Move remaining expressions out of vortex-array to vortex-expr

.github/workflows/codspeed.yml

@@ -56,7 +56,7 @@ jobs:
           - { shard: 3, name: "Array dictionaries", packages: "vortex-array", array-group: "dictionary", features: "--features _test-harness" }
           - { shard: 4, name: "Array builders & views", packages: "vortex-array", array-group: "builders", features: "--features _test-harness" }
           - { shard: 5, name: "Core & encodings", packages: "vortex-buffer vortex-error vortex-mask vortex-compute vortex-file vortex-alp vortex-bytebool vortex-datetime-parts" }
-          - { shard: 6, name: "Main library, functions & storage", packages: "vortex vortex-functions vortex-arrow vortex-btrblocks vortex-compressor vortex-row" }
+          - { shard: 6, name: "Main library, functions & storage", packages: "vortex vortex-expr vortex-functions vortex-arrow vortex-btrblocks vortex-compressor vortex-row" }
           - { shard: 7, name: "FastLanes & decimals", packages: "vortex-decimal-byte-parts vortex-fastlanes", features: "--features _test-harness" }
           - { shard: 8, name: "Encodings & layout", packages: "vortex-onpair vortex-pco vortex-runend vortex-sequence vortex-sparse vortex-zigzag vortex-zstd vortex-layout" }
           - { shard: 9, name: "Strings, tensor & spatial", packages: "vortex-fsst vortex-tensor vortex-spatial", features: "--features _test-harness" }
@@ -92,7 +92,7 @@ jobs:
           groups = {
               "dictionary": ("dict_", "chunked_dict_", "patches_"),
               "selection": ("take_", "filter_", "piecewise_sequence_"),
-              "compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool"),
+              "compute": ("aggregate_", "cast_", "binary_ops", "compare", "kleene_bool"),
           }
           manifest = tomllib.loads(pathlib.Path("vortex-array/Cargo.toml").read_text())
           benches = []

AGENTS.md

@@ -22,6 +22,8 @@ documentation in `docs/`, and benchmark tooling in `vortex-bench/` and `benchmar
 - `vortex-array` contains the core `Array` trait and the base encodings, including most
   Apache Arrow-style encodings.
 - `encodings/*` contains more specialized compressed encodings.
+- `vortex-expr` contains expression trees, their optimizer, and the analyses and rewrites built on
+  them. `vortex-functions` contains the scalar functions that `vortex-array` itself is not built on.
 - `vortex-file` implements file IO. It uses `LayoutReader` from `vortex-layout`.
 - `vortex-io` holds the core async and blocking IO traits, plus the generic `object_store`
   adapters that implement them.

docs/concepts/expressions.md

@@ -9,8 +9,10 @@ to the returned data.
 Expressions are defined as abstract scalar functions. These vtables define the signature of the function, properties
 such as whether the function is strict, and the actual logic for executing the function over input arrays.
 
-The built-in scalar functions can be found in the `vortex-array::expr` module, with additional use-case specific
-functions provided by integration and plugin crates.
+The scalar function framework and the handful of functions the array crate itself is built on live in
+`vortex_array::scalar_fn`; expression trees, their optimizer and the analyses built on them live in the
+`vortex-expr` crate; `vortex-functions` holds the rest of the built-in scalar functions, with additional
+use-case specific functions provided by integration and plugin crates.
 
 In the future, we plan to add full support for spatial functions, date-time functions, and more.
 

encodings/fsst/Cargo.toml

@@ -22,6 +22,7 @@ rand = { workspace = true, optional = true }
 vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-error = { workspace = true }
+vortex-expr = { workspace = true }
 vortex-functions = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-session = { workspace = true }
@@ -36,6 +37,7 @@ rand = { workspace = true }
 rstest = { workspace = true }
 test-with = { workspace = true }
 vortex-array = { workspace = true, features = ["_test-harness"] }
+vortex-expr = { workspace = true }
 
 [[bench]]
 name = "fsst_compress"

encodings/fsst/benches/fsst_url_compare.rs

@@ -16,13 +16,14 @@ use vortex_array::arrays::VarBinArray;
 use vortex_array::arrays::VarBinViewArray;
 use vortex_array::arrays::varbin::VarBinArrayExt;
 use vortex_array::builtins::ArrayBuiltins;
-use vortex_array::expr::lit;
-use vortex_array::expr::root;
 use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
 use vortex_array::scalar::Scalar;
 use vortex_array::scalar_fn::builtin::operators::Operator;
 use vortex_array::session::ArraySession;
+use vortex_expr::ArrayExprExt;
+use vortex_expr::lit;
+use vortex_expr::root;
 use vortex_fsst::fsst_compress;
 use vortex_fsst::fsst_train_compressor;
 use vortex_fsst::test_utils::HIGH_MATCH_DOMAIN;

encodings/fsst/src/kernel.rs

@@ -44,8 +44,9 @@ mod tests {
     use vortex_array::assert_arrays_eq;
     use vortex_array::dtype::DType;
     use vortex_array::dtype::Nullability;
-    use vortex_array::expr::root;
     use vortex_error::VortexResult;
+    use vortex_expr::ArrayExprExt;
+    use vortex_expr::root;
     use vortex_functions::expr::byte_length;
     use vortex_mask::Mask;
     use vortex_session::VortexSession;

encodings/parquet-variant/Cargo.toml

@@ -37,6 +37,7 @@ vortex-edition = { workspace = true }
 rstest = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 vortex-array = { workspace = true, features = ["_test-harness"] }
+vortex-expr = { workspace = true }
 vortex-file = { workspace = true, features = ["tokio"] }
 vortex-io = { workspace = true, features = ["tokio"] }
 vortex-layout = { workspace = true }

encodings/parquet-variant/src/json_to_variant_tests.rs

@@ -26,10 +26,11 @@ use vortex_array::assert_nth_scalar_is_null;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
 use vortex_array::dtype::PType;
-use vortex_array::expr::root;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 use vortex_error::vortex_err;
+use vortex_expr::ArrayExprExt;
+use vortex_expr::root;
 use vortex_functions::expr::variant_get;
 use vortex_functions::variant_get::VariantPath;
 use vortex_json::Json;

encodings/parquet-variant/src/kernel.rs

@@ -325,13 +325,14 @@ mod tests {
     use vortex_array::dtype::DType as VortexDType;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType;
-    use vortex_array::expr::root;
     use vortex_array::validity::Validity;
     use vortex_arrow::ArrowSessionExt;
     use vortex_error::VortexResult;
     use vortex_error::vortex_bail;
     use vortex_error::vortex_ensure;
     use vortex_error::vortex_err;
+    use vortex_expr::ArrayExprExt;
+    use vortex_expr::root;
     use vortex_functions::expr::variant_get;
     use vortex_functions::variant_get::VariantPath;
     use vortex_functions::variant_get::VariantPathElement;

encodings/sequence/Cargo.toml

@@ -20,6 +20,7 @@ vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-compute = { workspace = true }
 vortex-error = { workspace = true }
+vortex-expr = { workspace = true }
 vortex-functions = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-session = { workspace = true }

encodings/sequence/src/compute/list_contains.rs

@@ -97,10 +97,11 @@ mod tests {
     use vortex_array::dtype::DType;
     use vortex_array::dtype::Nullability;
     use vortex_array::dtype::PType::I32;
-    use vortex_array::expr::lit;
-    use vortex_array::expr::root;
     use vortex_array::scalar::Scalar;
     use vortex_error::VortexResult;
+    use vortex_expr::ArrayExprExt;
+    use vortex_expr::lit;
+    use vortex_expr::root;
     use vortex_functions::expr::list_contains;
     use vortex_functions::expr::list_contains_opts;
     use vortex_functions::list_contains::ListContainsOptions;

fuzz/Cargo.toml

@@ -35,6 +35,7 @@ vortex-array = { workspace = true, features = ["arbitrary", "_test-harness"] }
 vortex-btrblocks = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-error = { workspace = true }
+vortex-expr = { workspace = true, features = ["arbitrary"] }
 vortex-fsst = { workspace = true, features = ["_test-harness"] }
 vortex-functions = { workspace = true }
 vortex-io = { workspace = true, features = ["wasm-bindgen"] }

fuzz/fuzz_targets/file_io.rs

@@ -6,6 +6,8 @@
 use itertools::Itertools;
 use libfuzzer_sys::Corpus;
 use libfuzzer_sys::fuzz_target;
+use vortex::expr::lit;
+use vortex::expr::root;
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
 use vortex_array::VortexSessionExecute;
@@ -15,10 +17,9 @@ use vortex_array::arrays::bool::BoolArrayExt;
 use vortex_array::assert_arrays_eq;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::StructFields;
-use vortex_array::expr::lit;
-use vortex_array::expr::root;
 use vortex_btrblocks::BtrBlocksCompressorBuilder;
 use vortex_error::VortexExpect;
+use vortex_expr::ArrayExprExt;
 use vortex_file::OpenOptionsSessionExt;
 use vortex_file::WriteOptionsSessionExt;
 use vortex_file::WriteStrategyBuilder;

fuzz/src/file/mod.rs

@@ -3,13 +3,13 @@
 
 use arbitrary::Arbitrary;
 use arbitrary::Unstructured;
+use vortex::expr::Expression;
 use vortex_array::ArrayRef;
 use vortex_array::arrays::arbitrary::ArbitraryArray;
 use vortex_array::arrays::arbitrary::ArbitraryArrayConfig;
 use vortex_array::arrays::arbitrary::ArbitraryWith;
-use vortex_array::expr::Expression;
-use vortex_array::expr::arbitrary::filter_expr;
-use vortex_array::expr::arbitrary::projection_expr;
+use vortex_expr::arbitrary::filter_expr;
+use vortex_expr::arbitrary::projection_expr;
 
 use crate::FUZZ_FILE_ARRAY_MAX_LEN;
 use crate::array::CompressorStrategy;

fuzz/src/fsst_like.rs

@@ -9,6 +9,8 @@ use std::sync::LazyLock;
 
 use arbitrary::Arbitrary;
 use arbitrary::Unstructured;
+use vortex::functions::like::Like;
+use vortex::functions::like::LikeOptions;
 use vortex_array::ArrayRef;
 use vortex_array::Canonical;
 use vortex_array::IntoArray;
@@ -23,8 +25,6 @@ use vortex_error::VortexResult;
 use vortex_fsst::FSSTArray;
 use vortex_fsst::fsst_compress;
 use vortex_fsst::fsst_train_compressor;
-use vortex_functions::like::Like;
-use vortex_functions::like::LikeOptions;
 use vortex_session::VortexSession;
 
 use crate::error::Backtrace;

vortex-array/Cargo.toml

@@ -189,16 +189,6 @@ harness = false
 name = "varbinview_compact"
 harness = false
 
-[[bench]]
-name = "expr_large_struct_pack"
-path = "benches/expr/large_struct_pack.rs"
-harness = false
-
-[[bench]]
-name = "expr_optimize"
-path = "benches/expr/optimize_bench.rs"
-harness = false
-
 [[bench]]
 name = "chunked_dict_builder"
 harness = false

vortex-array/benches/dict_compare.rs

@@ -19,9 +19,6 @@ use vortex_array::arrays::dict_test::gen_varbin_words;
 use vortex_array::arrays::varbin::VarBinArrayExt;
 use vortex_array::builders::dict::dict_encode;
 use vortex_array::builtins::ArrayBuiltins;
-use vortex_array::expr::eq;
-use vortex_array::expr::lit;
-use vortex_array::expr::root;
 use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
 use vortex_array::scalar_fn::builtin::operators::Operator;
@@ -146,7 +143,10 @@ fn bench_compare_sliced_dict_primitive(
         .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
-                .apply(&eq(root(), lit(value)))
+                .binary(
+                    ConstantArray::new(value, codes_len).into_array(),
+                    Operator::Eq,
+                )
                 .unwrap()
                 .execute::<RecursiveCanonical>(ctx)
                 .unwrap()
@@ -169,7 +169,10 @@ fn bench_compare_sliced_dict_varbinview(
         .with_inputs(|| (&dict, SESSION.create_execution_ctx()))
         .bench_refs(|(dict, ctx)| {
             dict.clone()
-                .apply(&eq(root(), lit(value)))
+                .binary(
+                    ConstantArray::new(value, codes_len).into_array(),
+                    Operator::Eq,
+                )
                 .unwrap()
                 .execute::<RecursiveCanonical>(ctx)
                 .unwrap()

vortex-array/src/array/erased.rs

@@ -533,7 +533,7 @@ impl ArrayRef {
     /// # Safety
     /// The caller must put back a slot with the same logical dtype and length before exposing the
     /// parent array, and must only use this for physical rewrites.
-    pub(crate) unsafe fn take_slot_unchecked(
+    pub unsafe fn take_slot_unchecked(
         mut self,
         slot_idx: usize,
     ) -> VortexResult<(ArrayRef, ArrayRef)> {
@@ -567,7 +567,7 @@ impl ArrayRef {
     /// # Safety
     /// The replacement must have the same logical dtype and length as the taken slot, and this
     /// must only be used for physical rewrites.
-    pub(crate) unsafe fn put_slot_unchecked(
+    pub unsafe fn put_slot_unchecked(
         mut self,
         slot_idx: usize,
         replacement: ArrayRef,