Expressions are lazily resolved via session

benchmarks/compress-bench/src/vortex.rs

@@ -92,7 +92,7 @@ async fn vortex_decompress_read(data: Bytes) -> Result<()> {
         // Columns are named "0".."num_columns-1"; project the given subset.
         let names: FieldNames = cols.iter().map(|i| i.to_string()).collect();
         let projection = select(names, root())
-            .bind(&source_dtype)?
+            .bind(&source_dtype, &SESSION)?
             .optimize_recursive()?;
         scan = scan.with_projection(projection);
     }

docs/concepts/expressions.md

@@ -12,6 +12,12 @@ such as whether the function is strict, and the actual logic for executing the f
 The built-in scalar functions can be found in the `vortex-array::expr` module, with additional use-case specific
 functions provided by integration and plugin crates.
 
+An expression node holds the id of the function it calls and the options it is called with, not the function
+itself. Building an expression therefore needs neither a session nor the crate that defines the function. Binding
+the expression against a scope dtype and a `VortexSession` looks each function up in the session's scalar function
+registry, checks the options and the number of children against it, and type-checks the tree. The resulting bound
+expression is what scans optimize and execute, and what gives each function its own display.
+
 In the future, we plan to add full support for spatial functions, date-time functions, and more.
 
 ## Scalar Function Arrays

docs/developer-guide/internals/session.md

@@ -77,9 +77,9 @@ session.write_options()
     .write(&mut file, array_stream)
     .await?;
 
-// Scanning a layout
+// Scanning a layout: binding resolves the expression's scalar functions in the session
 let filter = expr
-    .bind(layout_reader.dtype())?
+    .bind(layout_reader.dtype(), &session)?
     .optimize_recursive()?;
 ScanBuilder::new(session.clone(), layout_reader)
     .with_filter(filter)

encodings/fsst/benches/fsst_url_compare.rs

@@ -245,14 +245,18 @@ fn like_substr_high_match(bencher: Bencher) {
         .unwrap()
         .into_array();
     let pattern = format!("%{HIGH_MATCH_DOMAIN}%");
-    let expr = like(root(), lit(pattern.as_str()));
+    let expr = like(root(), lit(pattern.as_str()))
+        .bind(fsst_array.dtype(), &SESSION)
+        .unwrap()
+        .optimize_recursive()
+        .unwrap();
 
     bencher
         .with_inputs(|| (&fsst_array, SESSION.create_execution_ctx()))
         .bench_refs(|(fsst_array, ctx)| {
             fsst_array
                 .clone()
-                .apply(&expr)
+                .apply_bound(&expr)
                 .unwrap()
                 .execute::<RecursiveCanonical>(ctx)
                 .unwrap()
@@ -269,14 +273,18 @@ fn like_substr_low_match(bencher: Bencher) {
         .unwrap()
         .into_array();
     let pattern = format!("%{LOW_MATCH_DOMAIN}%");
-    let expr = like(root(), lit(pattern.as_str()));
+    let expr = like(root(), lit(pattern.as_str()))
+        .bind(fsst_array.dtype(), &SESSION)
+        .unwrap()
+        .optimize_recursive()
+        .unwrap();
 
     bencher
         .with_inputs(|| (&fsst_array, SESSION.create_execution_ctx()))
         .bench_refs(|(fsst_array, ctx)| {
             fsst_array
                 .clone()
-                .apply(&expr)
+                .apply_bound(&expr)
                 .unwrap()
                 .execute::<RecursiveCanonical>(ctx)
                 .unwrap()

encodings/fsst/src/kernel.rs

@@ -239,7 +239,7 @@ mod tests {
         let mut ctx = SESSION.create_execution_ctx();
         let compressor = fsst_train_compressor(&varbin, &mut ctx)?;
         let fsst = fsst_compress(&varbin, &compressor, &mut ctx)?.into_array();
-        let result = fsst.apply(&byte_length(root()))?;
+        let result = fsst.apply(&byte_length(root()), &SESSION)?;
         let expected = PrimitiveArray::from_iter(vec![5u64, 7, 18, 0]);
         assert_arrays_eq!(result, expected, &mut ctx);
         Ok(())

encodings/parquet-variant/src/json_to_variant_tests.rs

@@ -61,7 +61,7 @@ fn json_input(storage: ArrayRef) -> VortexResult<ArrayRef> {
 fn execute_json_to_variant(input: ArrayRef, shredding: ShreddingSpec) -> VortexResult<ArrayRef> {
     let expr = json_to_variant(root(), shredding);
     input
-        .apply(&expr)?
+        .apply(&expr, &SESSION)?
         .execute::<ArrayRef>(&mut SESSION.create_execution_ctx())
 }
 
@@ -220,11 +220,10 @@ fn shredding_produces_typed_value_child() -> VortexResult<()> {
     assert!(shredded.unmasked_field_by_name_opt("a").is_some());
 
     // Typed extraction must serve shredded rows and fall back for mismatched rows.
-    let typed = result.clone().apply(&variant_get(
-        root(),
-        VariantPath::field("a"),
-        Some(i64_dtype()),
-    ))?;
+    let typed = result.clone().apply(
+        &variant_get(root(), VariantPath::field("a"), Some(i64_dtype())),
+        &SESSION,
+    )?;
     assert_arrays_eq!(
         typed,
         PrimitiveArray::from_option_iter([Some(1i64), Some(2), None, None]),
@@ -233,11 +232,14 @@ fn shredding_produces_typed_value_child() -> VortexResult<()> {
 
     // Mismatched rows keep their original value through the variant fallback.
     let untyped = result
-        .apply(&variant_get(
-            root(),
-            VariantPath::field("a"),
-            Some(DType::Utf8(Nullability::Nullable)),
-        ))?
+        .apply(
+            &variant_get(
+                root(),
+                VariantPath::field("a"),
+                Some(DType::Utf8(Nullability::Nullable)),
+            ),
+            &SESSION,
+        )?
         .execute::<ArrayRef>(&mut ctx)?;
     let row2 = untyped.execute_scalar(2, &mut ctx)?;
     assert_eq!(
@@ -259,11 +261,10 @@ fn shredding_preserves_null_rows() -> VortexResult<()> {
     assert_eq!(result.dtype(), &DType::Variant(Nullability::Nullable));
     let mut ctx = SESSION.create_execution_ctx();
     assert_nth_scalar_is_null!(result, 1, &mut ctx);
-    let typed = result.apply(&variant_get(
-        root(),
-        VariantPath::field("a"),
-        Some(i64_dtype()),
-    ))?;
+    let typed = result.apply(
+        &variant_get(root(), VariantPath::field("a"), Some(i64_dtype())),
+        &SESSION,
+    )?;
     assert_arrays_eq!(
         typed,
         PrimitiveArray::from_option_iter([Some(1i64), None, Some(3)]),

encodings/parquet-variant/src/kernel.rs

@@ -533,7 +533,7 @@ mod tests {
     ) -> VortexResult<ArrayRef> {
         let expr = variant_get(root(), parse_path(path)?, dtype);
         array
-            .apply(&expr)?
+            .apply(&expr, &SESSION)?
             .execute::<ArrayRef>(&mut SESSION.create_execution_ctx())
     }
 

encodings/sequence/src/compute/list_contains.rs

@@ -131,7 +131,7 @@ mod tests {
                 .into_array();
 
             let expr = list_contains(lit(list_scalar.clone()), root());
-            let result = array.into_array().apply(&expr).unwrap();
+            let result = array.into_array().apply(&expr, &SESSION).unwrap();
             let expected = BoolArray::from_iter([Some(true), Some(false), Some(true)]);
             assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
         }
@@ -145,7 +145,7 @@ mod tests {
                 .into_array();
 
             let expr = list_contains(lit(list_scalar), root());
-            let result = array.into_array().apply(&expr).unwrap();
+            let result = array.into_array().apply(&expr, &SESSION).unwrap();
             let expected = BoolArray::from_iter([Some(true), Some(true), Some(false)]);
             assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
         }
@@ -155,9 +155,9 @@ mod tests {
     fn apply_list_contains(array: ArrayRef, set: Scalar, optimize: bool) -> VortexResult<ArrayRef> {
         let expr = list_contains(lit(set), root());
         if !optimize {
-            return array.apply(&expr);
+            return array.apply(&expr, &SESSION);
         }
-        let expr = expr.bind(array.dtype())?.optimize()?;
+        let expr = expr.bind(array.dtype(), &SESSION)?.optimize()?;
         array.apply_bound(&expr)
     }
 
@@ -191,7 +191,7 @@ mod tests {
 
         let null_list = Scalar::null(DType::List(Arc::new(I32.into()), Nullability::Nullable));
         let expr = list_contains(lit(null_list), root());
-        let result = array.apply(&expr).unwrap();
+        let result = array.apply(&expr, &SESSION).unwrap();
         let expected = BoolArray::from_iter([None::<bool>, None, None]);
         assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
     }
@@ -215,7 +215,7 @@ mod tests {
 
         let result = array
             .clone()
-            .apply(&list_contains(lit(set.clone()), root()))
+            .apply(&list_contains(lit(set.clone()), root()), &SESSION)
             .unwrap();
         let expected = BoolArray::from_iter([true, false, false]);
         assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
@@ -224,7 +224,7 @@ mod tests {
             sql_null_semantics: true,
         };
         let result = array
-            .apply(&list_contains_opts(lit(set), root(), sql))
+            .apply(&list_contains_opts(lit(set), root(), sql), &SESSION)
             .unwrap();
         let expected = BoolArray::from_iter([Some(true), None, None]);
         assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
@@ -243,7 +243,7 @@ mod tests {
             .into_array();
 
         let expr = list_contains(lit(list_scalar), root());
-        let result = array.apply(&expr).unwrap();
+        let result = array.apply(&expr, &SESSION).unwrap();
         let expected = BoolArray::from_iter([Some(true), Some(true), Some(true)]);
         assert_arrays_eq!(result, expected, &mut SESSION.create_execution_ctx());
     }

fuzz/fuzz_targets/file_io.rs

@@ -46,7 +46,7 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
     let expected_array = {
         let bool_mask = array_data
             .clone()
-            .apply(&filter_expr.clone().unwrap_or_else(|| lit(true)))
+            .apply(&filter_expr.clone().unwrap_or_else(|| lit(true)), &SESSION)
             .vortex_expect("filter expression evaluation should succeed in fuzz test");
         let bool_mask_bool = bool_mask
             .execute::<BoolArray>(&mut ctx)
@@ -56,7 +56,7 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
             .filter(mask)
             .vortex_expect("filter operation should succeed in fuzz test");
         filtered
-            .apply(&projection_expr.clone().unwrap_or_else(root))
+            .apply(&projection_expr.clone().unwrap_or_else(root), &SESSION)
             .vortex_expect("projection expression evaluation should succeed in fuzz test")
     };
 
@@ -83,13 +83,13 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
         .vortex_expect("open_buffer should succeed in fuzz test");
     let projection = projection_expr
         .unwrap_or_else(root)
-        .bind(file.dtype())
+        .bind(file.dtype(), &SESSION)
         .and_then(|expr| expr.optimize_recursive())
         .vortex_expect("projection should bind in fuzz test");
     let filter = filter_expr
         .map(|filter| {
             filter
-                .bind(file.dtype())
+                .bind(file.dtype(), &SESSION)
                 .and_then(|expr| expr.optimize_recursive())
         })
         .transpose()

lang/cpp/examples/writer.cpp

@@ -40,7 +40,7 @@ int main(int argc, char **argv) {
     });
 
     Expression age_gt_10 = expr::col("age") > expr::lit<uint8_t>(10);
-    Array validity_array = array.apply(age_gt_10);
+    Array validity_array = array.apply(session, age_gt_10);
 
     const Validity validity = Validity::from_array(validity_array);
     Array array2 = make_struct({

lang/cpp/include/vortex/array.hpp

@@ -223,7 +223,7 @@ class Array {
     Array slice(size_t begin, size_t end) const;
 
     /**
-     * Apply an expression to an array.
+     * Apply an expression to an array, resolving its scalar functions in the session.
      *
      * This function operates in constant time and doesn't execute the result
      * array. To execute the array, canonicalise it.
@@ -232,12 +232,13 @@ class Array {
      *
      * using namespace vortex::expr::ops;
      *
+     * Session session;
      * std::array<uint16_t, 3> buffer = {0, 1, 2};
      * Array array = Array::primitive<uint16_t>(buffer);
      * Expression expr = expr::root() > expr::lit<uint16_t>(0);
-     * Array result = array.apply(expr);
+     * Array result = array.apply(session, expr);
      */
-    Array apply(const Expression &expr) const;
+    Array apply(const Session &session, const Expression &expr) const;
 
     /**
      * Bulk view over values. Canonicalizes the array.

lang/cpp/src/array.cpp

@@ -264,9 +264,9 @@ Array Array::slice(size_t begin, size_t end) const {
     return Access::adopt<Array>(out);
 }
 
-Array Array::apply(const Expression &expr) const {
+Array Array::apply(const Session &session, const Expression &expr) const {
     vx_error *error = nullptr;
-    const vx_array *out = vx_array_apply(handle_.get(), Access::c_ptr(expr), &error);
+    const vx_array *out = vx_array_apply(Access::c_ptr(session), handle_.get(), Access::c_ptr(expr), &error);
     throw_on_error(error);
     return Access::adopt<Array>(out);
 }

lang/cpp/tests/array.cpp

@@ -77,7 +77,7 @@ TEST_CASE("Validity from a boolean mask", "[array]") {
     std::vector<uint8_t> mask_bytes = {1, 0, 1};
 
     Array mask_u8 = Array::primitive<uint8_t>(std::span<const uint8_t>(mask_bytes));
-    Array mask = mask_u8.apply(expr::root() == expr::lit<uint8_t>(1));
+    Array mask = mask_u8.apply(session, expr::root() == expr::lit<uint8_t>(1));
 
     Array a = Array::primitive<int32_t>(std::span<const int32_t>(data), Validity::from_array(mask));
     REQUIRE(a.nullable());

lang/cpp/tests/expression.cpp

@@ -23,7 +23,7 @@ TEST_CASE("Expression copy", "[expr]") {
     Session session;
     const std::vector<uint8_t> ages = {10, 20, 30};
     Array input = make_struct({{"age", Array::primitive<uint8_t>(ages)}});
-    auto values = input.apply(another).values<uint8_t>(session);
+    auto values = input.apply(session, another).values<uint8_t>(session);
     REQUIRE(values.size() == ages.size());
     for (size_t i = 0; i < ages.size(); ++i) {
         REQUIRE(values.values()[i] == ages[i]);
@@ -35,7 +35,7 @@ TEST_CASE("Apply root()", "[expr]") {
     std::vector<int32_t> data = {10, 20, 30, 40, 50};
     Array array = Array::primitive<int32_t>(data);
 
-    Array applied = array.apply(expr::root());
+    Array applied = array.apply(session, expr::root());
 
     REQUIRE(applied.size() == data.size());
     REQUIRE(applied.is_primitive(I32));
@@ -56,7 +56,7 @@ TEST_CASE("Apply projection", "[expr]") {
         {"height", Array::primitive<uint16_t>(heights)},
     });
 
-    Array projected = struct_arr.apply(expr::col("age"));
+    Array projected = struct_arr.apply(session, expr::col("age"));
     REQUIRE(projected.size() == ages.size());
     REQUIRE(projected.is_primitive(U8));
     auto values = projected.values<uint8_t>(session);
@@ -71,7 +71,7 @@ TEST_CASE("Apply arithmetic", "[expr]") {
     Array array = Array::primitive<int32_t>(data);
 
     Expression e = expr::add(expr::root(), expr::lit<int32_t>(10));
-    Array applied = array.apply(e);
+    Array applied = array.apply(session, e);
 
     REQUIRE(applied.is_primitive(I32));
     auto values = applied.values<int32_t>(session);
@@ -87,7 +87,7 @@ TEST_CASE("Operator overloading", "[expr]") {
     std::vector<uint32_t> data = {1, 2, 2, 7};
     Array array = Array::primitive<uint32_t>(data);
 
-    Array applied = array.apply(expr::root() == expr::lit<uint32_t>(2));
+    Array applied = array.apply(session, expr::root() == expr::lit<uint32_t>(2));
     PrimitiveView<bool> view = applied.bools(session);
     REQUIRE(view.size() == data.size());
     BoolView values = view.values();
@@ -98,11 +98,12 @@ TEST_CASE("Operator overloading", "[expr]") {
 }
 
 TEST_CASE("Apply error", "[expr]") {
+    Session session;
     std::vector<uint8_t> data = {1, 2, 3};
     Array array = Array::primitive<uint8_t>(data);
 
     Expression bad = expr::add(expr::root(), expr::lit<int32_t>(1));
-    REQUIRE_THROWS_AS(array.apply(bad), VortexException);
+    REQUIRE_THROWS_AS(array.apply(session, bad), VortexException);
 }
 
 TEST_CASE("Empty conjunction", "[expr]") {