From 40789f60a7161d31b98c5dbb10068da7bd921f17 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:22:29 +0000 Subject: [PATCH 01/57] feat: analyze range transformation locality --- .../expr-common/src/interval_arithmetic.rs | 62 +- datafusion/expr/src/udf.rs | 70 ++ .../functions/src/datetime/date_trunc.rs | 195 ++++- .../src/equivalence/properties/mod.rs | 24 + datafusion/physical-expr/src/partitioning.rs | 664 +++++++++++++++++- .../physical-expr/src/scalar_function.rs | 50 ++ 6 files changed, 1052 insertions(+), 13 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 797c48cbe0dd9..3e1ba77a2504d 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -1238,6 +1238,34 @@ fn prev_value(value: ScalarValue) -> ScalarValue { value_transition!(MIN, false, value) } +/// Returns the previous distinct value, or `None` for nulls, non-finite +/// floats, type minima, and types without a discrete predecessor. +pub fn checked_predecessor(value: &ScalarValue) -> Option { + checked_adjacent(value, false) +} + +/// Returns the next distinct value, or `None` for nulls, non-finite floats, +/// type maxima, and types without a discrete successor. +pub fn checked_successor(value: &ScalarValue) -> Option { + checked_adjacent(value, true) +} + +fn checked_adjacent(value: &ScalarValue, successor: bool) -> Option { + if value.is_null() + || matches!(value, ScalarValue::Float32(Some(v)) if !v.is_finite()) + || matches!(value, ScalarValue::Float64(Some(v)) if !v.is_finite()) + { + return None; + } + + let adjacent = if successor { + next_value(value.clone()) + } else { + prev_value(value.clone()) + }; + (!adjacent.is_null() && adjacent != *value).then_some(adjacent) +} + trait OneTrait: Sized + std::ops::Add + std::ops::Sub { fn one() -> Self; } @@ -2273,8 +2301,8 @@ impl NullableInterval { mod tests { use crate::{ interval_arithmetic::{ - Interval, cast_scalar_value, handle_overflow, next_value, prev_value, - satisfy_greater, + Interval, cast_scalar_value, checked_predecessor, checked_successor, + handle_overflow, next_value, prev_value, satisfy_greater, }, operator::Operator, }; @@ -2594,6 +2622,36 @@ mod tests { Ok(()) } + #[test] + fn test_checked_adjacent_values() { + assert_eq!( + checked_predecessor(&ScalarValue::Int64(Some(10))), + Some(ScalarValue::Int64(Some(9))) + ); + assert_eq!( + checked_successor(&ScalarValue::Int64(Some(10))), + Some(ScalarValue::Int64(Some(11))) + ); + assert_eq!(checked_predecessor(&ScalarValue::Int64(None)), None); + assert_eq!( + checked_predecessor(&ScalarValue::Int64(Some(i64::MIN))), + None + ); + assert_eq!(checked_successor(&ScalarValue::Int64(Some(i64::MAX))), None); + assert_eq!( + checked_predecessor(&ScalarValue::Utf8(Some("a".into()))), + None + ); + assert_eq!( + checked_successor(&ScalarValue::Float64(Some(f64::NAN))), + None + ); + assert_eq!( + checked_predecessor(&ScalarValue::Float32(Some(f32::NEG_INFINITY))), + None + ); + } + #[test] fn test_new_interval() -> Result<()> { use ScalarValue::*; diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index d937a54029398..78c94a1ee3ea4 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -364,6 +364,15 @@ impl ScalarUDF { self.inner.output_ordering(inputs) } + /// See [`ScalarUDFImpl::supports_range_partitioning_analysis`] for more details. + pub fn supports_range_partitioning_analysis( + &self, + argument_types: &[DataType], + ) -> bool { + self.inner + .supports_range_partitioning_analysis(argument_types) + } + pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { self.inner.preserves_lex_ordering(inputs) } @@ -983,6 +992,38 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { Ok(inner(inputs)) } + /// Returns whether this function and argument-type combination may + /// participate in range-partitioning transformation analysis. + /// + /// Ordinary [`SortProperties::Ordered`] metadata is not sufficient for this + /// analysis. A non-strictly ordered function can map values from opposite + /// sides of a range split to the same output, and successful evaluation of + /// some functions can produce null from non-null input. Either behavior can + /// place one transformed equality group in multiple partitions. + /// + /// Returning `true` asserts that this argument-type combination has been + /// audited with the following contract: + /// + /// - whenever the concrete expression reports same-direction ordering with + /// all non-range inputs singleton, it is globally nondecreasing over the + /// admitted non-null input domain; + /// - successful evaluation preserves nullness: null range inputs remain + /// null and non-null range inputs do not produce null; and + /// - successful scalar evaluation at a split boundary is consistent with + /// array evaluation of that value. + /// + /// This method does not assert that a concrete range partitioning is + /// preserved. Callers must still identify the range input, establish + /// same-direction ordering and singleton arguments, verify exact split + /// types, and prove at every split that the images of adjacent partitions + /// do not overlap. Evaluation errors must fail closed. + /// + /// The default is `false`. This capability is not currently transported + /// across FFI, so foreign UDFs also remain denied by default. + fn supports_range_partitioning_analysis(&self, _argument_types: &[DataType]) -> bool { + false + } + /// Returns true if the function preserves lexicographical ordering based on /// the input ordering. /// @@ -1205,6 +1246,11 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl { self.inner.output_ordering(inputs) } + fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { + self.inner + .supports_range_partitioning_analysis(argument_types) + } + fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { self.inner.preserves_lex_ordering(inputs) } @@ -1237,6 +1283,7 @@ mod tests { name: &'static str, field: &'static str, signature: Signature, + supports_range_partitioning_analysis: bool, } impl ScalarUDFImpl for TestScalarUDFImpl { fn name(&self) -> &str { @@ -1254,6 +1301,14 @@ mod tests { fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { unimplemented!() } + + fn supports_range_partitioning_analysis( + &self, + argument_types: &[DataType], + ) -> bool { + self.supports_range_partitioning_analysis + && argument_types == [DataType::Int64] + } } // PartialEq and Hash must be consistent, and also PartialEq and PartialOrd @@ -1287,11 +1342,26 @@ mod tests { assert_eq!(b.partial_cmp(&o), Some(Ordering::Less)); } + #[test] + fn aliases_preserve_range_partitioning_analysis_support() { + let function = ScalarUDF::from(TestScalarUDFImpl { + name: "range_function", + field: "a", + signature: Signature::any(1, Volatility::Immutable), + supports_range_partitioning_analysis: true, + }) + .with_aliases(["range_function_alias"]); + + assert!(function.supports_range_partitioning_analysis(&[DataType::Int64])); + assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8])); + } + fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF { ScalarUDF::from(TestScalarUDFImpl { name, field: parameter, signature: Signature::any(1, Volatility::Immutable), + supports_range_partitioning_analysis: false, }) } diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 082671cdc0c0c..1dd1ac7069a7a 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -907,7 +907,10 @@ mod tests { }; use arrow::array::cast::as_primitive_array; - use arrow::array::types::{ArrowTimestampType, TimestampNanosecondType}; + use arrow::array::types::{ + ArrowTimestampType, TimestampMicrosecondType, TimestampMillisecondType, + TimestampNanosecondType, TimestampSecondType, + }; use arrow::array::{ Array, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, @@ -915,9 +918,9 @@ mod tests { use arrow::buffer::NullBuffer; use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos; use arrow::compute::{DatePart, SortOptions}; - use arrow::datatypes::{DataType, Field, TimeUnit}; - use datafusion_common::ScalarValue; + use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit}; use datafusion_common::config::ConfigOptions; + use datafusion_common::{Result, ScalarValue}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; @@ -962,6 +965,192 @@ mod tests { ); } + #[test] + fn supports_range_partitioning_analysis_for_timezone_less_timestamps() { + let function = DateTruncFunc::new(); + + for unit in [ + TimeUnit::Second, + TimeUnit::Millisecond, + TimeUnit::Microsecond, + TimeUnit::Nanosecond, + ] { + for string_type in [DataType::Utf8, DataType::Utf8View, DataType::LargeUtf8] { + assert!(function.supports_range_partitioning_analysis(&[ + string_type, + DataType::Timestamp(unit, None), + ])); + } + + for timezone in ["UTC", "+05:30", "America/Goose_Bay"] { + assert!(!function.supports_range_partitioning_analysis(&[ + DataType::Utf8, + DataType::Timestamp(unit, Some(timezone.into())), + ])); + } + } + + assert!(!function.supports_range_partitioning_analysis(&[ + DataType::Utf8, + DataType::Time32(TimeUnit::Second), + ])); + assert!(!function.supports_range_partitioning_analysis(&[ + DataType::Utf8, + DataType::Time64(TimeUnit::Nanosecond), + ])); + assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8,])); + assert!(!function.supports_range_partitioning_analysis(&[ + DataType::Int64, + DataType::Timestamp(TimeUnit::Nanosecond, None), + ])); + } + + #[test] + fn date_bin_does_not_support_range_partitioning_analysis() { + assert!( + !crate::datetime::date_bin::DateBinFunc::new() + .supports_range_partitioning_analysis(&[ + DataType::Interval(IntervalUnit::MonthDayNano), + DataType::Timestamp(TimeUnit::Nanosecond, None), + ]) + ); + } + + fn invoke_date_trunc( + granularity: &str, + timestamp: ColumnarValue, + timestamp_type: DataType, + number_rows: usize, + ) -> Result { + DateTruncFunc::new().invoke_with_args(ScalarFunctionArgs { + args: vec![ScalarValue::from(granularity).into(), timestamp], + arg_fields: vec![ + Field::new("granularity", DataType::Utf8, false).into(), + Field::new("timestamp", timestamp_type.clone(), true).into(), + ], + number_rows, + return_field: Field::new("date_trunc", timestamp_type, true).into(), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn assert_timezone_less_range_analysis_contract() { + let scale = match T::UNIT { + TimeUnit::Second => 1_000_000_000, + TimeUnit::Millisecond => 1_000_000, + TimeUnit::Microsecond => 1_000, + TimeUnit::Nanosecond => 1, + }; + // Probe both sides of calendar and fixed-width boundaries at the + // input's own resolution, including negative fractional timestamps. + let mut timestamps = [ + "1900-03-01T00:00:00Z", // Non-leap century. + "1969-12-29T00:00:00Z", // Monday before the epoch. + "1970-01-01T00:00:00Z", + "1970-01-01T00:00:00.000001Z", + "1970-01-01T00:00:00.001Z", + "1970-01-01T00:00:01Z", + "1970-01-01T00:01:00Z", + "1970-01-01T01:00:00Z", + "2000-01-01T00:00:00Z", + "2000-02-29T00:00:00Z", // Leap century. + "2000-03-01T00:00:00Z", + "2000-04-01T00:00:00Z", // Quarter boundary. + "2024-01-01T00:00:00Z", // Monday and year boundary. + ] + .into_iter() + .flat_map(|value| { + let boundary = string_to_timestamp_nanos(value).unwrap() / scale; + [boundary - 1, boundary, boundary + 1] + }) + .collect::>(); + // Stay inside the scalar nanosecond domain with room to truncate a + // full year at its lower edge; the upper edge needs no such margin. + timestamps.extend([ + i64::MIN / scale + 366 * 86_400 * (1_000_000_000 / scale), + i64::MAX / scale, + ]); + timestamps.sort_unstable(); + timestamps.dedup(); + let values = timestamps + .into_iter() + .flat_map(|value| [Some(value), None]) + .collect::>(); + let timestamp_type = DataType::Timestamp(T::UNIT, None); + + for granularity in [ + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year", + ] { + let input = PrimitiveArray::::from_iter(values.iter().copied()); + let output = invoke_date_trunc( + granularity, + ColumnarValue::Array(Arc::new(input.clone())), + timestamp_type.clone(), + input.len(), + ) + .unwrap(); + let ColumnarValue::Array(output) = output else { + panic!("expected array output") + }; + let output = as_primitive_array::(&output); + + assert_eq!( + input.nulls(), + output.nulls(), + "{granularity} must preserve nullness for {:?}", + T::UNIT + ); + + let mut previous = None; + for value in output.iter().flatten() { + if let Some(previous) = previous { + assert!( + previous <= value, + "{granularity} must be nondecreasing for {:?}", + T::UNIT + ); + } + previous = Some(value); + } + + for (index, value) in values.iter().enumerate() { + let scalar_output = invoke_date_trunc( + granularity, + ColumnarValue::Scalar(ScalarValue::new_timestamp::(*value, None)), + timestamp_type.clone(), + 1, + ) + .unwrap(); + let ColumnarValue::Scalar(scalar_output) = scalar_output else { + panic!("expected scalar output") + }; + assert_eq!( + scalar_output, + ScalarValue::try_from_array(output, index).unwrap(), + "scalar and array evaluation must agree for {granularity} {:?}", + T::UNIT + ); + } + } + } + + #[test] + fn timezone_less_timestamps_satisfy_range_analysis_contract() { + assert_timezone_less_range_analysis_contract::(); + assert_timezone_less_range_analysis_contract::(); + assert_timezone_less_range_analysis_contract::(); + assert_timezone_less_range_analysis_contract::(); + } + #[test] fn date_trunc_test() { let cases = vec![ diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1c2d970ccffa4..ed824e5712f96 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -39,6 +39,7 @@ use crate::{ PhysicalSortRequirement, }; +use arrow::compute::SortOptions; use arrow::datatypes::SchemaRef; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err}; @@ -1408,6 +1409,29 @@ impl EquivalenceProperties { .unwrap_or_else(|_| ExprProperties::new_unknown()) } + /// Returns whether `transformed_expr` reports the same ordering direction + /// when `source_expr` is ordered and all other inputs are singleton. + pub(crate) fn is_same_direction_ordered_transform( + &self, + transformed_expr: &Arc, + source_expr: &Arc, + ) -> bool { + if transformed_expr.eq(source_expr) { + return false; + } + + let options = SortOptions::default(); + let dependencies = Dependencies::new(std::iter::once(PhysicalSortExpr::new( + Arc::clone(source_expr), + options, + ))); + matches!( + get_expr_properties(transformed_expr, &dependencies, &self.schema) + .map(|properties| properties.sort_properties), + Ok(SortProperties::Ordered(actual)) if actual == options + ) + } + /// Transforms this `EquivalenceProperties` by mapping columns in the /// original schema to columns in the new schema by index. pub fn with_new_schema(mut self, schema: SchemaRef) -> Result { diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 336d09b987734..13235874a176f 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -17,13 +17,20 @@ //! [`Partitioning`] and [`Distribution`] for `ExecutionPlans` +use crate::expressions::{Literal, UnKnownColumn}; +use crate::simplifier::const_evaluator::create_dummy_batch; use crate::{ - EquivalenceProperties, PhysicalExpr, equivalence::ProjectionMapping, - expressions::UnKnownColumn, physical_exprs_contains, physical_exprs_equal, + EquivalenceProperties, PhysicalExpr, ScalarFunctionExpr, + equivalence::ProjectionMapping, physical_exprs_contains, physical_exprs_equal, }; use arrow::datatypes::Schema; pub use datafusion_common::SplitPoint; -use datafusion_common::{Result, plan_err, validate_range_split_points}; +use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::{ + Result, ScalarValue, plan_err, validate_range_split_points, +}; +use datafusion_expr::ColumnarValue; +use datafusion_expr::interval_arithmetic::{checked_predecessor, checked_successor}; use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; #[cfg(feature = "proto")] @@ -532,6 +539,165 @@ fn normalize_exprs( .collect() } +fn equivalent_expr( + left: &Arc, + right: &Arc, + eq_properties: &EquivalenceProperties, +) -> bool { + equivalent_exprs( + std::slice::from_ref(left), + std::slice::from_ref(right), + eq_properties, + ) +} + +fn evaluate_with_range_value( + expr: &Arc, + range_key: &Arc, + value: &ScalarValue, +) -> Option { + let literal: Arc = Arc::new(Literal::new(value.clone())); + let rewritten = Arc::clone(expr) + .transform(|node| { + if node.eq(range_key) { + Ok(Transformed::yes(Arc::clone(&literal))) + } else { + Ok(Transformed::no(node)) + } + }) + .ok()?; + if !rewritten.transformed { + return None; + } + + match rewritten.data.evaluate(create_dummy_batch().ok()?).ok()? { + ColumnarValue::Scalar(value) => Some(value), + ColumnarValue::Array(array) => ScalarValue::try_from_array(&array, 0).ok(), + } +} + +/// For a nondecreasing, null-preserving transform, adjacent partition images +/// are disjoint when the values immediately across each split remain distinct. +/// The open side is below an ASC split and above a DESC split. +fn range_transform_keeps_groups_local( + function: &Arc, + range_key: &Arc, + range: &RangePartitioning, + key_index: usize, + schema: &Schema, +) -> Option<()> { + let range_type = range_key.data_type(schema).ok()?; + range.split_points().iter().try_for_each(|split_point| { + let split_value = split_point.values().get(key_index)?; + if split_value.data_type() != range_type { + return None; + } + let adjacent = if range.ordering()[key_index].options.descending { + checked_successor(split_value) + } else { + checked_predecessor(split_value) + }?; + let at_split = evaluate_with_range_value(function, range_key, split_value)?; + let across_split = evaluate_with_range_value(function, range_key, &adjacent)?; + if at_split.is_null() + || across_split.is_null() + || at_split.data_type() != across_split.data_type() + || at_split == across_split + { + return None; + } + + Some(()) + }) +} + +fn is_audited_range_transform( + candidate: &Arc, + range_key: &Arc, + range: &RangePartitioning, + key_index: usize, + eq_properties: &EquivalenceProperties, +) -> bool { + let Some(function) = candidate.downcast_ref::() else { + return false; + }; + let schema = eq_properties.schema(); + if datafusion_physical_expr_common::physical_expr::is_volatile(candidate) + || !function + .supports_range_partitioning_analysis(schema) + .unwrap_or(false) + || !eq_properties.is_same_direction_ordered_transform(candidate, range_key) + { + return false; + } + + let mut range_argument_count = 0; + for argument in function.args() { + if equivalent_expr(argument, range_key, eq_properties) { + range_argument_count += 1; + } else if !argument.is::() { + return false; + } + } + + range_argument_count == 1 + && range_transform_keeps_groups_local( + candidate, range_key, range, key_index, schema, + ) + .is_some() +} + +fn range_transform_satisfaction( + range: &RangePartitioning, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + allow_subset: bool, +) -> PartitioningSatisfaction { + if required_exprs.is_empty() || range.ordering().len() > required_exprs.len() { + return PartitioningSatisfaction::NotSatisfied; + } + + let mut unmatched_required = (0..required_exprs.len()).collect::>(); + let last_range_key = range.ordering().len() - 1; + let mut used_transform = false; + + for (key_index, range_sort) in range.ordering().iter().enumerate() { + let mut match_position = unmatched_required.iter().position(|required_index| { + equivalent_expr( + &range_sort.expr, + &required_exprs[*required_index], + eq_properties, + ) + }); + if match_position.is_none() && key_index == last_range_key { + match_position = unmatched_required.iter().position(|required_index| { + is_audited_range_transform( + &required_exprs[*required_index], + &range_sort.expr, + range, + key_index, + eq_properties, + ) + }); + } + let Some(match_position) = match_position else { + return PartitioningSatisfaction::NotSatisfied; + }; + used_transform |= !equivalent_expr( + &range_sort.expr, + &required_exprs[unmatched_required[match_position]], + eq_properties, + ); + unmatched_required.swap_remove(match_position); + } + + if used_transform && (unmatched_required.is_empty() || allow_subset) { + PartitioningSatisfaction::KeyLocal + } else { + PartitioningSatisfaction::NotSatisfied + } +} + /// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PartitioningSatisfaction { @@ -541,6 +707,12 @@ pub enum PartitioningSatisfaction { Exact, /// The partitioning satisfies the distribution requirement via subset logic Subset, + /// Equal values of the required keys are partition-local, but the concrete + /// partition layout is not claimed to match the requested distribution. + /// + /// This is sufficient for independent keyed aggregation. It is not general + /// distribution satisfaction and must not be used for co-partitioned joins. + KeyLocal, } impl PartitioningSatisfaction { @@ -548,6 +720,11 @@ impl PartitioningSatisfaction { matches!(self, Self::Exact | Self::Subset) } + /// Returns whether equal required-key tuples cannot occur in two partitions. + pub fn is_key_local(&self) -> bool { + !matches!(self, Self::NotSatisfied) + } + pub fn is_subset(&self) -> bool { *self == Self::Subset } @@ -590,8 +767,15 @@ impl Partitioning { == PartitioningSatisfaction::Exact } - /// Returns how this [`Partitioning`] satisfies the partitioning scheme mandated - /// by the `required` [`Distribution`]. + /// Returns the relationship between this [`Partitioning`] and the + /// `required` [`Distribution`]. + /// + /// [`PartitioningSatisfaction::KeyLocal`] is deliberately weaker than + /// distribution satisfaction: it only proves that equal required-key tuples + /// cannot occur in different partitions. Callers that require a concrete + /// partition layout must use [`PartitioningSatisfaction::is_satisfied`]. + /// Operators that only require independent processing of each key may use + /// [`PartitioningSatisfaction::is_key_local`]. #[expect( deprecated, reason = "HashPartitioned is accepted during the KeyPartitioned migration" @@ -630,12 +814,22 @@ impl Partitioning { .iter() .map(|sort_expr| Arc::clone(&sort_expr.expr)) .collect::>(); - Self::key_satisfaction( + let satisfaction = Self::key_satisfaction( &partition_exprs, required_exprs, eq_properties, allow_subset, - ) + ); + if satisfaction.is_satisfied() { + satisfaction + } else { + range_transform_satisfaction( + range, + required_exprs, + eq_properties, + allow_subset, + ) + } } Partitioning::RoundRobinBatch(_) | Partitioning::UnknownPartitioning(_) => { @@ -991,7 +1185,8 @@ mod tests { use crate::projection::ProjectionTargets; use arrow::compute::SortOptions; - use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; + use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, ScalarValue}; struct PartitioningTestFixture { @@ -1112,6 +1307,459 @@ mod tests { ); } + fn date_trunc_of( + timestamp: Arc, + precision: &str, + timezone: Option<&str>, + ) -> Arc { + let timezone = timezone.map(Into::into); + Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![ + Arc::new(Literal::new(ScalarValue::Utf8(Some(precision.into())))), + timestamp, + ], + Field::new( + "date_trunc", + DataType::Timestamp(TimeUnit::Nanosecond, timezone), + true, + ) + .into(), + Arc::new(ConfigOptions::default()), + )) + } + + fn date_bin_of(timestamp: Arc) -> Arc { + Arc::new(ScalarFunctionExpr::new( + "date_bin", + datafusion_functions::datetime::date_bin(), + vec![ + Arc::new(Literal::new(ScalarValue::new_interval_mdn( + 0, + 0, + 60_000_000_000, + ))), + timestamp, + ], + Field::new( + "date_bin", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + ) + .into(), + Arc::new(ConfigOptions::default()), + )) + } + + fn timestamp_split(value: Option, timezone: Option<&str>) -> SplitPoint { + SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + value, + timezone.map(Into::into), + )]) + } + + fn assert_date_trunc_partition_images() + -> Result<()> { + use arrow::array::{PrimitiveArray, RecordBatch}; + use datafusion_common::HashSet; + + let ticks_per_second = match T::UNIT { + TimeUnit::Second => 1, + TimeUnit::Millisecond => 1_000, + TimeUnit::Microsecond => 1_000_000, + TimeUnit::Nanosecond => 1_000_000_000, + }; + let hour = 3600 * ticks_per_second; + let data_type = DataType::Timestamp(T::UNIT, None); + let schema = + Arc::new(Schema::new(vec![Field::new("ts", data_type.clone(), true)])); + let timestamp: Arc = Arc::new(Column::new("ts", 0)); + let properties = EquivalenceProperties::new(Arc::clone(&schema)); + + let mut cases = vec![ + ("hour", false, vec![-hour, 0, hour], true), + ("hour", false, vec![-hour + 1, 1, hour + 1], false), + ("day", false, vec![-hour, 0, hour], false), + ("hour", true, vec![-hour, 0, hour], false), + ("hour", true, vec![-hour - 1, -1, hour - 1], true), + ]; + for (precision, divisor) in [("millisecond", 1_000), ("microsecond", 1_000_000)] { + let width = (ticks_per_second / divisor).max(1); + cases.extend([ + (precision, false, vec![-width, 0, width], true), + // In particular, microsecond input with millisecond precision + // must reject split -999: both adjacent array values map to -1000. + (precision, false, vec![-width + 1], width == 1), + (precision, true, vec![-width, 0, width], width == 1), + (precision, true, vec![-width - 1, -1, width - 1], true), + ]); + } + for (precision, descending, source_splits, expect_disjoint) in cases { + for nulls_first in [false, true] { + let options = SortOptions::new(descending, nulls_first); + let mut splits = source_splits.clone(); + if descending { + splits.reverse(); + } + let range = Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new(Arc::clone(×tamp), options)].into(), + splits + .iter() + .map(|value| { + SplitPoint::new(vec![ScalarValue::new_timestamp::( + Some(*value), + None, + )]) + }) + .collect(), + )?); + let function: Arc = Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![ + Arc::new(Literal::new(ScalarValue::from(precision))), + Arc::clone(×tamp), + ], + Field::new("bucket", data_type.clone(), true).into(), + Arc::new(ConfigOptions::default()), + )); + let required = Distribution::KeyPartitioned(vec![Arc::clone(&function)]); + assert_eq!( + range + .satisfaction(&required, &properties, false) + .is_key_local(), + expect_disjoint + ); + + // Assign rows by source-domain comparisons, independently of + // the optimizer's adjacency check, then evaluate whole arrays. + let mut partitions = vec![vec![]; splits.len() + 1]; + let rows = splits + .iter() + .flat_map(|split| [Some(split - 1), Some(*split), Some(split + 1)]) + .chain([None, None]); + for value in rows { + let index = match value { + None if nulls_first => 0, + None => splits.len(), + Some(value) => splits + .iter() + .filter(|split| { + if descending { + value <= **split + } else { + value >= **split + } + }) + .count(), + }; + partitions[index].push(value); + } + let images = partitions + .into_iter() + .map(|values| { + let input = Arc::new(PrimitiveArray::::from_iter(values)); + let batch = + RecordBatch::try_new(Arc::clone(&schema), vec![input])?; + let output = + function.evaluate(&batch)?.into_array(batch.num_rows())?; + (0..output.len()) + .map(|i| ScalarValue::try_from_array(&output, i)) + .collect::>>() + }) + .collect::>>()?; + let disjoint = images.iter().enumerate().all(|(i, left)| { + images + .iter() + .skip(i + 1) + .all(|right| left.is_disjoint(right)) + }); + assert_eq!( + disjoint, + expect_disjoint, + "{precision}, {:?}, {options:?}, splits={splits:?}", + T::UNIT + ); + } + } + Ok(()) + } + + #[test] + fn date_trunc_partition_images_are_disjoint_only_for_safe_splits() -> Result<()> { + use arrow::array::types::{ + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, + }; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::() + } + + #[test] + fn range_transform_requires_literal_parameters() -> Result<()> { + use crate::expressions::CastExpr; + let fixture = PartitioningTestFixture::new(vec![( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let function = date_trunc_of(fixture.col(0), "hour", None); + let scalar_but_not_literal: Arc = Arc::new(CastExpr::new( + Arc::new(Literal::new(ScalarValue::from("hour"))), + DataType::Utf8View, + None, + )); + let function = + function.with_new_children(vec![scalar_but_not_literal, fixture.col(0)])?; + let range = fixture.range_partitioning([0], vec![timestamp_split(Some(0), None)]); + assert_eq!( + range.satisfaction( + &Distribution::KeyPartitioned(vec![function]), + &fixture.eq_properties, + false, + ), + PartitioningSatisfaction::NotSatisfied + ); + Ok(()) + } + + #[test] + fn range_transform_satisfaction_is_key_local_only() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![ + ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), + ("tag", DataType::Utf8), + ])?; + let hour = 1_704_070_800_000_000_000; + let aligned = + fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]); + let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); + + let exact_keys = Distribution::KeyPartitioned(vec![Arc::clone(&trunc_hour)]); + let satisfaction = + aligned.satisfaction(&exact_keys, &fixture.eq_properties, false); + assert_eq!(satisfaction, PartitioningSatisfaction::KeyLocal); + assert!(satisfaction.is_key_local()); + assert!(!satisfaction.is_satisfied()); + + assert_satisfaction( + "additional grouping keys require subset permission", + &aligned, + &Distribution::KeyPartitioned(vec![trunc_hour, fixture.col(1)]), + &fixture.eq_properties, + PartitioningSatisfaction::KeyLocal, + PartitioningSatisfaction::NotSatisfied, + ); + + assert_satisfaction( + "date_bin remains default-denied", + &aligned, + &Distribution::KeyPartitioned(vec![date_bin_of(fixture.col(0))]), + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + + let trunc_day = date_trunc_of(fixture.col(0), "day", None); + assert_satisfaction( + "an hour split straddles a day bucket", + &aligned, + &Distribution::KeyPartitioned(vec![trunc_day]), + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + + let two_aligned = fixture.range_partitioning( + [0], + vec![ + timestamp_split(Some(hour), None), + timestamp_split(Some(hour + 3_600_000_000_000), None), + ], + ); + assert_satisfaction( + "every split must be aligned", + &two_aligned, + &exact_keys, + &fixture.eq_properties, + PartitioningSatisfaction::KeyLocal, + PartitioningSatisfaction::KeyLocal, + ); + + let unaligned = + fixture.range_partitioning([0], vec![timestamp_split(Some(hour + 1), None)]); + assert_satisfaction( + "an unaligned split straddles an hour bucket", + &unaligned, + &exact_keys, + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + Ok(()) + } + + #[test] + fn range_transform_supports_only_the_final_compound_key() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![ + ("key", DataType::Utf8), + ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), + ])?; + let hour = 1_704_070_800_000_000_000; + let split = SplitPoint::new(vec![ + ScalarValue::Utf8(Some("m".into())), + ScalarValue::TimestampNanosecond(Some(hour), None), + ]); + let range = fixture.range_partitioning([0, 1], vec![split]); + assert_satisfaction( + "the final compound key can be transformed", + &range, + &Distribution::KeyPartitioned(vec![ + fixture.col(0), + date_trunc_of(fixture.col(1), "hour", None), + ]), + &fixture.eq_properties, + PartitioningSatisfaction::KeyLocal, + PartitioningSatisfaction::KeyLocal, + ); + + let split = SplitPoint::new(vec![ + ScalarValue::TimestampNanosecond(Some(hour), None), + ScalarValue::Utf8(Some("m".into())), + ]); + let range = fixture.range_partitioning([1, 0], vec![split]); + assert_satisfaction( + "a non-final compound key cannot be transformed", + &range, + &Distribution::KeyPartitioned(vec![ + date_trunc_of(fixture.col(1), "hour", None), + fixture.col(0), + ]), + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + Ok(()) + } + + #[test] + fn range_transform_satisfaction_fails_closed() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); + let required = Distribution::KeyPartitioned(vec![trunc_hour]); + + for (description, split) in [ + ("null", timestamp_split(None, None)), + ("minimum", timestamp_split(Some(i64::MIN), None)), + ] { + assert_satisfaction( + description, + &fixture.range_partitioning([0], vec![split]), + &required, + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + } + + let mismatched_split = Partitioning::Range(RangePartitioning::new( + fixture.range_ordering([0]), + vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])], + )); + assert_satisfaction( + "split scalar type must match the range key", + &mismatched_split, + &required, + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + + let invalid_precision = Distribution::KeyPartitioned(vec![date_trunc_of( + fixture.col(0), + "not-a-precision", + None, + )]); + let aligned = fixture.range_partitioning( + [0], + vec![timestamp_split(Some(1_704_070_800_000_000_000), None)], + ); + assert_satisfaction( + "split evaluation errors fail closed", + &aligned, + &invalid_precision, + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + + let timezone_fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + )])?; + let timezone_trunc = date_trunc_of(timezone_fixture.col(0), "hour", Some("UTC")); + let timezone_range = timezone_fixture.range_partitioning( + [0], + vec![timestamp_split( + Some(1_704_070_800_000_000_000), + Some("UTC"), + )], + ); + assert_satisfaction( + "timezone-aware date_trunc is not audited", + &timezone_range, + &Distribution::KeyPartitioned(vec![timezone_trunc]), + &timezone_fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + Ok(()) + } + + #[test] + fn descending_range_uses_the_open_side_successor() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let hour = 1_704_070_800_000_000_000; + let ordering: LexOrdering = + [fixture.range_sort_expr(0, SortOptions::new(true, false))].into(); + let required = Distribution::KeyPartitioned(vec![date_trunc_of( + fixture.col(0), + "hour", + None, + )]); + + assert_satisfaction( + "the successor of an hour boundary is in the same bucket", + &fixture.range_partitioning_with_ordering( + ordering.clone(), + vec![timestamp_split(Some(hour), None)], + ), + &required, + &fixture.eq_properties, + PartitioningSatisfaction::NotSatisfied, + PartitioningSatisfaction::NotSatisfied, + ); + assert_satisfaction( + "the successor of the final nanosecond starts a new bucket", + &fixture.range_partitioning_with_ordering( + ordering, + vec![timestamp_split(Some(hour + 3_600_000_000_000 - 1), None)], + ), + &required, + &fixture.eq_properties, + PartitioningSatisfaction::KeyLocal, + PartitioningSatisfaction::KeyLocal, + ); + Ok(()) + } + #[test] #[expect( deprecated, diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index f7bd5469cd71e..e9f8c0257f935 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -158,6 +158,22 @@ impl ScalarFunctionExpr { &self.config_options } + /// Returns whether this function expression's argument types have been + /// explicitly audited as candidates for range-partitioning analysis. + pub fn supports_range_partitioning_analysis( + &self, + input_schema: &Schema, + ) -> Result { + let argument_types = self + .args + .iter() + .map(|argument| argument.data_type(input_schema)) + .collect::>>()?; + Ok(self + .fun + .supports_range_partitioning_analysis(&argument_types)) + } + /// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr /// and verify that its inner function is of type T. /// If the downcast fails, or the function is not of type T, returns `None`. @@ -641,6 +657,40 @@ mod tests { assert_eq!(expr.get_properties(&[child]).unwrap().range, bounds); assert_eq!(expr.evaluate_bounds(&[&bounds]).unwrap(), bounds); } + + fn supports_range_partitioning_analysis( + &self, + argument_types: &[DataType], + ) -> bool { + argument_types == [DataType::Float32] + } + } + + #[test] + fn test_range_partitioning_analysis_support_uses_argument_types() { + let function = Arc::new(ScalarUDF::from(MockScalarUDF { + signature: Signature::any(1, Volatility::Immutable), + })); + let config_options = Arc::new(ConfigOptions::new()); + + for (data_type, expected) in [(DataType::Float32, true), (DataType::Int32, false)] + { + let schema = Schema::new(vec![Field::new("a", data_type, false)]); + let expression = ScalarFunctionExpr::try_new( + Arc::clone(&function), + vec![Arc::new(Column::new("a", 0))], + &schema, + Arc::clone(&config_options), + ) + .unwrap(); + + assert_eq!( + expression + .supports_range_partitioning_analysis(&schema) + .unwrap(), + expected + ); + } } #[test] From c4d8ecb9ae73ec7fca143b8dce9f72e7bc9b402a Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:45:18 +0000 Subject: [PATCH 02/57] docs: explain range transformation locality proof --- datafusion/physical-expr/src/partitioning.rs | 42 ++++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 13235874a176f..4ad243ae88930 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -576,9 +576,45 @@ fn evaluate_with_range_value( } } -/// For a nondecreasing, null-preserving transform, adjacent partition images -/// are disjoint when the values immediately across each split remain distinct. -/// The open side is below an ASC split and above a DESC split. +/// Determines whether a nondecreasing transform keeps equal output keys local. +/// +/// For one ascending range key, split points `x₁, ..., xₙ` define: +/// +/// ```text +/// P₀ P₁ Pₙ +/// (-∞, x₁) [x₁, x₂) ... [xₙ, +∞) +/// │ │ │ │ +/// └─ x₁ ─┘ └─ xₙ ─┘ +/// ``` +/// +/// Aggregation can reuse these partitions after applying `f` only if no output +/// key occurs in two partitions. In other words, the partition images must be +/// disjoint: `f(Pᵢ) ∩ f(Pⱼ) = ∅` for `i ≠ j`. +/// +/// Monotonicity alone is insufficient. It guarantees only that values do not +/// reverse order across a split: +/// +/// ```text +/// f(predecessor(xᵢ)) ≤ f(xᵢ) +/// ``` +/// +/// Equality means a group crosses the split. For example, an hourly truncation +/// maps timestamps immediately before and at a `01:30` split to the same +/// `01:00` bucket. A split at `01:00` is safe because its predecessor maps to +/// `00:00`, while the split itself maps to `01:00`. +/// +/// Timestamp values form a discrete domain. For every `x < xᵢ`, monotonicity +/// gives `f(x) ≤ f(predecessor(xᵢ))`; for every `y ≥ xᵢ`, it gives +/// `f(xᵢ) ≤ f(y)`. Therefore checking +/// +/// ```text +/// f(predecessor(xᵢ)) < f(xᵢ) +/// ``` +/// +/// at every split proves all partition images are disjoint. Descending ranges +/// use the mirrored check with `successor(xᵢ)`, because their open side lies +/// above the split. Null preservation is also required so the transform cannot +/// introduce a null group outside the source partition selected by null order. fn range_transform_keeps_groups_local( function: &Arc, range_key: &Arc, From cb3ec1091694d4eba564c2a50eded70fb2de6a30 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:48:58 +0000 Subject: [PATCH 03/57] docs: clarify adjacent range boundary checks --- datafusion/physical-expr/src/partitioning.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 4ad243ae88930..f68d24ff228d9 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -611,10 +611,15 @@ fn evaluate_with_range_value( /// f(predecessor(xᵢ)) < f(xᵢ) /// ``` /// -/// at every split proves all partition images are disjoint. Descending ranges -/// use the mirrored check with `successor(xᵢ)`, because their open side lies -/// above the split. Null preservation is also required so the transform cannot -/// introduce a null group outside the source partition selected by null order. +/// at every split proves all partition images are disjoint. The successor of +/// an ascending split is already in the same partition as `xᵢ`, so checking it +/// would not say anything about groups crossing that split. +/// +/// Descending ranges use the mirrored check with `successor(xᵢ)`, because their +/// open side lies above the split; there, the predecessor is already in the +/// same partition as `xᵢ`. Null preservation is also required so the transform +/// cannot introduce a null group outside the source partition selected by null +/// order. fn range_transform_keeps_groups_local( function: &Arc, range_key: &Arc, From 568f77ef1b59b6681775417046eeeecc4d0215a9 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:17:52 +0000 Subject: [PATCH 04/57] refactor: make adjacent value checks explicit --- .../expr-common/src/interval_arithmetic.rs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 3e1ba77a2504d..08930c7f00cf8 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -1241,29 +1241,29 @@ fn prev_value(value: ScalarValue) -> ScalarValue { /// Returns the previous distinct value, or `None` for nulls, non-finite /// floats, type minima, and types without a discrete predecessor. pub fn checked_predecessor(value: &ScalarValue) -> Option { - checked_adjacent(value, false) + if !is_finite_non_null(value) { + return None; + } + + let predecessor = prev_value(value.clone()); + (!predecessor.is_null() && predecessor != *value).then_some(predecessor) } /// Returns the next distinct value, or `None` for nulls, non-finite floats, /// type maxima, and types without a discrete successor. pub fn checked_successor(value: &ScalarValue) -> Option { - checked_adjacent(value, true) -} - -fn checked_adjacent(value: &ScalarValue, successor: bool) -> Option { - if value.is_null() - || matches!(value, ScalarValue::Float32(Some(v)) if !v.is_finite()) - || matches!(value, ScalarValue::Float64(Some(v)) if !v.is_finite()) - { + if !is_finite_non_null(value) { return None; } - let adjacent = if successor { - next_value(value.clone()) - } else { - prev_value(value.clone()) - }; - (!adjacent.is_null() && adjacent != *value).then_some(adjacent) + let successor = next_value(value.clone()); + (!successor.is_null() && successor != *value).then_some(successor) +} + +fn is_finite_non_null(value: &ScalarValue) -> bool { + !value.is_null() + && !matches!(value, ScalarValue::Float32(Some(v)) if !v.is_finite()) + && !matches!(value, ScalarValue::Float64(Some(v)) if !v.is_finite()) } trait OneTrait: Sized + std::ops::Add + std::ops::Sub { From cab108cbd3d018e06516cc3114c019d0752e2055 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:22:01 +0000 Subject: [PATCH 05/57] refactor: clarify adjacent input validation --- datafusion/expr-common/src/interval_arithmetic.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 08930c7f00cf8..88c1ff7ca9cc5 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -1261,9 +1261,11 @@ pub fn checked_successor(value: &ScalarValue) -> Option { } fn is_finite_non_null(value: &ScalarValue) -> bool { - !value.is_null() - && !matches!(value, ScalarValue::Float32(Some(v)) if !v.is_finite()) - && !matches!(value, ScalarValue::Float64(Some(v)) if !v.is_finite()) + match value { + ScalarValue::Float32(Some(value)) => value.is_finite(), + ScalarValue::Float64(Some(value)) => value.is_finite(), + _ => !value.is_null(), + } } trait OneTrait: Sized + std::ops::Add + std::ops::Sub { From 5bd8a8576bf77b42a559681e229d9386f9f100e3 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:24:15 +0000 Subject: [PATCH 06/57] test: expand checked adjacent value coverage --- .../expr-common/src/interval_arithmetic.rs | 278 ++++++++++++++++-- 1 file changed, 252 insertions(+), 26 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 88c1ff7ca9cc5..7ae56fcd379f2 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -2626,32 +2626,258 @@ mod tests { #[test] fn test_checked_adjacent_values() { - assert_eq!( - checked_predecessor(&ScalarValue::Int64(Some(10))), - Some(ScalarValue::Int64(Some(9))) - ); - assert_eq!( - checked_successor(&ScalarValue::Int64(Some(10))), - Some(ScalarValue::Int64(Some(11))) - ); - assert_eq!(checked_predecessor(&ScalarValue::Int64(None)), None); - assert_eq!( - checked_predecessor(&ScalarValue::Int64(Some(i64::MIN))), - None - ); - assert_eq!(checked_successor(&ScalarValue::Int64(Some(i64::MAX))), None); - assert_eq!( - checked_predecessor(&ScalarValue::Utf8(Some("a".into()))), - None - ); - assert_eq!( - checked_successor(&ScalarValue::Float64(Some(f64::NAN))), - None - ); - assert_eq!( - checked_predecessor(&ScalarValue::Float32(Some(f32::NEG_INFINITY))), - None - ); + use ScalarValue::*; + + struct Case { + name: &'static str, + value: ScalarValue, + predecessor: Option, + successor: Option, + } + + let day_time = arrow::datatypes::IntervalDayTime { + days: 2, + milliseconds: 3, + }; + let previous_day_time = arrow::datatypes::IntervalDayTime { + days: 2, + milliseconds: 2, + }; + let next_day_time = arrow::datatypes::IntervalDayTime { + days: 2, + milliseconds: 4, + }; + let month_day_nano = arrow::datatypes::IntervalMonthDayNano { + months: 2, + days: 3, + nanoseconds: 4, + }; + let previous_month_day_nano = arrow::datatypes::IntervalMonthDayNano { + months: 2, + days: 3, + nanoseconds: 3, + }; + let next_month_day_nano = arrow::datatypes::IntervalMonthDayNano { + months: 2, + days: 3, + nanoseconds: 5, + }; + + let cases = vec![ + Case { + name: "int8", + value: Int8(Some(-4)), + predecessor: Some(Int8(Some(-5))), + successor: Some(Int8(Some(-3))), + }, + Case { + name: "int16", + value: Int16(Some(10)), + predecessor: Some(Int16(Some(9))), + successor: Some(Int16(Some(11))), + }, + Case { + name: "int32", + value: Int32(Some(-10)), + predecessor: Some(Int32(Some(-11))), + successor: Some(Int32(Some(-9))), + }, + Case { + name: "int64", + value: Int64(Some(10)), + predecessor: Some(Int64(Some(9))), + successor: Some(Int64(Some(11))), + }, + Case { + name: "uint8 minimum", + value: UInt8(Some(0)), + predecessor: None, + successor: Some(UInt8(Some(1))), + }, + Case { + name: "uint16", + value: UInt16(Some(10)), + predecessor: Some(UInt16(Some(9))), + successor: Some(UInt16(Some(11))), + }, + Case { + name: "uint32", + value: UInt32(Some(10)), + predecessor: Some(UInt32(Some(9))), + successor: Some(UInt32(Some(11))), + }, + Case { + name: "uint64 maximum", + value: UInt64(Some(u64::MAX)), + predecessor: Some(UInt64(Some(u64::MAX - 1))), + successor: None, + }, + Case { + name: "float32", + value: Float32(Some(1.0)), + predecessor: Some(Float32(Some(next_down(1.0_f32)))), + successor: Some(Float32(Some(next_up(1.0_f32)))), + }, + Case { + name: "float64", + value: Float64(Some(-1.0)), + predecessor: Some(Float64(Some(next_down(-1.0_f64)))), + successor: Some(Float64(Some(next_up(-1.0_f64)))), + }, + Case { + name: "duration second", + value: DurationSecond(Some(10)), + predecessor: Some(DurationSecond(Some(9))), + successor: Some(DurationSecond(Some(11))), + }, + Case { + name: "duration millisecond", + value: DurationMillisecond(Some(10)), + predecessor: Some(DurationMillisecond(Some(9))), + successor: Some(DurationMillisecond(Some(11))), + }, + Case { + name: "duration microsecond", + value: DurationMicrosecond(Some(10)), + predecessor: Some(DurationMicrosecond(Some(9))), + successor: Some(DurationMicrosecond(Some(11))), + }, + Case { + name: "duration nanosecond", + value: DurationNanosecond(Some(10)), + predecessor: Some(DurationNanosecond(Some(9))), + successor: Some(DurationNanosecond(Some(11))), + }, + Case { + name: "timestamp second", + value: TimestampSecond(Some(10), None), + predecessor: Some(TimestampSecond(Some(9), None)), + successor: Some(TimestampSecond(Some(11), None)), + }, + Case { + name: "timestamp millisecond with timezone", + value: TimestampMillisecond(Some(10), Some("UTC".into())), + predecessor: Some(TimestampMillisecond(Some(9), Some("UTC".into()))), + successor: Some(TimestampMillisecond(Some(11), Some("UTC".into()))), + }, + Case { + name: "timestamp microsecond", + value: TimestampMicrosecond(Some(10), None), + predecessor: Some(TimestampMicrosecond(Some(9), None)), + successor: Some(TimestampMicrosecond(Some(11), None)), + }, + Case { + name: "timestamp nanosecond", + value: TimestampNanosecond(Some(10), None), + predecessor: Some(TimestampNanosecond(Some(9), None)), + successor: Some(TimestampNanosecond(Some(11), None)), + }, + Case { + name: "interval year month", + value: IntervalYearMonth(Some(10)), + predecessor: Some(IntervalYearMonth(Some(9))), + successor: Some(IntervalYearMonth(Some(11))), + }, + Case { + name: "interval day time", + value: IntervalDayTime(Some(day_time)), + predecessor: Some(IntervalDayTime(Some(previous_day_time))), + successor: Some(IntervalDayTime(Some(next_day_time))), + }, + Case { + name: "interval month day nano", + value: IntervalMonthDayNano(Some(month_day_nano)), + predecessor: Some(IntervalMonthDayNano(Some(previous_month_day_nano))), + successor: Some(IntervalMonthDayNano(Some(next_month_day_nano))), + }, + Case { + name: "signed minimum", + value: Int64(Some(i64::MIN)), + predecessor: None, + successor: Some(Int64(Some(i64::MIN + 1))), + }, + Case { + name: "signed maximum", + value: Int64(Some(i64::MAX)), + predecessor: Some(Int64(Some(i64::MAX - 1))), + successor: None, + }, + Case { + name: "timestamp minimum", + value: TimestampNanosecond(Some(i64::MIN), None), + predecessor: None, + successor: Some(TimestampNanosecond(Some(i64::MIN + 1), None)), + }, + Case { + name: "timestamp maximum", + value: TimestampNanosecond(Some(i64::MAX), None), + predecessor: Some(TimestampNanosecond(Some(i64::MAX - 1), None)), + successor: None, + }, + Case { + name: "typed null", + value: Int64(None), + predecessor: None, + successor: None, + }, + Case { + name: "float32 negative infinity", + value: Float32(Some(f32::NEG_INFINITY)), + predecessor: None, + successor: None, + }, + Case { + name: "float64 positive infinity", + value: Float64(Some(f64::INFINITY)), + predecessor: None, + successor: None, + }, + Case { + name: "float64 NaN", + value: Float64(Some(f64::NAN)), + predecessor: None, + successor: None, + }, + Case { + name: "utf8 has no discrete adjacent value", + value: Utf8(Some("a".into())), + predecessor: None, + successor: None, + }, + Case { + name: "boolean has no discrete adjacent value", + value: Boolean(Some(false)), + predecessor: None, + successor: None, + }, + Case { + name: "date has no supported adjacent value", + value: Date32(Some(1)), + predecessor: None, + successor: None, + }, + Case { + name: "decimal has no supported adjacent value", + value: Decimal128(Some(100), 10, 2), + predecessor: None, + successor: None, + }, + ]; + + for case in cases { + assert_eq!( + checked_predecessor(&case.value), + case.predecessor, + "predecessor for {}", + case.name + ); + assert_eq!( + checked_successor(&case.value), + case.successor, + "successor for {}", + case.name + ); + } } #[test] From 45824bfc386b760038b5419cbdb41e43d2e8049e Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:30:53 +0000 Subject: [PATCH 07/57] test: focus checked adjacent value coverage --- .../expr-common/src/interval_arithmetic.rs | 284 +++--------------- 1 file changed, 41 insertions(+), 243 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 7ae56fcd379f2..69fe894125937 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -2628,254 +2628,52 @@ mod tests { fn test_checked_adjacent_values() { use ScalarValue::*; - struct Case { - name: &'static str, - value: ScalarValue, - predecessor: Option, - successor: Option, - } - - let day_time = arrow::datatypes::IntervalDayTime { - days: 2, - milliseconds: 3, - }; - let previous_day_time = arrow::datatypes::IntervalDayTime { - days: 2, - milliseconds: 2, - }; - let next_day_time = arrow::datatypes::IntervalDayTime { - days: 2, - milliseconds: 4, - }; - let month_day_nano = arrow::datatypes::IntervalMonthDayNano { - months: 2, - days: 3, - nanoseconds: 4, - }; - let previous_month_day_nano = arrow::datatypes::IntervalMonthDayNano { - months: 2, - days: 3, - nanoseconds: 3, - }; - let next_month_day_nano = arrow::datatypes::IntervalMonthDayNano { - months: 2, - days: 3, - nanoseconds: 5, - }; - - let cases = vec![ - Case { - name: "int8", - value: Int8(Some(-4)), - predecessor: Some(Int8(Some(-5))), - successor: Some(Int8(Some(-3))), - }, - Case { - name: "int16", - value: Int16(Some(10)), - predecessor: Some(Int16(Some(9))), - successor: Some(Int16(Some(11))), - }, - Case { - name: "int32", - value: Int32(Some(-10)), - predecessor: Some(Int32(Some(-11))), - successor: Some(Int32(Some(-9))), - }, - Case { - name: "int64", - value: Int64(Some(10)), - predecessor: Some(Int64(Some(9))), - successor: Some(Int64(Some(11))), - }, - Case { - name: "uint8 minimum", - value: UInt8(Some(0)), - predecessor: None, - successor: Some(UInt8(Some(1))), - }, - Case { - name: "uint16", - value: UInt16(Some(10)), - predecessor: Some(UInt16(Some(9))), - successor: Some(UInt16(Some(11))), - }, - Case { - name: "uint32", - value: UInt32(Some(10)), - predecessor: Some(UInt32(Some(9))), - successor: Some(UInt32(Some(11))), - }, - Case { - name: "uint64 maximum", - value: UInt64(Some(u64::MAX)), - predecessor: Some(UInt64(Some(u64::MAX - 1))), - successor: None, - }, - Case { - name: "float32", - value: Float32(Some(1.0)), - predecessor: Some(Float32(Some(next_down(1.0_f32)))), - successor: Some(Float32(Some(next_up(1.0_f32)))), - }, - Case { - name: "float64", - value: Float64(Some(-1.0)), - predecessor: Some(Float64(Some(next_down(-1.0_f64)))), - successor: Some(Float64(Some(next_up(-1.0_f64)))), - }, - Case { - name: "duration second", - value: DurationSecond(Some(10)), - predecessor: Some(DurationSecond(Some(9))), - successor: Some(DurationSecond(Some(11))), - }, - Case { - name: "duration millisecond", - value: DurationMillisecond(Some(10)), - predecessor: Some(DurationMillisecond(Some(9))), - successor: Some(DurationMillisecond(Some(11))), - }, - Case { - name: "duration microsecond", - value: DurationMicrosecond(Some(10)), - predecessor: Some(DurationMicrosecond(Some(9))), - successor: Some(DurationMicrosecond(Some(11))), - }, - Case { - name: "duration nanosecond", - value: DurationNanosecond(Some(10)), - predecessor: Some(DurationNanosecond(Some(9))), - successor: Some(DurationNanosecond(Some(11))), - }, - Case { - name: "timestamp second", - value: TimestampSecond(Some(10), None), - predecessor: Some(TimestampSecond(Some(9), None)), - successor: Some(TimestampSecond(Some(11), None)), - }, - Case { - name: "timestamp millisecond with timezone", - value: TimestampMillisecond(Some(10), Some("UTC".into())), - predecessor: Some(TimestampMillisecond(Some(9), Some("UTC".into()))), - successor: Some(TimestampMillisecond(Some(11), Some("UTC".into()))), - }, - Case { - name: "timestamp microsecond", - value: TimestampMicrosecond(Some(10), None), - predecessor: Some(TimestampMicrosecond(Some(9), None)), - successor: Some(TimestampMicrosecond(Some(11), None)), - }, - Case { - name: "timestamp nanosecond", - value: TimestampNanosecond(Some(10), None), - predecessor: Some(TimestampNanosecond(Some(9), None)), - successor: Some(TimestampNanosecond(Some(11), None)), - }, - Case { - name: "interval year month", - value: IntervalYearMonth(Some(10)), - predecessor: Some(IntervalYearMonth(Some(9))), - successor: Some(IntervalYearMonth(Some(11))), - }, - Case { - name: "interval day time", - value: IntervalDayTime(Some(day_time)), - predecessor: Some(IntervalDayTime(Some(previous_day_time))), - successor: Some(IntervalDayTime(Some(next_day_time))), - }, - Case { - name: "interval month day nano", - value: IntervalMonthDayNano(Some(month_day_nano)), - predecessor: Some(IntervalMonthDayNano(Some(previous_month_day_nano))), - successor: Some(IntervalMonthDayNano(Some(next_month_day_nano))), - }, - Case { - name: "signed minimum", - value: Int64(Some(i64::MIN)), - predecessor: None, - successor: Some(Int64(Some(i64::MIN + 1))), - }, - Case { - name: "signed maximum", - value: Int64(Some(i64::MAX)), - predecessor: Some(Int64(Some(i64::MAX - 1))), - successor: None, - }, - Case { - name: "timestamp minimum", - value: TimestampNanosecond(Some(i64::MIN), None), - predecessor: None, - successor: Some(TimestampNanosecond(Some(i64::MIN + 1), None)), - }, - Case { - name: "timestamp maximum", - value: TimestampNanosecond(Some(i64::MAX), None), - predecessor: Some(TimestampNanosecond(Some(i64::MAX - 1), None)), - successor: None, - }, - Case { - name: "typed null", - value: Int64(None), - predecessor: None, - successor: None, - }, - Case { - name: "float32 negative infinity", - value: Float32(Some(f32::NEG_INFINITY)), - predecessor: None, - successor: None, - }, - Case { - name: "float64 positive infinity", - value: Float64(Some(f64::INFINITY)), - predecessor: None, - successor: None, - }, - Case { - name: "float64 NaN", - value: Float64(Some(f64::NAN)), - predecessor: None, - successor: None, - }, - Case { - name: "utf8 has no discrete adjacent value", - value: Utf8(Some("a".into())), - predecessor: None, - successor: None, - }, - Case { - name: "boolean has no discrete adjacent value", - value: Boolean(Some(false)), - predecessor: None, - successor: None, - }, - Case { - name: "date has no supported adjacent value", - value: Date32(Some(1)), - predecessor: None, - successor: None, - }, - Case { - name: "decimal has no supported adjacent value", - value: Decimal128(Some(100), 10, 2), - predecessor: None, - successor: None, - }, + let cases = [ + (Int64(Some(10)), Some(Int64(Some(9))), Some(Int64(Some(11)))), + ( + TimestampNanosecond(Some(10), None), + Some(TimestampNanosecond(Some(9), None)), + Some(TimestampNanosecond(Some(11), None)), + ), + ( + TimestampMillisecond(Some(10), Some("UTC".into())), + Some(TimestampMillisecond(Some(9), Some("UTC".into()))), + Some(TimestampMillisecond(Some(11), Some("UTC".into()))), + ), + ( + Float64(Some(1.0)), + Some(Float64(Some(next_down(1.0_f64)))), + Some(Float64(Some(next_up(1.0_f64)))), + ), + (Int64(Some(i64::MIN)), None, Some(Int64(Some(i64::MIN + 1)))), + (Int64(Some(i64::MAX)), Some(Int64(Some(i64::MAX - 1))), None), + ( + TimestampNanosecond(Some(i64::MIN), None), + None, + Some(TimestampNanosecond(Some(i64::MIN + 1), None)), + ), + ( + TimestampNanosecond(Some(i64::MAX), None), + Some(TimestampNanosecond(Some(i64::MAX - 1), None)), + None, + ), + (Int64(None), None, None), + (Float32(Some(f32::NEG_INFINITY)), None, None), + (Float64(Some(f64::INFINITY)), None, None), + (Float64(Some(f64::NAN)), None, None), + (Utf8(Some("a".into())), None, None), ]; - for case in cases { + for (value, predecessor, successor) in cases { assert_eq!( - checked_predecessor(&case.value), - case.predecessor, - "predecessor for {}", - case.name + checked_predecessor(&value), + predecessor, + "predecessor for {value:?}" ); assert_eq!( - checked_successor(&case.value), - case.successor, - "successor for {}", - case.name + checked_successor(&value), + successor, + "successor for {value:?}" ); } } From 870cf1a24aa196ce27176da6d4ad60b89ac767f3 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:38:50 +0000 Subject: [PATCH 08/57] test: simplify checked adjacent value coverage --- .../expr-common/src/interval_arithmetic.rs | 107 ++++++++++-------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 69fe894125937..469b7531f5723 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -2626,56 +2626,67 @@ mod tests { #[test] fn test_checked_adjacent_values() { - use ScalarValue::*; + let timestamp = ScalarValue::TimestampNanosecond(Some(10), None); + assert_eq!( + checked_predecessor(×tamp), + Some(ScalarValue::TimestampNanosecond(Some(9), None)) + ); + assert_eq!( + checked_successor(×tamp), + Some(ScalarValue::TimestampNanosecond(Some(11), None)) + ); - let cases = [ - (Int64(Some(10)), Some(Int64(Some(9))), Some(Int64(Some(11)))), - ( - TimestampNanosecond(Some(10), None), - Some(TimestampNanosecond(Some(9), None)), - Some(TimestampNanosecond(Some(11), None)), - ), - ( - TimestampMillisecond(Some(10), Some("UTC".into())), - Some(TimestampMillisecond(Some(9), Some("UTC".into()))), - Some(TimestampMillisecond(Some(11), Some("UTC".into()))), - ), - ( - Float64(Some(1.0)), - Some(Float64(Some(next_down(1.0_f64)))), - Some(Float64(Some(next_up(1.0_f64)))), - ), - (Int64(Some(i64::MIN)), None, Some(Int64(Some(i64::MIN + 1)))), - (Int64(Some(i64::MAX)), Some(Int64(Some(i64::MAX - 1))), None), - ( - TimestampNanosecond(Some(i64::MIN), None), - None, - Some(TimestampNanosecond(Some(i64::MIN + 1), None)), - ), - ( - TimestampNanosecond(Some(i64::MAX), None), - Some(TimestampNanosecond(Some(i64::MAX - 1), None)), - None, - ), - (Int64(None), None, None), - (Float32(Some(f32::NEG_INFINITY)), None, None), - (Float64(Some(f64::INFINITY)), None, None), - (Float64(Some(f64::NAN)), None, None), - (Utf8(Some("a".into())), None, None), - ]; + let timestamp_with_timezone = + ScalarValue::TimestampMillisecond(Some(10), Some("UTC".into())); + assert_eq!( + checked_predecessor(×tamp_with_timezone), + Some(ScalarValue::TimestampMillisecond( + Some(9), + Some("UTC".into()) + )) + ); + assert_eq!( + checked_successor(×tamp_with_timezone), + Some(ScalarValue::TimestampMillisecond( + Some(11), + Some("UTC".into()) + )) + ); - for (value, predecessor, successor) in cases { - assert_eq!( - checked_predecessor(&value), - predecessor, - "predecessor for {value:?}" - ); - assert_eq!( - checked_successor(&value), - successor, - "successor for {value:?}" - ); - } + let finite_float = ScalarValue::Float64(Some(1.0)); + assert_eq!( + checked_predecessor(&finite_float), + Some(ScalarValue::Float64(Some(next_down(1.0_f64)))) + ); + assert_eq!( + checked_successor(&finite_float), + Some(ScalarValue::Float64(Some(next_up(1.0_f64)))) + ); + + assert_eq!( + checked_predecessor(&ScalarValue::TimestampNanosecond(Some(i64::MIN), None)), + None + ); + assert_eq!( + checked_successor(&ScalarValue::TimestampNanosecond(Some(i64::MAX), None)), + None + ); + + let null = ScalarValue::TimestampNanosecond(None, None); + assert_eq!(checked_predecessor(&null), None); + assert_eq!(checked_successor(&null), None); + assert_eq!( + checked_predecessor(&ScalarValue::Float32(Some(f32::NEG_INFINITY))), + None + ); + assert_eq!( + checked_successor(&ScalarValue::Float64(Some(f64::NAN))), + None + ); + + let unsupported = ScalarValue::Utf8(Some("a".into())); + assert_eq!(checked_predecessor(&unsupported), None); + assert_eq!(checked_successor(&unsupported), None); } #[test] From 80f5979c8d41551a28719e03082f0285651f7829 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:49:54 +0000 Subject: [PATCH 09/57] docs: clarify range analysis capability goal --- datafusion/expr/src/udf.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 78c94a1ee3ea4..b15b340a00d7b 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -992,8 +992,16 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { Ok(inner(inputs)) } - /// Returns whether this function and argument-type combination may - /// participate in range-partitioning transformation analysis. + /// Returns whether this function and argument-type combination may be + /// analyzed to determine if transforming a range partition key keeps every + /// equal output key within one existing partition. + /// + /// This property allows keyed operators such as grouped aggregation to + /// reuse range-partitioned input after transforming the key, provided the + /// concrete split points prove that no output group crosses a partition + /// boundary. It does not claim that the transformed keys have a particular + /// partition layout or that two inputs are compatibly partitioned for a + /// join. /// /// Ordinary [`SortProperties::Ordered`] metadata is not sufficient for this /// analysis. A non-strictly ordered function can map values from opposite From e43fa5a80834e2ca393cd687c37b279ef1832cc0 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:37:29 +0000 Subject: [PATCH 10/57] refactor: separate range key locality analysis --- .../functions/src/datetime/date_trunc.rs | 17 +- datafusion/physical-expr/src/partitioning.rs | 323 ++++++++---------- 2 files changed, 158 insertions(+), 182 deletions(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 1dd1ac7069a7a..5d86273372ecd 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -451,6 +451,19 @@ impl ScalarUDFImpl for DateTruncFunc { Ok(SortProperties::Unordered) } } + + fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { + // Named timezone transitions can make date_trunc non-monotonic (#25353). + // Keep the audited domain to timezone-less timestamp columns. + matches!( + argument_types, + [ + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, + Timestamp(_, None) + ] + ) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -919,8 +932,8 @@ mod tests { use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos; use arrow::compute::{DatePart, SortOptions}; use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit}; + use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; - use datafusion_common::{Result, ScalarValue}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; @@ -1021,7 +1034,7 @@ mod tests { timestamp: ColumnarValue, timestamp_type: DataType, number_rows: usize, - ) -> Result { + ) -> datafusion_common::Result { DateTruncFunc::new().invoke_with_args(ScalarFunctionArgs { args: vec![ScalarValue::from(granularity).into(), timestamp], arg_fields: vec![ diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index f68d24ff228d9..7fb036c265972 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -620,20 +620,20 @@ fn evaluate_with_range_value( /// same partition as `xᵢ`. Null preservation is also required so the transform /// cannot introduce a null group outside the source partition selected by null /// order. -fn range_transform_keeps_groups_local( +fn range_transform_keeps_keys_local( function: &Arc, range_key: &Arc, range: &RangePartitioning, - key_index: usize, schema: &Schema, ) -> Option<()> { let range_type = range_key.data_type(schema).ok()?; + let sort_options = range.ordering().first().options; range.split_points().iter().try_for_each(|split_point| { - let split_value = split_point.values().get(key_index)?; + let split_value = split_point.values().first()?; if split_value.data_type() != range_type { return None; } - let adjacent = if range.ordering()[key_index].options.descending { + let adjacent = if sort_options.descending { checked_successor(split_value) } else { checked_predecessor(split_value) @@ -656,7 +656,6 @@ fn is_audited_range_transform( candidate: &Arc, range_key: &Arc, range: &RangePartitioning, - key_index: usize, eq_properties: &EquivalenceProperties, ) -> bool { let Some(function) = candidate.downcast_ref::() else { @@ -682,61 +681,25 @@ fn is_audited_range_transform( } range_argument_count == 1 - && range_transform_keeps_groups_local( - candidate, range_key, range, key_index, schema, - ) - .is_some() + && range_transform_keeps_keys_local(candidate, range_key, range, schema).is_some() } -fn range_transform_satisfaction( +fn range_partitioning_keeps_keys_local( range: &RangePartitioning, required_exprs: &[Arc], eq_properties: &EquivalenceProperties, allow_subset: bool, -) -> PartitioningSatisfaction { - if required_exprs.is_empty() || range.ordering().len() > required_exprs.len() { - return PartitioningSatisfaction::NotSatisfied; +) -> bool { + if range.ordering().len() != 1 || required_exprs.is_empty() { + return false; } - let mut unmatched_required = (0..required_exprs.len()).collect::>(); - let last_range_key = range.ordering().len() - 1; - let mut used_transform = false; + let range_key = &range.ordering()[0].expr; + let has_audited_transform = required_exprs.iter().any(|required| { + is_audited_range_transform(required, range_key, range, eq_properties) + }); - for (key_index, range_sort) in range.ordering().iter().enumerate() { - let mut match_position = unmatched_required.iter().position(|required_index| { - equivalent_expr( - &range_sort.expr, - &required_exprs[*required_index], - eq_properties, - ) - }); - if match_position.is_none() && key_index == last_range_key { - match_position = unmatched_required.iter().position(|required_index| { - is_audited_range_transform( - &required_exprs[*required_index], - &range_sort.expr, - range, - key_index, - eq_properties, - ) - }); - } - let Some(match_position) = match_position else { - return PartitioningSatisfaction::NotSatisfied; - }; - used_transform |= !equivalent_expr( - &range_sort.expr, - &required_exprs[unmatched_required[match_position]], - eq_properties, - ); - unmatched_required.swap_remove(match_position); - } - - if used_transform && (unmatched_required.is_empty() || allow_subset) { - PartitioningSatisfaction::KeyLocal - } else { - PartitioningSatisfaction::NotSatisfied - } + has_audited_transform && (required_exprs.len() == 1 || allow_subset) } /// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement. @@ -748,12 +711,6 @@ pub enum PartitioningSatisfaction { Exact, /// The partitioning satisfies the distribution requirement via subset logic Subset, - /// Equal values of the required keys are partition-local, but the concrete - /// partition layout is not claimed to match the requested distribution. - /// - /// This is sufficient for independent keyed aggregation. It is not general - /// distribution satisfaction and must not be used for co-partitioned joins. - KeyLocal, } impl PartitioningSatisfaction { @@ -761,11 +718,6 @@ impl PartitioningSatisfaction { matches!(self, Self::Exact | Self::Subset) } - /// Returns whether equal required-key tuples cannot occur in two partitions. - pub fn is_key_local(&self) -> bool { - !matches!(self, Self::NotSatisfied) - } - pub fn is_subset(&self) -> bool { *self == Self::Subset } @@ -810,13 +762,6 @@ impl Partitioning { /// Returns the relationship between this [`Partitioning`] and the /// `required` [`Distribution`]. - /// - /// [`PartitioningSatisfaction::KeyLocal`] is deliberately weaker than - /// distribution satisfaction: it only proves that equal required-key tuples - /// cannot occur in different partitions. Callers that require a concrete - /// partition layout must use [`PartitioningSatisfaction::is_satisfied`]. - /// Operators that only require independent processing of each key may use - /// [`PartitioningSatisfaction::is_key_local`]. #[expect( deprecated, reason = "HashPartitioned is accepted during the KeyPartitioned migration" @@ -855,22 +800,12 @@ impl Partitioning { .iter() .map(|sort_expr| Arc::clone(&sort_expr.expr)) .collect::>(); - let satisfaction = Self::key_satisfaction( + Self::key_satisfaction( &partition_exprs, required_exprs, eq_properties, allow_subset, - ); - if satisfaction.is_satisfied() { - satisfaction - } else { - range_transform_satisfaction( - range, - required_exprs, - eq_properties, - allow_subset, - ) - } + ) } Partitioning::RoundRobinBatch(_) | Partitioning::UnknownPartitioning(_) => { @@ -881,6 +816,41 @@ impl Partitioning { } } + /// Returns true when it can prove that equal tuples of `required_exprs` + /// cannot occur in different partitions. + /// + /// Unlike [`Self::satisfaction`], this does not claim a concrete partition + /// layout that can be compared with another input. It is sufficient for + /// operators such as grouped aggregation that process each key + /// independently, but must not establish co-partitioning between inputs. + /// Transform analysis is currently limited to single-key range partitioning. + pub fn keeps_keys_local( + &self, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + allow_subset: bool, + ) -> bool { + let required = Distribution::KeyPartitioned(required_exprs.to_vec()); + if self + .satisfaction(&required, eq_properties, allow_subset) + .is_satisfied() + { + return true; + } + + match self { + Self::Range(range) => range_partitioning_keeps_keys_local( + range, + required_exprs, + eq_properties, + allow_subset, + ), + Self::RoundRobinBatch(_) + | Self::Hash(_, _) + | Self::UnknownPartitioning(_) => false, + } + } + fn key_satisfaction( partition_exprs: &[Arc], required_exprs: &[Arc], @@ -1348,6 +1318,26 @@ mod tests { ); } + fn assert_key_locality( + desc: &str, + partitioning: &Partitioning, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + expected_with_subset: bool, + expected_without_subset: bool, + ) { + assert_eq!( + partitioning.keeps_keys_local(required_exprs, eq_properties, true), + expected_with_subset, + "Failed for {desc} with subset enabled" + ); + assert_eq!( + partitioning.keeps_keys_local(required_exprs, eq_properties, false), + expected_without_subset, + "Failed for {desc} with subset disabled" + ); + } + fn date_trunc_of( timestamp: Arc, precision: &str, @@ -1465,11 +1455,8 @@ mod tests { Field::new("bucket", data_type.clone(), true).into(), Arc::new(ConfigOptions::default()), )); - let required = Distribution::KeyPartitioned(vec![Arc::clone(&function)]); assert_eq!( - range - .satisfaction(&required, &properties, false) - .is_key_local(), + range.keeps_keys_local(&[Arc::clone(&function)], &properties, false), expect_disjoint ); @@ -1555,19 +1542,12 @@ mod tests { let function = function.with_new_children(vec![scalar_but_not_literal, fixture.col(0)])?; let range = fixture.range_partitioning([0], vec![timestamp_split(Some(0), None)]); - assert_eq!( - range.satisfaction( - &Distribution::KeyPartitioned(vec![function]), - &fixture.eq_properties, - false, - ), - PartitioningSatisfaction::NotSatisfied - ); + assert!(!range.keeps_keys_local(&[function], &fixture.eq_properties, false)); Ok(()) } #[test] - fn range_transform_satisfaction_is_key_local_only() -> Result<()> { + fn range_transform_establishes_key_locality_only() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![ ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), ("tag", DataType::Utf8), @@ -1577,39 +1557,50 @@ mod tests { fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]); let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); - let exact_keys = Distribution::KeyPartitioned(vec![Arc::clone(&trunc_hour)]); - let satisfaction = - aligned.satisfaction(&exact_keys, &fixture.eq_properties, false); - assert_eq!(satisfaction, PartitioningSatisfaction::KeyLocal); - assert!(satisfaction.is_key_local()); - assert!(!satisfaction.is_satisfied()); + let exact_key_exprs = vec![Arc::clone(&trunc_hour)]; + assert_eq!( + aligned.satisfaction( + &Distribution::KeyPartitioned(exact_key_exprs.clone()), + &fixture.eq_properties, + false, + ), + PartitioningSatisfaction::NotSatisfied + ); + assert_key_locality( + "aligned transformed key", + &aligned, + &exact_key_exprs, + &fixture.eq_properties, + true, + true, + ); - assert_satisfaction( + assert_key_locality( "additional grouping keys require subset permission", &aligned, - &Distribution::KeyPartitioned(vec![trunc_hour, fixture.col(1)]), + &[trunc_hour, fixture.col(1)], &fixture.eq_properties, - PartitioningSatisfaction::KeyLocal, - PartitioningSatisfaction::NotSatisfied, + true, + false, ); - assert_satisfaction( + assert_key_locality( "date_bin remains default-denied", &aligned, - &Distribution::KeyPartitioned(vec![date_bin_of(fixture.col(0))]), + &[date_bin_of(fixture.col(0))], &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); let trunc_day = date_trunc_of(fixture.col(0), "day", None); - assert_satisfaction( + assert_key_locality( "an hour split straddles a day bucket", &aligned, - &Distribution::KeyPartitioned(vec![trunc_day]), + &[trunc_day], &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); let two_aligned = fixture.range_partitioning( @@ -1619,91 +1610,70 @@ mod tests { timestamp_split(Some(hour + 3_600_000_000_000), None), ], ); - assert_satisfaction( + assert_key_locality( "every split must be aligned", &two_aligned, - &exact_keys, + &exact_key_exprs, &fixture.eq_properties, - PartitioningSatisfaction::KeyLocal, - PartitioningSatisfaction::KeyLocal, + true, + true, ); let unaligned = fixture.range_partitioning([0], vec![timestamp_split(Some(hour + 1), None)]); - assert_satisfaction( + assert_key_locality( "an unaligned split straddles an hour bucket", &unaligned, - &exact_keys, + &exact_key_exprs, &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); Ok(()) } #[test] - fn range_transform_supports_only_the_final_compound_key() -> Result<()> { + fn range_transform_rejects_compound_range_keys() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![ ("key", DataType::Utf8), ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), ])?; - let hour = 1_704_070_800_000_000_000; let split = SplitPoint::new(vec![ ScalarValue::Utf8(Some("m".into())), - ScalarValue::TimestampNanosecond(Some(hour), None), + ScalarValue::TimestampNanosecond(Some(1_704_070_800_000_000_000), None), ]); let range = fixture.range_partitioning([0, 1], vec![split]); - assert_satisfaction( - "the final compound key can be transformed", - &range, - &Distribution::KeyPartitioned(vec![ - fixture.col(0), - date_trunc_of(fixture.col(1), "hour", None), - ]), - &fixture.eq_properties, - PartitioningSatisfaction::KeyLocal, - PartitioningSatisfaction::KeyLocal, - ); - let split = SplitPoint::new(vec![ - ScalarValue::TimestampNanosecond(Some(hour), None), - ScalarValue::Utf8(Some("m".into())), - ]); - let range = fixture.range_partitioning([1, 0], vec![split]); - assert_satisfaction( - "a non-final compound key cannot be transformed", - &range, - &Distribution::KeyPartitioned(vec![ - date_trunc_of(fixture.col(1), "hour", None), - fixture.col(0), - ]), + // Start with single-key ranges; compound transforms can follow once + // tuple-boundary safety is established. + assert!(!range.keeps_keys_local( + &[fixture.col(0), date_trunc_of(fixture.col(1), "hour", None),], &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, - ); + false, + )); Ok(()) } #[test] - fn range_transform_satisfaction_fails_closed() -> Result<()> { + fn range_transform_key_locality_fails_closed() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![( "timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None), )])?; let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); - let required = Distribution::KeyPartitioned(vec![trunc_hour]); + let required = vec![trunc_hour]; for (description, split) in [ ("null", timestamp_split(None, None)), ("minimum", timestamp_split(Some(i64::MIN), None)), ] { - assert_satisfaction( + assert_key_locality( description, &fixture.range_partitioning([0], vec![split]), &required, &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); } @@ -1711,31 +1681,28 @@ mod tests { fixture.range_ordering([0]), vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])], )); - assert_satisfaction( + assert_key_locality( "split scalar type must match the range key", &mismatched_split, &required, &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); - let invalid_precision = Distribution::KeyPartitioned(vec![date_trunc_of( - fixture.col(0), - "not-a-precision", - None, - )]); + let invalid_precision = + vec![date_trunc_of(fixture.col(0), "not-a-precision", None)]; let aligned = fixture.range_partitioning( [0], vec![timestamp_split(Some(1_704_070_800_000_000_000), None)], ); - assert_satisfaction( + assert_key_locality( "split evaluation errors fail closed", &aligned, &invalid_precision, &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); let timezone_fixture = PartitioningTestFixture::new(vec![( @@ -1750,13 +1717,13 @@ mod tests { Some("UTC"), )], ); - assert_satisfaction( + assert_key_locality( "timezone-aware date_trunc is not audited", &timezone_range, - &Distribution::KeyPartitioned(vec![timezone_trunc]), + &[timezone_trunc], &timezone_fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); Ok(()) } @@ -1770,13 +1737,9 @@ mod tests { let hour = 1_704_070_800_000_000_000; let ordering: LexOrdering = [fixture.range_sort_expr(0, SortOptions::new(true, false))].into(); - let required = Distribution::KeyPartitioned(vec![date_trunc_of( - fixture.col(0), - "hour", - None, - )]); + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; - assert_satisfaction( + assert_key_locality( "the successor of an hour boundary is in the same bucket", &fixture.range_partitioning_with_ordering( ordering.clone(), @@ -1784,10 +1747,10 @@ mod tests { ), &required, &fixture.eq_properties, - PartitioningSatisfaction::NotSatisfied, - PartitioningSatisfaction::NotSatisfied, + false, + false, ); - assert_satisfaction( + assert_key_locality( "the successor of the final nanosecond starts a new bucket", &fixture.range_partitioning_with_ordering( ordering, @@ -1795,8 +1758,8 @@ mod tests { ), &required, &fixture.eq_properties, - PartitioningSatisfaction::KeyLocal, - PartitioningSatisfaction::KeyLocal, + true, + true, ); Ok(()) } From 8bdbc6945489a0165268dd31c110738ed851034a Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:06:44 +0000 Subject: [PATCH 11/57] test: verify range analysis is denied by default --- datafusion/expr/src/udf.rs | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index b15b340a00d7b..e57ace01c04e2 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -1291,7 +1291,6 @@ mod tests { name: &'static str, field: &'static str, signature: Signature, - supports_range_partitioning_analysis: bool, } impl ScalarUDFImpl for TestScalarUDFImpl { fn name(&self) -> &str { @@ -1309,14 +1308,6 @@ mod tests { fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { unimplemented!() } - - fn supports_range_partitioning_analysis( - &self, - argument_types: &[DataType], - ) -> bool { - self.supports_range_partitioning_analysis - && argument_types == [DataType::Int64] - } } // PartialEq and Hash must be consistent, and also PartialEq and PartialOrd @@ -1351,17 +1342,9 @@ mod tests { } #[test] - fn aliases_preserve_range_partitioning_analysis_support() { - let function = ScalarUDF::from(TestScalarUDFImpl { - name: "range_function", - field: "a", - signature: Signature::any(1, Volatility::Immutable), - supports_range_partitioning_analysis: true, - }) - .with_aliases(["range_function_alias"]); - - assert!(function.supports_range_partitioning_analysis(&[DataType::Int64])); - assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8])); + fn range_partitioning_analysis_is_disabled_by_default() { + let function = test_func("range_function", "a"); + assert!(!function.supports_range_partitioning_analysis(&[DataType::Int64])); } fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF { @@ -1369,7 +1352,6 @@ mod tests { name, field: parameter, signature: Signature::any(1, Volatility::Immutable), - supports_range_partitioning_analysis: false, }) } From 12aa6a4ba5915ec04653e6cbce2c4e3e3edf6277 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:17:50 +0000 Subject: [PATCH 12/57] test: focus range analysis contract --- datafusion/expr/src/udf.rs | 4 +--- .../functions/src/datetime/date_trunc.rs | 19 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index e57ace01c04e2..6f1fbd10b3d8a 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -1016,9 +1016,7 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// all non-range inputs singleton, it is globally nondecreasing over the /// admitted non-null input domain; /// - successful evaluation preserves nullness: null range inputs remain - /// null and non-null range inputs do not produce null; and - /// - successful scalar evaluation at a split boundary is consistent with - /// array evaluation of that value. + /// null and non-null range inputs do not produce null. /// /// This method does not assert that a concrete range partitioning is /// preserved. Callers must still identify the range input, establish diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 5d86273372ecd..348d46c5491aa 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -1134,25 +1134,6 @@ mod tests { } previous = Some(value); } - - for (index, value) in values.iter().enumerate() { - let scalar_output = invoke_date_trunc( - granularity, - ColumnarValue::Scalar(ScalarValue::new_timestamp::(*value, None)), - timestamp_type.clone(), - 1, - ) - .unwrap(); - let ColumnarValue::Scalar(scalar_output) = scalar_output else { - panic!("expected scalar output") - }; - assert_eq!( - scalar_output, - ScalarValue::try_from_array(output, index).unwrap(), - "scalar and array evaluation must agree for {granularity} {:?}", - T::UNIT - ); - } } } From e8cff7d5c8d7a43a82a659fb2de77210ce616d53 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:38:49 +0000 Subject: [PATCH 13/57] docs: simplify range locality contracts --- datafusion/expr/src/udf.rs | 52 +++++++------------- datafusion/physical-expr/src/partitioning.rs | 8 +-- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 6f1fbd10b3d8a..20d91c346714d 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -992,40 +992,24 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { Ok(inner(inputs)) } - /// Returns whether this function and argument-type combination may be - /// analyzed to determine if transforming a range partition key keeps every - /// equal output key within one existing partition. - /// - /// This property allows keyed operators such as grouped aggregation to - /// reuse range-partitioned input after transforming the key, provided the - /// concrete split points prove that no output group crosses a partition - /// boundary. It does not claim that the transformed keys have a particular - /// partition layout or that two inputs are compatibly partitioned for a - /// join. - /// - /// Ordinary [`SortProperties::Ordered`] metadata is not sufficient for this - /// analysis. A non-strictly ordered function can map values from opposite - /// sides of a range split to the same output, and successful evaluation of - /// some functions can produce null from non-null input. Either behavior can - /// place one transformed equality group in multiple partitions. - /// - /// Returning `true` asserts that this argument-type combination has been - /// audited with the following contract: - /// - /// - whenever the concrete expression reports same-direction ordering with - /// all non-range inputs singleton, it is globally nondecreasing over the - /// admitted non-null input domain; - /// - successful evaluation preserves nullness: null range inputs remain - /// null and non-null range inputs do not produce null. - /// - /// This method does not assert that a concrete range partitioning is - /// preserved. Callers must still identify the range input, establish - /// same-direction ordering and singleton arguments, verify exact split - /// types, and prove at every split that the images of adjacent partitions - /// do not overlap. Evaluation errors must fail closed. - /// - /// The default is `false`. This capability is not currently transported - /// across FFI, so foreign UDFs also remain denied by default. + /// Returns whether this function and argument-type combination may + /// participate in range-partitioning locality analysis. + /// + /// Returning `true` asserts that, whenever the concrete expression reports + /// same-direction ordering with all other inputs singleton: + /// + /// - the function is globally nondecreasing over its admitted non-null domain; + /// - successful evaluation preserves nullness. + /// + /// Callers may use this contract to prove key locality by evaluating values + /// on both sides of each concrete range split. Ordered metadata alone is + /// insufficient because a non-strict transform can collapse a split boundary. + /// + /// This does not claim a concrete partition layout or cross-input + /// co-partitioning compatibility. Boundary evaluation errors cause the + /// analysis to return `false`. + /// + /// The default is `false`. This capability is not transported across FFI. fn supports_range_partitioning_analysis(&self, _argument_types: &[DataType]) -> bool { false } diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 7fb036c265972..b0a3376f298e2 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -820,10 +820,10 @@ impl Partitioning { /// cannot occur in different partitions. /// /// Unlike [`Self::satisfaction`], this does not claim a concrete partition - /// layout that can be compared with another input. It is sufficient for - /// operators such as grouped aggregation that process each key - /// independently, but must not establish co-partitioning between inputs. - /// Transform analysis is currently limited to single-key range partitioning. + /// layout. This weaker guarantee is sufficient for independent keyed + /// operations such as aggregation, but not for co-partitioning multiple + /// inputs. Transform analysis is currently limited to single-key range + /// partitioning. pub fn keeps_keys_local( &self, required_exprs: &[Arc], From 14308b91929d09b51dd800fb22a0514df67d25c9 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:42:56 +0000 Subject: [PATCH 14/57] docs: clarify range analysis null contract --- datafusion/expr/src/udf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 20d91c346714d..9dc3385f74d3f 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -999,7 +999,7 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// same-direction ordering with all other inputs singleton: /// /// - the function is globally nondecreasing over its admitted non-null domain; - /// - successful evaluation preserves nullness. + /// - successful evaluation returns null exactly when the range input is null. /// /// Callers may use this contract to prove key locality by evaluating values /// on both sides of each concrete range split. Ordered metadata alone is From 1f266d428dd262b583118ffc1a2fcb7b2218c69d Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:46:28 +0000 Subject: [PATCH 15/57] docs: use ordered metadata terminology --- datafusion/expr/src/udf.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 9dc3385f74d3f..4b248a4055087 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -995,10 +995,10 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// Returns whether this function and argument-type combination may /// participate in range-partitioning locality analysis. /// - /// Returning `true` asserts that, whenever the concrete expression reports - /// same-direction ordering with all other inputs singleton: + /// Returning `true` asserts that: /// - /// - the function is globally nondecreasing over its admitted non-null domain; + /// - same-direction [`SortProperties::Ordered`] metadata is valid over the + /// admitted non-null domain when all other inputs are singleton; /// - successful evaluation returns null exactly when the range input is null. /// /// Callers may use this contract to prove key locality by evaluating values From a9a9ee5b6f1ae8cb633d9b4da4428252cbce213c Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:56:54 +0000 Subject: [PATCH 16/57] refactor: simplify range analysis type gate --- .../functions/src/datetime/date_trunc.rs | 56 ++++--------------- 1 file changed, 11 insertions(+), 45 deletions(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 348d46c5491aa..5ab86c7e5254e 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -455,13 +455,11 @@ impl ScalarUDFImpl for DateTruncFunc { fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { // Named timezone transitions can make date_trunc non-monotonic (#25353). // Keep the audited domain to timezone-less timestamp columns. - matches!( - argument_types, - [ - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8, - Timestamp(_, None) - ] - ) + let [precision, timestamp] = argument_types else { + return false; + }; + + precision.is_string() && matches!(timestamp, Timestamp(_, None)) } fn documentation(&self) -> Option<&Documentation> { @@ -931,7 +929,7 @@ mod tests { use arrow::buffer::NullBuffer; use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos; use arrow::compute::{DatePart, SortOptions}; - use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit}; + use arrow::datatypes::{DataType, Field, TimeUnit}; use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; use datafusion_expr::interval_arithmetic::Interval; @@ -979,54 +977,22 @@ mod tests { } #[test] - fn supports_range_partitioning_analysis_for_timezone_less_timestamps() { + fn range_partitioning_analysis_requires_timezone_less_timestamp() { let function = DateTruncFunc::new(); - for unit in [ - TimeUnit::Second, - TimeUnit::Millisecond, - TimeUnit::Microsecond, - TimeUnit::Nanosecond, - ] { - for string_type in [DataType::Utf8, DataType::Utf8View, DataType::LargeUtf8] { - assert!(function.supports_range_partitioning_analysis(&[ - string_type, - DataType::Timestamp(unit, None), - ])); - } - - for timezone in ["UTC", "+05:30", "America/Goose_Bay"] { - assert!(!function.supports_range_partitioning_analysis(&[ - DataType::Utf8, - DataType::Timestamp(unit, Some(timezone.into())), - ])); - } - } - - assert!(!function.supports_range_partitioning_analysis(&[ + assert!(function.supports_range_partitioning_analysis(&[ DataType::Utf8, - DataType::Time32(TimeUnit::Second), + DataType::Timestamp(TimeUnit::Nanosecond, None), ])); assert!(!function.supports_range_partitioning_analysis(&[ DataType::Utf8, - DataType::Time64(TimeUnit::Nanosecond), + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), ])); - assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8,])); assert!(!function.supports_range_partitioning_analysis(&[ DataType::Int64, DataType::Timestamp(TimeUnit::Nanosecond, None), ])); - } - - #[test] - fn date_bin_does_not_support_range_partitioning_analysis() { - assert!( - !crate::datetime::date_bin::DateBinFunc::new() - .supports_range_partitioning_analysis(&[ - DataType::Interval(IntervalUnit::MonthDayNano), - DataType::Timestamp(TimeUnit::Nanosecond, None), - ]) - ); + assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8])); } fn invoke_date_trunc( From 7e18d653605ad6e4d12c291e0c4b6617a937d037 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:59:01 +0000 Subject: [PATCH 17/57] test: clarify date trunc capability fixture --- datafusion/functions/src/datetime/date_trunc.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 5ab86c7e5254e..1443d4bed9cdd 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -978,21 +978,21 @@ mod tests { #[test] fn range_partitioning_analysis_requires_timezone_less_timestamp() { - let function = DateTruncFunc::new(); + let date_trunc = DateTruncFunc::new(); - assert!(function.supports_range_partitioning_analysis(&[ + assert!(date_trunc.supports_range_partitioning_analysis(&[ DataType::Utf8, DataType::Timestamp(TimeUnit::Nanosecond, None), ])); - assert!(!function.supports_range_partitioning_analysis(&[ + assert!(!date_trunc.supports_range_partitioning_analysis(&[ DataType::Utf8, DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), ])); - assert!(!function.supports_range_partitioning_analysis(&[ + assert!(!date_trunc.supports_range_partitioning_analysis(&[ DataType::Int64, DataType::Timestamp(TimeUnit::Nanosecond, None), ])); - assert!(!function.supports_range_partitioning_analysis(&[DataType::Utf8])); + assert!(!date_trunc.supports_range_partitioning_analysis(&[DataType::Utf8])); } fn invoke_date_trunc( From 5bd1290ac51b8de96cae4e79c13dfa1002e145a2 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:09:07 +0000 Subject: [PATCH 18/57] test: document date trunc range analysis contract --- .../functions/src/datetime/date_trunc.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 1443d4bed9cdd..c675364f3270d 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -1013,7 +1013,7 @@ mod tests { }) } - fn assert_timezone_less_range_analysis_contract() { + fn assert_timezone_less_date_trunc_contract() { let scale = match T::UNIT { TimeUnit::Second => 1_000_000_000, TimeUnit::Millisecond => 1_000_000, @@ -1103,12 +1103,19 @@ mod tests { } } + /// Verifies the function-level contract required by + /// [`ScalarUDFImpl::supports_range_partitioning_analysis`] across every + /// supported granularity and timezone-less timestamp unit. Ordered input + /// remains ordered, and output nullness exactly matches input nullness. + /// Concrete range-split separation is tested separately. + /// + /// See . #[test] - fn timezone_less_timestamps_satisfy_range_analysis_contract() { - assert_timezone_less_range_analysis_contract::(); - assert_timezone_less_range_analysis_contract::(); - assert_timezone_less_range_analysis_contract::(); - assert_timezone_less_range_analysis_contract::(); + fn timezone_less_date_trunc_is_ordered_and_preserves_nullness() { + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); } #[test] From 74232b71f9ee86b6fcef822f20e8fc172f261094 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:15:50 +0000 Subject: [PATCH 19/57] test: focus date trunc audit on null contract --- datafusion/expr/src/udf.rs | 14 ++-- .../functions/src/datetime/date_trunc.rs | 75 +++---------------- 2 files changed, 16 insertions(+), 73 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 4b248a4055087..b4437ef013985 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -995,15 +995,13 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// Returns whether this function and argument-type combination may /// participate in range-partitioning locality analysis. /// - /// Returning `true` asserts that: + /// Returning `true` asserts that successful evaluation returns null exactly + /// when the range input is null. /// - /// - same-direction [`SortProperties::Ordered`] metadata is valid over the - /// admitted non-null domain when all other inputs are singleton; - /// - successful evaluation returns null exactly when the range input is null. - /// - /// Callers may use this contract to prove key locality by evaluating values - /// on both sides of each concrete range split. Ordered metadata alone is - /// insufficient because a non-strict transform can collapse a split boundary. + /// Callers may combine this contract with same-direction + /// [`SortProperties::Ordered`] metadata and concrete range-split evaluation + /// to prove key locality. Ordered metadata alone is insufficient because a + /// non-strict transform can collapse a split boundary. /// /// This does not claim a concrete partition layout or cross-input /// co-partitioning compatibility. Boundary evaluation errors cause the diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index c675364f3270d..f36ee4e3beda5 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -1013,48 +1013,8 @@ mod tests { }) } - fn assert_timezone_less_date_trunc_contract() { - let scale = match T::UNIT { - TimeUnit::Second => 1_000_000_000, - TimeUnit::Millisecond => 1_000_000, - TimeUnit::Microsecond => 1_000, - TimeUnit::Nanosecond => 1, - }; - // Probe both sides of calendar and fixed-width boundaries at the - // input's own resolution, including negative fractional timestamps. - let mut timestamps = [ - "1900-03-01T00:00:00Z", // Non-leap century. - "1969-12-29T00:00:00Z", // Monday before the epoch. - "1970-01-01T00:00:00Z", - "1970-01-01T00:00:00.000001Z", - "1970-01-01T00:00:00.001Z", - "1970-01-01T00:00:01Z", - "1970-01-01T00:01:00Z", - "1970-01-01T01:00:00Z", - "2000-01-01T00:00:00Z", - "2000-02-29T00:00:00Z", // Leap century. - "2000-03-01T00:00:00Z", - "2000-04-01T00:00:00Z", // Quarter boundary. - "2024-01-01T00:00:00Z", // Monday and year boundary. - ] - .into_iter() - .flat_map(|value| { - let boundary = string_to_timestamp_nanos(value).unwrap() / scale; - [boundary - 1, boundary, boundary + 1] - }) - .collect::>(); - // Stay inside the scalar nanosecond domain with room to truncate a - // full year at its lower edge; the upper edge needs no such margin. - timestamps.extend([ - i64::MIN / scale + 366 * 86_400 * (1_000_000_000 / scale), - i64::MAX / scale, - ]); - timestamps.sort_unstable(); - timestamps.dedup(); - let values = timestamps - .into_iter() - .flat_map(|value| [Some(value), None]) - .collect::>(); + fn assert_timezone_less_date_trunc_preserves_nullness() { + let input = PrimitiveArray::::from_iter([Some(0), None]); let timestamp_type = DataType::Timestamp(T::UNIT, None); for granularity in [ @@ -1069,7 +1029,6 @@ mod tests { "quarter", "year", ] { - let input = PrimitiveArray::::from_iter(values.iter().copied()); let output = invoke_date_trunc( granularity, ColumnarValue::Array(Arc::new(input.clone())), @@ -1080,7 +1039,6 @@ mod tests { let ColumnarValue::Array(output) = output else { panic!("expected array output") }; - let output = as_primitive_array::(&output); assert_eq!( input.nulls(), @@ -1088,34 +1046,21 @@ mod tests { "{granularity} must preserve nullness for {:?}", T::UNIT ); - - let mut previous = None; - for value in output.iter().flatten() { - if let Some(previous) = previous { - assert!( - previous <= value, - "{granularity} must be nondecreasing for {:?}", - T::UNIT - ); - } - previous = Some(value); - } } } - /// Verifies the function-level contract required by + /// Verifies the exact null contract required by /// [`ScalarUDFImpl::supports_range_partitioning_analysis`] across every - /// supported granularity and timezone-less timestamp unit. Ordered input - /// remains ordered, and output nullness exactly matches input nullness. - /// Concrete range-split separation is tested separately. + /// supported granularity and timezone-less timestamp unit. Ordering metadata + /// and concrete range-split separation are tested separately. /// /// See . #[test] - fn timezone_less_date_trunc_is_ordered_and_preserves_nullness() { - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); + fn timezone_less_date_trunc_preserves_nullness() { + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); } #[test] From 7d5b0c58166e197ccb5c4b8461135902fafd59bf Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:26:52 +0000 Subject: [PATCH 20/57] test: restore audited date trunc range contract --- datafusion/expr/src/udf.rs | 19 +++-- .../functions/src/datetime/date_trunc.rs | 75 ++++++++++++++++--- 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index b4437ef013985..daf78781f9e44 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -992,15 +992,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { Ok(inner(inputs)) } - /// Returns whether this function and argument-type combination may - /// participate in range-partitioning locality analysis. + /// Returns whether this function and argument-type combination has been + /// audited for range-partitioning locality analysis. /// - /// Returning `true` asserts that successful evaluation returns null exactly - /// when the range input is null. + /// Returning `true` explicitly opts the combination into this analysis and + /// asserts that: /// - /// Callers may combine this contract with same-direction - /// [`SortProperties::Ordered`] metadata and concrete range-split evaluation - /// to prove key locality. Ordered metadata alone is insufficient because a + /// - same-direction [`SortProperties::Ordered`] metadata accurately describes + /// successful evaluation over the admitted non-null domain when all other + /// inputs are singleton; + /// - successful evaluation returns null exactly when the range input is null. + /// + /// This explicit audit is required in addition to ordered metadata. Callers + /// may then prove key locality by evaluating values on both sides of each + /// concrete range split. Ordered metadata alone is insufficient because a /// non-strict transform can collapse a split boundary. /// /// This does not claim a concrete partition layout or cross-input diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index f36ee4e3beda5..c675364f3270d 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -1013,8 +1013,48 @@ mod tests { }) } - fn assert_timezone_less_date_trunc_preserves_nullness() { - let input = PrimitiveArray::::from_iter([Some(0), None]); + fn assert_timezone_less_date_trunc_contract() { + let scale = match T::UNIT { + TimeUnit::Second => 1_000_000_000, + TimeUnit::Millisecond => 1_000_000, + TimeUnit::Microsecond => 1_000, + TimeUnit::Nanosecond => 1, + }; + // Probe both sides of calendar and fixed-width boundaries at the + // input's own resolution, including negative fractional timestamps. + let mut timestamps = [ + "1900-03-01T00:00:00Z", // Non-leap century. + "1969-12-29T00:00:00Z", // Monday before the epoch. + "1970-01-01T00:00:00Z", + "1970-01-01T00:00:00.000001Z", + "1970-01-01T00:00:00.001Z", + "1970-01-01T00:00:01Z", + "1970-01-01T00:01:00Z", + "1970-01-01T01:00:00Z", + "2000-01-01T00:00:00Z", + "2000-02-29T00:00:00Z", // Leap century. + "2000-03-01T00:00:00Z", + "2000-04-01T00:00:00Z", // Quarter boundary. + "2024-01-01T00:00:00Z", // Monday and year boundary. + ] + .into_iter() + .flat_map(|value| { + let boundary = string_to_timestamp_nanos(value).unwrap() / scale; + [boundary - 1, boundary, boundary + 1] + }) + .collect::>(); + // Stay inside the scalar nanosecond domain with room to truncate a + // full year at its lower edge; the upper edge needs no such margin. + timestamps.extend([ + i64::MIN / scale + 366 * 86_400 * (1_000_000_000 / scale), + i64::MAX / scale, + ]); + timestamps.sort_unstable(); + timestamps.dedup(); + let values = timestamps + .into_iter() + .flat_map(|value| [Some(value), None]) + .collect::>(); let timestamp_type = DataType::Timestamp(T::UNIT, None); for granularity in [ @@ -1029,6 +1069,7 @@ mod tests { "quarter", "year", ] { + let input = PrimitiveArray::::from_iter(values.iter().copied()); let output = invoke_date_trunc( granularity, ColumnarValue::Array(Arc::new(input.clone())), @@ -1039,6 +1080,7 @@ mod tests { let ColumnarValue::Array(output) = output else { panic!("expected array output") }; + let output = as_primitive_array::(&output); assert_eq!( input.nulls(), @@ -1046,21 +1088,34 @@ mod tests { "{granularity} must preserve nullness for {:?}", T::UNIT ); + + let mut previous = None; + for value in output.iter().flatten() { + if let Some(previous) = previous { + assert!( + previous <= value, + "{granularity} must be nondecreasing for {:?}", + T::UNIT + ); + } + previous = Some(value); + } } } - /// Verifies the exact null contract required by + /// Verifies the function-level contract required by /// [`ScalarUDFImpl::supports_range_partitioning_analysis`] across every - /// supported granularity and timezone-less timestamp unit. Ordering metadata - /// and concrete range-split separation are tested separately. + /// supported granularity and timezone-less timestamp unit. Ordered input + /// remains ordered, and output nullness exactly matches input nullness. + /// Concrete range-split separation is tested separately. /// /// See . #[test] - fn timezone_less_date_trunc_preserves_nullness() { - assert_timezone_less_date_trunc_preserves_nullness::(); - assert_timezone_less_date_trunc_preserves_nullness::(); - assert_timezone_less_date_trunc_preserves_nullness::(); - assert_timezone_less_date_trunc_preserves_nullness::(); + fn timezone_less_date_trunc_is_ordered_and_preserves_nullness() { + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); + assert_timezone_less_date_trunc_contract::(); } #[test] From 1a51496d98778bc0d07c78b6f8e8956d9ca85e14 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:32:05 +0000 Subject: [PATCH 21/57] docs: illustrate timezone ordering reversal --- datafusion/functions/src/datetime/date_trunc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index c675364f3270d..375112c5f5aaa 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -453,7 +453,8 @@ impl ScalarUDFImpl for DateTruncFunc { } fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { - // Named timezone transitions can make date_trunc non-monotonic (#25353). + // Across an America/Goose_Bay transition, ascending epoch seconds + // 562129259 < 562129260 truncate to 562129200 > 562125600. // Keep the audited domain to timezone-less timestamp columns. let [precision, timestamp] = argument_types else { return false; From e99e608a26007884c78e4bb3da009eedba1c3b34 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:39:13 +0000 Subject: [PATCH 22/57] test: separate date trunc range contracts --- .../functions/src/datetime/date_trunc.rs | 77 +++---------------- 1 file changed, 10 insertions(+), 67 deletions(-) diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 375112c5f5aaa..b293a0d3022ce 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -1014,48 +1014,8 @@ mod tests { }) } - fn assert_timezone_less_date_trunc_contract() { - let scale = match T::UNIT { - TimeUnit::Second => 1_000_000_000, - TimeUnit::Millisecond => 1_000_000, - TimeUnit::Microsecond => 1_000, - TimeUnit::Nanosecond => 1, - }; - // Probe both sides of calendar and fixed-width boundaries at the - // input's own resolution, including negative fractional timestamps. - let mut timestamps = [ - "1900-03-01T00:00:00Z", // Non-leap century. - "1969-12-29T00:00:00Z", // Monday before the epoch. - "1970-01-01T00:00:00Z", - "1970-01-01T00:00:00.000001Z", - "1970-01-01T00:00:00.001Z", - "1970-01-01T00:00:01Z", - "1970-01-01T00:01:00Z", - "1970-01-01T01:00:00Z", - "2000-01-01T00:00:00Z", - "2000-02-29T00:00:00Z", // Leap century. - "2000-03-01T00:00:00Z", - "2000-04-01T00:00:00Z", // Quarter boundary. - "2024-01-01T00:00:00Z", // Monday and year boundary. - ] - .into_iter() - .flat_map(|value| { - let boundary = string_to_timestamp_nanos(value).unwrap() / scale; - [boundary - 1, boundary, boundary + 1] - }) - .collect::>(); - // Stay inside the scalar nanosecond domain with room to truncate a - // full year at its lower edge; the upper edge needs no such margin. - timestamps.extend([ - i64::MIN / scale + 366 * 86_400 * (1_000_000_000 / scale), - i64::MAX / scale, - ]); - timestamps.sort_unstable(); - timestamps.dedup(); - let values = timestamps - .into_iter() - .flat_map(|value| [Some(value), None]) - .collect::>(); + fn assert_timezone_less_date_trunc_preserves_nullness() { + let input = PrimitiveArray::::from_iter([Some(0), None]); let timestamp_type = DataType::Timestamp(T::UNIT, None); for granularity in [ @@ -1070,7 +1030,6 @@ mod tests { "quarter", "year", ] { - let input = PrimitiveArray::::from_iter(values.iter().copied()); let output = invoke_date_trunc( granularity, ColumnarValue::Array(Arc::new(input.clone())), @@ -1081,7 +1040,6 @@ mod tests { let ColumnarValue::Array(output) = output else { panic!("expected array output") }; - let output = as_primitive_array::(&output); assert_eq!( input.nulls(), @@ -1089,34 +1047,19 @@ mod tests { "{granularity} must preserve nullness for {:?}", T::UNIT ); - - let mut previous = None; - for value in output.iter().flatten() { - if let Some(previous) = previous { - assert!( - previous <= value, - "{granularity} must be nondecreasing for {:?}", - T::UNIT - ); - } - previous = Some(value); - } } } - /// Verifies the function-level contract required by + /// Verifies the exact null contract required by /// [`ScalarUDFImpl::supports_range_partitioning_analysis`] across every - /// supported granularity and timezone-less timestamp unit. Ordered input - /// remains ordered, and output nullness exactly matches input nullness. - /// Concrete range-split separation is tested separately. - /// - /// See . + /// supported granularity and timezone-less timestamp unit. Ordering metadata + /// and concrete range-split separation are tested separately. #[test] - fn timezone_less_date_trunc_is_ordered_and_preserves_nullness() { - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); - assert_timezone_less_date_trunc_contract::(); + fn timezone_less_date_trunc_preserves_nullness() { + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); } #[test] From ca929ca4a0153d24090ba67b71c1973f9669518a Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:22:53 +0000 Subject: [PATCH 23/57] feat: reuse range locality for aggregation --- datafusion/core/Cargo.toml | 4 + datafusion/core/benches/range_date_trunc.rs | 152 ++++++++++++++++++ .../enforce_distribution.rs | 104 ++++++++++++ .../enforce_distribution.rs | 39 +++-- .../physical-optimizer/src/sanity_checker.rs | 20 ++- .../physical-plan/src/aggregates/mod.rs | 47 ++++++ .../src/test_context/range_partitioning.rs | 6 +- .../test_files/range_sorted_time_bin_agg.slt | 81 ++++++++-- 8 files changed, 415 insertions(+), 38 deletions(-) create mode 100644 datafusion/core/benches/range_date_trunc.rs diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 7275c4b674d20..ca29bc1a68d18 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -217,6 +217,10 @@ required-features = ["docs_generation"] harness = false name = "aggregate_query_sql" +[[bench]] +harness = false +name = "range_date_trunc" + [[bench]] harness = false name = "csv_load" diff --git a/datafusion/core/benches/range_date_trunc.rs b/datafusion/core/benches/range_date_trunc.rs new file mode 100644 index 0000000000000..4e7c3bc2f897d --- /dev/null +++ b/datafusion/core/benches/range_date_trunc.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Group hourly range partitions without moving groups across partitions. +//! The same benchmark can run on main: only the optimized plan differs. + +use std::hint::black_box; +use std::sync::Arc; + +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion::arrow::array::{ArrayRef, Int64Array, TimestampNanosecondArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::catalog::streaming::StreamingTable; +use datafusion::common::{Result, ScalarValue, SplitPoint}; +use datafusion::logical_expr::col; +use datafusion::physical_expr::{ + Partitioning, PhysicalSortExpr, RangePartitioning, expressions::col as physical_col, +}; +use datafusion::physical_plan::{ + collect, execution_plan::reset_plan_states, streaming::PartitionStream, + test::TestPartitionStream, +}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use tokio::runtime::Runtime; + +const PARTITIONS: usize = 8; +const REPETITIONS: i64 = 8; + +fn context(keys_per_partition: i64) -> Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int64, false), + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), false), + Field::new("value", DataType::Int64, false), + ])); + let start = 1_704_067_200_000_000_000_i64; + let hour = 3_600_000_000_000_i64; + let mut streams: Vec> = vec![]; + for partition in 0..PARTITIONS { + let mut batches = vec![]; + let rows = keys_per_partition * REPETITIONS; + for offset in (0..rows).step_by(8192) { + let end = (offset + 8192).min(rows); + let columns: Vec = vec![ + Arc::new(Int64Array::from_iter_values( + (offset..end).map(|i| i / REPETITIONS), + )), + Arc::new(TimestampNanosecondArray::from_iter_values( + (offset..end).map(|i| { + start + + partition as i64 * hour + + (i % REPETITIONS) * 1_000_000_000 + }), + )), + Arc::new(Int64Array::from_iter_values((offset..end).map(|_| 1))), + ]; + batches.push(RecordBatch::try_new(Arc::clone(&schema), columns)?); + } + streams.push(Arc::new(TestPartitionStream::new_with_batches(batches))); + } + let partitioning = Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new_default(physical_col("ts", &schema)?)].into(), + (1..PARTITIONS) + .map(|partition| { + SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + Some(start + partition as i64 * hour), + None, + )]) + }) + .collect(), + )?); + let mut config = SessionConfig::new().with_target_partitions(PARTITIONS); + config + .options_mut() + .optimizer + .enable_round_robin_repartition = false; + config.options_mut().optimizer.subset_repartition_threshold = PARTITIONS; + let context = SessionContext::new_with_config(config); + context.register_table( + "t", + Arc::new( + StreamingTable::try_new(schema, streams)? + .with_sort_order(vec![ + col("key").sort(true, true), + col("ts").sort(true, true), + ]) + .with_output_partitioning(partitioning), + ), + )?; + Ok(context) +} + +fn criterion_benchmark(criterion: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + for keys_per_partition in [1024, 65_536] { + let context = context(keys_per_partition).unwrap(); + let plan = + runtime.block_on(async { + context + .sql("SELECT key, date_trunc('hour', ts) AS bucket, sum(value) AS total \ + FROM t GROUP BY key, bucket") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap() + }); + let results = runtime + .block_on(collect(Arc::clone(&plan), context.task_ctx())) + .unwrap(); + assert_eq!( + results.iter().map(RecordBatch::num_rows).sum::(), + PARTITIONS * keys_per_partition as usize + ); + for batch in results { + let sums = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(sums.values().iter().all(|sum| *sum == REPETITIONS)); + } + criterion.bench_function( + &format!("range_date_trunc/{keys_per_partition}_keys_per_partition"), + |bencher| { + bencher.iter(|| { + let plan = reset_plan_states(Arc::clone(&plan)).unwrap(); + black_box( + runtime.block_on(collect(plan, context.task_ctx())).unwrap(), + ) + }); + }, + ); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 283d22767f9b1..dd0609cd5e68d 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -407,6 +407,110 @@ fn parquet_exec_with_output_partitioning( DataSourceExec::from_data_source(config) } +#[test] +fn range_date_trunc_locality_survives_aggregate_combination() -> Result<()> { + use arrow_schema::TimeUnit; + use datafusion_physical_expr::ScalarFunctionExpr; + use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate; + + let timestamp_type = DataType::Timestamp(TimeUnit::Nanosecond, None); + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + timestamp_type.clone(), + true, + )])); + let timestamp = col("ts", &schema)?; + let range = Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new_default(Arc::clone(×tamp))].into(), + vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + Some(1_704_070_800_000_000_000), + None, + )])], + )?); + let scan = FileScanConfigBuilder::new( + ObjectStoreUrl::parse("test:///")?, + Arc::new(ParquetSource::new(Arc::clone(&schema))), + ) + .with_file_groups(vec![ + FileGroup::new(vec![PartitionedFile::new("p0", 100)]), + FileGroup::new(vec![PartitionedFile::new("p1", 100)]), + ]) + .with_output_partitioning(Some(range)) + .build(); + let source: Arc = DataSourceExec::from_data_source(scan); + let bucket: Arc = Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![lit("hour"), timestamp], + Field::new("bucket", timestamp_type, true).into(), + Arc::new(ConfigOptions::default()), + )); + let group_by = + PhysicalGroupBy::new_single(vec![(Arc::clone(&bucket), "bucket".into())]); + let partial = AggregateExec::try_new( + AggregateMode::Partial, + group_by.clone(), + vec![], + vec![], + Arc::clone(&source), + Arc::clone(&schema), + )?; + let final_plan: Arc = Arc::new(AggregateExec::try_new( + AggregateMode::FinalPartitioned, + group_by.as_final(), + vec![], + vec![], + Arc::new(partial), + schema, + )?); + let mut config = ConfigOptions::default(); + config.execution.target_partitions = 2; + config.optimizer.enable_round_robin_repartition = false; + config.optimizer.repartition_file_scans = false; + let enforced = EnsureRequirements::new().optimize(final_plan, &config)?; + let plan_text = displayable(enforced.as_ref()).indent(true).to_string(); + assert!(plan_text.contains("mode=FinalPartitioned"), "{plan_text}"); + assert!(plan_text.contains("mode=Partial"), "{plan_text}"); + assert!(!plan_text.contains("RepartitionExec"), "{plan_text}"); + SanityCheckPlan::new().optimize(Arc::clone(&enforced), &config)?; + + let combined = CombinePartialFinalAggregate::new().optimize(enforced, &config)?; + let combined_text = displayable(combined.as_ref()).indent(true).to_string(); + assert!( + combined_text.contains("mode=SinglePartitioned"), + "{combined_text}" + ); + let reenforced = EnsureRequirements::new().optimize(combined, &config)?; + assert_eq!( + displayable(reenforced.as_ref()).indent(true).to_string(), + combined_text + ); + SanityCheckPlan::new().optimize(reenforced, &config)?; + + // Group locality does not establish a join-compatible partition layout. + let join = crate::physical_optimizer::test_utils::hash_join_exec( + Arc::clone(&source), + source, + vec![(Arc::clone(&bucket), bucket)], + None, + &JoinType::Inner, + )?; + assert!( + SanityCheckPlan::new() + .optimize(Arc::clone(&join), &config) + .is_err() + ); + let enforced_join = EnsureRequirements::new().optimize(join, &config)?; + assert!( + displayable(enforced_join.as_ref()) + .indent(true) + .to_string() + .contains("partitioning=Hash") + ); + SanityCheckPlan::new().optimize(enforced_join, &config)?; + Ok(()) +} + fn range_partitioning( column: &str, split_values: impl IntoIterator, diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs index 49776d8843d40..0306817a2e07d 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs @@ -885,12 +885,11 @@ fn add_roundrobin_on_top( } } -// Partial aggregates require unspecified input distribution, but their output -// may already satisfy the final aggregate's key distribution because partial -// aggregation preserves/projects input partitioning. Keep that reusable output -// partitioning intact when preserve_file_partitions would otherwise insert -// RoundRobin below the partial aggregate. -fn partial_aggregate_output_satisfies_final_partitioning( +// Partial aggregates require unspecified input distribution, but their input +// may already keep every group partition-local. Preserve that layout when +// preserve_file_partitions would otherwise insert RoundRobin below the partial +// aggregate, and reuse it for the final aggregate. +fn partial_aggregate_has_partition_local_groups( plan: &Arc, allow_subset_satisfy_partitioning: bool, ) -> bool { @@ -904,15 +903,23 @@ fn partial_aggregate_output_satisfies_final_partitioning( return false; } - let key_distribution = Distribution::KeyPartitioned(aggregate.output_group_expr()); - - plan.output_partitioning() + let output_distribution = Distribution::KeyPartitioned(aggregate.output_group_expr()); + if plan + .output_partitioning() .satisfaction( - &key_distribution, + &output_distribution, plan.equivalence_properties(), allow_subset_satisfy_partitioning, ) .is_satisfied() + { + return true; + } + + aggregate.groups_are_partition_local( + aggregate.input().as_ref(), + allow_subset_satisfy_partitioning, + ) } /// Adds a [`SortPreservingMergeExec`] or a [`CoalescePartitionsExec`] operator @@ -1686,7 +1693,7 @@ pub fn ensure_distribution_with_stats( let preserve_partial_aggregate_partitioning = preserve_file_partition_threshold_met - && partial_aggregate_output_satisfies_final_partitioning( + && partial_aggregate_has_partition_local_groups( &plan, allow_subset_satisfy_partitioning, ); @@ -1728,7 +1735,15 @@ pub fn ensure_distribution_with_stats( ChildSatisfactionOptions::new() .with_allow_subset(allow_subset_satisfy_partitioning), )? - .is_satisfied(); + .is_satisfied() + || plan.downcast_ref::().is_some_and( + |aggregate| { + aggregate.groups_are_partition_local( + child.plan.as_ref(), + allow_subset_satisfy_partitioning, + ) + }, + ); let preserve_satisfying_file_partitioning = preserve_file_partition_threshold_met && !requires_grouping_id diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs index 713213b70612d..0d2138014f7ee 100644 --- a/datafusion/physical-optimizer/src/sanity_checker.rs +++ b/datafusion/physical-optimizer/src/sanity_checker.rs @@ -30,6 +30,7 @@ use datafusion_common::config::{ConfigOptions, OptimizerOptions}; use datafusion_common::plan_err; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported}; +use datafusion_physical_plan::aggregates::AggregateExec; use datafusion_physical_plan::execution_plan::{ Boundedness, EmissionType, InvariantLevel, }; @@ -167,14 +168,17 @@ pub fn check_plan_sanity( } } - if !input_distributions - .child_satisfaction( - idx, - child.as_ref(), - ChildSatisfactionOptions::new().with_allow_subset(true), - )? - .is_satisfied() - { + let distribution_satisfaction = input_distributions.child_satisfaction( + idx, + child.as_ref(), + ChildSatisfactionOptions::new().with_allow_subset(true), + )?; + let accepts_partition_local_groups = plan + .downcast_ref::() + .is_some_and(|aggregate| { + aggregate.groups_are_partition_local(child.as_ref(), true) + }); + if !distribution_satisfaction.is_satisfied() && !accepts_partition_local_groups { let plan_str = get_plan_string(plan); return plan_err!( "Plan: {:?} does not satisfy distribution requirements: {}. Child-{} output partitioning: {}", diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 7957c804d5d3b..fce270ac8b197 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1654,6 +1654,53 @@ impl AggregateExec { &self.input } + /// Whether a candidate input keeps each group in one partition. + /// + /// Unlike co-partitioning, aggregation only needs equality groups to be + /// local. A final stage can inherit that guarantee from a partial stage + /// even when projecting the grouping expressions loses range metadata. + /// This check does not require the two stages to be combined. + pub fn groups_are_partition_local( + &self, + input: &dyn ExecutionPlan, + allow_subset: bool, + ) -> bool { + if !matches!( + self.mode, + AggregateMode::Partial + | AggregateMode::SinglePartitioned + | AggregateMode::FinalPartitioned + ) || self.group_by.is_empty() + || self.group_by.has_grouping_set() + { + return false; + } + + let required = Distribution::KeyPartitioned(self.group_by.input_exprs()); + if input + .output_partitioning() + .satisfaction(&required, input.equivalence_properties(), allow_subset) + .is_key_local() + { + return true; + } + + self.mode == AggregateMode::FinalPartitioned + && input + .downcast_ref::() + .is_some_and(|partial| { + partial.mode == AggregateMode::Partial + && datafusion_physical_expr::physical_exprs_equal( + &partial.output_group_expr(), + &self.group_by.input_exprs(), + ) + && partial.groups_are_partition_local( + partial.input.as_ref(), + allow_subset, + ) + }) + } + /// Get the input schema before any aggregates are applied pub fn input_schema(&self) -> SchemaRef { Arc::clone(&self.input_schema) diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs index 2cb4f57e29001..4989294146814 100644 --- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs +++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs @@ -423,9 +423,9 @@ type TimeBinRow = ( /// - partition 1: `[2024-01-01 01:00, 02:00)` /// /// Files are range-partitioned on `timestamp` and sorted on `(key, timestamp)`. -/// Because `date_bin(60 seconds, timestamp)` does not straddle the hour split, -/// grouping by `(key, time_bin)` is partition-disjoint. Today's planner still -/// inserts a hash shuffle; the test pins that plan so a follow-up can remove it. +/// The hour split permits partition-local `date_trunc('hour')` aggregation, +/// but `date_trunc('day')` straddles it. Aligned `date_bin` remains a negative +/// control because the function is not admitted to range analysis. pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) { let schema = Arc::new(Schema::new(vec![ Field::new("key", DataType::Utf8, false), diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt index 400bbab1a3592..30a173dedca5c 100644 --- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt +++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt @@ -29,15 +29,10 @@ # 1. Range([timestamp]) and output_ordering=[key, timestamp] # 2. Two file_groups, so the two 60-minute streams run in parallel # -# Improvement opportunity: -# date_bin(60s) is monotonic in timestamp and the hour split is aligned to bin -# boundaries, so (key, time_bin) is partition-disjoint. Aggregation could be a -# single streaming SinglePartitioned step with no hash shuffle. -# -# Today's plan still hash-repartitions: -# Partial AggregateExec (group_clustering_mode=Full) -# -> RepartitionExec Hash([key, date_bin(...)]) -# -> FinalPartitioned AggregateExec (group_clustering_mode=Full) +# date_bin is a negative control: even aligned bins retain the hash shuffle +# because date_bin can produce NULL from non-null inputs and is not admitted +# to range analysis. Timezone-less date_trunc is admitted, but only removes +# the shuffle when every split keeps the resulting groups partition-local. statement ok set datafusion.explain.physical_plan_only = true; @@ -82,11 +77,9 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion ########## # TEST 2: Filtered time-bin aggregation. # GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys -# (date_bin is monotonic in timestamp) and range-partitioned so bins do not -# overlap across the two 60-minute streams. -# -# Today this is still Partial + hash RepartitionExec + Final, even though -# group_clustering_mode=Full is already recognized. +# and the split aligns with the bins in this data. The function remains +# default-denied by range analysis, so Partial + hash RepartitionExec + Final +# is retained even though ordering_mode=Sorted is recognized. ########## query TT @@ -118,7 +111,7 @@ k2 2024-01-01T01:45:00 5 ########## # TEST 3: Same aggregation without the col4 filter. The scan still has two -# 60-minute file groups, and today's plan still hash-repartitions. +# 60-minute file groups, and date_bin still requires a hash repartition. ########## query TT @@ -146,6 +139,64 @@ k2 2024-01-01T00:30:00 7 k2 2024-01-01T01:30:00 30 k2 2024-01-01T01:45:00 5 +########## +# TEST 4: date_trunc('hour') is audited and the split is aligned, so each +# group is partition-local and aggregation needs no hash shuffle. +########## + +query TT +EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin; +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin +ORDER BY key, time_bin; +---- +k1 2024-01-01T00:00:00 3 +k1 2024-01-01T01:00:00 30 +k2 2024-01-01T00:00:00 7 +k2 2024-01-01T01:00:00 35 + +########## +# TEST 5: date_trunc('day') straddles the hour split, so the hash shuffle +# remains. +########## + +query TT +EXPLAIN SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin; +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 ASC +04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_trunc(day, timestamp@1) as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin +ORDER BY key, time_bin; +---- +k1 2024-01-01T00:00:00 33 +k2 2024-01-01T00:00:00 42 + ########## # CLEANUP ########## From 73dff8a764018ae343ef74941d130a319edb27a5 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:00:35 +0000 Subject: [PATCH 24/57] refactor: consume explicit range key locality --- datafusion/physical-plan/src/aggregates/mod.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index fce270ac8b197..dcf372eb2cf07 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1676,12 +1676,11 @@ impl AggregateExec { return false; } - let required = Distribution::KeyPartitioned(self.group_by.input_exprs()); - if input - .output_partitioning() - .satisfaction(&required, input.equivalence_properties(), allow_subset) - .is_key_local() - { + if input.output_partitioning().keeps_keys_local( + &self.group_by.input_exprs(), + input.equivalence_properties(), + allow_subset, + ) { return true; } From ea70fd866c1be9a470ac6422798b19bbae6c1e06 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:57:14 +0000 Subject: [PATCH 25/57] docs: explain default-deny range analysis policy --- datafusion/expr/src/udf.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index daf78781f9e44..4879045f90cc8 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -995,24 +995,22 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// Returns whether this function and argument-type combination has been /// audited for range-partitioning locality analysis. /// - /// Returning `true` explicitly opts the combination into this analysis and - /// asserts that: + /// Range-boundary analysis has subtle correctness requirements, so the + /// policy is default-deny. Functions may opt in only after individual audit. + /// At minimum, an opted-in combination must: /// - /// - same-direction [`SortProperties::Ordered`] metadata accurately describes - /// successful evaluation over the admitted non-null domain when all other - /// inputs are singleton; - /// - successful evaluation returns null exactly when the range input is null. + /// - have same-direction [`SortProperties::Ordered`] metadata over the + /// admitted non-null domain when all other inputs are singleton; + /// - preserve nullness exactly, so successful evaluation returns null if and + /// only if the range input is null. /// - /// This explicit audit is required in addition to ordered metadata. Callers - /// may then prove key locality by evaluating values on both sides of each - /// concrete range split. Ordered metadata alone is insufficient because a - /// non-strict transform can collapse a split boundary. + /// Callers may then prove key locality by evaluating values on both sides of + /// each concrete range split. Ordered metadata alone is insufficient because + /// a non-strict transform can collapse a split boundary. /// /// This does not claim a concrete partition layout or cross-input /// co-partitioning compatibility. Boundary evaluation errors cause the - /// analysis to return `false`. - /// - /// The default is `false`. This capability is not transported across FFI. + /// analysis to return `false`. This capability is not transported across FFI. fn supports_range_partitioning_analysis(&self, _argument_types: &[DataType]) -> bool { false } From 1f46dcb577d1fb3940732c273e98160fa1b4c24c Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:30:32 +0000 Subject: [PATCH 26/57] docs: clarify range partition intervals --- datafusion/physical-expr/src/partitioning.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index b0a3376f298e2..47beb244aad3d 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -581,10 +581,9 @@ fn evaluate_with_range_value( /// For one ascending range key, split points `x₁, ..., xₙ` define: /// /// ```text -/// P₀ P₁ Pₙ -/// (-∞, x₁) [x₁, x₂) ... [xₙ, +∞) -/// │ │ │ │ -/// └─ x₁ ─┘ └─ xₙ ─┘ +/// P₀ = (-∞, x₁) +/// Pᵢ = [xᵢ, xᵢ₊₁) for 1 ≤ i < n +/// Pₙ = [xₙ, +∞) /// ``` /// /// Aggregation can reuse these partitions after applying `f` only if no output From 6e11b1f05d929aa18c59d75025e42f3f3bc34fb4 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:32:31 +0000 Subject: [PATCH 27/57] refactor: return bool from range locality check --- datafusion/physical-expr/src/partitioning.rs | 44 ++++++++++++-------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 47beb244aad3d..0fa1c6669863d 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -624,30 +624,40 @@ fn range_transform_keeps_keys_local( range_key: &Arc, range: &RangePartitioning, schema: &Schema, -) -> Option<()> { - let range_type = range_key.data_type(schema).ok()?; +) -> bool { + let Ok(range_type) = range_key.data_type(schema) else { + return false; + }; let sort_options = range.ordering().first().options; - range.split_points().iter().try_for_each(|split_point| { - let split_value = split_point.values().first()?; + range.split_points().iter().all(|split_point| { + let Some(split_value) = split_point.values().first() else { + return false; + }; if split_value.data_type() != range_type { - return None; + return false; } let adjacent = if sort_options.descending { checked_successor(split_value) } else { checked_predecessor(split_value) - }?; - let at_split = evaluate_with_range_value(function, range_key, split_value)?; - let across_split = evaluate_with_range_value(function, range_key, &adjacent)?; - if at_split.is_null() - || across_split.is_null() - || at_split.data_type() != across_split.data_type() - || at_split == across_split - { - return None; - } + }; + let Some(adjacent) = adjacent else { + return false; + }; + let Some(at_split) = evaluate_with_range_value(function, range_key, split_value) + else { + return false; + }; + let Some(across_split) = + evaluate_with_range_value(function, range_key, &adjacent) + else { + return false; + }; - Some(()) + !at_split.is_null() + && !across_split.is_null() + && at_split.data_type() == across_split.data_type() + && at_split != across_split }) } @@ -680,7 +690,7 @@ fn is_audited_range_transform( } range_argument_count == 1 - && range_transform_keeps_keys_local(candidate, range_key, range, schema).is_some() + && range_transform_keeps_keys_local(candidate, range_key, range, schema) } fn range_partitioning_keeps_keys_local( From 8d58a03779362c7e98285ca5e64ea38a9f3e6ec5 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:44:20 +0000 Subject: [PATCH 28/57] refactor: reuse expression equivalence comparison --- datafusion/physical-expr/src/partitioning.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 0fa1c6669863d..ae35f35c75583 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -539,18 +539,6 @@ fn normalize_exprs( .collect() } -fn equivalent_expr( - left: &Arc, - right: &Arc, - eq_properties: &EquivalenceProperties, -) -> bool { - equivalent_exprs( - std::slice::from_ref(left), - std::slice::from_ref(right), - eq_properties, - ) -} - fn evaluate_with_range_value( expr: &Arc, range_key: &Arc, @@ -682,7 +670,7 @@ fn is_audited_range_transform( let mut range_argument_count = 0; for argument in function.args() { - if equivalent_expr(argument, range_key, eq_properties) { + if eq_properties.eq_group().exprs_equal(argument, range_key) { range_argument_count += 1; } else if !argument.is::() { return false; From 8e18b005d396c5f098e5fb7923bfba7c0467fa0c Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 20:49:15 +0000 Subject: [PATCH 29/57] refactor: limit range analysis bridge visibility --- datafusion/physical-expr/src/scalar_function.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index e9f8c0257f935..ff7327d099705 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -160,7 +160,7 @@ impl ScalarFunctionExpr { /// Returns whether this function expression's argument types have been /// explicitly audited as candidates for range-partitioning analysis. - pub fn supports_range_partitioning_analysis( + pub(crate) fn supports_range_partitioning_analysis( &self, input_schema: &Schema, ) -> Result { From 6982816cc408165cf10a08101c484b849525679f Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:00:37 +0000 Subject: [PATCH 30/57] docs: clarify strict range boundary proof --- datafusion/physical-expr/src/partitioning.rs | 30 +++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index ae35f35c75583..dcb00c0f98049 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -578,27 +578,35 @@ fn evaluate_with_range_value( /// key occurs in two partitions. In other words, the partition images must be /// disjoint: `f(Pᵢ) ∩ f(Pⱼ) = ∅` for `i ≠ j`. /// -/// Monotonicity alone is insufficient. It guarantees only that values do not -/// reverse order across a split: +/// Monotonicity alone is insufficient. Ordered metadata gives only /// /// ```text /// f(predecessor(xᵢ)) ≤ f(xᵢ) /// ``` /// -/// Equality means a group crosses the split. For example, an hourly truncation -/// maps timestamps immediately before and at a `01:30` split to the same -/// `01:00` bucket. A split at `01:00` is safe because its predecessor maps to -/// `00:00`, while the split itself maps to `01:00`. -/// -/// Timestamp values form a discrete domain. For every `x < xᵢ`, monotonicity -/// gives `f(x) ≤ f(predecessor(xᵢ))`; for every `y ≥ xᵢ`, it gives -/// `f(xᵢ) ≤ f(y)`. Therefore checking +/// but locality requires strict separation at every split: /// /// ```text /// f(predecessor(xᵢ)) < f(xᵢ) /// ``` /// -/// at every split proves all partition images are disjoint. The successor of +/// Since ordering already supplies the non-strict inequality, the analysis +/// proves strictness by explicitly checking that the two outputs are not equal. +/// Otherwise, the same transformed key could occur in both adjacent partitions, +/// so the transform would not preserve key locality. For example, an hourly +/// truncation maps timestamps immediately before and at a `01:30` split to the +/// same `01:00` bucket. A split at `01:00` is safe because its predecessor maps +/// to `00:00`, while the split itself maps to `01:00`. +/// +/// `predecessor(xᵢ)` is the greatest representable timestamp below the split. +/// Therefore, for every `x < xᵢ` and `y ≥ xᵢ`, ordering and the strict boundary +/// check give +/// +/// ```text +/// f(x) ≤ f(predecessor(xᵢ)) < f(xᵢ) ≤ f(y) +/// ``` +/// +/// Thus no transformed key can occur on both sides of the split. The successor of /// an ascending split is already in the same partition as `xᵢ`, so checking it /// would not say anything about groups crossing that split. /// From 271c5dabed4c93c59ba76e5a81a3a54df84e13bd Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:01:49 +0000 Subject: [PATCH 31/57] docs: simplify descending boundary explanation --- datafusion/physical-expr/src/partitioning.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index dcb00c0f98049..fced71116780d 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -610,11 +610,9 @@ fn evaluate_with_range_value( /// an ascending split is already in the same partition as `xᵢ`, so checking it /// would not say anything about groups crossing that split. /// -/// Descending ranges use the mirrored check with `successor(xᵢ)`, because their -/// open side lies above the split; there, the predecessor is already in the -/// same partition as `xᵢ`. Null preservation is also required so the transform -/// cannot introduce a null group outside the source partition selected by null -/// order. +/// The argument is the same for descending ranges, but `successor(xᵢ)` must be +/// used instead. Null preservation is also required so the transform cannot +/// introduce a null group outside the source partition selected by null order. fn range_transform_keeps_keys_local( function: &Arc, range_key: &Arc, From fbeab85c38bf0074709d6ccc2324710186331fdd Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:23:56 +0000 Subject: [PATCH 32/57] fix(expr-common): reject unsafe composite interval adjacency --- .../expr-common/src/interval_arithmetic.rs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 469b7531f5723..7f4e08233a1e2 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -1239,9 +1239,10 @@ fn prev_value(value: ScalarValue) -> ScalarValue { } /// Returns the previous distinct value, or `None` for nulls, non-finite -/// floats, type minima, and types without a discrete predecessor. +/// floats, composite intervals, type minima, and types without a discrete +/// predecessor. pub fn checked_predecessor(value: &ScalarValue) -> Option { - if !is_finite_non_null(value) { + if !is_supported_adjacent_value(value) { return None; } @@ -1250,9 +1251,9 @@ pub fn checked_predecessor(value: &ScalarValue) -> Option { } /// Returns the next distinct value, or `None` for nulls, non-finite floats, -/// type maxima, and types without a discrete successor. +/// composite intervals, type maxima, and types without a discrete successor. pub fn checked_successor(value: &ScalarValue) -> Option { - if !is_finite_non_null(value) { + if !is_supported_adjacent_value(value) { return None; } @@ -1260,10 +1261,14 @@ pub fn checked_successor(value: &ScalarValue) -> Option { (!successor.is_null() && successor != *value).then_some(successor) } -fn is_finite_non_null(value: &ScalarValue) -> bool { +fn is_supported_adjacent_value(value: &ScalarValue) -> bool { match value { ScalarValue::Float32(Some(value)) => value.is_finite(), ScalarValue::Float64(Some(value)) => value.is_finite(), + // A component can reach its bound before the composite value reaches MIN + // or MAX (see #25573). Reject these until adjacent values carry between + // components. + ScalarValue::IntervalDayTime(_) | ScalarValue::IntervalMonthDayNano(_) => false, _ => !value.is_null(), } } @@ -2687,6 +2692,16 @@ mod tests { let unsupported = ScalarValue::Utf8(Some("a".into())); assert_eq!(checked_predecessor(&unsupported), None); assert_eq!(checked_successor(&unsupported), None); + + for value in [ + ScalarValue::new_interval_dt(0, i32::MIN), + ScalarValue::new_interval_dt(0, i32::MAX), + ScalarValue::new_interval_mdn(0, 0, i64::MIN), + ScalarValue::new_interval_mdn(0, 0, i64::MAX), + ] { + assert_eq!(checked_predecessor(&value), None); + assert_eq!(checked_successor(&value), None); + } } #[test] From 58115e253f3ba46ddeca7f995c79daefc395b5c1 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:58:22 +0000 Subject: [PATCH 33/57] chore: remove range date_trunc benchmark --- datafusion/core/Cargo.toml | 4 - datafusion/core/benches/range_date_trunc.rs | 152 -------------------- 2 files changed, 156 deletions(-) delete mode 100644 datafusion/core/benches/range_date_trunc.rs diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index ca29bc1a68d18..7275c4b674d20 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -217,10 +217,6 @@ required-features = ["docs_generation"] harness = false name = "aggregate_query_sql" -[[bench]] -harness = false -name = "range_date_trunc" - [[bench]] harness = false name = "csv_load" diff --git a/datafusion/core/benches/range_date_trunc.rs b/datafusion/core/benches/range_date_trunc.rs deleted file mode 100644 index 4e7c3bc2f897d..0000000000000 --- a/datafusion/core/benches/range_date_trunc.rs +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Group hourly range partitions without moving groups across partitions. -//! The same benchmark can run on main: only the optimized plan differs. - -use std::hint::black_box; -use std::sync::Arc; - -use criterion::{Criterion, criterion_group, criterion_main}; -use datafusion::arrow::array::{ArrayRef, Int64Array, TimestampNanosecondArray}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::catalog::streaming::StreamingTable; -use datafusion::common::{Result, ScalarValue, SplitPoint}; -use datafusion::logical_expr::col; -use datafusion::physical_expr::{ - Partitioning, PhysicalSortExpr, RangePartitioning, expressions::col as physical_col, -}; -use datafusion::physical_plan::{ - collect, execution_plan::reset_plan_states, streaming::PartitionStream, - test::TestPartitionStream, -}; -use datafusion::prelude::{SessionConfig, SessionContext}; -use tokio::runtime::Runtime; - -const PARTITIONS: usize = 8; -const REPETITIONS: i64 = 8; - -fn context(keys_per_partition: i64) -> Result { - let schema = Arc::new(Schema::new(vec![ - Field::new("key", DataType::Int64, false), - Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), false), - Field::new("value", DataType::Int64, false), - ])); - let start = 1_704_067_200_000_000_000_i64; - let hour = 3_600_000_000_000_i64; - let mut streams: Vec> = vec![]; - for partition in 0..PARTITIONS { - let mut batches = vec![]; - let rows = keys_per_partition * REPETITIONS; - for offset in (0..rows).step_by(8192) { - let end = (offset + 8192).min(rows); - let columns: Vec = vec![ - Arc::new(Int64Array::from_iter_values( - (offset..end).map(|i| i / REPETITIONS), - )), - Arc::new(TimestampNanosecondArray::from_iter_values( - (offset..end).map(|i| { - start - + partition as i64 * hour - + (i % REPETITIONS) * 1_000_000_000 - }), - )), - Arc::new(Int64Array::from_iter_values((offset..end).map(|_| 1))), - ]; - batches.push(RecordBatch::try_new(Arc::clone(&schema), columns)?); - } - streams.push(Arc::new(TestPartitionStream::new_with_batches(batches))); - } - let partitioning = Partitioning::Range(RangePartitioning::try_new( - [PhysicalSortExpr::new_default(physical_col("ts", &schema)?)].into(), - (1..PARTITIONS) - .map(|partition| { - SplitPoint::new(vec![ScalarValue::TimestampNanosecond( - Some(start + partition as i64 * hour), - None, - )]) - }) - .collect(), - )?); - let mut config = SessionConfig::new().with_target_partitions(PARTITIONS); - config - .options_mut() - .optimizer - .enable_round_robin_repartition = false; - config.options_mut().optimizer.subset_repartition_threshold = PARTITIONS; - let context = SessionContext::new_with_config(config); - context.register_table( - "t", - Arc::new( - StreamingTable::try_new(schema, streams)? - .with_sort_order(vec![ - col("key").sort(true, true), - col("ts").sort(true, true), - ]) - .with_output_partitioning(partitioning), - ), - )?; - Ok(context) -} - -fn criterion_benchmark(criterion: &mut Criterion) { - let runtime = Runtime::new().unwrap(); - for keys_per_partition in [1024, 65_536] { - let context = context(keys_per_partition).unwrap(); - let plan = - runtime.block_on(async { - context - .sql("SELECT key, date_trunc('hour', ts) AS bucket, sum(value) AS total \ - FROM t GROUP BY key, bucket") - .await - .unwrap() - .create_physical_plan() - .await - .unwrap() - }); - let results = runtime - .block_on(collect(Arc::clone(&plan), context.task_ctx())) - .unwrap(); - assert_eq!( - results.iter().map(RecordBatch::num_rows).sum::(), - PARTITIONS * keys_per_partition as usize - ); - for batch in results { - let sums = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - assert!(sums.values().iter().all(|sum| *sum == REPETITIONS)); - } - criterion.bench_function( - &format!("range_date_trunc/{keys_per_partition}_keys_per_partition"), - |bencher| { - bencher.iter(|| { - let plan = reset_plan_states(Arc::clone(&plan)).unwrap(); - black_box( - runtime.block_on(collect(plan, context.task_ctx())).unwrap(), - ) - }); - }, - ); - } -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); From a2d87b2288169b46674ab4659dc76274e87ec84d Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:37:39 +0000 Subject: [PATCH 34/57] refactor: limit range adjacency to timestamps --- .../expr-common/src/interval_arithmetic.rs | 114 +----------------- datafusion/physical-expr/src/partitioning.rs | 45 +++++-- .../physical-expr/src/scalar_function.rs | 34 ------ 3 files changed, 39 insertions(+), 154 deletions(-) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 7f4e08233a1e2..797c48cbe0dd9 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -1238,41 +1238,6 @@ fn prev_value(value: ScalarValue) -> ScalarValue { value_transition!(MIN, false, value) } -/// Returns the previous distinct value, or `None` for nulls, non-finite -/// floats, composite intervals, type minima, and types without a discrete -/// predecessor. -pub fn checked_predecessor(value: &ScalarValue) -> Option { - if !is_supported_adjacent_value(value) { - return None; - } - - let predecessor = prev_value(value.clone()); - (!predecessor.is_null() && predecessor != *value).then_some(predecessor) -} - -/// Returns the next distinct value, or `None` for nulls, non-finite floats, -/// composite intervals, type maxima, and types without a discrete successor. -pub fn checked_successor(value: &ScalarValue) -> Option { - if !is_supported_adjacent_value(value) { - return None; - } - - let successor = next_value(value.clone()); - (!successor.is_null() && successor != *value).then_some(successor) -} - -fn is_supported_adjacent_value(value: &ScalarValue) -> bool { - match value { - ScalarValue::Float32(Some(value)) => value.is_finite(), - ScalarValue::Float64(Some(value)) => value.is_finite(), - // A component can reach its bound before the composite value reaches MIN - // or MAX (see #25573). Reject these until adjacent values carry between - // components. - ScalarValue::IntervalDayTime(_) | ScalarValue::IntervalMonthDayNano(_) => false, - _ => !value.is_null(), - } -} - trait OneTrait: Sized + std::ops::Add + std::ops::Sub { fn one() -> Self; } @@ -2308,8 +2273,8 @@ impl NullableInterval { mod tests { use crate::{ interval_arithmetic::{ - Interval, cast_scalar_value, checked_predecessor, checked_successor, - handle_overflow, next_value, prev_value, satisfy_greater, + Interval, cast_scalar_value, handle_overflow, next_value, prev_value, + satisfy_greater, }, operator::Operator, }; @@ -2629,81 +2594,6 @@ mod tests { Ok(()) } - #[test] - fn test_checked_adjacent_values() { - let timestamp = ScalarValue::TimestampNanosecond(Some(10), None); - assert_eq!( - checked_predecessor(×tamp), - Some(ScalarValue::TimestampNanosecond(Some(9), None)) - ); - assert_eq!( - checked_successor(×tamp), - Some(ScalarValue::TimestampNanosecond(Some(11), None)) - ); - - let timestamp_with_timezone = - ScalarValue::TimestampMillisecond(Some(10), Some("UTC".into())); - assert_eq!( - checked_predecessor(×tamp_with_timezone), - Some(ScalarValue::TimestampMillisecond( - Some(9), - Some("UTC".into()) - )) - ); - assert_eq!( - checked_successor(×tamp_with_timezone), - Some(ScalarValue::TimestampMillisecond( - Some(11), - Some("UTC".into()) - )) - ); - - let finite_float = ScalarValue::Float64(Some(1.0)); - assert_eq!( - checked_predecessor(&finite_float), - Some(ScalarValue::Float64(Some(next_down(1.0_f64)))) - ); - assert_eq!( - checked_successor(&finite_float), - Some(ScalarValue::Float64(Some(next_up(1.0_f64)))) - ); - - assert_eq!( - checked_predecessor(&ScalarValue::TimestampNanosecond(Some(i64::MIN), None)), - None - ); - assert_eq!( - checked_successor(&ScalarValue::TimestampNanosecond(Some(i64::MAX), None)), - None - ); - - let null = ScalarValue::TimestampNanosecond(None, None); - assert_eq!(checked_predecessor(&null), None); - assert_eq!(checked_successor(&null), None); - assert_eq!( - checked_predecessor(&ScalarValue::Float32(Some(f32::NEG_INFINITY))), - None - ); - assert_eq!( - checked_successor(&ScalarValue::Float64(Some(f64::NAN))), - None - ); - - let unsupported = ScalarValue::Utf8(Some("a".into())); - assert_eq!(checked_predecessor(&unsupported), None); - assert_eq!(checked_successor(&unsupported), None); - - for value in [ - ScalarValue::new_interval_dt(0, i32::MIN), - ScalarValue::new_interval_dt(0, i32::MAX), - ScalarValue::new_interval_mdn(0, 0, i64::MIN), - ScalarValue::new_interval_mdn(0, 0, i64::MAX), - ] { - assert_eq!(checked_predecessor(&value), None); - assert_eq!(checked_successor(&value), None); - } - } - #[test] fn test_new_interval() -> Result<()> { use ScalarValue::*; diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index fced71116780d..a3b7db72b4f29 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -30,7 +30,6 @@ use datafusion_common::{ Result, ScalarValue, plan_err, validate_range_split_points, }; use datafusion_expr::ColumnarValue; -use datafusion_expr::interval_arithmetic::{checked_predecessor, checked_successor}; use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; #[cfg(feature = "proto")] @@ -564,6 +563,28 @@ fn evaluate_with_range_value( } } +fn checked_adjacent_timestamp( + value: &ScalarValue, + is_successor: bool, +) -> Option { + let offset = if is_successor { 1 } else { -1 }; + match value { + ScalarValue::TimestampSecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampSecond(Some(value), None)), + ScalarValue::TimestampMillisecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampMillisecond(Some(value), None)), + ScalarValue::TimestampMicrosecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampMicrosecond(Some(value), None)), + ScalarValue::TimestampNanosecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampNanosecond(Some(value), None)), + _ => None, + } +} + /// Determines whether a nondecreasing transform keeps equal output keys local. /// /// For one ascending range key, split points `x₁, ..., xₙ` define: @@ -630,12 +651,9 @@ fn range_transform_keeps_keys_local( if split_value.data_type() != range_type { return false; } - let adjacent = if sort_options.descending { - checked_successor(split_value) - } else { - checked_predecessor(split_value) - }; - let Some(adjacent) = adjacent else { + let Some(adjacent) = + checked_adjacent_timestamp(split_value, sort_options.descending) + else { return false; }; let Some(at_split) = evaluate_with_range_value(function, range_key, split_value) @@ -1756,7 +1774,7 @@ mod tests { assert_key_locality( "the successor of the final nanosecond starts a new bucket", &fixture.range_partitioning_with_ordering( - ordering, + ordering.clone(), vec![timestamp_split(Some(hour + 3_600_000_000_000 - 1), None)], ), &required, @@ -1764,6 +1782,17 @@ mod tests { true, true, ); + assert_key_locality( + "the maximum timestamp has no successor", + &fixture.range_partitioning_with_ordering( + ordering, + vec![timestamp_split(Some(i64::MAX), None)], + ), + &required, + &fixture.eq_properties, + false, + false, + ); Ok(()) } diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index ff7327d099705..dcaaf55e6e20c 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -657,40 +657,6 @@ mod tests { assert_eq!(expr.get_properties(&[child]).unwrap().range, bounds); assert_eq!(expr.evaluate_bounds(&[&bounds]).unwrap(), bounds); } - - fn supports_range_partitioning_analysis( - &self, - argument_types: &[DataType], - ) -> bool { - argument_types == [DataType::Float32] - } - } - - #[test] - fn test_range_partitioning_analysis_support_uses_argument_types() { - let function = Arc::new(ScalarUDF::from(MockScalarUDF { - signature: Signature::any(1, Volatility::Immutable), - })); - let config_options = Arc::new(ConfigOptions::new()); - - for (data_type, expected) in [(DataType::Float32, true), (DataType::Int32, false)] - { - let schema = Schema::new(vec![Field::new("a", data_type, false)]); - let expression = ScalarFunctionExpr::try_new( - Arc::clone(&function), - vec![Arc::new(Column::new("a", 0))], - &schema, - Arc::clone(&config_options), - ) - .unwrap(); - - assert_eq!( - expression - .supports_range_partitioning_analysis(&schema) - .unwrap(), - expected - ); - } } #[test] From d3eddfc10bf36f279c8fddee7e402fe8339d70b3 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:52:55 +0000 Subject: [PATCH 35/57] test: remove redundant range aggregate coverage --- .../enforce_distribution.rs | 104 ------------------ 1 file changed, 104 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index dd0609cd5e68d..283d22767f9b1 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -407,110 +407,6 @@ fn parquet_exec_with_output_partitioning( DataSourceExec::from_data_source(config) } -#[test] -fn range_date_trunc_locality_survives_aggregate_combination() -> Result<()> { - use arrow_schema::TimeUnit; - use datafusion_physical_expr::ScalarFunctionExpr; - use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate; - - let timestamp_type = DataType::Timestamp(TimeUnit::Nanosecond, None); - let schema = Arc::new(Schema::new(vec![Field::new( - "ts", - timestamp_type.clone(), - true, - )])); - let timestamp = col("ts", &schema)?; - let range = Partitioning::Range(RangePartitioning::try_new( - [PhysicalSortExpr::new_default(Arc::clone(×tamp))].into(), - vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond( - Some(1_704_070_800_000_000_000), - None, - )])], - )?); - let scan = FileScanConfigBuilder::new( - ObjectStoreUrl::parse("test:///")?, - Arc::new(ParquetSource::new(Arc::clone(&schema))), - ) - .with_file_groups(vec![ - FileGroup::new(vec![PartitionedFile::new("p0", 100)]), - FileGroup::new(vec![PartitionedFile::new("p1", 100)]), - ]) - .with_output_partitioning(Some(range)) - .build(); - let source: Arc = DataSourceExec::from_data_source(scan); - let bucket: Arc = Arc::new(ScalarFunctionExpr::new( - "date_trunc", - datafusion_functions::datetime::date_trunc(), - vec![lit("hour"), timestamp], - Field::new("bucket", timestamp_type, true).into(), - Arc::new(ConfigOptions::default()), - )); - let group_by = - PhysicalGroupBy::new_single(vec![(Arc::clone(&bucket), "bucket".into())]); - let partial = AggregateExec::try_new( - AggregateMode::Partial, - group_by.clone(), - vec![], - vec![], - Arc::clone(&source), - Arc::clone(&schema), - )?; - let final_plan: Arc = Arc::new(AggregateExec::try_new( - AggregateMode::FinalPartitioned, - group_by.as_final(), - vec![], - vec![], - Arc::new(partial), - schema, - )?); - let mut config = ConfigOptions::default(); - config.execution.target_partitions = 2; - config.optimizer.enable_round_robin_repartition = false; - config.optimizer.repartition_file_scans = false; - let enforced = EnsureRequirements::new().optimize(final_plan, &config)?; - let plan_text = displayable(enforced.as_ref()).indent(true).to_string(); - assert!(plan_text.contains("mode=FinalPartitioned"), "{plan_text}"); - assert!(plan_text.contains("mode=Partial"), "{plan_text}"); - assert!(!plan_text.contains("RepartitionExec"), "{plan_text}"); - SanityCheckPlan::new().optimize(Arc::clone(&enforced), &config)?; - - let combined = CombinePartialFinalAggregate::new().optimize(enforced, &config)?; - let combined_text = displayable(combined.as_ref()).indent(true).to_string(); - assert!( - combined_text.contains("mode=SinglePartitioned"), - "{combined_text}" - ); - let reenforced = EnsureRequirements::new().optimize(combined, &config)?; - assert_eq!( - displayable(reenforced.as_ref()).indent(true).to_string(), - combined_text - ); - SanityCheckPlan::new().optimize(reenforced, &config)?; - - // Group locality does not establish a join-compatible partition layout. - let join = crate::physical_optimizer::test_utils::hash_join_exec( - Arc::clone(&source), - source, - vec![(Arc::clone(&bucket), bucket)], - None, - &JoinType::Inner, - )?; - assert!( - SanityCheckPlan::new() - .optimize(Arc::clone(&join), &config) - .is_err() - ); - let enforced_join = EnsureRequirements::new().optimize(join, &config)?; - assert!( - displayable(enforced_join.as_ref()) - .indent(true) - .to_string() - .contains("partitioning=Hash") - ); - SanityCheckPlan::new().optimize(enforced_join, &config)?; - Ok(()) -} - fn range_partitioning( column: &str, split_values: impl IntoIterator, From 7134bdebf53d899b21853b6edc93996827ded503 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 18:24:56 +0000 Subject: [PATCH 36/57] refactor(optimizer): separate aggregate locality checks --- datafusion/physical-expr/src/partitioning.rs | 4 +- .../enforce_distribution.rs | 46 +++++++++++-------- .../physical-plan/src/aggregates/mod.rs | 8 ++-- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index a3b7db72b4f29..165682464a23e 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -26,9 +26,7 @@ use crate::{ use arrow::datatypes::Schema; pub use datafusion_common::SplitPoint; use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{ - Result, ScalarValue, plan_err, validate_range_split_points, -}; +use datafusion_common::{Result, ScalarValue, plan_err, validate_range_split_points}; use datafusion_expr::ColumnarValue; use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs index 0306817a2e07d..8661a11650a1a 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs @@ -885,11 +885,12 @@ fn add_roundrobin_on_top( } } -// Partial aggregates require unspecified input distribution, but their input -// may already keep every group partition-local. Preserve that layout when -// preserve_file_partitions would otherwise insert RoundRobin below the partial -// aggregate, and reuse it for the final aggregate. -fn partial_aggregate_has_partition_local_groups( +// Partial aggregates require unspecified input distribution, but their output +// may already satisfy the final aggregate's key distribution because partial +// aggregation preserves/projects input partitioning. Keep that reusable output +// partitioning intact when preserve_file_partitions would otherwise insert +// RoundRobin below the partial aggregate. +fn partial_aggregate_output_satisfies_final_partitioning( plan: &Arc, allow_subset_satisfy_partitioning: bool, ) -> bool { @@ -903,23 +904,29 @@ fn partial_aggregate_has_partition_local_groups( return false; } - let output_distribution = Distribution::KeyPartitioned(aggregate.output_group_expr()); - if plan - .output_partitioning() + let key_distribution = Distribution::KeyPartitioned(aggregate.output_group_expr()); + + plan.output_partitioning() .satisfaction( - &output_distribution, + &key_distribution, plan.equivalence_properties(), allow_subset_satisfy_partitioning, ) .is_satisfied() - { - return true; - } +} - aggregate.groups_are_partition_local( - aggregate.input().as_ref(), - allow_subset_satisfy_partitioning, - ) +fn partial_aggregate_input_keeps_groups_local( + plan: &Arc, + allow_subset_satisfy_partitioning: bool, +) -> bool { + plan.downcast_ref::() + .is_some_and(|aggregate| { + aggregate.mode() == &AggregateMode::Partial + && aggregate.groups_are_partition_local( + aggregate.input().as_ref(), + allow_subset_satisfy_partitioning, + ) + }) } /// Adds a [`SortPreservingMergeExec`] or a [`CoalescePartitionsExec`] operator @@ -1693,10 +1700,13 @@ pub fn ensure_distribution_with_stats( let preserve_partial_aggregate_partitioning = preserve_file_partition_threshold_met - && partial_aggregate_has_partition_local_groups( + && (partial_aggregate_output_satisfies_final_partitioning( + &plan, + allow_subset_satisfy_partitioning, + ) || partial_aggregate_input_keeps_groups_local( &plan, allow_subset_satisfy_partitioning, - ); + )); let add_roundrobin = enable_round_robin // Operator benefits from partitioning (e.g. filter): diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index dcf372eb2cf07..5e2b917e663e3 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1670,14 +1670,14 @@ impl AggregateExec { AggregateMode::Partial | AggregateMode::SinglePartitioned | AggregateMode::FinalPartitioned - ) || self.group_by.is_empty() - || self.group_by.has_grouping_set() + ) || self.group_by().is_empty() + || self.group_by().has_grouping_set() { return false; } if input.output_partitioning().keeps_keys_local( - &self.group_by.input_exprs(), + &self.group_by().input_exprs(), input.equivalence_properties(), allow_subset, ) { @@ -1691,7 +1691,7 @@ impl AggregateExec { partial.mode == AggregateMode::Partial && datafusion_physical_expr::physical_exprs_equal( &partial.output_group_expr(), - &self.group_by.input_exprs(), + &self.group_by().input_exprs(), ) && partial.groups_are_partition_local( partial.input.as_ref(), From 994bfca6fef32e7106d6b23934fde32712a99bdd Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:46:52 +0000 Subject: [PATCH 37/57] test: remove redundant date_bin locality check --- datafusion/physical-expr/src/partitioning.rs | 46 +++++++------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 165682464a23e..d083759b7d634 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -1315,6 +1315,17 @@ mod tests { ) -> Partitioning { Partitioning::Range(self.range(indices, split_points)) } + + fn range_partitioning_with_ordering( + &self, + ordering: LexOrdering, + split_points: Vec, + ) -> Partitioning { + Partitioning::Range( + RangePartitioning::try_new(ordering, split_points) + .expect("test range partitioning should be valid"), + ) + } } fn assert_satisfaction( @@ -1380,28 +1391,6 @@ mod tests { )) } - fn date_bin_of(timestamp: Arc) -> Arc { - Arc::new(ScalarFunctionExpr::new( - "date_bin", - datafusion_functions::datetime::date_bin(), - vec![ - Arc::new(Literal::new(ScalarValue::new_interval_mdn( - 0, - 0, - 60_000_000_000, - ))), - timestamp, - ], - Field::new( - "date_bin", - DataType::Timestamp(TimeUnit::Nanosecond, None), - true, - ) - .into(), - Arc::new(ConfigOptions::default()), - )) - } - fn timestamp_split(value: Option, timezone: Option<&str>) -> SplitPoint { SplitPoint::new(vec![ScalarValue::TimestampNanosecond( value, @@ -1603,15 +1592,6 @@ mod tests { false, ); - assert_key_locality( - "date_bin remains default-denied", - &aligned, - &[date_bin_of(fixture.col(0))], - &fixture.eq_properties, - false, - false, - ); - let trunc_day = date_trunc_of(fixture.col(0), "day", None); assert_key_locality( "an hour split straddles a day bucket", @@ -1674,6 +1654,10 @@ mod tests { } #[test] + #[expect( + deprecated, + reason = "test intentionally constructs a malformed range split" + )] fn range_transform_key_locality_fails_closed() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![( "timestamp", From 6cce1b4facf9d52bc0bcb3f60c74cf6a74a937f2 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:12:34 +0000 Subject: [PATCH 38/57] chore(optimizer): address mutable-key Clippy warnings Mark intentional expression hash keys with narrowly scoped lint expectations. Interior-mutable memory accounting does not affect expression hashing or equality. --- datafusion/optimizer/src/push_down_filter.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index becc789d81029..0206f986f03a9 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1175,10 +1175,18 @@ impl OptimizerRule for PushDownFilter { // multiple window functions, each with potentially different partition keys. // Therefore, we need to ensure that any potential partition key returned is used in // ALL window functions. Otherwise, filters cannot be pushed by through that column. + #[expect( + clippy::mutable_key_type, + reason = "Expr's interior mutability does not affect hashing or equality" + )] fn extract_partition_keys(func: &WindowFunction) -> HashSet<&Expr> { func.params.partition_by.iter().collect() } + #[expect( + clippy::mutable_key_type, + reason = "Expr's interior mutability does not affect hashing or equality" + )] let potential_partition_keys = window .window_expr .iter() @@ -1678,6 +1686,10 @@ fn with_filters(predicates: Vec, plan: LogicalPlan) -> LogicalPlan { /// /// A predicate containing a subquery is never pushed: what the subquery reads is /// not visible from the expression tree. +#[expect( + clippy::mutable_key_type, + reason = "Expr's interior mutability does not affect hashing or equality" +)] fn reads_only_partition_keys( expr: &Expr, partition_keys: &HashSet<&Expr>, From 542a9fabd4760b5e0eef066b42010a2885a8d6d0 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:12:36 +0000 Subject: [PATCH 39/57] fix(physical-expr): validate range locality boundaries Reject malformed effective split points before proving transformed-key locality. Cover uncombined aggregation, repeated optimization, incompatible join layouts, and grouping-set subtotals with executable regressions. --- .../enforce_distribution.rs | 183 +++++++++++++++++- datafusion/physical-expr/src/partitioning.rs | 65 +++++++ .../test_files/range_sorted_time_bin_agg.slt | 33 ++++ 3 files changed, 278 insertions(+), 3 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 283d22767f9b1..cc06cfe3acdf7 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -28,9 +28,11 @@ use crate::physical_optimizer::test_utils::{ sort_preserving_merge_exec, union_exec, }; -use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array}; +use arrow::array::{ + Int64Array, RecordBatch, TimestampNanosecondArray, UInt8Array, UInt64Array, +}; use arrow::compute::SortOptions; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion::config::ConfigOptions; use datafusion::datasource::MemTable; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; @@ -75,7 +77,7 @@ use datafusion_physical_plan::operator_statistics::{ use datafusion_physical_expr::{ AcrossPartitions, ConstExpr, Distribution, Partitioning, RangePartitioning, - SplitPoint, + ScalarFunctionExpr, SplitPoint, }; use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion_physical_plan::execution_plan::ExecutionPlan; @@ -959,6 +961,181 @@ fn range_aggregate_reuses_range_partitioning() -> Result<()> { Ok(()) } +fn timestamp_range_input(split: i64) -> Result> { + let schema = Arc::new(Schema::new(vec![Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + )])); + let hour = 3_600_000_000_000; + let (before_split, after_split): (Vec<_>, Vec<_>) = + [0, 1, hour, hour + 1, 2 * hour, 2 * hour + 1] + .into_iter() + .partition(|timestamp| *timestamp < split); + let partitions = [before_split, after_split] + .into_iter() + .map(|values| -> Result<_> { + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(TimestampNanosecondArray::from(values))], + )?; + Ok(vec![batch]) + }) + .collect::>>()?; + let range = RangePartitioning::try_new( + [PhysicalSortExpr::new_default(col("timestamp", &schema)?)].into(), + vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + Some(split), + None, + )])], + )?; + let source = MemorySourceConfig::try_new_exec(&partitions, schema, None)?; + Ok(Arc::new( + source + .as_ref() + .clone() + .with_partitioning(Partitioning::Range(range)), + )) +} + +fn date_trunc_expr( + input: &dyn ExecutionPlan, + precision: &str, +) -> Result> { + Ok(Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![lit(precision), col("timestamp", &input.schema())?], + Field::new( + "bucket", + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ) + .into(), + Arc::new(ConfigOptions::default()), + ))) +} + +#[tokio::test] +async fn range_transform_uncombined_aggregate_is_idempotent() -> Result<()> { + let hour = 3_600_000_000_000; + for (precision, expected_exchanges, expected_groups) in [ + ("hour", 0, vec![(0, 2), (hour, 2), (2 * hour, 2)]), + ("day", 1, vec![(0, 6)]), + ] { + let input = timestamp_range_input(hour)?; + let input_schema = input.schema(); + let group_by = PhysicalGroupBy::new_single(vec![( + date_trunc_expr(input.as_ref(), precision)?, + "bucket".to_string(), + )]); + let count = Arc::new( + AggregateExprBuilder::new( + count_udaf(), + vec![col("timestamp", &input_schema)?], + ) + .schema(Arc::clone(&input_schema)) + .alias("count") + .build()?, + ); + let partial = Arc::new(AggregateExec::try_new( + AggregateMode::Partial, + group_by.clone(), + vec![Arc::clone(&count)], + vec![None], + input, + Arc::clone(&input_schema), + )?); + assert!(matches!( + partial.properties().output_partitioning(), + Partitioning::UnknownPartitioning(2) + )); + let aggregate = Arc::new(AggregateExec::try_new( + AggregateMode::FinalPartitioned, + group_by.as_final(), + vec![count], + vec![None], + partial, + input_schema, + )?); + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + let rendered = displayable(optimized.as_ref()).indent(true).to_string(); + assert_eq!( + rendered.matches("RepartitionExec:").count(), + expected_exchanges, + "{precision}: {rendered}" + ); + assert!(rendered.contains("mode=Partial,"), "{rendered}"); + assert!(rendered.contains("mode=FinalPartitioned,"), "{rendered}"); + let reoptimized = + EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&reoptimized), &config)?; + assert_plan!(optimized, reoptimized); + + let batches = collect(reoptimized, SessionContext::new().task_ctx()).await?; + let mut groups = batches + .iter() + .flat_map(|batch| { + let buckets = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let counts = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + buckets + .values() + .iter() + .copied() + .zip(counts.values().iter().copied()) + }) + .collect::>(); + groups.sort_unstable(); + assert_eq!(groups, expected_groups, "{precision}"); + } + Ok(()) +} + +#[tokio::test] +async fn range_transform_locality_does_not_satisfy_join_distribution() -> Result<()> { + let hour = 3_600_000_000_000; + let left = timestamp_range_input(hour)?; + let right = timestamp_range_input(2 * hour)?; + let left_key = date_trunc_expr(left.as_ref(), "hour")?; + let right_key = date_trunc_expr(right.as_ref(), "hour")?; + for (input, key) in [(&left, &left_key), (&right, &right_key)] { + assert!(input.output_partitioning().keeps_keys_local( + &[Arc::clone(key)], + input.equivalence_properties(), + false, + )); + } + let join = + hash_join_exec(left, right, &vec![(left_key, right_key)], &JoinType::Inner); + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(join, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + for child in optimized.children() { + assert!(child.is::()); + assert!(matches!( + child.output_partitioning(), + Partitioning::Hash(_, 2) + )); + } + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 12); + Ok(()) +} + #[test] fn range_grouping_set_aggregate_rehashes_with_grouping_id() -> Result<()> { let input = parquet_exec_with_output_partitioning(range_partitioning( diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index d083759b7d634..dfd132faa4fa5 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -642,6 +642,9 @@ fn range_transform_keeps_keys_local( return false; }; let sort_options = range.ordering().first().options; + if validate_range_split_points(range.split_points(), &[sort_options]).is_err() { + return false; + } range.split_points().iter().all(|split_point| { let Some(split_value) = split_point.values().first() else { return false; @@ -1653,6 +1656,68 @@ mod tests { Ok(()) } + #[test] + #[expect( + deprecated, + reason = "test intentionally constructs malformed range splits" + )] + fn range_transform_rejects_malformed_split_points() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; + let hour = 3_600_000_000_000; + let timestamp = |value| ScalarValue::TimestampNanosecond(Some(value), None); + + for (description, descending, splits) in [ + ( + "reversed ascending splits", + false, + vec![vec![timestamp(2 * hour)], vec![timestamp(hour)]], + ), + ( + "reversed descending splits", + true, + vec![vec![timestamp(hour - 1)], vec![timestamp(2 * hour - 1)]], + ), + ( + "duplicate ascending splits", + false, + vec![vec![timestamp(hour)], vec![timestamp(hour)]], + ), + ( + "duplicate descending splits", + true, + vec![vec![timestamp(hour - 1)], vec![timestamp(hour - 1)]], + ), + ( + "overwide split tuple", + false, + vec![vec![timestamp(hour), ScalarValue::Int64(Some(0))]], + ), + ("empty split tuple", false, vec![vec![]]), + ] { + let range = Partitioning::Range(RangePartitioning::new( + [PhysicalSortExpr::new( + fixture.col(0), + SortOptions::new(descending, false), + )] + .into(), + splits.into_iter().map(SplitPoint::new).collect(), + )); + assert_key_locality( + description, + &range, + &required, + &fixture.eq_properties, + false, + false, + ); + } + Ok(()) + } + #[test] #[expect( deprecated, diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt index 30a173dedca5c..69a6556d4bc30 100644 --- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt +++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt @@ -197,6 +197,39 @@ ORDER BY key, time_bin; k1 2024-01-01T00:00:00 33 k2 2024-01-01T00:00:00 42 +########## +# TEST 6: Grouping sets can omit the range transform. The per-key subtotal +# crosses the hour split, so it still needs a hash shuffle. +########## + +query TT +EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY GROUPING SETS ((key, time_bin), (key)); +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@3 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp), __grouping_id@2 as __grouping_id], aggr=[sum(range_sorted_time_bin.value)] +03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1, __grouping_id@2], 2), input_partitions=2 +04)------AggregateExec: mode=Partial, gby=[(key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)), (key@0 as key, NULL as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp))], aggr=[sum(range_sorted_time_bin.value)] +05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY GROUPING SETS ((key, time_bin), (key)) +ORDER BY key, time_bin NULLS LAST; +---- +k1 2024-01-01T00:00:00 3 +k1 2024-01-01T01:00:00 30 +k1 NULL 33 +k2 2024-01-01T00:00:00 7 +k2 2024-01-01T01:00:00 35 +k2 NULL 42 + ########## # CLEANUP ########## From 0ef2204d5c982a98df286ace2bd9622a0c1fe08d Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:16:18 +0000 Subject: [PATCH 40/57] chore(optimizer): make lint suppressions feature-compatible --- datafusion/optimizer/src/push_down_filter.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 0206f986f03a9..04442e9599cde 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1176,6 +1176,10 @@ impl OptimizerRule for PushDownFilter { // Therefore, we need to ensure that any potential partition key returned is used in // ALL window functions. Otherwise, filters cannot be pushed by through that column. #[expect( + clippy::allow_attributes, + reason = "Mutable-key warnings depend on enabled features" + )] + #[allow( clippy::mutable_key_type, reason = "Expr's interior mutability does not affect hashing or equality" )] @@ -1184,6 +1188,10 @@ impl OptimizerRule for PushDownFilter { } #[expect( + clippy::allow_attributes, + reason = "Mutable-key warnings depend on enabled features" + )] + #[allow( clippy::mutable_key_type, reason = "Expr's interior mutability does not affect hashing or equality" )] @@ -1687,6 +1695,10 @@ fn with_filters(predicates: Vec, plan: LogicalPlan) -> LogicalPlan { /// A predicate containing a subquery is never pushed: what the subquery reads is /// not visible from the expression tree. #[expect( + clippy::allow_attributes, + reason = "Mutable-key warnings depend on enabled features" +)] +#[allow( clippy::mutable_key_type, reason = "Expr's interior mutability does not affect hashing or equality" )] From 3e210d05c7586d5614afdaec107e21fd743be588 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:16:21 +0000 Subject: [PATCH 41/57] perf(core): benchmark range-partitioned time aggregation --- .../core/benches/aggregate_query_sql.rs | 225 +++++++++++++++++- 1 file changed, 223 insertions(+), 2 deletions(-) diff --git a/datafusion/core/benches/aggregate_query_sql.rs b/datafusion/core/benches/aggregate_query_sql.rs index d7e24aceba170..6de3e19d088f0 100644 --- a/datafusion/core/benches/aggregate_query_sql.rs +++ b/datafusion/core/benches/aggregate_query_sql.rs @@ -17,10 +17,25 @@ mod data_utils; -use criterion::{Criterion, criterion_group, criterion_main}; +use arrow::array::{Int64Array, TimestampNanosecondArray, UInt64Array}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use arrow::record_batch::RecordBatch; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use data_utils::create_table_provider; +use datafusion::catalog::streaming::StreamingTable; use datafusion::error::Result; use datafusion::execution::context::SessionContext; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_plan::execution_plan::reset_plan_states; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::PartitionStream; +use datafusion::physical_plan::{collect, displayable}; +use datafusion::prelude::{SessionConfig, col}; +use datafusion_common::ScalarValue; +use datafusion_physical_expr::expressions::Column; +use datafusion_physical_expr::{ + Partitioning, PhysicalSortExpr, RangePartitioning, SplitPoint, +}; use parking_lot::Mutex; use std::hint::black_box; use std::sync::Arc; @@ -330,5 +345,211 @@ fn criterion_benchmark(c: &mut Criterion) { }); } -criterion_group!(benches, criterion_benchmark); +#[derive(Debug)] +struct TimestampRangePartition { + schema: SchemaRef, + batches: Vec, +} + +impl PartitionStream for TimestampRangePartition { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + fn execute(&self, _: Arc) -> SendableRecordBatchStream { + Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.schema), + futures::stream::iter(self.batches.clone().into_iter().map(Ok)), + )) + } +} + +const RANGE_AGGREGATE_ROWS: usize = 1 << 20; +const ROWS_PER_SERIES: usize = 8; +const HOUR_NANOS: i64 = 3_600_000_000_000; + +fn range_aggregate_context(partition_count: usize) -> Result { + let schema = Arc::new(Schema::new(vec![ + Field::new("series_id", DataType::UInt64, false), + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ), + Field::new("value", DataType::Int64, false), + ])); + let rows_per_partition = RANGE_AGGREGATE_ROWS / partition_count; + let partitions = (0..partition_count) + .map(|partition| -> Result> { + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(UInt64Array::from_iter_values( + (0..rows_per_partition).map(|row| (row / ROWS_PER_SERIES) as u64), + )), + Arc::new(TimestampNanosecondArray::from_iter_values( + (0..rows_per_partition).map(|row| { + partition as i64 * HOUR_NANOS + + (row % ROWS_PER_SERIES) as i64 + * (HOUR_NANOS / ROWS_PER_SERIES as i64) + }), + )), + Arc::new(Int64Array::from_value(1, rows_per_partition)), + ], + )?; + let batches = (0..rows_per_partition) + .step_by(8192) + .map(|offset| batch.slice(offset, 8192.min(rows_per_partition - offset))) + .collect(); + Ok(Arc::new(TimestampRangePartition { + schema: Arc::clone(&schema), + batches, + })) + }) + .collect::>>()?; + let range = RangePartitioning::try_new( + [PhysicalSortExpr::new_default(Arc::new(Column::new( + "timestamp", + 1, + )))] + .into(), + (1..partition_count) + .map(|partition| { + SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + Some(partition as i64 * HOUR_NANOS), + None, + )]) + }) + .collect(), + )?; + let table = StreamingTable::try_new(schema, partitions)? + .with_sort_order(vec![ + col("series_id").sort(true, true), + col("timestamp").sort(true, true), + ]) + .with_output_partitioning(Partitioning::Range(range)); + let mut config = SessionConfig::new().with_target_partitions(partition_count); + config.options_mut().optimizer.subset_repartition_threshold = 1; + config.options_mut().optimizer.preserve_file_partitions = 1; + config + .options_mut() + .optimizer + .enable_round_robin_repartition = false; + let context = SessionContext::new_with_config(config); + context.register_table("range_points", Arc::new(table))?; + Ok(context) +} + +fn check_range_aggregate_results( + batches: &[RecordBatch], + partition_count: usize, + hours_per_bucket: usize, +) { + let mut actual = batches + .iter() + .flat_map(|batch| { + let series = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let buckets = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let sums = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(move |row| (series.value(row), buckets.value(row), sums.value(row))) + }) + .collect::>(); + let series_count = RANGE_AGGREGATE_ROWS / partition_count / ROWS_PER_SERIES; + let mut expected = (0..partition_count) + .step_by(hours_per_bucket) + .flat_map(|first_hour| { + let contributing_hours = hours_per_bucket.min(partition_count - first_hour); + (0..series_count).map(move |series| { + ( + series as u64, + first_hour as i64 * HOUR_NANOS, + (contributing_hours * ROWS_PER_SERIES) as i64, + ) + }) + }) + .collect::>(); + actual.sort_unstable(); + expected.sort_unstable(); + assert_eq!(actual, expected); +} + +/// Measures SQL planning and execution over one million rows in hourly ranges. +/// Hourly buckets are local; daily buckets cross ranges and must still shuffle. +/// Run the identical benchmark on main and the feature branch for an A/B comparison. +fn range_partitioned_aggregate(c: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + for partition_count in [8, 128] { + let context = range_aggregate_context(partition_count).unwrap(); + for (precision, hours_per_bucket) in [("hour", 1), ("day", 24)] { + let sql = format!( + "SELECT series_id, date_trunc('{precision}', timestamp) AS bucket, \ + sum(value) FROM range_points GROUP BY series_id, bucket" + ); + let plan = runtime.block_on(async { + context + .sql(&sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap() + }); + println!( + "\nRange aggregate: partitions={partition_count}, precision={precision}\n{}", + displayable(plan.as_ref()).indent(true) + ); + let batches = runtime + .block_on(collect( + reset_plan_states(Arc::clone(&plan)).unwrap(), + context.task_ctx(), + )) + .unwrap(); + check_range_aggregate_results(&batches, partition_count, hours_per_bucket); + + let mut group = c.benchmark_group(format!( + "range_partitioned_aggregate/{partition_count}/{precision}" + )); + group.bench_function("planning", |b| { + b.iter(|| { + black_box(runtime.block_on(async { + context + .sql(&sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap() + })) + }) + }); + group.bench_function("execution", |b| { + b.iter_batched( + || reset_plan_states(Arc::clone(&plan)).unwrap(), + |plan| { + black_box( + runtime.block_on(collect(plan, context.task_ctx())).unwrap(), + ) + }, + BatchSize::SmallInput, + ) + }); + group.finish(); + } + } +} + +criterion_group!(benches, criterion_benchmark, range_partitioned_aggregate); criterion_main!(benches); From 88f543067a8b58cf2d6ba778bae734b7fa5d5432 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:35:16 +0000 Subject: [PATCH 42/57] chore: remove unrelated lint and comment changes --- datafusion/optimizer/src/push_down_filter.rs | 24 -------------------- datafusion/physical-expr/src/partitioning.rs | 4 ++-- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 04442e9599cde..becc789d81029 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1175,26 +1175,10 @@ impl OptimizerRule for PushDownFilter { // multiple window functions, each with potentially different partition keys. // Therefore, we need to ensure that any potential partition key returned is used in // ALL window functions. Otherwise, filters cannot be pushed by through that column. - #[expect( - clippy::allow_attributes, - reason = "Mutable-key warnings depend on enabled features" - )] - #[allow( - clippy::mutable_key_type, - reason = "Expr's interior mutability does not affect hashing or equality" - )] fn extract_partition_keys(func: &WindowFunction) -> HashSet<&Expr> { func.params.partition_by.iter().collect() } - #[expect( - clippy::allow_attributes, - reason = "Mutable-key warnings depend on enabled features" - )] - #[allow( - clippy::mutable_key_type, - reason = "Expr's interior mutability does not affect hashing or equality" - )] let potential_partition_keys = window .window_expr .iter() @@ -1694,14 +1678,6 @@ fn with_filters(predicates: Vec, plan: LogicalPlan) -> LogicalPlan { /// /// A predicate containing a subquery is never pushed: what the subquery reads is /// not visible from the expression tree. -#[expect( - clippy::allow_attributes, - reason = "Mutable-key warnings depend on enabled features" -)] -#[allow( - clippy::mutable_key_type, - reason = "Expr's interior mutability does not affect hashing or equality" -)] fn reads_only_partition_keys( expr: &Expr, partition_keys: &HashSet<&Expr>, diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index dfd132faa4fa5..58e47dee7feca 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -782,8 +782,8 @@ impl Partitioning { == PartitioningSatisfaction::Exact } - /// Returns the relationship between this [`Partitioning`] and the - /// `required` [`Distribution`]. + /// Returns how this [`Partitioning`] satisfies the partitioning scheme mandated + /// by the `required` [`Distribution`]. #[expect( deprecated, reason = "HashPartitioned is accepted during the KeyPartitioned migration" From d1840e23a3d717a22901111056d24842d2a957d4 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:52:50 +0000 Subject: [PATCH 43/57] refactor: clarify range-key locality analysis naming and docs --- datafusion/expr/src/udf.rs | 16 ++++++++-------- datafusion/functions/src/datetime/date_trunc.rs | 14 +++++++------- datafusion/physical-expr/src/partitioning.rs | 8 +++++--- datafusion/physical-expr/src/scalar_function.rs | 9 ++++++--- .../ensure_requirements/enforce_distribution.rs | 3 +++ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 4879045f90cc8..f900564d453b6 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -364,13 +364,13 @@ impl ScalarUDF { self.inner.output_ordering(inputs) } - /// See [`ScalarUDFImpl::supports_range_partitioning_analysis`] for more details. - pub fn supports_range_partitioning_analysis( + /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for more details. + pub fn supports_range_key_locality_analysis( &self, argument_types: &[DataType], ) -> bool { self.inner - .supports_range_partitioning_analysis(argument_types) + .supports_range_key_locality_analysis(argument_types) } pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { @@ -1011,7 +1011,7 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// This does not claim a concrete partition layout or cross-input /// co-partitioning compatibility. Boundary evaluation errors cause the /// analysis to return `false`. This capability is not transported across FFI. - fn supports_range_partitioning_analysis(&self, _argument_types: &[DataType]) -> bool { + fn supports_range_key_locality_analysis(&self, _argument_types: &[DataType]) -> bool { false } @@ -1237,9 +1237,9 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl { self.inner.output_ordering(inputs) } - fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { + fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool { self.inner - .supports_range_partitioning_analysis(argument_types) + .supports_range_key_locality_analysis(argument_types) } fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { @@ -1325,9 +1325,9 @@ mod tests { } #[test] - fn range_partitioning_analysis_is_disabled_by_default() { + fn range_key_locality_analysis_is_disabled_by_default() { let function = test_func("range_function", "a"); - assert!(!function.supports_range_partitioning_analysis(&[DataType::Int64])); + assert!(!function.supports_range_key_locality_analysis(&[DataType::Int64])); } fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF { diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index b293a0d3022ce..1023d347991cd 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -452,7 +452,7 @@ impl ScalarUDFImpl for DateTruncFunc { } } - fn supports_range_partitioning_analysis(&self, argument_types: &[DataType]) -> bool { + fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool { // Across an America/Goose_Bay transition, ascending epoch seconds // 562129259 < 562129260 truncate to 562129200 > 562125600. // Keep the audited domain to timezone-less timestamp columns. @@ -978,22 +978,22 @@ mod tests { } #[test] - fn range_partitioning_analysis_requires_timezone_less_timestamp() { + fn range_key_locality_analysis_requires_timezone_less_timestamp() { let date_trunc = DateTruncFunc::new(); - assert!(date_trunc.supports_range_partitioning_analysis(&[ + assert!(date_trunc.supports_range_key_locality_analysis(&[ DataType::Utf8, DataType::Timestamp(TimeUnit::Nanosecond, None), ])); - assert!(!date_trunc.supports_range_partitioning_analysis(&[ + assert!(!date_trunc.supports_range_key_locality_analysis(&[ DataType::Utf8, DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), ])); - assert!(!date_trunc.supports_range_partitioning_analysis(&[ + assert!(!date_trunc.supports_range_key_locality_analysis(&[ DataType::Int64, DataType::Timestamp(TimeUnit::Nanosecond, None), ])); - assert!(!date_trunc.supports_range_partitioning_analysis(&[DataType::Utf8])); + assert!(!date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8])); } fn invoke_date_trunc( @@ -1051,7 +1051,7 @@ mod tests { } /// Verifies the exact null contract required by - /// [`ScalarUDFImpl::supports_range_partitioning_analysis`] across every + /// [`ScalarUDFImpl::supports_range_key_locality_analysis`] across every /// supported granularity and timezone-less timestamp unit. Ordering metadata /// and concrete range-split separation are tested separately. #[test] diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 58e47dee7feca..324f3c3f40072 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -674,7 +674,7 @@ fn range_transform_keeps_keys_local( }) } -fn is_audited_range_transform( +fn range_transform_proves_key_locality( candidate: &Arc, range_key: &Arc, range: &RangePartitioning, @@ -686,7 +686,7 @@ fn is_audited_range_transform( let schema = eq_properties.schema(); if datafusion_physical_expr_common::physical_expr::is_volatile(candidate) || !function - .supports_range_partitioning_analysis(schema) + .supports_range_key_locality_analysis(schema) .unwrap_or(false) || !eq_properties.is_same_direction_ordered_transform(candidate, range_key) { @@ -712,13 +712,15 @@ fn range_partitioning_keeps_keys_local( eq_properties: &EquivalenceProperties, allow_subset: bool, ) -> bool { + // Currently, transform locality analysis only supports single-key ranges. + // Extending the proof to compound range keys is tracked in #24644. if range.ordering().len() != 1 || required_exprs.is_empty() { return false; } let range_key = &range.ordering()[0].expr; let has_audited_transform = required_exprs.iter().any(|required| { - is_audited_range_transform(required, range_key, range, eq_properties) + range_transform_proves_key_locality(required, range_key, range, eq_properties) }); has_audited_transform && (required_exprs.len() == 1 || allow_subset) diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index dcaaf55e6e20c..1d9bc2f3fd55a 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -159,8 +159,11 @@ impl ScalarFunctionExpr { } /// Returns whether this function expression's argument types have been - /// explicitly audited as candidates for range-partitioning analysis. - pub(crate) fn supports_range_partitioning_analysis( + /// explicitly audited as candidates for range-key locality analysis. + /// + /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for the safety + /// requirements. + pub(crate) fn supports_range_key_locality_analysis( &self, input_schema: &Schema, ) -> Result { @@ -171,7 +174,7 @@ impl ScalarFunctionExpr { .collect::>>()?; Ok(self .fun - .supports_range_partitioning_analysis(&argument_types)) + .supports_range_key_locality_analysis(&argument_types)) } /// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs index 8661a11650a1a..3bff212385c0e 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs @@ -1738,6 +1738,9 @@ pub fn ensure_distribution_with_stats( Distribution::HashPartitioned(_) | Distribution::KeyPartitioned(_) => { let child_partitions = child.plan.output_partitioning().partition_count(); + // An aggregate can avoid repartitioning if all rows for each group + // are already in one input partition, even when the ordinary + // distribution check fails (#25344). This does not prove join co-partitioning. let partitioning_satisfied = input_distributions .child_satisfaction( child_idx, From 125b59cef9d5b89adf89027892b22d944b164109 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:58:08 +0000 Subject: [PATCH 44/57] refactor: clarify locality proof names and time-bin test comments --- datafusion/physical-expr/src/partitioning.rs | 8 ++--- .../enforce_distribution.rs | 4 +-- .../physical-optimizer/src/sanity_checker.rs | 2 +- .../physical-plan/src/aggregates/mod.rs | 4 +-- .../test_files/range_sorted_time_bin_agg.slt | 31 +++++++++++++------ 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 324f3c3f40072..ea98778edf778 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -632,7 +632,7 @@ fn checked_adjacent_timestamp( /// The argument is the same for descending ranges, but `successor(xᵢ)` must be /// used instead. Null preservation is also required so the transform cannot /// introduce a null group outside the source partition selected by null order. -fn range_transform_keeps_keys_local( +fn range_splits_separate_transformed_keys( function: &Arc, range_key: &Arc, range: &RangePartitioning, @@ -703,7 +703,7 @@ fn range_transform_proves_key_locality( } range_argument_count == 1 - && range_transform_keeps_keys_local(candidate, range_key, range, schema) + && range_splits_separate_transformed_keys(candidate, range_key, range, schema) } fn range_partitioning_keeps_keys_local( @@ -719,11 +719,11 @@ fn range_partitioning_keeps_keys_local( } let range_key = &range.ordering()[0].expr; - let has_audited_transform = required_exprs.iter().any(|required| { + let has_key_locality_proof = required_exprs.iter().any(|required| { range_transform_proves_key_locality(required, range_key, range, eq_properties) }); - has_audited_transform && (required_exprs.len() == 1 || allow_subset) + has_key_locality_proof && (required_exprs.len() == 1 || allow_subset) } /// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement. diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs index 3bff212385c0e..168eef838d43a 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs @@ -922,7 +922,7 @@ fn partial_aggregate_input_keeps_groups_local( plan.downcast_ref::() .is_some_and(|aggregate| { aggregate.mode() == &AggregateMode::Partial - && aggregate.groups_are_partition_local( + && aggregate.input_keeps_groups_local( aggregate.input().as_ref(), allow_subset_satisfy_partitioning, ) @@ -1751,7 +1751,7 @@ pub fn ensure_distribution_with_stats( .is_satisfied() || plan.downcast_ref::().is_some_and( |aggregate| { - aggregate.groups_are_partition_local( + aggregate.input_keeps_groups_local( child.plan.as_ref(), allow_subset_satisfy_partitioning, ) diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs index 0d2138014f7ee..746746761b864 100644 --- a/datafusion/physical-optimizer/src/sanity_checker.rs +++ b/datafusion/physical-optimizer/src/sanity_checker.rs @@ -176,7 +176,7 @@ pub fn check_plan_sanity( let accepts_partition_local_groups = plan .downcast_ref::() .is_some_and(|aggregate| { - aggregate.groups_are_partition_local(child.as_ref(), true) + aggregate.input_keeps_groups_local(child.as_ref(), true) }); if !distribution_satisfaction.is_satisfied() && !accepts_partition_local_groups { let plan_str = get_plan_string(plan); diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 5e2b917e663e3..226ce3d271973 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1660,7 +1660,7 @@ impl AggregateExec { /// local. A final stage can inherit that guarantee from a partial stage /// even when projecting the grouping expressions loses range metadata. /// This check does not require the two stages to be combined. - pub fn groups_are_partition_local( + pub fn input_keeps_groups_local( &self, input: &dyn ExecutionPlan, allow_subset: bool, @@ -1693,7 +1693,7 @@ impl AggregateExec { &partial.output_group_expr(), &self.group_by().input_exprs(), ) - && partial.groups_are_partition_local( + && partial.input_keeps_groups_local( partial.input.as_ref(), allow_subset, ) diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt index 69a6556d4bc30..51411888b595f 100644 --- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt +++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt @@ -29,10 +29,20 @@ # 1. Range([timestamp]) and output_ordering=[key, timestamp] # 2. Two file_groups, so the two 60-minute streams run in parallel # -# date_bin is a negative control: even aligned bins retain the hash shuffle -# because date_bin can produce NULL from non-null inputs and is not admitted -# to range analysis. Timezone-less date_trunc is admitted, but only removes -# the shuffle when every split keeps the resulting groups partition-local. +# Improvement opportunity: +# In this fixture, date_bin(60s) is monotonic in timestamp and the hour split +# is aligned to bin boundaries, so (key, time_bin) is partition-disjoint. +# Aggregation could be a single streaming SinglePartitioned step with no +# hash shuffle. +# +# This optimization exists for timezone-less date_trunc when every split +# keeps groups partition-local. Extending it to date_bin is future work +# requiring a safety audit, including its null behavior (#25344). +# +# Today's date_bin plan still hash-repartitions: +# Partial AggregateExec (ordering_mode=Sorted) +# -> RepartitionExec Hash([key, date_bin(...)]) +# -> FinalPartitioned AggregateExec (ordering_mode=Sorted) statement ok set datafusion.explain.physical_plan_only = true; @@ -77,9 +87,11 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion ########## # TEST 2: Filtered time-bin aggregation. # GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys -# and the split aligns with the bins in this data. The function remains -# default-denied by range analysis, so Partial + hash RepartitionExec + Final -# is retained even though ordering_mode=Sorted is recognized. +# and the split aligns with the bins in this data, so the groups are +# partition-local. Timezone-less date_trunc can already avoid the shuffle; +# extending this optimization to date_bin awaits its safety audit (#25344). +# Until then, Partial + hash RepartitionExec + Final is retained even though +# ordering_mode=Sorted is recognized. ########## query TT @@ -110,8 +122,9 @@ k2 2024-01-01T01:30:00 30 k2 2024-01-01T01:45:00 5 ########## -# TEST 3: Same aggregation without the col4 filter. The scan still has two -# 60-minute file groups, and date_bin still requires a hash repartition. +# TEST 3: Same aggregation without the col4 filter. The groups remain +# partition-local, but date_bin retains the hash shuffle pending its +# safety audit (#25344). ########## query TT From a9919b90abb073daf54d058e33787d8a0cd6fc71 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:05:44 +0000 Subject: [PATCH 45/57] test: clarify range-transform aggregate regression coverage --- .../enforce_distribution.rs | 193 +++++++++++------- 1 file changed, 119 insertions(+), 74 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index cc06cfe3acdf7..3a96385c26775 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -33,6 +33,7 @@ use arrow::array::{ }; use arrow::compute::SortOptions; use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use datafusion::assert_batches_sorted_eq; use datafusion::config::ConfigOptions; use datafusion::datasource::MemTable; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; @@ -1016,89 +1017,133 @@ fn date_trunc_expr( ))) } -#[tokio::test] -async fn range_transform_uncombined_aggregate_is_idempotent() -> Result<()> { - let hour = 3_600_000_000_000; - for (precision, expected_exchanges, expected_groups) in [ - ("hour", 0, vec![(0, 2), (hour, 2), (2 * hour, 2)]), - ("day", 1, vec![(0, 6)]), - ] { - let input = timestamp_range_input(hour)?; - let input_schema = input.schema(); - let group_by = PhysicalGroupBy::new_single(vec![( - date_trunc_expr(input.as_ref(), precision)?, - "bucket".to_string(), - )]); - let count = Arc::new( - AggregateExprBuilder::new( - count_udaf(), - vec![col("timestamp", &input_schema)?], - ) +/// Builds separate Partial and Final stages without projected range metadata. +/// This exercises checking group locality through the partial stage's input. +fn partial_final_date_trunc_count( + input: Arc, + precision: &str, +) -> Result> { + let input_schema = input.schema(); + let group_by = PhysicalGroupBy::new_single(vec![( + date_trunc_expr(input.as_ref(), precision)?, + "bucket".to_string(), + )]); + let count = Arc::new( + AggregateExprBuilder::new(count_udaf(), vec![col("timestamp", &input_schema)?]) .schema(Arc::clone(&input_schema)) .alias("count") .build()?, - ); - let partial = Arc::new(AggregateExec::try_new( - AggregateMode::Partial, - group_by.clone(), - vec![Arc::clone(&count)], - vec![None], - input, - Arc::clone(&input_schema), - )?); - assert!(matches!( - partial.properties().output_partitioning(), - Partitioning::UnknownPartitioning(2) - )); - let aggregate = Arc::new(AggregateExec::try_new( - AggregateMode::FinalPartitioned, - group_by.as_final(), - vec![count], - vec![None], - partial, - input_schema, - )?); - let config = TestConfig::default() - .with_query_execution_partitions(2) - .config; + ); + let partial = Arc::new(AggregateExec::try_new( + AggregateMode::Partial, + group_by.clone(), + vec![Arc::clone(&count)], + vec![None], + input, + Arc::clone(&input_schema), + )?); + assert!(matches!( + partial.properties().output_partitioning(), + Partitioning::UnknownPartitioning(2) + )); + + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::FinalPartitioned, + group_by.as_final(), + vec![count], + vec![None], + partial, + input_schema, + )?)) +} + +#[tokio::test] +async fn aligned_hour_groups_avoid_shuffle_through_partial_aggregate() -> Result<()> { + let hour = 3_600_000_000_000; + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, "hour")?; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + + assert_plan!( + optimized, + @r" + AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count] + AggregateExec: mode=Partial, gby=[date_trunc(hour, timestamp@0) as bucket], aggr=[count] + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_batches_sorted_eq!( + [ + "+---------------------+-------+", + "| bucket | count |", + "+---------------------+-------+", + "| 1970-01-01T00:00:00 | 2 |", + "| 1970-01-01T01:00:00 | 2 |", + "| 1970-01-01T02:00:00 | 2 |", + "+---------------------+-------+", + ], + &batches + ); + Ok(()) +} + +#[tokio::test] +async fn crossing_day_groups_keep_shuffle_through_partial_aggregate() -> Result<()> { + let hour = 3_600_000_000_000; + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, "day")?; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + + assert_plan!( + optimized, + @r" + AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count] + RepartitionExec: partitioning=Hash([bucket@0], 2), input_partitions=2 + AggregateExec: mode=Partial, gby=[date_trunc(day, timestamp@0) as bucket], aggr=[count] + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_batches_sorted_eq!( + [ + "+---------------------+-------+", + "| bucket | count |", + "+---------------------+-------+", + "| 1970-01-01T00:00:00 | 6 |", + "+---------------------+-------+", + ], + &batches + ); + Ok(()) +} + +#[test] +fn range_transform_aggregate_optimization_is_idempotent() -> Result<()> { + let hour = 3_600_000_000_000; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + + for precision in ["hour", "day"] { + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, precision)?; let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; - let rendered = displayable(optimized.as_ref()).indent(true).to_string(); - assert_eq!( - rendered.matches("RepartitionExec:").count(), - expected_exchanges, - "{precision}: {rendered}" - ); - assert!(rendered.contains("mode=Partial,"), "{rendered}"); - assert!(rendered.contains("mode=FinalPartitioned,"), "{rendered}"); let reoptimized = EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?; SanityCheckPlan::new().optimize(Arc::clone(&reoptimized), &config)?; assert_plan!(optimized, reoptimized); - - let batches = collect(reoptimized, SessionContext::new().task_ctx()).await?; - let mut groups = batches - .iter() - .flat_map(|batch| { - let buckets = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let counts = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - buckets - .values() - .iter() - .copied() - .zip(counts.values().iter().copied()) - }) - .collect::>(); - groups.sort_unstable(); - assert_eq!(groups, expected_groups, "{precision}"); } Ok(()) } From bc2d5f82438fd02fa4e1851485a03a1600c4df14 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:32:47 +0000 Subject: [PATCH 46/57] test: clarify range-locality boundary and join coverage --- .../enforce_distribution.rs | 24 +- datafusion/physical-expr/src/partitioning.rs | 236 ++++++++++++------ 2 files changed, 179 insertions(+), 81 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 3a96385c26775..20ea0fa35319d 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -1169,15 +1169,23 @@ async fn range_transform_locality_does_not_satisfy_join_distribution() -> Result .config; let optimized = EnsureRequirements::new().optimize(join, &config)?; SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; - for child in optimized.children() { - assert!(child.is::()); - assert!(matches!( - child.output_partitioning(), - Partitioning::Hash(_, 2) - )); - } + assert_plan!( + optimized, + @r" + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(hour, timestamp@0), date_trunc(hour, timestamp@0))] + RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2 + DataSourceExec: partitions=2, partition_sizes=[1, 1] + RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2 + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; - assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 12); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 3 * 2 * 2, + "three hour buckets, each joining two left rows with two right rows" + ); Ok(()) } diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index ea98778edf778..f6419a3134cb7 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -1403,11 +1403,63 @@ mod tests { )]) } - fn assert_date_trunc_partition_images() - -> Result<()> { + /// Assigns sample rows to source ranges and evaluates whole arrays, + /// independently of the optimizer's boundary-adjacency proof. + fn date_trunc_partition_images_are_disjoint< + T: arrow::array::types::ArrowTimestampType, + >( + function: &Arc, + splits: &[i64], + options: SortOptions, + schema: &SchemaRef, + ) -> Result { use arrow::array::{PrimitiveArray, RecordBatch}; use datafusion_common::HashSet; + let mut partitions = vec![vec![]; splits.len() + 1]; + let rows = splits + .iter() + .flat_map(|split| [Some(split - 1), Some(*split), Some(split + 1)]) + .chain([None, None]); + for value in rows { + let index = match value { + None if options.nulls_first => 0, + None => splits.len(), + Some(value) => splits + .iter() + .filter(|split| { + if options.descending { + value <= **split + } else { + value >= **split + } + }) + .count(), + }; + partitions[index].push(value); + } + let images = partitions + .into_iter() + .map(|values| { + let input = Arc::new(PrimitiveArray::::from_iter(values)); + let batch = RecordBatch::try_new(Arc::clone(schema), vec![input])?; + let output = function.evaluate(&batch)?.into_array(batch.num_rows())?; + (0..output.len()) + .map(|i| ScalarValue::try_from_array(&output, i)) + .collect::>>() + }) + .collect::>>()?; + + Ok(images.iter().enumerate().all(|(i, left)| { + images + .iter() + .skip(i + 1) + .all(|right| left.is_disjoint(right)) + })) + } + + fn assert_date_trunc_partition_images() + -> Result<()> { let ticks_per_second = match T::UNIT { TimeUnit::Second => 1, TimeUnit::Millisecond => 1_000, @@ -1422,24 +1474,79 @@ mod tests { let properties = EquivalenceProperties::new(Arc::clone(&schema)); let mut cases = vec![ - ("hour", false, vec![-hour, 0, hour], true), - ("hour", false, vec![-hour + 1, 1, hour + 1], false), - ("day", false, vec![-hour, 0, hour], false), - ("hour", true, vec![-hour, 0, hour], false), - ("hour", true, vec![-hour - 1, -1, hour - 1], true), + ( + "aligned ascending hours", + "hour", + false, + vec![-hour, 0, hour], + true, + ), + ( + "unaligned ascending hours", + "hour", + false, + vec![-hour + 1, 1, hour + 1], + false, + ), + ( + "days cross hourly splits", + "day", + false, + vec![-hour, 0, hour], + false, + ), + ( + "descending splits inside hour buckets", + "hour", + true, + vec![-hour, 0, hour], + false, + ), + ( + "descending splits between hour buckets", + "hour", + true, + vec![-hour - 1, -1, hour - 1], + true, + ), ]; for (precision, divisor) in [("millisecond", 1_000), ("microsecond", 1_000_000)] { let width = (ticks_per_second / divisor).max(1); cases.extend([ - (precision, false, vec![-width, 0, width], true), + ( + "aligned ascending subsecond splits", + precision, + false, + vec![-width, 0, width], + true, + ), // In particular, microsecond input with millisecond precision // must reject split -999: both adjacent array values map to -1000. - (precision, false, vec![-width + 1], width == 1), - (precision, true, vec![-width, 0, width], width == 1), - (precision, true, vec![-width - 1, -1, width - 1], true), + ( + "offset ascending subsecond split", + precision, + false, + vec![-width + 1], + width == 1, + ), + ( + "descending splits at subsecond bucket starts", + precision, + true, + vec![-width, 0, width], + width == 1, + ), + ( + "descending splits at subsecond bucket ends", + precision, + true, + vec![-width - 1, -1, width - 1], + true, + ), ]); } - for (precision, descending, source_splits, expect_disjoint) in cases { + for (description, precision, descending, source_splits, expect_disjoint) in cases + { for nulls_first in [false, true] { let options = SortOptions::new(descending, nulls_first); let mut splits = source_splits.clone(); @@ -1470,56 +1577,16 @@ mod tests { )); assert_eq!( range.keeps_keys_local(&[Arc::clone(&function)], &properties, false), - expect_disjoint + expect_disjoint, + "optimizer: {description}, {precision}, {:?}, {options:?}, splits={splits:?}", + T::UNIT ); - - // Assign rows by source-domain comparisons, independently of - // the optimizer's adjacency check, then evaluate whole arrays. - let mut partitions = vec![vec![]; splits.len() + 1]; - let rows = splits - .iter() - .flat_map(|split| [Some(split - 1), Some(*split), Some(split + 1)]) - .chain([None, None]); - for value in rows { - let index = match value { - None if nulls_first => 0, - None => splits.len(), - Some(value) => splits - .iter() - .filter(|split| { - if descending { - value <= **split - } else { - value >= **split - } - }) - .count(), - }; - partitions[index].push(value); - } - let images = partitions - .into_iter() - .map(|values| { - let input = Arc::new(PrimitiveArray::::from_iter(values)); - let batch = - RecordBatch::try_new(Arc::clone(&schema), vec![input])?; - let output = - function.evaluate(&batch)?.into_array(batch.num_rows())?; - (0..output.len()) - .map(|i| ScalarValue::try_from_array(&output, i)) - .collect::>>() - }) - .collect::>>()?; - let disjoint = images.iter().enumerate().all(|(i, left)| { - images - .iter() - .skip(i + 1) - .all(|right| left.is_disjoint(right)) - }); assert_eq!( - disjoint, + date_trunc_partition_images_are_disjoint::( + &function, &splits, options, &schema, + )?, expect_disjoint, - "{precision}, {:?}, {options:?}, splits={splits:?}", + "array evaluation: {description}, {precision}, {:?}, {options:?}, splits={splits:?}", T::UNIT ); } @@ -1721,21 +1788,19 @@ mod tests { } #[test] - #[expect( - deprecated, - reason = "test intentionally constructs a malformed range split" - )] - fn range_transform_key_locality_fails_closed() -> Result<()> { + fn range_transform_rejects_null_or_minimum_timestamp_splits() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![( "timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None), )])?; - let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); - let required = vec![trunc_hour]; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; for (description, split) in [ ("null", timestamp_split(None, None)), - ("minimum", timestamp_split(Some(i64::MIN), None)), + ( + "minimum has no predecessor", + timestamp_split(Some(i64::MIN), None), + ), ] { assert_key_locality( description, @@ -1746,7 +1811,20 @@ mod tests { false, ); } + Ok(()) + } + #[test] + #[expect( + deprecated, + reason = "test intentionally constructs a malformed range split" + )] + fn range_transform_rejects_split_type_mismatch() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; let mismatched_split = Partitioning::Range(RangePartitioning::new( fixture.range_ordering([0]), vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])], @@ -1759,7 +1837,15 @@ mod tests { false, false, ); + Ok(()) + } + #[test] + fn range_transform_rejects_boundary_evaluation_errors() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; let invalid_precision = vec![date_trunc_of(fixture.col(0), "not-a-precision", None)]; let aligned = fixture.range_partitioning( @@ -1774,13 +1860,17 @@ mod tests { false, false, ); + Ok(()) + } - let timezone_fixture = PartitioningTestFixture::new(vec![( + #[test] + fn range_transform_rejects_timezone_annotated_timestamps() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( "timestamp", DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), )])?; - let timezone_trunc = date_trunc_of(timezone_fixture.col(0), "hour", Some("UTC")); - let timezone_range = timezone_fixture.range_partitioning( + let trunc_hour = date_trunc_of(fixture.col(0), "hour", Some("UTC")); + let range = fixture.range_partitioning( [0], vec![timestamp_split( Some(1_704_070_800_000_000_000), @@ -1789,9 +1879,9 @@ mod tests { ); assert_key_locality( "timezone-aware date_trunc is not audited", - &timezone_range, - &[timezone_trunc], - &timezone_fixture.eq_properties, + &range, + &[trunc_hour], + &fixture.eq_properties, false, false, ); From 65c3c78c13974bff29a340f1a3d3c6e43fbaefef Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:07:12 +0000 Subject: [PATCH 47/57] chore: remove range aggregation benchmark --- .../core/benches/aggregate_query_sql.rs | 225 +----------------- 1 file changed, 2 insertions(+), 223 deletions(-) diff --git a/datafusion/core/benches/aggregate_query_sql.rs b/datafusion/core/benches/aggregate_query_sql.rs index 6de3e19d088f0..d7e24aceba170 100644 --- a/datafusion/core/benches/aggregate_query_sql.rs +++ b/datafusion/core/benches/aggregate_query_sql.rs @@ -17,25 +17,10 @@ mod data_utils; -use arrow::array::{Int64Array, TimestampNanosecondArray, UInt64Array}; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use arrow::record_batch::RecordBatch; -use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use criterion::{Criterion, criterion_group, criterion_main}; use data_utils::create_table_provider; -use datafusion::catalog::streaming::StreamingTable; use datafusion::error::Result; use datafusion::execution::context::SessionContext; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_plan::execution_plan::reset_plan_states; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::streaming::PartitionStream; -use datafusion::physical_plan::{collect, displayable}; -use datafusion::prelude::{SessionConfig, col}; -use datafusion_common::ScalarValue; -use datafusion_physical_expr::expressions::Column; -use datafusion_physical_expr::{ - Partitioning, PhysicalSortExpr, RangePartitioning, SplitPoint, -}; use parking_lot::Mutex; use std::hint::black_box; use std::sync::Arc; @@ -345,211 +330,5 @@ fn criterion_benchmark(c: &mut Criterion) { }); } -#[derive(Debug)] -struct TimestampRangePartition { - schema: SchemaRef, - batches: Vec, -} - -impl PartitionStream for TimestampRangePartition { - fn schema(&self) -> &SchemaRef { - &self.schema - } - - fn execute(&self, _: Arc) -> SendableRecordBatchStream { - Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.schema), - futures::stream::iter(self.batches.clone().into_iter().map(Ok)), - )) - } -} - -const RANGE_AGGREGATE_ROWS: usize = 1 << 20; -const ROWS_PER_SERIES: usize = 8; -const HOUR_NANOS: i64 = 3_600_000_000_000; - -fn range_aggregate_context(partition_count: usize) -> Result { - let schema = Arc::new(Schema::new(vec![ - Field::new("series_id", DataType::UInt64, false), - Field::new( - "timestamp", - DataType::Timestamp(TimeUnit::Nanosecond, None), - false, - ), - Field::new("value", DataType::Int64, false), - ])); - let rows_per_partition = RANGE_AGGREGATE_ROWS / partition_count; - let partitions = (0..partition_count) - .map(|partition| -> Result> { - let batch = RecordBatch::try_new( - Arc::clone(&schema), - vec![ - Arc::new(UInt64Array::from_iter_values( - (0..rows_per_partition).map(|row| (row / ROWS_PER_SERIES) as u64), - )), - Arc::new(TimestampNanosecondArray::from_iter_values( - (0..rows_per_partition).map(|row| { - partition as i64 * HOUR_NANOS - + (row % ROWS_PER_SERIES) as i64 - * (HOUR_NANOS / ROWS_PER_SERIES as i64) - }), - )), - Arc::new(Int64Array::from_value(1, rows_per_partition)), - ], - )?; - let batches = (0..rows_per_partition) - .step_by(8192) - .map(|offset| batch.slice(offset, 8192.min(rows_per_partition - offset))) - .collect(); - Ok(Arc::new(TimestampRangePartition { - schema: Arc::clone(&schema), - batches, - })) - }) - .collect::>>()?; - let range = RangePartitioning::try_new( - [PhysicalSortExpr::new_default(Arc::new(Column::new( - "timestamp", - 1, - )))] - .into(), - (1..partition_count) - .map(|partition| { - SplitPoint::new(vec![ScalarValue::TimestampNanosecond( - Some(partition as i64 * HOUR_NANOS), - None, - )]) - }) - .collect(), - )?; - let table = StreamingTable::try_new(schema, partitions)? - .with_sort_order(vec![ - col("series_id").sort(true, true), - col("timestamp").sort(true, true), - ]) - .with_output_partitioning(Partitioning::Range(range)); - let mut config = SessionConfig::new().with_target_partitions(partition_count); - config.options_mut().optimizer.subset_repartition_threshold = 1; - config.options_mut().optimizer.preserve_file_partitions = 1; - config - .options_mut() - .optimizer - .enable_round_robin_repartition = false; - let context = SessionContext::new_with_config(config); - context.register_table("range_points", Arc::new(table))?; - Ok(context) -} - -fn check_range_aggregate_results( - batches: &[RecordBatch], - partition_count: usize, - hours_per_bucket: usize, -) { - let mut actual = batches - .iter() - .flat_map(|batch| { - let series = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let buckets = batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let sums = batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap(); - (0..batch.num_rows()) - .map(move |row| (series.value(row), buckets.value(row), sums.value(row))) - }) - .collect::>(); - let series_count = RANGE_AGGREGATE_ROWS / partition_count / ROWS_PER_SERIES; - let mut expected = (0..partition_count) - .step_by(hours_per_bucket) - .flat_map(|first_hour| { - let contributing_hours = hours_per_bucket.min(partition_count - first_hour); - (0..series_count).map(move |series| { - ( - series as u64, - first_hour as i64 * HOUR_NANOS, - (contributing_hours * ROWS_PER_SERIES) as i64, - ) - }) - }) - .collect::>(); - actual.sort_unstable(); - expected.sort_unstable(); - assert_eq!(actual, expected); -} - -/// Measures SQL planning and execution over one million rows in hourly ranges. -/// Hourly buckets are local; daily buckets cross ranges and must still shuffle. -/// Run the identical benchmark on main and the feature branch for an A/B comparison. -fn range_partitioned_aggregate(c: &mut Criterion) { - let runtime = Runtime::new().unwrap(); - for partition_count in [8, 128] { - let context = range_aggregate_context(partition_count).unwrap(); - for (precision, hours_per_bucket) in [("hour", 1), ("day", 24)] { - let sql = format!( - "SELECT series_id, date_trunc('{precision}', timestamp) AS bucket, \ - sum(value) FROM range_points GROUP BY series_id, bucket" - ); - let plan = runtime.block_on(async { - context - .sql(&sql) - .await - .unwrap() - .create_physical_plan() - .await - .unwrap() - }); - println!( - "\nRange aggregate: partitions={partition_count}, precision={precision}\n{}", - displayable(plan.as_ref()).indent(true) - ); - let batches = runtime - .block_on(collect( - reset_plan_states(Arc::clone(&plan)).unwrap(), - context.task_ctx(), - )) - .unwrap(); - check_range_aggregate_results(&batches, partition_count, hours_per_bucket); - - let mut group = c.benchmark_group(format!( - "range_partitioned_aggregate/{partition_count}/{precision}" - )); - group.bench_function("planning", |b| { - b.iter(|| { - black_box(runtime.block_on(async { - context - .sql(&sql) - .await - .unwrap() - .create_physical_plan() - .await - .unwrap() - })) - }) - }); - group.bench_function("execution", |b| { - b.iter_batched( - || reset_plan_states(Arc::clone(&plan)).unwrap(), - |plan| { - black_box( - runtime.block_on(collect(plan, context.task_ctx())).unwrap(), - ) - }, - BatchSize::SmallInput, - ) - }); - group.finish(); - } - } -} - -criterion_group!(benches, criterion_benchmark, range_partitioned_aggregate); +criterion_group!(benches, criterion_benchmark); criterion_main!(benches); From a6482e56a4cb9ef06b4efdeeecdd37b64f67a21e Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:29:41 +0000 Subject: [PATCH 48/57] test: update time-bin plans for aggregate clustering terminology --- .../test_files/range_sorted_time_bin_agg.slt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt index 51411888b595f..e8c656450ba23 100644 --- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt +++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt @@ -40,9 +40,9 @@ # requiring a safety audit, including its null behavior (#25344). # # Today's date_bin plan still hash-repartitions: -# Partial AggregateExec (ordering_mode=Sorted) +# Partial AggregateExec (group_clustering_mode=Full) # -> RepartitionExec Hash([key, date_bin(...)]) -# -> FinalPartitioned AggregateExec (ordering_mode=Sorted) +# -> FinalPartitioned AggregateExec (group_clustering_mode=Full) statement ok set datafusion.explain.physical_plan_only = true; @@ -91,7 +91,7 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion # partition-local. Timezone-less date_trunc can already avoid the shuffle; # extending this optimization to date_bin awaits its safety audit (#25344). # Until then, Partial + hash RepartitionExec + Final is retained even though -# ordering_mode=Sorted is recognized. +# group_clustering_mode=Full is recognized. ########## query TT @@ -165,7 +165,7 @@ GROUP BY key, time_bin; ---- physical_plan 01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] -02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full 03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] 04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] @@ -194,9 +194,9 @@ GROUP BY key, time_bin; ---- physical_plan 01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] -02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full 03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 ASC -04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_trunc(day, timestamp@1) as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], ordering_mode=Sorted +04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_trunc(day, timestamp@1) as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full 05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] 06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] From c7a7f9440ab1e01f7e373c455801c2c9c6903d12 Mon Sep 17 00:00:00 2001 From: Ben Bellick <36523439+benbellick@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:34:04 +0000 Subject: [PATCH 49/57] test: cover adjacent timestamp truncation overflow --- datafusion/physical-expr/src/partitioning.rs | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index f6419a3134cb7..15d7849ebeac1 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -1863,6 +1863,43 @@ mod tests { Ok(()) } + #[test] + fn range_transform_rejects_adjacent_timestamp_evaluation_errors() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let nanoseconds_per_hour = 3_600_000_000_000; + let first_representable_hour = + i64::MIN / nanoseconds_per_hour * nanoseconds_per_hour; + let split = + ScalarValue::TimestampNanosecond(Some(first_representable_hour), None); + let predecessor = + ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None); + let timestamp = fixture.col(0); + let trunc_hour = date_trunc_of(Arc::clone(×tamp), "hour", None); + + assert_eq!( + evaluate_with_range_value(&trunc_hour, ×tamp, &split), + Some(split.clone()), + ); + assert_eq!( + evaluate_with_range_value(&trunc_hour, ×tamp, &predecessor), + None, + ); + + let range = fixture.range_partitioning([0], vec![SplitPoint::new(vec![split])]); + assert_key_locality( + "predecessor truncation underflows even though split truncation succeeds", + &range, + &[trunc_hour], + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + #[test] fn range_transform_rejects_timezone_annotated_timestamps() -> Result<()> { let fixture = PartitioningTestFixture::new(vec![( From f43f67b844189af217e252f9894fe66d02546124 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Thu, 22 Oct 2026 08:45:59 +0000 Subject: [PATCH 50/57] chore: check for circular dependencies with `cargo publish` (#26582) ## Which issue does this PR close? - Related to #25088, #25148 ## Rationale for this change Normal `cargo` commands reject circular dependencies between packages, but they allow cycles through dev-dependencies. This has bitten us in the past, which is why the `dev/depcheck` tool and its associated CI job were added to detect dev-dependency cycles. Running `dev/depcheck` in CI is relatively expensive, in part because it depends on the `cargo` crate, which has a large dependency tree. Building and running `depcheck` takes ~130 seconds on a 4 vCPU container, and also requires an `ubuntu-latest` runner. Cargo 1.90 added support for multiple packages in a single `cargo publish`, which does check for dev-dependency cycles. It follows that we can use a wrapper script over `cargo publish --workspace --dry-run` to replace `depcheck`. This is more efficient and can run on `ubuntu-slim`. Moreover, it avoids the need to maintain a separate tool and ensures that the CI check is consistent with how packages are actually published. (We should probably also switch to using multi-package `cargo publish` to make releases, but I'll defer that for now.) One downside is that error reporting when a cycle is found is less helpful, but I don't think that is a major concern. ## What changes are included in this PR? * Delete `dev/depcheck` * Change `ci/scripts/check_circular_dependencies.sh` to call `cargo publish --dry-run --workspace` * The "Circular Dependency Check" now runs on `ubuntu-slim` ## What is the testing strategy for this PR? CI / tooling only. Manual local testing: - Small test workspace with a dev-dependency cycle: `cargo test` passes, and `cargo package` / `cargo publish --workspace --dry-run` fail. - Same workspace without the cycle: packaging passes. - Same workspace with a dev-dependency that has a path but no version: packaging passes, because Cargo drops that dependency. - DataFusion main: the dry run passes in 15 seconds and skips the 11 `publish = false` crates. - DataFusion with a dev-dependency cycle added: `cargo check --tests` passes, and the dry run fails. - DataFusion with a normal-dependency cycle added: `cargo check` fails by itself. - DataFusion bumped to unpublished versions: the dry run passes. - Bumped versions plus the dev-dependency cycle: the dry run fails. ## Are there any user-facing changes? No. Co-authored-by: Kumar Ujjawal --- .github/dependabot.yml | 8 - .github/workflows/rust.yml | 14 +- Cargo.toml | 1 - ci/scripts/check_circular_dependencies.sh | 30 +- dev/depcheck/Cargo.lock | 4165 --------------------- dev/depcheck/Cargo.toml | 26 - dev/depcheck/README.md | 26 - dev/depcheck/rust-toolchain.toml | 23 - dev/depcheck/src/main.rs | 98 - docs/source/contributor-guide/testing.md | 11 +- 10 files changed, 35 insertions(+), 4367 deletions(-) delete mode 100644 dev/depcheck/Cargo.lock delete mode 100644 dev/depcheck/Cargo.toml delete mode 100644 dev/depcheck/README.md delete mode 100644 dev/depcheck/rust-toolchain.toml delete mode 100644 dev/depcheck/src/main.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cf222c4b32b7b..e2ab325701ed5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -62,14 +62,6 @@ updates: update-types: - "minor" - "patch" - # dev/depcheck is a standalone crate excluded from the workspace, so it is - # not covered by the workspace entry above - - package-ecosystem: cargo - directory: "/dev/depcheck" - schedule: - interval: weekly - target-branch: main - labels: [auto-dependencies] # the wasm test app has its own npm lockfile - package-ecosystem: npm directory: "/datafusion/wasmtest/datafusion-wasm-app" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7df585ddad6f9..ba77e4ddb7d3c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -980,21 +980,15 @@ jobs: working-directory: datafusion/proto run: cargo msrv --output-format json --log-target stdout verify - # Check for dependency cycles between DataFusion crates with `dev/depcheck`. + # Check for dependency cycles between DataFusion crates, which would block + # publishing them to crates.io, with a publishing dry run. depcheck: name: Circular Dependency Check if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }} - runs-on: ubuntu-latest - container: - image: amd64/rust + runs-on: ubuntu-slim steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: true - fetch-depth: 1 - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable + uses: ./.github/actions/setup-rustup - name: Check dependencies run: bash ci/scripts/check_circular_dependencies.sh diff --git a/Cargo.toml b/Cargo.toml index 763206a0fdeb8..14eaad0245976 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,6 @@ members = [ "datafusion/doc", "xtask", ] -exclude = ["dev/depcheck"] resolver = "2" [workspace.package] diff --git a/ci/scripts/check_circular_dependencies.sh b/ci/scripts/check_circular_dependencies.sh index 7cdc97b0d6ba7..cb891e6b40e4d 100755 --- a/ci/scripts/check_circular_dependencies.sh +++ b/ci/scripts/check_circular_dependencies.sh @@ -17,8 +17,26 @@ # specific language governing permissions and limitations # under the License. -# Checks for circular dependencies between DataFusion crates with -# `dev/depcheck`, the same way the "Circular Dependency Check" job does. +# Checks for circular dependencies between DataFusion crates, the same way the +# "Circular Dependency Check" job does. +# +# A cycle prevents publishing the crates to crates.io. A crate can be +# published only after every DataFusion crate it depends on, including through +# dev-dependencies, optional dependencies, and dependencies for other targets, +# so no crate in a cycle can be published first. Cargo refuses to build a +# cycle of normal dependencies, but it allows one that passes through a +# dev-dependency, because a dev-dependency is built only for the crate's own +# tests, examples, and benchmarks. Such a cycle builds and passes tests, and +# fails only at release time. +# +# The check is a publishing dry run. `cargo publish --workspace` packages every +# crate that does not set `publish = false`, in an order where each crate comes +# after the DataFusion crates it depends on, and fails if there is no such +# order. `--dry-run` stops before uploading, and `--no-verify` skips building +# the packages. The dry run also fails on any other problem that would stop the +# crates from being published, such as a dependency without a version. +# `--allow-dirty` lets the check run on a working tree with uncommitted +# changes. set -euo pipefail @@ -26,9 +44,7 @@ SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" -# depcheck is outside the root workspace and finds the root manifest from its -# own directory, so run Cargo from there. -cd "${ROOT_DIR}/dev/depcheck" +cd "${ROOT_DIR}" -echo "[${SCRIPT_NAME}] \`cargo run --locked\` in dev/depcheck" -cargo run --locked +echo "[${SCRIPT_NAME}] \`cargo publish --workspace --dry-run --no-verify --allow-dirty\`" +cargo publish --workspace --dry-run --no-verify --allow-dirty diff --git a/dev/depcheck/Cargo.lock b/dev/depcheck/Cargo.lock deleted file mode 100644 index 31d0ee9af7e55..0000000000000 --- a/dev/depcheck/Cargo.lock +++ /dev/null @@ -1,4165 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" -dependencies = [ - "libc", -] - -[[package]] -name = "annotate-snippets" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" -dependencies = [ - "anstyle", - "memchr", - "unicode-width", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-hyperlink" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a14cad0fa1d1fb02de486a95ea7fd8dd28567b82f65b4ca34bffa7da81823e4" -dependencies = [ - "libc", - "percent-encoding", -] - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-progress" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082ebe8b7607a68881fbcccc7b3a3e784aa58593ed4261df0679a803fdb80853" - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "arc-swap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" -dependencies = [ - "rustversion", -] - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" - -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - -[[package]] -name = "blake3" -version = "1.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.0", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "block2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" -dependencies = [ - "objc2", -] - -[[package]] -name = "bstr" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" -dependencies = [ - "memchr", - "regex-automata", - "serde_core", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cargo" -version = "0.100.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4d8037642eef9a2744e1bbc775522cb20867ecc7ea35373f1769609d7386e60" -dependencies = [ - "anstream", - "anstyle", - "anstyle-progress", - "anyhow", - "async-trait", - "base64", - "blake3", - "cargo-credential", - "cargo-credential-libsecret", - "cargo-credential-macos-keychain", - "cargo-credential-wincred", - "cargo-platform", - "cargo-util", - "cargo-util-schemas", - "cargo-util-terminal", - "clap", - "clap_complete", - "color-print", - "crates-io", - "curl", - "curl-sys", - "filetime", - "flate2", - "futures", - "futures-timer", - "git2", - "git2-curl", - "gix", - "glob", - "heck", - "hex", - "hmac 0.13.0", - "home", - "http", - "http-auth", - "ignore", - "im-rc", - "indexmap", - "itertools", - "jiff", - "jobserver", - "libc", - "libgit2-sys", - "memchr", - "opener", - "os_info", - "pasetors", - "pathdiff", - "portable-atomic", - "rand", - "regex", - "rusqlite", - "rustc-hash", - "rustc-stable-hash", - "rustfix", - "same-file", - "semver", - "serde", - "serde-untagged", - "serde_ignored", - "serde_json", - "sha1 0.11.0", - "shell-escape", - "tar", - "tempfile", - "thiserror", - "time", - "toml", - "toml_edit", - "toml_parser", - "toml_writer", - "tracing", - "tracing-chrome", - "tracing-subscriber", - "unicase", - "unicode-ident", - "unicode-width", - "url", - "walkdir", - "windows-sys", - "winnow", -] - -[[package]] -name = "cargo-credential" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de74202293b2bc090da20d0b475aa74821caf0e901f29ba97003353886a5b42" -dependencies = [ - "anyhow", - "libc", - "serde", - "serde_json", - "thiserror", - "time", - "windows-sys", -] - -[[package]] -name = "cargo-credential-libsecret" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a84968127ab001ecc938ea245ad6d93be6039eb7732ead0f16f9b83fc8d5334" -dependencies = [ - "anyhow", - "cargo-credential", - "libloading", -] - -[[package]] -name = "cargo-credential-macos-keychain" -version = "0.4.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf917177a4bb7f238e1b8b3084cae37a688910e3b7bd95403f43f7c3d5a93b0e" -dependencies = [ - "cargo-credential", - "security-framework", -] - -[[package]] -name = "cargo-credential-wincred" -version = "0.4.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9234ec1ab98e92524fbf7ceb5dd06e0fcf2a046d6f341670d32796e9c5a9a384" -dependencies = [ - "cargo-credential", - "windows-sys", -] - -[[package]] -name = "cargo-platform" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "cargo-util" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe819082beae53247a2c440bece59bc2d1d90736cf5bed6b87f5e5bb53f0d30" -dependencies = [ - "anyhow", - "core-foundation", - "filetime", - "hex", - "ignore", - "jobserver", - "libc", - "miow", - "portable-atomic", - "same-file", - "sha2 0.11.0", - "shell-escape", - "tempfile", - "tracing", - "walkdir", - "windows-sys", -] - -[[package]] -name = "cargo-util-schemas" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46262ca4393669799a8df4852e81672ddea46657e73294b1a846f0d5c71509ed" -dependencies = [ - "jiff", - "semver", - "serde", - "serde-untagged", - "serde-value", - "thiserror", - "toml", - "unicode-ident", - "url", -] - -[[package]] -name = "cargo-util-terminal" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82be174c17e22a0366a1a0929e0306e0d53f120d159416b00f35e9708e7b2940" -dependencies = [ - "annotate-snippets", - "anstream", - "anstyle", - "anstyle-hyperlink", - "anstyle-progress", - "anyhow", - "cargo-util", - "libc", - "serde", - "serde_json", - "supports-hyperlinks", - "supports-unicode", - "windows-sys", -] - -[[package]] -name = "cc" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - -[[package]] -name = "clap" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", - "terminal_size", -] - -[[package]] -name = "clap_complete" -version = "4.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" -dependencies = [ - "clap", - "clap_lex", - "is_executable", - "shlex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "clru" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - -[[package]] -name = "color-print" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa954171903797d5623e047d9ab69d91b493657917bdfb8c2c80ecaf9cdb6f4" -dependencies = [ - "color-print-proc-macro", -] - -[[package]] -name = "color-print-proc-macro" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692186b5ebe54007e45a59aea47ece9eb4108e141326c304cdc91699a7118a22" -dependencies = [ - "nom", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crates-io" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c047c636cac168acc89c2df97a9017f2751e96742faaa35941fabec8c9d057a9" -dependencies = [ - "http", - "percent-encoding", - "serde", - "serde_json", - "thiserror", - "url", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "ct-codecs" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fb0c6640b4507ebd99ff67677009e381ba5eee1d14df78de4a3d16eb123c39" - -[[package]] -name = "ctutils" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" -dependencies = [ - "cmov", -] - -[[package]] -name = "curl" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a45ee8994e5307cb4c60cfc1c20bf7263ffb771ddc135c9f768a14bcbc15b09" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "windows-sys", -] - -[[package]] -name = "curl-sys" -version = "0.4.90+curl-8.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "windows-sys", -] - -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror", -] - -[[package]] -name = "depcheck" -version = "0.0.0" -dependencies = [ - "cargo", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "serde_core", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "const-oid 0.9.6", - "crypto-common 0.1.6", - "subtle", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid 0.10.2", - "crypto-common 0.2.2", - "ctutils", -] - -[[package]] -name = "dispatch2" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" -dependencies = [ - "bitflags 2.13.1", - "objc2", -] - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] - -[[package]] -name = "ed25519-compact" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c24599140dc39d7a81e4476e7573d41bbc18e07c803900298e522a5fbcfbfb6" -dependencies = [ - "getrandom 0.4.3", -] - -[[package]] -name = "either" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest 0.10.7", - "ff", - "generic-array", - "group", - "hkdf", - "pem-rfc7468", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "erased-serde" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" -dependencies = [ - "serde", - "serde_core", - "typeid", -] - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "faster-hex" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" -dependencies = [ - "heapless", - "serde", -] - -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" - -[[package]] -name = "filetime" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" -dependencies = [ - "cfg-if", - "libc", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" - -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "miniz_oxide", - "zlib-rs", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-executor" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" - -[[package]] -name = "futures-macro" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "futures-sink" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-timer" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "rand_core 0.10.1", - "wasm-bindgen", -] - -[[package]] -name = "git2" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" -dependencies = [ - "bitflags 2.13.1", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", -] - -[[package]] -name = "git2-curl" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f1f6fe01327a2f7a1d6ee519c14651f79530aa4d64e870f4cf672efe697bea5" -dependencies = [ - "curl", - "git2", - "log", - "url", -] - -[[package]] -name = "gix" -version = "0.85.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8b2e38ebfc4484dfef8580ddcaf8abb7285e6f3eb6413ff6775d104ae96ca6" -dependencies = [ - "gix-actor", - "gix-attributes", - "gix-command", - "gix-commitgraph", - "gix-config", - "gix-credentials", - "gix-date", - "gix-diff", - "gix-dir", - "gix-discover", - "gix-error", - "gix-features", - "gix-filter", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-hashtable", - "gix-ignore", - "gix-index", - "gix-lock", - "gix-negotiate", - "gix-object", - "gix-odb", - "gix-pack", - "gix-path", - "gix-pathspec", - "gix-prompt", - "gix-protocol", - "gix-ref", - "gix-refspec", - "gix-revision", - "gix-revwalk", - "gix-sec", - "gix-shallow", - "gix-status", - "gix-submodule", - "gix-tempfile", - "gix-trace", - "gix-transport", - "gix-traverse", - "gix-url", - "gix-utils", - "gix-validate", - "gix-worktree", - "gix-worktree-stream", - "nonempty", - "prodash", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-actor" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f9308ad6fd35b2a865cbe4117ac61b2be59e4a9ef1621c7a9794f7c8e52c5b" -dependencies = [ - "bstr", - "gix-date", - "gix-error", -] - -[[package]] -name = "gix-attributes" -version = "0.33.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-quote", - "gix-trace", - "kstring", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-bitmap" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1d118d0f5d88b96e6f6e13b566475fef4797ead4a02c26fed36c1375066f7" -dependencies = [ - "gix-error", -] - -[[package]] -name = "gix-chunk" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a871e5cab12ba568845714473505deefffb3c04eb47f4708ce344cd459c1cc" -dependencies = [ - "gix-error", -] - -[[package]] -name = "gix-command" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4363accdf6ef7ba861871d2d521ab7418a04aaaed919fadb022af71d379b12" -dependencies = [ - "bstr", - "gix-path", - "gix-quote", - "gix-trace", - "shell-words", -] - -[[package]] -name = "gix-commitgraph" -version = "0.37.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" -dependencies = [ - "bstr", - "gix-chunk", - "gix-error", - "gix-hash", - "memmap2", - "nonempty", -] - -[[package]] -name = "gix-config" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29bf266c4cdaf759e535c24ad4ce655b987aeb6911075643403cc7cc5ade583" -dependencies = [ - "bstr", - "gix-config-value", - "gix-features", - "gix-glob", - "gix-path", - "gix-ref", - "gix-sec", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-config-value" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-path", - "libc", - "thiserror", -] - -[[package]] -name = "gix-credentials" -version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a0fcfcd73229d1a9c34e04edd05dbcb80364ec6f3788d24c546de2916671eb" -dependencies = [ - "bstr", - "gix-command", - "gix-config-value", - "gix-date", - "gix-path", - "gix-prompt", - "gix-sec", - "gix-trace", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-date" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e47b9e8cdc688296609b706428de570f88b1e0eed7156dde7b4a89d26fa4567" -dependencies = [ - "bstr", - "gix-error", - "itoa", - "jiff", -] - -[[package]] -name = "gix-diff" -version = "0.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c6d56c94edf92d78203a1cd416f770e35e10b6955ede6b9d7d0c22ff88a5f3" -dependencies = [ - "bstr", - "gix-attributes", - "gix-command", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-imara-diff", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-tempfile", - "gix-trace", - "gix-traverse", - "gix-worktree", - "thiserror", -] - -[[package]] -name = "gix-dir" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20098fba2b9c6e29361ccb4c0379d42dfb81fc7f86f7ab11f2dff4528c0bf01f" -dependencies = [ - "bstr", - "gix-discover", - "gix-fs", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-trace", - "gix-utils", - "gix-worktree", - "thiserror", -] - -[[package]] -name = "gix-discover" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d624d5b23b10c1d85337645227abe353ac95ab8ff66a7bdd5ce689b2db33a722" -dependencies = [ - "bstr", - "dunce", - "gix-fs", - "gix-path", - "gix-ref", - "gix-sec", - "thiserror", -] - -[[package]] -name = "gix-error" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9292309fd944e71b2a3c96d3c03a6feb8852db646febdde7cbb9f79cb5f329" -dependencies = [ - "bstr", -] - -[[package]] -name = "gix-features" -version = "0.48.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" -dependencies = [ - "bytes", - "crc32fast", - "crossbeam-channel", - "gix-path", - "gix-trace", - "gix-utils", - "libc", - "once_cell", - "parking_lot", - "prodash", - "thiserror", - "walkdir", - "zlib-rs", -] - -[[package]] -name = "gix-filter" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6644fb2ef97928c278675b239f366b457103d7e436f811d27331a8daf212759c" -dependencies = [ - "bstr", - "encoding_rs", - "gix-attributes", - "gix-command", - "gix-hash", - "gix-object", - "gix-packetline", - "gix-path", - "gix-quote", - "gix-trace", - "gix-utils", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-fs" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" -dependencies = [ - "bstr", - "fastrand", - "gix-features", - "gix-path", - "gix-utils", - "thiserror", -] - -[[package]] -name = "gix-glob" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-features", - "gix-path", -] - -[[package]] -name = "gix-hash" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" -dependencies = [ - "faster-hex", - "gix-features", - "sha1-checked", - "thiserror", -] - -[[package]] -name = "gix-hashtable" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" -dependencies = [ - "gix-hash", - "hashbrown 0.17.1", - "parking_lot", -] - -[[package]] -name = "gix-ignore" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-trace", - "unicode-bom", -] - -[[package]] -name = "gix-imara-diff" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a791e6620676a875f362f3156ed213e73ca099a09bf992c18812abe65cc37b1" -dependencies = [ - "bstr", - "hashbrown 0.17.1", -] - -[[package]] -name = "gix-index" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d45f82ec5a4d7542ea595e9ad16e03e26c8cb4f221e5bc9fcdcf469f63a681" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "filetime", - "fnv", - "gix-bitmap", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-traverse", - "gix-utils", - "gix-validate", - "hashbrown 0.17.1", - "itoa", - "libc", - "memmap2", - "rustix", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-lock" -version = "23.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" -dependencies = [ - "gix-tempfile", - "gix-utils", - "thiserror", -] - -[[package]] -name = "gix-negotiate" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d6081882a5f575ace9f53924a7c85f69bbd0f96071b982df12f258ab338cc9" -dependencies = [ - "bitflags 2.13.1", - "gix-commitgraph", - "gix-date", - "gix-hash", - "gix-object", - "gix-revwalk", -] - -[[package]] -name = "gix-object" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019b38afc3eac1e41f9fe09a327664b313ba4a120fa5f40e3678795d0e42783e" -dependencies = [ - "bstr", - "gix-actor", - "gix-date", - "gix-features", - "gix-hash", - "gix-hashtable", - "gix-utils", - "gix-validate", - "itoa", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-odb" -version = "0.82.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fadc59f6fa0f9dd445eceee61060a2b59ca557f48da9fc677f567db535b782a" -dependencies = [ - "arc-swap", - "gix-features", - "gix-fs", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-pack", - "gix-path", - "gix-quote", - "memmap2", - "parking_lot", - "tempfile", - "thiserror", -] - -[[package]] -name = "gix-pack" -version = "0.72.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3e7f1726cd2c0cd1cf1fc20be8a8e623f0b163f1f8d6fc836cfb9bc8cd758b" -dependencies = [ - "clru", - "gix-chunk", - "gix-error", - "gix-features", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-path", - "gix-tempfile", - "memmap2", - "parking_lot", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-packetline" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" -dependencies = [ - "bstr", - "faster-hex", - "gix-trace", - "thiserror", -] - -[[package]] -name = "gix-path" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751d6bd162106f8c1e7e9aaccb5bbdd605267e91a930a17a4560c46e33a9100c" -dependencies = [ - "bstr", - "gix-trace", - "gix-validate", - "thiserror", -] - -[[package]] -name = "gix-pathspec" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-attributes", - "gix-config-value", - "gix-glob", - "gix-path", - "thiserror", -] - -[[package]] -name = "gix-prompt" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" -dependencies = [ - "gix-command", - "gix-config-value", - "parking_lot", - "rustix", - "thiserror", -] - -[[package]] -name = "gix-protocol" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978468bae4ea2df20c72db3b20d0bdb548a0c1090b85a83643b553e6e0e041f2" -dependencies = [ - "bstr", - "gix-credentials", - "gix-date", - "gix-features", - "gix-hash", - "gix-lock", - "gix-negotiate", - "gix-object", - "gix-ref", - "gix-refspec", - "gix-revwalk", - "gix-shallow", - "gix-trace", - "gix-transport", - "gix-utils", - "maybe-async", - "nonempty", - "thiserror", -] - -[[package]] -name = "gix-quote" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" -dependencies = [ - "bstr", - "gix-error", - "gix-utils", -] - -[[package]] -name = "gix-ref" -version = "0.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbfbce1dfd7d7f8469ddef6d3518376aff664348f153cbe0fc3e58ef993d24e" -dependencies = [ - "gix-actor", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-path", - "gix-tempfile", - "gix-utils", - "gix-validate", - "memmap2", - "thiserror", -] - -[[package]] -name = "gix-refspec" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc36a4fb1a1540b59cf2da498783080743fa274b02a3f19ca444fc4015a9d4f" -dependencies = [ - "bstr", - "gix-error", - "gix-glob", - "gix-hash", - "gix-revision", - "gix-validate", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-revision" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "885075c3c21eb9c06e0be3b3728ba5932c04e1c1011dcee7c81801980e3e986f" -dependencies = [ - "bstr", - "gix-commitgraph", - "gix-date", - "gix-error", - "gix-hash", - "gix-object", - "gix-revwalk", - "nonempty", -] - -[[package]] -name = "gix-revwalk" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f11fe7ca2585193d3d70bbe0be175a2008d883a704cc7a55e454e113e689455" -dependencies = [ - "gix-commitgraph", - "gix-date", - "gix-error", - "gix-hash", - "gix-hashtable", - "gix-object", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-sec" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4fe6c152c1d50aea36f299825702cd37e303307832fec1d0fdd5844e47ce2f" -dependencies = [ - "bitflags 2.13.1", - "gix-path", - "libc", - "windows-sys", -] - -[[package]] -name = "gix-shallow" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" -dependencies = [ - "bstr", - "gix-hash", - "gix-lock", - "nonempty", - "thiserror", -] - -[[package]] -name = "gix-status" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec3293f75db1212f99217832cbc70c30faeb95cefc97c7f1a17fd3bcf13a72e" -dependencies = [ - "bstr", - "filetime", - "gix-diff", - "gix-dir", - "gix-features", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-worktree", - "portable-atomic", - "thiserror", -] - -[[package]] -name = "gix-submodule" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9f594f7cbda0b38ba6b633b3e9a7b7901acdc5d27bc186a16633800cd1ac8" -dependencies = [ - "bstr", - "gix-config", - "gix-path", - "gix-pathspec", - "gix-refspec", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-tempfile" -version = "23.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" -dependencies = [ - "dashmap", - "gix-fs", - "libc", - "parking_lot", - "tempfile", -] - -[[package]] -name = "gix-trace" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3eb81d9dc914335923e50d52829c551feefd6a72d176c4130c546b67a60814" - -[[package]] -name = "gix-transport" -version = "0.57.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" -dependencies = [ - "base64", - "bstr", - "curl", - "gix-command", - "gix-credentials", - "gix-features", - "gix-packetline", - "gix-quote", - "gix-sec", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-traverse" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5062cca8f2977565bbaf666ec31dbdb9bc9d9293beb65f9bec52e6c1121b62a1" -dependencies = [ - "bitflags 2.13.1", - "gix-commitgraph", - "gix-date", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-revwalk", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-url" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d68e70e96da0e5f9c871f1566349e0fd0e1a20bb483c7f54af1dd0b85b4b29" -dependencies = [ - "bstr", - "gix-path", - "percent-encoding", - "thiserror", -] - -[[package]] -name = "gix-utils" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1795bd2a970ca8b2185318c2abb97d955c71992f1cf28de73ad3b593a9f3ce8" -dependencies = [ - "bstr", - "fastrand", - "getrandom 0.4.3", - "unicode-normalization", -] - -[[package]] -name = "gix-validate" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a034e84d1e04e1b1f20f51f12491da230b6ac8b925d0c8e1b89bcd87a7c5ccc" -dependencies = [ - "bstr", -] - -[[package]] -name = "gix-worktree" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92399ed66f259592050c6ed9dc80105e095a2f8e87e6b83d98aa2e21d8e27036" -dependencies = [ - "bstr", - "gix-attributes", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "gix-validate", -] - -[[package]] -name = "gix-worktree-stream" -version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f3a878c89a05470ad98c644b0015777c530da24854dd29e41fe4f41176840f" -dependencies = [ - "gix-attributes", - "gix-error", - "gix-features", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-object", - "gix-path", - "gix-traverse", - "parking_lot", -] - -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - -[[package]] -name = "globset" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "hash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashlink" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" -dependencies = [ - "hashbrown 0.17.1", -] - -[[package]] -name = "heapless" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" -dependencies = [ - "hash32", - "stable_deref_trait", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "hmac" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" -dependencies = [ - "digest 0.11.3", -] - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-auth" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" -dependencies = [ - "memchr", -] - -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "ignore" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata", - "same-file", - "walkdir", - "winapi-util", -] - -[[package]] -name = "im-rc" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" -dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "is_executable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cb6a9f675da968c63b6208c641b9dca58fc0133ae53375736b1767b0cab8bd" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jiff" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" -dependencies = [ - "defmt", - "jiff-core", - "jiff-static", - "jiff-tzdb-platform", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "windows-link", -] - -[[package]] -name = "jiff-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" -dependencies = [ - "defmt", -] - -[[package]] -name = "jiff-static" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" -dependencies = [ - "jiff-core", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" -dependencies = [ - "cfg-if", - "wasm-bindgen", -] - -[[package]] -name = "kstring" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "libgit2-sys" -version = "0.18.7+1.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - -[[package]] -name = "libloading" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" -dependencies = [ - "cfg-if", - "windows-link", -] - -[[package]] -name = "libnghttp2-sys" -version = "0.1.13+1.68.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "492e00167f1418c15648144f42bbfc63099806ecee9bf8d09a6353d6b4856b3c" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libssh2-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c04141a07bb0c0bc461cb657808764de571702a59bc5c726c400ac9a7625e3ab" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "maybe-async" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "miow" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536bfad37a309d62069485248eeaba1e8d9853aaf951caaeaed0585a95346f08" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.1", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nonempty" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" - -[[package]] -name = "normpath" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9985ef7269fa99f3b12437bb698381da2428743ab90f20393f399fa14cab21a" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "objc2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" -dependencies = [ - "objc2-encode", -] - -[[package]] -name = "objc2-cloud-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-data" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags 2.13.1", - "dispatch2", - "objc2", -] - -[[package]] -name = "objc2-core-graphics" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" -dependencies = [ - "bitflags 2.13.1", - "dispatch2", - "objc2", - "objc2-core-foundation", - "objc2-io-surface", -] - -[[package]] -name = "objc2-core-image" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-location" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca347214e24bc973fc025fd0d36ebb179ff30536ed1f80252706db19ee452009" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-text" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", - "objc2-core-graphics", -] - -[[package]] -name = "objc2-encode" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" - -[[package]] -name = "objc2-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" -dependencies = [ - "bitflags 2.13.1", - "block2", - "libc", - "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-io-surface" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-quartz-core" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", - "objc2-foundation", -] - -[[package]] -name = "objc2-ui-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" -dependencies = [ - "bitflags 2.13.1", - "block2", - "objc2", - "objc2-cloud-kit", - "objc2-core-data", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-core-image", - "objc2-core-location", - "objc2-core-text", - "objc2-foundation", - "objc2-quartz-core", - "objc2-user-notifications", -] - -[[package]] -name = "objc2-user-notifications" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9df9128cbbfef73cda168416ccf7f837b62737d748333bfe9ab71c245d76613e" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "opener" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b03ff07a220d0d0ec9a1f0f238951b7967a5a2e96aefcd21a117b1083415e9" -dependencies = [ - "bstr", - "normpath", - "windows-sys", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "orion" -version = "0.17.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6758747fd1ce1efaf2bd43219ac4aa9e28263b236b2b6a1e486bcd06820707" -dependencies = [ - "fiat-crypto", - "subtle", -] - -[[package]] -name = "os_info" -version = "3.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf20a545b305cf1da722b236b5155c9bb35f1d5ceb28c048bd96ca842f41b5b" -dependencies = [ - "android_system_properties", - "log", - "nix", - "objc2", - "objc2-foundation", - "objc2-ui-kit", - "windows-sys", -] - -[[package]] -name = "p384" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "pasetors" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e838401fb2873bad417e6a03179014c748746f67311cb7317ab14fc0881fa9f0" -dependencies = [ - "ct-codecs", - "ed25519-compact", - "getrandom 0.4.3", - "orion", - "p384", - "rand_core 0.6.4", - "regex", - "serde", - "serde_derive", - "serde_json", - "sha2 0.10.9", - "subtle", - "time", - "zeroize", -] - -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "portable-atomic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prodash" -version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" -dependencies = [ - "parking_lot", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.13.1", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - -[[package]] -name = "rsqlite-vfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" -dependencies = [ - "hashbrown 0.16.1", - "thiserror", -] - -[[package]] -name = "rusqlite" -version = "0.40.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" -dependencies = [ - "bitflags 2.13.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc-stable-hash" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "781442f29170c5c93b7185ad559492601acdc71d5bb0706f5868094f45cfcd08" - -[[package]] -name = "rustfix" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c614151d9f1b1e1f23041dacbf5f8dfab85687455b9e553f19f3d70275d1fe1c" -dependencies = [ - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags 2.13.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde-untagged" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" -dependencies = [ - "erased-serde", - "serde", - "serde_core", - "typeid", -] - -[[package]] -name = "serde-value" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" -dependencies = [ - "ordered-float", - "serde", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_ignored" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_spanned" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" -dependencies = [ - "serde_core", -] - -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - -[[package]] -name = "sha1-checked" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" -dependencies = [ - "digest 0.10.7", - "sha1 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shell-escape" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" - -[[package]] -name = "shell-words" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlite-wasm-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" -dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "supports-hyperlinks" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" - -[[package]] -name = "supports-unicode" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tar" -version = "0.4.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" -dependencies = [ - "filetime", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys", -] - -[[package]] -name = "terminal_size" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" -dependencies = [ - "rustix", - "windows-sys", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - -[[package]] -name = "time-macros" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "toml" -version = "1.1.4+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" -dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime", - "toml_parser", - "toml_writer", - "winnow", -] - -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.25.13+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" -dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime", - "toml_parser", - "toml_writer", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.1.3+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" -dependencies = [ - "winnow", -] - -[[package]] -name = "toml_writer" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tracing-chrome" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" -dependencies = [ - "serde_json", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "typeid" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - -[[package]] -name = "unicode-bom" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasm-bindgen" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "winnow" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -dependencies = [ - "memchr", -] - -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/dev/depcheck/Cargo.toml b/dev/depcheck/Cargo.toml deleted file mode 100644 index d7db5ef570251..0000000000000 --- a/dev/depcheck/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Circular dependency checker for DataFusion -[package] -name = "depcheck" -edition = "2024" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -cargo = "0.100.0" diff --git a/dev/depcheck/README.md b/dev/depcheck/README.md deleted file mode 100644 index 4c1571051f222..0000000000000 --- a/dev/depcheck/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - -This directory contains a tool that ensures there are no circular dependencies -in the DataFusion codebase. - -Specifically, it checks that no create's tests depend on another crate which -depends on the first, which prevents publishing to crates.io, for example - -[issue 9272]: https://github.com/apache/datafusion/issues/9277: diff --git a/dev/depcheck/rust-toolchain.toml b/dev/depcheck/rust-toolchain.toml deleted file mode 100644 index d3483e9f75a0d..0000000000000 --- a/dev/depcheck/rust-toolchain.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file specifies the default version of Rust used -# to compile this workspace and run CI jobs. - -[toolchain] -channel = "1.99.0" -components = ["rustfmt", "clippy"] diff --git a/dev/depcheck/src/main.rs b/dev/depcheck/src/main.rs deleted file mode 100644 index de5eacb71d062..0000000000000 --- a/dev/depcheck/src/main.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -extern crate cargo; - -use cargo::CargoResult; -/// Check for circular dependencies between DataFusion crates -use std::collections::{HashMap, HashSet}; -use std::env; -use std::path::Path; - -use cargo::GlobalContext; - -/// Verifies that there are no circular dependencies between DataFusion crates -/// (which prevents publishing on crates.io) by parsing the Cargo.toml files and -/// checking the dependency graph. -/// -/// See https://github.com/apache/datafusion/issues/9278 for more details -fn main() -> CargoResult<()> { - let gctx = GlobalContext::default()?; - // This is the path for the depcheck binary - let path = env::var("CARGO_MANIFEST_DIR").unwrap(); - let root_cargo_toml = Path::new(&path) - // dev directory - .parent() - .expect("Can not find dev directory") - // project root directory - .parent() - .expect("Can not find project root directory") - .join("Cargo.toml"); - - println!( - "Checking for circular dependencies in {}", - root_cargo_toml.display() - ); - let workspace = cargo::workspace::Workspace::new(&root_cargo_toml, &gctx)?; - let (_, resolve) = cargo::ops::resolve_ws(&workspace, false)?; - - let mut package_deps = HashMap::new(); - for package_id in resolve - .iter() - .filter(|id| id.name().starts_with("datafusion")) - { - let deps: Vec = resolve - .deps(package_id) - .filter(|(package_id, _)| package_id.name().starts_with("datafusion")) - .map(|(package_id, _)| package_id.name().to_string()) - .collect(); - package_deps.insert(package_id.name().to_string(), deps); - } - - // check for circular dependencies - for (root_package, deps) in &package_deps { - let mut seen = HashSet::new(); - for dep in deps { - check_circular_deps(root_package, dep, &package_deps, &mut seen); - } - } - println!("No circular dependencies found"); - Ok(()) -} - -fn check_circular_deps( - root_package: &str, - current_dep: &str, - package_deps: &HashMap>, - seen: &mut HashSet, -) { - if root_package == current_dep { - panic!( - "circular dependency detected from {root_package} to self via one of {:?}", - seen - ); - } - if seen.contains(current_dep) { - return; - } - seen.insert(current_dep.to_string()); - if let Some(deps) = package_deps.get(current_dep) { - for dep in deps { - check_circular_deps(root_package, dep, package_deps, seen); - } - } -} diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index f4b5c1cbc0437..7650eb1977a53 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -332,8 +332,14 @@ Only committed files are checked. Commit a change before running it. CI runs two dependency checks, and `./dev/rust_lint.sh` runs both: -- `ci/scripts/check_circular_dependencies.sh` builds and runs [`dev/depcheck`], - which fails on dependency cycles between DataFusion crates. +- `ci/scripts/check_circular_dependencies.sh` fails on dependency cycles + between DataFusion crates, including one through a dev-dependency, which + Cargo allows when building but which prevents publishing the crates. It runs + `cargo publish --workspace --dry-run --no-verify`, which packages every crate + that does not set `publish = false` without building or uploading anything, + so it also fails on any other problem that would stop the crates from being + published to crates.io. It reads the crates.io index and accepts uncommitted + changes. - `ci/scripts/check_unused_dependencies.sh` runs `cargo machete --with-metadata` from the repository root. The lint suite installs [cargo-machete] with the version in `ci/scripts/utils/tool_versions.sh` if it is missing. @@ -345,7 +351,6 @@ To run either check on its own: ./ci/scripts/check_unused_dependencies.sh ``` -[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck [cargo-machete]: https://github.com/bnjbvr/cargo-machete ## Examples README Check From 290e26347b4031ccf1f7ba37e956ebee2ce1736c Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 22 Oct 2026 15:12:43 +0000 Subject: [PATCH 51/57] perf: add a GroupsAccumulator for array_agg(DISTINCT) (#26357) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #26347. ## Rationale for this change `array_agg(DISTINCT x)` had no `GroupsAccumulator`. Grouped queries fell back to `GroupsAccumulatorAdapter`, which keeps one `DistinctArrayAggAccumulator` per group, each with its own hash table and encoded rows. With many groups this dominates memory and CPU. ## What changes are included in this PR? New `DistinctArrayAggGroupsAccumulator`: * Each distinct value is encoded once with a `RowConverter` into a `Rows` buffer shared by all groups, so a value that appears in many groups is stored once. * Groups record `(group_index, value_index)` pairs, deduplicated in one hash set. * `evaluate` builds the `ListArray` with a counting sort by group. * It also covers `array_agg(DISTINCT x ORDER BY x)`, sorting each group by the encoded rows. * `EmitTo::First` drops values only referenced by emitted groups. Small refactors: the DISTINCT / ORDER BY check is shared by the row and groups paths (`distinct_sort_options`), and the one-row-per-list `convert_to_state` is shared with `ArrayAggGroupsAccumulator`. `state_compat` now compares list results as multisets for DISTINCT without ORDER BY, since the element order is not defined there. The row accumulator returns hash table order. ## Are these changes tested? * **Unit tests** cover dedup across batches, nulls with and without IGNORE NULLS, FILTER, ORDER BY DESC, `EmitTo::First`, state/merge, `convert_to_state`, dictionary types, and a differential check against `DistinctArrayAggAccumulator`. * **`state_compat`** passes with the new accumulator included. * **`array_agg.slt`** gains grouped cases mixing DISTINCT with `count(*)`, FILTER, ORDER BY and IGNORE NULLS. The existing aggregate slt files pass. ## Benchmarks Measured locally with a criterion benchmark (not included here, happy to add it): one 8192-row Utf8 batch, `update_batch` plus `evaluate`, the new accumulator against `GroupsAccumulatorAdapter` over `DistinctArrayAggAccumulator`. Apple M-series: | groups | input | groups accumulator | adapter (before) | speedup | | -- | -- | -- | -- | -- | | 100 | ~25 distinct | 154 µs | 251 µs | 1.6x | | 100 | ~7800 distinct | 384 µs | 969 µs | 2.5x | | 1000 | ~25 distinct | 173 µs | 1.42 ms | 8.2x | | 1000 | ~7800 distinct | 382 µs | 2.26 ms | 5.9x | | 8192 | ~25 distinct | 279 µs | 10.2 ms | 36x | | 8192 | ~7800 distinct | 414 µs | 10.4 ms | 25x | Live heap after `update_batch`, measured with a counting global allocator: 65 536 Utf8 rows in 8 batches. `size()` tracks these within ~10% for both. | groups | input | groups accumulator | adapter (before) | | -- | -- | -- | -- | | 100 | ~25 distinct | 11 KB | 720 KB | | 1000 | ~7800 distinct | 2.2 MB | 9.9 MB | | 8192 | ~25 distinct | 1.7 MB | 18.3 MB | | 65536 | ~7800 distinct | 2.2 MB | 100.7 MB | Memory now grows with the number of distinct values instead of the number of groups. ## Are there any user-facing changes? No API changes. `array_agg(DISTINCT x)` without ORDER BY may return elements in a different (still unspecified) order. --------- Co-authored-by: Andrew Lamb Co-authored-by: Claude Fable 5.1 --- .../sql/aggregates/nested_nullability.rs | 3 +- .../functions-aggregate/benches/array_agg.rs | 111 ++- .../functions-aggregate/src/array_agg.rs | 813 ++++++++++++++++-- .../functions-aggregate/tests/state_compat.rs | 44 +- .../src/aggregates/grouped_hash_stream.rs | 29 +- .../sqllogictest/test_files/array_agg.slt | 37 + 6 files changed, 959 insertions(+), 78 deletions(-) diff --git a/datafusion/core/tests/sql/aggregates/nested_nullability.rs b/datafusion/core/tests/sql/aggregates/nested_nullability.rs index 3677bf4281ed8..ceb98263f8975 100644 --- a/datafusion/core/tests/sql/aggregates/nested_nullability.rs +++ b/datafusion/core/tests/sql/aggregates/nested_nullability.rs @@ -235,7 +235,8 @@ async fn array_agg_struct_from_stricter_batches_with_spilling() -> Result<()> { #[tokio::test] async fn array_agg_distinct_struct_from_stricter_batches_with_spilling() -> Result<()> { AggregateBatchesTest::new() - .with_num_rows(10_000) + // Enough groups that even the compact native accumulator exceeds the limit + .with_num_rows(100_000) // One partition keeps the native aggregate's memory pressure deterministic. .with_target_partitions(1) .without_single_distinct_to_groupby() diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index 4fcb2b32f2ace..2d92222561dfe 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -25,10 +25,11 @@ use arrow::array::{ use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, Schema}; use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use datafusion_expr::function::AccumulatorArgs; -use datafusion_expr::{Accumulator, AggregateUDFImpl}; +use datafusion_expr::{Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator}; use datafusion_functions_aggregate::array_agg::{ ArrayAgg, ArrayAggAccumulator, DistinctArrayAggAccumulator, }; +use datafusion_functions_aggregate_common::aggregate::groups_accumulator::GroupsAccumulatorAdapter; use datafusion_physical_expr::{PhysicalSortExpr, expressions::col}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -323,6 +324,113 @@ fn distinct_array_agg_benchmark(c: &mut Criterion) { ); } +/// `ARRAY_AGG(DISTINCT x) ... GROUP BY g` over one 8 192-row Utf8 batch, with +/// the groups accumulator against the previous per-group path +/// (`GroupsAccumulatorAdapter` over one `DistinctArrayAggAccumulator` per group), +/// across group counts, distinct cardinalities and partial emission. +fn distinct_array_agg_groups_benchmark(c: &mut Criterion) { + let schema = Schema::new(vec![Field::new("value", DataType::Utf8, true)]); + let value_expr = col("value", &schema).unwrap(); + let value_field = value_expr.return_field(&schema).unwrap(); + let return_field: FieldRef = Field::new( + "array_agg", + DataType::List(Field::new_list_field(DataType::Utf8, true).into()), + true, + ) + .into(); + let args = AccumulatorArgs { + return_field, + schema: &schema, + expr_fields: std::slice::from_ref(&value_field), + ignore_nulls: false, + order_bys: &[], + is_reversed: false, + name: "array_agg(DISTINCT value)", + is_distinct: true, + exprs: std::slice::from_ref(&value_expr), + }; + let array_agg = ArrayAgg::default(); + + let inputs = [ + ( + "low cardinality (~25 distinct)", + Arc::new(create_string_array_low_cardinality(8192, 0.0)) as ArrayRef, + ), + ( + "high cardinality (~7800 distinct)", + Arc::new(create_string_array_high_cardinality(8192, 0.05, 0.0)) as ArrayRef, + ), + ]; + for num_groups in [100, 1000, 8192] { + let mut rng = StdRng::seed_from_u64(42); + let group_indices = (0..8192) + .map(|_| rng.random_range(0..num_groups)) + .collect::>(); + for (input_name, values) in &inputs { + // Emits every group at once, or in eight `EmitTo::First` chunks, which + // makes the accumulator rebuild its state after each partial emission. + for emit_chunks in [1, 8] { + let run = |acc: &mut dyn GroupsAccumulator| { + acc.update_batch( + std::slice::from_ref(values), + &group_indices, + None, + num_groups, + ) + .unwrap(); + if emit_chunks == 1 { + return vec![acc.evaluate(EmitTo::All).unwrap()]; + } + let chunk = num_groups.div_ceil(emit_chunks); + let mut remaining = num_groups; + let mut emitted = Vec::with_capacity(emit_chunks); + while remaining > 0 { + let n = chunk.min(remaining); + emitted.push(acc.evaluate(EmitTo::First(n)).unwrap()); + remaining -= n; + } + emitted + }; + let emit = if emit_chunks == 1 { + "emit all".to_string() + } else { + format!("emit in {emit_chunks} chunks") + }; + c.bench_function( + &format!( + "distinct_array_agg groups {num_groups} groups, {input_name}, {emit}" + ), + |b| { + b.iter(|| { + let mut acc = + array_agg.create_groups_accumulator(args.clone()).unwrap(); + run(acc.as_mut()) + }) + }, + ); + c.bench_function( + &format!( + "distinct_array_agg adapter {num_groups} groups, {input_name}, {emit}" + ), + |b| { + b.iter(|| { + let factory = || { + Ok(Box::new(DistinctArrayAggAccumulator::try_new( + &DataType::Utf8, + None, + false, + )?) as Box) + }; + let mut acc = GroupsAccumulatorAdapter::new(factory); + run(&mut acc) + }) + }, + ); + } + } + } +} + /// Precomputes the schema, physical expressions, sort expression, and aggregate /// metadata so each benchmark iteration can focus on accumulator operations. struct OrderedArrayAggBenchFixture { @@ -545,6 +653,7 @@ criterion_group!( benches, array_agg_benchmark, distinct_array_agg_benchmark, + distinct_array_agg_groups_benchmark, ordered_array_agg_benchmark, ordered_array_agg_wide_utf8_benchmark ); diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index f81a60c222e41..9c27b8b249f33 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -18,6 +18,7 @@ //! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`] use std::collections::VecDeque; +use std::hash::BuildHasher; use std::mem::{size_of, size_of_val, take}; use std::ops::Range; use std::sync::Arc; @@ -27,7 +28,7 @@ use arrow::array::{ UInt32Array, make_array, new_empty_array, }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; -use arrow::compute::{SortOptions, cast, filter}; +use arrow::compute::{SortOptions, cast, filter, take as take_array}; use arrow::datatypes::{DataType, Field, FieldRef, Fields}; use arrow::row::{OwnedRow, Row, RowConverter, Rows, SortField}; use arrow_select::dictionary::garbage_collect_any_dictionary; @@ -35,10 +36,12 @@ use arrow_select::dictionary::garbage_collect_any_dictionary; use datafusion_common::cast::as_list_array; use datafusion_common::hash_utils::{RandomState, create_hashes}; use datafusion_common::scalar::copy_array_data; +use datafusion_common::utils::memory::estimate_memory_size; use datafusion_common::utils::proxy::HashTableAllocExt; use datafusion_common::utils::{SingleRowListArrayBuilder, offset_span}; use datafusion_common::{ - Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err, + HashMap, HashSet, Result, ScalarValue, assert_eq_or_internal_err, exec_err, + internal_datafusion_err, internal_err, }; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; @@ -177,40 +180,9 @@ impl AggregateUDFImpl for ArrayAgg { let ignore_nulls = acc_args.ignore_nulls && field.is_nullable(); if acc_args.is_distinct { - // Limitation similar to Postgres. The aggregation function can only mix - // DISTINCT and ORDER BY if all the expressions in the ORDER BY appear - // also in the arguments of the function. This implies that if the - // aggregation function only accepts one argument, only one argument - // can be used in the ORDER BY, For example: - // - // ARRAY_AGG(DISTINCT col) - // - // can only be mixed with an ORDER BY if the order expression is "col". - // - // ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid - // ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid - // ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid - // ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid - let sort_option = match acc_args.order_bys { - // This accumulator sorts the values itself, so a reversed - // aggregate must still sort them in the requested order. - [single] if single.expr.eq(&acc_args.exprs[0]) => { - Some(if acc_args.is_reversed { - !single.options - } else { - single.options - }) - } - [] => None, - _ => { - return exec_err!( - "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list" - ); - } - }; return Ok(Box::new(DistinctArrayAggAccumulator::try_new( data_type, - sort_option, + distinct_sort_options(&acc_args)?, ignore_nulls, )?)); } @@ -243,7 +215,11 @@ impl AggregateUDFImpl for ArrayAgg { } fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool { - !args.is_distinct && args.order_bys.is_empty() + if args.is_distinct { + distinct_sort_options(&args).is_ok() + } else { + args.order_bys.is_empty() + } } fn create_groups_accumulator( @@ -253,6 +229,13 @@ impl AggregateUDFImpl for ArrayAgg { let field = &args.expr_fields[0]; let data_type = field.data_type().clone(); let ignore_nulls = args.ignore_nulls && field.is_nullable(); + if args.is_distinct { + return Ok(Box::new(DistinctArrayAggGroupsAccumulator::new( + data_type, + distinct_sort_options(&args)?, + ignore_nulls, + ))); + } Ok(Box::new(ArrayAggGroupsAccumulator::new( data_type, ignore_nulls, @@ -347,6 +330,42 @@ impl ArrayAggAccumulator { } } +/// Returns the sort options for `ARRAY_AGG(DISTINCT ...)`, or an error if the +/// ORDER BY cannot be combined with DISTINCT. +/// +/// Limitation similar to Postgres. The aggregation function can only mix +/// DISTINCT and ORDER BY if all the expressions in the ORDER BY appear +/// also in the arguments of the function. This implies that if the +/// aggregation function only accepts one argument, only one argument +/// can be used in the ORDER BY, For example: +/// +/// ARRAY_AGG(DISTINCT col) +/// +/// can only be mixed with an ORDER BY if the order expression is "col". +/// +/// ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid +/// ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid +/// ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid +/// ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid +/// +/// The DISTINCT accumulators sort the values themselves, so a reversed +/// aggregate must still sort them in the requested order. +fn distinct_sort_options(acc_args: &AccumulatorArgs) -> Result> { + match acc_args.order_bys { + [single] if single.expr.eq(&acc_args.exprs[0]) => { + Ok(Some(if acc_args.is_reversed { + !single.options + } else { + single.options + })) + } + [] => Ok(None), + _ => exec_err!( + "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list" + ), + } +} + /// Returns the mask of non-null rows in `values` when `ignore_nulls` is set /// and some row is null. fn non_null_mask(values: &dyn Array, ignore_nulls: bool) -> Option { @@ -803,27 +822,12 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { ) -> Result> { assert_eq!(values.len(), 1, "one argument to convert_to_state"); - let input = &values[0]; - - // Each row becomes a 1-element list: offsets are [0, 1, 2, ..., n]. - let offsets = OffsetBuffer::from_repeated_length(1, input.len()); - - // Filtered rows become null list entries, which merge_batch will skip. - let filter_nulls = opt_filter.map(filter_to_nulls); - - // With ignore_nulls, null values also become null list entries. Without - // ignore_nulls, null values stay as [NULL] so merge_batch retains them. - let nulls = if self.ignore_nulls { - let logical = input.logical_nulls(); - NullBuffer::union(filter_nulls.as_ref(), logical.as_ref()) - } else { - filter_nulls - }; - - let field = Arc::new(Field::new_list_field(self.datatype.clone(), true)); - let list_array = ListArray::new(field, offsets, Arc::clone(input), nulls); - - Ok(vec![Arc::new(list_array)]) + Ok(vec![single_value_lists( + &values[0], + opt_filter, + self.ignore_nulls, + &self.datatype, + )]) } fn size(&self) -> usize { self.batches @@ -840,6 +844,419 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { } } +/// Converts each input row into a one-element list, the state shape of +/// `ARRAY_AGG`. +/// +/// Filtered rows become null list entries, which `merge_batch` skips. With +/// `ignore_nulls`, null values also become null list entries; without it they +/// stay as `[NULL]` so `merge_batch` retains them. +fn single_value_lists( + input: &ArrayRef, + opt_filter: Option<&BooleanArray>, + ignore_nulls: bool, + datatype: &DataType, +) -> ArrayRef { + // Offsets are [0, 1, 2, ..., n]. + let offsets = OffsetBuffer::from_repeated_length(1, input.len()); + let filter_nulls = opt_filter.map(filter_to_nulls); + let nulls = if ignore_nulls { + let logical = input.logical_nulls(); + NullBuffer::union(filter_nulls.as_ref(), logical.as_ref()) + } else { + filter_nulls + }; + let field = Arc::new(Field::new_list_field(datatype.clone(), true)); + Arc::new(ListArray::new(field, offsets, Arc::clone(input), nulls)) +} + +/// [`GroupsAccumulator`] for `ARRAY_AGG(DISTINCT x)`, optionally with +/// `ORDER BY x`. +/// +/// Distinct values are encoded once with a [`RowConverter`] and stored in a +/// single [`Rows`] buffer shared by all groups, so a value that appears in many +/// groups is held once. Each group records the values it has seen as +/// `(group_index, value_index)` pairs, deduplicated in one hash set rather than +/// one set per group. +#[derive(Debug)] +struct DistinctArrayAggGroupsAccumulator { + datatype: DataType, + sort_options: Option, + ignore_nulls: bool, + /// Encodes values to rows. Created on the first non-empty input, once the + /// runtime type is known. + converter: Option, + /// Every distinct value seen so far, indexed by value index. + values: Option, + /// `(hash, value_index)` for every row in `values`. + value_map: HashTable<(u64, u32)>, + /// Heap size of `value_map` in bytes. + value_map_size: usize, + /// Distinct `(group_index, value_index)` pairs, in first-seen order. + pairs: Vec<(u32, u32)>, + /// Deduplicates `pairs`. + pair_set: HashSet<(u32, u32), RandomState>, + /// Total number of groups tracked. + num_groups: usize, + random_state: RandomState, + /// Time spent deduplicating values, reported as the DISTINCT submetric. + distinct_metric: Option>, +} + +impl DistinctArrayAggGroupsAccumulator { + fn new( + datatype: DataType, + sort_options: Option, + ignore_nulls: bool, + ) -> Self { + Self { + datatype, + sort_options, + ignore_nulls, + converter: None, + values: None, + value_map: HashTable::new(), + value_map_size: 0, + pairs: Vec::new(), + pair_set: HashSet::default(), + num_groups: 0, + random_state: RandomState::default(), + distinct_metric: None, + } + } + + fn converter(&mut self, data_type: &DataType) -> Result<&RowConverter> { + if self.converter.is_none() { + let sort_field = match self.sort_options { + Some(opts) => SortField::new_with_options(data_type.clone(), opts), + None => SortField::new(data_type.clone()), + }; + let converter = RowConverter::new(vec![sort_field])?; + self.values = Some(converter.empty_rows(0, 0)); + self.converter = Some(converter); + } + self.converter + .as_ref() + .ok_or_else(|| internal_datafusion_err!("row converter must be set")) + } + + /// Like [`Self::insert`], but first gathers the selected rows when they + /// are only part of `input`, so rejected values are never encoded. + fn insert_selected( + &mut self, + input: &ArrayRef, + rows: Vec<(usize, usize)>, + ) -> Result<()> { + if rows.len() == input.len() { + return self.insert(input, rows); + } + let indices = UInt32Array::from_iter_values( + rows.iter().map(|&(_, row_index)| row_index as u32), + ); + let selected = take_array(input.as_ref(), &indices, None)?; + self.insert( + &selected, + rows.into_iter() + .enumerate() + .map(|(position, (group_index, _))| (group_index, position)), + ) + } + + /// Records that `group_index` contains each value of `input` selected by + /// `rows`, given as `(group_index, row_in_input)` pairs. + fn insert( + &mut self, + input: &ArrayRef, + rows: impl IntoIterator, + ) -> Result<()> { + let encoded = self + .converter(input.data_type())? + .convert_columns(std::slice::from_ref(input))?; + let Self { + values, + value_map, + value_map_size, + pairs, + pair_set, + random_state, + .. + } = self; + let values = values + .as_mut() + .ok_or_else(|| internal_datafusion_err!("value rows must be set"))?; + + for (group_index, row_index) in rows { + let row = encoded.row(row_index); + let hash = random_state.hash_one(row.as_ref()); + let value_index = match value_map.find(hash, |&(h, index)| { + h == hash && values.row(index as usize) == row + }) { + Some(&(_, index)) => index, + None => { + let index = values.num_rows() as u32; + values.push(row); + value_map.insert_accounted( + (hash, index), + |&(h, _)| h, + value_map_size, + ); + index + } + }; + let pair = (group_index as u32, value_index); + if pair_set.insert(pair) { + pairs.push(pair); + } + } + Ok(()) + } + + /// Drops the state of the first `emit_groups` groups and renumbers the + /// rest so they start from 0. Values only referenced by emitted groups are + /// dropped too. + fn retain_after(&mut self, emit_groups: usize) { + let emit_groups = emit_groups as u32; + let old_pairs = take(&mut self.pairs); + let old_values = self.values.take(); + self.value_map = HashTable::new(); + self.value_map_size = 0; + self.pair_set = HashSet::default(); + self.num_groups -= emit_groups as usize; + + let (Some(converter), Some(old_values)) = (&self.converter, old_values) else { + return; + }; + let mut values = converter.empty_rows(0, 0); + let mut remap: HashMap = HashMap::new(); + let mut pairs = Vec::new(); + for (group_index, value_index) in old_pairs { + if group_index < emit_groups { + continue; + } + let new_index = *remap.entry(value_index).or_insert_with(|| { + let row = old_values.row(value_index as usize); + let index = values.num_rows() as u32; + values.push(row); + let hash = self.random_state.hash_one(row.as_ref()); + self.value_map.insert_accounted( + (hash, index), + |&(h, _)| h, + &mut self.value_map_size, + ); + index + }); + let pair = (group_index - emit_groups, new_index); + self.pair_set.insert(pair); + pairs.push(pair); + } + self.pairs = pairs; + self.values = Some(values); + } +} + +impl GroupsAccumulator for DistinctArrayAggGroupsAccumulator { + fn update_batch( + &mut self, + values: &[ArrayRef], + group_indices: &[usize], + opt_filter: Option<&BooleanArray>, + total_num_groups: usize, + ) -> Result<()> { + // With `ORDER BY x` the ordering column is passed too. It is the + // argument itself, so only the first column is needed. + let Some(input) = values.first() else { + return internal_err!("array_agg(DISTINCT) expects an argument"); + }; + self.num_groups = self.num_groups.max(total_num_groups); + + let nulls = if self.ignore_nulls { + input.logical_nulls() + } else { + None + }; + let rows = group_indices + .iter() + .enumerate() + .filter(|&(row_index, _)| { + let filtered_out = opt_filter.is_some_and(|filter| { + filter.is_null(row_index) || !filter.value(row_index) + }); + let skipped_null = + nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)); + !filtered_out && !skipped_null + }) + .map(|(row_index, &group_index)| (group_index, row_index)) + .collect::>(); + if rows.is_empty() { + return Ok(()); + } + let mut metric_recorder = + AggregateMetricRecorder::new(self.distinct_metric.clone()); + let _timer = metric_recorder.timer(); + self.insert_selected(input, rows) + } + + /// Produces a `ListArray` ordered by group index. Groups without values + /// are null, matching [`DistinctArrayAggAccumulator`]. + fn evaluate(&mut self, emit_to: EmitTo) -> Result { + let emit_groups = match emit_to { + EmitTo::All => self.num_groups, + EmitTo::First(n) => n, + }; + + // Counting sort of the pairs into group order. + let mut counts = vec![0u32; emit_groups]; + for &(group_index, _) in &self.pairs { + if let Some(count) = counts.get_mut(group_index as usize) { + *count += 1; + } + } + let mut offsets = Vec::::with_capacity(emit_groups + 1); + offsets.push(0); + let mut nulls = NullBufferBuilder::new(emit_groups); + let mut write_positions = Vec::with_capacity(emit_groups); + let mut total = 0u32; + for &count in &counts { + if count == 0 { + nulls.append_null(); + } else { + nulls.append_non_null(); + } + write_positions.push(total); + total += count; + offsets.push(total as i32); + } + let mut ordered = vec![0u32; total as usize]; + for &(group_index, value_index) in &self.pairs { + if let Some(position) = write_positions.get_mut(group_index as usize) { + ordered[*position as usize] = value_index; + *position += 1; + } + } + + let flat_values = match (&self.converter, &self.values) { + (Some(converter), Some(values)) if total > 0 => { + if self.sort_options.is_some() { + // The sort direction is encoded in the row bytes, so + // comparing rows gives the requested order. + for window in offsets.windows(2) { + ordered[window[0] as usize..window[1] as usize].sort_unstable_by( + |&a, &b| values.row(a as usize).cmp(&values.row(b as usize)), + ); + } + } + let decoded = converter + .convert_rows(ordered.iter().map(|&i| values.row(i as usize)))? + .swap_remove(0); + // `convert_rows` always returns the physical (non-dictionary) + // type, so cast back when the declared type has a dictionary. + if decoded.data_type() != &self.datatype + && datatype_contains_dictionary(&self.datatype) + { + cast(decoded.as_ref(), &self.datatype)? + } else { + decoded + } + } + _ => new_empty_array(&self.datatype), + }; + + match emit_to { + EmitTo::All => { + let converter = self.converter.take(); + let distinct_metric = self.distinct_metric.take(); + *self = Self::new( + self.datatype.clone(), + self.sort_options, + self.ignore_nulls, + ); + self.distinct_metric = distinct_metric; + // Keep the converter: the runtime type does not change. + if let Some(converter) = converter { + self.values = Some(converter.empty_rows(0, 0)); + self.converter = Some(converter); + } + } + EmitTo::First(_) => self.retain_after(emit_groups), + } + + let field = Arc::new(Field::new_list_field(self.datatype.clone(), true)); + Ok(Arc::new(ListArray::new( + field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + flat_values, + nulls.finish(), + ))) + } + + fn state(&mut self, emit_to: EmitTo) -> Result> { + Ok(vec![self.evaluate(emit_to)?]) + } + + fn merge_batch( + &mut self, + values: &[ArrayRef], + group_indices: &[usize], + total_num_groups: usize, + ) -> Result<()> { + assert_eq_or_internal_err!(values.len(), 1, "one argument to merge_batch"); + let lists = values[0].as_list::(); + self.num_groups = self.num_groups.max(total_num_groups); + + // A sliced list keeps its whole child array, so encode only the span + // its offsets cover. + let offsets = lists.offsets(); + let (span_start, span_len) = offset_span(offsets); + let child = lists.values().slice(span_start, span_len); + let rows = group_indices + .iter() + .enumerate() + .filter(|&(list_index, _)| lists.is_valid(list_index)) + .flat_map(|(list_index, &group_index)| { + let start = offsets[list_index] as usize - span_start; + let end = offsets[list_index + 1] as usize - span_start; + (start..end).map(move |position| (group_index, position)) + }) + .collect::>(); + if rows.is_empty() { + return Ok(()); + } + let mut metric_recorder = + AggregateMetricRecorder::new(self.distinct_metric.clone()); + let _timer = metric_recorder.timer(); + self.insert_selected(&child, rows) + } + + fn convert_to_state( + &self, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result> { + let Some(input) = values.first() else { + return internal_err!("array_agg(DISTINCT) expects an argument"); + }; + Ok(vec![single_value_lists( + input, + opt_filter, + self.ignore_nulls, + &self.datatype, + )]) + } + + fn set_metrics(&mut self, metrics: Arc) { + self.distinct_metric = Some(metrics.metric("distinct")); + } + + fn size(&self) -> usize { + size_of_val(self) + + self.converter.as_ref().map_or(0, RowConverter::size) + + self.values.as_ref().map_or(0, Rows::size) + + self.value_map_size + + self.pairs.capacity() * size_of::<(u32, u32)>() + + estimate_memory_size::<(u32, u32)>(self.pair_set.capacity(), 0) + .unwrap_or_default() + } +} + /// Resources that are allocated lazily on the first `update_batch` call, /// once the concrete runtime Arrow type is known. /// @@ -1791,8 +2208,8 @@ impl Accumulator for OrderSensitiveArrayAggAccumulator { #[cfg(test)] mod tests { use super::*; - use arrow::array::{Int32Builder, ListBuilder, StringBuilder}; - use arrow::datatypes::Schema; + use arrow::array::{DictionaryArray, Int32Builder, ListBuilder, StringBuilder}; + use arrow::datatypes::{Int32Type, Schema}; use datafusion_common::cast::as_generic_string_array; use datafusion_common::internal_err; use datafusion_physical_expr::PhysicalExpr; @@ -3417,13 +3834,291 @@ mod tests { } fn eval_i32_lists( - acc: &mut ArrayAggGroupsAccumulator, + acc: &mut dyn GroupsAccumulator, emit_to: EmitTo, ) -> Result>>>> { let result = acc.evaluate(emit_to)?; Ok(list_array_to_i32_vecs(result.as_list::())) } + fn distinct_groups_accumulator( + ignore_nulls: bool, + ) -> DistinctArrayAggGroupsAccumulator { + DistinctArrayAggGroupsAccumulator::new(DataType::Int32, None, ignore_nulls) + } + + #[test] + fn distinct_groups_accumulator_dedups_per_group() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 1, 2, 1, 2, 3])); + acc.update_batch(&[values], &[0, 0, 0, 1, 1, 1], None, 2)?; + let values: ArrayRef = Arc::new(Int32Array::from(vec![2, 3, 3])); + acc.update_batch(&[values], &[0, 1, 1], None, 2)?; + + // Values shared by both groups are stored once. + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(3)); + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(2)])); + assert_eq!(vals[1], Some(vec![Some(1), Some(2), Some(3)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_nulls() -> Result<()> { + let values: ArrayRef = + Arc::new(Int32Array::from(vec![None, Some(1), None, None])); + let groups = [0, 0, 0, 1]; + + // Without IGNORE NULLS a null is one more distinct value. + let mut acc = distinct_groups_accumulator(false); + acc.update_batch(&[Arc::clone(&values)], &groups, None, 3)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![None, Some(1)])); + assert_eq!(vals[1], Some(vec![None])); + // A group with no rows is null. + assert_eq!(vals[2], None); + + // With IGNORE NULLS a group of only nulls is null. + let mut acc = distinct_groups_accumulator(true); + acc.update_batch(&[values], &groups, None, 2)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_opt_filter() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 2, 3])); + let filter = BooleanArray::from(vec![Some(true), Some(false), Some(true), None]); + acc.update_batch(&[values], &[0, 0, 0, 1], Some(&filter), 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(2)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_sparse_filter() -> Result<()> { + // Only the selected rows are gathered and encoded, so their positions + // move; check each value still lands in its own group. + let mut acc = distinct_groups_accumulator(true); + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(5), + Some(6), + Some(7), + None, + Some(8), + ])); + let filter = BooleanArray::from(vec![false, true, false, true, true]); + acc.update_batch(&[values], &[0, 1, 0, 2, 2], Some(&filter), 3)?; + + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2)); + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals, vec![None, Some(vec![Some(6)]), Some(vec![Some(8)])]); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_merges_sliced_state() -> Result<()> { + let mut partial = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 4])); + partial.update_batch(&[values], &[0, 1, 2, 3, 3], None, 4)?; + let state = partial.state(EmitTo::All)?; + + // Partial aggregation emits slices of one state array. + let mut acc = distinct_groups_accumulator(false); + acc.merge_batch(&[state[0].slice(2, 2)], &[0, 1], 2)?; + acc.merge_batch(&[state[0].slice(0, 2)], &[1, 0], 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(3), Some(2)])); + assert_eq!(vals[1], Some(vec![Some(4), Some(1)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_records_metric() -> Result<()> { + let metric_updates = Arc::new(AtomicUsize::new(0)); + let mut acc = distinct_groups_accumulator(false); + acc.set_metrics(Arc::new(CountingMetrics(Arc::clone(&metric_updates)))); + + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); + acc.update_batch(&[Arc::clone(&values)], &[0, 1], None, 2)?; + let state = acc.state(EmitTo::All)?; + acc.merge_batch(&state, &[0, 1], 2)?; + assert_eq!(metric_updates.load(Ordering::Relaxed), 2); + + // Emitting everything resets the state but keeps the metric. + acc.evaluate(EmitTo::All)?; + acc.update_batch(&[values], &[0, 1], None, 2)?; + assert_eq!(metric_updates.load(Ordering::Relaxed), 3); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_order_by() -> Result<()> { + let descending = SortOptions { + descending: true, + nulls_first: false, + }; + let mut acc = DistinctArrayAggGroupsAccumulator::new( + DataType::Int32, + Some(descending), + false, + ); + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(2), + None, + Some(5), + Some(2), + Some(1), + ])); + acc.update_batch(&[values], &[0, 0, 0, 0, 1], None, 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(5), Some(2), None])); + assert_eq!(vals[1], Some(vec![Some(1)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_emit_first() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 10, 20, 30, 20])); + acc.update_batch(&[values], &[0, 0, 1, 2, 2], None, 3)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::First(2))?; + assert_eq!(vals, vec![Some(vec![Some(10)]), Some(vec![Some(20)])]); + + // 10 was only referenced by an emitted group, so it is dropped. + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2)); + + // The remaining group is renumbered to 0 and keeps deduplicating. + let values: ArrayRef = Arc::new(Int32Array::from(vec![20, 40])); + acc.update_batch(&[values], &[0, 0], None, 1)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals, vec![Some(vec![Some(30), Some(20), Some(40)])]); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_state_merge_roundtrip() -> Result<()> { + let mut acc1 = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); + acc1.update_batch(&[values], &[0, 1], None, 2)?; + + let mut acc2 = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 3, 2])); + acc2.update_batch(&[values], &[0, 0, 1], None, 2)?; + + let state = acc2.state(EmitTo::All)?; + acc1.merge_batch(&state, &[0, 1], 2)?; + + let vals = eval_i32_lists(&mut acc1, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(3)])); + assert_eq!(vals[1], Some(vec![Some(2)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_convert_to_state_merge() -> Result<()> { + let partial = distinct_groups_accumulator(true); + let values: ArrayRef = + Arc::new(Int32Array::from(vec![Some(7), None, Some(7), Some(8)])); + let filter = BooleanArray::from(vec![true, true, true, false]); + let state = partial.convert_to_state(&[values], Some(&filter))?; + + let mut acc = distinct_groups_accumulator(true); + acc.merge_batch(&state, &[0, 0, 0, 1], 2)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(7)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_keeps_dictionary_type() -> Result<()> { + let datatype = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let mut acc = + DistinctArrayAggGroupsAccumulator::new(datatype.clone(), None, false); + let values: ArrayRef = Arc::new( + vec!["a", "b", "a", "a"] + .into_iter() + .collect::>(), + ); + acc.update_batch(&[values], &[0, 0, 0, 1], None, 2)?; + + let result = acc.evaluate(EmitTo::All)?; + let lists = result.as_list::(); + assert_eq!(lists.values().data_type(), &datatype); + let flat = cast(lists.values().as_ref(), &DataType::Utf8)?; + let flat = flat.as_string::().iter().collect::>(); + assert_eq!(flat, vec![Some("a"), Some("b"), Some("a")]); + assert_eq!(lists.value_offsets(), &[0, 2, 3]); + Ok(()) + } + + /// Compares the groups accumulator with one [`DistinctArrayAggAccumulator`] + /// per group over batches with repeated values, nulls and a filter. + #[test] + fn distinct_groups_accumulator_matches_row_accumulator() -> Result<()> { + let num_groups = 7; + let mut acc = distinct_groups_accumulator(false); + let mut expected = (0..num_groups) + .map(|_| DistinctArrayAggAccumulator::try_new(&DataType::Int32, None, false)) + .collect::>>()?; + + for batch in 0..5 { + let len = 50; + let values = (0..len) + .map(|i| ((i * 7 + batch * 3) % 11 != 0).then_some((i * 13 + batch) % 9)) + .collect::>>(); + let groups = (0..len) + .map(|i| ((i * 5 + batch) % num_groups as i32) as usize) + .collect::>(); + let filter = (0..len).map(|i| (i + batch) % 4 != 0).collect::>(); + + let array: ArrayRef = Arc::new(Int32Array::from(values.clone())); + acc.update_batch( + &[array], + &groups, + Some(&BooleanArray::from(filter.clone())), + num_groups, + )?; + for ((value, group), keep) in values.into_iter().zip(groups).zip(filter) { + if keep { + let value: ArrayRef = Arc::new(Int32Array::from(vec![value])); + expected[group].update_batch(&[value])?; + } + } + } + + let sorted = |list: Option>>| { + list.map(|mut values| { + values.sort(); + values + }) + }; + let actual = eval_i32_lists(&mut acc, EmitTo::All)?; + for (group, acc) in expected.iter_mut().enumerate() { + let ScalarValue::List(list) = acc.evaluate()? else { + return internal_err!("expected a list"); + }; + let expected_values = list_array_to_i32_vecs(&list).swap_remove(0); + assert_eq!( + sorted(actual[group].clone()), + sorted(expected_values), + "group {group}" + ); + } + Ok(()) + } + #[test] fn groups_accumulator_multiple_batches() -> Result<()> { let mut acc = ArrayAggGroupsAccumulator::new(DataType::Int32, false); diff --git a/datafusion/functions-aggregate/tests/state_compat.rs b/datafusion/functions-aggregate/tests/state_compat.rs index e1915da9cfaa8..5e6f794d82b1e 100644 --- a/datafusion/functions-aggregate/tests/state_compat.rs +++ b/datafusion/functions-aggregate/tests/state_compat.rs @@ -27,8 +27,8 @@ use std::collections::BTreeSet; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, UInt32Array}; -use arrow::compute::{cast, concat, take}; +use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, ListArray, UInt32Array}; +use arrow::compute::{cast, concat, sort, take}; use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; use arrow::record_batch::{RecordBatch, RecordBatchOptions}; use datafusion_common::{DataFusionError, Result, ScalarValue}; @@ -518,7 +518,10 @@ fn take_all(arrays: &[ArrayRef], indices: &UInt32Array) -> Result> /// Runs every state exchange for one case. fn check_case(expr: &AggregateFunctionExpr, case: &Case) -> Errors { - let mut errors = Errors::default(); + let mut errors = Errors { + ignore_list_order: case.distinct && !case.ordered, + ..Default::default() + }; let input = match Input::new(expr, case) { Ok(input) => input, Err(e) => { @@ -693,7 +696,12 @@ fn guard(f: impl FnOnce() -> Result) -> Result { /// Failures for one case, deduplicated by message across groups. #[derive(Default)] -struct Errors(Vec<(String, Vec)>); +struct Errors { + failures: Vec<(String, Vec)>, + /// Compare list results as multisets. `array_agg(DISTINCT x)` without + /// ORDER BY does not define the order of its elements. + ignore_list_order: bool, +} impl Errors { fn push(&mut self, msg: &str) { @@ -708,14 +716,14 @@ impl Errors { // Only keep the first line, dropping e.g. the "please file a bug" text // of internal errors. let msg = msg.lines().next().unwrap_or_default().to_string(); - match self.0.iter_mut().find(|(m, _)| *m == msg) { + match self.failures.iter_mut().find(|(m, _)| *m == msg) { Some((_, groups)) => groups.extend(group), - None => self.0.push((msg, group.into_iter().collect())), + None => self.failures.push((msg, group.into_iter().collect())), } } fn lines(&self) -> Vec { - self.0 + self.failures .iter() .map(|(msg, groups)| { if groups.is_empty() { @@ -771,7 +779,7 @@ impl Errors { ) { match got { Err(e) => self.push_group(&format!("{what}: {e}"), group), - Ok(got) if !scalars_match(&got, want) => { + Ok(got) if !scalars_match(&got, want, self.ignore_list_order) => { self.push_group(&format!("{what}: got {got:?}, expected {want:?}"), group) } Ok(_) => {} @@ -780,8 +788,15 @@ impl Errors { } /// Exact equality, except that floats are compared with a relative tolerance -/// because merging in a different order can change rounding. -fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool { +/// because merging in a different order can change rounding, and lists are +/// compared as multisets when `ignore_list_order` is set. +fn scalars_match(a: &ScalarValue, b: &ScalarValue, ignore_list_order: bool) -> bool { + if ignore_list_order + && let (ScalarValue::List(x), ScalarValue::List(y)) = (a, b) + && let (Some(x), Some(y)) = (sorted_list_values(x), sorted_list_values(y)) + { + return x.as_ref() == y.as_ref(); + } match (a, b) { (ScalarValue::Float64(Some(x)), ScalarValue::Float64(Some(y))) => { floats_match(*x, *y) @@ -793,6 +808,15 @@ fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool { } } +/// The values of a single-row list, sorted, or `None` if the row is null or the +/// values cannot be sorted. +fn sorted_list_values(list: &ListArray) -> Option { + if list.len() != 1 || list.is_null(0) { + return None; + } + sort(list.value(0).as_ref(), None).ok() +} + fn floats_match(x: f64, y: f64) -> bool { if x.is_nan() || y.is_nan() { return x.is_nan() && y.is_nan(); diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs index 89131b38bce1a..1638ba545551f 100644 --- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs +++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs @@ -1500,7 +1500,10 @@ mod tests { use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_expr::{AggregateMetric, AggregateMetrics}; use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf}; - use datafusion_physical_expr::aggregate::AggregateExprBuilder; + use datafusion_physical_expr::GroupsAccumulatorAdapter; + use datafusion_physical_expr::aggregate::{ + AggregateExprBuilder, AggregateFunctionExpr, + }; use datafusion_physical_expr::expressions::col; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -1527,6 +1530,18 @@ mod tests { } } + /// `array_agg(DISTINCT)` has a native groups accumulator, so build the + /// per-group adapter explicitly to cover its metric handling. + fn distinct_adapter( + aggregate_expr: Arc, + metrics: Arc, + ) -> GroupsAccumulatorAdapter { + let mut adapter = + GroupsAccumulatorAdapter::new(move || aggregate_expr.create_accumulator()); + adapter.set_metrics(metrics); + adapter + } + #[test] fn legacy_grouped_distinct_installs_metrics_for_conversion_and_update() -> Result<()> { @@ -1544,13 +1559,13 @@ mod tests { ); let metric_resolutions = Arc::new(AtomicUsize::new(0)); let metric_durations = Arc::new(AtomicUsize::new(0)); - let mut accumulator = create_group_accumulator( - &aggregate_expr, + let mut accumulator = distinct_adapter( + aggregate_expr, Arc::new(CountingMetrics { resolutions: Arc::clone(&metric_resolutions), durations: Arc::clone(&metric_durations), }), - )?; + ); let values: ArrayRef = Arc::new(UInt32Array::from(vec![1, 2, 3])); accumulator.convert_to_state(&[Arc::clone(&values)], None)?; @@ -1579,13 +1594,13 @@ mod tests { ); let metric_resolutions = Arc::new(AtomicUsize::new(0)); let metric_durations = Arc::new(AtomicUsize::new(0)); - let mut accumulator = create_group_accumulator( - &aggregate_expr, + let mut accumulator = distinct_adapter( + aggregate_expr, Arc::new(CountingMetrics { resolutions: Arc::clone(&metric_resolutions), durations: Arc::clone(&metric_durations), }), - )?; + ); let mut state = ListBuilder::new(UInt32Builder::new()); for value in [1, 2, 3] { diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.slt index 8342380e31a43..89b7bcf3f50a5 100644 --- a/datafusion/sqllogictest/test_files/array_agg.slt +++ b/datafusion/sqllogictest/test_files/array_agg.slt @@ -678,3 +678,40 @@ List(Dictionary(Int32, Utf8)) statement ok drop table array_agg_distinct_dict_table; + +# array_agg(DISTINCT) per group, mixed with other aggregates and a FILTER +statement ok +create table array_agg_distinct_groups(g int, v varchar) as values + (1, 'b'), (1, 'a'), (1, 'b'), (1, null), + (2, 'a'), (2, 'a'), + (3, null), + (4, 'c'); + +query II?? +select + g, + count(*), + array_sort(array_agg(DISTINCT v) FILTER (WHERE v <> 'c')), + array_sort(array_agg(DISTINCT v)) +from array_agg_distinct_groups +group by g +order by g; +---- +1 4 [a, b] [NULL, a, b] +2 2 [a] [a] +3 1 NULL [NULL] +4 1 NULL [c] + +query I? +select g, array_agg(DISTINCT v ORDER BY v DESC) ignore nulls +from array_agg_distinct_groups +group by g +order by g; +---- +1 [b, a] +2 [a] +3 NULL +4 [c] + +statement ok +drop table array_agg_distinct_groups; From 3e6058ddde4900aac3dc4132e68658d4e9b79c52 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Thu, 22 Oct 2026 17:46:33 +0000 Subject: [PATCH 52/57] chore: skip slow test under `force_hash_collisions` (#26594) ## Which issue does this PR close? - Addresses an issue introduced in #25820 - Related to #25088, #25148 ## Rationale for this change #25820 added a test, `sort_after_group_by_reserves_shared_output_once`, which takes ~17 minutes when run under `force_hash_collisions`. This increased the runtime of the extended "hash collisions" job from 3-5 minutes to ~20 minutes, which slows down the merge queue and consumes CI resources. This PR disables the test under `force_collisions` mode. ## What changes are included in this PR? See above. ## What is the testing strategy for this PR? Test config change only. ## Are there any user-facing changes? No. --- datafusion/core/tests/memory_limit/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs index 0faa4a64cf983..858439f907d3a 100644 --- a/datafusion/core/tests/memory_limit/mod.rs +++ b/datafusion/core/tests/memory_limit/mod.rs @@ -453,6 +453,10 @@ async fn group_by_count_distinct_utf8_view() { /// the buffered slices are sorted: coalesced into runs for a single sort key, /// and one run per slice otherwise. The in-place threshold is below the slice /// size, so that coalescing keeps the slices as separate runs. +/// +/// Skipped under `force_hash_collisions`, where the 100,000 colliding group +/// keys make the aggregate quadratic and the test take many minutes. +#[cfg(not(feature = "force_hash_collisions"))] #[tokio::test] async fn sort_after_group_by_reserves_shared_output_once() { for order_by in ["group_key", "s desc, group_key"] { From a9fc673ccd17d3610f47f14978a74d2820a3bf9d Mon Sep 17 00:00:00 2001 From: Oleks V Date: Thu, 22 Oct 2026 19:24:07 +0000 Subject: [PATCH 53/57] feat: decorrelate a window below a correlated equality filter (#26013) ## Which issue does this PR close? - Closes #25897. ## Rationale for this change Since #25838, a correlated filter below a window function keeps the subquery correlated unless the filter reads only `PARTITION BY` columns. Such queries fail with "Physical plan does not support logical expression ...". For example (#25792): ```sql SELECT t1.id, sub.id, sub.rn FROM t1, LATERAL ( SELECT t2.id, row_number() OVER (ORDER BY t2.id) AS rn FROM (SELECT * FROM t2 WHERE t2.t1_id = t1.id) AS t2 ) AS sub; ``` ## What changes are included in this PR? When every correlated conjunct has the form `col = `, the filter keeps the rows of one `col` value for each outer row. `PullUpCorrelatedExpr` then adds `col` to the `PARTITION BY` of every window function and pulls the filter above the window, as Spark (`DecorrelateInnerQuery`) and DuckDB (`FlattenDependentJoins`) do. Each extended window expression is aliased to its old name, so the output schema does not change. The subquery stays correlated when: - the outer expression is volatile. - `col` has a float leaf. `-0.0 = 0.0` holds, but a partition keeps the two apart. - a conjunct casts `col` with a lossy cast. A lossy cast lets several `col` values match one outer row, which is #26014. Widening casts that keep distinct values distinct are accepted through `is_injective_cast`, which was a private Map key helper in `datafusion-common`. - an aggregate without `GROUP BY` sits below the window. It returns a row for an outer row without a match, which the pulled up filter would drop. A Projection over a count bug aggregate now records the expressions that read no column, such as `1 AS x`, in `collected_count_expr_map`. Before, such a Projection dropped the entry. A scalar subquery then returned NULL instead of `1` for an outer row without a match, and the window guard above did not see the aggregate. The core of the change is @adriangb's commit 1a2ad03ab, which #25838 reverted to stay a bug fix. ## What is the testing strategy for this PR? - `subquery.slt` and `lateral_join.slt`: the window cases from #25838 that expected an error now check results, plus an `EXPLAIN` that shows `PARTITION BY [b.y]`. - New cases that stay correlated: a lossy `CAST`, a float column with signed zeros, a volatile outer expression, and a window over an aggregate without `GROUP BY` behind a Projection, `DISTINCT` or a Filter, or in an `EXISTS`, `NOT EXISTS`, `IN` or `LATERAL` subquery. - New cases with results: a widening cast, a lower window that already outputs the extended name, a constant projected over a count bug aggregate, and a window over a grouped aggregate. ## Are there any user-facing changes? Queries with a correlated equality below a window function now run instead of failing with a not-implemented error. A scalar subquery that projects a constant over a correlated aggregate now returns the constant for outer rows without a match, instead of NULL. `datafusion_common::nested_struct::is_injective_cast` is now public. --------- Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- datafusion/common/src/nested_struct.rs | 22 +-- datafusion/optimizer/src/decorrelate.rs | 146 ++++++++++++++---- .../sqllogictest/test_files/lateral_join.slt | 19 ++- .../sqllogictest/test_files/subquery.slt | 144 ++++++++++++++--- 4 files changed, 260 insertions(+), 71 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index e79c21dfb7f08..f6f666840010b 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -903,7 +903,7 @@ fn validate_map_key_data_type( } Ok(()) } - _ if is_injective_map_key_cast(source_type, target_type) => Ok(()), + _ if is_injective_cast(source_type, target_type) => Ok(()), _ => _plan_err!( "Cannot safely evolve Map key type from {} to {}", source_type, @@ -912,15 +912,15 @@ fn validate_map_key_data_type( } } -/// Returns true when casting a Map key from `source_type` to `target_type` is -/// known to preserve key identity. +/// Returns true when casting from `source_type` to `target_type` is known to +/// preserve value identity, so distinct source values stay distinct. /// -/// Keep this list conservative. Map keys define equality/lookup identity, so -/// only representation widenings that cannot merge distinct source keys belong -/// here. Other Arrow-supported casts, including decimal/timestamp widening and -/// dictionaries, should stay rejected until their identity semantics are -/// deliberately reviewed. -fn is_injective_map_key_cast(source_type: &DataType, target_type: &DataType) -> bool { +/// Keep this list conservative. Map keys and decorrelated equality keys depend +/// on identity, so only representation widenings that cannot merge distinct +/// source values belong here. Other Arrow-supported casts, including +/// decimal/timestamp widening and dictionaries, should stay rejected until +/// their identity semantics are deliberately reviewed. +pub fn is_injective_cast(source_type: &DataType, target_type: &DataType) -> bool { matches!( (source_type, target_type), ( @@ -2283,14 +2283,14 @@ mod tests { (DataType::UInt16, DataType::Int64), (DataType::UInt32, DataType::Int64), ] { - assert!(is_injective_map_key_cast(&source_type, &target_type)); + assert!(is_injective_cast(&source_type, &target_type)); } for (source_type, target_type) in [ (DataType::UInt8, DataType::Int8), (DataType::UInt16, DataType::Int16), (DataType::UInt32, DataType::Int32), ] { - assert!(!is_injective_map_key_cast(&source_type, &target_type)); + assert!(!is_injective_cast(&source_type, &target_type)); } } diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 7dcf8fde935e5..e6f235da4dd42 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -23,11 +23,14 @@ use std::sync::Arc; use crate::push_down_filter::lr_is_preserved; use crate::simplify_expressions::ExprSimplifier; +use datafusion_common::nested_struct::is_injective_cast; use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, }; +use datafusion_common::utils::has_float_leaf; use datafusion_common::{ - Column, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, plan_err, + Column, DFSchema, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, + plan_err, }; use datafusion_expr::expr::{Alias, GroupingSet}; use datafusion_expr::logical_plan::{Join, JoinType}; @@ -38,7 +41,7 @@ use datafusion_expr::utils::{ use datafusion_expr::{ Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension, FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType, - UserDefinedLogicalNodeCore, expr, lit, + UserDefinedLogicalNodeCore, Window, expr, lit, }; /// This struct rewrite the sub query plan by pull up the correlated @@ -59,6 +62,10 @@ pub struct PullUpCorrelatedExpr { /// Indicates if we encounter any correlated expression that can not be pulled up /// above a aggregation without changing the meaning of the query. can_pull_over_aggregation: bool, + /// Whether every correlated conjunct seen so far passes + /// [`is_partitionable_equality`], so a Window above them can partition by their + /// columns. + can_extend_partition_by: bool, /// Do we need to handle [the count bug] during the pull up process. /// /// The "count bug" was described in [Optimization of Nested SQL @@ -117,6 +124,7 @@ impl PullUpCorrelatedExpr { exists_sub_query: false, can_pull_up: true, can_pull_over_aggregation: true, + can_extend_partition_by: true, need_handle_count_bug: false, collected_count_expr_map: HashMap::new(), pull_up_having_expr: None, @@ -321,7 +329,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { // outer row. A non-equality filter can return duplicates, see // https://github.com/apache/datafusion/issues/25808. // - A Window keeps or drops whole partitions when the filter reads - // only its `PARTITION BY` columns. `f_up` checks this. + // only its `PARTITION BY` columns. For other filters, `f_up` adds + // the columns of a column equality to `PARTITION BY` and keeps + // the subquery correlated in the remaining cases. // - An Unnest maps each input row to its own output rows, so a // filter on a column it passes through keeps the same rows. // The arm above checks that the filter does not read an unnested @@ -384,11 +394,11 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { match &plan { LogicalPlan::Filter(plan_filter) => { let subquery_filter_exprs = split_conjunction(&plan_filter.predicate); - self.can_pull_over_aggregation = self.can_pull_over_aggregation - && subquery_filter_exprs - .iter() - .filter(|e| e.contains_outer()) - .all(|&e| can_pullup_over_aggregation(e)); + for &expr in subquery_filter_exprs.iter().filter(|e| e.contains_outer()) { + self.can_pull_over_aggregation &= can_pullup_over_aggregation(expr); + self.can_extend_partition_by &= + is_partitionable_equality(expr, plan_filter.input.schema()); + } for expr in &subquery_filter_exprs { if expr.contains_outer() && !self.correlated_filters.contains(expr) { self.correlated_filters.push((*expr).clone()); @@ -713,6 +723,16 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { .iter() .all(|expr| partitions_by_all(expr, &local_correlated_cols)) { + // `inner.y = outer.k` keeps the rows of one `y` value for + // each outer row, so partitioning by `y` as well gives each + // window function the same rows. An aggregate without + // `GROUP BY` below returns a row for an outer row without + // a match, which the pulled up filter would drop. + if self.can_extend_partition_by && !self.pulled_up_scalar_agg { + let extended = + partition_window_by(window, &local_correlated_cols)?; + return self.pass_through_pulled_up_cols(extended); + } self.can_pull_up = false; } self.pass_through_pulled_up_cols(plan) @@ -960,6 +980,46 @@ fn correlated_filter_columns(plan: &LogicalPlan) -> Vec { cols } +/// Adds `cols` to the `PARTITION BY` of every window function of `window`. Each +/// window expression keeps its old output name, so nothing above sees a change. +fn partition_window_by(window: &Window, cols: &BTreeSet) -> Result { + let window_expr = window + .window_expr + .iter() + .map(|expr| { + add_partition_by(expr, cols).alias_if_changed(expr.schema_name().to_string()) + }) + .collect::>>()?; + Window::try_new(window_expr, Arc::clone(&window.input)).map(LogicalPlan::Window) +} + +fn add_partition_by(expr: &Expr, cols: &BTreeSet) -> Expr { + match expr { + Expr::Alias(alias) => Expr::Alias(Alias { + expr: Box::new(add_partition_by(&alias.expr, cols)), + ..alias.clone() + }), + Expr::WindowFunction(window_fun) => { + let mut window_fun = window_fun.clone(); + for col in cols { + let partitioned = window_fun + .params + .partition_by + .iter() + .any(|expr| matches!(expr, Expr::Column(c) if c == col)); + if !partitioned { + window_fun + .params + .partition_by + .push(Expr::Column(col.clone())); + } + } + Expr::WindowFunction(window_fun) + } + _ => expr.clone(), + } +} + /// Whether the window function `expr` lists each of `cols` as a plain /// `PARTITION BY` column. fn partitions_by_all(expr: &Expr, cols: &BTreeSet) -> bool { @@ -989,30 +1049,54 @@ fn collides_with_existing(exprs: &[Expr], candidate: &Expr) -> bool { } fn can_pullup_over_aggregation(expr: &Expr) -> bool { - if let Expr::BinaryExpr(BinaryExpr { + split_column_equality(expr).is_some() +} + +/// Whether partitioning by the columns of `expr` keeps the rows `expr` matches +/// for one outer row in one partition. That holds for `col = ` or +/// ` = col`, where: +/// - `` reads no subquery column and is not volatile. +/// - `col` has no float leaf, because `-0.0 = 0.0` holds but a partition +/// keeps the two apart. +/// - `col` is cast only by a cast that keeps distinct values distinct. A lossy +/// cast lets several `col` values match one outer row, see #26014. +fn is_partitionable_equality(expr: &Expr, schema: &DFSchema) -> bool { + let Some((col_side, other)) = split_column_equality(expr) else { + return false; + }; + let (col, cast_to) = match col_side { + Expr::Cast(Cast { expr, field }) => (&**expr, Some(field.data_type())), + _ => (col_side, None), + }; + let Ok(col_type) = col.get_type(schema) else { + return false; + }; + !other.is_volatile() + && !has_float_leaf(&col_type) + && cast_to.is_none_or(|to| is_injective_cast(&col_type, to)) +} + +/// Splits `col = ` or ` = col` into the subquery column side, a +/// column or a cast of one, and the other side, which reads no subquery column. +fn split_column_equality(expr: &Expr) -> Option<(&Expr, &Expr)> { + let Expr::BinaryExpr(BinaryExpr { left, op: Operator::Eq, right, }) = expr - { - match (&**left, &**right) { - (Expr::Column(_), right) => !right.any_column_refs(), - (left, Expr::Column(_)) => !left.any_column_refs(), - (Expr::Cast(Cast { expr, .. }), right) - if matches!(&**expr, Expr::Column(_)) => - { - !right.any_column_refs() - } - (left, Expr::Cast(Cast { expr, .. })) - if matches!(&**expr, Expr::Column(_)) => - { - !left.any_column_refs() - } - (_, _) => false, - } - } else { - false - } + else { + return None; + }; + [(&**left, &**right), (&**right, &**left)] + .into_iter() + .find(|(col_side, other)| { + let is_col = match col_side { + Expr::Column(_) => true, + Expr::Cast(Cast { expr, .. }) => matches!(&**expr, Expr::Column(_)), + _ => false, + }; + is_col && !other.any_column_refs() + }) } fn collect_local_correlated_cols( @@ -1121,7 +1205,11 @@ fn proj_exprs_evaluation_result_on_empty_batch( }) .data()?; - if result_expr.ne(expr) { + // An expression that reads no column, like `1 AS x`, keeps its value + // on an empty input, while the join that replaces the subquery gives + // it NULL for an outer row without a match. + let is_constant = !expr.any_column_refs() && !expr.contains_outer(); + if result_expr.ne(expr) || is_constant { let info = SimplifyContext::builder() .with_schema(Arc::clone(schema)) .build(); diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.slt index 0e0ceb9cfa4fa..40fb9d7d556aa 100644 --- a/datafusion/sqllogictest/test_files/lateral_join.slt +++ b/datafusion/sqllogictest/test_files/lateral_join.slt @@ -755,16 +755,19 @@ ORDER BY t1.id, sub.x; # --- Correlated filter below a window function --- # https://github.com/apache/datafusion/issues/25792 -# Pulling `t2.t1_id = t1.id` above the window would number the t2 rows of all -# t1 rows together instead of the t2 rows of each t1 row. -# Expected result once supported: (1, 10, 1), (1, 20, 2), (2, 30, 1). -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn +# The pull up adds `t2.t1_id` to the `PARTITION BY` of the window, so each t1 +# row still numbers only its own t2 rows. +query III SELECT t1.id, sub.id, sub.rn FROM t1, LATERAL ( SELECT t2.id, row_number() OVER (ORDER BY t2.id) AS rn FROM (SELECT * FROM t2 WHERE t2.t1_id = t1.id) AS t2 ) AS sub ORDER BY t1.id, sub.id; +---- +1 10 1 +1 20 2 +2 30 1 # A correlated filter above the window is still decorrelated. The window # numbers all t2 rows, then the filter keeps the rows of each t1 row. @@ -850,8 +853,8 @@ ORDER BY t1.id, sub.id; 1 20 2 2 30 1 -# The filter must read a `PARTITION BY` column of every window function. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn +# Every window function is partitioned by the columns of the filter. +query IIII SELECT t1.id, sub.id, sub.rn, sub.c FROM t1, LATERAL ( SELECT t2.id, @@ -860,6 +863,10 @@ FROM t1, LATERAL ( FROM t2 WHERE t2.t1_id = t1.id ) AS sub ORDER BY t1.id, sub.id; +---- +1 10 1 1 +1 20 2 1 +2 30 1 1 # --- Implicit LATERAL (comma without LATERAL keyword) --- # DuckDB auto-detects lateral correlation; DataFusion requires diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 3cd3bfbf193bb..a4474c4b0388d 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -3718,10 +3718,9 @@ DROP TABLE oj_a; statement ok DROP TABLE oj_b; -# Regression test for #25792: a correlated filter below a window function must -# not be pulled above the window. The filter selects the rows the window -# function sees for each outer row, so once it is above the window the window -# function runs over all rows and computes different values. +# Regression test for #25792: a correlated filter below a window function +# selects the rows that the window function sees for each outer row. For +# `y = k`, the pull up partitions the window by `y` as well. statement ok CREATE TABLE win_outer(k INT) AS VALUES (1), (2), (5); @@ -3731,39 +3730,49 @@ CREATE TABLE win_inner(id INT, y INT) AS VALUES (1, 1), (2, 2); statement ok set datafusion.explain.logical_plan_only = true; -# The correlated filter is below the window, so the subquery stays correlated. +# The correlated filter is below the window, so the window is partitioned by b.y. query TT EXPLAIN SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1); ---- logical_plan -01)Filter: EXISTS () -02)--Subquery: -03)----Projection: Int64(1) -04)------SubqueryAlias: w -05)--------Projection: b.id, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn -06)----------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1) -07)------------WindowAggr: windowExpr=[[row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] -08)--------------SubqueryAlias: b -09)----------------Projection: win_inner.id, win_inner.y -10)------------------Filter: win_inner.y = outer_ref(win_outer.k) -11)--------------------TableScan: win_inner -12)--TableScan: win_outer projection=[k] - -# It errors rather than returning wrong results. The right answer is `1, 2`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +01)LeftSemi Join: win_outer.k = __correlated_sq_1.y +02)--TableScan: win_outer projection=[k] +03)--SubqueryAlias: __correlated_sq_1 +04)----SubqueryAlias: w +05)------Projection: b.y +06)--------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1) +07)----------Projection: b.y, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +08)------------WindowAggr: windowExpr=[[row_number() PARTITION BY [b.y] ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] +09)--------------SubqueryAlias: b +10)----------------TableScan: win_inner projection=[id, y] + +# The right answer is `1, 2`. +query I SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k; +---- +1 +2 # The same for NOT EXISTS. The right answer is `5`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +query I SELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k; +---- +5 # The same for IN. The right answer is `1, 2`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery +query I SELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT row_number() OVER (ORDER BY win_inner.id) FROM win_inner WHERE win_inner.y = win_outer.k) ORDER BY win_outer.k; +---- +1 +2 # And for a scalar subquery. The right answer is `(1, 1), (2, 1), (5, NULL)`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +query II SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL # A correlated filter above the window is still pulled up, because the window # input does not depend on the outer row. @@ -3808,14 +3817,99 @@ SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION 2 1 5 NULL -# A window that partitions by an expression of the column does not count. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +# A window that partitions by an expression of the column is partitioned by +# the column as well. +query II SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y + 0 ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + +# A window below another window whose output has the name the extended window +# would get. The extended window keeps its old name, so the names stay unique. +query II +SELECT win_outer.k, (SELECT max(w.a + w.b) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS a, row_number() OVER (ORDER BY win_inner.id) AS b FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 2 +2 2 +5 NULL + +# A lossless widening cast, as coercion adds for `INT = BIGINT`, keeps the +# distinct values of `y` apart, so the window is partitioned by `y`. +query II +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = CAST(win_outer.k AS BIGINT)) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + +# `-0.0 = 0.0` holds but a window partition keeps the two apart, so a float +# `y` keeps the subquery correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 0.0), (2, -0.0)) AS v(id, f) WHERE v.f = CAST(win_outer.k - 1 AS DOUBLE)) AS w) AS m FROM win_outer ORDER BY win_outer.k; + +# A volatile outer expression can match several `y` values for one outer row, so +# the subquery stays correlated. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k + CAST(random() * 0 AS INT)) AS w) AS m FROM win_outer ORDER BY win_outer.k; + +# A lossy `CAST(f AS INT) = k` matches several `f` values for one outer row, so +# the subquery stays correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 1.2), (2, 1.4)) AS v(id, f) WHERE CAST(v.f AS INT) = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; # A window over an aggregate with count bug handling stays correlated. statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS m FROM win_outer ORDER BY win_outer.k; +# A projection that drops the aggregate column keeps the count bug handling, so +# the subquery stays correlated. The right answer is `(1, 1), (2, 1), (5, 1)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# Without the window, the constant keeps its value for an outer row without a match. +query II +SELECT win_outer.k, (SELECT b.x FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 1 + +# An aggregate without `GROUP BY` returns a row for an outer row without a match. +# A window above it keeps the subquery correlated, whatever node sits between +# them. The right answer for each scalar case is `(1, 1), (2, 1), (5, 1)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT DISTINCT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# A Filter that cannot be evaluated on an empty input. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a WHERE random() >= 0) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# The same for EXISTS, NOT EXISTS and IN, which do not track count bug handling. +# The right answers are `1, 2, 5`, no rows, and `1, 2, 5`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +SELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery +SELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +# And for LATERAL. +statement error DataFusion error: This feature is not implemented: Physical plan does not support undecorrelated Subquery +SELECT win_outer.k, s.n FROM win_outer, LATERAL (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS s ORDER BY win_outer.k; + +# An aggregate with `GROUP BY` returns no row for an outer row without a match, +# so the window above it is partitioned. +query II +SELECT win_outer.k, (SELECT max(w.n) FROM (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k GROUP BY win_inner.y) AS a) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + # LEFT JOIN LATERAL keeps the outer rows without a match. query III SELECT win_outer.k, s.id, s.rn FROM win_outer LEFT JOIN LATERAL (SELECT win_inner.id, row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS s ON true ORDER BY win_outer.k; From d090d2948824e3ee1c283f3fdd21e9af31b3e793 Mon Sep 17 00:00:00 2001 From: Geoffrey Claude Date: Thu, 22 Oct 2026 19:33:27 +0000 Subject: [PATCH 54/57] IN LIST: retain short integer, floating-point, and fixed-size binary lists (#25187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #19241. ## Rationale for this change DataFusion currently rewrites short `IN` lists like `x IN (1, 2)` as `x = 1 OR x = 2`. Evaluating `IN` directly can be faster. This PR keeps the `IN` form for the types below, so short lists can use DataFusion's existing optimized `IN` filters. ## What changes are included in this PR? - Keep two- and three-value `IN` and `NOT IN` lists for `Int32`, `Int64`, `UInt64`, `Float32`, `Float64`, and `FixedSizeBinary` columns with widths of 1, 2, 4, 8, or 16 bytes. The list must contain constant values of the column's type. - Add benchmarks that use normal optimization to compare performance before and after this change. - Add optimizer and SQL logic tests covering list sizes, types, NULLs, and non-constant list values. ## What is the testing strategy for this PR? All 935 optimizer unit tests and all 533 SQL logic files passed, including TPC-H. Formatting, Clippy, and lint checks passed. All 336 benchmark cases passed on both builds. The benchmark bot's ARM [two-value](https://github.com/apache/datafusion/pull/25187#issuecomment-6085097362) and [three-value](https://github.com/apache/datafusion/pull/25187#issuecomment-6085097845) results show: - Integer lists take about 32–55% less time when half the rows match or none do. In the extreme case where every row matches the first value, they can take up to about twice as long. - Floating-point and fixed-size binary lists are faster in every tested case, including NULLs and small batches.
Benchmark setup Both builds use the same 336 cases. The [baseline](https://github.com/apache/datafusion/commit/31a333da6952047803dcd93b7a80e59ef056ce8e) contains `main` at `06aa131cda0` plus the same benchmark code as the [PR](https://github.com/apache/datafusion/commit/8aea24b85b1e16e0c0e13773935b130e67160853). Queries use normal optimization. Timings cover running the chosen expressions; query planning and table setup happen before timing.
## Are there any user-facing changes? `EXPLAIN` shows `IN` or `NOT IN` for these short lists in place of the expanded `OR` or `AND` comparisons. This performance PR aligns constant two- and three-value `Float32`/`Float64` lists with existing behavior for 4+ values, including the signed-zero bug (`+0.0` versus `-0.0`). That bug [predates this optimization series](https://github.com/apache/datafusion/pull/7378) and already affects longer lists with constant or column values; fixing it is left to #25186. --------- Co-authored-by: Andrew Lamb --- datafusion/core/Cargo.toml | 4 + datafusion/core/benches/in_list_rewrite.rs | 417 ++++++++++++++++++ .../simplify_expressions/expr_simplifier.rs | 87 +++- .../simplify_expressions/inlist_simplifier.rs | 42 +- .../test_files/eliminate_outer_join.slt | 30 +- .../sqllogictest/test_files/in_list.slt | 220 ++++++++- datafusion/sqllogictest/test_files/pwmj.slt | 4 +- .../test_files/range_partitioning.slt | 4 +- .../test_files/string_numeric_coercion.slt | 4 +- datafusion/sqllogictest/test_files/union.slt | 8 +- datafusion/sqllogictest/test_files/window.slt | 4 +- 11 files changed, 784 insertions(+), 40 deletions(-) create mode 100644 datafusion/core/benches/in_list_rewrite.rs diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 7275c4b674d20..89f42761535e3 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -245,6 +245,10 @@ name = "math_query_sql" harness = false name = "filter_query_sql" +[[bench]] +harness = false +name = "in_list_rewrite" + [[bench]] harness = false name = "struct_query_sql" diff --git a/datafusion/core/benches/in_list_rewrite.rs b/datafusion/core/benches/in_list_rewrite.rs new file mode 100644 index 0000000000000..078c8f5e7db4b --- /dev/null +++ b/datafusion/core/benches/in_list_rewrite.rs @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks short SQL `IN` lists using the default optimizer. Comparing the +//! same cases across commits measures changes to the chosen evaluation strategy. +//! Criterion times expression evaluation, with planning and table setup outside +//! the measured loop. Input types and expected Boolean results are checked first. +//! +//! All types cover misses, balanced matches, and uniform or skewed +//! first-item hits. Floating-point and fixed-size-binary types also cover +//! nulls and small batches; a misaligned binary batch exercises alignment copies. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray}; +use arrow::buffer::{Buffer, MutableBuffer}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use datafusion::prelude::SessionContext; +use datafusion_common::ScalarValue; +use datafusion_expr::LogicalPlan; +use datafusion_physical_expr::PhysicalExpr; +use rand::prelude::*; +use tokio::runtime::Runtime; + +const MISS_VALUE_BASE: usize = 10_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + Int32, + Int64, + UInt64, + Float32, + Float64, + Utf8, + Utf8View, + FixedSizeBinary(i32), +} + +impl ValueKind { + const ALL: [Self; 12] = [ + Self::Int32, + Self::Int64, + Self::UInt64, + Self::Float32, + Self::Float64, + Self::Utf8, + Self::Utf8View, + Self::FixedSizeBinary(1), + Self::FixedSizeBinary(2), + Self::FixedSizeBinary(4), + Self::FixedSizeBinary(8), + Self::FixedSizeBinary(16), + ]; + + fn name(self) -> String { + match self { + Self::Int32 => "int32", + Self::Int64 => "int64", + Self::UInt64 => "uint64", + Self::Float32 => "float32", + Self::Float64 => "float64", + Self::Utf8 => "utf8", + Self::Utf8View => "utf8view_inline", + Self::FixedSizeBinary(width) => return format!("fixed_size_binary_{width}"), + } + .to_owned() + } + + fn seed_tag(self) -> u64 { + match self { + Self::Int32 => 1, + Self::Float64 => 2, + Self::Utf8 => 3, + Self::Utf8View => 4, + Self::FixedSizeBinary(8) => 5, + Self::Int64 => 6, + Self::Float32 => 7, + Self::UInt64 => 8, + Self::FixedSizeBinary(width) => 20 + width as u64, + } + } + + // The same typed values build batches and bind SQL parameters, avoiding + // implicit SQL literal widening for Float32 and UInt64. + fn scalar(self, value: Option) -> ScalarValue { + match self { + Self::Int32 => ScalarValue::Int32(value.map(|v| v as i32)), + Self::Int64 => ScalarValue::Int64(value.map(|v| v as i64)), + Self::UInt64 => ScalarValue::UInt64(value.map(|v| v as u64)), + Self::Float32 => ScalarValue::Float32(value.map(|v| v as f32)), + Self::Float64 => ScalarValue::Float64(value.map(|v| v as f64)), + Self::Utf8 => ScalarValue::Utf8(value.map(string_value)), + Self::Utf8View => ScalarValue::Utf8View(value.map(string_value)), + Self::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + width, + value.map(|v| { + // Narrow misses stay disjoint from the hit values 0, 1, and 2. + let v = if width == 1 && v >= MISS_VALUE_BASE { + 4 + (v - MISS_VALUE_BASE) % 252 + } else { + v + }; + (v as u128).to_be_bytes()[16 - width as usize..].to_vec() + }), + ), + } + } + + fn data_type(self) -> DataType { + self.scalar(Some(0)).data_type() + } + + fn make_array(self, values: &[Option]) -> ArrayRef { + let array = + ScalarValue::iter_to_array(values.iter().map(|&v| self.scalar(v))).unwrap(); + if let Self::FixedSizeBinary(width) = self { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + binary.values().as_ptr().align_offset(width as usize), + 0, + "builder-created binary buffer must be aligned" + ); + } + array + } +} + +// Eight-byte strings use Arrow's inline byte-view representation. +fn string_value(value: usize) -> String { + format!("v{value:07}") +} + +#[derive(Clone, Copy)] +enum Profile { + Miss, + Balanced, + AllFirstHit, + SkewedHit, + Nullable, + SmallBatch, + SingleMiss, + SingleHit, + ListNull, + Unaligned, +} + +impl Profile { + const ALL: [Self; 10] = [ + Self::Miss, + Self::Balanced, + Self::AllFirstHit, + Self::SkewedHit, + Self::Nullable, + Self::SmallBatch, + Self::SingleMiss, + Self::SingleHit, + Self::ListNull, + Self::Unaligned, + ]; + + fn name(self) -> &'static str { + match self { + Self::Miss => "miss", + Self::Balanced => "balanced", + Self::AllFirstHit => "all_first_hit", + Self::SkewedHit => "skewed_hit", + Self::Nullable => "balanced_nullable", + Self::SmallBatch => "balanced_small_batch", + Self::SingleMiss => "single_row_miss", + Self::SingleHit => "single_row_hit", + Self::ListNull => "list_null_balanced", + Self::Unaligned => "balanced_unaligned", + } + } + + fn batch_size(self) -> usize { + match self { + Self::SmallBatch => 64, + Self::SingleMiss | Self::SingleHit => 1, + _ => 8192, + } + } + + fn null_percent(self) -> usize { + if matches!(self, Self::Nullable) { + 20 + } else { + 0 + } + } + + fn match_percent(self) -> usize { + match self { + Self::Miss | Self::SingleMiss => 0, + Self::AllFirstHit | Self::SingleHit => 100, + Self::SkewedHit => 99, + _ => 50, + } + } + + fn permits_kind(self, kind: ValueKind) -> bool { + match self { + Self::Unaligned => kind == ValueKind::FixedSizeBinary(16), + Self::Miss | Self::Balanced | Self::AllFirstHit | Self::SkewedHit => true, + _ => matches!( + kind, + ValueKind::Float32 | ValueKind::Float64 | ValueKind::FixedSizeBinary(_) + ), + } + } +} + +// Project the predicate so optimization preserves its Boolean result, including +// NULLs, rather than simplifying it based on WHERE semantics. +fn plan_in_list( + ctx: &SessionContext, + runtime: &Runtime, + kind: ValueKind, + batch: &RecordBatch, + list_len: usize, + negated: bool, + list_has_null: bool, +) -> Arc { + ctx.deregister_table(kind.name()).unwrap(); + ctx.register_batch(kind.name(), batch.clone()).unwrap(); + let params = (0..list_len) + .map(|i| kind.scalar((!list_has_null || i + 1 < list_len).then_some(i))) + .collect::>(); + assert!( + params + .iter() + .all(|value| value.data_type() == kind.data_type()) + ); + let placeholders = (1..=list_len).map(|i| format!("${i}")).collect::>(); + let sql = format!( + "SELECT a {}IN ({}) AS matches FROM {}", + if negated { "NOT " } else { "" }, + placeholders.join(", "), + kind.name() + ); + let plan = runtime + .block_on(ctx.sql(&sql)) + .unwrap() + .with_param_values(params) + .unwrap() + .into_optimized_plan() + .unwrap(); + let LogicalPlan::Projection(projection) = plan else { + panic!("SQL must project the predicate"); + }; + assert_eq!(projection.expr.len(), 1); + assert_eq!( + projection.input.schema().field(0).data_type(), + &kind.data_type() + ); + let expr = ctx + .create_physical_expr( + projection.expr[0].clone().unalias(), + projection.input.schema(), + ) + .unwrap(); + let schema = batch.schema(); + assert_eq!(expr.data_type(schema.as_ref()).unwrap(), DataType::Boolean); + // Check types without requiring either IN or comparison-chain expressions. + let mut nodes = vec![Arc::clone(&expr)]; + while let Some(node) = nodes.pop() { + let data_type = node.data_type(schema.as_ref()).unwrap(); + assert!(data_type == DataType::Boolean || data_type == kind.data_type()); + nodes.extend(node.children().into_iter().cloned()); + } + expr +} + +fn make_batch( + kind: ValueKind, + profile: Profile, + profile_index: usize, + list_len: usize, + negated: bool, +) -> (RecordBatch, BooleanArray) { + let null_count = profile.batch_size() * profile.null_percent() / 100; + let non_null_count = profile.batch_size() - null_count; + let match_count = non_null_count * profile.match_percent() / 100; + let list_has_null = matches!(profile, Profile::ListNull); + let first_hits = matches!( + profile, + Profile::AllFirstHit | Profile::SkewedHit | Profile::SingleHit + ); + let mut values = Vec::with_capacity(profile.batch_size()); + values.extend(std::iter::repeat_n(None, null_count)); + values.extend((0..match_count).map(|i| { + Some(if first_hits { + 0 + } else { + i % (list_len - usize::from(list_has_null)) + }) + })); + values.extend((0..non_null_count - match_count).map(|i| Some(MISS_VALUE_BASE + i))); + let seed = 0x1A11_1575_5EED_u64 + ^ (kind.seed_tag() << 48) + ^ ((profile_index as u64) << 32) + ^ list_len as u64; + values.shuffle(&mut StdRng::seed_from_u64(seed)); + let expected = values + .iter() + .map(|value| { + value.and_then(|v| { + if v < list_len - usize::from(list_has_null) { + Some(!negated) + } else if list_has_null { + None + } else { + Some(negated) + } + }) + }) + .collect::(); + let mut array = kind.make_array(&values); + if matches!(profile, Profile::Unaligned) { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + let mut bytes = MutableBuffer::with_capacity(binary.values().len() + 1); + bytes.push(0_u8); + bytes.extend_from_slice(binary.values()); + let buffer = Buffer::from(bytes).slice(1); + assert_ne!(buffer.as_ptr().align_offset(16), 0); + array = Arc::new(FixedSizeBinaryArray::new( + 16, + buffer, + binary.nulls().cloned(), + )); + } + let schema = Arc::new(Schema::new(vec![Field::new("a", kind.data_type(), true)])); + (RecordBatch::try_new(schema, vec![array]).unwrap(), expected) +} + +fn criterion_benchmark(c: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + let ctx = SessionContext::new(); + for kind in ValueKind::ALL { + for (profile_index, profile) in Profile::ALL.into_iter().enumerate() { + if !profile.permits_kind(kind) { + continue; + } + let mut group = c.benchmark_group(format!( + "in_list_rewrite/{}/{}/batch={}/nulls={}%/match={}%", + kind.name(), + profile.name(), + profile.batch_size(), + profile.null_percent(), + profile.match_percent() + )); + group.throughput(Throughput::Elements(profile.batch_size() as u64)); + for list_len in [2, 3] { + for negated in [false, true] { + let (batch, expected) = + make_batch(kind, profile, profile_index, list_len, negated); + let expr = plan_in_list( + &ctx, + &runtime, + kind, + &batch, + list_len, + negated, + matches!(profile, Profile::ListNull), + ); + let output = expr + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(output.as_boolean(), &expected); + let case = format!( + "{}/list={list_len}", + if negated { "not_in" } else { "in" } + ); + group.bench_function(case, |b| { + b.iter(|| black_box(expr.evaluate(black_box(&batch)).unwrap())) + }); + } + } + group.finish(); + } + } +} + +criterion_group! { + name = benches; + config = Criterion::default() + .warm_up_time(Duration::from_millis(100)) + .measurement_time(Duration::from_millis(500)); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index ad23bb3c2577a..63929263d48c6 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -117,6 +117,12 @@ pub struct ExprSimplifier { max_simplifier_cycles: u32, } +/// Largest `IN` list considered for expansion into binary comparisons. +/// +/// The final decision also accounts for the measured performance of the column +/// representation and whether the list values are literals. +/// See [PR #25187](https://github.com/apache/datafusion/pull/25187) for the +/// benchmark evidence. pub const THRESHOLD_INLINE_INLIST: usize = 3; pub const DEFAULT_MAX_SIMPLIFIER_CYCLES: u32 = 3; @@ -199,7 +205,8 @@ impl ExprSimplifier { let mut simplifier = Simplifier::new(&self.info); let config_options = Some(Arc::clone(self.info.config_options())); let mut const_evaluator = ConstEvaluator::try_new(config_options)?; - let mut shorten_in_list_simplifier = ShortenInListSimplifier::new(); + let mut shorten_in_list_simplifier = + ShortenInListSimplifier::new(self.info.schema()); let guarantees_map: HashMap<&Expr, &NullableInterval> = self.guarantees.iter().map(|(k, v)| (k, v)).collect(); @@ -5006,6 +5013,8 @@ mod tests { }; let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?; let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?; + let retains_short_lists = + matches!(data_type, DataType::Float32 | DataType::Float64); let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)])); let batch = RecordBatch::try_new(schema, vec![values])?; let schema = batch.schema().to_dfschema_ref()?; @@ -5030,7 +5039,7 @@ mod tests { (left(true).or(right(true)), vec![false, false]), ] { let simplified = simplifier.simplify(expr.clone())?; - if list_len > THRESHOLD_INLINE_INLIST { + if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists { // Floating-point literals must bypass structural set rewrites. assert_eq!(simplified, expr); } else { @@ -5099,6 +5108,80 @@ mod tests { } } + #[test] + fn simplify_short_inlist_types() { + for (data_type, retain_short) in [ + (DataType::Int32, true), + (DataType::Int64, true), + (DataType::UInt64, true), + (DataType::Float32, true), + (DataType::Float64, true), + (DataType::FixedSizeBinary(1), true), + (DataType::FixedSizeBinary(2), true), + (DataType::FixedSizeBinary(4), true), + (DataType::FixedSizeBinary(8), true), + (DataType::FixedSizeBinary(16), true), + (DataType::Float16, false), + (DataType::FixedSizeBinary(3), false), + (DataType::Utf8View, false), + ( + DataType::Dictionary( + Box::new(DataType::Int8), + Box::new(DataType::Float64), + ), + false, + ), + ] { + let values = (1..=3) + .map(|value| { + lit(match &data_type { + DataType::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + *width, + Some(vec![value; *width as usize]), + ), + DataType::Utf8View => { + ScalarValue::Utf8View(Some(value.to_string())) + } + _ => ScalarValue::Int32(Some(i32::from(value))) + .cast_to(&data_type) + .unwrap(), + }) + }) + .collect::>(); + let schema = Schema::new(vec![Field::new("value", data_type.clone(), true)]) + .to_dfschema_ref() + .unwrap(); + let simplifier = ExprSimplifier::new( + SimplifyContext::builder().with_schema(schema).build(), + ); + for list_len in [2, 3] { + for negated in [false, true] { + let expr = + in_list(col("value"), values[..list_len].to_vec(), negated); + let simplified = simplifier.simplify(expr.clone()).unwrap(); + if retain_short { + assert_eq!(simplified, expr, "{data_type:?}"); + } else { + let op = if negated { Operator::And } else { Operator::Or }; + assert!( + matches!(simplified, Expr::BinaryExpr(binary) if binary.op == op), + "{data_type:?}, list_len={list_len}, negated={negated}" + ); + } + } + } + } + } + + #[test] + fn simplify_short_inlist_coerced_literals() { + for negated in [false, true] { + let expr = in_list(col("c3"), vec![lit(1_i32), lit(2_i32)], negated); + let expected = in_list(col("c3"), vec![lit(1_i64), lit(2_i64)], negated); + assert_eq!(simplify(coerce(expr)), expected); + } + } + #[test] fn simplify_inlist_set_operation_propagates_nullability_errors() { let info = SimplifyContext::builder() diff --git a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs index 17112d4f0ae24..087fb192c9a7e 100644 --- a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs @@ -19,24 +19,49 @@ use super::THRESHOLD_INLINE_INLIST; -use datafusion_common::Result; +use arrow::datatypes::DataType; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; -use datafusion_expr::Expr; +use datafusion_common::{DFSchema, Result}; use datafusion_expr::expr::InList; +use datafusion_expr::{Expr, ExprSchemable}; -pub(super) struct ShortenInListSimplifier {} +pub(super) struct ShortenInListSimplifier<'a> { + schema: &'a DFSchema, +} + +impl<'a> ShortenInListSimplifier<'a> { + pub(super) fn new(schema: &'a DFSchema) -> Self { + Self { schema } + } -impl ShortenInListSimplifier { - pub(super) fn new() -> Self { - Self {} + /// Benchmarks in `in_list_rewrite` favor evaluating short literal lists as + /// membership expressions over OR/AND chains for these types on batches with + /// a mix of matches and misses, or no matches. Integer chains can still be + /// faster when every row matches the first value, allowing the remaining + /// comparisons to short circuit. + /// + /// See [PR #25187](https://github.com/apache/datafusion/pull/25187) for measurements. + fn should_retain_short_inlist(&self, expr: &Expr, list: &[Expr]) -> bool { + let Ok(data_type) = expr.get_type(self.schema) else { + return false; + }; + matches!( + data_type, + DataType::Int32 + | DataType::Int64 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) + ) && list.iter().all(|expr| expr.as_literal().is_some()) } } -impl TreeNodeRewriter for ShortenInListSimplifier { +impl TreeNodeRewriter for ShortenInListSimplifier<'_> { type Node = Expr; fn f_up(&mut self, expr: Expr) -> Result> { - // if expr is a single column reference: + // Rewrite eligible short lists to left-deep comparison chains: // expr IN (A, B, ...) --> (expr = A) OR (expr = B) OR (expr = C) if let Expr::InList(InList { ref expr, @@ -52,6 +77,7 @@ impl TreeNodeRewriter for ShortenInListSimplifier { list.len() == 1 || list.len() <= THRESHOLD_INLINE_INLIST && expr.try_as_col().is_some() + && !self.should_retain_short_inlist(expr, list) ) { let first_val = list[0].clone(); diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt index a17ce11a0bff0..7e4d100b74237 100644 --- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt +++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt @@ -45,9 +45,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, 3); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] # Verify result correctness @@ -63,9 +63,9 @@ explain select * from t1 right join t2 on t1.a = t2.x where t1.a in (1, 2); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -80,9 +80,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t1.a in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -97,9 +97,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, null); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(NULL)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(NULL)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -168,9 +168,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t1.a in (1, 2, 3); ---- logical_plan 01)Left Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -185,7 +185,7 @@ query TT explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) or t2.x is null; ---- logical_plan -01)Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x IS NULL +01)Filter: t2.x IN ([Int32(1), Int32(2)]) OR t2.x IS NULL 02)--Left Join: t1.a = t2.x 03)----TableScan: t1 projection=[a, b, c] 04)----TableScan: t2 projection=[x, y, z] @@ -459,9 +459,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t2.x in (1, 2); ---- logical_plan 01)Right Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -512,9 +512,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: (t2.x = Int32(1) OR t2.x = Int32(2)) AND t2.y >= Int32(50) AND t2.y <= Int32(250) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) AND t2.y >= Int32(50) AND t2.y <= Int32(250) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort diff --git a/datafusion/sqllogictest/test_files/in_list.slt b/datafusion/sqllogictest/test_files/in_list.slt index 8fde8b11fe717..2a41eb3426105 100644 --- a/datafusion/sqllogictest/test_files/in_list.slt +++ b/datafusion/sqllogictest/test_files/in_list.slt @@ -20,9 +20,11 @@ # # This file focuses on the IN operator and its various specializations # -# Note that "short" IN LISTS do not go through the InList implementation at all, -# instead they are rewritten into a series of OR expressions. See: -# https://github.com/apache/datafusion/blob/ed37b6c9555bc278130dc774ed833b8c0bd29bfa/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs#L39-L88 +# One-item IN lists are rewritten to a comparison. Two- and three-item literal +# lists on Int32, Int64, UInt64, Float32, Float64, or FixedSizeBinary columns +# with widths 1, 2, 4, 8, or 16 retain their static InList filters. Other short +# lists on columns are rewritten to OR/AND expressions. See: +# https://github.com/apache/datafusion/blob/main/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs ########## @@ -211,6 +213,69 @@ CREATE TABLE in_list_ints_nullable ( ('no_match', 7, 7, 7, 7, 7, 7, 7, 7), ('nulls', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +# Measured integer representations retain two- and three-item literal lists. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match false true false true false true +nulls NULL NULL NULL NULL NULL NULL + +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(NULL, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(NULL, 'Int32'), arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(NULL, 'Int64'), arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(NULL, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(NULL, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(NULL, 'UInt64'), arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match NULL NULL NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list keeps the comparison path for these representations. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), i16), + i32 NOT IN (arrow_cast(3, 'Int32'), i16), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + u64 IN (arrow_cast(3, 'UInt64'), u32), + u64 NOT IN (arrow_cast(3, 'UInt64'), u32) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match true false true false true false +nulls NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')) AS i32_in_two, + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')) AS i64_not_in_three, + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) AS u64_in_three, + i64 IN (arrow_cast(1, 'Int64')) AS i64_in_one, + i64 IN (arrow_cast(3, 'Int64'), i32) AS i64_in_column, + i64 NOT IN (arrow_cast(3, 'Int64'), i32) AS i64_not_in_column +FROM in_list_ints_nullable; +---- +logical_plan +01)Projection: in_list_ints_nullable.i32 IN ([Int32(3), Int32(11)]) AS i32_in_two, in_list_ints_nullable.i64 NOT IN ([Int64(3), Int64(4), Int64(11)]) AS i64_not_in_three, in_list_ints_nullable.u64 IN ([UInt64(3), UInt64(4), UInt64(11)]) AS u64_in_three, in_list_ints_nullable.i64 = Int64(1) AS i64_in_one, in_list_ints_nullable.i64 = Int64(3) OR in_list_ints_nullable.i64 = CAST(in_list_ints_nullable.i32 AS Int64) AS i64_in_column, in_list_ints_nullable.i64 != Int64(3) AND in_list_ints_nullable.i64 != CAST(in_list_ints_nullable.i32 AS Int64) AS i64_not_in_column +02)--TableScan: in_list_ints_nullable projection=[i32, i64, u64] +physical_plan +01)ProjectionExec: expr=[i32@0 IN (SET) ([3, 11]) as i32_in_two, i64@1 NOT IN (SET) ([3, 4, 11]) as i64_not_in_three, u64@2 IN (SET) ([3, 4, 11]) as u64_in_three, i64@1 = 1 as i64_in_one, i64@1 = 3 OR i64@1 = CAST(i32@0 AS Int64) as i64_in_column, i64@1 != 3 AND i64@1 != CAST(i32@0 AS Int64) as i64_not_in_column] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Null input values return NULL when the IN list has no nulls. query TBBBBBBBB SELECT @@ -308,6 +373,79 @@ FROM (VALUES ('nulls', NULL) ) AS t(label, value); +# Short Float32 and Float64 lists retain membership filters for IN and NOT IN. +query TBBBBBBBB +SELECT + label, + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TBBBBBBBB +SELECT + label, + f64 IN (3.0, 11.0), + f64 NOT IN (3.0, 11.0), + f64 IN (3.0, 4.0, 11.0), + f64 NOT IN (3.0, 4.0, 11.0), + f64 IN (NULL, 11.0), + f64 NOT IN (NULL, 11.0), + f64 IN (NULL, 3.0, 11.0), + f64 NOT IN (NULL, 3.0, 11.0) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +# A list containing a column is evaluated through comparisons. +query TBBBB +SELECT label, + f32 IN (arrow_cast(3.0, 'Float32'), f16), + f32 NOT IN (arrow_cast(3.0, 'Float32'), f16), + f64 IN (3.0, f32), f64 NOT IN (3.0, f32) +FROM in_list_floats +ORDER BY label +---- +match true false true false +no_match true false true false +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS in_two, + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f32 IN ([Float32(3), Float32(11)]) AS in_two, in_list_floats.f32 NOT IN ([Float32(3), Float32(4), Float32(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f32] +physical_plan +01)ProjectionExec: expr=[f32@0 IN (SET) ([3, 11]) as in_two, f32@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +EXPLAIN SELECT f64 IN (3.0, 11.0) AS in_two, f64 NOT IN (3.0, 4.0, 11.0) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f64 IN ([Float64(3), Float64(11)]) AS in_two, in_list_floats.f64 NOT IN ([Float64(3), Float64(4), Float64(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f64] +physical_plan +01)ProjectionExec: expr=[f64@0 IN (SET) ([3, 11]) as in_two, f64@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Five element IN lists cover the specialized Float16/32/64 paths. query TTBBB SELECT @@ -616,6 +754,82 @@ LIMIT 1 ---- FixedSizeBinary(1) FixedSizeBinary(2) FixedSizeBinary(3) FixedSizeBinary(4) FixedSizeBinary(8) FixedSizeBinary(16) +# Short lists at the width endpoints cover both list lengths and typed nulls. +query TBBBBBB +SELECT + label, + fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'04', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 IN (arrow_cast(NULL, 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true true false false +no_match false true NULL false true NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list retains the comparison path. +query TBBBB +SELECT l.label, + l.fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16), + l.fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16) +FROM in_list_fsb l CROSS JOIN in_list_fsb r +WHERE r.label = 'match' +ORDER BY l.label +---- +match true false true false +no_match false true false true +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')) AS in_two, + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb1 IN ([FixedSizeBinary(1, "3"), FixedSizeBinary(1, "11")]) AS in_two, in_list_fsb.fsb16 NOT IN ([FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb1, fsb16] +physical_plan +01)ProjectionExec: expr=[fsb1@0 IN (SET) ([03, 0B]) as in_two, fsb16@1 NOT IN (SET) ([00000000000000000000..., 00000000000000000000..., 00000000000000000000...]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Short eight-byte lists retain membership filters, including list nulls. +query TBBBBBBBB +SELECT + label, + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS in_two, + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb8 IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS in_two, in_list_fsb.fsb8 NOT IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,4"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb8] +physical_plan +01)ProjectionExec: expr=[fsb8@0 IN (SET) ([0000000000000003, 000000000000000B]) as in_two, fsb8@0 NOT IN (SET) ([0000000000000003, 0000000000000004, 000000000000000B]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Eight element lists, all matching or none matching query TBBBBBB SELECT diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index 104b38e1a5036..619ebab6ba567 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -257,7 +257,7 @@ logical_plan 01)Sort: t1.t1_id ASC NULLS LAST, t2.t2_id ASC NULLS LAST 02)--Inner Join: Filter: t1.t1_id <= t2.t2_id 03)----SubqueryAlias: t1 -04)------Filter: join_t1.t1_id = Int32(11) OR join_t1.t1_id = Int32(44) +04)------Filter: join_t1.t1_id IN ([Int32(11), Int32(44)]) 05)--------TableScan: join_t1 projection=[t1_id] 06)----SubqueryAlias: t2 07)------Projection: join_t2.t2_id @@ -268,7 +268,7 @@ physical_plan 02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=LtEq, join_type=Inner, on=(t1_id <= t2_id) 04)------SortExec: expr=[t1_id@0 DESC NULLS LAST], preserve_partitioning=[false] -05)--------FilterExec: t1_id@0 = 11 OR t1_id@0 = 44 +05)--------FilterExec: t1_id@0 IN (SET) ([11, 44]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] 07)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 08)--------FilterExec: t2_name@1 != y, projection=[t2_id@0] diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 56184845a6fe3..21f9f439a2085 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -1928,7 +1928,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20, pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20]), pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] 03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TT @@ -1942,7 +1942,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- Plan with Metrics 01)HashJoinExec: mode=Partitionedmetrics=[output_rows=2,] -02)--DataSourceExec: file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20metrics=[output_rows=2,] +02)--DataSourceExec: file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20])metrics=[output_rows=2,] 03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] query III diff --git a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt index 196da416a2037..c09f2c986e7c0 100644 --- a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt +++ b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt @@ -229,10 +229,10 @@ query TT EXPLAIN SELECT * FROM t_int WHERE column1 IN ('5', '325'); ---- logical_plan -01)Filter: t_int.column1 = Int64(5) OR t_int.column1 = Int64(325) +01)Filter: t_int.column1 IN ([Int64(5), Int64(325)]) 02)--TableScan: t_int projection=[column1] physical_plan -01)FilterExec: column1@0 = 5 OR column1@0 = 325 +01)FilterExec: column1@0 IN (SET) ([5, 325]) 02)--DataSourceExec: partitions=1, partition_sizes=[1] # Error on invalid string in IN list diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index ca83e4b6527bf..2c706596b5409 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -365,14 +365,14 @@ EXPLAIN SELECT id, name FROM t1 WHERE id = 1 UNION SELECT id, name FROM t1 WHERE ---- logical_plan 01)Aggregate: groupBy=[[id, name]], aggr=[[]] -02)--Filter: t1.id = Int32(1) OR t1.id = Int32(2) +02)--Filter: t1.id IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[id, name] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Regression: schema recomputation must preserve the unqualified UNION @@ -415,14 +415,14 @@ SELECT x.id FROM t1 AS x WHERE x.id = 2 logical_plan 01)Aggregate: groupBy=[[id]], aggr=[[]] 02)--SubqueryAlias: x -03)----Filter: t1.id = Int32(1) OR t1.id = Int32(2) +03)----Filter: t1.id IN ([Int32(1), Int32(2)]) 04)------TableScan: t1 projection=[id] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Matching computed projections remain eligible for the rewrite and must diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index b7a6ad531bb84..864a6e56bf574 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -5673,7 +5673,7 @@ logical_plan 01)Sort: t1.c1 ASC NULLS LAST, t1.c2 ASC NULLS LAST, rank ASC NULLS LAST 02)--Projection: t1.c1, t1.c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rank 03)----WindowAggr: windowExpr=[[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] -04)------Filter: t1.c1 = Int32(2) OR t1.c1 = Int32(3) +04)------Filter: t1.c1 IN ([Int32(2), Int32(3)]) 05)--------TableScan: t1 projection=[c1, c2] physical_plan 01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank@2 ASC NULLS LAST] @@ -5681,7 +5681,7 @@ physical_plan 03)----BoundedWindowAggExec: wdw=[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 04)------SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST], preserve_partitioning=[true] 05)--------RepartitionExec: partitioning=Hash([c1@0], 2), input_partitions=2 -06)----------FilterExec: c1@0 = 2 OR c1@0 = 3 +06)----------FilterExec: c1@0 IN (SET) ([2, 3]) 07)------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 08)--------------DataSourceExec: partitions=1, partition_sizes=[1] From c0ec5127ac99be38e10c1d10d4a9cd4060b71dde Mon Sep 17 00:00:00 2001 From: Braedon Wooding Date: Thu, 22 Oct 2026 19:45:20 +0000 Subject: [PATCH 55/57] perf: don't build a parquet stream for a file range with no row group left (#26476) I've got a few fixes to upstream so just starting with small ones. ## Which issue does this PR close? - Closes #26475 ## Rationale for this change If row-group statistics or Bloom filters prune every row group in a file range, the opener still builds the decoder read plans, the decoder and the row filter, then hands back a stream morsel that yields nothing. `PruneFile` already ends early when file statistics prune the range; this does the same once row-group pruning has. ## What changes are included in this PR? `PruneWithStatistics` and `PruneWithBloomFilters` return `ParquetOpenState::Done` when no row group survives. ## What is the testing strategy for this PR? Testing to ensure that we do actually skip them! ## Are there any user-facing changes? Nope! --------- Co-authored-by: Andrew Lamb Co-authored-by: Claude Fable 5.1 --- .../datasource-parquet/src/opener/mod.rs | 74 ++++++++++++++++++- .../src/row_group_filter.rs | 25 +++++++ 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index b048a23eea896..3a1766869d319 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -660,6 +660,9 @@ impl ParquetOpenState { } ParquetOpenState::PruneWithStatistics(prepared) => { let mut prepared_row_groups = (*prepared).prune_row_groups()?; + if prepared_row_groups.row_groups.is_fully_pruned() { + return Ok(ParquetOpenState::Done); + } if prepared_row_groups.should_load_page_index()? { Ok(ParquetOpenState::LoadPageIndex( prepared_row_groups.load_page_index().boxed(), @@ -690,9 +693,13 @@ impl ParquetOpenState { ParquetOpenState::LoadBloomFilters(future) => { Ok(ParquetOpenState::LoadBloomFilters(future)) } - ParquetOpenState::PruneWithBloomFilters(loaded) => Ok( - ParquetOpenState::BuildStream(Box::new(loaded.prune_bloom_filters())), - ), + ParquetOpenState::PruneWithBloomFilters(loaded) => { + let pruned = loaded.prune_bloom_filters(); + if pruned.row_groups.is_fully_pruned() { + return Ok(ParquetOpenState::Done); + } + Ok(ParquetOpenState::BuildStream(Box::new(pruned))) + } ParquetOpenState::BuildStream(prepared) => { Ok(ParquetOpenState::Ready(prepared.build_stream()?)) } @@ -2690,6 +2697,26 @@ mod test { } } + /// Drives a [`ParquetMorselizer`] to completion and returns how many + /// morsels it produced. + async fn count_morsels( + morselizer: &dyn Morselizer, + file: PartitionedFile, + ) -> Result { + let mut planners = VecDeque::from([morselizer.plan_file(file)?]); + let mut morsels = 0; + while let Some(planner) = planners.pop_front() { + if let Some(mut plan) = planner.plan()? { + morsels += plan.take_morsels().len(); + planners.extend(plan.take_ready_planners()); + if let Some(pending_planner) = plan.take_pending_planner() { + planners.push_front(pending_planner.await?); + } + } + } + Ok(morsels) + } + fn constant_int_stats() -> (Statistics, SchemaRef) { let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Int32, false), @@ -3234,6 +3261,47 @@ mod test { assert_eq!(num_rows, 0); } + #[tokio::test] + async fn test_no_morsel_when_every_row_group_is_pruned() { + let store = Arc::new(InMemory::new()) as Arc; + let batches = vec![ + record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(), + record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(), + ]; + let schema = batches[0].schema(); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(3)) + .set_bloom_filter_enabled(true) + .set_statistics_enabled(EnabledStatistics::Chunk) + .build(); + let data_size = write_parquet_batches( + Arc::clone(&store), + "test.parquet", + batches, + Some(props), + ) + .await; + let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64); + + let morsels = |expr: Expr| { + let morselizer = ParquetMorselizerBuilder::new() + .with_store(Arc::clone(&store)) + .with_schema(Arc::clone(&schema)) + .with_predicate(logical2physical(&expr, &schema)) + .with_row_group_stats_pruning(true) + .with_enable_bloom_filter(true) + .build(); + let file = file.clone(); + async move { count_morsels(&morselizer, file).await.unwrap() } + }; + + // Statistics prune both row groups. + assert_eq!(morsels(col("a").gt(lit(10))).await, 0); + // Statistics keep both row groups; their Bloom filters prune both. + assert_eq!(morsels(col("a").eq(lit(1))).await, 0); + assert_eq!(morsels(col("a").eq(lit(2))).await, 1); + } + #[tokio::test] async fn test_prune_on_partition_statistics_with_dynamic_expression() { let store = Arc::new(InMemory::new()) as Arc; diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index 02fc2700d1228..feba768955e18 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -99,6 +99,12 @@ impl RowGroupAccessPlanFilter { self.access_plan.is_empty() } + /// Return true if no row groups remain to be scanned, either because the + /// file has none or because pruning has skipped every one of them. + pub fn is_fully_pruned(&self) -> bool { + self.access_plan.row_group_index_iter().next().is_none() + } + /// Return the number of row groups that are currently expected to be scanned pub fn remaining_row_group_count(&self) -> usize { self.access_plan.row_group_index_iter().count() @@ -706,6 +712,25 @@ mod tests { assert_eq!(filter.remaining_row_group_count(), 2); } + #[test] + fn is_fully_pruned_only_when_every_row_group_is_skipped() { + let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2)); + assert!(!filter.is_fully_pruned()); + + filter.access_plan.skip(0); + assert!(!filter.is_fully_pruned()); + + filter.access_plan.skip(1); + assert!(filter.is_fully_pruned()); + // Skipped row groups are still part of the plan + assert!(!filter.is_empty()); + + assert!( + RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(0)) + .is_fully_pruned() + ); + } + /// A row group with one column chunk of `size` bytes at the given offsets fn row_group_at( dictionary_page_offset: Option, From 5026ac452492048a3a9fcbe2e653ecfa921cc709 Mon Sep 17 00:00:00 2001 From: George Stamatakis <126914070+gstamatakis95@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:06:26 +0000 Subject: [PATCH 56/57] fix: declare Hive partitioned scans as Range (#25279) ## Which issue does this PR close? - Closes #23436. ## Rationale for this change - Hive grouped scans declare Partitioning::Hash, but no hash ever ran. - The optimizer treats any two Hash partitionings with equal counts as co-partitioned, so it skips the repartition. - Partitioned hash joins then drop rows. Fact {A, B, C} joined to dimension {A, C, D} returns only A, because fact group 1 holds B while dimension group 1 holds C. - Joining a Hive scan to a VALUES list also loses rows, since the planner hash repartitions the other side. ## What changes are included in this PR? - Add FileGroup::group_by_partition_values_with_split_points, returning groups plus the SplitPoints between them. - Merge values into contiguous chunks rather than round robin. Only contiguous chunks can be described by split points. Group counts are unchanged. - Give ListFilesResult a partition_split_points field. - In ListingTable::scan build a logical Range partitioning from the partition columns and the split points and convert it with create_physical_partitioning, the same path a user declared output partitioning takes. No new public helper. - Declare nothing when split_file_groups_by_statistics re-cuts the groups. Comparing counts misses this, since the re-cut can return the same number. - Drop the dynamic filter guard in HashJoinExec since it existed only because Hive groups reported Hash. ## What is the testing strategy for this PR? - TEST 14 and 15 in preserve_file_partitioning.slt cover the two failures above. - TEST 16 self joins the merged high cardinality table. TEST 17 asserts the statistics re-cut declares nothing. - Reverting only table.rs makes TEST 14 return one row instead of two and breaks TEST 12. - Six unit tests route every file through its split points and assert it lands in its own group, covering NULLs, compound keys, a NULL inside a split point, and unordered split points. - Two datafusion-proto round trips cover Dictionary(UInt16, Utf8) and Utf8View split points. ## Are there any user-facing changes? - EXPLAIN prints Range([col ASC], [(v1), (v2)], N) instead of Hash([col], N). - Joins between Hive tables with different values gain a RepartitionExec(Range) rather than returning wrong results. - Partitioned dynamic filter pushdown works again under preserve_file_partitions. - File group membership changes when distinct values exceed target_partitions. Byte balance can shift on date partitions with larger recent days. Set preserve_file_partitions = 0 to restore the old behaviour. - API break: ListFilesResult gains a public field. Add partition_split_points: vec![] or use ... - Providers declaring Hash over value grouped files should declare Range from the split points instead. They already return missing rows today when the other join input is hash repartitioned. --------- Co-authored-by: Andrew Lamb Co-authored-by: Claude Fable 5.1 --- datafusion/catalog-listing/src/table.rs | 53 ++-- datafusion/datasource/src/file_groups.rs | 228 +++++++++++++++--- .../datasource/src/file_scan_config/mod.rs | 121 ++++++++-- .../physical-plan/src/joins/hash_join/exec.rs | 10 +- datafusion/proto/tests/cases/plans/sources.rs | 81 +++++++ .../test_files/preserve_file_partitioning.slt | 223 +++++++++++++++-- .../repartition_subset_satisfaction.slt | 16 +- 7 files changed, 626 insertions(+), 106 deletions(-) diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index a72d2c2bc7d40..c94ddcef22d6b 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -25,13 +25,13 @@ use async_trait::async_trait; use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err, - project_schema, + Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err, + plan_err, project_schema, }; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_groups::FileGroup; use datafusion_datasource::file_scan_config::{ - FileScanConfig, FileScanConfigBuilder, output_partitioning_from_partition_fields, + FileScanConfig, FileScanConfigBuilder, output_range_partitioning_from_split_points, }; use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig}; #[expect(deprecated)] @@ -69,6 +69,8 @@ pub struct ListFilesResult { pub statistics: Statistics, /// Whether files are grouped by partition values. pub grouped_by_partition: bool, + /// Boundaries between file groups, empty unless `grouped_by_partition` is true. + pub partition_split_points: Vec, } /// Built in [`TableProvider`] that reads data from one or more files as a single table. @@ -668,6 +670,7 @@ impl ListingTable { file_groups: mut partitioned_file_lists, statistics, grouped_by_partition: partitioned_by_file_group, + partition_split_points, } = self .list_files_for_scan(state, &partition_filters, statistic_file_limit) .await?; @@ -687,6 +690,7 @@ impl ListingTable { .config_options() .execution .split_file_groups_by_statistics; + let mut regrouped_by_statistics = false; match split_file_groups_by_statistics .then(|| { output_ordering.first().map(|output_ordering| { @@ -704,6 +708,7 @@ impl ListingTable { Some(Ok(new_groups)) => { if new_groups.len() <= file_group_count { partitioned_file_lists = new_groups; + regrouped_by_statistics = true; } else { log::debug!( "attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered" @@ -737,6 +742,17 @@ impl ListingTable { )? } }; + Some(output_partitioning) + } else if partitioned_by_file_group && !regrouped_by_statistics { + output_range_partitioning_from_split_points( + &self.table_schema, + &table_partition_cols.into(), + partition_split_points, + )? + } else { + None + }; + if let Some(output_partitioning) = &output_partitioning { let partition_count = output_partitioning.partition_count(); if partitioned_file_lists.len() != partition_count { return plan_err!( @@ -744,19 +760,7 @@ impl ListingTable { partitioned_file_lists.len() ); } - Some(output_partitioning) - } else if partitioned_by_file_group { - // Files are grouped by partition column values: declare output - // partitioning on those columns so the optimizer can skip - // repartitioning for aggregates and joins on the partition columns. - output_partitioning_from_partition_fields( - &self.table_schema, - &table_partition_cols.clone().into(), - partitioned_file_lists.len(), - ) - } else { - None - }; + } let Some(object_store_url) = self.table_paths.first().map(ListingTableUrl::object_store) @@ -1005,6 +1009,7 @@ impl ListingTable { file_groups: vec![], statistics: Statistics::new_unknown(&self.file_schema), grouped_by_partition: false, + partition_split_points: vec![], }); }; let (file_group, inexact_stats) = self @@ -1018,21 +1023,23 @@ impl ListingTable { // skip repartitioning for aggregates and joins on partition columns. let threshold = ctx.config_options().optimizer.preserve_file_partitions; - let (file_groups, grouped_by_partition) = + let (file_groups, grouped_by_partition, partition_split_points) = if threshold > 0 && !self.options.table_partition_cols.is_empty() { - let grouped = file_group.group_by_partition_values(file_group_count); + let (grouped, split_points) = file_group + .group_by_partition_values_with_split_points(file_group_count); if grouped.len() >= threshold { - (grouped, true) + (grouped, true, split_points) } else { let all_files: Vec<_> = grouped.into_iter().flat_map(|g| g.into_inner()).collect(); ( FileGroup::new(all_files).split_files(file_group_count), false, + vec![], ) } } else { - (file_group.split_files(file_group_count), false) + (file_group.split_files(file_group_count), false, vec![]) }; self.list_files_result_from_groups( @@ -1040,6 +1047,7 @@ impl ListingTable { file_groups, inexact_stats, grouped_by_partition, + partition_split_points, ) } @@ -1067,6 +1075,7 @@ impl ListingTable { file_groups: vec![], statistics: Statistics::new_unknown(&self.file_schema), grouped_by_partition: false, + partition_split_points: vec![], }); }; let (file_group, inexact_stats) = @@ -1078,7 +1087,7 @@ impl ListingTable { let file_groups = self.filter_declared_file_groups_by_partition_filters(file_groups, filters)?; - self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false) + self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false, vec![]) } fn filter_declared_file_groups_by_partition_filters( @@ -1113,6 +1122,7 @@ impl ListingTable { file_groups: Vec, inexact_stats: bool, grouped_by_partition: bool, + partition_split_points: Vec, ) -> datafusion_common::Result { let (file_groups, stats) = compute_all_files_statistics( file_groups, @@ -1129,6 +1139,7 @@ impl ListingTable { file_groups, statistics: stats, grouped_by_partition, + partition_split_points, }) } diff --git a/datafusion/datasource/src/file_groups.rs b/datafusion/datasource/src/file_groups.rs index 84594be54b504..73cf4848465be 100644 --- a/datafusion/datasource/src/file_groups.rs +++ b/datafusion/datasource/src/file_groups.rs @@ -19,8 +19,8 @@ use crate::{FileRange, PartitionedFile}; use arrow::compute::SortOptions; -use datafusion_common::Statistics; use datafusion_common::utils::compare_rows; +use datafusion_common::{ScalarValue, SplitPoint, Statistics}; use itertools::Itertools; use std::cmp::{Ordering, min}; use std::collections::{BinaryHeap, HashMap}; @@ -488,19 +488,33 @@ impl FileGroup { /// /// Note: May return fewer groups than `max_target_partitions` when the /// number of unique partition values is less than the target. - #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key + /// + /// Wrapper around [`Self::group_by_partition_values_with_split_points`] that discards + /// the split points. pub fn group_by_partition_values( self, max_target_partitions: usize, ) -> Vec { + self.group_by_partition_values_with_split_points(max_target_partitions) + .0 + } + + /// Groups files by partition value and returns the [`SplitPoint`]s between groups. + /// + /// Values are sorted with [`SortOptions::default()`] and cut into at most + /// `max_target_partitions` contiguous chunks so routing a file's `partition_values` + /// through the split points yields the group it was placed in. + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] + pub fn group_by_partition_values_with_split_points( + self, + max_target_partitions: usize, + ) -> (Vec, Vec) { if self.is_empty() || max_target_partitions == 0 { - return vec![]; + return (vec![], vec![]); } - let mut partition_groups: HashMap< - Vec, - Vec, - > = HashMap::new(); + let mut partition_groups: HashMap, Vec> = + HashMap::new(); for file in self.files { partition_groups @@ -511,7 +525,9 @@ impl FileGroup { let num_unique_partitions = partition_groups.len(); - // Sort for deterministic bucket assignment across query executions. + // Sort for deterministic bucket assignment across query executions. Must match + // the ordering the caller declares, otherwise the split points would not + // describe the groups. let mut sorted_partitions: Vec<_> = partition_groups.into_iter().collect(); let sort_options = vec![ @@ -522,24 +538,31 @@ impl FileGroup { compare_rows(&a.0, &b.0, &sort_options).unwrap_or(Ordering::Equal) }); - if num_unique_partitions <= max_target_partitions { - sorted_partitions - .into_iter() - .map(|(_, files)| FileGroup::new(files)) - .collect() - } else { - // Merge into max_target_partitions buckets using round-robin. - // This maintains grouping by partition value as we are merging groups which already - // contain all values for a partition key. - let mut target_groups = vec![vec![]; max_target_partitions]; - - for (idx, (_, files)) in sorted_partitions.into_iter().enumerate() { - let bucket = idx % max_target_partitions; - target_groups[bucket].extend(files); + let bucket_count = min(num_unique_partitions, max_target_partitions); + let mut groups = Vec::with_capacity(bucket_count); + let mut split_points = Vec::with_capacity(bucket_count.saturating_sub(1)); + + // Every chunk is non-empty because `bucket_count <= num_unique_partitions`, so + // its first tuple is a well defined lower bound. + let mut iter = sorted_partitions.into_iter(); + for bucket in 0..bucket_count { + let start = bucket * num_unique_partitions / bucket_count; + let end = (bucket + 1) * num_unique_partitions / bucket_count; + let mut files = Vec::new(); + for offset in start..end { + let (values, group_files) = iter + .next() + .expect("contiguous chunking never runs past the sorted partitions"); + if offset == start && bucket > 0 { + split_points.push(SplitPoint::new(values)); + } + files.extend(group_files); } - - target_groups.into_iter().map(FileGroup::new).collect() + groups.push(FileGroup::new(files)); } + debug_assert_eq!(split_points.len() + 1, groups.len()); + + (groups, split_points) } } @@ -632,7 +655,9 @@ impl DerefMut for CompareByRangeSize { #[cfg(test)] mod test { use super::*; - use datafusion_common::ScalarValue; + use datafusion_physical_expr::RangePartitioning; + use datafusion_physical_expr::expressions::Column; + use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; /// Empty file won't get partitioned #[test] @@ -1276,8 +1301,8 @@ mod test { #[test] fn test_group_by_partition_values_more_groups_than_target() { - // Each file has a single partition value. The number of partition values > max_target_partitions, so - // they should be round-robin distributed into groups. + // More values than `max_target_partitions`, so they are cut into contiguous + // chunks whose sizes differ by at most one. let fg = FileGroup::new(vec![ pfile_with_pv("a", "p1"), pfile_with_pv("b", "p2"), @@ -1287,8 +1312,153 @@ mod test { ]); let groups = fg.group_by_partition_values(3); assert_eq!(groups.len(), 3); - assert_eq!(groups[0].len(), 2); + assert_eq!(groups[0].len(), 1); assert_eq!(groups[1].len(), 2); - assert_eq!(groups[2].len(), 1); + assert_eq!(groups[2].len(), 2); + // Groups must be contiguous key intervals in sorted order. + assert_eq!(group_pvs(&groups[0]), vec!["p1"]); + assert_eq!(group_pvs(&groups[1]), vec!["p2", "p3"]); + assert_eq!(group_pvs(&groups[2]), vec!["p4", "p5"]); + } + + fn group_pvs(group: &FileGroup) -> Vec { + group + .iter() + .map(|f| f.partition_values[0].to_string()) + .collect() + } + + /// Mirrors `RangeRouter` semantics: the partition index is the number of split points + /// that are `<=` the key under the given sort options. + fn range_route(key: &[ScalarValue], split_points: &[SplitPoint]) -> usize { + let sort_options = vec![SortOptions::default(); key.len()]; + split_points + .iter() + .take_while(|sp| { + compare_rows(sp.values(), key, &sort_options).unwrap() + != Ordering::Greater + }) + .count() + } + + #[test] + fn test_group_by_partition_values_split_points_describe_groups() { + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p2"), + pfile_with_pv("c", "p3"), + pfile_with_pv("d", "p4"), + pfile_with_pv("e", "p5"), + pfile_with_pv("f", "p5"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(3); + assert_eq!(groups.len(), 3); + assert_eq!(split_points.len(), 2); + assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]); + assert_eq!(split_points[1].values(), &[ScalarValue::from("p4")]); + + // Every file routes (RangeRouter semantics) to the group it was placed in. + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_points_fewer_values_than_target() { + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p1"), + pfile_with_pv("c", "p2"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(4); + assert_eq!(groups.len(), 2); + assert_eq!(split_points.len(), 1); + assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_points_single_target_partition() { + // One partition holds every value, so there is no boundary to describe. + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p2"), + pfile_with_pv("c", "p3"), + pfile_with_pv("d", "p3"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(1); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 4); + assert!(split_points.is_empty()); + } + + fn pfile_with_pvs(path: &str, pvs: &[Option<&str>]) -> PartitionedFile { + let mut file = pfile(path, 10); + file.partition_values = pvs + .iter() + .map(|v| ScalarValue::Utf8(v.map(str::to_string))) + .collect(); + file + } + + /// Two partition columns with NULLs in either position. + fn compound_key_file_group() -> FileGroup { + FileGroup::new(vec![ + pfile_with_pvs("a", &[Some("2026"), Some("09")]), + pfile_with_pvs("b", &[Some("2026"), Some("10")]), + pfile_with_pvs("c", &[None, Some("09")]), + pfile_with_pvs("d", &[Some("2025"), None]), + pfile_with_pvs("e", &[Some("2025"), Some("12")]), + ]) + } + + #[test] + fn test_group_by_partition_values_split_points_null_and_compound_keys() { + let fg = compound_key_file_group(); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(2); + assert_eq!(groups.len(), 2); + assert_eq!(split_points.len(), 1); + assert_eq!( + split_points[0].values(), + &[ScalarValue::from("2025"), ScalarValue::from("12")] + ); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_point_containing_null() { + let fg = compound_key_file_group(); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(3); + assert_eq!(groups.len(), 3); + assert_eq!( + split_points[0].values(), + &[ScalarValue::from("2025"), ScalarValue::Utf8(None)] + ); + assert_eq!( + split_points[1].values(), + &[ScalarValue::from("2026"), ScalarValue::from("09")] + ); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + let ordering = LexOrdering::new(vec![ + PhysicalSortExpr::new_default(Arc::new(Column::new("year", 0))), + PhysicalSortExpr::new_default(Arc::new(Column::new("month", 1))), + ]) + .unwrap(); + let range = RangePartitioning::try_new(ordering, split_points).unwrap(); + assert_eq!(range.partition_count(), 3); } } diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index 07144d8191a76..64b12951efc8e 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -39,8 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ - Constraint, Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, - internal_err, + Constraint, Constraints, Result, ScalarValue, SplitPoint, Statistics, + internal_datafusion_err, internal_err, }; use datafusion_execution::{ SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl, @@ -52,7 +52,9 @@ use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_physical_expr::expressions::{BinaryExpr, Column}; use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping}; use datafusion_physical_expr::utils::reassign_expr_columns; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction}; +use datafusion_physical_expr::{ + EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction, +}; use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory; use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile}; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; @@ -613,15 +615,52 @@ impl From for FileScanConfigBuilder { } } -/// Builds output partitioning over `partition_cols` (resolved to their indices in -/// `schema`) with `partition_count` partitions. Returns `None` when there are no -/// partition columns. Callers use this to declare the output partitioning of a scan -/// whose file groups are organized by partition column values. +/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in +/// `schema`). Returns `None` when there are no partition columns. +#[deprecated( + since = "56.0.0", + note = "Use `output_range_partitioning_from_split_points` instead" +)] pub fn output_partitioning_from_partition_fields( schema: &Schema, partition_cols: &Fields, partition_count: usize, ) -> Option { + let exprs = partition_column_exprs(schema, partition_cols)?; + Some(Partitioning::Hash(exprs, partition_count)) +} + +/// Builds the [`Partitioning::Range`] for file groups from +/// [`FileGroup::group_by_partition_values_with_split_points`], each column sorted with +/// `SortOptions::default()` to match how the groups were cut. Returns `None` when there +/// are no partition columns or one is missing from `schema`, and errors when +/// `split_points` are not strictly ordered. Split point values must have the partition +/// column types in `schema`. +pub fn output_range_partitioning_from_split_points( + schema: &Schema, + partition_cols: &Fields, + split_points: Vec, +) -> Result> { + let Some(exprs) = partition_column_exprs(schema, partition_cols) else { + return Ok(None); + }; + let sort_exprs = exprs + .into_iter() + .map(PhysicalSortExpr::new_default) + .collect::>(); + let Some(ordering) = LexOrdering::new(sort_exprs) else { + return Ok(None); + }; + let range = RangePartitioning::try_new(ordering, split_points)?; + Ok(Some(Partitioning::Range(range))) +} + +/// Resolves `partition_cols` to `Column` expressions in `schema`, or `None` when there +/// are no partition columns or one is missing. +fn partition_column_exprs( + schema: &Schema, + partition_cols: &Fields, +) -> Option>> { if partition_cols.is_empty() { return None; } @@ -635,8 +674,7 @@ pub fn output_partitioning_from_partition_fields( .position(|field| field.name() == name)?; exprs.push(Arc::new(Column::new(name, idx))); } - - Some(Partitioning::Hash(exprs, partition_count)) + Some(exprs) } fn project_output_partitioning( @@ -3150,6 +3188,14 @@ mod tests { let partitioning = config.output_partitioning(); assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_))); + + let partitioning = output_range_partitioning_from_split_points( + config.file_source.table_schema().table_schema(), + config.table_partition_cols(), + vec![], + ) + .unwrap(); + assert!(partitioning.is_none()); } #[test] @@ -3174,22 +3220,44 @@ mod tests { FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]), FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]), ]; - config.output_partitioning = output_partitioning_from_partition_fields( + let dict = |v: &str| wrap_partition_value_in_dict(ScalarValue::from(v)); + config.output_partitioning = output_range_partitioning_from_split_points( config.file_source.table_schema().table_schema(), config.table_partition_cols(), - config.file_groups.len(), - ); + vec![ + SplitPoint::new(vec![dict("2026-09")]), + SplitPoint::new(vec![dict("2026-10")]), + ], + ) + .unwrap(); let partitioning = config.output_partitioning(); match partitioning { - Partitioning::Hash(exprs, num_partitions) => { - assert_eq!(num_partitions, 3); - assert_eq!(exprs.len(), 1); - assert_eq!(exprs[0].downcast_ref::().unwrap().name(), "date"); + Partitioning::Range(range) => { + assert_eq!(range.partition_count(), 3); + assert_eq!(range.ordering().len(), 1); + let sort_expr = &range.ordering()[0]; + assert_eq!( + sort_expr.expr.downcast_ref::().unwrap().name(), + "date" + ); + assert_eq!(sort_expr.options, arrow::compute::SortOptions::default()); + assert_eq!(range.split_points().len(), 2); } - _ => panic!("Expected Hash partitioning"), + other => panic!("Expected Range partitioning, got {other}"), } + let err = output_range_partitioning_from_split_points( + config.file_source.table_schema().table_schema(), + config.table_partition_cols(), + vec![ + SplitPoint::new(vec![dict("2026-10")]), + SplitPoint::new(vec![dict("2026-09")]), + ], + ) + .unwrap_err(); + assert!(err.to_string().contains("strictly ordered"), "{err}"); + // Test multiple partition columns let multiple_partition_cols = vec![ Field::new("year", wrap_partition_type_in_dict(DataType::Utf8), false), @@ -3206,24 +3274,25 @@ mod tests { FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]), FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]), ]; - config.output_partitioning = output_partitioning_from_partition_fields( + config.output_partitioning = output_range_partitioning_from_split_points( config.file_source.table_schema().table_schema(), config.table_partition_cols(), - config.file_groups.len(), - ); + vec![SplitPoint::new(vec![dict("2026"), dict("09")])], + ) + .unwrap(); let partitioning = config.output_partitioning(); match partitioning { - Partitioning::Hash(exprs, num_partitions) => { - assert_eq!(num_partitions, 2); - assert_eq!(exprs.len(), 2); - let col_names: Vec<_> = exprs + Partitioning::Range(range) => { + assert_eq!(range.partition_count(), 2); + let col_names: Vec<_> = range + .ordering() .iter() - .map(|e| e.downcast_ref::().unwrap().name()) + .map(|e| e.expr.downcast_ref::().unwrap().name()) .collect(); assert_eq!(col_names, vec!["year", "month"]); } - _ => panic!("Expected Hash partitioning"), + other => panic!("Expected Range partitioning, got {other}"), } } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index eca97556d024a..5309f4c6f5109 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1217,13 +1217,9 @@ impl HashJoinExec { return false; } - // `preserve_file_partitions` can report Hive-style file groups as Hash - // partitioned even though their partition indexes do not follow the - // hash router used by partitioned dynamic filters. Reject Hash inputs - // because the metadata cannot distinguish those scans from a real hash - // repartition. Compatible Range inputs remain safe because matching - // ordering and split points align each build filter with its probe - // partition. Other unsupported layouts are rejected. + // Under `preserve_file_partitions`, file groups can report Hash without following + // the dynamic filter hash router and look like a real hash repartition, so reject + // Hash. Range inputs with matching split points are safe. // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195). // https://github.com/apache/datafusion/issues/20195 if config.optimizer.preserve_file_partitions > 0 diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs index a215a01e84260..40821583678d1 100644 --- a/datafusion/proto/tests/cases/plans/sources.rs +++ b/datafusion/proto/tests/cases/plans/sources.rs @@ -1144,3 +1144,84 @@ fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> { Ok(()) } + +#[test] +fn roundtrip_parquet_exec_range_output_partitioning_dict_split_points() -> Result<()> { + let file_schema = Arc::new(Schema::new(vec![Field::new( + "col", + wrap_partition_type_in_dict(DataType::Utf8), + false, + )])); + let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema))); + let output_partitioning = Partitioning::Range(RangePartitioning::try_new( + LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( + "col", 0, + )))]) + .unwrap(), + vec![SplitPoint::new(vec![wrap_partition_value_in_dict( + ScalarValue::from("B"), + )])], + )?); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![ + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-1.parquet".to_string(), + 1024, + )]), + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-2.parquet".to_string(), + 1024, + )]), + ]) + .with_output_partitioning(Some(output_partitioning.clone())) + .build(); + + assert_eq!( + roundtrip_file_scan_config(scan_config)?.output_partitioning, + Some(output_partitioning) + ); + + Ok(()) +} + +#[test] +fn roundtrip_parquet_exec_range_output_partitioning_utf8view_split_points() -> Result<()> +{ + let file_schema = Arc::new(Schema::new(vec![Field::new( + "col", + DataType::Utf8View, + false, + )])); + let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema))); + let output_partitioning = Partitioning::Range(RangePartitioning::try_new( + LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( + "col", 0, + )))]) + .unwrap(), + vec![SplitPoint::new(vec![ScalarValue::Utf8View(Some( + "B".to_string(), + ))])], + )?); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![ + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-1.parquet".to_string(), + 1024, + )]), + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-2.parquet".to_string(), + 1024, + )]), + ]) + .with_output_partitioning(Some(output_partitioning.clone())) + .build(); + + assert_eq!( + roundtrip_file_scan_config(scan_config)?.output_partitioning, + Some(output_partitioning) + ); + + Ok(()) +} diff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt index f4aab94ea627d..3fb4a3e17c3b9 100644 --- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt +++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt @@ -125,7 +125,7 @@ STORED AS PARQUET; 1 # Create high-cardinality fact table (5 partitions > 3 target_partitions) -# For testing partition merging with consistent hashing +# For testing partition merging into contiguous value ranges query I COPY (SELECT column1 as timestamp, column2 as value FROM (VALUES (TIMESTAMP '2023-01-01T09:00:00', 100.0) @@ -258,7 +258,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(fact_table.value)@2 as sum(fact_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(fact_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet # Verify results with optimization match results without optimization query TIR rowsort @@ -320,7 +320,7 @@ physical_plan 01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST] 02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)] 03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet query TIR SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; @@ -418,7 +418,7 @@ physical_plan 06)----------FilterExec: service@2 = log 07)------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TTTIR rowsort SELECT f.f_dkey, MAX(d.env), MAX(d.service), count(*), sum(f.value) @@ -493,7 +493,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet query TPRI rowsort SELECT f_dkey, timestamp, value, @@ -513,8 +513,7 @@ C 2023-01-01T09:00:20 310.2 3 ########## # TEST 9: High-Cardinality Partitions (more partitions than target_partitions) -# Since num_partitions > target_partitions (5 > 3), files are merged using -# round-robin assignment to ensure exactly target_partitions groups are created. +# Sorted partition values are cut into contiguous chunks. ########## # First verify results without optimization @@ -537,7 +536,7 @@ statement ok set datafusion.optimizer.preserve_file_partitions = 1; # Verify the plan uses SinglePartitioned mode with no RepartitionExec -# The 5 partitions are merged into 3 file groups using round-robin assignment +# 5 values merge into 3 contiguous groups: [A], [B, C], [D, E] query TT EXPLAIN SELECT f_dkey, count(*), sum(value) FROM high_cardinality_table GROUP BY f_dkey; ---- @@ -548,7 +547,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(high_cardinality_table.value)@2 as sum(high_cardinality_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(high_cardinality_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (D)], 3), file_type=parquet # Verify results with optimization match results without optimization query TIR rowsort @@ -658,9 +657,7 @@ C prod 2017.6 ########## # TEST 12: Partitioned Join with Matching Partition Counts - With Optimization # Both tables have 3 partitions matching target_partitions=3 -# No RepartitionExec needed for join - partitions already satisfy the requirement -# Dynamic filter pushdown is disabled in this mode because preserve_file_partitions -# reports Hash partitioning for Hive-style file groups, which are not hash-routed. +# Identical split points prove co-partitioning, so no RepartitionExec is needed. ########## statement ok @@ -685,8 +682,8 @@ physical_plan 02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] 04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0] -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Hash([d_dkey@1], 3), file_type=parquet -06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), file_type=parquet +06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TTR rowsort SELECT f.f_dkey, d.env, sum(f.value) @@ -722,7 +719,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, timestamp@1 as timestamp, count(Int64(1))@2 as count(*), avg(fact_table.value)@3 as avg(fact_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, timestamp@0 as timestamp], aggr=[count(Int64(1)), avg(fact_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet query TPIR rowsort SELECT f_dkey, timestamp, @@ -752,6 +749,196 @@ C 2023-01-01T09:00:40 1 300 C 2023-01-01T09:12:40 1 250 C 2023-01-01T09:12:50 1 275.4 +########## +# TEST 14: Partitioned join, same group count, different partition values +# fact has {A, B, C}, dimension has {A, C, D}. Split points differ, so the scans are not +# co-partitioned and one side must be range repartitioned. +########## + +query I +COPY (SELECT 'dev' as env, 'log' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet' +STORED AS PARQUET; +---- +1 + +query I +COPY (SELECT 'prod' as env, 'log' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet' +STORED AS PARQUET; +---- +1 + +query I +COPY (SELECT 'prod' as env, 'trace' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet' +STORED AS PARQUET; +---- +1 + +statement ok +CREATE EXTERNAL TABLE dimension_table_skewed (env STRING, service STRING) +STORED AS PARQUET +PARTITIONED BY (d_dkey STRING) +LOCATION 'test_files/scratch/preserve_file_partitioning/dimension_skewed/'; + +statement ok +set datafusion.execution.target_partitions = 3; + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold = 0; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0; + +# Exactly one join input gets a RepartitionExec(Range). +query TT +EXPLAIN SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +logical_plan +01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]] +02)--Projection: f.value, f.f_dkey, d.env +03)----Inner Join: f.f_dkey = d.d_dkey +04)------SubqueryAlias: f +05)--------TableScan: fact_table projection=[value, f_dkey] +06)------SubqueryAlias: d +07)--------TableScan: dimension_table_skewed projection=[env, d_dkey] +physical_plan +01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)] +02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 +03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] +04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0] +05)--------RepartitionExec: partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), input_partitions=3 +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(C), (D)], 3), file_type=parquet +07)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + +# Only A and C match. +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +C prod 2017.6 + +# Same result with the optimization disabled. +statement ok +set datafusion.optimizer.preserve_file_partitions = 0; + +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +C prod 2017.6 + +########## +# TEST 15: Partitioned join against a non partitioned VALUES input +# The inserted RepartitionExec reuses the scan's split points, so partition i on both +# sides covers the same key interval. +########## + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +query TT +EXPLAIN SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env) + ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +logical_plan +01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]] +02)--Projection: f.value, f.f_dkey, d.env +03)----Inner Join: f.f_dkey = CAST(d.d_dkey AS Utf8View) +04)------SubqueryAlias: f +05)--------TableScan: fact_table projection=[value, f_dkey] +06)------SubqueryAlias: d +07)--------Projection: column1 AS d_dkey, column2 AS env +08)----------Values: (Utf8("A"), Utf8("dev")), (Utf8("B"), Utf8("prod")), (Utf8("C"), Utf8("prod")), (Utf8("Z"), Utf8("none")) +physical_plan +01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)] +02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 +03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] +04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(d.d_dkey AS Utf8View)@2, f_dkey@1)], projection=[value@3, f_dkey@4, env@1] +05)--------RepartitionExec: partitioning=Range([CAST(d.d_dkey AS Utf8View)@2 ASC], [(B), (C)], 3), input_partitions=1 +06)----------ProjectionExec: expr=[column1@0 as d_dkey, column2@1 as env, CAST(column1@0 AS Utf8View) as CAST(d.d_dkey AS Utf8View)] +07)------------DataSourceExec: partitions=1, partition_sizes=[1] +08)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env) + ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +B prod 614.4 +C prod 2017.6 + +########## +# TEST 16: Self join on merged groups keeps every row +# 5 values merge into [A], [B, C], [D, E], and all must stay routable. +########## + +query TII rowsort +SELECT a.f_dkey, count(*), count(b.value) +FROM high_cardinality_table a +INNER JOIN high_cardinality_table b ON a.f_dkey = b.f_dkey +GROUP BY a.f_dkey; +---- +A 1 1 +B 1 1 +C 1 1 +D 1 1 +E 1 1 + +########## +# TEST 17: Statistics regrouping drops the Range claim +# The re-cut groups no longer follow partition values, so no output partitioning is +# declared, even when the group count is unchanged. +########## + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +statement ok +set datafusion.execution.split_file_groups_by_statistics = true; + +query TT +EXPLAIN SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; +---- +logical_plan +01)Sort: fact_table_ordered.f_dkey ASC NULLS LAST +02)--Projection: fact_table_ordered.f_dkey, count(Int64(1)) AS count(*), avg(fact_table_ordered.value) +03)----Aggregate: groupBy=[[fact_table_ordered.f_dkey]], aggr=[[count(Int64(1)), avg(fact_table_ordered.value)]] +04)------TableScan: fact_table_ordered projection=[value, f_dkey] +physical_plan +01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST] +02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)] +03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full +04)------RepartitionExec: partitioning=Hash([f_dkey@0], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST +05)--------AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet + +query TIR +SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; +---- +A 7 110.342857142857 +B 7 87.771428571429 +C 7 288.228571428571 + ########## # CLEANUP ########## @@ -769,6 +956,9 @@ reset datafusion.optimizer.hash_join_single_partition_threshold; statement ok reset datafusion.optimizer.hash_join_single_partition_threshold_rows; +statement ok +reset datafusion.execution.split_file_groups_by_statistics; + statement ok DROP TABLE fact_table; @@ -781,5 +971,8 @@ DROP TABLE dimension_table; statement ok DROP TABLE dimension_table_partitioned; +statement ok +DROP TABLE dimension_table_skewed; + statement ok DROP TABLE high_cardinality_table; diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt index 09bf131e5e95f..aaebd04e51923 100644 --- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt +++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt @@ -164,7 +164,7 @@ physical_plan 03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full 04)------RepartitionExec: partitioning=Hash([f_dkey@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 ASC NULLS LAST 05)--------AggregateExec: mode=Partial, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results without subset satisfaction query TPIR rowsort @@ -204,7 +204,7 @@ physical_plan 01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST] 02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as time_bin, count(Int64(1))@2 as count(*), avg(fact_table_ordered.value)@3 as avg(fact_table_ordered.value)] 03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results match with subset satisfaction query TPIR rowsort @@ -251,7 +251,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[f_dkey@2 ASC NULLS LAST, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) ASC NULLS LAST, timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([f_dkey@2, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0)], 3), input_partitions=3 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results without subset satisfaction query TPRI rowsort @@ -292,7 +292,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results match with subset satisfaction query TPRI rowsort @@ -379,8 +379,8 @@ physical_plan 11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] 12)----------------------CoalescePartitionsExec 13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2] -14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] +15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible # Verify results without subset satisfaction query TPR rowsort @@ -474,8 +474,8 @@ physical_plan 09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] 10)------------------CoalescePartitionsExec 11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2] -12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] +13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible # Verify results match with subset satisfaction query TPR rowsort From 8ce9f60384c8450baf10ba302979164b435fcc75 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:03:21 +0000 Subject: [PATCH 57/57] fix: `PushDownLeafProjections` yields when the extraction cannot move (#26085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of https://github.com/apache/datafusion/issues/25459. This is a follow-up to https://github.com/apache/datafusion/pull/25455 (https://github.com/apache/datafusion/issues/14540). ## Rationale for this change `PushDownLeafProjections` and `OptimizeProjections` undo each other in every optimizer pass for a common plan shape: a projection of a struct field (or another `MoveTowardsLeafNodes` expression) directly over a node that the extraction cannot go through, such as a `TableScan` or an `Aggregate`. ```text Projection: leaf_udf(test.a) TableScan: test ``` In each pass, `push_down_leaf_projections` splits this into a recovery projection over an in-place extraction projection: ```text Projection: __datafusion_extracted_1 AS leaf_udf(test.a) Projection: leaf_udf(test.a) AS __datafusion_extracted_1, test.a TableScan: test ``` Then `optimize_projections` merges the two back into the original plan. The loop stops only because the plan at the end of a pass is the same as at the end of the previous pass. The final plan is correct, but each pass does the work of both rules again. The test that https://github.com/apache/datafusion/pull/25455 added found this shape. That test records which rules change the plan in each pass. On `main` it prints, for the plan above: ```text pass 0: push_down_leaf_projections, optimize_projections pass 1: push_down_leaf_projections, optimize_projections ``` With this PR: ```text pass 0: optimize_projections ``` ### Which rule yields `PushDownLeafProjections` yields. The split has no use when the extraction projection cannot move: a plain `Projection(get_field(...))` directly over a Parquet scan already reaches `DataSourceExec projection=[..., get_field(s@1, value) ...]`. The existing `EXPLAIN` checks in `projection_pushdown.slt` show this and do not change. If `OptimizeProjections` yielded instead, the plan would keep a projection node that does no work. ## What changes are included in this PR? In `split_and_push_projection` (`extract_leaf_expressions.rs`), the rule still builds the in-place extraction projection, because a mixed projection over a `Filter` must become a pure extraction projection before it can go through the filter. The rule now immediately tries to push that extraction projection into its input. If it cannot move, the rule returns the original projection unchanged. The early return for a projection without new extractions stays: it also ends the recursion of that push attempt. - The decision is recorded in the module documentation of `extract_leaf_expressions.rs` and `optimize_projections`, and as a second entry in the "Rule Precedence" section of `docs/source/library-user-guide/query-optimizer.md`. ## What is the testing strategy for this PR? - `filter_and_extraction_projection_reach_fixed_point` in `push_down_filter.rs` now checks that no rule at all changes the plan in the last optimizer pass (before, it checked only `PushDownFilter` and `PushDownLeafProjections`). It also covers a projection over a scan with no filter. Without the fix it fails with `[["push_down_leaf_projections","optimize_projections"],["push_down_leaf_projections","optimize_projections"]]`. - Six stage snapshots in `extract_leaf_expressions.rs` change. In each one, only the "After Pushdown" stage changes, to "(same as after extraction)". The "Optimized" plan is the same in all six. - One line in `unnest.slt` changes (logical plan line 03). A nested alias is now kept: `get_field(__unnest_placeholder(...,depth=1) AS UNNEST(recursive_unnest_table.column3), Utf8("c1"))`. The split and merge used to remove that alias as a side effect. The physical plan is the same. This is the same display-only nested alias that https://github.com/apache/datafusion/pull/25455 showed for `length(...)`. Commands run: - `cargo test --profile ci -p datafusion-optimizer --lib`: 932 passed. - `cargo test --profile ci -p datafusion-sqllogictest --test sqllogictests`: 526 of 526 files pass. - `cargo fmt --all` and `cargo clippy --profile ci -p datafusion-optimizer --all-targets -- -D warnings`: clean. ## Are there any user-facing changes? No. Final plans and results do not change, except for the alias display in the `unnest` plan above. Planning does less work for queries that read struct fields. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../optimizer/src/extract_leaf_expressions.rs | 85 ++++++++++--------- .../optimizer/src/optimize_projections/mod.rs | 11 +++ datafusion/optimizer/src/push_down_filter.rs | 31 +++++-- datafusion/sqllogictest/test_files/unnest.slt | 2 +- .../library-user-guide/query-optimizer.md | 3 +- 5 files changed, 83 insertions(+), 49 deletions(-) diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs index ca12fb0bbb527..dad3cc03042c6 100644 --- a/datafusion/optimizer/src/extract_leaf_expressions.rs +++ b/datafusion/optimizer/src/extract_leaf_expressions.rs @@ -53,7 +53,19 @@ //! that runs before the extraction projection exists, so row group pruning and //! source level filtering still happen. //! +//! # Yields to [`OptimizeProjections`] +//! +//! When the extraction projection cannot move below the input of a projection +//! (for example a `TableScan` or an `Aggregate`), [`PushDownLeafProjections`] +//! leaves the projection unchanged. Splitting it in place would give a +//! recovery projection over an extraction projection on the same input, and +//! `OptimizeProjections` merges those two back into the original projection. +//! The two rules would then undo each other on every optimizer pass. The split +//! has no use there: the source absorbs the leaf expressions of the original +//! projection just as well. +//! //! [`PushDownFilter`]: crate::push_down_filter::PushDownFilter +//! [`OptimizeProjections`]: crate::optimize_projections::OptimizeProjections use indexmap::{IndexMap, IndexSet}; use std::collections::{BTreeSet, HashMap}; @@ -1150,18 +1162,33 @@ fn split_and_push_projection( // Only pre-existing __datafusion_extracted aliases and columns, no new // extractions from routing_extract. The original projection is // already an extraction projection that couldn't be pushed - // further. Return None. + // further. Return None. This return also ends the recursion + // of the `try_push_input` call below, which comes back here + // with a pure extraction projection. return Ok(None); } - // Build extraction projection in-place (couldn't push down) + // Build the extraction projection in place, then push it. It is a + // pure extraction projection, which can go through nodes that the + // original projection cannot. let input_arc = Arc::clone(input); - let extraction = build_extraction_projection_impl( + let extraction = LogicalPlan::Projection(build_extraction_projection_impl( &extraction_pairs, columns_needed, &input_arc, input_schema.as_ref(), - )?; - LogicalPlan::Projection(extraction) + )?); + match try_push_input(&extraction, alias_generator)? { + Some(pushed) => pushed, + // The extraction projection cannot move below `input`. Leave + // the original projection unchanged: the split would put the + // extraction projection on the same input, and + // `OptimizeProjections` merges the recovery and extraction + // projections back into the original projection. The rules + // would then undo each other in every optimizer pass. The + // source absorbs the leaf expressions of the original + // projection just as well. + None => return Ok(None), + } } }; @@ -1730,13 +1757,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -1760,13 +1784,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 IS NOT NULL AS has_name - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) IS NOT NULL AS has_name - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -1965,13 +1986,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS username - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) AS username - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2031,13 +2049,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")), __datafusion_extracted_1 AS name2 - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")), leaf_udf(test.user, Utf8("name")) AS name2 - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2245,13 +2260,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2405,8 +2417,8 @@ mod tests { } /// Projection containing a MoveTowardsLeafNodes sub-expression above an - /// Aggregate. Aggregate blocks pushdown, so the (None, true) recovery - /// fallback path fires: in-place extraction + recovery projection. + /// Aggregate. The Aggregate blocks pushdown, so the extraction cannot move + /// and the original projection is kept unchanged. #[test] fn test_projection_with_leaf_expr_above_aggregate() -> Result<()> { use datafusion_expr::test::function_stub::count; @@ -2432,15 +2444,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 IS NOT NULL AS has_name, COUNT(Int32(1)) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user, COUNT(Int32(1)) - Aggregate: groupBy=[[test.user]], aggr=[[COUNT(Int32(1))]] - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) IS NOT NULL AS has_name, COUNT(Int32(1)) - Aggregate: groupBy=[[test.user]], aggr=[[COUNT(Int32(1))]] - TableScan: test projection=[user] + (same as after pushdown) "#) } diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index c913a4ce961cc..8fdb5775c971c 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -16,6 +16,17 @@ // under the License. //! [`OptimizeProjections`] identifies and eliminates unused columns +//! +//! # Precedence over `PushDownLeafProjections` +//! +//! `OptimizeProjections` merges adjacent projections, including a recovery +//! projection over an extraction projection that +//! [`PushDownLeafProjections`] creates. To keep the two rules from undoing each +//! other, `PushDownLeafProjections` only creates that pair when the extraction +//! projection moves below the input. See the module documentation of +//! [`extract_leaf_expressions`](crate::extract_leaf_expressions). +//! +//! [`PushDownLeafProjections`]: crate::extract_leaf_expressions::PushDownLeafProjections mod required_indices; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 54f220a1fe9dd..ccde8aa07f64e 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1751,7 +1751,7 @@ mod tests { use crate::simplify_expressions::SimplifyExpressions; use crate::test::udfs::leaf_udf_expr; use crate::test::*; - use datafusion_expr::test::function_stub::sum; + use datafusion_expr::test::function_stub::{count, sum}; use insta::assert_snapshot; use super::*; @@ -4864,9 +4864,11 @@ mod tests { /// `PushDownFilter` and `PushDownLeafProjections` must not undo each other /// for a filter next to a pure extraction projection - /// (). Neither rule may - /// change the plan in the last optimizer pass. The source does not absorb - /// filters, so the `Filter` node stays in the plan. + /// (). The source does + /// not absorb filters, so the `Filter` node stays in the plan. Also, + /// `PushDownLeafProjections` must not split a projection that + /// `OptimizeProjections` merges back. No rule may change the plan in the + /// last optimizer pass. #[test] fn filter_and_extraction_projection_reach_fixed_point() -> Result<()> { let scan = || { @@ -4887,13 +4889,26 @@ mod tests { .project(vec![leaf_udf_expr(col("a")), col("b")])? .build()?; - for plan in [simple, two_filters] { + // No filter: the extraction projection cannot move below the scan. + let no_filter = LogicalPlanBuilder::from(test_table_scan()?) + .project(vec![leaf_udf_expr(col("a"))])? + .build()?; + + // Leaf expression above an Aggregate: the Aggregate blocks movement. + let above_aggregate = LogicalPlanBuilder::from(test_table_scan_with_struct()?) + .aggregate(vec![col("user")], vec![count(lit(1))])? + .project(vec![ + leaf_udf_expr(col("user")).is_not_null().alias("has_name"), + col("COUNT(Int32(1))"), + ])? + .build()?; + + for plan in [simple, two_filters, no_filter, above_aggregate] { let passes = rules_that_changed_plan_per_pass(plan)?; let last = passes.last().unwrap(); assert!( - !last.iter().any(|rule| rule == "push_down_filter" - || rule == "push_down_leaf_projections"), - "pushdown rules changed the plan in the last pass: {passes:?}" + last.is_empty(), + "rules changed the plan in the last pass: {passes:?}" ); } Ok(()) diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt index 39ed64a526755..cf190949aad0a 100644 --- a/datafusion/sqllogictest/test_files/unnest.slt +++ b/datafusion/sqllogictest/test_files/unnest.slt @@ -668,7 +668,7 @@ explain select unnest(unnest(unnest(column3)['c1'])), column3 from recursive_unn logical_plan 01)Projection: __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1],depth=2) AS UNNEST(UNNEST(UNNEST(recursive_unnest_table.column3)[c1])), recursive_unnest_table.column3 02)--Unnest: lists[__unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1])|depth=2] structs[] -03)----Projection: get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1), Utf8("c1")) AS __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), recursive_unnest_table.column3 +03)----Projection: get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1) AS UNNEST(recursive_unnest_table.column3), Utf8("c1")) AS __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), recursive_unnest_table.column3 04)------Unnest: lists[__unnest_placeholder(recursive_unnest_table.column3)|depth=1] structs[] 05)--------Projection: recursive_unnest_table.column3 AS __unnest_placeholder(recursive_unnest_table.column3), recursive_unnest_table.column3 06)----------TableScan: recursive_unnest_table projection=[column3] diff --git a/docs/source/library-user-guide/query-optimizer.md b/docs/source/library-user-guide/query-optimizer.md index 73c3ed6ed11dc..03fb6f2e33fe8 100644 --- a/docs/source/library-user-guide/query-optimizer.md +++ b/docs/source/library-user-guide/query-optimizer.md @@ -168,9 +168,10 @@ Two rules can want the opposite order for the same pair of adjacent plan nodes. Do not let two rules compete. Give one rule precedence, make the competing rule yield, and record the decision in the module documentation of both rules. -There is one such decision today: +There are two such decisions today: - `PushDownFilter` yields to a _pure extraction projection_. A pure extraction projection is a projection whose expressions are only `__datafusion_extracted_N` aliases and pass-through columns. `ExtractLeafExpressions` creates it, and `PushDownLeafProjections` moves it towards the leaves. `PushDownFilter` does not move a filter below such a projection, so the projection stays next to the scan. A Parquet scan then merges the projection into the file projection and reads only the struct leaf. The filter loses nothing, because `PushDownFilter` records the predicate in `TableScan::filters` in the pass that runs before the extraction projection exists. See [issue #14540](https://github.com/apache/datafusion/issues/14540). +- `PushDownLeafProjections` yields to `OptimizeProjections`. When the extraction projection cannot move below the input of a projection, for example a `TableScan` or an `Aggregate`, `PushDownLeafProjections` leaves the projection unchanged. A split in place would give a recovery projection over an extraction projection on the same input, which `OptimizeProjections` merges back. The source absorbs the leaf expressions of the original projection just as well. See [PR #26085](https://github.com/apache/datafusion/pull/26085). ### Expression Naming