diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cf222c4b32b7b..e2ab325701ed5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -62,14 +62,6 @@ updates: update-types: - "minor" - "patch" - # dev/depcheck is a standalone crate excluded from the workspace, so it is - # not covered by the workspace entry above - - package-ecosystem: cargo - directory: "/dev/depcheck" - schedule: - interval: weekly - target-branch: main - labels: [auto-dependencies] # the wasm test app has its own npm lockfile - package-ecosystem: npm directory: "/datafusion/wasmtest/datafusion-wasm-app" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7df585ddad6f9..ba77e4ddb7d3c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -980,21 +980,15 @@ jobs: working-directory: datafusion/proto run: cargo msrv --output-format json --log-target stdout verify - # Check for dependency cycles between DataFusion crates with `dev/depcheck`. + # Check for dependency cycles between DataFusion crates, which would block + # publishing them to crates.io, with a publishing dry run. depcheck: name: Circular Dependency Check if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }} - runs-on: ubuntu-latest - container: - image: amd64/rust + runs-on: ubuntu-slim steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: true - fetch-depth: 1 - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable + uses: ./.github/actions/setup-rustup - name: Check dependencies run: bash ci/scripts/check_circular_dependencies.sh diff --git a/Cargo.toml b/Cargo.toml index 763206a0fdeb8..14eaad0245976 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,6 @@ members = [ "datafusion/doc", "xtask", ] -exclude = ["dev/depcheck"] resolver = "2" [workspace.package] diff --git a/ci/scripts/check_circular_dependencies.sh b/ci/scripts/check_circular_dependencies.sh index 7cdc97b0d6ba7..cb891e6b40e4d 100755 --- a/ci/scripts/check_circular_dependencies.sh +++ b/ci/scripts/check_circular_dependencies.sh @@ -17,8 +17,26 @@ # specific language governing permissions and limitations # under the License. -# Checks for circular dependencies between DataFusion crates with -# `dev/depcheck`, the same way the "Circular Dependency Check" job does. +# Checks for circular dependencies between DataFusion crates, the same way the +# "Circular Dependency Check" job does. +# +# A cycle prevents publishing the crates to crates.io. A crate can be +# published only after every DataFusion crate it depends on, including through +# dev-dependencies, optional dependencies, and dependencies for other targets, +# so no crate in a cycle can be published first. Cargo refuses to build a +# cycle of normal dependencies, but it allows one that passes through a +# dev-dependency, because a dev-dependency is built only for the crate's own +# tests, examples, and benchmarks. Such a cycle builds and passes tests, and +# fails only at release time. +# +# The check is a publishing dry run. `cargo publish --workspace` packages every +# crate that does not set `publish = false`, in an order where each crate comes +# after the DataFusion crates it depends on, and fails if there is no such +# order. `--dry-run` stops before uploading, and `--no-verify` skips building +# the packages. The dry run also fails on any other problem that would stop the +# crates from being published, such as a dependency without a version. +# `--allow-dirty` lets the check run on a working tree with uncommitted +# changes. set -euo pipefail @@ -26,9 +44,7 @@ SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" -# depcheck is outside the root workspace and finds the root manifest from its -# own directory, so run Cargo from there. -cd "${ROOT_DIR}/dev/depcheck" +cd "${ROOT_DIR}" -echo "[${SCRIPT_NAME}] \`cargo run --locked\` in dev/depcheck" -cargo run --locked +echo "[${SCRIPT_NAME}] \`cargo publish --workspace --dry-run --no-verify --allow-dirty\`" +cargo publish --workspace --dry-run --no-verify --allow-dirty diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index a72d2c2bc7d40..c94ddcef22d6b 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -25,13 +25,13 @@ use async_trait::async_trait; use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err, - project_schema, + Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err, + plan_err, project_schema, }; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_groups::FileGroup; use datafusion_datasource::file_scan_config::{ - FileScanConfig, FileScanConfigBuilder, output_partitioning_from_partition_fields, + FileScanConfig, FileScanConfigBuilder, output_range_partitioning_from_split_points, }; use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig}; #[expect(deprecated)] @@ -69,6 +69,8 @@ pub struct ListFilesResult { pub statistics: Statistics, /// Whether files are grouped by partition values. pub grouped_by_partition: bool, + /// Boundaries between file groups, empty unless `grouped_by_partition` is true. + pub partition_split_points: Vec, } /// Built in [`TableProvider`] that reads data from one or more files as a single table. @@ -668,6 +670,7 @@ impl ListingTable { file_groups: mut partitioned_file_lists, statistics, grouped_by_partition: partitioned_by_file_group, + partition_split_points, } = self .list_files_for_scan(state, &partition_filters, statistic_file_limit) .await?; @@ -687,6 +690,7 @@ impl ListingTable { .config_options() .execution .split_file_groups_by_statistics; + let mut regrouped_by_statistics = false; match split_file_groups_by_statistics .then(|| { output_ordering.first().map(|output_ordering| { @@ -704,6 +708,7 @@ impl ListingTable { Some(Ok(new_groups)) => { if new_groups.len() <= file_group_count { partitioned_file_lists = new_groups; + regrouped_by_statistics = true; } else { log::debug!( "attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered" @@ -737,6 +742,17 @@ impl ListingTable { )? } }; + Some(output_partitioning) + } else if partitioned_by_file_group && !regrouped_by_statistics { + output_range_partitioning_from_split_points( + &self.table_schema, + &table_partition_cols.into(), + partition_split_points, + )? + } else { + None + }; + if let Some(output_partitioning) = &output_partitioning { let partition_count = output_partitioning.partition_count(); if partitioned_file_lists.len() != partition_count { return plan_err!( @@ -744,19 +760,7 @@ impl ListingTable { partitioned_file_lists.len() ); } - Some(output_partitioning) - } else if partitioned_by_file_group { - // Files are grouped by partition column values: declare output - // partitioning on those columns so the optimizer can skip - // repartitioning for aggregates and joins on the partition columns. - output_partitioning_from_partition_fields( - &self.table_schema, - &table_partition_cols.clone().into(), - partitioned_file_lists.len(), - ) - } else { - None - }; + } let Some(object_store_url) = self.table_paths.first().map(ListingTableUrl::object_store) @@ -1005,6 +1009,7 @@ impl ListingTable { file_groups: vec![], statistics: Statistics::new_unknown(&self.file_schema), grouped_by_partition: false, + partition_split_points: vec![], }); }; let (file_group, inexact_stats) = self @@ -1018,21 +1023,23 @@ impl ListingTable { // skip repartitioning for aggregates and joins on partition columns. let threshold = ctx.config_options().optimizer.preserve_file_partitions; - let (file_groups, grouped_by_partition) = + let (file_groups, grouped_by_partition, partition_split_points) = if threshold > 0 && !self.options.table_partition_cols.is_empty() { - let grouped = file_group.group_by_partition_values(file_group_count); + let (grouped, split_points) = file_group + .group_by_partition_values_with_split_points(file_group_count); if grouped.len() >= threshold { - (grouped, true) + (grouped, true, split_points) } else { let all_files: Vec<_> = grouped.into_iter().flat_map(|g| g.into_inner()).collect(); ( FileGroup::new(all_files).split_files(file_group_count), false, + vec![], ) } } else { - (file_group.split_files(file_group_count), false) + (file_group.split_files(file_group_count), false, vec![]) }; self.list_files_result_from_groups( @@ -1040,6 +1047,7 @@ impl ListingTable { file_groups, inexact_stats, grouped_by_partition, + partition_split_points, ) } @@ -1067,6 +1075,7 @@ impl ListingTable { file_groups: vec![], statistics: Statistics::new_unknown(&self.file_schema), grouped_by_partition: false, + partition_split_points: vec![], }); }; let (file_group, inexact_stats) = @@ -1078,7 +1087,7 @@ impl ListingTable { let file_groups = self.filter_declared_file_groups_by_partition_filters(file_groups, filters)?; - self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false) + self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false, vec![]) } fn filter_declared_file_groups_by_partition_filters( @@ -1113,6 +1122,7 @@ impl ListingTable { file_groups: Vec, inexact_stats: bool, grouped_by_partition: bool, + partition_split_points: Vec, ) -> datafusion_common::Result { let (file_groups, stats) = compute_all_files_statistics( file_groups, @@ -1129,6 +1139,7 @@ impl ListingTable { file_groups, statistics: stats, grouped_by_partition, + partition_split_points, }) } diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index e79c21dfb7f08..f6f666840010b 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -903,7 +903,7 @@ fn validate_map_key_data_type( } Ok(()) } - _ if is_injective_map_key_cast(source_type, target_type) => Ok(()), + _ if is_injective_cast(source_type, target_type) => Ok(()), _ => _plan_err!( "Cannot safely evolve Map key type from {} to {}", source_type, @@ -912,15 +912,15 @@ fn validate_map_key_data_type( } } -/// Returns true when casting a Map key from `source_type` to `target_type` is -/// known to preserve key identity. +/// Returns true when casting from `source_type` to `target_type` is known to +/// preserve value identity, so distinct source values stay distinct. /// -/// Keep this list conservative. Map keys define equality/lookup identity, so -/// only representation widenings that cannot merge distinct source keys belong -/// here. Other Arrow-supported casts, including decimal/timestamp widening and -/// dictionaries, should stay rejected until their identity semantics are -/// deliberately reviewed. -fn is_injective_map_key_cast(source_type: &DataType, target_type: &DataType) -> bool { +/// Keep this list conservative. Map keys and decorrelated equality keys depend +/// on identity, so only representation widenings that cannot merge distinct +/// source values belong here. Other Arrow-supported casts, including +/// decimal/timestamp widening and dictionaries, should stay rejected until +/// their identity semantics are deliberately reviewed. +pub fn is_injective_cast(source_type: &DataType, target_type: &DataType) -> bool { matches!( (source_type, target_type), ( @@ -2283,14 +2283,14 @@ mod tests { (DataType::UInt16, DataType::Int64), (DataType::UInt32, DataType::Int64), ] { - assert!(is_injective_map_key_cast(&source_type, &target_type)); + assert!(is_injective_cast(&source_type, &target_type)); } for (source_type, target_type) in [ (DataType::UInt8, DataType::Int8), (DataType::UInt16, DataType::Int16), (DataType::UInt32, DataType::Int32), ] { - assert!(!is_injective_map_key_cast(&source_type, &target_type)); + assert!(!is_injective_cast(&source_type, &target_type)); } } diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 7275c4b674d20..89f42761535e3 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -245,6 +245,10 @@ name = "math_query_sql" harness = false name = "filter_query_sql" +[[bench]] +harness = false +name = "in_list_rewrite" + [[bench]] harness = false name = "struct_query_sql" diff --git a/datafusion/core/benches/in_list_rewrite.rs b/datafusion/core/benches/in_list_rewrite.rs new file mode 100644 index 0000000000000..078c8f5e7db4b --- /dev/null +++ b/datafusion/core/benches/in_list_rewrite.rs @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks short SQL `IN` lists using the default optimizer. Comparing the +//! same cases across commits measures changes to the chosen evaluation strategy. +//! Criterion times expression evaluation, with planning and table setup outside +//! the measured loop. Input types and expected Boolean results are checked first. +//! +//! All types cover misses, balanced matches, and uniform or skewed +//! first-item hits. Floating-point and fixed-size-binary types also cover +//! nulls and small batches; a misaligned binary batch exercises alignment copies. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray}; +use arrow::buffer::{Buffer, MutableBuffer}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use datafusion::prelude::SessionContext; +use datafusion_common::ScalarValue; +use datafusion_expr::LogicalPlan; +use datafusion_physical_expr::PhysicalExpr; +use rand::prelude::*; +use tokio::runtime::Runtime; + +const MISS_VALUE_BASE: usize = 10_000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + Int32, + Int64, + UInt64, + Float32, + Float64, + Utf8, + Utf8View, + FixedSizeBinary(i32), +} + +impl ValueKind { + const ALL: [Self; 12] = [ + Self::Int32, + Self::Int64, + Self::UInt64, + Self::Float32, + Self::Float64, + Self::Utf8, + Self::Utf8View, + Self::FixedSizeBinary(1), + Self::FixedSizeBinary(2), + Self::FixedSizeBinary(4), + Self::FixedSizeBinary(8), + Self::FixedSizeBinary(16), + ]; + + fn name(self) -> String { + match self { + Self::Int32 => "int32", + Self::Int64 => "int64", + Self::UInt64 => "uint64", + Self::Float32 => "float32", + Self::Float64 => "float64", + Self::Utf8 => "utf8", + Self::Utf8View => "utf8view_inline", + Self::FixedSizeBinary(width) => return format!("fixed_size_binary_{width}"), + } + .to_owned() + } + + fn seed_tag(self) -> u64 { + match self { + Self::Int32 => 1, + Self::Float64 => 2, + Self::Utf8 => 3, + Self::Utf8View => 4, + Self::FixedSizeBinary(8) => 5, + Self::Int64 => 6, + Self::Float32 => 7, + Self::UInt64 => 8, + Self::FixedSizeBinary(width) => 20 + width as u64, + } + } + + // The same typed values build batches and bind SQL parameters, avoiding + // implicit SQL literal widening for Float32 and UInt64. + fn scalar(self, value: Option) -> ScalarValue { + match self { + Self::Int32 => ScalarValue::Int32(value.map(|v| v as i32)), + Self::Int64 => ScalarValue::Int64(value.map(|v| v as i64)), + Self::UInt64 => ScalarValue::UInt64(value.map(|v| v as u64)), + Self::Float32 => ScalarValue::Float32(value.map(|v| v as f32)), + Self::Float64 => ScalarValue::Float64(value.map(|v| v as f64)), + Self::Utf8 => ScalarValue::Utf8(value.map(string_value)), + Self::Utf8View => ScalarValue::Utf8View(value.map(string_value)), + Self::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + width, + value.map(|v| { + // Narrow misses stay disjoint from the hit values 0, 1, and 2. + let v = if width == 1 && v >= MISS_VALUE_BASE { + 4 + (v - MISS_VALUE_BASE) % 252 + } else { + v + }; + (v as u128).to_be_bytes()[16 - width as usize..].to_vec() + }), + ), + } + } + + fn data_type(self) -> DataType { + self.scalar(Some(0)).data_type() + } + + fn make_array(self, values: &[Option]) -> ArrayRef { + let array = + ScalarValue::iter_to_array(values.iter().map(|&v| self.scalar(v))).unwrap(); + if let Self::FixedSizeBinary(width) = self { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + binary.values().as_ptr().align_offset(width as usize), + 0, + "builder-created binary buffer must be aligned" + ); + } + array + } +} + +// Eight-byte strings use Arrow's inline byte-view representation. +fn string_value(value: usize) -> String { + format!("v{value:07}") +} + +#[derive(Clone, Copy)] +enum Profile { + Miss, + Balanced, + AllFirstHit, + SkewedHit, + Nullable, + SmallBatch, + SingleMiss, + SingleHit, + ListNull, + Unaligned, +} + +impl Profile { + const ALL: [Self; 10] = [ + Self::Miss, + Self::Balanced, + Self::AllFirstHit, + Self::SkewedHit, + Self::Nullable, + Self::SmallBatch, + Self::SingleMiss, + Self::SingleHit, + Self::ListNull, + Self::Unaligned, + ]; + + fn name(self) -> &'static str { + match self { + Self::Miss => "miss", + Self::Balanced => "balanced", + Self::AllFirstHit => "all_first_hit", + Self::SkewedHit => "skewed_hit", + Self::Nullable => "balanced_nullable", + Self::SmallBatch => "balanced_small_batch", + Self::SingleMiss => "single_row_miss", + Self::SingleHit => "single_row_hit", + Self::ListNull => "list_null_balanced", + Self::Unaligned => "balanced_unaligned", + } + } + + fn batch_size(self) -> usize { + match self { + Self::SmallBatch => 64, + Self::SingleMiss | Self::SingleHit => 1, + _ => 8192, + } + } + + fn null_percent(self) -> usize { + if matches!(self, Self::Nullable) { + 20 + } else { + 0 + } + } + + fn match_percent(self) -> usize { + match self { + Self::Miss | Self::SingleMiss => 0, + Self::AllFirstHit | Self::SingleHit => 100, + Self::SkewedHit => 99, + _ => 50, + } + } + + fn permits_kind(self, kind: ValueKind) -> bool { + match self { + Self::Unaligned => kind == ValueKind::FixedSizeBinary(16), + Self::Miss | Self::Balanced | Self::AllFirstHit | Self::SkewedHit => true, + _ => matches!( + kind, + ValueKind::Float32 | ValueKind::Float64 | ValueKind::FixedSizeBinary(_) + ), + } + } +} + +// Project the predicate so optimization preserves its Boolean result, including +// NULLs, rather than simplifying it based on WHERE semantics. +fn plan_in_list( + ctx: &SessionContext, + runtime: &Runtime, + kind: ValueKind, + batch: &RecordBatch, + list_len: usize, + negated: bool, + list_has_null: bool, +) -> Arc { + ctx.deregister_table(kind.name()).unwrap(); + ctx.register_batch(kind.name(), batch.clone()).unwrap(); + let params = (0..list_len) + .map(|i| kind.scalar((!list_has_null || i + 1 < list_len).then_some(i))) + .collect::>(); + assert!( + params + .iter() + .all(|value| value.data_type() == kind.data_type()) + ); + let placeholders = (1..=list_len).map(|i| format!("${i}")).collect::>(); + let sql = format!( + "SELECT a {}IN ({}) AS matches FROM {}", + if negated { "NOT " } else { "" }, + placeholders.join(", "), + kind.name() + ); + let plan = runtime + .block_on(ctx.sql(&sql)) + .unwrap() + .with_param_values(params) + .unwrap() + .into_optimized_plan() + .unwrap(); + let LogicalPlan::Projection(projection) = plan else { + panic!("SQL must project the predicate"); + }; + assert_eq!(projection.expr.len(), 1); + assert_eq!( + projection.input.schema().field(0).data_type(), + &kind.data_type() + ); + let expr = ctx + .create_physical_expr( + projection.expr[0].clone().unalias(), + projection.input.schema(), + ) + .unwrap(); + let schema = batch.schema(); + assert_eq!(expr.data_type(schema.as_ref()).unwrap(), DataType::Boolean); + // Check types without requiring either IN or comparison-chain expressions. + let mut nodes = vec![Arc::clone(&expr)]; + while let Some(node) = nodes.pop() { + let data_type = node.data_type(schema.as_ref()).unwrap(); + assert!(data_type == DataType::Boolean || data_type == kind.data_type()); + nodes.extend(node.children().into_iter().cloned()); + } + expr +} + +fn make_batch( + kind: ValueKind, + profile: Profile, + profile_index: usize, + list_len: usize, + negated: bool, +) -> (RecordBatch, BooleanArray) { + let null_count = profile.batch_size() * profile.null_percent() / 100; + let non_null_count = profile.batch_size() - null_count; + let match_count = non_null_count * profile.match_percent() / 100; + let list_has_null = matches!(profile, Profile::ListNull); + let first_hits = matches!( + profile, + Profile::AllFirstHit | Profile::SkewedHit | Profile::SingleHit + ); + let mut values = Vec::with_capacity(profile.batch_size()); + values.extend(std::iter::repeat_n(None, null_count)); + values.extend((0..match_count).map(|i| { + Some(if first_hits { + 0 + } else { + i % (list_len - usize::from(list_has_null)) + }) + })); + values.extend((0..non_null_count - match_count).map(|i| Some(MISS_VALUE_BASE + i))); + let seed = 0x1A11_1575_5EED_u64 + ^ (kind.seed_tag() << 48) + ^ ((profile_index as u64) << 32) + ^ list_len as u64; + values.shuffle(&mut StdRng::seed_from_u64(seed)); + let expected = values + .iter() + .map(|value| { + value.and_then(|v| { + if v < list_len - usize::from(list_has_null) { + Some(!negated) + } else if list_has_null { + None + } else { + Some(negated) + } + }) + }) + .collect::(); + let mut array = kind.make_array(&values); + if matches!(profile, Profile::Unaligned) { + let binary = array + .as_any() + .downcast_ref::() + .unwrap(); + let mut bytes = MutableBuffer::with_capacity(binary.values().len() + 1); + bytes.push(0_u8); + bytes.extend_from_slice(binary.values()); + let buffer = Buffer::from(bytes).slice(1); + assert_ne!(buffer.as_ptr().align_offset(16), 0); + array = Arc::new(FixedSizeBinaryArray::new( + 16, + buffer, + binary.nulls().cloned(), + )); + } + let schema = Arc::new(Schema::new(vec![Field::new("a", kind.data_type(), true)])); + (RecordBatch::try_new(schema, vec![array]).unwrap(), expected) +} + +fn criterion_benchmark(c: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + let ctx = SessionContext::new(); + for kind in ValueKind::ALL { + for (profile_index, profile) in Profile::ALL.into_iter().enumerate() { + if !profile.permits_kind(kind) { + continue; + } + let mut group = c.benchmark_group(format!( + "in_list_rewrite/{}/{}/batch={}/nulls={}%/match={}%", + kind.name(), + profile.name(), + profile.batch_size(), + profile.null_percent(), + profile.match_percent() + )); + group.throughput(Throughput::Elements(profile.batch_size() as u64)); + for list_len in [2, 3] { + for negated in [false, true] { + let (batch, expected) = + make_batch(kind, profile, profile_index, list_len, negated); + let expr = plan_in_list( + &ctx, + &runtime, + kind, + &batch, + list_len, + negated, + matches!(profile, Profile::ListNull), + ); + let output = expr + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(output.as_boolean(), &expected); + let case = format!( + "{}/list={list_len}", + if negated { "not_in" } else { "in" } + ); + group.bench_function(case, |b| { + b.iter(|| black_box(expr.evaluate(black_box(&batch)).unwrap())) + }); + } + } + group.finish(); + } + } +} + +criterion_group! { + name = benches; + config = Criterion::default() + .warm_up_time(Duration::from_millis(100)) + .measurement_time(Duration::from_millis(500)); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs index 0faa4a64cf983..858439f907d3a 100644 --- a/datafusion/core/tests/memory_limit/mod.rs +++ b/datafusion/core/tests/memory_limit/mod.rs @@ -453,6 +453,10 @@ async fn group_by_count_distinct_utf8_view() { /// the buffered slices are sorted: coalesced into runs for a single sort key, /// and one run per slice otherwise. The in-place threshold is below the slice /// size, so that coalescing keeps the slices as separate runs. +/// +/// Skipped under `force_hash_collisions`, where the 100,000 colliding group +/// keys make the aggregate quadratic and the test take many minutes. +#[cfg(not(feature = "force_hash_collisions"))] #[tokio::test] async fn sort_after_group_by_reserves_shared_output_once() { for order_by in ["group_key", "s desc, group_key"] { diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 283d22767f9b1..20ea0fa35319d 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -28,9 +28,12 @@ use crate::physical_optimizer::test_utils::{ sort_preserving_merge_exec, union_exec, }; -use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array}; +use arrow::array::{ + Int64Array, RecordBatch, TimestampNanosecondArray, UInt8Array, UInt64Array, +}; use arrow::compute::SortOptions; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use datafusion::assert_batches_sorted_eq; use datafusion::config::ConfigOptions; use datafusion::datasource::MemTable; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; @@ -75,7 +78,7 @@ use datafusion_physical_plan::operator_statistics::{ use datafusion_physical_expr::{ AcrossPartitions, ConstExpr, Distribution, Partitioning, RangePartitioning, - SplitPoint, + ScalarFunctionExpr, SplitPoint, }; use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion_physical_plan::execution_plan::ExecutionPlan; @@ -959,6 +962,233 @@ fn range_aggregate_reuses_range_partitioning() -> Result<()> { Ok(()) } +fn timestamp_range_input(split: i64) -> Result> { + let schema = Arc::new(Schema::new(vec![Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + )])); + let hour = 3_600_000_000_000; + let (before_split, after_split): (Vec<_>, Vec<_>) = + [0, 1, hour, hour + 1, 2 * hour, 2 * hour + 1] + .into_iter() + .partition(|timestamp| *timestamp < split); + let partitions = [before_split, after_split] + .into_iter() + .map(|values| -> Result<_> { + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(TimestampNanosecondArray::from(values))], + )?; + Ok(vec![batch]) + }) + .collect::>>()?; + let range = RangePartitioning::try_new( + [PhysicalSortExpr::new_default(col("timestamp", &schema)?)].into(), + vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + Some(split), + None, + )])], + )?; + let source = MemorySourceConfig::try_new_exec(&partitions, schema, None)?; + Ok(Arc::new( + source + .as_ref() + .clone() + .with_partitioning(Partitioning::Range(range)), + )) +} + +fn date_trunc_expr( + input: &dyn ExecutionPlan, + precision: &str, +) -> Result> { + Ok(Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![lit(precision), col("timestamp", &input.schema())?], + Field::new( + "bucket", + DataType::Timestamp(TimeUnit::Nanosecond, None), + false, + ) + .into(), + Arc::new(ConfigOptions::default()), + ))) +} + +/// Builds separate Partial and Final stages without projected range metadata. +/// This exercises checking group locality through the partial stage's input. +fn partial_final_date_trunc_count( + input: Arc, + precision: &str, +) -> Result> { + let input_schema = input.schema(); + let group_by = PhysicalGroupBy::new_single(vec![( + date_trunc_expr(input.as_ref(), precision)?, + "bucket".to_string(), + )]); + let count = Arc::new( + AggregateExprBuilder::new(count_udaf(), vec![col("timestamp", &input_schema)?]) + .schema(Arc::clone(&input_schema)) + .alias("count") + .build()?, + ); + let partial = Arc::new(AggregateExec::try_new( + AggregateMode::Partial, + group_by.clone(), + vec![Arc::clone(&count)], + vec![None], + input, + Arc::clone(&input_schema), + )?); + assert!(matches!( + partial.properties().output_partitioning(), + Partitioning::UnknownPartitioning(2) + )); + + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::FinalPartitioned, + group_by.as_final(), + vec![count], + vec![None], + partial, + input_schema, + )?)) +} + +#[tokio::test] +async fn aligned_hour_groups_avoid_shuffle_through_partial_aggregate() -> Result<()> { + let hour = 3_600_000_000_000; + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, "hour")?; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + + assert_plan!( + optimized, + @r" + AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count] + AggregateExec: mode=Partial, gby=[date_trunc(hour, timestamp@0) as bucket], aggr=[count] + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_batches_sorted_eq!( + [ + "+---------------------+-------+", + "| bucket | count |", + "+---------------------+-------+", + "| 1970-01-01T00:00:00 | 2 |", + "| 1970-01-01T01:00:00 | 2 |", + "| 1970-01-01T02:00:00 | 2 |", + "+---------------------+-------+", + ], + &batches + ); + Ok(()) +} + +#[tokio::test] +async fn crossing_day_groups_keep_shuffle_through_partial_aggregate() -> Result<()> { + let hour = 3_600_000_000_000; + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, "day")?; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + + assert_plan!( + optimized, + @r" + AggregateExec: mode=FinalPartitioned, gby=[bucket@0 as bucket], aggr=[count] + RepartitionExec: partitioning=Hash([bucket@0], 2), input_partitions=2 + AggregateExec: mode=Partial, gby=[date_trunc(day, timestamp@0) as bucket], aggr=[count] + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_batches_sorted_eq!( + [ + "+---------------------+-------+", + "| bucket | count |", + "+---------------------+-------+", + "| 1970-01-01T00:00:00 | 6 |", + "+---------------------+-------+", + ], + &batches + ); + Ok(()) +} + +#[test] +fn range_transform_aggregate_optimization_is_idempotent() -> Result<()> { + let hour = 3_600_000_000_000; + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + + for precision in ["hour", "day"] { + let input = timestamp_range_input(hour)?; + let aggregate = partial_final_date_trunc_count(input, precision)?; + let optimized = EnsureRequirements::new().optimize(aggregate, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + let reoptimized = + EnsureRequirements::new().optimize(Arc::clone(&optimized), &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&reoptimized), &config)?; + assert_plan!(optimized, reoptimized); + } + Ok(()) +} + +#[tokio::test] +async fn range_transform_locality_does_not_satisfy_join_distribution() -> Result<()> { + let hour = 3_600_000_000_000; + let left = timestamp_range_input(hour)?; + let right = timestamp_range_input(2 * hour)?; + let left_key = date_trunc_expr(left.as_ref(), "hour")?; + let right_key = date_trunc_expr(right.as_ref(), "hour")?; + for (input, key) in [(&left, &left_key), (&right, &right_key)] { + assert!(input.output_partitioning().keeps_keys_local( + &[Arc::clone(key)], + input.equivalence_properties(), + false, + )); + } + let join = + hash_join_exec(left, right, &vec![(left_key, right_key)], &JoinType::Inner); + let config = TestConfig::default() + .with_query_execution_partitions(2) + .config; + let optimized = EnsureRequirements::new().optimize(join, &config)?; + SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config)?; + assert_plan!( + optimized, + @r" + HashJoinExec: mode=Partitioned, join_type=Inner, on=[(date_trunc(hour, timestamp@0), date_trunc(hour, timestamp@0))] + RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2 + DataSourceExec: partitions=2, partition_sizes=[1, 1] + RepartitionExec: partitioning=Hash([date_trunc(hour, timestamp@0)], 2), input_partitions=2 + DataSourceExec: partitions=2, partition_sizes=[1, 1] + " + ); + + let batches = collect(optimized, SessionContext::new().task_ctx()).await?; + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 3 * 2 * 2, + "three hour buckets, each joining two left rows with two right rows" + ); + Ok(()) +} + #[test] fn range_grouping_set_aggregate_rehashes_with_grouping_id() -> Result<()> { let input = parquet_exec_with_output_partitioning(range_partitioning( diff --git a/datafusion/core/tests/sql/aggregates/nested_nullability.rs b/datafusion/core/tests/sql/aggregates/nested_nullability.rs index 3677bf4281ed8..ceb98263f8975 100644 --- a/datafusion/core/tests/sql/aggregates/nested_nullability.rs +++ b/datafusion/core/tests/sql/aggregates/nested_nullability.rs @@ -235,7 +235,8 @@ async fn array_agg_struct_from_stricter_batches_with_spilling() -> Result<()> { #[tokio::test] async fn array_agg_distinct_struct_from_stricter_batches_with_spilling() -> Result<()> { AggregateBatchesTest::new() - .with_num_rows(10_000) + // Enough groups that even the compact native accumulator exceeds the limit + .with_num_rows(100_000) // One partition keeps the native aggregate's memory pressure deterministic. .with_target_partitions(1) .without_single_distinct_to_groupby() diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index b048a23eea896..3a1766869d319 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -660,6 +660,9 @@ impl ParquetOpenState { } ParquetOpenState::PruneWithStatistics(prepared) => { let mut prepared_row_groups = (*prepared).prune_row_groups()?; + if prepared_row_groups.row_groups.is_fully_pruned() { + return Ok(ParquetOpenState::Done); + } if prepared_row_groups.should_load_page_index()? { Ok(ParquetOpenState::LoadPageIndex( prepared_row_groups.load_page_index().boxed(), @@ -690,9 +693,13 @@ impl ParquetOpenState { ParquetOpenState::LoadBloomFilters(future) => { Ok(ParquetOpenState::LoadBloomFilters(future)) } - ParquetOpenState::PruneWithBloomFilters(loaded) => Ok( - ParquetOpenState::BuildStream(Box::new(loaded.prune_bloom_filters())), - ), + ParquetOpenState::PruneWithBloomFilters(loaded) => { + let pruned = loaded.prune_bloom_filters(); + if pruned.row_groups.is_fully_pruned() { + return Ok(ParquetOpenState::Done); + } + Ok(ParquetOpenState::BuildStream(Box::new(pruned))) + } ParquetOpenState::BuildStream(prepared) => { Ok(ParquetOpenState::Ready(prepared.build_stream()?)) } @@ -2690,6 +2697,26 @@ mod test { } } + /// Drives a [`ParquetMorselizer`] to completion and returns how many + /// morsels it produced. + async fn count_morsels( + morselizer: &dyn Morselizer, + file: PartitionedFile, + ) -> Result { + let mut planners = VecDeque::from([morselizer.plan_file(file)?]); + let mut morsels = 0; + while let Some(planner) = planners.pop_front() { + if let Some(mut plan) = planner.plan()? { + morsels += plan.take_morsels().len(); + planners.extend(plan.take_ready_planners()); + if let Some(pending_planner) = plan.take_pending_planner() { + planners.push_front(pending_planner.await?); + } + } + } + Ok(morsels) + } + fn constant_int_stats() -> (Statistics, SchemaRef) { let schema = Arc::new(Schema::new(vec![ Field::new("a", DataType::Int32, false), @@ -3234,6 +3261,47 @@ mod test { assert_eq!(num_rows, 0); } + #[tokio::test] + async fn test_no_morsel_when_every_row_group_is_pruned() { + let store = Arc::new(InMemory::new()) as Arc; + let batches = vec![ + record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(), + record_batch!(("a", Int32, vec![0, 2, 4])).unwrap(), + ]; + let schema = batches[0].schema(); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(3)) + .set_bloom_filter_enabled(true) + .set_statistics_enabled(EnabledStatistics::Chunk) + .build(); + let data_size = write_parquet_batches( + Arc::clone(&store), + "test.parquet", + batches, + Some(props), + ) + .await; + let file = PartitionedFile::new("test.parquet".to_string(), data_size as u64); + + let morsels = |expr: Expr| { + let morselizer = ParquetMorselizerBuilder::new() + .with_store(Arc::clone(&store)) + .with_schema(Arc::clone(&schema)) + .with_predicate(logical2physical(&expr, &schema)) + .with_row_group_stats_pruning(true) + .with_enable_bloom_filter(true) + .build(); + let file = file.clone(); + async move { count_morsels(&morselizer, file).await.unwrap() } + }; + + // Statistics prune both row groups. + assert_eq!(morsels(col("a").gt(lit(10))).await, 0); + // Statistics keep both row groups; their Bloom filters prune both. + assert_eq!(morsels(col("a").eq(lit(1))).await, 0); + assert_eq!(morsels(col("a").eq(lit(2))).await, 1); + } + #[tokio::test] async fn test_prune_on_partition_statistics_with_dynamic_expression() { let store = Arc::new(InMemory::new()) as Arc; diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index 02fc2700d1228..feba768955e18 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -99,6 +99,12 @@ impl RowGroupAccessPlanFilter { self.access_plan.is_empty() } + /// Return true if no row groups remain to be scanned, either because the + /// file has none or because pruning has skipped every one of them. + pub fn is_fully_pruned(&self) -> bool { + self.access_plan.row_group_index_iter().next().is_none() + } + /// Return the number of row groups that are currently expected to be scanned pub fn remaining_row_group_count(&self) -> usize { self.access_plan.row_group_index_iter().count() @@ -706,6 +712,25 @@ mod tests { assert_eq!(filter.remaining_row_group_count(), 2); } + #[test] + fn is_fully_pruned_only_when_every_row_group_is_skipped() { + let mut filter = RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(2)); + assert!(!filter.is_fully_pruned()); + + filter.access_plan.skip(0); + assert!(!filter.is_fully_pruned()); + + filter.access_plan.skip(1); + assert!(filter.is_fully_pruned()); + // Skipped row groups are still part of the plan + assert!(!filter.is_empty()); + + assert!( + RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(0)) + .is_fully_pruned() + ); + } + /// A row group with one column chunk of `size` bytes at the given offsets fn row_group_at( dictionary_page_offset: Option, diff --git a/datafusion/datasource/src/file_groups.rs b/datafusion/datasource/src/file_groups.rs index 84594be54b504..73cf4848465be 100644 --- a/datafusion/datasource/src/file_groups.rs +++ b/datafusion/datasource/src/file_groups.rs @@ -19,8 +19,8 @@ use crate::{FileRange, PartitionedFile}; use arrow::compute::SortOptions; -use datafusion_common::Statistics; use datafusion_common::utils::compare_rows; +use datafusion_common::{ScalarValue, SplitPoint, Statistics}; use itertools::Itertools; use std::cmp::{Ordering, min}; use std::collections::{BinaryHeap, HashMap}; @@ -488,19 +488,33 @@ impl FileGroup { /// /// Note: May return fewer groups than `max_target_partitions` when the /// number of unique partition values is less than the target. - #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key + /// + /// Wrapper around [`Self::group_by_partition_values_with_split_points`] that discards + /// the split points. pub fn group_by_partition_values( self, max_target_partitions: usize, ) -> Vec { + self.group_by_partition_values_with_split_points(max_target_partitions) + .0 + } + + /// Groups files by partition value and returns the [`SplitPoint`]s between groups. + /// + /// Values are sorted with [`SortOptions::default()`] and cut into at most + /// `max_target_partitions` contiguous chunks so routing a file's `partition_values` + /// through the split points yields the group it was placed in. + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] + pub fn group_by_partition_values_with_split_points( + self, + max_target_partitions: usize, + ) -> (Vec, Vec) { if self.is_empty() || max_target_partitions == 0 { - return vec![]; + return (vec![], vec![]); } - let mut partition_groups: HashMap< - Vec, - Vec, - > = HashMap::new(); + let mut partition_groups: HashMap, Vec> = + HashMap::new(); for file in self.files { partition_groups @@ -511,7 +525,9 @@ impl FileGroup { let num_unique_partitions = partition_groups.len(); - // Sort for deterministic bucket assignment across query executions. + // Sort for deterministic bucket assignment across query executions. Must match + // the ordering the caller declares, otherwise the split points would not + // describe the groups. let mut sorted_partitions: Vec<_> = partition_groups.into_iter().collect(); let sort_options = vec![ @@ -522,24 +538,31 @@ impl FileGroup { compare_rows(&a.0, &b.0, &sort_options).unwrap_or(Ordering::Equal) }); - if num_unique_partitions <= max_target_partitions { - sorted_partitions - .into_iter() - .map(|(_, files)| FileGroup::new(files)) - .collect() - } else { - // Merge into max_target_partitions buckets using round-robin. - // This maintains grouping by partition value as we are merging groups which already - // contain all values for a partition key. - let mut target_groups = vec![vec![]; max_target_partitions]; - - for (idx, (_, files)) in sorted_partitions.into_iter().enumerate() { - let bucket = idx % max_target_partitions; - target_groups[bucket].extend(files); + let bucket_count = min(num_unique_partitions, max_target_partitions); + let mut groups = Vec::with_capacity(bucket_count); + let mut split_points = Vec::with_capacity(bucket_count.saturating_sub(1)); + + // Every chunk is non-empty because `bucket_count <= num_unique_partitions`, so + // its first tuple is a well defined lower bound. + let mut iter = sorted_partitions.into_iter(); + for bucket in 0..bucket_count { + let start = bucket * num_unique_partitions / bucket_count; + let end = (bucket + 1) * num_unique_partitions / bucket_count; + let mut files = Vec::new(); + for offset in start..end { + let (values, group_files) = iter + .next() + .expect("contiguous chunking never runs past the sorted partitions"); + if offset == start && bucket > 0 { + split_points.push(SplitPoint::new(values)); + } + files.extend(group_files); } - - target_groups.into_iter().map(FileGroup::new).collect() + groups.push(FileGroup::new(files)); } + debug_assert_eq!(split_points.len() + 1, groups.len()); + + (groups, split_points) } } @@ -632,7 +655,9 @@ impl DerefMut for CompareByRangeSize { #[cfg(test)] mod test { use super::*; - use datafusion_common::ScalarValue; + use datafusion_physical_expr::RangePartitioning; + use datafusion_physical_expr::expressions::Column; + use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; /// Empty file won't get partitioned #[test] @@ -1276,8 +1301,8 @@ mod test { #[test] fn test_group_by_partition_values_more_groups_than_target() { - // Each file has a single partition value. The number of partition values > max_target_partitions, so - // they should be round-robin distributed into groups. + // More values than `max_target_partitions`, so they are cut into contiguous + // chunks whose sizes differ by at most one. let fg = FileGroup::new(vec![ pfile_with_pv("a", "p1"), pfile_with_pv("b", "p2"), @@ -1287,8 +1312,153 @@ mod test { ]); let groups = fg.group_by_partition_values(3); assert_eq!(groups.len(), 3); - assert_eq!(groups[0].len(), 2); + assert_eq!(groups[0].len(), 1); assert_eq!(groups[1].len(), 2); - assert_eq!(groups[2].len(), 1); + assert_eq!(groups[2].len(), 2); + // Groups must be contiguous key intervals in sorted order. + assert_eq!(group_pvs(&groups[0]), vec!["p1"]); + assert_eq!(group_pvs(&groups[1]), vec!["p2", "p3"]); + assert_eq!(group_pvs(&groups[2]), vec!["p4", "p5"]); + } + + fn group_pvs(group: &FileGroup) -> Vec { + group + .iter() + .map(|f| f.partition_values[0].to_string()) + .collect() + } + + /// Mirrors `RangeRouter` semantics: the partition index is the number of split points + /// that are `<=` the key under the given sort options. + fn range_route(key: &[ScalarValue], split_points: &[SplitPoint]) -> usize { + let sort_options = vec![SortOptions::default(); key.len()]; + split_points + .iter() + .take_while(|sp| { + compare_rows(sp.values(), key, &sort_options).unwrap() + != Ordering::Greater + }) + .count() + } + + #[test] + fn test_group_by_partition_values_split_points_describe_groups() { + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p2"), + pfile_with_pv("c", "p3"), + pfile_with_pv("d", "p4"), + pfile_with_pv("e", "p5"), + pfile_with_pv("f", "p5"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(3); + assert_eq!(groups.len(), 3); + assert_eq!(split_points.len(), 2); + assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]); + assert_eq!(split_points[1].values(), &[ScalarValue::from("p4")]); + + // Every file routes (RangeRouter semantics) to the group it was placed in. + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_points_fewer_values_than_target() { + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p1"), + pfile_with_pv("c", "p2"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(4); + assert_eq!(groups.len(), 2); + assert_eq!(split_points.len(), 1); + assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_points_single_target_partition() { + // One partition holds every value, so there is no boundary to describe. + let fg = FileGroup::new(vec![ + pfile_with_pv("a", "p1"), + pfile_with_pv("b", "p2"), + pfile_with_pv("c", "p3"), + pfile_with_pv("d", "p3"), + ]); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(1); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].len(), 4); + assert!(split_points.is_empty()); + } + + fn pfile_with_pvs(path: &str, pvs: &[Option<&str>]) -> PartitionedFile { + let mut file = pfile(path, 10); + file.partition_values = pvs + .iter() + .map(|v| ScalarValue::Utf8(v.map(str::to_string))) + .collect(); + file + } + + /// Two partition columns with NULLs in either position. + fn compound_key_file_group() -> FileGroup { + FileGroup::new(vec![ + pfile_with_pvs("a", &[Some("2026"), Some("09")]), + pfile_with_pvs("b", &[Some("2026"), Some("10")]), + pfile_with_pvs("c", &[None, Some("09")]), + pfile_with_pvs("d", &[Some("2025"), None]), + pfile_with_pvs("e", &[Some("2025"), Some("12")]), + ]) + } + + #[test] + fn test_group_by_partition_values_split_points_null_and_compound_keys() { + let fg = compound_key_file_group(); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(2); + assert_eq!(groups.len(), 2); + assert_eq!(split_points.len(), 1); + assert_eq!( + split_points[0].values(), + &[ScalarValue::from("2025"), ScalarValue::from("12")] + ); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + } + + #[test] + fn test_group_by_partition_values_split_point_containing_null() { + let fg = compound_key_file_group(); + let (groups, split_points) = fg.group_by_partition_values_with_split_points(3); + assert_eq!(groups.len(), 3); + assert_eq!( + split_points[0].values(), + &[ScalarValue::from("2025"), ScalarValue::Utf8(None)] + ); + assert_eq!( + split_points[1].values(), + &[ScalarValue::from("2026"), ScalarValue::from("09")] + ); + for (idx, group) in groups.iter().enumerate() { + for file in group.iter() { + assert_eq!(range_route(&file.partition_values, &split_points), idx); + } + } + let ordering = LexOrdering::new(vec![ + PhysicalSortExpr::new_default(Arc::new(Column::new("year", 0))), + PhysicalSortExpr::new_default(Arc::new(Column::new("month", 1))), + ]) + .unwrap(); + let range = RangePartitioning::try_new(ordering, split_points).unwrap(); + assert_eq!(range.partition_count(), 3); } } diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index 07144d8191a76..64b12951efc8e 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -39,8 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ - Constraint, Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, - internal_err, + Constraint, Constraints, Result, ScalarValue, SplitPoint, Statistics, + internal_datafusion_err, internal_err, }; use datafusion_execution::{ SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl, @@ -52,7 +52,9 @@ use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_physical_expr::expressions::{BinaryExpr, Column}; use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping}; use datafusion_physical_expr::utils::reassign_expr_columns; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction}; +use datafusion_physical_expr::{ + EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction, +}; use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory; use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile}; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; @@ -613,15 +615,52 @@ impl From for FileScanConfigBuilder { } } -/// Builds output partitioning over `partition_cols` (resolved to their indices in -/// `schema`) with `partition_count` partitions. Returns `None` when there are no -/// partition columns. Callers use this to declare the output partitioning of a scan -/// whose file groups are organized by partition column values. +/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in +/// `schema`). Returns `None` when there are no partition columns. +#[deprecated( + since = "56.0.0", + note = "Use `output_range_partitioning_from_split_points` instead" +)] pub fn output_partitioning_from_partition_fields( schema: &Schema, partition_cols: &Fields, partition_count: usize, ) -> Option { + let exprs = partition_column_exprs(schema, partition_cols)?; + Some(Partitioning::Hash(exprs, partition_count)) +} + +/// Builds the [`Partitioning::Range`] for file groups from +/// [`FileGroup::group_by_partition_values_with_split_points`], each column sorted with +/// `SortOptions::default()` to match how the groups were cut. Returns `None` when there +/// are no partition columns or one is missing from `schema`, and errors when +/// `split_points` are not strictly ordered. Split point values must have the partition +/// column types in `schema`. +pub fn output_range_partitioning_from_split_points( + schema: &Schema, + partition_cols: &Fields, + split_points: Vec, +) -> Result> { + let Some(exprs) = partition_column_exprs(schema, partition_cols) else { + return Ok(None); + }; + let sort_exprs = exprs + .into_iter() + .map(PhysicalSortExpr::new_default) + .collect::>(); + let Some(ordering) = LexOrdering::new(sort_exprs) else { + return Ok(None); + }; + let range = RangePartitioning::try_new(ordering, split_points)?; + Ok(Some(Partitioning::Range(range))) +} + +/// Resolves `partition_cols` to `Column` expressions in `schema`, or `None` when there +/// are no partition columns or one is missing. +fn partition_column_exprs( + schema: &Schema, + partition_cols: &Fields, +) -> Option>> { if partition_cols.is_empty() { return None; } @@ -635,8 +674,7 @@ pub fn output_partitioning_from_partition_fields( .position(|field| field.name() == name)?; exprs.push(Arc::new(Column::new(name, idx))); } - - Some(Partitioning::Hash(exprs, partition_count)) + Some(exprs) } fn project_output_partitioning( @@ -3150,6 +3188,14 @@ mod tests { let partitioning = config.output_partitioning(); assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_))); + + let partitioning = output_range_partitioning_from_split_points( + config.file_source.table_schema().table_schema(), + config.table_partition_cols(), + vec![], + ) + .unwrap(); + assert!(partitioning.is_none()); } #[test] @@ -3174,22 +3220,44 @@ mod tests { FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]), FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]), ]; - config.output_partitioning = output_partitioning_from_partition_fields( + let dict = |v: &str| wrap_partition_value_in_dict(ScalarValue::from(v)); + config.output_partitioning = output_range_partitioning_from_split_points( config.file_source.table_schema().table_schema(), config.table_partition_cols(), - config.file_groups.len(), - ); + vec![ + SplitPoint::new(vec![dict("2026-09")]), + SplitPoint::new(vec![dict("2026-10")]), + ], + ) + .unwrap(); let partitioning = config.output_partitioning(); match partitioning { - Partitioning::Hash(exprs, num_partitions) => { - assert_eq!(num_partitions, 3); - assert_eq!(exprs.len(), 1); - assert_eq!(exprs[0].downcast_ref::().unwrap().name(), "date"); + Partitioning::Range(range) => { + assert_eq!(range.partition_count(), 3); + assert_eq!(range.ordering().len(), 1); + let sort_expr = &range.ordering()[0]; + assert_eq!( + sort_expr.expr.downcast_ref::().unwrap().name(), + "date" + ); + assert_eq!(sort_expr.options, arrow::compute::SortOptions::default()); + assert_eq!(range.split_points().len(), 2); } - _ => panic!("Expected Hash partitioning"), + other => panic!("Expected Range partitioning, got {other}"), } + let err = output_range_partitioning_from_split_points( + config.file_source.table_schema().table_schema(), + config.table_partition_cols(), + vec![ + SplitPoint::new(vec![dict("2026-10")]), + SplitPoint::new(vec![dict("2026-09")]), + ], + ) + .unwrap_err(); + assert!(err.to_string().contains("strictly ordered"), "{err}"); + // Test multiple partition columns let multiple_partition_cols = vec![ Field::new("year", wrap_partition_type_in_dict(DataType::Utf8), false), @@ -3206,24 +3274,25 @@ mod tests { FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]), FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]), ]; - config.output_partitioning = output_partitioning_from_partition_fields( + config.output_partitioning = output_range_partitioning_from_split_points( config.file_source.table_schema().table_schema(), config.table_partition_cols(), - config.file_groups.len(), - ); + vec![SplitPoint::new(vec![dict("2026"), dict("09")])], + ) + .unwrap(); let partitioning = config.output_partitioning(); match partitioning { - Partitioning::Hash(exprs, num_partitions) => { - assert_eq!(num_partitions, 2); - assert_eq!(exprs.len(), 2); - let col_names: Vec<_> = exprs + Partitioning::Range(range) => { + assert_eq!(range.partition_count(), 2); + let col_names: Vec<_> = range + .ordering() .iter() - .map(|e| e.downcast_ref::().unwrap().name()) + .map(|e| e.expr.downcast_ref::().unwrap().name()) .collect(); assert_eq!(col_names, vec!["year", "month"]); } - _ => panic!("Expected Hash partitioning"), + other => panic!("Expected Range partitioning, got {other}"), } } diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index d937a54029398..f900564d453b6 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -364,6 +364,15 @@ impl ScalarUDF { self.inner.output_ordering(inputs) } + /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for more details. + pub fn supports_range_key_locality_analysis( + &self, + argument_types: &[DataType], + ) -> bool { + self.inner + .supports_range_key_locality_analysis(argument_types) + } + pub fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { self.inner.preserves_lex_ordering(inputs) } @@ -983,6 +992,29 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { Ok(inner(inputs)) } + /// Returns whether this function and argument-type combination has been + /// audited for range-partitioning locality analysis. + /// + /// Range-boundary analysis has subtle correctness requirements, so the + /// policy is default-deny. Functions may opt in only after individual audit. + /// At minimum, an opted-in combination must: + /// + /// - have same-direction [`SortProperties::Ordered`] metadata over the + /// admitted non-null domain when all other inputs are singleton; + /// - preserve nullness exactly, so successful evaluation returns null if and + /// only if the range input is null. + /// + /// Callers may then prove key locality by evaluating values on both sides of + /// each concrete range split. Ordered metadata alone is insufficient because + /// a non-strict transform can collapse a split boundary. + /// + /// This does not claim a concrete partition layout or cross-input + /// co-partitioning compatibility. Boundary evaluation errors cause the + /// analysis to return `false`. This capability is not transported across FFI. + fn supports_range_key_locality_analysis(&self, _argument_types: &[DataType]) -> bool { + false + } + /// Returns true if the function preserves lexicographical ordering based on /// the input ordering. /// @@ -1205,6 +1237,11 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl { self.inner.output_ordering(inputs) } + fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool { + self.inner + .supports_range_key_locality_analysis(argument_types) + } + fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { self.inner.preserves_lex_ordering(inputs) } @@ -1287,6 +1324,12 @@ mod tests { assert_eq!(b.partial_cmp(&o), Some(Ordering::Less)); } + #[test] + fn range_key_locality_analysis_is_disabled_by_default() { + let function = test_func("range_function", "a"); + assert!(!function.supports_range_key_locality_analysis(&[DataType::Int64])); + } + fn test_func(name: &'static str, parameter: &'static str) -> ScalarUDF { ScalarUDF::from(TestScalarUDFImpl { name, diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index 4fcb2b32f2ace..2d92222561dfe 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -25,10 +25,11 @@ use arrow::array::{ use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, Schema}; use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use datafusion_expr::function::AccumulatorArgs; -use datafusion_expr::{Accumulator, AggregateUDFImpl}; +use datafusion_expr::{Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator}; use datafusion_functions_aggregate::array_agg::{ ArrayAgg, ArrayAggAccumulator, DistinctArrayAggAccumulator, }; +use datafusion_functions_aggregate_common::aggregate::groups_accumulator::GroupsAccumulatorAdapter; use datafusion_physical_expr::{PhysicalSortExpr, expressions::col}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -323,6 +324,113 @@ fn distinct_array_agg_benchmark(c: &mut Criterion) { ); } +/// `ARRAY_AGG(DISTINCT x) ... GROUP BY g` over one 8 192-row Utf8 batch, with +/// the groups accumulator against the previous per-group path +/// (`GroupsAccumulatorAdapter` over one `DistinctArrayAggAccumulator` per group), +/// across group counts, distinct cardinalities and partial emission. +fn distinct_array_agg_groups_benchmark(c: &mut Criterion) { + let schema = Schema::new(vec![Field::new("value", DataType::Utf8, true)]); + let value_expr = col("value", &schema).unwrap(); + let value_field = value_expr.return_field(&schema).unwrap(); + let return_field: FieldRef = Field::new( + "array_agg", + DataType::List(Field::new_list_field(DataType::Utf8, true).into()), + true, + ) + .into(); + let args = AccumulatorArgs { + return_field, + schema: &schema, + expr_fields: std::slice::from_ref(&value_field), + ignore_nulls: false, + order_bys: &[], + is_reversed: false, + name: "array_agg(DISTINCT value)", + is_distinct: true, + exprs: std::slice::from_ref(&value_expr), + }; + let array_agg = ArrayAgg::default(); + + let inputs = [ + ( + "low cardinality (~25 distinct)", + Arc::new(create_string_array_low_cardinality(8192, 0.0)) as ArrayRef, + ), + ( + "high cardinality (~7800 distinct)", + Arc::new(create_string_array_high_cardinality(8192, 0.05, 0.0)) as ArrayRef, + ), + ]; + for num_groups in [100, 1000, 8192] { + let mut rng = StdRng::seed_from_u64(42); + let group_indices = (0..8192) + .map(|_| rng.random_range(0..num_groups)) + .collect::>(); + for (input_name, values) in &inputs { + // Emits every group at once, or in eight `EmitTo::First` chunks, which + // makes the accumulator rebuild its state after each partial emission. + for emit_chunks in [1, 8] { + let run = |acc: &mut dyn GroupsAccumulator| { + acc.update_batch( + std::slice::from_ref(values), + &group_indices, + None, + num_groups, + ) + .unwrap(); + if emit_chunks == 1 { + return vec![acc.evaluate(EmitTo::All).unwrap()]; + } + let chunk = num_groups.div_ceil(emit_chunks); + let mut remaining = num_groups; + let mut emitted = Vec::with_capacity(emit_chunks); + while remaining > 0 { + let n = chunk.min(remaining); + emitted.push(acc.evaluate(EmitTo::First(n)).unwrap()); + remaining -= n; + } + emitted + }; + let emit = if emit_chunks == 1 { + "emit all".to_string() + } else { + format!("emit in {emit_chunks} chunks") + }; + c.bench_function( + &format!( + "distinct_array_agg groups {num_groups} groups, {input_name}, {emit}" + ), + |b| { + b.iter(|| { + let mut acc = + array_agg.create_groups_accumulator(args.clone()).unwrap(); + run(acc.as_mut()) + }) + }, + ); + c.bench_function( + &format!( + "distinct_array_agg adapter {num_groups} groups, {input_name}, {emit}" + ), + |b| { + b.iter(|| { + let factory = || { + Ok(Box::new(DistinctArrayAggAccumulator::try_new( + &DataType::Utf8, + None, + false, + )?) as Box) + }; + let mut acc = GroupsAccumulatorAdapter::new(factory); + run(&mut acc) + }) + }, + ); + } + } + } +} + /// Precomputes the schema, physical expressions, sort expression, and aggregate /// metadata so each benchmark iteration can focus on accumulator operations. struct OrderedArrayAggBenchFixture { @@ -545,6 +653,7 @@ criterion_group!( benches, array_agg_benchmark, distinct_array_agg_benchmark, + distinct_array_agg_groups_benchmark, ordered_array_agg_benchmark, ordered_array_agg_wide_utf8_benchmark ); diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index f81a60c222e41..9c27b8b249f33 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -18,6 +18,7 @@ //! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`] use std::collections::VecDeque; +use std::hash::BuildHasher; use std::mem::{size_of, size_of_val, take}; use std::ops::Range; use std::sync::Arc; @@ -27,7 +28,7 @@ use arrow::array::{ UInt32Array, make_array, new_empty_array, }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; -use arrow::compute::{SortOptions, cast, filter}; +use arrow::compute::{SortOptions, cast, filter, take as take_array}; use arrow::datatypes::{DataType, Field, FieldRef, Fields}; use arrow::row::{OwnedRow, Row, RowConverter, Rows, SortField}; use arrow_select::dictionary::garbage_collect_any_dictionary; @@ -35,10 +36,12 @@ use arrow_select::dictionary::garbage_collect_any_dictionary; use datafusion_common::cast::as_list_array; use datafusion_common::hash_utils::{RandomState, create_hashes}; use datafusion_common::scalar::copy_array_data; +use datafusion_common::utils::memory::estimate_memory_size; use datafusion_common::utils::proxy::HashTableAllocExt; use datafusion_common::utils::{SingleRowListArrayBuilder, offset_span}; use datafusion_common::{ - Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err, + HashMap, HashSet, Result, ScalarValue, assert_eq_or_internal_err, exec_err, + internal_datafusion_err, internal_err, }; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; @@ -177,40 +180,9 @@ impl AggregateUDFImpl for ArrayAgg { let ignore_nulls = acc_args.ignore_nulls && field.is_nullable(); if acc_args.is_distinct { - // Limitation similar to Postgres. The aggregation function can only mix - // DISTINCT and ORDER BY if all the expressions in the ORDER BY appear - // also in the arguments of the function. This implies that if the - // aggregation function only accepts one argument, only one argument - // can be used in the ORDER BY, For example: - // - // ARRAY_AGG(DISTINCT col) - // - // can only be mixed with an ORDER BY if the order expression is "col". - // - // ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid - // ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid - // ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid - // ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid - let sort_option = match acc_args.order_bys { - // This accumulator sorts the values itself, so a reversed - // aggregate must still sort them in the requested order. - [single] if single.expr.eq(&acc_args.exprs[0]) => { - Some(if acc_args.is_reversed { - !single.options - } else { - single.options - }) - } - [] => None, - _ => { - return exec_err!( - "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list" - ); - } - }; return Ok(Box::new(DistinctArrayAggAccumulator::try_new( data_type, - sort_option, + distinct_sort_options(&acc_args)?, ignore_nulls, )?)); } @@ -243,7 +215,11 @@ impl AggregateUDFImpl for ArrayAgg { } fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool { - !args.is_distinct && args.order_bys.is_empty() + if args.is_distinct { + distinct_sort_options(&args).is_ok() + } else { + args.order_bys.is_empty() + } } fn create_groups_accumulator( @@ -253,6 +229,13 @@ impl AggregateUDFImpl for ArrayAgg { let field = &args.expr_fields[0]; let data_type = field.data_type().clone(); let ignore_nulls = args.ignore_nulls && field.is_nullable(); + if args.is_distinct { + return Ok(Box::new(DistinctArrayAggGroupsAccumulator::new( + data_type, + distinct_sort_options(&args)?, + ignore_nulls, + ))); + } Ok(Box::new(ArrayAggGroupsAccumulator::new( data_type, ignore_nulls, @@ -347,6 +330,42 @@ impl ArrayAggAccumulator { } } +/// Returns the sort options for `ARRAY_AGG(DISTINCT ...)`, or an error if the +/// ORDER BY cannot be combined with DISTINCT. +/// +/// Limitation similar to Postgres. The aggregation function can only mix +/// DISTINCT and ORDER BY if all the expressions in the ORDER BY appear +/// also in the arguments of the function. This implies that if the +/// aggregation function only accepts one argument, only one argument +/// can be used in the ORDER BY, For example: +/// +/// ARRAY_AGG(DISTINCT col) +/// +/// can only be mixed with an ORDER BY if the order expression is "col". +/// +/// ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid +/// ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid +/// ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid +/// ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid +/// +/// The DISTINCT accumulators sort the values themselves, so a reversed +/// aggregate must still sort them in the requested order. +fn distinct_sort_options(acc_args: &AccumulatorArgs) -> Result> { + match acc_args.order_bys { + [single] if single.expr.eq(&acc_args.exprs[0]) => { + Ok(Some(if acc_args.is_reversed { + !single.options + } else { + single.options + })) + } + [] => Ok(None), + _ => exec_err!( + "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list" + ), + } +} + /// Returns the mask of non-null rows in `values` when `ignore_nulls` is set /// and some row is null. fn non_null_mask(values: &dyn Array, ignore_nulls: bool) -> Option { @@ -803,27 +822,12 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { ) -> Result> { assert_eq!(values.len(), 1, "one argument to convert_to_state"); - let input = &values[0]; - - // Each row becomes a 1-element list: offsets are [0, 1, 2, ..., n]. - let offsets = OffsetBuffer::from_repeated_length(1, input.len()); - - // Filtered rows become null list entries, which merge_batch will skip. - let filter_nulls = opt_filter.map(filter_to_nulls); - - // With ignore_nulls, null values also become null list entries. Without - // ignore_nulls, null values stay as [NULL] so merge_batch retains them. - let nulls = if self.ignore_nulls { - let logical = input.logical_nulls(); - NullBuffer::union(filter_nulls.as_ref(), logical.as_ref()) - } else { - filter_nulls - }; - - let field = Arc::new(Field::new_list_field(self.datatype.clone(), true)); - let list_array = ListArray::new(field, offsets, Arc::clone(input), nulls); - - Ok(vec![Arc::new(list_array)]) + Ok(vec![single_value_lists( + &values[0], + opt_filter, + self.ignore_nulls, + &self.datatype, + )]) } fn size(&self) -> usize { self.batches @@ -840,6 +844,419 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { } } +/// Converts each input row into a one-element list, the state shape of +/// `ARRAY_AGG`. +/// +/// Filtered rows become null list entries, which `merge_batch` skips. With +/// `ignore_nulls`, null values also become null list entries; without it they +/// stay as `[NULL]` so `merge_batch` retains them. +fn single_value_lists( + input: &ArrayRef, + opt_filter: Option<&BooleanArray>, + ignore_nulls: bool, + datatype: &DataType, +) -> ArrayRef { + // Offsets are [0, 1, 2, ..., n]. + let offsets = OffsetBuffer::from_repeated_length(1, input.len()); + let filter_nulls = opt_filter.map(filter_to_nulls); + let nulls = if ignore_nulls { + let logical = input.logical_nulls(); + NullBuffer::union(filter_nulls.as_ref(), logical.as_ref()) + } else { + filter_nulls + }; + let field = Arc::new(Field::new_list_field(datatype.clone(), true)); + Arc::new(ListArray::new(field, offsets, Arc::clone(input), nulls)) +} + +/// [`GroupsAccumulator`] for `ARRAY_AGG(DISTINCT x)`, optionally with +/// `ORDER BY x`. +/// +/// Distinct values are encoded once with a [`RowConverter`] and stored in a +/// single [`Rows`] buffer shared by all groups, so a value that appears in many +/// groups is held once. Each group records the values it has seen as +/// `(group_index, value_index)` pairs, deduplicated in one hash set rather than +/// one set per group. +#[derive(Debug)] +struct DistinctArrayAggGroupsAccumulator { + datatype: DataType, + sort_options: Option, + ignore_nulls: bool, + /// Encodes values to rows. Created on the first non-empty input, once the + /// runtime type is known. + converter: Option, + /// Every distinct value seen so far, indexed by value index. + values: Option, + /// `(hash, value_index)` for every row in `values`. + value_map: HashTable<(u64, u32)>, + /// Heap size of `value_map` in bytes. + value_map_size: usize, + /// Distinct `(group_index, value_index)` pairs, in first-seen order. + pairs: Vec<(u32, u32)>, + /// Deduplicates `pairs`. + pair_set: HashSet<(u32, u32), RandomState>, + /// Total number of groups tracked. + num_groups: usize, + random_state: RandomState, + /// Time spent deduplicating values, reported as the DISTINCT submetric. + distinct_metric: Option>, +} + +impl DistinctArrayAggGroupsAccumulator { + fn new( + datatype: DataType, + sort_options: Option, + ignore_nulls: bool, + ) -> Self { + Self { + datatype, + sort_options, + ignore_nulls, + converter: None, + values: None, + value_map: HashTable::new(), + value_map_size: 0, + pairs: Vec::new(), + pair_set: HashSet::default(), + num_groups: 0, + random_state: RandomState::default(), + distinct_metric: None, + } + } + + fn converter(&mut self, data_type: &DataType) -> Result<&RowConverter> { + if self.converter.is_none() { + let sort_field = match self.sort_options { + Some(opts) => SortField::new_with_options(data_type.clone(), opts), + None => SortField::new(data_type.clone()), + }; + let converter = RowConverter::new(vec![sort_field])?; + self.values = Some(converter.empty_rows(0, 0)); + self.converter = Some(converter); + } + self.converter + .as_ref() + .ok_or_else(|| internal_datafusion_err!("row converter must be set")) + } + + /// Like [`Self::insert`], but first gathers the selected rows when they + /// are only part of `input`, so rejected values are never encoded. + fn insert_selected( + &mut self, + input: &ArrayRef, + rows: Vec<(usize, usize)>, + ) -> Result<()> { + if rows.len() == input.len() { + return self.insert(input, rows); + } + let indices = UInt32Array::from_iter_values( + rows.iter().map(|&(_, row_index)| row_index as u32), + ); + let selected = take_array(input.as_ref(), &indices, None)?; + self.insert( + &selected, + rows.into_iter() + .enumerate() + .map(|(position, (group_index, _))| (group_index, position)), + ) + } + + /// Records that `group_index` contains each value of `input` selected by + /// `rows`, given as `(group_index, row_in_input)` pairs. + fn insert( + &mut self, + input: &ArrayRef, + rows: impl IntoIterator, + ) -> Result<()> { + let encoded = self + .converter(input.data_type())? + .convert_columns(std::slice::from_ref(input))?; + let Self { + values, + value_map, + value_map_size, + pairs, + pair_set, + random_state, + .. + } = self; + let values = values + .as_mut() + .ok_or_else(|| internal_datafusion_err!("value rows must be set"))?; + + for (group_index, row_index) in rows { + let row = encoded.row(row_index); + let hash = random_state.hash_one(row.as_ref()); + let value_index = match value_map.find(hash, |&(h, index)| { + h == hash && values.row(index as usize) == row + }) { + Some(&(_, index)) => index, + None => { + let index = values.num_rows() as u32; + values.push(row); + value_map.insert_accounted( + (hash, index), + |&(h, _)| h, + value_map_size, + ); + index + } + }; + let pair = (group_index as u32, value_index); + if pair_set.insert(pair) { + pairs.push(pair); + } + } + Ok(()) + } + + /// Drops the state of the first `emit_groups` groups and renumbers the + /// rest so they start from 0. Values only referenced by emitted groups are + /// dropped too. + fn retain_after(&mut self, emit_groups: usize) { + let emit_groups = emit_groups as u32; + let old_pairs = take(&mut self.pairs); + let old_values = self.values.take(); + self.value_map = HashTable::new(); + self.value_map_size = 0; + self.pair_set = HashSet::default(); + self.num_groups -= emit_groups as usize; + + let (Some(converter), Some(old_values)) = (&self.converter, old_values) else { + return; + }; + let mut values = converter.empty_rows(0, 0); + let mut remap: HashMap = HashMap::new(); + let mut pairs = Vec::new(); + for (group_index, value_index) in old_pairs { + if group_index < emit_groups { + continue; + } + let new_index = *remap.entry(value_index).or_insert_with(|| { + let row = old_values.row(value_index as usize); + let index = values.num_rows() as u32; + values.push(row); + let hash = self.random_state.hash_one(row.as_ref()); + self.value_map.insert_accounted( + (hash, index), + |&(h, _)| h, + &mut self.value_map_size, + ); + index + }); + let pair = (group_index - emit_groups, new_index); + self.pair_set.insert(pair); + pairs.push(pair); + } + self.pairs = pairs; + self.values = Some(values); + } +} + +impl GroupsAccumulator for DistinctArrayAggGroupsAccumulator { + fn update_batch( + &mut self, + values: &[ArrayRef], + group_indices: &[usize], + opt_filter: Option<&BooleanArray>, + total_num_groups: usize, + ) -> Result<()> { + // With `ORDER BY x` the ordering column is passed too. It is the + // argument itself, so only the first column is needed. + let Some(input) = values.first() else { + return internal_err!("array_agg(DISTINCT) expects an argument"); + }; + self.num_groups = self.num_groups.max(total_num_groups); + + let nulls = if self.ignore_nulls { + input.logical_nulls() + } else { + None + }; + let rows = group_indices + .iter() + .enumerate() + .filter(|&(row_index, _)| { + let filtered_out = opt_filter.is_some_and(|filter| { + filter.is_null(row_index) || !filter.value(row_index) + }); + let skipped_null = + nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index)); + !filtered_out && !skipped_null + }) + .map(|(row_index, &group_index)| (group_index, row_index)) + .collect::>(); + if rows.is_empty() { + return Ok(()); + } + let mut metric_recorder = + AggregateMetricRecorder::new(self.distinct_metric.clone()); + let _timer = metric_recorder.timer(); + self.insert_selected(input, rows) + } + + /// Produces a `ListArray` ordered by group index. Groups without values + /// are null, matching [`DistinctArrayAggAccumulator`]. + fn evaluate(&mut self, emit_to: EmitTo) -> Result { + let emit_groups = match emit_to { + EmitTo::All => self.num_groups, + EmitTo::First(n) => n, + }; + + // Counting sort of the pairs into group order. + let mut counts = vec![0u32; emit_groups]; + for &(group_index, _) in &self.pairs { + if let Some(count) = counts.get_mut(group_index as usize) { + *count += 1; + } + } + let mut offsets = Vec::::with_capacity(emit_groups + 1); + offsets.push(0); + let mut nulls = NullBufferBuilder::new(emit_groups); + let mut write_positions = Vec::with_capacity(emit_groups); + let mut total = 0u32; + for &count in &counts { + if count == 0 { + nulls.append_null(); + } else { + nulls.append_non_null(); + } + write_positions.push(total); + total += count; + offsets.push(total as i32); + } + let mut ordered = vec![0u32; total as usize]; + for &(group_index, value_index) in &self.pairs { + if let Some(position) = write_positions.get_mut(group_index as usize) { + ordered[*position as usize] = value_index; + *position += 1; + } + } + + let flat_values = match (&self.converter, &self.values) { + (Some(converter), Some(values)) if total > 0 => { + if self.sort_options.is_some() { + // The sort direction is encoded in the row bytes, so + // comparing rows gives the requested order. + for window in offsets.windows(2) { + ordered[window[0] as usize..window[1] as usize].sort_unstable_by( + |&a, &b| values.row(a as usize).cmp(&values.row(b as usize)), + ); + } + } + let decoded = converter + .convert_rows(ordered.iter().map(|&i| values.row(i as usize)))? + .swap_remove(0); + // `convert_rows` always returns the physical (non-dictionary) + // type, so cast back when the declared type has a dictionary. + if decoded.data_type() != &self.datatype + && datatype_contains_dictionary(&self.datatype) + { + cast(decoded.as_ref(), &self.datatype)? + } else { + decoded + } + } + _ => new_empty_array(&self.datatype), + }; + + match emit_to { + EmitTo::All => { + let converter = self.converter.take(); + let distinct_metric = self.distinct_metric.take(); + *self = Self::new( + self.datatype.clone(), + self.sort_options, + self.ignore_nulls, + ); + self.distinct_metric = distinct_metric; + // Keep the converter: the runtime type does not change. + if let Some(converter) = converter { + self.values = Some(converter.empty_rows(0, 0)); + self.converter = Some(converter); + } + } + EmitTo::First(_) => self.retain_after(emit_groups), + } + + let field = Arc::new(Field::new_list_field(self.datatype.clone(), true)); + Ok(Arc::new(ListArray::new( + field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + flat_values, + nulls.finish(), + ))) + } + + fn state(&mut self, emit_to: EmitTo) -> Result> { + Ok(vec![self.evaluate(emit_to)?]) + } + + fn merge_batch( + &mut self, + values: &[ArrayRef], + group_indices: &[usize], + total_num_groups: usize, + ) -> Result<()> { + assert_eq_or_internal_err!(values.len(), 1, "one argument to merge_batch"); + let lists = values[0].as_list::(); + self.num_groups = self.num_groups.max(total_num_groups); + + // A sliced list keeps its whole child array, so encode only the span + // its offsets cover. + let offsets = lists.offsets(); + let (span_start, span_len) = offset_span(offsets); + let child = lists.values().slice(span_start, span_len); + let rows = group_indices + .iter() + .enumerate() + .filter(|&(list_index, _)| lists.is_valid(list_index)) + .flat_map(|(list_index, &group_index)| { + let start = offsets[list_index] as usize - span_start; + let end = offsets[list_index + 1] as usize - span_start; + (start..end).map(move |position| (group_index, position)) + }) + .collect::>(); + if rows.is_empty() { + return Ok(()); + } + let mut metric_recorder = + AggregateMetricRecorder::new(self.distinct_metric.clone()); + let _timer = metric_recorder.timer(); + self.insert_selected(&child, rows) + } + + fn convert_to_state( + &self, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result> { + let Some(input) = values.first() else { + return internal_err!("array_agg(DISTINCT) expects an argument"); + }; + Ok(vec![single_value_lists( + input, + opt_filter, + self.ignore_nulls, + &self.datatype, + )]) + } + + fn set_metrics(&mut self, metrics: Arc) { + self.distinct_metric = Some(metrics.metric("distinct")); + } + + fn size(&self) -> usize { + size_of_val(self) + + self.converter.as_ref().map_or(0, RowConverter::size) + + self.values.as_ref().map_or(0, Rows::size) + + self.value_map_size + + self.pairs.capacity() * size_of::<(u32, u32)>() + + estimate_memory_size::<(u32, u32)>(self.pair_set.capacity(), 0) + .unwrap_or_default() + } +} + /// Resources that are allocated lazily on the first `update_batch` call, /// once the concrete runtime Arrow type is known. /// @@ -1791,8 +2208,8 @@ impl Accumulator for OrderSensitiveArrayAggAccumulator { #[cfg(test)] mod tests { use super::*; - use arrow::array::{Int32Builder, ListBuilder, StringBuilder}; - use arrow::datatypes::Schema; + use arrow::array::{DictionaryArray, Int32Builder, ListBuilder, StringBuilder}; + use arrow::datatypes::{Int32Type, Schema}; use datafusion_common::cast::as_generic_string_array; use datafusion_common::internal_err; use datafusion_physical_expr::PhysicalExpr; @@ -3417,13 +3834,291 @@ mod tests { } fn eval_i32_lists( - acc: &mut ArrayAggGroupsAccumulator, + acc: &mut dyn GroupsAccumulator, emit_to: EmitTo, ) -> Result>>>> { let result = acc.evaluate(emit_to)?; Ok(list_array_to_i32_vecs(result.as_list::())) } + fn distinct_groups_accumulator( + ignore_nulls: bool, + ) -> DistinctArrayAggGroupsAccumulator { + DistinctArrayAggGroupsAccumulator::new(DataType::Int32, None, ignore_nulls) + } + + #[test] + fn distinct_groups_accumulator_dedups_per_group() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 1, 2, 1, 2, 3])); + acc.update_batch(&[values], &[0, 0, 0, 1, 1, 1], None, 2)?; + let values: ArrayRef = Arc::new(Int32Array::from(vec![2, 3, 3])); + acc.update_batch(&[values], &[0, 1, 1], None, 2)?; + + // Values shared by both groups are stored once. + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(3)); + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(2)])); + assert_eq!(vals[1], Some(vec![Some(1), Some(2), Some(3)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_nulls() -> Result<()> { + let values: ArrayRef = + Arc::new(Int32Array::from(vec![None, Some(1), None, None])); + let groups = [0, 0, 0, 1]; + + // Without IGNORE NULLS a null is one more distinct value. + let mut acc = distinct_groups_accumulator(false); + acc.update_batch(&[Arc::clone(&values)], &groups, None, 3)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![None, Some(1)])); + assert_eq!(vals[1], Some(vec![None])); + // A group with no rows is null. + assert_eq!(vals[2], None); + + // With IGNORE NULLS a group of only nulls is null. + let mut acc = distinct_groups_accumulator(true); + acc.update_batch(&[values], &groups, None, 2)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_opt_filter() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 2, 3])); + let filter = BooleanArray::from(vec![Some(true), Some(false), Some(true), None]); + acc.update_batch(&[values], &[0, 0, 0, 1], Some(&filter), 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(2)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_sparse_filter() -> Result<()> { + // Only the selected rows are gathered and encoded, so their positions + // move; check each value still lands in its own group. + let mut acc = distinct_groups_accumulator(true); + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(5), + Some(6), + Some(7), + None, + Some(8), + ])); + let filter = BooleanArray::from(vec![false, true, false, true, true]); + acc.update_batch(&[values], &[0, 1, 0, 2, 2], Some(&filter), 3)?; + + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2)); + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals, vec![None, Some(vec![Some(6)]), Some(vec![Some(8)])]); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_merges_sliced_state() -> Result<()> { + let mut partial = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 4])); + partial.update_batch(&[values], &[0, 1, 2, 3, 3], None, 4)?; + let state = partial.state(EmitTo::All)?; + + // Partial aggregation emits slices of one state array. + let mut acc = distinct_groups_accumulator(false); + acc.merge_batch(&[state[0].slice(2, 2)], &[0, 1], 2)?; + acc.merge_batch(&[state[0].slice(0, 2)], &[1, 0], 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(3), Some(2)])); + assert_eq!(vals[1], Some(vec![Some(4), Some(1)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_records_metric() -> Result<()> { + let metric_updates = Arc::new(AtomicUsize::new(0)); + let mut acc = distinct_groups_accumulator(false); + acc.set_metrics(Arc::new(CountingMetrics(Arc::clone(&metric_updates)))); + + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); + acc.update_batch(&[Arc::clone(&values)], &[0, 1], None, 2)?; + let state = acc.state(EmitTo::All)?; + acc.merge_batch(&state, &[0, 1], 2)?; + assert_eq!(metric_updates.load(Ordering::Relaxed), 2); + + // Emitting everything resets the state but keeps the metric. + acc.evaluate(EmitTo::All)?; + acc.update_batch(&[values], &[0, 1], None, 2)?; + assert_eq!(metric_updates.load(Ordering::Relaxed), 3); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_order_by() -> Result<()> { + let descending = SortOptions { + descending: true, + nulls_first: false, + }; + let mut acc = DistinctArrayAggGroupsAccumulator::new( + DataType::Int32, + Some(descending), + false, + ); + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(2), + None, + Some(5), + Some(2), + Some(1), + ])); + acc.update_batch(&[values], &[0, 0, 0, 0, 1], None, 2)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(5), Some(2), None])); + assert_eq!(vals[1], Some(vec![Some(1)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_emit_first() -> Result<()> { + let mut acc = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 10, 20, 30, 20])); + acc.update_batch(&[values], &[0, 0, 1, 2, 2], None, 3)?; + + let vals = eval_i32_lists(&mut acc, EmitTo::First(2))?; + assert_eq!(vals, vec![Some(vec![Some(10)]), Some(vec![Some(20)])]); + + // 10 was only referenced by an emitted group, so it is dropped. + assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2)); + + // The remaining group is renumbered to 0 and keeps deduplicating. + let values: ArrayRef = Arc::new(Int32Array::from(vec![20, 40])); + acc.update_batch(&[values], &[0, 0], None, 1)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals, vec![Some(vec![Some(30), Some(20), Some(40)])]); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_state_merge_roundtrip() -> Result<()> { + let mut acc1 = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); + acc1.update_batch(&[values], &[0, 1], None, 2)?; + + let mut acc2 = distinct_groups_accumulator(false); + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 3, 2])); + acc2.update_batch(&[values], &[0, 0, 1], None, 2)?; + + let state = acc2.state(EmitTo::All)?; + acc1.merge_batch(&state, &[0, 1], 2)?; + + let vals = eval_i32_lists(&mut acc1, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(1), Some(3)])); + assert_eq!(vals[1], Some(vec![Some(2)])); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_convert_to_state_merge() -> Result<()> { + let partial = distinct_groups_accumulator(true); + let values: ArrayRef = + Arc::new(Int32Array::from(vec![Some(7), None, Some(7), Some(8)])); + let filter = BooleanArray::from(vec![true, true, true, false]); + let state = partial.convert_to_state(&[values], Some(&filter))?; + + let mut acc = distinct_groups_accumulator(true); + acc.merge_batch(&state, &[0, 0, 0, 1], 2)?; + let vals = eval_i32_lists(&mut acc, EmitTo::All)?; + assert_eq!(vals[0], Some(vec![Some(7)])); + assert_eq!(vals[1], None); + Ok(()) + } + + #[test] + fn distinct_groups_accumulator_keeps_dictionary_type() -> Result<()> { + let datatype = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let mut acc = + DistinctArrayAggGroupsAccumulator::new(datatype.clone(), None, false); + let values: ArrayRef = Arc::new( + vec!["a", "b", "a", "a"] + .into_iter() + .collect::>(), + ); + acc.update_batch(&[values], &[0, 0, 0, 1], None, 2)?; + + let result = acc.evaluate(EmitTo::All)?; + let lists = result.as_list::(); + assert_eq!(lists.values().data_type(), &datatype); + let flat = cast(lists.values().as_ref(), &DataType::Utf8)?; + let flat = flat.as_string::().iter().collect::>(); + assert_eq!(flat, vec![Some("a"), Some("b"), Some("a")]); + assert_eq!(lists.value_offsets(), &[0, 2, 3]); + Ok(()) + } + + /// Compares the groups accumulator with one [`DistinctArrayAggAccumulator`] + /// per group over batches with repeated values, nulls and a filter. + #[test] + fn distinct_groups_accumulator_matches_row_accumulator() -> Result<()> { + let num_groups = 7; + let mut acc = distinct_groups_accumulator(false); + let mut expected = (0..num_groups) + .map(|_| DistinctArrayAggAccumulator::try_new(&DataType::Int32, None, false)) + .collect::>>()?; + + for batch in 0..5 { + let len = 50; + let values = (0..len) + .map(|i| ((i * 7 + batch * 3) % 11 != 0).then_some((i * 13 + batch) % 9)) + .collect::>>(); + let groups = (0..len) + .map(|i| ((i * 5 + batch) % num_groups as i32) as usize) + .collect::>(); + let filter = (0..len).map(|i| (i + batch) % 4 != 0).collect::>(); + + let array: ArrayRef = Arc::new(Int32Array::from(values.clone())); + acc.update_batch( + &[array], + &groups, + Some(&BooleanArray::from(filter.clone())), + num_groups, + )?; + for ((value, group), keep) in values.into_iter().zip(groups).zip(filter) { + if keep { + let value: ArrayRef = Arc::new(Int32Array::from(vec![value])); + expected[group].update_batch(&[value])?; + } + } + } + + let sorted = |list: Option>>| { + list.map(|mut values| { + values.sort(); + values + }) + }; + let actual = eval_i32_lists(&mut acc, EmitTo::All)?; + for (group, acc) in expected.iter_mut().enumerate() { + let ScalarValue::List(list) = acc.evaluate()? else { + return internal_err!("expected a list"); + }; + let expected_values = list_array_to_i32_vecs(&list).swap_remove(0); + assert_eq!( + sorted(actual[group].clone()), + sorted(expected_values), + "group {group}" + ); + } + Ok(()) + } + #[test] fn groups_accumulator_multiple_batches() -> Result<()> { let mut acc = ArrayAggGroupsAccumulator::new(DataType::Int32, false); diff --git a/datafusion/functions-aggregate/tests/state_compat.rs b/datafusion/functions-aggregate/tests/state_compat.rs index e1915da9cfaa8..5e6f794d82b1e 100644 --- a/datafusion/functions-aggregate/tests/state_compat.rs +++ b/datafusion/functions-aggregate/tests/state_compat.rs @@ -27,8 +27,8 @@ use std::collections::BTreeSet; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, UInt32Array}; -use arrow::compute::{cast, concat, take}; +use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, ListArray, UInt32Array}; +use arrow::compute::{cast, concat, sort, take}; use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit}; use arrow::record_batch::{RecordBatch, RecordBatchOptions}; use datafusion_common::{DataFusionError, Result, ScalarValue}; @@ -518,7 +518,10 @@ fn take_all(arrays: &[ArrayRef], indices: &UInt32Array) -> Result> /// Runs every state exchange for one case. fn check_case(expr: &AggregateFunctionExpr, case: &Case) -> Errors { - let mut errors = Errors::default(); + let mut errors = Errors { + ignore_list_order: case.distinct && !case.ordered, + ..Default::default() + }; let input = match Input::new(expr, case) { Ok(input) => input, Err(e) => { @@ -693,7 +696,12 @@ fn guard(f: impl FnOnce() -> Result) -> Result { /// Failures for one case, deduplicated by message across groups. #[derive(Default)] -struct Errors(Vec<(String, Vec)>); +struct Errors { + failures: Vec<(String, Vec)>, + /// Compare list results as multisets. `array_agg(DISTINCT x)` without + /// ORDER BY does not define the order of its elements. + ignore_list_order: bool, +} impl Errors { fn push(&mut self, msg: &str) { @@ -708,14 +716,14 @@ impl Errors { // Only keep the first line, dropping e.g. the "please file a bug" text // of internal errors. let msg = msg.lines().next().unwrap_or_default().to_string(); - match self.0.iter_mut().find(|(m, _)| *m == msg) { + match self.failures.iter_mut().find(|(m, _)| *m == msg) { Some((_, groups)) => groups.extend(group), - None => self.0.push((msg, group.into_iter().collect())), + None => self.failures.push((msg, group.into_iter().collect())), } } fn lines(&self) -> Vec { - self.0 + self.failures .iter() .map(|(msg, groups)| { if groups.is_empty() { @@ -771,7 +779,7 @@ impl Errors { ) { match got { Err(e) => self.push_group(&format!("{what}: {e}"), group), - Ok(got) if !scalars_match(&got, want) => { + Ok(got) if !scalars_match(&got, want, self.ignore_list_order) => { self.push_group(&format!("{what}: got {got:?}, expected {want:?}"), group) } Ok(_) => {} @@ -780,8 +788,15 @@ impl Errors { } /// Exact equality, except that floats are compared with a relative tolerance -/// because merging in a different order can change rounding. -fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool { +/// because merging in a different order can change rounding, and lists are +/// compared as multisets when `ignore_list_order` is set. +fn scalars_match(a: &ScalarValue, b: &ScalarValue, ignore_list_order: bool) -> bool { + if ignore_list_order + && let (ScalarValue::List(x), ScalarValue::List(y)) = (a, b) + && let (Some(x), Some(y)) = (sorted_list_values(x), sorted_list_values(y)) + { + return x.as_ref() == y.as_ref(); + } match (a, b) { (ScalarValue::Float64(Some(x)), ScalarValue::Float64(Some(y))) => { floats_match(*x, *y) @@ -793,6 +808,15 @@ fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool { } } +/// The values of a single-row list, sorted, or `None` if the row is null or the +/// values cannot be sorted. +fn sorted_list_values(list: &ListArray) -> Option { + if list.len() != 1 || list.is_null(0) { + return None; + } + sort(list.value(0).as_ref(), None).ok() +} + fn floats_match(x: f64, y: f64) -> bool { if x.is_nan() || y.is_nan() { return x.is_nan() && y.is_nan(); diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index 3d304989afda8..89fd40603f12a 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -571,6 +571,18 @@ impl ScalarUDFImpl for DateTruncFunc { Ok(SortProperties::Unordered) } } + + fn supports_range_key_locality_analysis(&self, argument_types: &[DataType]) -> bool { + // Across an America/Goose_Bay transition, ascending epoch seconds + // 562129259 < 562129260 truncate to 562129200 > 562125600. + // Keep the audited domain to timezone-less timestamp columns. + let [precision, timestamp] = argument_types else { + return false; + }; + + precision.is_string() && matches!(timestamp, Timestamp(_, None)) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -1339,7 +1351,10 @@ mod tests { }; use arrow::array::cast::as_primitive_array; - use arrow::array::types::{ArrowTimestampType, TimestampNanosecondType}; + use arrow::array::types::{ + ArrowTimestampType, TimestampMicrosecondType, TimestampMillisecondType, + TimestampNanosecondType, TimestampSecondType, + }; use arrow::array::{ Array, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, @@ -1394,6 +1409,91 @@ mod tests { ); } + #[test] + fn range_key_locality_analysis_requires_timezone_less_timestamp() { + let date_trunc = DateTruncFunc::new(); + + assert!(date_trunc.supports_range_key_locality_analysis(&[ + DataType::Utf8, + DataType::Timestamp(TimeUnit::Nanosecond, None), + ])); + assert!(!date_trunc.supports_range_key_locality_analysis(&[ + DataType::Utf8, + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ])); + assert!(!date_trunc.supports_range_key_locality_analysis(&[ + DataType::Int64, + DataType::Timestamp(TimeUnit::Nanosecond, None), + ])); + assert!(!date_trunc.supports_range_key_locality_analysis(&[DataType::Utf8])); + } + + fn invoke_date_trunc( + granularity: &str, + timestamp: ColumnarValue, + timestamp_type: DataType, + number_rows: usize, + ) -> datafusion_common::Result { + DateTruncFunc::new().invoke_with_args(ScalarFunctionArgs { + args: vec![ScalarValue::from(granularity).into(), timestamp], + arg_fields: vec![ + Field::new("granularity", DataType::Utf8, false).into(), + Field::new("timestamp", timestamp_type.clone(), true).into(), + ], + number_rows, + return_field: Field::new("date_trunc", timestamp_type, true).into(), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn assert_timezone_less_date_trunc_preserves_nullness() { + let input = PrimitiveArray::::from_iter([Some(0), None]); + let timestamp_type = DataType::Timestamp(T::UNIT, None); + + for granularity in [ + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year", + ] { + let output = invoke_date_trunc( + granularity, + ColumnarValue::Array(Arc::new(input.clone())), + timestamp_type.clone(), + input.len(), + ) + .unwrap(); + let ColumnarValue::Array(output) = output else { + panic!("expected array output") + }; + + assert_eq!( + input.nulls(), + output.nulls(), + "{granularity} must preserve nullness for {:?}", + T::UNIT + ); + } + } + + /// Verifies the exact null contract required by + /// [`ScalarUDFImpl::supports_range_key_locality_analysis`] across every + /// supported granularity and timezone-less timestamp unit. Ordering metadata + /// and concrete range-split separation are tested separately. + #[test] + fn timezone_less_date_trunc_preserves_nullness() { + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + assert_timezone_less_date_trunc_preserves_nullness::(); + } + #[test] fn date_trunc_test() { let cases = vec![ diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 7dcf8fde935e5..e6f235da4dd42 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -23,11 +23,14 @@ use std::sync::Arc; use crate::push_down_filter::lr_is_preserved; use crate::simplify_expressions::ExprSimplifier; +use datafusion_common::nested_struct::is_injective_cast; use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, }; +use datafusion_common::utils::has_float_leaf; use datafusion_common::{ - Column, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, plan_err, + Column, DFSchema, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, + plan_err, }; use datafusion_expr::expr::{Alias, GroupingSet}; use datafusion_expr::logical_plan::{Join, JoinType}; @@ -38,7 +41,7 @@ use datafusion_expr::utils::{ use datafusion_expr::{ Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension, FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType, - UserDefinedLogicalNodeCore, expr, lit, + UserDefinedLogicalNodeCore, Window, expr, lit, }; /// This struct rewrite the sub query plan by pull up the correlated @@ -59,6 +62,10 @@ pub struct PullUpCorrelatedExpr { /// Indicates if we encounter any correlated expression that can not be pulled up /// above a aggregation without changing the meaning of the query. can_pull_over_aggregation: bool, + /// Whether every correlated conjunct seen so far passes + /// [`is_partitionable_equality`], so a Window above them can partition by their + /// columns. + can_extend_partition_by: bool, /// Do we need to handle [the count bug] during the pull up process. /// /// The "count bug" was described in [Optimization of Nested SQL @@ -117,6 +124,7 @@ impl PullUpCorrelatedExpr { exists_sub_query: false, can_pull_up: true, can_pull_over_aggregation: true, + can_extend_partition_by: true, need_handle_count_bug: false, collected_count_expr_map: HashMap::new(), pull_up_having_expr: None, @@ -321,7 +329,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { // outer row. A non-equality filter can return duplicates, see // https://github.com/apache/datafusion/issues/25808. // - A Window keeps or drops whole partitions when the filter reads - // only its `PARTITION BY` columns. `f_up` checks this. + // only its `PARTITION BY` columns. For other filters, `f_up` adds + // the columns of a column equality to `PARTITION BY` and keeps + // the subquery correlated in the remaining cases. // - An Unnest maps each input row to its own output rows, so a // filter on a column it passes through keeps the same rows. // The arm above checks that the filter does not read an unnested @@ -384,11 +394,11 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { match &plan { LogicalPlan::Filter(plan_filter) => { let subquery_filter_exprs = split_conjunction(&plan_filter.predicate); - self.can_pull_over_aggregation = self.can_pull_over_aggregation - && subquery_filter_exprs - .iter() - .filter(|e| e.contains_outer()) - .all(|&e| can_pullup_over_aggregation(e)); + for &expr in subquery_filter_exprs.iter().filter(|e| e.contains_outer()) { + self.can_pull_over_aggregation &= can_pullup_over_aggregation(expr); + self.can_extend_partition_by &= + is_partitionable_equality(expr, plan_filter.input.schema()); + } for expr in &subquery_filter_exprs { if expr.contains_outer() && !self.correlated_filters.contains(expr) { self.correlated_filters.push((*expr).clone()); @@ -713,6 +723,16 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { .iter() .all(|expr| partitions_by_all(expr, &local_correlated_cols)) { + // `inner.y = outer.k` keeps the rows of one `y` value for + // each outer row, so partitioning by `y` as well gives each + // window function the same rows. An aggregate without + // `GROUP BY` below returns a row for an outer row without + // a match, which the pulled up filter would drop. + if self.can_extend_partition_by && !self.pulled_up_scalar_agg { + let extended = + partition_window_by(window, &local_correlated_cols)?; + return self.pass_through_pulled_up_cols(extended); + } self.can_pull_up = false; } self.pass_through_pulled_up_cols(plan) @@ -960,6 +980,46 @@ fn correlated_filter_columns(plan: &LogicalPlan) -> Vec { cols } +/// Adds `cols` to the `PARTITION BY` of every window function of `window`. Each +/// window expression keeps its old output name, so nothing above sees a change. +fn partition_window_by(window: &Window, cols: &BTreeSet) -> Result { + let window_expr = window + .window_expr + .iter() + .map(|expr| { + add_partition_by(expr, cols).alias_if_changed(expr.schema_name().to_string()) + }) + .collect::>>()?; + Window::try_new(window_expr, Arc::clone(&window.input)).map(LogicalPlan::Window) +} + +fn add_partition_by(expr: &Expr, cols: &BTreeSet) -> Expr { + match expr { + Expr::Alias(alias) => Expr::Alias(Alias { + expr: Box::new(add_partition_by(&alias.expr, cols)), + ..alias.clone() + }), + Expr::WindowFunction(window_fun) => { + let mut window_fun = window_fun.clone(); + for col in cols { + let partitioned = window_fun + .params + .partition_by + .iter() + .any(|expr| matches!(expr, Expr::Column(c) if c == col)); + if !partitioned { + window_fun + .params + .partition_by + .push(Expr::Column(col.clone())); + } + } + Expr::WindowFunction(window_fun) + } + _ => expr.clone(), + } +} + /// Whether the window function `expr` lists each of `cols` as a plain /// `PARTITION BY` column. fn partitions_by_all(expr: &Expr, cols: &BTreeSet) -> bool { @@ -989,30 +1049,54 @@ fn collides_with_existing(exprs: &[Expr], candidate: &Expr) -> bool { } fn can_pullup_over_aggregation(expr: &Expr) -> bool { - if let Expr::BinaryExpr(BinaryExpr { + split_column_equality(expr).is_some() +} + +/// Whether partitioning by the columns of `expr` keeps the rows `expr` matches +/// for one outer row in one partition. That holds for `col = ` or +/// ` = col`, where: +/// - `` reads no subquery column and is not volatile. +/// - `col` has no float leaf, because `-0.0 = 0.0` holds but a partition +/// keeps the two apart. +/// - `col` is cast only by a cast that keeps distinct values distinct. A lossy +/// cast lets several `col` values match one outer row, see #26014. +fn is_partitionable_equality(expr: &Expr, schema: &DFSchema) -> bool { + let Some((col_side, other)) = split_column_equality(expr) else { + return false; + }; + let (col, cast_to) = match col_side { + Expr::Cast(Cast { expr, field }) => (&**expr, Some(field.data_type())), + _ => (col_side, None), + }; + let Ok(col_type) = col.get_type(schema) else { + return false; + }; + !other.is_volatile() + && !has_float_leaf(&col_type) + && cast_to.is_none_or(|to| is_injective_cast(&col_type, to)) +} + +/// Splits `col = ` or ` = col` into the subquery column side, a +/// column or a cast of one, and the other side, which reads no subquery column. +fn split_column_equality(expr: &Expr) -> Option<(&Expr, &Expr)> { + let Expr::BinaryExpr(BinaryExpr { left, op: Operator::Eq, right, }) = expr - { - match (&**left, &**right) { - (Expr::Column(_), right) => !right.any_column_refs(), - (left, Expr::Column(_)) => !left.any_column_refs(), - (Expr::Cast(Cast { expr, .. }), right) - if matches!(&**expr, Expr::Column(_)) => - { - !right.any_column_refs() - } - (left, Expr::Cast(Cast { expr, .. })) - if matches!(&**expr, Expr::Column(_)) => - { - !left.any_column_refs() - } - (_, _) => false, - } - } else { - false - } + else { + return None; + }; + [(&**left, &**right), (&**right, &**left)] + .into_iter() + .find(|(col_side, other)| { + let is_col = match col_side { + Expr::Column(_) => true, + Expr::Cast(Cast { expr, .. }) => matches!(&**expr, Expr::Column(_)), + _ => false, + }; + is_col && !other.any_column_refs() + }) } fn collect_local_correlated_cols( @@ -1121,7 +1205,11 @@ fn proj_exprs_evaluation_result_on_empty_batch( }) .data()?; - if result_expr.ne(expr) { + // An expression that reads no column, like `1 AS x`, keeps its value + // on an empty input, while the join that replaces the subquery gives + // it NULL for an outer row without a match. + let is_constant = !expr.any_column_refs() && !expr.contains_outer(); + if result_expr.ne(expr) || is_constant { let info = SimplifyContext::builder() .with_schema(Arc::clone(schema)) .build(); diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs index ca12fb0bbb527..dad3cc03042c6 100644 --- a/datafusion/optimizer/src/extract_leaf_expressions.rs +++ b/datafusion/optimizer/src/extract_leaf_expressions.rs @@ -53,7 +53,19 @@ //! that runs before the extraction projection exists, so row group pruning and //! source level filtering still happen. //! +//! # Yields to [`OptimizeProjections`] +//! +//! When the extraction projection cannot move below the input of a projection +//! (for example a `TableScan` or an `Aggregate`), [`PushDownLeafProjections`] +//! leaves the projection unchanged. Splitting it in place would give a +//! recovery projection over an extraction projection on the same input, and +//! `OptimizeProjections` merges those two back into the original projection. +//! The two rules would then undo each other on every optimizer pass. The split +//! has no use there: the source absorbs the leaf expressions of the original +//! projection just as well. +//! //! [`PushDownFilter`]: crate::push_down_filter::PushDownFilter +//! [`OptimizeProjections`]: crate::optimize_projections::OptimizeProjections use indexmap::{IndexMap, IndexSet}; use std::collections::{BTreeSet, HashMap}; @@ -1150,18 +1162,33 @@ fn split_and_push_projection( // Only pre-existing __datafusion_extracted aliases and columns, no new // extractions from routing_extract. The original projection is // already an extraction projection that couldn't be pushed - // further. Return None. + // further. Return None. This return also ends the recursion + // of the `try_push_input` call below, which comes back here + // with a pure extraction projection. return Ok(None); } - // Build extraction projection in-place (couldn't push down) + // Build the extraction projection in place, then push it. It is a + // pure extraction projection, which can go through nodes that the + // original projection cannot. let input_arc = Arc::clone(input); - let extraction = build_extraction_projection_impl( + let extraction = LogicalPlan::Projection(build_extraction_projection_impl( &extraction_pairs, columns_needed, &input_arc, input_schema.as_ref(), - )?; - LogicalPlan::Projection(extraction) + )?); + match try_push_input(&extraction, alias_generator)? { + Some(pushed) => pushed, + // The extraction projection cannot move below `input`. Leave + // the original projection unchanged: the split would put the + // extraction projection on the same input, and + // `OptimizeProjections` merges the recovery and extraction + // projections back into the original projection. The rules + // would then undo each other in every optimizer pass. The + // source absorbs the leaf expressions of the original + // projection just as well. + None => return Ok(None), + } } }; @@ -1730,13 +1757,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -1760,13 +1784,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 IS NOT NULL AS has_name - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) IS NOT NULL AS has_name - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -1965,13 +1986,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS username - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) AS username - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2031,13 +2049,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")), __datafusion_extracted_1 AS name2 - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")), leaf_udf(test.user, Utf8("name")) AS name2 - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2245,13 +2260,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 AS leaf_udf(test.user,Utf8("name")) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) - TableScan: test projection=[user] + (same as after pushdown) "#) } @@ -2405,8 +2417,8 @@ mod tests { } /// Projection containing a MoveTowardsLeafNodes sub-expression above an - /// Aggregate. Aggregate blocks pushdown, so the (None, true) recovery - /// fallback path fires: in-place extraction + recovery projection. + /// Aggregate. The Aggregate blocks pushdown, so the extraction cannot move + /// and the original projection is kept unchanged. #[test] fn test_projection_with_leaf_expr_above_aggregate() -> Result<()> { use datafusion_expr::test::function_stub::count; @@ -2432,15 +2444,10 @@ mod tests { (same as original) ## After Pushdown - Projection: __datafusion_extracted_1 IS NOT NULL AS has_name, COUNT(Int32(1)) - Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_1, test.user, COUNT(Int32(1)) - Aggregate: groupBy=[[test.user]], aggr=[[COUNT(Int32(1))]] - TableScan: test projection=[user] + (same as after extraction) ## Optimized - Projection: leaf_udf(test.user, Utf8("name")) IS NOT NULL AS has_name, COUNT(Int32(1)) - Aggregate: groupBy=[[test.user]], aggr=[[COUNT(Int32(1))]] - TableScan: test projection=[user] + (same as after pushdown) "#) } diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index c913a4ce961cc..8fdb5775c971c 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -16,6 +16,17 @@ // under the License. //! [`OptimizeProjections`] identifies and eliminates unused columns +//! +//! # Precedence over `PushDownLeafProjections` +//! +//! `OptimizeProjections` merges adjacent projections, including a recovery +//! projection over an extraction projection that +//! [`PushDownLeafProjections`] creates. To keep the two rules from undoing each +//! other, `PushDownLeafProjections` only creates that pair when the extraction +//! projection moves below the input. See the module documentation of +//! [`extract_leaf_expressions`](crate::extract_leaf_expressions). +//! +//! [`PushDownLeafProjections`]: crate::extract_leaf_expressions::PushDownLeafProjections mod required_indices; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 54f220a1fe9dd..ccde8aa07f64e 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1751,7 +1751,7 @@ mod tests { use crate::simplify_expressions::SimplifyExpressions; use crate::test::udfs::leaf_udf_expr; use crate::test::*; - use datafusion_expr::test::function_stub::sum; + use datafusion_expr::test::function_stub::{count, sum}; use insta::assert_snapshot; use super::*; @@ -4864,9 +4864,11 @@ mod tests { /// `PushDownFilter` and `PushDownLeafProjections` must not undo each other /// for a filter next to a pure extraction projection - /// (). Neither rule may - /// change the plan in the last optimizer pass. The source does not absorb - /// filters, so the `Filter` node stays in the plan. + /// (). The source does + /// not absorb filters, so the `Filter` node stays in the plan. Also, + /// `PushDownLeafProjections` must not split a projection that + /// `OptimizeProjections` merges back. No rule may change the plan in the + /// last optimizer pass. #[test] fn filter_and_extraction_projection_reach_fixed_point() -> Result<()> { let scan = || { @@ -4887,13 +4889,26 @@ mod tests { .project(vec![leaf_udf_expr(col("a")), col("b")])? .build()?; - for plan in [simple, two_filters] { + // No filter: the extraction projection cannot move below the scan. + let no_filter = LogicalPlanBuilder::from(test_table_scan()?) + .project(vec![leaf_udf_expr(col("a"))])? + .build()?; + + // Leaf expression above an Aggregate: the Aggregate blocks movement. + let above_aggregate = LogicalPlanBuilder::from(test_table_scan_with_struct()?) + .aggregate(vec![col("user")], vec![count(lit(1))])? + .project(vec![ + leaf_udf_expr(col("user")).is_not_null().alias("has_name"), + col("COUNT(Int32(1))"), + ])? + .build()?; + + for plan in [simple, two_filters, no_filter, above_aggregate] { let passes = rules_that_changed_plan_per_pass(plan)?; let last = passes.last().unwrap(); assert!( - !last.iter().any(|rule| rule == "push_down_filter" - || rule == "push_down_leaf_projections"), - "pushdown rules changed the plan in the last pass: {passes:?}" + last.is_empty(), + "rules changed the plan in the last pass: {passes:?}" ); } Ok(()) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index ad23bb3c2577a..63929263d48c6 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -117,6 +117,12 @@ pub struct ExprSimplifier { max_simplifier_cycles: u32, } +/// Largest `IN` list considered for expansion into binary comparisons. +/// +/// The final decision also accounts for the measured performance of the column +/// representation and whether the list values are literals. +/// See [PR #25187](https://github.com/apache/datafusion/pull/25187) for the +/// benchmark evidence. pub const THRESHOLD_INLINE_INLIST: usize = 3; pub const DEFAULT_MAX_SIMPLIFIER_CYCLES: u32 = 3; @@ -199,7 +205,8 @@ impl ExprSimplifier { let mut simplifier = Simplifier::new(&self.info); let config_options = Some(Arc::clone(self.info.config_options())); let mut const_evaluator = ConstEvaluator::try_new(config_options)?; - let mut shorten_in_list_simplifier = ShortenInListSimplifier::new(); + let mut shorten_in_list_simplifier = + ShortenInListSimplifier::new(self.info.schema()); let guarantees_map: HashMap<&Expr, &NullableInterval> = self.guarantees.iter().map(|(k, v)| (k, v)).collect(); @@ -5006,6 +5013,8 @@ mod tests { }; let left_list = literals(&[0.0, 1.0, 2.0, 3.0])?; let right_list = literals(&[-0.0, 4.0, 5.0, 6.0])?; + let retains_short_lists = + matches!(data_type, DataType::Float32 | DataType::Float64); let schema = Arc::new(Schema::new(vec![Field::new("x", data_type, false)])); let batch = RecordBatch::try_new(schema, vec![values])?; let schema = batch.schema().to_dfschema_ref()?; @@ -5030,7 +5039,7 @@ mod tests { (left(true).or(right(true)), vec![false, false]), ] { let simplified = simplifier.simplify(expr.clone())?; - if list_len > THRESHOLD_INLINE_INLIST { + if list_len > THRESHOLD_INLINE_INLIST || retains_short_lists { // Floating-point literals must bypass structural set rewrites. assert_eq!(simplified, expr); } else { @@ -5099,6 +5108,80 @@ mod tests { } } + #[test] + fn simplify_short_inlist_types() { + for (data_type, retain_short) in [ + (DataType::Int32, true), + (DataType::Int64, true), + (DataType::UInt64, true), + (DataType::Float32, true), + (DataType::Float64, true), + (DataType::FixedSizeBinary(1), true), + (DataType::FixedSizeBinary(2), true), + (DataType::FixedSizeBinary(4), true), + (DataType::FixedSizeBinary(8), true), + (DataType::FixedSizeBinary(16), true), + (DataType::Float16, false), + (DataType::FixedSizeBinary(3), false), + (DataType::Utf8View, false), + ( + DataType::Dictionary( + Box::new(DataType::Int8), + Box::new(DataType::Float64), + ), + false, + ), + ] { + let values = (1..=3) + .map(|value| { + lit(match &data_type { + DataType::FixedSizeBinary(width) => ScalarValue::FixedSizeBinary( + *width, + Some(vec![value; *width as usize]), + ), + DataType::Utf8View => { + ScalarValue::Utf8View(Some(value.to_string())) + } + _ => ScalarValue::Int32(Some(i32::from(value))) + .cast_to(&data_type) + .unwrap(), + }) + }) + .collect::>(); + let schema = Schema::new(vec![Field::new("value", data_type.clone(), true)]) + .to_dfschema_ref() + .unwrap(); + let simplifier = ExprSimplifier::new( + SimplifyContext::builder().with_schema(schema).build(), + ); + for list_len in [2, 3] { + for negated in [false, true] { + let expr = + in_list(col("value"), values[..list_len].to_vec(), negated); + let simplified = simplifier.simplify(expr.clone()).unwrap(); + if retain_short { + assert_eq!(simplified, expr, "{data_type:?}"); + } else { + let op = if negated { Operator::And } else { Operator::Or }; + assert!( + matches!(simplified, Expr::BinaryExpr(binary) if binary.op == op), + "{data_type:?}, list_len={list_len}, negated={negated}" + ); + } + } + } + } + } + + #[test] + fn simplify_short_inlist_coerced_literals() { + for negated in [false, true] { + let expr = in_list(col("c3"), vec![lit(1_i32), lit(2_i32)], negated); + let expected = in_list(col("c3"), vec![lit(1_i64), lit(2_i64)], negated); + assert_eq!(simplify(coerce(expr)), expected); + } + } + #[test] fn simplify_inlist_set_operation_propagates_nullability_errors() { let info = SimplifyContext::builder() diff --git a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs index 17112d4f0ae24..087fb192c9a7e 100644 --- a/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs @@ -19,24 +19,49 @@ use super::THRESHOLD_INLINE_INLIST; -use datafusion_common::Result; +use arrow::datatypes::DataType; use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; -use datafusion_expr::Expr; +use datafusion_common::{DFSchema, Result}; use datafusion_expr::expr::InList; +use datafusion_expr::{Expr, ExprSchemable}; -pub(super) struct ShortenInListSimplifier {} +pub(super) struct ShortenInListSimplifier<'a> { + schema: &'a DFSchema, +} + +impl<'a> ShortenInListSimplifier<'a> { + pub(super) fn new(schema: &'a DFSchema) -> Self { + Self { schema } + } -impl ShortenInListSimplifier { - pub(super) fn new() -> Self { - Self {} + /// Benchmarks in `in_list_rewrite` favor evaluating short literal lists as + /// membership expressions over OR/AND chains for these types on batches with + /// a mix of matches and misses, or no matches. Integer chains can still be + /// faster when every row matches the first value, allowing the remaining + /// comparisons to short circuit. + /// + /// See [PR #25187](https://github.com/apache/datafusion/pull/25187) for measurements. + fn should_retain_short_inlist(&self, expr: &Expr, list: &[Expr]) -> bool { + let Ok(data_type) = expr.get_type(self.schema) else { + return false; + }; + matches!( + data_type, + DataType::Int32 + | DataType::Int64 + | DataType::UInt64 + | DataType::Float32 + | DataType::Float64 + | DataType::FixedSizeBinary(1 | 2 | 4 | 8 | 16) + ) && list.iter().all(|expr| expr.as_literal().is_some()) } } -impl TreeNodeRewriter for ShortenInListSimplifier { +impl TreeNodeRewriter for ShortenInListSimplifier<'_> { type Node = Expr; fn f_up(&mut self, expr: Expr) -> Result> { - // if expr is a single column reference: + // Rewrite eligible short lists to left-deep comparison chains: // expr IN (A, B, ...) --> (expr = A) OR (expr = B) OR (expr = C) if let Expr::InList(InList { ref expr, @@ -52,6 +77,7 @@ impl TreeNodeRewriter for ShortenInListSimplifier { list.len() == 1 || list.len() <= THRESHOLD_INLINE_INLIST && expr.try_as_col().is_some() + && !self.should_retain_short_inlist(expr, list) ) { let first_val = list[0].clone(); diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 1c2d970ccffa4..ed824e5712f96 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -39,6 +39,7 @@ use crate::{ PhysicalSortRequirement, }; +use arrow::compute::SortOptions; use arrow::datatypes::SchemaRef; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err}; @@ -1408,6 +1409,29 @@ impl EquivalenceProperties { .unwrap_or_else(|_| ExprProperties::new_unknown()) } + /// Returns whether `transformed_expr` reports the same ordering direction + /// when `source_expr` is ordered and all other inputs are singleton. + pub(crate) fn is_same_direction_ordered_transform( + &self, + transformed_expr: &Arc, + source_expr: &Arc, + ) -> bool { + if transformed_expr.eq(source_expr) { + return false; + } + + let options = SortOptions::default(); + let dependencies = Dependencies::new(std::iter::once(PhysicalSortExpr::new( + Arc::clone(source_expr), + options, + ))); + matches!( + get_expr_properties(transformed_expr, &dependencies, &self.schema) + .map(|properties| properties.sort_properties), + Ok(SortProperties::Ordered(actual)) if actual == options + ) + } + /// Transforms this `EquivalenceProperties` by mapping columns in the /// original schema to columns in the new schema by index. pub fn with_new_schema(mut self, schema: SchemaRef) -> Result { diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs index 336d09b987734..15d7849ebeac1 100644 --- a/datafusion/physical-expr/src/partitioning.rs +++ b/datafusion/physical-expr/src/partitioning.rs @@ -17,13 +17,17 @@ //! [`Partitioning`] and [`Distribution`] for `ExecutionPlans` +use crate::expressions::{Literal, UnKnownColumn}; +use crate::simplifier::const_evaluator::create_dummy_batch; use crate::{ - EquivalenceProperties, PhysicalExpr, equivalence::ProjectionMapping, - expressions::UnKnownColumn, physical_exprs_contains, physical_exprs_equal, + EquivalenceProperties, PhysicalExpr, ScalarFunctionExpr, + equivalence::ProjectionMapping, physical_exprs_contains, physical_exprs_equal, }; use arrow::datatypes::Schema; pub use datafusion_common::SplitPoint; -use datafusion_common::{Result, plan_err, validate_range_split_points}; +use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::{Result, ScalarValue, plan_err, validate_range_split_points}; +use datafusion_expr::ColumnarValue; use datafusion_physical_expr_common::physical_expr::format_physical_expr_list; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; #[cfg(feature = "proto")] @@ -532,6 +536,196 @@ fn normalize_exprs( .collect() } +fn evaluate_with_range_value( + expr: &Arc, + range_key: &Arc, + value: &ScalarValue, +) -> Option { + let literal: Arc = Arc::new(Literal::new(value.clone())); + let rewritten = Arc::clone(expr) + .transform(|node| { + if node.eq(range_key) { + Ok(Transformed::yes(Arc::clone(&literal))) + } else { + Ok(Transformed::no(node)) + } + }) + .ok()?; + if !rewritten.transformed { + return None; + } + + match rewritten.data.evaluate(create_dummy_batch().ok()?).ok()? { + ColumnarValue::Scalar(value) => Some(value), + ColumnarValue::Array(array) => ScalarValue::try_from_array(&array, 0).ok(), + } +} + +fn checked_adjacent_timestamp( + value: &ScalarValue, + is_successor: bool, +) -> Option { + let offset = if is_successor { 1 } else { -1 }; + match value { + ScalarValue::TimestampSecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampSecond(Some(value), None)), + ScalarValue::TimestampMillisecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampMillisecond(Some(value), None)), + ScalarValue::TimestampMicrosecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampMicrosecond(Some(value), None)), + ScalarValue::TimestampNanosecond(Some(value), None) => value + .checked_add(offset) + .map(|value| ScalarValue::TimestampNanosecond(Some(value), None)), + _ => None, + } +} + +/// Determines whether a nondecreasing transform keeps equal output keys local. +/// +/// For one ascending range key, split points `x₁, ..., xₙ` define: +/// +/// ```text +/// P₀ = (-∞, x₁) +/// Pᵢ = [xᵢ, xᵢ₊₁) for 1 ≤ i < n +/// Pₙ = [xₙ, +∞) +/// ``` +/// +/// Aggregation can reuse these partitions after applying `f` only if no output +/// key occurs in two partitions. In other words, the partition images must be +/// disjoint: `f(Pᵢ) ∩ f(Pⱼ) = ∅` for `i ≠ j`. +/// +/// Monotonicity alone is insufficient. Ordered metadata gives only +/// +/// ```text +/// f(predecessor(xᵢ)) ≤ f(xᵢ) +/// ``` +/// +/// but locality requires strict separation at every split: +/// +/// ```text +/// f(predecessor(xᵢ)) < f(xᵢ) +/// ``` +/// +/// Since ordering already supplies the non-strict inequality, the analysis +/// proves strictness by explicitly checking that the two outputs are not equal. +/// Otherwise, the same transformed key could occur in both adjacent partitions, +/// so the transform would not preserve key locality. For example, an hourly +/// truncation maps timestamps immediately before and at a `01:30` split to the +/// same `01:00` bucket. A split at `01:00` is safe because its predecessor maps +/// to `00:00`, while the split itself maps to `01:00`. +/// +/// `predecessor(xᵢ)` is the greatest representable timestamp below the split. +/// Therefore, for every `x < xᵢ` and `y ≥ xᵢ`, ordering and the strict boundary +/// check give +/// +/// ```text +/// f(x) ≤ f(predecessor(xᵢ)) < f(xᵢ) ≤ f(y) +/// ``` +/// +/// Thus no transformed key can occur on both sides of the split. The successor of +/// an ascending split is already in the same partition as `xᵢ`, so checking it +/// would not say anything about groups crossing that split. +/// +/// The argument is the same for descending ranges, but `successor(xᵢ)` must be +/// used instead. Null preservation is also required so the transform cannot +/// introduce a null group outside the source partition selected by null order. +fn range_splits_separate_transformed_keys( + function: &Arc, + range_key: &Arc, + range: &RangePartitioning, + schema: &Schema, +) -> bool { + let Ok(range_type) = range_key.data_type(schema) else { + return false; + }; + let sort_options = range.ordering().first().options; + if validate_range_split_points(range.split_points(), &[sort_options]).is_err() { + return false; + } + range.split_points().iter().all(|split_point| { + let Some(split_value) = split_point.values().first() else { + return false; + }; + if split_value.data_type() != range_type { + return false; + } + let Some(adjacent) = + checked_adjacent_timestamp(split_value, sort_options.descending) + else { + return false; + }; + let Some(at_split) = evaluate_with_range_value(function, range_key, split_value) + else { + return false; + }; + let Some(across_split) = + evaluate_with_range_value(function, range_key, &adjacent) + else { + return false; + }; + + !at_split.is_null() + && !across_split.is_null() + && at_split.data_type() == across_split.data_type() + && at_split != across_split + }) +} + +fn range_transform_proves_key_locality( + candidate: &Arc, + range_key: &Arc, + range: &RangePartitioning, + eq_properties: &EquivalenceProperties, +) -> bool { + let Some(function) = candidate.downcast_ref::() else { + return false; + }; + let schema = eq_properties.schema(); + if datafusion_physical_expr_common::physical_expr::is_volatile(candidate) + || !function + .supports_range_key_locality_analysis(schema) + .unwrap_or(false) + || !eq_properties.is_same_direction_ordered_transform(candidate, range_key) + { + return false; + } + + let mut range_argument_count = 0; + for argument in function.args() { + if eq_properties.eq_group().exprs_equal(argument, range_key) { + range_argument_count += 1; + } else if !argument.is::() { + return false; + } + } + + range_argument_count == 1 + && range_splits_separate_transformed_keys(candidate, range_key, range, schema) +} + +fn range_partitioning_keeps_keys_local( + range: &RangePartitioning, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + allow_subset: bool, +) -> bool { + // Currently, transform locality analysis only supports single-key ranges. + // Extending the proof to compound range keys is tracked in #24644. + if range.ordering().len() != 1 || required_exprs.is_empty() { + return false; + } + + let range_key = &range.ordering()[0].expr; + let has_key_locality_proof = required_exprs.iter().any(|required| { + range_transform_proves_key_locality(required, range_key, range, eq_properties) + }); + + has_key_locality_proof && (required_exprs.len() == 1 || allow_subset) +} + /// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PartitioningSatisfaction { @@ -646,6 +840,41 @@ impl Partitioning { } } + /// Returns true when it can prove that equal tuples of `required_exprs` + /// cannot occur in different partitions. + /// + /// Unlike [`Self::satisfaction`], this does not claim a concrete partition + /// layout. This weaker guarantee is sufficient for independent keyed + /// operations such as aggregation, but not for co-partitioning multiple + /// inputs. Transform analysis is currently limited to single-key range + /// partitioning. + pub fn keeps_keys_local( + &self, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + allow_subset: bool, + ) -> bool { + let required = Distribution::KeyPartitioned(required_exprs.to_vec()); + if self + .satisfaction(&required, eq_properties, allow_subset) + .is_satisfied() + { + return true; + } + + match self { + Self::Range(range) => range_partitioning_keeps_keys_local( + range, + required_exprs, + eq_properties, + allow_subset, + ), + Self::RoundRobinBatch(_) + | Self::Hash(_, _) + | Self::UnknownPartitioning(_) => false, + } + } + fn key_satisfaction( partition_exprs: &[Arc], required_exprs: &[Arc], @@ -991,7 +1220,8 @@ mod tests { use crate::projection::ProjectionTargets; use arrow::compute::SortOptions; - use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; + use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, ScalarValue}; struct PartitioningTestFixture { @@ -1090,6 +1320,17 @@ mod tests { ) -> Partitioning { Partitioning::Range(self.range(indices, split_points)) } + + fn range_partitioning_with_ordering( + &self, + ordering: LexOrdering, + split_points: Vec, + ) -> Partitioning { + Partitioning::Range( + RangePartitioning::try_new(ordering, split_points) + .expect("test range partitioning should be valid"), + ) + } } fn assert_satisfaction( @@ -1112,6 +1353,625 @@ mod tests { ); } + fn assert_key_locality( + desc: &str, + partitioning: &Partitioning, + required_exprs: &[Arc], + eq_properties: &EquivalenceProperties, + expected_with_subset: bool, + expected_without_subset: bool, + ) { + assert_eq!( + partitioning.keeps_keys_local(required_exprs, eq_properties, true), + expected_with_subset, + "Failed for {desc} with subset enabled" + ); + assert_eq!( + partitioning.keeps_keys_local(required_exprs, eq_properties, false), + expected_without_subset, + "Failed for {desc} with subset disabled" + ); + } + + fn date_trunc_of( + timestamp: Arc, + precision: &str, + timezone: Option<&str>, + ) -> Arc { + let timezone = timezone.map(Into::into); + Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![ + Arc::new(Literal::new(ScalarValue::Utf8(Some(precision.into())))), + timestamp, + ], + Field::new( + "date_trunc", + DataType::Timestamp(TimeUnit::Nanosecond, timezone), + true, + ) + .into(), + Arc::new(ConfigOptions::default()), + )) + } + + fn timestamp_split(value: Option, timezone: Option<&str>) -> SplitPoint { + SplitPoint::new(vec![ScalarValue::TimestampNanosecond( + value, + timezone.map(Into::into), + )]) + } + + /// Assigns sample rows to source ranges and evaluates whole arrays, + /// independently of the optimizer's boundary-adjacency proof. + fn date_trunc_partition_images_are_disjoint< + T: arrow::array::types::ArrowTimestampType, + >( + function: &Arc, + splits: &[i64], + options: SortOptions, + schema: &SchemaRef, + ) -> Result { + use arrow::array::{PrimitiveArray, RecordBatch}; + use datafusion_common::HashSet; + + let mut partitions = vec![vec![]; splits.len() + 1]; + let rows = splits + .iter() + .flat_map(|split| [Some(split - 1), Some(*split), Some(split + 1)]) + .chain([None, None]); + for value in rows { + let index = match value { + None if options.nulls_first => 0, + None => splits.len(), + Some(value) => splits + .iter() + .filter(|split| { + if options.descending { + value <= **split + } else { + value >= **split + } + }) + .count(), + }; + partitions[index].push(value); + } + let images = partitions + .into_iter() + .map(|values| { + let input = Arc::new(PrimitiveArray::::from_iter(values)); + let batch = RecordBatch::try_new(Arc::clone(schema), vec![input])?; + let output = function.evaluate(&batch)?.into_array(batch.num_rows())?; + (0..output.len()) + .map(|i| ScalarValue::try_from_array(&output, i)) + .collect::>>() + }) + .collect::>>()?; + + Ok(images.iter().enumerate().all(|(i, left)| { + images + .iter() + .skip(i + 1) + .all(|right| left.is_disjoint(right)) + })) + } + + fn assert_date_trunc_partition_images() + -> Result<()> { + let ticks_per_second = match T::UNIT { + TimeUnit::Second => 1, + TimeUnit::Millisecond => 1_000, + TimeUnit::Microsecond => 1_000_000, + TimeUnit::Nanosecond => 1_000_000_000, + }; + let hour = 3600 * ticks_per_second; + let data_type = DataType::Timestamp(T::UNIT, None); + let schema = + Arc::new(Schema::new(vec![Field::new("ts", data_type.clone(), true)])); + let timestamp: Arc = Arc::new(Column::new("ts", 0)); + let properties = EquivalenceProperties::new(Arc::clone(&schema)); + + let mut cases = vec![ + ( + "aligned ascending hours", + "hour", + false, + vec![-hour, 0, hour], + true, + ), + ( + "unaligned ascending hours", + "hour", + false, + vec![-hour + 1, 1, hour + 1], + false, + ), + ( + "days cross hourly splits", + "day", + false, + vec![-hour, 0, hour], + false, + ), + ( + "descending splits inside hour buckets", + "hour", + true, + vec![-hour, 0, hour], + false, + ), + ( + "descending splits between hour buckets", + "hour", + true, + vec![-hour - 1, -1, hour - 1], + true, + ), + ]; + for (precision, divisor) in [("millisecond", 1_000), ("microsecond", 1_000_000)] { + let width = (ticks_per_second / divisor).max(1); + cases.extend([ + ( + "aligned ascending subsecond splits", + precision, + false, + vec![-width, 0, width], + true, + ), + // In particular, microsecond input with millisecond precision + // must reject split -999: both adjacent array values map to -1000. + ( + "offset ascending subsecond split", + precision, + false, + vec![-width + 1], + width == 1, + ), + ( + "descending splits at subsecond bucket starts", + precision, + true, + vec![-width, 0, width], + width == 1, + ), + ( + "descending splits at subsecond bucket ends", + precision, + true, + vec![-width - 1, -1, width - 1], + true, + ), + ]); + } + for (description, precision, descending, source_splits, expect_disjoint) in cases + { + for nulls_first in [false, true] { + let options = SortOptions::new(descending, nulls_first); + let mut splits = source_splits.clone(); + if descending { + splits.reverse(); + } + let range = Partitioning::Range(RangePartitioning::try_new( + [PhysicalSortExpr::new(Arc::clone(×tamp), options)].into(), + splits + .iter() + .map(|value| { + SplitPoint::new(vec![ScalarValue::new_timestamp::( + Some(*value), + None, + )]) + }) + .collect(), + )?); + let function: Arc = Arc::new(ScalarFunctionExpr::new( + "date_trunc", + datafusion_functions::datetime::date_trunc(), + vec![ + Arc::new(Literal::new(ScalarValue::from(precision))), + Arc::clone(×tamp), + ], + Field::new("bucket", data_type.clone(), true).into(), + Arc::new(ConfigOptions::default()), + )); + assert_eq!( + range.keeps_keys_local(&[Arc::clone(&function)], &properties, false), + expect_disjoint, + "optimizer: {description}, {precision}, {:?}, {options:?}, splits={splits:?}", + T::UNIT + ); + assert_eq!( + date_trunc_partition_images_are_disjoint::( + &function, &splits, options, &schema, + )?, + expect_disjoint, + "array evaluation: {description}, {precision}, {:?}, {options:?}, splits={splits:?}", + T::UNIT + ); + } + } + Ok(()) + } + + #[test] + fn date_trunc_partition_images_are_disjoint_only_for_safe_splits() -> Result<()> { + use arrow::array::types::{ + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, + }; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::()?; + assert_date_trunc_partition_images::() + } + + #[test] + fn range_transform_requires_literal_parameters() -> Result<()> { + use crate::expressions::CastExpr; + let fixture = PartitioningTestFixture::new(vec![( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let function = date_trunc_of(fixture.col(0), "hour", None); + let scalar_but_not_literal: Arc = Arc::new(CastExpr::new( + Arc::new(Literal::new(ScalarValue::from("hour"))), + DataType::Utf8View, + None, + )); + let function = + function.with_new_children(vec![scalar_but_not_literal, fixture.col(0)])?; + let range = fixture.range_partitioning([0], vec![timestamp_split(Some(0), None)]); + assert!(!range.keeps_keys_local(&[function], &fixture.eq_properties, false)); + Ok(()) + } + + #[test] + fn range_transform_establishes_key_locality_only() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![ + ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), + ("tag", DataType::Utf8), + ])?; + let hour = 1_704_070_800_000_000_000; + let aligned = + fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]); + let trunc_hour = date_trunc_of(fixture.col(0), "hour", None); + + let exact_key_exprs = vec![Arc::clone(&trunc_hour)]; + assert_eq!( + aligned.satisfaction( + &Distribution::KeyPartitioned(exact_key_exprs.clone()), + &fixture.eq_properties, + false, + ), + PartitioningSatisfaction::NotSatisfied + ); + assert_key_locality( + "aligned transformed key", + &aligned, + &exact_key_exprs, + &fixture.eq_properties, + true, + true, + ); + + assert_key_locality( + "additional grouping keys require subset permission", + &aligned, + &[trunc_hour, fixture.col(1)], + &fixture.eq_properties, + true, + false, + ); + + let trunc_day = date_trunc_of(fixture.col(0), "day", None); + assert_key_locality( + "an hour split straddles a day bucket", + &aligned, + &[trunc_day], + &fixture.eq_properties, + false, + false, + ); + + let two_aligned = fixture.range_partitioning( + [0], + vec![ + timestamp_split(Some(hour), None), + timestamp_split(Some(hour + 3_600_000_000_000), None), + ], + ); + assert_key_locality( + "every split must be aligned", + &two_aligned, + &exact_key_exprs, + &fixture.eq_properties, + true, + true, + ); + + let unaligned = + fixture.range_partitioning([0], vec![timestamp_split(Some(hour + 1), None)]); + assert_key_locality( + "an unaligned split straddles an hour bucket", + &unaligned, + &exact_key_exprs, + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + + #[test] + fn range_transform_rejects_compound_range_keys() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![ + ("key", DataType::Utf8), + ("timestamp", DataType::Timestamp(TimeUnit::Nanosecond, None)), + ])?; + let split = SplitPoint::new(vec![ + ScalarValue::Utf8(Some("m".into())), + ScalarValue::TimestampNanosecond(Some(1_704_070_800_000_000_000), None), + ]); + let range = fixture.range_partitioning([0, 1], vec![split]); + + // Start with single-key ranges; compound transforms can follow once + // tuple-boundary safety is established. + assert!(!range.keeps_keys_local( + &[fixture.col(0), date_trunc_of(fixture.col(1), "hour", None),], + &fixture.eq_properties, + false, + )); + Ok(()) + } + + #[test] + #[expect( + deprecated, + reason = "test intentionally constructs malformed range splits" + )] + fn range_transform_rejects_malformed_split_points() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; + let hour = 3_600_000_000_000; + let timestamp = |value| ScalarValue::TimestampNanosecond(Some(value), None); + + for (description, descending, splits) in [ + ( + "reversed ascending splits", + false, + vec![vec![timestamp(2 * hour)], vec![timestamp(hour)]], + ), + ( + "reversed descending splits", + true, + vec![vec![timestamp(hour - 1)], vec![timestamp(2 * hour - 1)]], + ), + ( + "duplicate ascending splits", + false, + vec![vec![timestamp(hour)], vec![timestamp(hour)]], + ), + ( + "duplicate descending splits", + true, + vec![vec![timestamp(hour - 1)], vec![timestamp(hour - 1)]], + ), + ( + "overwide split tuple", + false, + vec![vec![timestamp(hour), ScalarValue::Int64(Some(0))]], + ), + ("empty split tuple", false, vec![vec![]]), + ] { + let range = Partitioning::Range(RangePartitioning::new( + [PhysicalSortExpr::new( + fixture.col(0), + SortOptions::new(descending, false), + )] + .into(), + splits.into_iter().map(SplitPoint::new).collect(), + )); + assert_key_locality( + description, + &range, + &required, + &fixture.eq_properties, + false, + false, + ); + } + Ok(()) + } + + #[test] + fn range_transform_rejects_null_or_minimum_timestamp_splits() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; + + for (description, split) in [ + ("null", timestamp_split(None, None)), + ( + "minimum has no predecessor", + timestamp_split(Some(i64::MIN), None), + ), + ] { + assert_key_locality( + description, + &fixture.range_partitioning([0], vec![split]), + &required, + &fixture.eq_properties, + false, + false, + ); + } + Ok(()) + } + + #[test] + #[expect( + deprecated, + reason = "test intentionally constructs a malformed range split" + )] + fn range_transform_rejects_split_type_mismatch() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; + let mismatched_split = Partitioning::Range(RangePartitioning::new( + fixture.range_ordering([0]), + vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])], + )); + assert_key_locality( + "split scalar type must match the range key", + &mismatched_split, + &required, + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + + #[test] + fn range_transform_rejects_boundary_evaluation_errors() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let invalid_precision = + vec![date_trunc_of(fixture.col(0), "not-a-precision", None)]; + let aligned = fixture.range_partitioning( + [0], + vec![timestamp_split(Some(1_704_070_800_000_000_000), None)], + ); + assert_key_locality( + "split evaluation errors fail closed", + &aligned, + &invalid_precision, + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + + #[test] + fn range_transform_rejects_adjacent_timestamp_evaluation_errors() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let nanoseconds_per_hour = 3_600_000_000_000; + let first_representable_hour = + i64::MIN / nanoseconds_per_hour * nanoseconds_per_hour; + let split = + ScalarValue::TimestampNanosecond(Some(first_representable_hour), None); + let predecessor = + ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None); + let timestamp = fixture.col(0); + let trunc_hour = date_trunc_of(Arc::clone(×tamp), "hour", None); + + assert_eq!( + evaluate_with_range_value(&trunc_hour, ×tamp, &split), + Some(split.clone()), + ); + assert_eq!( + evaluate_with_range_value(&trunc_hour, ×tamp, &predecessor), + None, + ); + + let range = fixture.range_partitioning([0], vec![SplitPoint::new(vec![split])]); + assert_key_locality( + "predecessor truncation underflows even though split truncation succeeds", + &range, + &[trunc_hour], + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + + #[test] + fn range_transform_rejects_timezone_annotated_timestamps() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + )])?; + let trunc_hour = date_trunc_of(fixture.col(0), "hour", Some("UTC")); + let range = fixture.range_partitioning( + [0], + vec![timestamp_split( + Some(1_704_070_800_000_000_000), + Some("UTC"), + )], + ); + assert_key_locality( + "timezone-aware date_trunc is not audited", + &range, + &[trunc_hour], + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + + #[test] + fn descending_range_uses_the_open_side_successor() -> Result<()> { + let fixture = PartitioningTestFixture::new(vec![( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, None), + )])?; + let hour = 1_704_070_800_000_000_000; + let ordering: LexOrdering = + [fixture.range_sort_expr(0, SortOptions::new(true, false))].into(); + let required = vec![date_trunc_of(fixture.col(0), "hour", None)]; + + assert_key_locality( + "the successor of an hour boundary is in the same bucket", + &fixture.range_partitioning_with_ordering( + ordering.clone(), + vec![timestamp_split(Some(hour), None)], + ), + &required, + &fixture.eq_properties, + false, + false, + ); + assert_key_locality( + "the successor of the final nanosecond starts a new bucket", + &fixture.range_partitioning_with_ordering( + ordering.clone(), + vec![timestamp_split(Some(hour + 3_600_000_000_000 - 1), None)], + ), + &required, + &fixture.eq_properties, + true, + true, + ); + assert_key_locality( + "the maximum timestamp has no successor", + &fixture.range_partitioning_with_ordering( + ordering, + vec![timestamp_split(Some(i64::MAX), None)], + ), + &required, + &fixture.eq_properties, + false, + false, + ); + Ok(()) + } + #[test] #[expect( deprecated, diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index f7bd5469cd71e..1d9bc2f3fd55a 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -158,6 +158,25 @@ impl ScalarFunctionExpr { &self.config_options } + /// Returns whether this function expression's argument types have been + /// explicitly audited as candidates for range-key locality analysis. + /// + /// See [`ScalarUDFImpl::supports_range_key_locality_analysis`] for the safety + /// requirements. + pub(crate) fn supports_range_key_locality_analysis( + &self, + input_schema: &Schema, + ) -> Result { + let argument_types = self + .args + .iter() + .map(|argument| argument.data_type(input_schema)) + .collect::>>()?; + Ok(self + .fun + .supports_range_key_locality_analysis(&argument_types)) + } + /// Given an arbitrary PhysicalExpr attempt to downcast it to a ScalarFunctionExpr /// and verify that its inner function is of type T. /// If the downcast fails, or the function is not of type T, returns `None`. diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs index 49776d8843d40..168eef838d43a 100644 --- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs @@ -915,6 +915,20 @@ fn partial_aggregate_output_satisfies_final_partitioning( .is_satisfied() } +fn partial_aggregate_input_keeps_groups_local( + plan: &Arc, + allow_subset_satisfy_partitioning: bool, +) -> bool { + plan.downcast_ref::() + .is_some_and(|aggregate| { + aggregate.mode() == &AggregateMode::Partial + && aggregate.input_keeps_groups_local( + aggregate.input().as_ref(), + allow_subset_satisfy_partitioning, + ) + }) +} + /// Adds a [`SortPreservingMergeExec`] or a [`CoalescePartitionsExec`] operator /// on top of the given plan node to satisfy a single partition requirement /// while preserving ordering constraints. @@ -1686,10 +1700,13 @@ pub fn ensure_distribution_with_stats( let preserve_partial_aggregate_partitioning = preserve_file_partition_threshold_met - && partial_aggregate_output_satisfies_final_partitioning( + && (partial_aggregate_output_satisfies_final_partitioning( + &plan, + allow_subset_satisfy_partitioning, + ) || partial_aggregate_input_keeps_groups_local( &plan, allow_subset_satisfy_partitioning, - ); + )); let add_roundrobin = enable_round_robin // Operator benefits from partitioning (e.g. filter): @@ -1721,6 +1738,9 @@ pub fn ensure_distribution_with_stats( Distribution::HashPartitioned(_) | Distribution::KeyPartitioned(_) => { let child_partitions = child.plan.output_partitioning().partition_count(); + // An aggregate can avoid repartitioning if all rows for each group + // are already in one input partition, even when the ordinary + // distribution check fails (#25344). This does not prove join co-partitioning. let partitioning_satisfied = input_distributions .child_satisfaction( child_idx, @@ -1728,7 +1748,15 @@ pub fn ensure_distribution_with_stats( ChildSatisfactionOptions::new() .with_allow_subset(allow_subset_satisfy_partitioning), )? - .is_satisfied(); + .is_satisfied() + || plan.downcast_ref::().is_some_and( + |aggregate| { + aggregate.input_keeps_groups_local( + child.plan.as_ref(), + allow_subset_satisfy_partitioning, + ) + }, + ); let preserve_satisfying_file_partitioning = preserve_file_partition_threshold_met && !requires_grouping_id diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs index 713213b70612d..746746761b864 100644 --- a/datafusion/physical-optimizer/src/sanity_checker.rs +++ b/datafusion/physical-optimizer/src/sanity_checker.rs @@ -30,6 +30,7 @@ use datafusion_common::config::{ConfigOptions, OptimizerOptions}; use datafusion_common::plan_err; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported}; +use datafusion_physical_plan::aggregates::AggregateExec; use datafusion_physical_plan::execution_plan::{ Boundedness, EmissionType, InvariantLevel, }; @@ -167,14 +168,17 @@ pub fn check_plan_sanity( } } - if !input_distributions - .child_satisfaction( - idx, - child.as_ref(), - ChildSatisfactionOptions::new().with_allow_subset(true), - )? - .is_satisfied() - { + let distribution_satisfaction = input_distributions.child_satisfaction( + idx, + child.as_ref(), + ChildSatisfactionOptions::new().with_allow_subset(true), + )?; + let accepts_partition_local_groups = plan + .downcast_ref::() + .is_some_and(|aggregate| { + aggregate.input_keeps_groups_local(child.as_ref(), true) + }); + if !distribution_satisfaction.is_satisfied() && !accepts_partition_local_groups { let plan_str = get_plan_string(plan); return plan_err!( "Plan: {:?} does not satisfy distribution requirements: {}. Child-{} output partitioning: {}", diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs index 89131b38bce1a..1638ba545551f 100644 --- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs +++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs @@ -1500,7 +1500,10 @@ mod tests { use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_expr::{AggregateMetric, AggregateMetrics}; use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf}; - use datafusion_physical_expr::aggregate::AggregateExprBuilder; + use datafusion_physical_expr::GroupsAccumulatorAdapter; + use datafusion_physical_expr::aggregate::{ + AggregateExprBuilder, AggregateFunctionExpr, + }; use datafusion_physical_expr::expressions::col; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -1527,6 +1530,18 @@ mod tests { } } + /// `array_agg(DISTINCT)` has a native groups accumulator, so build the + /// per-group adapter explicitly to cover its metric handling. + fn distinct_adapter( + aggregate_expr: Arc, + metrics: Arc, + ) -> GroupsAccumulatorAdapter { + let mut adapter = + GroupsAccumulatorAdapter::new(move || aggregate_expr.create_accumulator()); + adapter.set_metrics(metrics); + adapter + } + #[test] fn legacy_grouped_distinct_installs_metrics_for_conversion_and_update() -> Result<()> { @@ -1544,13 +1559,13 @@ mod tests { ); let metric_resolutions = Arc::new(AtomicUsize::new(0)); let metric_durations = Arc::new(AtomicUsize::new(0)); - let mut accumulator = create_group_accumulator( - &aggregate_expr, + let mut accumulator = distinct_adapter( + aggregate_expr, Arc::new(CountingMetrics { resolutions: Arc::clone(&metric_resolutions), durations: Arc::clone(&metric_durations), }), - )?; + ); let values: ArrayRef = Arc::new(UInt32Array::from(vec![1, 2, 3])); accumulator.convert_to_state(&[Arc::clone(&values)], None)?; @@ -1579,13 +1594,13 @@ mod tests { ); let metric_resolutions = Arc::new(AtomicUsize::new(0)); let metric_durations = Arc::new(AtomicUsize::new(0)); - let mut accumulator = create_group_accumulator( - &aggregate_expr, + let mut accumulator = distinct_adapter( + aggregate_expr, Arc::new(CountingMetrics { resolutions: Arc::clone(&metric_resolutions), durations: Arc::clone(&metric_durations), }), - )?; + ); let mut state = ListBuilder::new(UInt32Builder::new()); for value in [1, 2, 3] { diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 7957c804d5d3b..226ce3d271973 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1654,6 +1654,52 @@ impl AggregateExec { &self.input } + /// Whether a candidate input keeps each group in one partition. + /// + /// Unlike co-partitioning, aggregation only needs equality groups to be + /// local. A final stage can inherit that guarantee from a partial stage + /// even when projecting the grouping expressions loses range metadata. + /// This check does not require the two stages to be combined. + pub fn input_keeps_groups_local( + &self, + input: &dyn ExecutionPlan, + allow_subset: bool, + ) -> bool { + if !matches!( + self.mode, + AggregateMode::Partial + | AggregateMode::SinglePartitioned + | AggregateMode::FinalPartitioned + ) || self.group_by().is_empty() + || self.group_by().has_grouping_set() + { + return false; + } + + if input.output_partitioning().keeps_keys_local( + &self.group_by().input_exprs(), + input.equivalence_properties(), + allow_subset, + ) { + return true; + } + + self.mode == AggregateMode::FinalPartitioned + && input + .downcast_ref::() + .is_some_and(|partial| { + partial.mode == AggregateMode::Partial + && datafusion_physical_expr::physical_exprs_equal( + &partial.output_group_expr(), + &self.group_by().input_exprs(), + ) + && partial.input_keeps_groups_local( + partial.input.as_ref(), + allow_subset, + ) + }) + } + /// Get the input schema before any aggregates are applied pub fn input_schema(&self) -> SchemaRef { Arc::clone(&self.input_schema) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index eca97556d024a..5309f4c6f5109 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1217,13 +1217,9 @@ impl HashJoinExec { return false; } - // `preserve_file_partitions` can report Hive-style file groups as Hash - // partitioned even though their partition indexes do not follow the - // hash router used by partitioned dynamic filters. Reject Hash inputs - // because the metadata cannot distinguish those scans from a real hash - // repartition. Compatible Range inputs remain safe because matching - // ordering and split points align each build filter with its probe - // partition. Other unsupported layouts are rejected. + // Under `preserve_file_partitions`, file groups can report Hash without following + // the dynamic filter hash router and look like a real hash repartition, so reject + // Hash. Range inputs with matching split points are safe. // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195). // https://github.com/apache/datafusion/issues/20195 if config.optimizer.preserve_file_partitions > 0 diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs index a215a01e84260..40821583678d1 100644 --- a/datafusion/proto/tests/cases/plans/sources.rs +++ b/datafusion/proto/tests/cases/plans/sources.rs @@ -1144,3 +1144,84 @@ fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> { Ok(()) } + +#[test] +fn roundtrip_parquet_exec_range_output_partitioning_dict_split_points() -> Result<()> { + let file_schema = Arc::new(Schema::new(vec![Field::new( + "col", + wrap_partition_type_in_dict(DataType::Utf8), + false, + )])); + let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema))); + let output_partitioning = Partitioning::Range(RangePartitioning::try_new( + LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( + "col", 0, + )))]) + .unwrap(), + vec![SplitPoint::new(vec![wrap_partition_value_in_dict( + ScalarValue::from("B"), + )])], + )?); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![ + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-1.parquet".to_string(), + 1024, + )]), + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-2.parquet".to_string(), + 1024, + )]), + ]) + .with_output_partitioning(Some(output_partitioning.clone())) + .build(); + + assert_eq!( + roundtrip_file_scan_config(scan_config)?.output_partitioning, + Some(output_partitioning) + ); + + Ok(()) +} + +#[test] +fn roundtrip_parquet_exec_range_output_partitioning_utf8view_split_points() -> Result<()> +{ + let file_schema = Arc::new(Schema::new(vec![Field::new( + "col", + DataType::Utf8View, + false, + )])); + let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema))); + let output_partitioning = Partitioning::Range(RangePartitioning::try_new( + LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( + "col", 0, + )))]) + .unwrap(), + vec![SplitPoint::new(vec![ScalarValue::Utf8View(Some( + "B".to_string(), + ))])], + )?); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![ + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-1.parquet".to_string(), + 1024, + )]), + FileGroup::new(vec![PartitionedFile::new( + "/path/to/file-2.parquet".to_string(), + 1024, + )]), + ]) + .with_output_partitioning(Some(output_partitioning.clone())) + .build(); + + assert_eq!( + roundtrip_file_scan_config(scan_config)?.output_partitioning, + Some(output_partitioning) + ); + + Ok(()) +} diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs index 2cb4f57e29001..4989294146814 100644 --- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs +++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs @@ -423,9 +423,9 @@ type TimeBinRow = ( /// - partition 1: `[2024-01-01 01:00, 02:00)` /// /// Files are range-partitioned on `timestamp` and sorted on `(key, timestamp)`. -/// Because `date_bin(60 seconds, timestamp)` does not straddle the hour split, -/// grouping by `(key, time_bin)` is partition-disjoint. Today's planner still -/// inserts a hash shuffle; the test pins that plan so a follow-up can remove it. +/// The hour split permits partition-local `date_trunc('hour')` aggregation, +/// but `date_trunc('day')` straddles it. Aligned `date_bin` remains a negative +/// control because the function is not admitted to range analysis. pub(super) fn register_range_sorted_time_bin_table(ctx: &SessionContext) { let schema = Arc::new(Schema::new(vec![ Field::new("key", DataType::Utf8, false), diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.slt index 8342380e31a43..89b7bcf3f50a5 100644 --- a/datafusion/sqllogictest/test_files/array_agg.slt +++ b/datafusion/sqllogictest/test_files/array_agg.slt @@ -678,3 +678,40 @@ List(Dictionary(Int32, Utf8)) statement ok drop table array_agg_distinct_dict_table; + +# array_agg(DISTINCT) per group, mixed with other aggregates and a FILTER +statement ok +create table array_agg_distinct_groups(g int, v varchar) as values + (1, 'b'), (1, 'a'), (1, 'b'), (1, null), + (2, 'a'), (2, 'a'), + (3, null), + (4, 'c'); + +query II?? +select + g, + count(*), + array_sort(array_agg(DISTINCT v) FILTER (WHERE v <> 'c')), + array_sort(array_agg(DISTINCT v)) +from array_agg_distinct_groups +group by g +order by g; +---- +1 4 [a, b] [NULL, a, b] +2 2 [a] [a] +3 1 NULL [NULL] +4 1 NULL [c] + +query I? +select g, array_agg(DISTINCT v ORDER BY v DESC) ignore nulls +from array_agg_distinct_groups +group by g +order by g; +---- +1 [b, a] +2 [a] +3 NULL +4 [c] + +statement ok +drop table array_agg_distinct_groups; diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt index a17ce11a0bff0..7e4d100b74237 100644 --- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt +++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt @@ -45,9 +45,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, 3); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] # Verify result correctness @@ -63,9 +63,9 @@ explain select * from t1 right join t2 on t1.a = t2.x where t1.a in (1, 2); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -80,9 +80,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t1.a in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -97,9 +97,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2, null); ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(NULL)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(NULL)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -168,9 +168,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t1.a in (1, 2, 3); ---- logical_plan 01)Left Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) OR t1.a = Int32(3) +02)--Filter: t1.a IN ([Int32(1), Int32(2), Int32(3)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x = Int32(3) +04)--Filter: t2.x IN ([Int32(1), Int32(2), Int32(3)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -185,7 +185,7 @@ query TT explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) or t2.x is null; ---- logical_plan -01)Filter: t2.x = Int32(1) OR t2.x = Int32(2) OR t2.x IS NULL +01)Filter: t2.x IN ([Int32(1), Int32(2)]) OR t2.x IS NULL 02)--Left Join: t1.a = t2.x 03)----TableScan: t1 projection=[a, b, c] 04)----TableScan: t2 projection=[x, y, z] @@ -459,9 +459,9 @@ explain select * from t1 full join t2 on t1.a = t2.x where t2.x in (1, 2); ---- logical_plan 01)Right Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: t2.x = Int32(1) OR t2.x = Int32(2) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort @@ -512,9 +512,9 @@ explain select * from t1 left join t2 on t1.a = t2.x where t2.x in (1, 2) and t2 ---- logical_plan 01)Inner Join: t1.a = t2.x -02)--Filter: t1.a = Int32(1) OR t1.a = Int32(2) +02)--Filter: t1.a IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[a, b, c] -04)--Filter: (t2.x = Int32(1) OR t2.x = Int32(2)) AND t2.y >= Int32(50) AND t2.y <= Int32(250) +04)--Filter: t2.x IN ([Int32(1), Int32(2)]) AND t2.y >= Int32(50) AND t2.y <= Int32(250) 05)----TableScan: t2 projection=[x, y, z] query IITIIT rowsort diff --git a/datafusion/sqllogictest/test_files/in_list.slt b/datafusion/sqllogictest/test_files/in_list.slt index 8fde8b11fe717..2a41eb3426105 100644 --- a/datafusion/sqllogictest/test_files/in_list.slt +++ b/datafusion/sqllogictest/test_files/in_list.slt @@ -20,9 +20,11 @@ # # This file focuses on the IN operator and its various specializations # -# Note that "short" IN LISTS do not go through the InList implementation at all, -# instead they are rewritten into a series of OR expressions. See: -# https://github.com/apache/datafusion/blob/ed37b6c9555bc278130dc774ed833b8c0bd29bfa/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs#L39-L88 +# One-item IN lists are rewritten to a comparison. Two- and three-item literal +# lists on Int32, Int64, UInt64, Float32, Float64, or FixedSizeBinary columns +# with widths 1, 2, 4, 8, or 16 retain their static InList filters. Other short +# lists on columns are rewritten to OR/AND expressions. See: +# https://github.com/apache/datafusion/blob/main/datafusion/optimizer/src/simplify_expressions/inlist_simplifier.rs ########## @@ -211,6 +213,69 @@ CREATE TABLE in_list_ints_nullable ( ('no_match', 7, 7, 7, 7, 7, 7, 7, 7), ('nulls', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +# Measured integer representations retain two- and three-item literal lists. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match false true false true false true +nulls NULL NULL NULL NULL NULL NULL + +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(NULL, 'Int32'), arrow_cast(11, 'Int32')), + i32 NOT IN (arrow_cast(NULL, 'Int32'), arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')), + i64 IN (arrow_cast(NULL, 'Int64'), arrow_cast(3, 'Int64'), arrow_cast(11, 'Int64')), + i64 NOT IN (arrow_cast(NULL, 'Int64'), arrow_cast(11, 'Int64')), + u64 IN (arrow_cast(NULL, 'UInt64'), arrow_cast(11, 'UInt64')), + u64 NOT IN (arrow_cast(NULL, 'UInt64'), arrow_cast(3, 'UInt64'), arrow_cast(11, 'UInt64')) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match NULL NULL NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list keeps the comparison path for these representations. +query TBBBBBB +SELECT label, + i32 IN (arrow_cast(3, 'Int32'), i16), + i32 NOT IN (arrow_cast(3, 'Int32'), i16), + i64 IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), i32), + u64 IN (arrow_cast(3, 'UInt64'), u32), + u64 NOT IN (arrow_cast(3, 'UInt64'), u32) +FROM in_list_ints_nullable +ORDER BY label +---- +match true false true false true false +no_match true false true false true false +nulls NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT i32 IN (arrow_cast(3, 'Int32'), arrow_cast(11, 'Int32')) AS i32_in_two, + i64 NOT IN (arrow_cast(3, 'Int64'), arrow_cast(4, 'Int64'), arrow_cast(11, 'Int64')) AS i64_not_in_three, + u64 IN (arrow_cast(3, 'UInt64'), arrow_cast(4, 'UInt64'), arrow_cast(11, 'UInt64')) AS u64_in_three, + i64 IN (arrow_cast(1, 'Int64')) AS i64_in_one, + i64 IN (arrow_cast(3, 'Int64'), i32) AS i64_in_column, + i64 NOT IN (arrow_cast(3, 'Int64'), i32) AS i64_not_in_column +FROM in_list_ints_nullable; +---- +logical_plan +01)Projection: in_list_ints_nullable.i32 IN ([Int32(3), Int32(11)]) AS i32_in_two, in_list_ints_nullable.i64 NOT IN ([Int64(3), Int64(4), Int64(11)]) AS i64_not_in_three, in_list_ints_nullable.u64 IN ([UInt64(3), UInt64(4), UInt64(11)]) AS u64_in_three, in_list_ints_nullable.i64 = Int64(1) AS i64_in_one, in_list_ints_nullable.i64 = Int64(3) OR in_list_ints_nullable.i64 = CAST(in_list_ints_nullable.i32 AS Int64) AS i64_in_column, in_list_ints_nullable.i64 != Int64(3) AND in_list_ints_nullable.i64 != CAST(in_list_ints_nullable.i32 AS Int64) AS i64_not_in_column +02)--TableScan: in_list_ints_nullable projection=[i32, i64, u64] +physical_plan +01)ProjectionExec: expr=[i32@0 IN (SET) ([3, 11]) as i32_in_two, i64@1 NOT IN (SET) ([3, 4, 11]) as i64_not_in_three, u64@2 IN (SET) ([3, 4, 11]) as u64_in_three, i64@1 = 1 as i64_in_one, i64@1 = 3 OR i64@1 = CAST(i32@0 AS Int64) as i64_in_column, i64@1 != 3 AND i64@1 != CAST(i32@0 AS Int64) as i64_not_in_column] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Null input values return NULL when the IN list has no nulls. query TBBBBBBBB SELECT @@ -308,6 +373,79 @@ FROM (VALUES ('nulls', NULL) ) AS t(label, value); +# Short Float32 and Float64 lists retain membership filters for IN and NOT IN. +query TBBBBBBBB +SELECT + label, + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(11.0, 'Float32')), + f32 IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')), + f32 NOT IN (NULL, arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TBBBBBBBB +SELECT + label, + f64 IN (3.0, 11.0), + f64 NOT IN (3.0, 11.0), + f64 IN (3.0, 4.0, 11.0), + f64 NOT IN (3.0, 4.0, 11.0), + f64 IN (NULL, 11.0), + f64 NOT IN (NULL, 11.0), + f64 IN (NULL, 3.0, 11.0), + f64 NOT IN (NULL, 3.0, 11.0) +FROM in_list_floats +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +# A list containing a column is evaluated through comparisons. +query TBBBB +SELECT label, + f32 IN (arrow_cast(3.0, 'Float32'), f16), + f32 NOT IN (arrow_cast(3.0, 'Float32'), f16), + f64 IN (3.0, f32), f64 NOT IN (3.0, f32) +FROM in_list_floats +ORDER BY label +---- +match true false true false +no_match true false true false +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT f32 IN (arrow_cast(3.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS in_two, + f32 NOT IN (arrow_cast(3.0, 'Float32'), arrow_cast(4.0, 'Float32'), arrow_cast(11.0, 'Float32')) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f32 IN ([Float32(3), Float32(11)]) AS in_two, in_list_floats.f32 NOT IN ([Float32(3), Float32(4), Float32(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f32] +physical_plan +01)ProjectionExec: expr=[f32@0 IN (SET) ([3, 11]) as in_two, f32@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +EXPLAIN SELECT f64 IN (3.0, 11.0) AS in_two, f64 NOT IN (3.0, 4.0, 11.0) AS not_in_three +FROM in_list_floats; +---- +logical_plan +01)Projection: in_list_floats.f64 IN ([Float64(3), Float64(11)]) AS in_two, in_list_floats.f64 NOT IN ([Float64(3), Float64(4), Float64(11)]) AS not_in_three +02)--TableScan: in_list_floats projection=[f64] +physical_plan +01)ProjectionExec: expr=[f64@0 IN (SET) ([3, 11]) as in_two, f64@0 NOT IN (SET) ([3, 4, 11]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Five element IN lists cover the specialized Float16/32/64 paths. query TTBBB SELECT @@ -616,6 +754,82 @@ LIMIT 1 ---- FixedSizeBinary(1) FixedSizeBinary(2) FixedSizeBinary(3) FixedSizeBinary(4) FixedSizeBinary(8) FixedSizeBinary(16) +# Short lists at the width endpoints cover both list lengths and typed nulls. +query TBBBBBB +SELECT + label, + fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'04', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb1 IN (arrow_cast(NULL, 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')), + fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')), + fsb16 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true true false false +no_match false true NULL false true NULL +nulls NULL NULL NULL NULL NULL NULL + +# A column-valued list retains the comparison path. +query TBBBB +SELECT l.label, + l.fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb1 NOT IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), r.fsb1), + l.fsb16 IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16), + l.fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), r.fsb16) +FROM in_list_fsb l CROSS JOIN in_list_fsb r +WHERE r.label = 'match' +ORDER BY l.label +---- +match true false true false +no_match false true false true +nulls NULL NULL NULL NULL + +query TT +EXPLAIN SELECT fsb1 IN (arrow_cast(X'03', 'FixedSizeBinary(1)'), arrow_cast(X'0B', 'FixedSizeBinary(1)')) AS in_two, + fsb16 NOT IN (arrow_cast(X'00000000000000000000000000000003', 'FixedSizeBinary(16)'), arrow_cast(X'00000000000000000000000000000004', 'FixedSizeBinary(16)'), arrow_cast(X'0000000000000000000000000000000B', 'FixedSizeBinary(16)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb1 IN ([FixedSizeBinary(1, "3"), FixedSizeBinary(1, "11")]) AS in_two, in_list_fsb.fsb16 NOT IN ([FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4"), FixedSizeBinary(16, "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb1, fsb16] +physical_plan +01)ProjectionExec: expr=[fsb1@0 IN (SET) ([03, 0B]) as in_two, fsb16@1 NOT IN (SET) ([00000000000000000000..., 00000000000000000000..., 00000000000000000000...]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Short eight-byte lists retain membership filters, including list nulls. +query TBBBBBBBB +SELECT + label, + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')), + fsb8 NOT IN (arrow_cast(NULL, 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) +FROM in_list_fsb +ORDER BY label +---- +match true false true false true false true false +no_match false true false true NULL NULL NULL NULL +nulls NULL NULL NULL NULL NULL NULL NULL NULL + +query TT +EXPLAIN SELECT + fsb8 IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS in_two, + fsb8 NOT IN (arrow_cast(X'0000000000000003', 'FixedSizeBinary(8)'), arrow_cast(X'0000000000000004', 'FixedSizeBinary(8)'), arrow_cast(X'000000000000000B', 'FixedSizeBinary(8)')) AS not_in_three +FROM in_list_fsb; +---- +logical_plan +01)Projection: in_list_fsb.fsb8 IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS in_two, in_list_fsb.fsb8 NOT IN ([FixedSizeBinary(8, "0,0,0,0,0,0,0,3"), FixedSizeBinary(8, "0,0,0,0,0,0,0,4"), FixedSizeBinary(8, "0,0,0,0,0,0,0,11")]) AS not_in_three +02)--TableScan: in_list_fsb projection=[fsb8] +physical_plan +01)ProjectionExec: expr=[fsb8@0 IN (SET) ([0000000000000003, 000000000000000B]) as in_two, fsb8@0 NOT IN (SET) ([0000000000000003, 0000000000000004, 000000000000000B]) as not_in_three] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + # Eight element lists, all matching or none matching query TBBBBBB SELECT diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.slt index 0e0ceb9cfa4fa..40fb9d7d556aa 100644 --- a/datafusion/sqllogictest/test_files/lateral_join.slt +++ b/datafusion/sqllogictest/test_files/lateral_join.slt @@ -755,16 +755,19 @@ ORDER BY t1.id, sub.x; # --- Correlated filter below a window function --- # https://github.com/apache/datafusion/issues/25792 -# Pulling `t2.t1_id = t1.id` above the window would number the t2 rows of all -# t1 rows together instead of the t2 rows of each t1 row. -# Expected result once supported: (1, 10, 1), (1, 20, 2), (2, 30, 1). -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn +# The pull up adds `t2.t1_id` to the `PARTITION BY` of the window, so each t1 +# row still numbers only its own t2 rows. +query III SELECT t1.id, sub.id, sub.rn FROM t1, LATERAL ( SELECT t2.id, row_number() OVER (ORDER BY t2.id) AS rn FROM (SELECT * FROM t2 WHERE t2.t1_id = t1.id) AS t2 ) AS sub ORDER BY t1.id, sub.id; +---- +1 10 1 +1 20 2 +2 30 1 # A correlated filter above the window is still decorrelated. The window # numbers all t2 rows, then the filter keeps the rows of each t1 row. @@ -850,8 +853,8 @@ ORDER BY t1.id, sub.id; 1 20 2 2 30 1 -# The filter must read a `PARTITION BY` column of every window function. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn +# Every window function is partitioned by the columns of the filter. +query IIII SELECT t1.id, sub.id, sub.rn, sub.c FROM t1, LATERAL ( SELECT t2.id, @@ -860,6 +863,10 @@ FROM t1, LATERAL ( FROM t2 WHERE t2.t1_id = t1.id ) AS sub ORDER BY t1.id, sub.id; +---- +1 10 1 1 +1 20 2 1 +2 30 1 1 # --- Implicit LATERAL (comma without LATERAL keyword) --- # DuckDB auto-detects lateral correlation; DataFusion requires diff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt index f4aab94ea627d..3fb4a3e17c3b9 100644 --- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt +++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt @@ -125,7 +125,7 @@ STORED AS PARQUET; 1 # Create high-cardinality fact table (5 partitions > 3 target_partitions) -# For testing partition merging with consistent hashing +# For testing partition merging into contiguous value ranges query I COPY (SELECT column1 as timestamp, column2 as value FROM (VALUES (TIMESTAMP '2023-01-01T09:00:00', 100.0) @@ -258,7 +258,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(fact_table.value)@2 as sum(fact_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(fact_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet # Verify results with optimization match results without optimization query TIR rowsort @@ -320,7 +320,7 @@ physical_plan 01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST] 02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)] 03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet query TIR SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; @@ -418,7 +418,7 @@ physical_plan 06)----------FilterExec: service@2 = log 07)------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TTTIR rowsort SELECT f.f_dkey, MAX(d.env), MAX(d.service), count(*), sum(f.value) @@ -493,7 +493,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet query TPRI rowsort SELECT f_dkey, timestamp, value, @@ -513,8 +513,7 @@ C 2023-01-01T09:00:20 310.2 3 ########## # TEST 9: High-Cardinality Partitions (more partitions than target_partitions) -# Since num_partitions > target_partitions (5 > 3), files are merged using -# round-robin assignment to ensure exactly target_partitions groups are created. +# Sorted partition values are cut into contiguous chunks. ########## # First verify results without optimization @@ -537,7 +536,7 @@ statement ok set datafusion.optimizer.preserve_file_partitions = 1; # Verify the plan uses SinglePartitioned mode with no RepartitionExec -# The 5 partitions are merged into 3 file groups using round-robin assignment +# 5 values merge into 3 contiguous groups: [A], [B, C], [D, E] query TT EXPLAIN SELECT f_dkey, count(*), sum(value) FROM high_cardinality_table GROUP BY f_dkey; ---- @@ -548,7 +547,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(high_cardinality_table.value)@2 as sum(high_cardinality_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(high_cardinality_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (D)], 3), file_type=parquet # Verify results with optimization match results without optimization query TIR rowsort @@ -658,9 +657,7 @@ C prod 2017.6 ########## # TEST 12: Partitioned Join with Matching Partition Counts - With Optimization # Both tables have 3 partitions matching target_partitions=3 -# No RepartitionExec needed for join - partitions already satisfy the requirement -# Dynamic filter pushdown is disabled in this mode because preserve_file_partitions -# reports Hash partitioning for Hive-style file groups, which are not hash-routed. +# Identical split points prove co-partitioning, so no RepartitionExec is needed. ########## statement ok @@ -685,8 +682,8 @@ physical_plan 02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] 04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0] -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Hash([d_dkey@1], 3), file_type=parquet -06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), file_type=parquet +06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TTR rowsort SELECT f.f_dkey, d.env, sum(f.value) @@ -722,7 +719,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@0 as f_dkey, timestamp@1 as timestamp, count(Int64(1))@2 as count(*), avg(fact_table.value)@3 as avg(fact_table.value)] 02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, timestamp@0 as timestamp], aggr=[count(Int64(1)), avg(fact_table.value)] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet query TPIR rowsort SELECT f_dkey, timestamp, @@ -752,6 +749,196 @@ C 2023-01-01T09:00:40 1 300 C 2023-01-01T09:12:40 1 250 C 2023-01-01T09:12:50 1 275.4 +########## +# TEST 14: Partitioned join, same group count, different partition values +# fact has {A, B, C}, dimension has {A, C, D}. Split points differ, so the scans are not +# co-partitioned and one side must be range repartitioned. +########## + +query I +COPY (SELECT 'dev' as env, 'log' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet' +STORED AS PARQUET; +---- +1 + +query I +COPY (SELECT 'prod' as env, 'log' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet' +STORED AS PARQUET; +---- +1 + +query I +COPY (SELECT 'prod' as env, 'trace' as service) +TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet' +STORED AS PARQUET; +---- +1 + +statement ok +CREATE EXTERNAL TABLE dimension_table_skewed (env STRING, service STRING) +STORED AS PARQUET +PARTITIONED BY (d_dkey STRING) +LOCATION 'test_files/scratch/preserve_file_partitioning/dimension_skewed/'; + +statement ok +set datafusion.execution.target_partitions = 3; + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold = 0; + +statement ok +set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0; + +# Exactly one join input gets a RepartitionExec(Range). +query TT +EXPLAIN SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +logical_plan +01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]] +02)--Projection: f.value, f.f_dkey, d.env +03)----Inner Join: f.f_dkey = d.d_dkey +04)------SubqueryAlias: f +05)--------TableScan: fact_table projection=[value, f_dkey] +06)------SubqueryAlias: d +07)--------TableScan: dimension_table_skewed projection=[env, d_dkey] +physical_plan +01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)] +02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 +03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] +04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0] +05)--------RepartitionExec: partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), input_partitions=3 +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(C), (D)], 3), file_type=parquet +07)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + +# Only A and C match. +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +C prod 2017.6 + +# Same result with the optimization disabled. +statement ok +set datafusion.optimizer.preserve_file_partitions = 0; + +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +C prod 2017.6 + +########## +# TEST 15: Partitioned join against a non partitioned VALUES input +# The inserted RepartitionExec reuses the scan's split points, so partition i on both +# sides covers the same key interval. +########## + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +query TT +EXPLAIN SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env) + ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +logical_plan +01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]] +02)--Projection: f.value, f.f_dkey, d.env +03)----Inner Join: f.f_dkey = CAST(d.d_dkey AS Utf8View) +04)------SubqueryAlias: f +05)--------TableScan: fact_table projection=[value, f_dkey] +06)------SubqueryAlias: d +07)--------Projection: column1 AS d_dkey, column2 AS env +08)----------Values: (Utf8("A"), Utf8("dev")), (Utf8("B"), Utf8("prod")), (Utf8("C"), Utf8("prod")), (Utf8("Z"), Utf8("none")) +physical_plan +01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)] +02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3 +03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)] +04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(d.d_dkey AS Utf8View)@2, f_dkey@1)], projection=[value@3, f_dkey@4, env@1] +05)--------RepartitionExec: partitioning=Range([CAST(d.d_dkey AS Utf8View)@2 ASC], [(B), (C)], 3), input_partitions=1 +06)----------ProjectionExec: expr=[column1@0 as d_dkey, column2@1 as env, CAST(column1@0 AS Utf8View) as CAST(d.d_dkey AS Utf8View)] +07)------------DataSourceExec: partitions=1, partition_sizes=[1] +08)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + +query TTR rowsort +SELECT f.f_dkey, d.env, sum(f.value) +FROM fact_table f +INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env) + ON f.f_dkey = d.d_dkey +GROUP BY f.f_dkey, d.env; +---- +A dev 772.4 +B prod 614.4 +C prod 2017.6 + +########## +# TEST 16: Self join on merged groups keeps every row +# 5 values merge into [A], [B, C], [D, E], and all must stay routable. +########## + +query TII rowsort +SELECT a.f_dkey, count(*), count(b.value) +FROM high_cardinality_table a +INNER JOIN high_cardinality_table b ON a.f_dkey = b.f_dkey +GROUP BY a.f_dkey; +---- +A 1 1 +B 1 1 +C 1 1 +D 1 1 +E 1 1 + +########## +# TEST 17: Statistics regrouping drops the Range claim +# The re-cut groups no longer follow partition values, so no output partitioning is +# declared, even when the group count is unchanged. +########## + +statement ok +set datafusion.optimizer.preserve_file_partitions = 1; + +statement ok +set datafusion.execution.split_file_groups_by_statistics = true; + +query TT +EXPLAIN SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; +---- +logical_plan +01)Sort: fact_table_ordered.f_dkey ASC NULLS LAST +02)--Projection: fact_table_ordered.f_dkey, count(Int64(1)) AS count(*), avg(fact_table_ordered.value) +03)----Aggregate: groupBy=[[fact_table_ordered.f_dkey]], aggr=[[count(Int64(1)), avg(fact_table_ordered.value)]] +04)------TableScan: fact_table_ordered projection=[value, f_dkey] +physical_plan +01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST] +02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)] +03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full +04)------RepartitionExec: partitioning=Hash([f_dkey@0], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST +05)--------AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet + +query TIR +SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey; +---- +A 7 110.342857142857 +B 7 87.771428571429 +C 7 288.228571428571 + ########## # CLEANUP ########## @@ -769,6 +956,9 @@ reset datafusion.optimizer.hash_join_single_partition_threshold; statement ok reset datafusion.optimizer.hash_join_single_partition_threshold_rows; +statement ok +reset datafusion.execution.split_file_groups_by_statistics; + statement ok DROP TABLE fact_table; @@ -781,5 +971,8 @@ DROP TABLE dimension_table; statement ok DROP TABLE dimension_table_partitioned; +statement ok +DROP TABLE dimension_table_skewed; + statement ok DROP TABLE high_cardinality_table; diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index 104b38e1a5036..619ebab6ba567 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -257,7 +257,7 @@ logical_plan 01)Sort: t1.t1_id ASC NULLS LAST, t2.t2_id ASC NULLS LAST 02)--Inner Join: Filter: t1.t1_id <= t2.t2_id 03)----SubqueryAlias: t1 -04)------Filter: join_t1.t1_id = Int32(11) OR join_t1.t1_id = Int32(44) +04)------Filter: join_t1.t1_id IN ([Int32(11), Int32(44)]) 05)--------TableScan: join_t1 projection=[t1_id] 06)----SubqueryAlias: t2 07)------Projection: join_t2.t2_id @@ -268,7 +268,7 @@ physical_plan 02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=LtEq, join_type=Inner, on=(t1_id <= t2_id) 04)------SortExec: expr=[t1_id@0 DESC NULLS LAST], preserve_partitioning=[false] -05)--------FilterExec: t1_id@0 = 11 OR t1_id@0 = 44 +05)--------FilterExec: t1_id@0 IN (SET) ([11, 44]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] 07)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 08)--------FilterExec: t2_name@1 != y, projection=[t2_id@0] diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt index 56184845a6fe3..21f9f439a2085 100644 --- a/datafusion/sqllogictest/test_files/range_partitioning.slt +++ b/datafusion/sqllogictest/test_files/range_partitioning.slt @@ -1928,7 +1928,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- physical_plan 01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3] -02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20, pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] +02)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20]), pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)] 03)--DataSourceExec: file_groups=, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible query TT @@ -1942,7 +1942,7 @@ JOIN range_partitioned p ON b.range_key = p.range_key; ---- Plan with Metrics 01)HashJoinExec: mode=Partitionedmetrics=[output_rows=2,] -02)--DataSourceExec: file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20metrics=[output_rows=2,] +02)--DataSourceExec: file_type=parquet, predicate=range_key@0 IN (SET) ([5, 20])metrics=[output_rows=2,] 03)--DataSourceExec: file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]metrics=[output_rows=2,] query III diff --git a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt index 400bbab1a3592..e8c656450ba23 100644 --- a/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt +++ b/datafusion/sqllogictest/test_files/range_sorted_time_bin_agg.slt @@ -30,11 +30,16 @@ # 2. Two file_groups, so the two 60-minute streams run in parallel # # Improvement opportunity: -# date_bin(60s) is monotonic in timestamp and the hour split is aligned to bin -# boundaries, so (key, time_bin) is partition-disjoint. Aggregation could be a -# single streaming SinglePartitioned step with no hash shuffle. +# In this fixture, date_bin(60s) is monotonic in timestamp and the hour split +# is aligned to bin boundaries, so (key, time_bin) is partition-disjoint. +# Aggregation could be a single streaming SinglePartitioned step with no +# hash shuffle. # -# Today's plan still hash-repartitions: +# This optimization exists for timezone-less date_trunc when every split +# keeps groups partition-local. Extending it to date_bin is future work +# requiring a safety audit, including its null behavior (#25344). +# +# Today's date_bin plan still hash-repartitions: # Partial AggregateExec (group_clustering_mode=Full) # -> RepartitionExec Hash([key, date_bin(...)]) # -> FinalPartitioned AggregateExec (group_clustering_mode=Full) @@ -82,11 +87,11 @@ physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion ########## # TEST 2: Filtered time-bin aggregation. # GROUP BY keys are (key, date_bin(timestamp)). Input is sorted on those keys -# (date_bin is monotonic in timestamp) and range-partitioned so bins do not -# overlap across the two 60-minute streams. -# -# Today this is still Partial + hash RepartitionExec + Final, even though -# group_clustering_mode=Full is already recognized. +# and the split aligns with the bins in this data, so the groups are +# partition-local. Timezone-less date_trunc can already avoid the shuffle; +# extending this optimization to date_bin awaits its safety audit (#25344). +# Until then, Partial + hash RepartitionExec + Final is retained even though +# group_clustering_mode=Full is recognized. ########## query TT @@ -117,8 +122,9 @@ k2 2024-01-01T01:30:00 30 k2 2024-01-01T01:45:00 5 ########## -# TEST 3: Same aggregation without the col4 filter. The scan still has two -# 60-minute file groups, and today's plan still hash-repartitions. +# TEST 3: Same aggregation without the col4 filter. The groups remain +# partition-local, but date_bin retains the hash shuffle pending its +# safety audit (#25344). ########## query TT @@ -146,6 +152,97 @@ k2 2024-01-01T00:30:00 7 k2 2024-01-01T01:30:00 30 k2 2024-01-01T01:45:00 5 +########## +# TEST 4: date_trunc('hour') is audited and the split is aligned, so each +# group is partition-local and aggregation needs no hash shuffle. +########## + +query TT +EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin; +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=SinglePartitioned, gby=[key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full +03)----FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin +ORDER BY key, time_bin; +---- +k1 2024-01-01T00:00:00 3 +k1 2024-01-01T01:00:00 30 +k2 2024-01-01T00:00:00 7 +k2 2024-01-01T01:00:00 35 + +########## +# TEST 5: date_trunc('day') straddles the hour split, so the hash shuffle +# remains. +########## + +query TT +EXPLAIN SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin; +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@2 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full +03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1], 2), input_partitions=2, preserve_order=true, sort_exprs=key@0 ASC, date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)@1 ASC +04)------AggregateExec: mode=Partial, gby=[key@0 as key, date_trunc(day, timestamp@1) as date_trunc(Utf8("day"),range_sorted_time_bin.timestamp)], aggr=[sum(range_sorted_time_bin.value)], group_clustering_mode=Full +05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('day', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY key, time_bin +ORDER BY key, time_bin; +---- +k1 2024-01-01T00:00:00 33 +k2 2024-01-01T00:00:00 42 + +########## +# TEST 6: Grouping sets can omit the range transform. The per-key subtotal +# crosses the hour split, so it still needs a hash shuffle. +########## + +query TT +EXPLAIN SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY GROUPING SETS ((key, time_bin), (key)); +---- +physical_plan +01)ProjectionExec: expr=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as time_bin, sum(range_sorted_time_bin.value)@3 as sum(range_sorted_time_bin.value)] +02)--AggregateExec: mode=FinalPartitioned, gby=[key@0 as key, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1 as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp), __grouping_id@2 as __grouping_id], aggr=[sum(range_sorted_time_bin.value)] +03)----RepartitionExec: partitioning=Hash([key@0, date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)@1, __grouping_id@2], 2), input_partitions=2 +04)------AggregateExec: mode=Partial, gby=[(key@0 as key, date_trunc(hour, timestamp@1) as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp)), (key@0 as key, NULL as date_trunc(Utf8("hour"),range_sorted_time_bin.timestamp))], aggr=[sum(range_sorted_time_bin.value)] +05)--------FilterExec: col4@1 = a, projection=[key@0, timestamp@2, value@3] +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_sorted_time_bin/part-1.parquet]]}, projection=[key, col4, timestamp, value], output_ordering=[key@0 ASC, timestamp@2 ASC], output_partitioning=Range([timestamp@2 ASC], [(1704070800000000000)], 2), file_type=parquet, predicate=col4@4 = a, pruning_predicate=col4_null_count@2 != row_count@3 AND col4_min@0 <= a AND a <= col4_max@1, required_guarantees=[col4 in (a)] + +query TPI +SELECT key, date_trunc('hour', timestamp) AS time_bin, sum(value) +FROM range_sorted_time_bin +WHERE col4 = 'a' +GROUP BY GROUPING SETS ((key, time_bin), (key)) +ORDER BY key, time_bin NULLS LAST; +---- +k1 2024-01-01T00:00:00 3 +k1 2024-01-01T01:00:00 30 +k1 NULL 33 +k2 2024-01-01T00:00:00 7 +k2 2024-01-01T01:00:00 35 +k2 NULL 42 + ########## # CLEANUP ########## diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt index 09bf131e5e95f..aaebd04e51923 100644 --- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt +++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt @@ -164,7 +164,7 @@ physical_plan 03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full 04)------RepartitionExec: partitioning=Hash([f_dkey@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 ASC NULLS LAST 05)--------AggregateExec: mode=Partial, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results without subset satisfaction query TPIR rowsort @@ -204,7 +204,7 @@ physical_plan 01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST] 02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as time_bin, count(Int64(1))@2 as count(*), avg(fact_table_ordered.value)@3 as avg(fact_table_ordered.value)] 03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results match with subset satisfaction query TPIR rowsort @@ -251,7 +251,7 @@ physical_plan 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 03)----SortExec: expr=[f_dkey@2 ASC NULLS LAST, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) ASC NULLS LAST, timestamp@0 ASC NULLS LAST], preserve_partitioning=[true] 04)------RepartitionExec: partitioning=Hash([f_dkey@2, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0)], 3), input_partitions=3 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results without subset satisfaction query TPRI rowsort @@ -292,7 +292,7 @@ logical_plan physical_plan 01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn] 02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] -03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet # Verify results match with subset satisfaction query TPRI rowsort @@ -379,8 +379,8 @@ physical_plan 11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] 12)----------------------CoalescePartitionsExec 13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2] -14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] +15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible # Verify results without subset satisfaction query TPR rowsort @@ -474,8 +474,8 @@ physical_plan 09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] 10)------------------CoalescePartitionsExec 11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2] -12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] -13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] +13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible # Verify results match with subset satisfaction query TPR rowsort diff --git a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt index 196da416a2037..c09f2c986e7c0 100644 --- a/datafusion/sqllogictest/test_files/string_numeric_coercion.slt +++ b/datafusion/sqllogictest/test_files/string_numeric_coercion.slt @@ -229,10 +229,10 @@ query TT EXPLAIN SELECT * FROM t_int WHERE column1 IN ('5', '325'); ---- logical_plan -01)Filter: t_int.column1 = Int64(5) OR t_int.column1 = Int64(325) +01)Filter: t_int.column1 IN ([Int64(5), Int64(325)]) 02)--TableScan: t_int projection=[column1] physical_plan -01)FilterExec: column1@0 = 5 OR column1@0 = 325 +01)FilterExec: column1@0 IN (SET) ([5, 325]) 02)--DataSourceExec: partitions=1, partition_sizes=[1] # Error on invalid string in IN list diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 3cd3bfbf193bb..a4474c4b0388d 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -3718,10 +3718,9 @@ DROP TABLE oj_a; statement ok DROP TABLE oj_b; -# Regression test for #25792: a correlated filter below a window function must -# not be pulled above the window. The filter selects the rows the window -# function sees for each outer row, so once it is above the window the window -# function runs over all rows and computes different values. +# Regression test for #25792: a correlated filter below a window function +# selects the rows that the window function sees for each outer row. For +# `y = k`, the pull up partitions the window by `y` as well. statement ok CREATE TABLE win_outer(k INT) AS VALUES (1), (2), (5); @@ -3731,39 +3730,49 @@ CREATE TABLE win_inner(id INT, y INT) AS VALUES (1, 1), (2, 2); statement ok set datafusion.explain.logical_plan_only = true; -# The correlated filter is below the window, so the subquery stays correlated. +# The correlated filter is below the window, so the window is partitioned by b.y. query TT EXPLAIN SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1); ---- logical_plan -01)Filter: EXISTS () -02)--Subquery: -03)----Projection: Int64(1) -04)------SubqueryAlias: w -05)--------Projection: b.id, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn -06)----------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1) -07)------------WindowAggr: windowExpr=[[row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] -08)--------------SubqueryAlias: b -09)----------------Projection: win_inner.id, win_inner.y -10)------------------Filter: win_inner.y = outer_ref(win_outer.k) -11)--------------------TableScan: win_inner -12)--TableScan: win_outer projection=[k] - -# It errors rather than returning wrong results. The right answer is `1, 2`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +01)LeftSemi Join: win_outer.k = __correlated_sq_1.y +02)--TableScan: win_outer projection=[k] +03)--SubqueryAlias: __correlated_sq_1 +04)----SubqueryAlias: w +05)------Projection: b.y +06)--------Filter: row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW = UInt64(1) +07)----------Projection: b.y, row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +08)------------WindowAggr: windowExpr=[[row_number() PARTITION BY [b.y] ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS row_number() ORDER BY [b.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] +09)--------------SubqueryAlias: b +10)----------------TableScan: win_inner projection=[id, y] + +# The right answer is `1, 2`. +query I SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k; +---- +1 +2 # The same for NOT EXISTS. The right answer is `5`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +query I SELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT 1 FROM (SELECT b.id, row_number() OVER (ORDER BY b.id) AS rn FROM (SELECT * FROM win_inner WHERE win_inner.y = win_outer.k) AS b) AS w WHERE w.rn = 1) ORDER BY win_outer.k; +---- +5 # The same for IN. The right answer is `1, 2`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery +query I SELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT row_number() OVER (ORDER BY win_inner.id) FROM win_inner WHERE win_inner.y = win_outer.k) ORDER BY win_outer.k; +---- +1 +2 # And for a scalar subquery. The right answer is `(1, 1), (2, 1), (5, NULL)`. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +query II SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL # A correlated filter above the window is still pulled up, because the window # input does not depend on the outer row. @@ -3808,14 +3817,99 @@ SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION 2 1 5 NULL -# A window that partitions by an expression of the column does not count. -statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +# A window that partitions by an expression of the column is partitioned by +# the column as well. +query II SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y + 0 ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + +# A window below another window whose output has the name the extended window +# would get. The extended window keeps its old name, so the names stay unique. +query II +SELECT win_outer.k, (SELECT max(w.a + w.b) FROM (SELECT row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS a, row_number() OVER (ORDER BY win_inner.id) AS b FROM win_inner WHERE win_inner.y = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 2 +2 2 +5 NULL + +# A lossless widening cast, as coercion adds for `INT = BIGINT`, keeps the +# distinct values of `y` apart, so the window is partitioned by `y`. +query II +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = CAST(win_outer.k AS BIGINT)) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + +# `-0.0 = 0.0` holds but a window partition keeps the two apart, so a float +# `y` keeps the subquery correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 0.0), (2, -0.0)) AS v(id, f) WHERE v.f = CAST(win_outer.k - 1 AS DOUBLE)) AS w) AS m FROM win_outer ORDER BY win_outer.k; + +# A volatile outer expression can match several `y` values for one outer row, so +# the subquery stays correlated. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k + CAST(random() * 0 AS INT)) AS w) AS m FROM win_outer ORDER BY win_outer.k; + +# A lossy `CAST(f AS INT) = k` matches several `f` values for one outer row, so +# the subquery stays correlated. The right answer is `(1, 2), (2, NULL), (5, NULL)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT max(w.rn) FROM (SELECT row_number() OVER (ORDER BY v.id) AS rn FROM (VALUES (1, 1.2), (2, 1.4)) AS v(id, f) WHERE CAST(v.f AS INT) = win_outer.k) AS w) AS m FROM win_outer ORDER BY win_outer.k; # A window over an aggregate with count bug handling stays correlated. statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS m FROM win_outer ORDER BY win_outer.k; +# A projection that drops the aggregate column keeps the count bug handling, so +# the subquery stays correlated. The right answer is `(1, 1), (2, 1), (5, 1)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# Without the window, the constant keeps its value for an outer row without a match. +query II +SELECT win_outer.k, (SELECT b.x FROM (SELECT 1 AS x FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 1 + +# An aggregate without `GROUP BY` returns a row for an outer row without a match. +# A window above it keeps the subquery correlated, whatever node sits between +# them. The right answer for each scalar case is `(1, 1), (2, 1), (5, 1)`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT DISTINCT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# A Filter that cannot be evaluated on an empty input. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery +SELECT win_outer.k, (SELECT count(*) OVER () FROM (SELECT a.c FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a WHERE random() >= 0) AS b) AS m FROM win_outer ORDER BY win_outer.k; + +# The same for EXISTS, NOT EXISTS and IN, which do not track count bug handling. +# The right answers are `1, 2, 5`, no rows, and `1, 2, 5`. +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +SELECT win_outer.k FROM win_outer WHERE EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists +SELECT win_outer.k FROM win_outer WHERE NOT EXISTS (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery +SELECT win_outer.k FROM win_outer WHERE 1 IN (SELECT count(*) OVER () FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) ORDER BY win_outer.k; + +# And for LATERAL. +statement error DataFusion error: This feature is not implemented: Physical plan does not support undecorrelated Subquery +SELECT win_outer.k, s.n FROM win_outer, LATERAL (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k) AS a) AS s ORDER BY win_outer.k; + +# An aggregate with `GROUP BY` returns no row for an outer row without a match, +# so the window above it is partitioned. +query II +SELECT win_outer.k, (SELECT max(w.n) FROM (SELECT count(*) OVER () AS n FROM (SELECT count(*) AS c FROM win_inner WHERE win_inner.y = win_outer.k GROUP BY win_inner.y) AS a) AS w) AS m FROM win_outer ORDER BY win_outer.k; +---- +1 1 +2 1 +5 NULL + # LEFT JOIN LATERAL keeps the outer rows without a match. query III SELECT win_outer.k, s.id, s.rn FROM win_outer LEFT JOIN LATERAL (SELECT win_inner.id, row_number() OVER (PARTITION BY win_inner.y ORDER BY win_inner.id) AS rn FROM win_inner WHERE win_inner.y = win_outer.k) AS s ON true ORDER BY win_outer.k; diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index ca83e4b6527bf..2c706596b5409 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -365,14 +365,14 @@ EXPLAIN SELECT id, name FROM t1 WHERE id = 1 UNION SELECT id, name FROM t1 WHERE ---- logical_plan 01)Aggregate: groupBy=[[id, name]], aggr=[[]] -02)--Filter: t1.id = Int32(1) OR t1.id = Int32(2) +02)--Filter: t1.id IN ([Int32(1), Int32(2)]) 03)----TableScan: t1 projection=[id, name] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Regression: schema recomputation must preserve the unqualified UNION @@ -415,14 +415,14 @@ SELECT x.id FROM t1 AS x WHERE x.id = 2 logical_plan 01)Aggregate: groupBy=[[id]], aggr=[[]] 02)--SubqueryAlias: x -03)----Filter: t1.id = Int32(1) OR t1.id = Int32(2) +03)----Filter: t1.id IN ([Int32(1), Int32(2)]) 04)------TableScan: t1 projection=[id] physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[] 02)--RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=4 03)----AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[] 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -05)--------FilterExec: id@0 = 1 OR id@0 = 2 +05)--------FilterExec: id@0 IN (SET) ([1, 2]) 06)----------DataSourceExec: partitions=1, partition_sizes=[1] # Matching computed projections remain eligible for the rewrite and must diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt index 39ed64a526755..cf190949aad0a 100644 --- a/datafusion/sqllogictest/test_files/unnest.slt +++ b/datafusion/sqllogictest/test_files/unnest.slt @@ -668,7 +668,7 @@ explain select unnest(unnest(unnest(column3)['c1'])), column3 from recursive_unn logical_plan 01)Projection: __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1],depth=2) AS UNNEST(UNNEST(UNNEST(recursive_unnest_table.column3)[c1])), recursive_unnest_table.column3 02)--Unnest: lists[__unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1])|depth=2] structs[] -03)----Projection: get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1), Utf8("c1")) AS __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), recursive_unnest_table.column3 +03)----Projection: get_field(__unnest_placeholder(recursive_unnest_table.column3,depth=1) AS UNNEST(recursive_unnest_table.column3), Utf8("c1")) AS __unnest_placeholder(UNNEST(recursive_unnest_table.column3)[c1]), recursive_unnest_table.column3 04)------Unnest: lists[__unnest_placeholder(recursive_unnest_table.column3)|depth=1] structs[] 05)--------Projection: recursive_unnest_table.column3 AS __unnest_placeholder(recursive_unnest_table.column3), recursive_unnest_table.column3 06)----------TableScan: recursive_unnest_table projection=[column3] diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index b7a6ad531bb84..864a6e56bf574 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -5673,7 +5673,7 @@ logical_plan 01)Sort: t1.c1 ASC NULLS LAST, t1.c2 ASC NULLS LAST, rank ASC NULLS LAST 02)--Projection: t1.c1, t1.c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rank 03)----WindowAggr: windowExpr=[[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] -04)------Filter: t1.c1 = Int32(2) OR t1.c1 = Int32(3) +04)------Filter: t1.c1 IN ([Int32(2), Int32(3)]) 05)--------TableScan: t1 projection=[c1, c2] physical_plan 01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank@2 ASC NULLS LAST] @@ -5681,7 +5681,7 @@ physical_plan 03)----BoundedWindowAggExec: wdw=[rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] 04)------SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST], preserve_partitioning=[true] 05)--------RepartitionExec: partitioning=Hash([c1@0], 2), input_partitions=2 -06)----------FilterExec: c1@0 = 2 OR c1@0 = 3 +06)----------FilterExec: c1@0 IN (SET) ([2, 3]) 07)------------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 08)--------------DataSourceExec: partitions=1, partition_sizes=[1] diff --git a/dev/depcheck/Cargo.lock b/dev/depcheck/Cargo.lock deleted file mode 100644 index 31d0ee9af7e55..0000000000000 --- a/dev/depcheck/Cargo.lock +++ /dev/null @@ -1,4165 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" -dependencies = [ - "libc", -] - -[[package]] -name = "annotate-snippets" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" -dependencies = [ - "anstyle", - "memchr", - "unicode-width", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-hyperlink" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a14cad0fa1d1fb02de486a95ea7fd8dd28567b82f65b4ca34bffa7da81823e4" -dependencies = [ - "libc", - "percent-encoding", -] - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-progress" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082ebe8b7607a68881fbcccc7b3a3e784aa58593ed4261df0679a803fdb80853" - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "arc-swap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" -dependencies = [ - "rustversion", -] - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" - -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - -[[package]] -name = "blake3" -version = "1.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.0", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "block2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" -dependencies = [ - "objc2", -] - -[[package]] -name = "bstr" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" -dependencies = [ - "memchr", - "regex-automata", - "serde_core", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cargo" -version = "0.100.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4d8037642eef9a2744e1bbc775522cb20867ecc7ea35373f1769609d7386e60" -dependencies = [ - "anstream", - "anstyle", - "anstyle-progress", - "anyhow", - "async-trait", - "base64", - "blake3", - "cargo-credential", - "cargo-credential-libsecret", - "cargo-credential-macos-keychain", - "cargo-credential-wincred", - "cargo-platform", - "cargo-util", - "cargo-util-schemas", - "cargo-util-terminal", - "clap", - "clap_complete", - "color-print", - "crates-io", - "curl", - "curl-sys", - "filetime", - "flate2", - "futures", - "futures-timer", - "git2", - "git2-curl", - "gix", - "glob", - "heck", - "hex", - "hmac 0.13.0", - "home", - "http", - "http-auth", - "ignore", - "im-rc", - "indexmap", - "itertools", - "jiff", - "jobserver", - "libc", - "libgit2-sys", - "memchr", - "opener", - "os_info", - "pasetors", - "pathdiff", - "portable-atomic", - "rand", - "regex", - "rusqlite", - "rustc-hash", - "rustc-stable-hash", - "rustfix", - "same-file", - "semver", - "serde", - "serde-untagged", - "serde_ignored", - "serde_json", - "sha1 0.11.0", - "shell-escape", - "tar", - "tempfile", - "thiserror", - "time", - "toml", - "toml_edit", - "toml_parser", - "toml_writer", - "tracing", - "tracing-chrome", - "tracing-subscriber", - "unicase", - "unicode-ident", - "unicode-width", - "url", - "walkdir", - "windows-sys", - "winnow", -] - -[[package]] -name = "cargo-credential" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de74202293b2bc090da20d0b475aa74821caf0e901f29ba97003353886a5b42" -dependencies = [ - "anyhow", - "libc", - "serde", - "serde_json", - "thiserror", - "time", - "windows-sys", -] - -[[package]] -name = "cargo-credential-libsecret" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a84968127ab001ecc938ea245ad6d93be6039eb7732ead0f16f9b83fc8d5334" -dependencies = [ - "anyhow", - "cargo-credential", - "libloading", -] - -[[package]] -name = "cargo-credential-macos-keychain" -version = "0.4.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf917177a4bb7f238e1b8b3084cae37a688910e3b7bd95403f43f7c3d5a93b0e" -dependencies = [ - "cargo-credential", - "security-framework", -] - -[[package]] -name = "cargo-credential-wincred" -version = "0.4.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9234ec1ab98e92524fbf7ceb5dd06e0fcf2a046d6f341670d32796e9c5a9a384" -dependencies = [ - "cargo-credential", - "windows-sys", -] - -[[package]] -name = "cargo-platform" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "cargo-util" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe819082beae53247a2c440bece59bc2d1d90736cf5bed6b87f5e5bb53f0d30" -dependencies = [ - "anyhow", - "core-foundation", - "filetime", - "hex", - "ignore", - "jobserver", - "libc", - "miow", - "portable-atomic", - "same-file", - "sha2 0.11.0", - "shell-escape", - "tempfile", - "tracing", - "walkdir", - "windows-sys", -] - -[[package]] -name = "cargo-util-schemas" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46262ca4393669799a8df4852e81672ddea46657e73294b1a846f0d5c71509ed" -dependencies = [ - "jiff", - "semver", - "serde", - "serde-untagged", - "serde-value", - "thiserror", - "toml", - "unicode-ident", - "url", -] - -[[package]] -name = "cargo-util-terminal" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82be174c17e22a0366a1a0929e0306e0d53f120d159416b00f35e9708e7b2940" -dependencies = [ - "annotate-snippets", - "anstream", - "anstyle", - "anstyle-hyperlink", - "anstyle-progress", - "anyhow", - "cargo-util", - "libc", - "serde", - "serde_json", - "supports-hyperlinks", - "supports-unicode", - "windows-sys", -] - -[[package]] -name = "cc" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - -[[package]] -name = "clap" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", - "terminal_size", -] - -[[package]] -name = "clap_complete" -version = "4.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" -dependencies = [ - "clap", - "clap_lex", - "is_executable", - "shlex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "clru" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - -[[package]] -name = "color-print" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa954171903797d5623e047d9ab69d91b493657917bdfb8c2c80ecaf9cdb6f4" -dependencies = [ - "color-print-proc-macro", -] - -[[package]] -name = "color-print-proc-macro" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692186b5ebe54007e45a59aea47ece9eb4108e141326c304cdc91699a7118a22" -dependencies = [ - "nom", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crates-io" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c047c636cac168acc89c2df97a9017f2751e96742faaa35941fabec8c9d057a9" -dependencies = [ - "http", - "percent-encoding", - "serde", - "serde_json", - "thiserror", - "url", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "ct-codecs" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fb0c6640b4507ebd99ff67677009e381ba5eee1d14df78de4a3d16eb123c39" - -[[package]] -name = "ctutils" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" -dependencies = [ - "cmov", -] - -[[package]] -name = "curl" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a45ee8994e5307cb4c60cfc1c20bf7263ffb771ddc135c9f768a14bcbc15b09" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "windows-sys", -] - -[[package]] -name = "curl-sys" -version = "0.4.90+curl-8.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" -dependencies = [ - "cc", - "libc", - "libnghttp2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "windows-sys", -] - -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror", -] - -[[package]] -name = "depcheck" -version = "0.0.0" -dependencies = [ - "cargo", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "serde_core", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer 0.10.4", - "const-oid 0.9.6", - "crypto-common 0.1.6", - "subtle", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer 0.12.1", - "const-oid 0.10.2", - "crypto-common 0.2.2", - "ctutils", -] - -[[package]] -name = "dispatch2" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" -dependencies = [ - "bitflags 2.13.1", - "objc2", -] - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest 0.10.7", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] - -[[package]] -name = "ed25519-compact" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c24599140dc39d7a81e4476e7573d41bbc18e07c803900298e522a5fbcfbfb6" -dependencies = [ - "getrandom 0.4.3", -] - -[[package]] -name = "either" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest 0.10.7", - "ff", - "generic-array", - "group", - "hkdf", - "pem-rfc7468", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "erased-serde" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" -dependencies = [ - "serde", - "serde_core", - "typeid", -] - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "faster-hex" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" -dependencies = [ - "heapless", - "serde", -] - -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" - -[[package]] -name = "filetime" -version = "0.2.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" -dependencies = [ - "cfg-if", - "libc", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" - -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "miniz_oxide", - "zlib-rs", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-executor" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" - -[[package]] -name = "futures-macro" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "futures-sink" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-timer" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "rand_core 0.10.1", - "wasm-bindgen", -] - -[[package]] -name = "git2" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" -dependencies = [ - "bitflags 2.13.1", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", -] - -[[package]] -name = "git2-curl" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f1f6fe01327a2f7a1d6ee519c14651f79530aa4d64e870f4cf672efe697bea5" -dependencies = [ - "curl", - "git2", - "log", - "url", -] - -[[package]] -name = "gix" -version = "0.85.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8b2e38ebfc4484dfef8580ddcaf8abb7285e6f3eb6413ff6775d104ae96ca6" -dependencies = [ - "gix-actor", - "gix-attributes", - "gix-command", - "gix-commitgraph", - "gix-config", - "gix-credentials", - "gix-date", - "gix-diff", - "gix-dir", - "gix-discover", - "gix-error", - "gix-features", - "gix-filter", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-hashtable", - "gix-ignore", - "gix-index", - "gix-lock", - "gix-negotiate", - "gix-object", - "gix-odb", - "gix-pack", - "gix-path", - "gix-pathspec", - "gix-prompt", - "gix-protocol", - "gix-ref", - "gix-refspec", - "gix-revision", - "gix-revwalk", - "gix-sec", - "gix-shallow", - "gix-status", - "gix-submodule", - "gix-tempfile", - "gix-trace", - "gix-transport", - "gix-traverse", - "gix-url", - "gix-utils", - "gix-validate", - "gix-worktree", - "gix-worktree-stream", - "nonempty", - "prodash", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-actor" -version = "0.41.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33f9308ad6fd35b2a865cbe4117ac61b2be59e4a9ef1621c7a9794f7c8e52c5b" -dependencies = [ - "bstr", - "gix-date", - "gix-error", -] - -[[package]] -name = "gix-attributes" -version = "0.33.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-quote", - "gix-trace", - "kstring", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-bitmap" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd1d118d0f5d88b96e6f6e13b566475fef4797ead4a02c26fed36c1375066f7" -dependencies = [ - "gix-error", -] - -[[package]] -name = "gix-chunk" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a871e5cab12ba568845714473505deefffb3c04eb47f4708ce344cd459c1cc" -dependencies = [ - "gix-error", -] - -[[package]] -name = "gix-command" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4363accdf6ef7ba861871d2d521ab7418a04aaaed919fadb022af71d379b12" -dependencies = [ - "bstr", - "gix-path", - "gix-quote", - "gix-trace", - "shell-words", -] - -[[package]] -name = "gix-commitgraph" -version = "0.37.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" -dependencies = [ - "bstr", - "gix-chunk", - "gix-error", - "gix-hash", - "memmap2", - "nonempty", -] - -[[package]] -name = "gix-config" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29bf266c4cdaf759e535c24ad4ce655b987aeb6911075643403cc7cc5ade583" -dependencies = [ - "bstr", - "gix-config-value", - "gix-features", - "gix-glob", - "gix-path", - "gix-ref", - "gix-sec", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-config-value" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-path", - "libc", - "thiserror", -] - -[[package]] -name = "gix-credentials" -version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a0fcfcd73229d1a9c34e04edd05dbcb80364ec6f3788d24c546de2916671eb" -dependencies = [ - "bstr", - "gix-command", - "gix-config-value", - "gix-date", - "gix-path", - "gix-prompt", - "gix-sec", - "gix-trace", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-date" -version = "0.15.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e47b9e8cdc688296609b706428de570f88b1e0eed7156dde7b4a89d26fa4567" -dependencies = [ - "bstr", - "gix-error", - "itoa", - "jiff", -] - -[[package]] -name = "gix-diff" -version = "0.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c6d56c94edf92d78203a1cd416f770e35e10b6955ede6b9d7d0c22ff88a5f3" -dependencies = [ - "bstr", - "gix-attributes", - "gix-command", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-imara-diff", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-tempfile", - "gix-trace", - "gix-traverse", - "gix-worktree", - "thiserror", -] - -[[package]] -name = "gix-dir" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20098fba2b9c6e29361ccb4c0379d42dfb81fc7f86f7ab11f2dff4528c0bf01f" -dependencies = [ - "bstr", - "gix-discover", - "gix-fs", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-trace", - "gix-utils", - "gix-worktree", - "thiserror", -] - -[[package]] -name = "gix-discover" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d624d5b23b10c1d85337645227abe353ac95ab8ff66a7bdd5ce689b2db33a722" -dependencies = [ - "bstr", - "dunce", - "gix-fs", - "gix-path", - "gix-ref", - "gix-sec", - "thiserror", -] - -[[package]] -name = "gix-error" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9292309fd944e71b2a3c96d3c03a6feb8852db646febdde7cbb9f79cb5f329" -dependencies = [ - "bstr", -] - -[[package]] -name = "gix-features" -version = "0.48.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" -dependencies = [ - "bytes", - "crc32fast", - "crossbeam-channel", - "gix-path", - "gix-trace", - "gix-utils", - "libc", - "once_cell", - "parking_lot", - "prodash", - "thiserror", - "walkdir", - "zlib-rs", -] - -[[package]] -name = "gix-filter" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6644fb2ef97928c278675b239f366b457103d7e436f811d27331a8daf212759c" -dependencies = [ - "bstr", - "encoding_rs", - "gix-attributes", - "gix-command", - "gix-hash", - "gix-object", - "gix-packetline", - "gix-path", - "gix-quote", - "gix-trace", - "gix-utils", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-fs" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" -dependencies = [ - "bstr", - "fastrand", - "gix-features", - "gix-path", - "gix-utils", - "thiserror", -] - -[[package]] -name = "gix-glob" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-features", - "gix-path", -] - -[[package]] -name = "gix-hash" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" -dependencies = [ - "faster-hex", - "gix-features", - "sha1-checked", - "thiserror", -] - -[[package]] -name = "gix-hashtable" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" -dependencies = [ - "gix-hash", - "hashbrown 0.17.1", - "parking_lot", -] - -[[package]] -name = "gix-ignore" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-trace", - "unicode-bom", -] - -[[package]] -name = "gix-imara-diff" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a791e6620676a875f362f3156ed213e73ca099a09bf992c18812abe65cc37b1" -dependencies = [ - "bstr", - "hashbrown 0.17.1", -] - -[[package]] -name = "gix-index" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d45f82ec5a4d7542ea595e9ad16e03e26c8cb4f221e5bc9fcdcf469f63a681" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "filetime", - "fnv", - "gix-bitmap", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-traverse", - "gix-utils", - "gix-validate", - "hashbrown 0.17.1", - "itoa", - "libc", - "memmap2", - "rustix", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-lock" -version = "23.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" -dependencies = [ - "gix-tempfile", - "gix-utils", - "thiserror", -] - -[[package]] -name = "gix-negotiate" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d6081882a5f575ace9f53924a7c85f69bbd0f96071b982df12f258ab338cc9" -dependencies = [ - "bitflags 2.13.1", - "gix-commitgraph", - "gix-date", - "gix-hash", - "gix-object", - "gix-revwalk", -] - -[[package]] -name = "gix-object" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "019b38afc3eac1e41f9fe09a327664b313ba4a120fa5f40e3678795d0e42783e" -dependencies = [ - "bstr", - "gix-actor", - "gix-date", - "gix-features", - "gix-hash", - "gix-hashtable", - "gix-utils", - "gix-validate", - "itoa", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-odb" -version = "0.82.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fadc59f6fa0f9dd445eceee61060a2b59ca557f48da9fc677f567db535b782a" -dependencies = [ - "arc-swap", - "gix-features", - "gix-fs", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-pack", - "gix-path", - "gix-quote", - "memmap2", - "parking_lot", - "tempfile", - "thiserror", -] - -[[package]] -name = "gix-pack" -version = "0.72.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3e7f1726cd2c0cd1cf1fc20be8a8e623f0b163f1f8d6fc836cfb9bc8cd758b" -dependencies = [ - "clru", - "gix-chunk", - "gix-error", - "gix-features", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-path", - "gix-tempfile", - "memmap2", - "parking_lot", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-packetline" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" -dependencies = [ - "bstr", - "faster-hex", - "gix-trace", - "thiserror", -] - -[[package]] -name = "gix-path" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751d6bd162106f8c1e7e9aaccb5bbdd605267e91a930a17a4560c46e33a9100c" -dependencies = [ - "bstr", - "gix-trace", - "gix-validate", - "thiserror", -] - -[[package]] -name = "gix-pathspec" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" -dependencies = [ - "bitflags 2.13.1", - "bstr", - "gix-attributes", - "gix-config-value", - "gix-glob", - "gix-path", - "thiserror", -] - -[[package]] -name = "gix-prompt" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" -dependencies = [ - "gix-command", - "gix-config-value", - "parking_lot", - "rustix", - "thiserror", -] - -[[package]] -name = "gix-protocol" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978468bae4ea2df20c72db3b20d0bdb548a0c1090b85a83643b553e6e0e041f2" -dependencies = [ - "bstr", - "gix-credentials", - "gix-date", - "gix-features", - "gix-hash", - "gix-lock", - "gix-negotiate", - "gix-object", - "gix-ref", - "gix-refspec", - "gix-revwalk", - "gix-shallow", - "gix-trace", - "gix-transport", - "gix-utils", - "maybe-async", - "nonempty", - "thiserror", -] - -[[package]] -name = "gix-quote" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" -dependencies = [ - "bstr", - "gix-error", - "gix-utils", -] - -[[package]] -name = "gix-ref" -version = "0.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bbfbce1dfd7d7f8469ddef6d3518376aff664348f153cbe0fc3e58ef993d24e" -dependencies = [ - "gix-actor", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-path", - "gix-tempfile", - "gix-utils", - "gix-validate", - "memmap2", - "thiserror", -] - -[[package]] -name = "gix-refspec" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc36a4fb1a1540b59cf2da498783080743fa274b02a3f19ca444fc4015a9d4f" -dependencies = [ - "bstr", - "gix-error", - "gix-glob", - "gix-hash", - "gix-revision", - "gix-validate", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-revision" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "885075c3c21eb9c06e0be3b3728ba5932c04e1c1011dcee7c81801980e3e986f" -dependencies = [ - "bstr", - "gix-commitgraph", - "gix-date", - "gix-error", - "gix-hash", - "gix-object", - "gix-revwalk", - "nonempty", -] - -[[package]] -name = "gix-revwalk" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f11fe7ca2585193d3d70bbe0be175a2008d883a704cc7a55e454e113e689455" -dependencies = [ - "gix-commitgraph", - "gix-date", - "gix-error", - "gix-hash", - "gix-hashtable", - "gix-object", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-sec" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4fe6c152c1d50aea36f299825702cd37e303307832fec1d0fdd5844e47ce2f" -dependencies = [ - "bitflags 2.13.1", - "gix-path", - "libc", - "windows-sys", -] - -[[package]] -name = "gix-shallow" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" -dependencies = [ - "bstr", - "gix-hash", - "gix-lock", - "nonempty", - "thiserror", -] - -[[package]] -name = "gix-status" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec3293f75db1212f99217832cbc70c30faeb95cefc97c7f1a17fd3bcf13a72e" -dependencies = [ - "bstr", - "filetime", - "gix-diff", - "gix-dir", - "gix-features", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-index", - "gix-object", - "gix-path", - "gix-pathspec", - "gix-worktree", - "portable-atomic", - "thiserror", -] - -[[package]] -name = "gix-submodule" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9f594f7cbda0b38ba6b633b3e9a7b7901acdc5d27bc186a16633800cd1ac8" -dependencies = [ - "bstr", - "gix-config", - "gix-path", - "gix-pathspec", - "gix-refspec", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-tempfile" -version = "23.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" -dependencies = [ - "dashmap", - "gix-fs", - "libc", - "parking_lot", - "tempfile", -] - -[[package]] -name = "gix-trace" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3eb81d9dc914335923e50d52829c551feefd6a72d176c4130c546b67a60814" - -[[package]] -name = "gix-transport" -version = "0.57.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" -dependencies = [ - "base64", - "bstr", - "curl", - "gix-command", - "gix-credentials", - "gix-features", - "gix-packetline", - "gix-quote", - "gix-sec", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-traverse" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5062cca8f2977565bbaf666ec31dbdb9bc9d9293beb65f9bec52e6c1121b62a1" -dependencies = [ - "bitflags 2.13.1", - "gix-commitgraph", - "gix-date", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-revwalk", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-url" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d68e70e96da0e5f9c871f1566349e0fd0e1a20bb483c7f54af1dd0b85b4b29" -dependencies = [ - "bstr", - "gix-path", - "percent-encoding", - "thiserror", -] - -[[package]] -name = "gix-utils" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1795bd2a970ca8b2185318c2abb97d955c71992f1cf28de73ad3b593a9f3ce8" -dependencies = [ - "bstr", - "fastrand", - "getrandom 0.4.3", - "unicode-normalization", -] - -[[package]] -name = "gix-validate" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a034e84d1e04e1b1f20f51f12491da230b6ac8b925d0c8e1b89bcd87a7c5ccc" -dependencies = [ - "bstr", -] - -[[package]] -name = "gix-worktree" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92399ed66f259592050c6ed9dc80105e095a2f8e87e6b83d98aa2e21d8e27036" -dependencies = [ - "bstr", - "gix-attributes", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "gix-validate", -] - -[[package]] -name = "gix-worktree-stream" -version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f3a878c89a05470ad98c644b0015777c530da24854dd29e41fe4f41176840f" -dependencies = [ - "gix-attributes", - "gix-error", - "gix-features", - "gix-filter", - "gix-fs", - "gix-hash", - "gix-object", - "gix-path", - "gix-traverse", - "parking_lot", -] - -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - -[[package]] -name = "globset" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "hash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashlink" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" -dependencies = [ - "hashbrown 0.17.1", -] - -[[package]] -name = "heapless" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" -dependencies = [ - "hash32", - "stable_deref_trait", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "hmac" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" -dependencies = [ - "digest 0.11.3", -] - -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-auth" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" -dependencies = [ - "memchr", -] - -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "ignore" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata", - "same-file", - "walkdir", - "winapi-util", -] - -[[package]] -name = "im-rc" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" -dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "is_executable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cb6a9f675da968c63b6208c641b9dca58fc0133ae53375736b1767b0cab8bd" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jiff" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" -dependencies = [ - "defmt", - "jiff-core", - "jiff-static", - "jiff-tzdb-platform", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "windows-link", -] - -[[package]] -name = "jiff-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" -dependencies = [ - "defmt", -] - -[[package]] -name = "jiff-static" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" -dependencies = [ - "jiff-core", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" -dependencies = [ - "cfg-if", - "wasm-bindgen", -] - -[[package]] -name = "kstring" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "libgit2-sys" -version = "0.18.7+1.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - -[[package]] -name = "libloading" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" -dependencies = [ - "cfg-if", - "windows-link", -] - -[[package]] -name = "libnghttp2-sys" -version = "0.1.13+1.68.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "492e00167f1418c15648144f42bbfc63099806ecee9bf8d09a6353d6b4856b3c" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libssh2-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c04141a07bb0c0bc461cb657808764de571702a59bc5c726c400ac9a7625e3ab" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "maybe-async" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "miow" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536bfad37a309d62069485248eeaba1e8d9853aaf951caaeaed0585a95346f08" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.1", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nonempty" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" - -[[package]] -name = "normpath" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9985ef7269fa99f3b12437bb698381da2428743ab90f20393f399fa14cab21a" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "objc2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" -dependencies = [ - "objc2-encode", -] - -[[package]] -name = "objc2-cloud-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-data" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags 2.13.1", - "dispatch2", - "objc2", -] - -[[package]] -name = "objc2-core-graphics" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" -dependencies = [ - "bitflags 2.13.1", - "dispatch2", - "objc2", - "objc2-core-foundation", - "objc2-io-surface", -] - -[[package]] -name = "objc2-core-image" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-location" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca347214e24bc973fc025fd0d36ebb179ff30536ed1f80252706db19ee452009" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-text" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", - "objc2-core-graphics", -] - -[[package]] -name = "objc2-encode" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" - -[[package]] -name = "objc2-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" -dependencies = [ - "bitflags 2.13.1", - "block2", - "libc", - "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-io-surface" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-quartz-core" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" -dependencies = [ - "bitflags 2.13.1", - "objc2", - "objc2-core-foundation", - "objc2-foundation", -] - -[[package]] -name = "objc2-ui-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" -dependencies = [ - "bitflags 2.13.1", - "block2", - "objc2", - "objc2-cloud-kit", - "objc2-core-data", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-core-image", - "objc2-core-location", - "objc2-core-text", - "objc2-foundation", - "objc2-quartz-core", - "objc2-user-notifications", -] - -[[package]] -name = "objc2-user-notifications" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9df9128cbbfef73cda168416ccf7f837b62737d748333bfe9ab71c245d76613e" -dependencies = [ - "objc2", - "objc2-foundation", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "opener" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b03ff07a220d0d0ec9a1f0f238951b7967a5a2e96aefcd21a117b1083415e9" -dependencies = [ - "bstr", - "normpath", - "windows-sys", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "orion" -version = "0.17.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6758747fd1ce1efaf2bd43219ac4aa9e28263b236b2b6a1e486bcd06820707" -dependencies = [ - "fiat-crypto", - "subtle", -] - -[[package]] -name = "os_info" -version = "3.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf20a545b305cf1da722b236b5155c9bb35f1d5ceb28c048bd96ca842f41b5b" -dependencies = [ - "android_system_properties", - "log", - "nix", - "objc2", - "objc2-foundation", - "objc2-ui-kit", - "windows-sys", -] - -[[package]] -name = "p384" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2 0.10.9", -] - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "pasetors" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e838401fb2873bad417e6a03179014c748746f67311cb7317ab14fc0881fa9f0" -dependencies = [ - "ct-codecs", - "ed25519-compact", - "getrandom 0.4.3", - "orion", - "p384", - "rand_core 0.6.4", - "regex", - "serde", - "serde_derive", - "serde_json", - "sha2 0.10.9", - "subtle", - "time", - "zeroize", -] - -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "portable-atomic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prodash" -version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" -dependencies = [ - "parking_lot", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.13.1", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - -[[package]] -name = "rsqlite-vfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" -dependencies = [ - "hashbrown 0.16.1", - "thiserror", -] - -[[package]] -name = "rusqlite" -version = "0.40.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" -dependencies = [ - "bitflags 2.13.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc-stable-hash" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "781442f29170c5c93b7185ad559492601acdc71d5bb0706f5868094f45cfcd08" - -[[package]] -name = "rustfix" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c614151d9f1b1e1f23041dacbf5f8dfab85687455b9e553f19f3d70275d1fe1c" -dependencies = [ - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags 2.13.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags 2.13.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde-untagged" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" -dependencies = [ - "erased-serde", - "serde", - "serde_core", - "typeid", -] - -[[package]] -name = "serde-value" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" -dependencies = [ - "ordered-float", - "serde", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_ignored" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115dffd5f3853e06e746965a20dcbae6ee747ae30b543d91b0e089668bb07798" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_spanned" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" -dependencies = [ - "serde_core", -] - -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - -[[package]] -name = "sha1-checked" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" -dependencies = [ - "digest 0.10.7", - "sha1 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shell-escape" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" - -[[package]] -name = "shell-words" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlite-wasm-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" -dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "supports-hyperlinks" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" - -[[package]] -name = "supports-unicode" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tar" -version = "0.4.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" -dependencies = [ - "filetime", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys", -] - -[[package]] -name = "terminal_size" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" -dependencies = [ - "rustix", - "windows-sys", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - -[[package]] -name = "time-macros" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "toml" -version = "1.1.4+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" -dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime", - "toml_parser", - "toml_writer", - "winnow", -] - -[[package]] -name = "toml_datetime" -version = "1.1.1+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" -dependencies = [ - "serde_core", -] - -[[package]] -name = "toml_edit" -version = "0.25.13+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" -dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime", - "toml_parser", - "toml_writer", - "winnow", -] - -[[package]] -name = "toml_parser" -version = "1.1.3+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" -dependencies = [ - "winnow", -] - -[[package]] -name = "toml_writer" -version = "1.1.2+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tracing-chrome" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" -dependencies = [ - "serde_json", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "typeid" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - -[[package]] -name = "unicode-bom" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasm-bindgen" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "winnow" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -dependencies = [ - "memchr", -] - -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/dev/depcheck/Cargo.toml b/dev/depcheck/Cargo.toml deleted file mode 100644 index d7db5ef570251..0000000000000 --- a/dev/depcheck/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Circular dependency checker for DataFusion -[package] -name = "depcheck" -edition = "2024" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -cargo = "0.100.0" diff --git a/dev/depcheck/README.md b/dev/depcheck/README.md deleted file mode 100644 index 4c1571051f222..0000000000000 --- a/dev/depcheck/README.md +++ /dev/null @@ -1,26 +0,0 @@ - - -This directory contains a tool that ensures there are no circular dependencies -in the DataFusion codebase. - -Specifically, it checks that no create's tests depend on another crate which -depends on the first, which prevents publishing to crates.io, for example - -[issue 9272]: https://github.com/apache/datafusion/issues/9277: diff --git a/dev/depcheck/rust-toolchain.toml b/dev/depcheck/rust-toolchain.toml deleted file mode 100644 index d3483e9f75a0d..0000000000000 --- a/dev/depcheck/rust-toolchain.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file specifies the default version of Rust used -# to compile this workspace and run CI jobs. - -[toolchain] -channel = "1.99.0" -components = ["rustfmt", "clippy"] diff --git a/dev/depcheck/src/main.rs b/dev/depcheck/src/main.rs deleted file mode 100644 index de5eacb71d062..0000000000000 --- a/dev/depcheck/src/main.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -extern crate cargo; - -use cargo::CargoResult; -/// Check for circular dependencies between DataFusion crates -use std::collections::{HashMap, HashSet}; -use std::env; -use std::path::Path; - -use cargo::GlobalContext; - -/// Verifies that there are no circular dependencies between DataFusion crates -/// (which prevents publishing on crates.io) by parsing the Cargo.toml files and -/// checking the dependency graph. -/// -/// See https://github.com/apache/datafusion/issues/9278 for more details -fn main() -> CargoResult<()> { - let gctx = GlobalContext::default()?; - // This is the path for the depcheck binary - let path = env::var("CARGO_MANIFEST_DIR").unwrap(); - let root_cargo_toml = Path::new(&path) - // dev directory - .parent() - .expect("Can not find dev directory") - // project root directory - .parent() - .expect("Can not find project root directory") - .join("Cargo.toml"); - - println!( - "Checking for circular dependencies in {}", - root_cargo_toml.display() - ); - let workspace = cargo::workspace::Workspace::new(&root_cargo_toml, &gctx)?; - let (_, resolve) = cargo::ops::resolve_ws(&workspace, false)?; - - let mut package_deps = HashMap::new(); - for package_id in resolve - .iter() - .filter(|id| id.name().starts_with("datafusion")) - { - let deps: Vec = resolve - .deps(package_id) - .filter(|(package_id, _)| package_id.name().starts_with("datafusion")) - .map(|(package_id, _)| package_id.name().to_string()) - .collect(); - package_deps.insert(package_id.name().to_string(), deps); - } - - // check for circular dependencies - for (root_package, deps) in &package_deps { - let mut seen = HashSet::new(); - for dep in deps { - check_circular_deps(root_package, dep, &package_deps, &mut seen); - } - } - println!("No circular dependencies found"); - Ok(()) -} - -fn check_circular_deps( - root_package: &str, - current_dep: &str, - package_deps: &HashMap>, - seen: &mut HashSet, -) { - if root_package == current_dep { - panic!( - "circular dependency detected from {root_package} to self via one of {:?}", - seen - ); - } - if seen.contains(current_dep) { - return; - } - seen.insert(current_dep.to_string()); - if let Some(deps) = package_deps.get(current_dep) { - for dep in deps { - check_circular_deps(root_package, dep, package_deps, seen); - } - } -} diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index f4b5c1cbc0437..7650eb1977a53 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -332,8 +332,14 @@ Only committed files are checked. Commit a change before running it. CI runs two dependency checks, and `./dev/rust_lint.sh` runs both: -- `ci/scripts/check_circular_dependencies.sh` builds and runs [`dev/depcheck`], - which fails on dependency cycles between DataFusion crates. +- `ci/scripts/check_circular_dependencies.sh` fails on dependency cycles + between DataFusion crates, including one through a dev-dependency, which + Cargo allows when building but which prevents publishing the crates. It runs + `cargo publish --workspace --dry-run --no-verify`, which packages every crate + that does not set `publish = false` without building or uploading anything, + so it also fails on any other problem that would stop the crates from being + published to crates.io. It reads the crates.io index and accepts uncommitted + changes. - `ci/scripts/check_unused_dependencies.sh` runs `cargo machete --with-metadata` from the repository root. The lint suite installs [cargo-machete] with the version in `ci/scripts/utils/tool_versions.sh` if it is missing. @@ -345,7 +351,6 @@ To run either check on its own: ./ci/scripts/check_unused_dependencies.sh ``` -[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck [cargo-machete]: https://github.com/bnjbvr/cargo-machete ## Examples README Check diff --git a/docs/source/library-user-guide/query-optimizer.md b/docs/source/library-user-guide/query-optimizer.md index 73c3ed6ed11dc..03fb6f2e33fe8 100644 --- a/docs/source/library-user-guide/query-optimizer.md +++ b/docs/source/library-user-guide/query-optimizer.md @@ -168,9 +168,10 @@ Two rules can want the opposite order for the same pair of adjacent plan nodes. Do not let two rules compete. Give one rule precedence, make the competing rule yield, and record the decision in the module documentation of both rules. -There is one such decision today: +There are two such decisions today: - `PushDownFilter` yields to a _pure extraction projection_. A pure extraction projection is a projection whose expressions are only `__datafusion_extracted_N` aliases and pass-through columns. `ExtractLeafExpressions` creates it, and `PushDownLeafProjections` moves it towards the leaves. `PushDownFilter` does not move a filter below such a projection, so the projection stays next to the scan. A Parquet scan then merges the projection into the file projection and reads only the struct leaf. The filter loses nothing, because `PushDownFilter` records the predicate in `TableScan::filters` in the pass that runs before the extraction projection exists. See [issue #14540](https://github.com/apache/datafusion/issues/14540). +- `PushDownLeafProjections` yields to `OptimizeProjections`. When the extraction projection cannot move below the input of a projection, for example a `TableScan` or an `Aggregate`, `PushDownLeafProjections` leaves the projection unchanged. A split in place would give a recovery projection over an extraction projection on the same input, which `OptimizeProjections` merges back. The source absorbs the leaf expressions of the original projection just as well. See [PR #26085](https://github.com/apache/datafusion/pull/26085). ### Expression Naming