From ea1bcd91679bc666b9e40bf339e1dfd5c496986e Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Thu, 24 Sep 2026 14:54:42 +0000 Subject: [PATCH 1/6] fix datafusion-cli CI: switch from minio to rustfs (#25706) Closes https://github.com/apache/datafusion/issues/25705 This morning it seems like Minio pulled its images from quay.io, which has caused recent CI runs to start failing, e.g. https://github.com/apache/datafusion/actions/runs/36006680489/job/107657362588 RustFS is a pretty comparable (and more permissively licensed) alternative that provides a suitable S3 interface for integration tests. This was substantially authored by Codex, but reviewed by me and tested locally. Existing integration tests No Signed-off-by: Andrew Duffy (cherry picked from commit e2ca7f38051744b2010cf09b80db8e9dfa4b5d38) --- .github/workflows/rust.yml | 15 + datafusion-cli/CONTRIBUTING.md | 6 +- datafusion-cli/Cargo.toml | 2 +- datafusion-cli/tests/cli_integration.rs | 388 ++++++++++++++++-------- 4 files changed, 279 insertions(+), 132 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a5deb5b3e538d..62c56c40c6e94 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -371,6 +371,21 @@ jobs: with: save-if: false # set in linux-test shared-key: "amd-ci" + # The storage integration tests start RustFS containers. Pulling the image + # once up front keeps the pull off the critical path of the tests, which + # would otherwise pull it several times concurrently and occasionally fail + # with transient Docker transport errors. The tests retry the pull + # themselves, so a failure here is only a warning. + # + # RUSTFS_IMAGE must match RUSTFS_IMAGE_NAME and RUSTFS_IMAGE_TAG in + # `datafusion-cli/tests/cli_integration.rs`. The + # `rustfs_image_matches_ci_prepull` test fails if they differ. + - name: Pre-pull RustFS image + env: + RUSTFS_IMAGE: docker.io/rustfs/rustfs:1.0.0 + run: | + ci/scripts/retry timeout 120 docker pull "$RUSTFS_IMAGE" \ + || echo "::warning::Could not pre-pull $RUSTFS_IMAGE, the tests will pull it themselves" - name: Run tests (excluding doctests) env: RUST_BACKTRACE: 1 diff --git a/datafusion-cli/CONTRIBUTING.md b/datafusion-cli/CONTRIBUTING.md index 8be656ec4ee34..fa615028a4ddd 100644 --- a/datafusion-cli/CONTRIBUTING.md +++ b/datafusion-cli/CONTRIBUTING.md @@ -35,7 +35,7 @@ cargo test --all-targets ## Running Storage Integration Tests -By default, storage integration tests are not run. These tests use the `testcontainers` crate to start up a local MinIO server using Docker on port 9000. +By default, storage integration tests are not run. These tests use the `testcontainers` crate to start up a local RustFS server using Docker with a dynamically assigned host port. To run them you will need to set `TEST_STORAGE_INTEGRATION`: @@ -47,8 +47,8 @@ For some of the tests, [snapshots](https://datafusion.apache.org/contributor-gui ### AWS -S3 integration is tested against [Minio](https://github.com/minio/minio) with [TestContainers](https://github.com/testcontainers/testcontainers-rs) -This requires Docker to be running on your machine and port 9000 to be free. +S3 integration is tested against [RustFS](https://github.com/rustfs/rustfs) with [TestContainers](https://github.com/testcontainers/testcontainers-rs) +This requires Docker to be running on your machine. If you see an error mentioning "failed to load IMDS session token" such as diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 62eedafe798d4..c21bc972161d6 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -77,7 +77,7 @@ ctor = { workspace = true } insta = { workspace = true } insta-cmd = "0.7.0" rstest = { workspace = true } -testcontainers-modules = { workspace = true, features = ["minio"] } +testcontainers-modules = { workspace = true } # Makes sure `test_display_pg_json` behaves in a consistent way regardless of # feature unification with dependencies serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 4dc244445a2eb..92de3ad68e92f 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -20,16 +20,22 @@ use std::process::Command; use rstest::rstest; use async_trait::async_trait; +use futures::TryStreamExt; use insta::internals::SettingsBindDropGuard; use insta::{Settings, glob}; use insta_cmd::{assert_cmd_snapshot, get_cargo_bin}; +use object_store::{ + ObjectStore, ObjectStoreExt, aws::AmazonS3Builder, local::LocalFileSystem, +}; use std::path::PathBuf; +use std::time::Duration; use std::{env, fs}; -use testcontainers_modules::minio; -use testcontainers_modules::testcontainers::core::{CmdWaitFor, ExecCommand, Mount}; +use testcontainers_modules::testcontainers::core::{ + CmdWaitFor, ExecCommand, IntoContainerPort, +}; use testcontainers_modules::testcontainers::runners::AsyncRunner; use testcontainers_modules::testcontainers::{ - ContainerAsync, ImageExt, TestcontainersError, + ContainerAsync, GenericImage, ImageExt, TestcontainersError, }; fn cli() -> Command { @@ -45,79 +51,241 @@ fn make_settings() -> Settings { settings } -async fn setup_minio_container() -> Result, String> { - const MINIO_ROOT_USER: &str = "TEST-DataFusionLogin"; - const MINIO_ROOT_PASSWORD: &str = "TEST-DataFusionPassword"; +const RUSTFS_ACCESS_KEY: &str = "TEST-DataFusionLogin"; +const RUSTFS_SECRET_KEY: &str = "TEST-DataFusionPassword"; - let data_path = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../datafusion/core/tests/data"); +/// Pinned RustFS image, also pulled by the CLI CI job. +const RUSTFS_IMAGE_NAME: &str = "docker.io/rustfs/rustfs"; +const RUSTFS_IMAGE_TAG: &str = "1.0.0"; - let absolute_data_path = data_path - .canonicalize() - .expect("Failed to get absolute path for test data"); - - let container = minio::MinIO::default() - .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) - .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) - .with_mount(Mount::bind_mount( - absolute_data_path.to_str().unwrap(), - "/source", - )) - .start() - .await; - - match container { - Ok(container) => { - // We wait for MinIO to be healthy and prepare test files. We do it via CLI to avoid s3 dependency - let commands = [ - ExecCommand::new(["/usr/bin/mc", "ready", "local"]), - ExecCommand::new([ - "/usr/bin/mc", - "alias", - "set", - "localminio", - "http://localhost:9000", - MINIO_ROOT_USER, - MINIO_ROOT_PASSWORD, - ]), - ExecCommand::new(["/usr/bin/mc", "mb", "localminio/data"]), - ExecCommand::new([ - "/usr/bin/mc", - "cp", - "-r", - "/source/", - "localminio/data/", - ]), - ]; - - for command in commands { - let command = - command.with_cmd_ready_condition(CmdWaitFor::Exit { code: Some(0) }); - - let cmd_ref = format!("{command:?}"); - - if let Err(e) = container.exec(command).await { - let stdout = container.stdout_to_vec().await.unwrap_or_default(); - let stderr = container.stderr_to_vec().await.unwrap_or_default(); - - return Err(format!( - "Failed to execute command: {}\nError: {}\nStdout: {:?}\nStderr: {:?}", - cmd_ref, - e, - String::from_utf8_lossy(&stdout), - String::from_utf8_lossy(&stderr) - )); - } +/// How many times to try bringing up the RustFS container before failing. +/// +/// Image pulls and fixture uploads can fail with transient errors such as +/// `bytes remaining on stream`. Retry these failures before failing the test. +const RUSTFS_SETUP_ATTEMPTS: u32 = 3; + +/// Delay before the first retry of the RustFS setup, doubled on each attempt. +const RUSTFS_SETUP_RETRY_DELAY: Duration = Duration::from_secs(5); + +/// Time budget for a single RustFS setup attempt. A stalled image pull or `curl` +/// invocation is retried instead of hanging the whole test run. +const RUSTFS_SETUP_TIMEOUT: Duration = Duration::from_mins(3); + +/// Starts a RustFS container preloaded with the test data, retrying transient +/// Docker failures. +/// +/// Returns `None` when the test should be skipped, that is when +/// `TEST_STORAGE_INTEGRATION` is unset or the registry is rate limiting the +/// image pull. Panics if the container cannot be started for any other reason. +async fn start_rustfs_or_skip() -> Option> { + if env::var("TEST_STORAGE_INTEGRATION").is_err() { + eprintln!("Skipping external storages integration tests"); + return None; + } + + match setup_rustfs_container().await { + Ok(container) => Some(container), + Err(e) if is_docker_pull_rate_limit(&e) => { + eprintln!("Skipping test: Docker pull rate limit reached: {e}"); + None + } + Err(e) => panic!("{e}"), + } +} + +/// A registry pull rate limit does not clear up within a test run, so the +/// affected tests are skipped rather than retried. +fn is_docker_pull_rate_limit(error: &str) -> bool { + error.contains("toomanyrequests") +} + +/// Retrying only pays off for transient failures. An exhausted pull quota or a +/// Docker daemon that cannot be reached at all stays broken for the whole run. +fn is_retryable(error: &str) -> bool { + !is_docker_pull_rate_limit(error) + && !error.contains("failed to initialize a docker client") +} + +async fn setup_rustfs_container() -> Result, String> { + let mut delay = RUSTFS_SETUP_RETRY_DELAY; + let mut last_error = String::from("RustFS container setup was not attempted at all"); + + for attempt in 1..=RUSTFS_SETUP_ATTEMPTS { + last_error = match tokio::time::timeout( + RUSTFS_SETUP_TIMEOUT, + try_setup_rustfs_container(), + ) + .await + { + Ok(Ok(container)) => return Ok(container), + Ok(Err(e)) => e, + Err(_) => format!( + "Timed out after {RUSTFS_SETUP_TIMEOUT:?} while starting the RustFS container" + ), + }; + + if attempt == RUSTFS_SETUP_ATTEMPTS || !is_retryable(&last_error) { + break; + } + + eprintln!( + "RustFS container setup failed (attempt {attempt}/{RUSTFS_SETUP_ATTEMPTS}), \ + retrying in {delay:?}: {last_error}" + ); + tokio::time::sleep(delay).await; + delay *= 2; + } + + Err(last_error) +} + +/// A single attempt at starting and provisioning a RustFS container. +/// +/// The container is removed again if provisioning fails, so that the next +/// attempt starts from a clean state. +async fn try_setup_rustfs_container() -> Result, String> { + let container = start_rustfs_container().await?; + + match provision_rustfs_container(&container).await { + Ok(()) => Ok(container), + Err(e) => { + if let Err(rm_error) = container.rm().await { + eprintln!("Failed to remove the RustFS container: {rm_error}"); } + Err(e) + } + } +} - Ok(container) +async fn start_rustfs_container() -> Result, String> { + GenericImage::new(RUSTFS_IMAGE_NAME, RUSTFS_IMAGE_TAG) + .with_exposed_port(9000.tcp()) + .with_env_var("RUSTFS_ACCESS_KEY", RUSTFS_ACCESS_KEY) + .with_env_var("RUSTFS_SECRET_KEY", RUSTFS_SECRET_KEY) + .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") + .start() + .await + .map_err(|e| match e { + TestcontainersError::Client(e) => format!( + "Failed to start RustFS container. Ensure Docker is running and accessible: {e}" + ), + e => format!("Failed to start RustFS container: {e}"), + }) +} + +/// Waits for RustFS to be healthy and uploads the test files. +/// +/// Use the image's `curl` to check readiness and create the bucket, then upload +/// the fixtures through `object_store`. +async fn provision_rustfs_container( + container: &ContainerAsync, +) -> Result<(), String> { + let credentials = format!("{RUSTFS_ACCESS_KEY}:{RUSTFS_SECRET_KEY}"); + let commands = [ + ExecCommand::new([ + "curl", + "--fail", + "--silent", + "--show-error", + "--retry", + "60", + "--retry-delay", + "1", + "--retry-all-errors", + "--max-time", + "5", + "http://localhost:9000/health/ready", + ]), + ExecCommand::new([ + "curl", + "--fail", + "--silent", + "--show-error", + "--aws-sigv4", + "aws:amz:us-east-1:s3", + "--user", + &credentials, + "--request", + "PUT", + "http://localhost:9000/data", + ]), + ]; + + for command in commands { + let command = + command.with_cmd_ready_condition(CmdWaitFor::Exit { code: Some(0) }); + + let cmd_ref = format!("{command:?}"); + + if let Err(e) = container.exec(command).await { + let stdout = container.stdout_to_vec().await.unwrap_or_default(); + let stderr = container.stderr_to_vec().await.unwrap_or_default(); + + return Err(format!( + "Failed to execute command: {}\nError: {}\nStdout: {:?}\nStderr: {:?}", + cmd_ref, + e, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + )); } + } - Err(TestcontainersError::Client(e)) => Err(format!( - "Failed to start MinIO container. Ensure Docker is running and accessible: {e}" - )), - Err(e) => Err(format!("Failed to start MinIO container: {e}")), + let port = container + .get_host_port_ipv4(9000) + .await + .map_err(|e| e.to_string())?; + let store = AmazonS3Builder::new() + .with_bucket_name("data") + .with_region("us-east-1") + .with_access_key_id(RUSTFS_ACCESS_KEY) + .with_secret_access_key(RUSTFS_SECRET_KEY) + .with_endpoint(format!("http://localhost:{port}")) + .with_allow_http(true) + .build() + .map_err(|e| e.to_string())?; + let data_path = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../datafusion/core/tests/data"); + let source = + LocalFileSystem::new_with_prefix(data_path).map_err(|e| e.to_string())?; + let mut files = source.list(None); + while let Some(file) = files.try_next().await.map_err(|e| e.to_string())? { + let data = source + .get(&file.location) + .await + .map_err(|e| e.to_string())? + .bytes() + .await + .map_err(|e| e.to_string())?; + store + .put(&file.location, data.into()) + .await + .map_err(|e| e.to_string())?; } + + Ok(()) +} + +/// CI pre-pulls the RustFS image so that the storage integration tests do not +/// have to pull it themselves. Keep the CI image and the test image in sync. +#[test] +fn rustfs_image_matches_ci_prepull() { + let workflow = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.github/workflows/rust.yml"); + + // The workflow is not shipped with the published crate. + let Ok(contents) = fs::read_to_string(&workflow) else { + return; + }; + + let image_ref = format!("{RUSTFS_IMAGE_NAME}:{RUSTFS_IMAGE_TAG}"); + + assert!( + contents.contains(&format!("RUSTFS_IMAGE: {image_ref}")), + "{} does not pre-pull `{image_ref}`. Update RUSTFS_IMAGE in the \ + `Pre-pull RustFS image` step to match the image used by the tests.", + workflow.display() + ); } #[cfg(test)] @@ -522,18 +690,8 @@ fn test_cli_wide_result_set_no_crash() { #[tokio::test] async fn test_cli() { - if env::var("TEST_STORAGE_INTEGRATION").is_err() { - eprintln!("Skipping external storages integration tests"); + let Some(container) = start_rustfs_or_skip().await else { return; - } - - let container = match setup_minio_container().await { - Ok(c) => c, - Err(e) if e.contains("toomanyrequests") => { - eprintln!("Skipping test: Docker pull rate limit reached: {e}"); - return; - } - e @ Err(_) => e.unwrap(), }; let settings = make_settings(); @@ -546,8 +704,8 @@ async fn test_cli() { assert_cmd_snapshot!( cli() .env_clear() - .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin") - .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword") + .env("AWS_ACCESS_KEY_ID", RUSTFS_ACCESS_KEY) + .env("AWS_SECRET_ACCESS_KEY", RUSTFS_SECRET_KEY) .env("AWS_ENDPOINT", format!("http://localhost:{port}")) .env("AWS_ALLOW_HTTP", "true") .pass_stdin(input) @@ -559,22 +717,13 @@ async fn test_cli() { async fn test_aws_options() { // Separate test is needed to pass aws as options in sql and not via env - if env::var("TEST_STORAGE_INTEGRATION").is_err() { - eprintln!("Skipping external storages integration tests"); + let Some(container) = start_rustfs_or_skip().await else { return; - } + }; let settings = make_settings(); let _bound = settings.bind_to_scope(); - let container = match setup_minio_container().await { - Ok(c) => c, - Err(e) if e.contains("toomanyrequests") => { - eprintln!("Skipping test: Docker pull rate limit reached: {e}"); - return; - } - e @ Err(_) => e.unwrap(), - }; let port = container.get_host_port_ipv4(9000).await.unwrap(); let input = format!( @@ -582,8 +731,8 @@ async fn test_aws_options() { STORED AS CSV LOCATION 's3://data/cars.csv' OPTIONS( - 'aws.access_key_id' 'TEST-DataFusionLogin', - 'aws.secret_access_key' 'TEST-DataFusionPassword', + 'aws.access_key_id' '{RUSTFS_ACCESS_KEY}', + 'aws.secret_access_key' '{RUSTFS_SECRET_KEY}', 'aws.endpoint' 'http://localhost:{port}', 'aws.allow_http' 'true' ); @@ -658,18 +807,8 @@ fn test_backtrace_output(#[case] query: &str) { #[tokio::test] async fn test_s3_url_fallback() { - if env::var("TEST_STORAGE_INTEGRATION").is_err() { - eprintln!("Skipping external storages integration tests"); + let Some(container) = start_rustfs_or_skip().await else { return; - } - - let container = match setup_minio_container().await { - Ok(c) => c, - Err(e) if e.contains("toomanyrequests") => { - eprintln!("Skipping test: Docker pull rate limit reached: {e}"); - return; - } - e @ Err(_) => e.unwrap(), }; let mut settings = make_settings(); @@ -689,25 +828,16 @@ OPTIONS ( SELECT * FROM partitioned_data ORDER BY column_1, column_2 LIMIT 5; "#; - assert_cmd_snapshot!(cli().with_minio(&container).await.pass_stdin(input)); + assert_cmd_snapshot!(cli().with_rustfs(&container).await.pass_stdin(input)); } /// Validate object store profiling output #[tokio::test] async fn test_object_store_profiling() { - if env::var("TEST_STORAGE_INTEGRATION").is_err() { - eprintln!("Skipping external storages integration tests"); + let Some(container) = start_rustfs_or_skip().await else { return; - } - - let container = match setup_minio_container().await { - Ok(c) => c, - Err(e) if e.contains("toomanyrequests") => { - eprintln!("Skipping test: Docker pull rate limit reached: {e}"); - return; - } - e @ Err(_) => e.unwrap(), }; + let mut settings = make_settings(); // as the object store profiling contains timestamps and durations, we must @@ -750,27 +880,29 @@ SELECT * from CARS LIMIT 1; SELECT * from CARS LIMIT 1; "#; - assert_cmd_snapshot!(cli().with_minio(&container).await.pass_stdin(input)); + assert_cmd_snapshot!(cli().with_rustfs(&container).await.pass_stdin(input)); } -/// Extension trait to Add the minio connection information to a Command +/// Add the RustFS connection information to a Command. #[async_trait] -trait MinioCommandExt { - async fn with_minio(&mut self, container: &ContainerAsync) - -> &mut Self; +trait RustfsCommandExt { + async fn with_rustfs( + &mut self, + container: &ContainerAsync, + ) -> &mut Self; } #[async_trait] -impl MinioCommandExt for Command { - async fn with_minio( +impl RustfsCommandExt for Command { + async fn with_rustfs( &mut self, - container: &ContainerAsync, + container: &ContainerAsync, ) -> &mut Self { let port = container.get_host_port_ipv4(9000).await.unwrap(); self.env_clear() - .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin") - .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword") + .env("AWS_ACCESS_KEY_ID", RUSTFS_ACCESS_KEY) + .env("AWS_SECRET_ACCESS_KEY", RUSTFS_SECRET_KEY) .env("AWS_ENDPOINT", format!("http://localhost:{port}")) .env("AWS_ALLOW_HTTP", "true") } From 8fd1f202bb3f253adaa2cb9d7b85a1e7d0875c05 Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 25 Oct 2026 18:43:42 +0300 Subject: [PATCH 2/6] Repair DataFusion catalog tests after identifier option update (cherry picked from commit 0bc89d6187902a0ca990ce5256ba8063e0ff4053) --- datafusion/catalog/src/information_schema.rs | 1 + datafusion/sqllogictest/test_files/information_schema.slt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 448dbca8805cd..1fa7a0ac8bbe8 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -1701,6 +1701,7 @@ mod tests { let config = InformationSchemaConfig { catalog_list: Arc::new(Fixture), table_functions: HashMap::new(), + uppercase_identifiers: false, }; let mut builder = InformationSchemaTablesBuilder { catalog_names: StringBuilder::new(), diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 83315f57cbffc..27a3fb00a35c3 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -367,6 +367,7 @@ datafusion.sql_parser.parse_float_as_decimal false datafusion.sql_parser.recursion_limit 50 datafusion.sql_parser.support_varchar_with_length true datafusion.sql_parser.trim_decimal_literal_trailing_zeros false +datafusion.sql_parser.uppercase_unquoted_identifiers false # show all variables with verbose query TTT rowsort @@ -525,7 +526,7 @@ datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate ke datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. -datafusion.sql_parser.enable_ident_normalization true When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) +datafusion.sql_parser.enable_ident_normalization true When set to true, normalize unquoted identifiers to lowercase by default. datafusion.sql_parser.enable_options_value_normalization false When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. datafusion.sql_parser.enable_subquery_sort_elimination true When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. datafusion.sql_parser.map_string_types_to_utf8view true If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. @@ -533,6 +534,7 @@ datafusion.sql_parser.parse_float_as_decimal false When set to true, SQL parser datafusion.sql_parser.recursion_limit 50 Specifies the recursion depth limit when parsing complex SQL Queries datafusion.sql_parser.support_varchar_with_length true If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. datafusion.sql_parser.trim_decimal_literal_trailing_zeros false When set to true, insignificant trailing zeros are removed from decimal literals. For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`. +datafusion.sql_parser.uppercase_unquoted_identifiers false When identifier normalization is enabled, fold unquoted relation, schema, column, and CTE identifiers to uppercase instead of lowercase. Quoted identifiers retain their exact spelling. Set this when creating the session; SQL SET/RESET cannot change it later. Catalogs and schemas used by SQL should be registered under their uppercase names. Function names still follow the existing function-registry lookup rules. # show_variable_in_config_options query TT From 2eebde48fc8fd69db9397944763b4128c79865bc Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 25 Oct 2026 18:59:29 +0300 Subject: [PATCH 3/6] Format datasource manifest for Taplo CI (cherry picked from commit 5ddc41bb19e776f2490acae6b95c12ebd3881eac) --- datafusion/datasource/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index 9216feb02115b..40e3952542e6b 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -56,9 +56,9 @@ async-compression = { version = "0.4.40", features = [ "tokio", ], optional = true } async-trait = { workspace = true } +brotli = { version = "8.0.2", optional = true } bytes = { workspace = true } bzip2 = { workspace = true, optional = true } -brotli = { version = "8.0.2", optional = true } chrono = { workspace = true } datafusion-common = { workspace = true, features = ["object_store"] } datafusion-common-runtime = { workspace = true } From 7ae693d32903f922e791aff6f81f2603a6420bdf Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 25 Oct 2026 19:06:08 +0300 Subject: [PATCH 4/6] Refresh parser option docs and clippy test (cherry picked from commit 52580648fa40ce24a28b466044872f59615d6a8c) --- datafusion/core/src/execution/context/mod.rs | 2 +- docs/source/user-guide/configs.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index c5ad0b57b4ead..6a3a00a42d2c0 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -2799,7 +2799,7 @@ mod tests { "SET datafusion.sql_parser.uppercase_unquoted_identifiers = false", "RESET datafusion.sql_parser.uppercase_unquoted_identifiers", ] { - let error = ctx.sql(sql).await.err().expect("startup-only setting"); + let error = ctx.sql(sql).await.expect_err("startup-only setting"); assert!( error .to_string() diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 5f98875b518de..96f6972c1051e 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -199,7 +199,8 @@ The following configuration settings are available: | datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | | datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | | datafusion.sql_parser.trim_decimal_literal_trailing_zeros | false | When set to true, insignificant trailing zeros are removed from decimal literals. For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`. | -| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | +| datafusion.sql_parser.enable_ident_normalization | true | When set to true, normalize unquoted identifiers to lowercase by default. | +| datafusion.sql_parser.uppercase_unquoted_identifiers | false | When identifier normalization is enabled, fold unquoted relation, schema, column, and CTE identifiers to uppercase instead of lowercase. Quoted identifiers retain their exact spelling. Set this when creating the session; SQL SET/RESET cannot change it later. Catalogs and schemas used by SQL should be registered under their uppercase names. Function names still follow the existing function-registry lookup rules. | | datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | | datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. | | datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | From 07a1a92487b4c6819be138be7750d7140e018f9d Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 25 Oct 2026 19:09:50 +0300 Subject: [PATCH 5/6] Compare ranged CSV diagnostics without backtraces (cherry picked from commit 69bb1e18cdb5358b3dd776de89c34ec8de8619c5) --- datafusion/datasource-csv/src/source.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index f8a84a3a40457..396acace20571 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -1261,7 +1261,7 @@ mod tests { .try_collect::>() .await .expect_err("full-file blank record must fail") - .to_string(); + .strip_backtrace(); let mut ranged_file = file; ranged_file.range = Some(FileRange { start: range_start, @@ -1273,7 +1273,7 @@ mod tests { .try_collect::>() .await .expect_err("ranged blank record must fail") - .to_string(); + .strip_backtrace(); assert_eq!(ranged_error, full_error, "{name}"); } Ok(()) From a7e306941f98ade96c3a31806d6eafd8559f5c4b Mon Sep 17 00:00:00 2001 From: osipovartem Date: Sun, 25 Oct 2026 19:51:10 +0300 Subject: [PATCH 6/6] Refresh generated array_reduce function docs --- .../source/user-guide/sql/scalar_functions.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 1bfec4ce43599..95c5c305d9e8d 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -3264,6 +3264,7 @@ _Alias of [current_date](#current_date)._ - [array_product](#array_product) - [array_push_back](#array_push_back) - [array_push_front](#array_push_front) +- [array_reduce](#array_reduce) - [array_remove](#array_remove) - [array_remove_all](#array_remove_all) - [array_remove_n](#array_remove_n) @@ -4215,6 +4216,31 @@ _Alias of [array_append](#array_append)._ _Alias of [array_prepend](#array_prepend)._ +### `array_reduce` + +Reduces an array to a single value by applying a binary lambda to the accumulator and each array element in order. + +```sql +array_reduce(array, initial, (acc, value) -> expression) +``` + +#### Arguments + +- **array**: Array expression to reduce. +- **initial**: Initial accumulator value. +- **merge**: Binary lambda whose parameters are the accumulator and current array element. + +#### Example + +```sql +> select array_reduce([1, 2, 3], 0, (acc, value) -> acc + value); ++----------------------------------------------------------------+ +| array_reduce([1,2,3],0,(acc,value) -> acc + value) | ++----------------------------------------------------------------+ +| 6 | ++----------------------------------------------------------------+ +``` + ### `array_remove` Removes the first element from the array equal to the given value. NULL elements already in the array are preserved when removing a non-NULL value. If `element` evaluates to NULL, the result is NULL rather than removing NULL entries.