Rust

test: skip count distinct spill memory test under forced hash collisions #75607

Workflow file for this run

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	# # CI runners
	#
	# Some actions use Runs-On to run on ASF infrastructure:
	# https://datafusion.apache.org/contributor-guide/#ci-runners
	#
	# # `cargo xtask`
	#
	# Individual CI commands are implemented with `cargo xtask` to make them easier
	# to reproduce locally.
	#
	# See `xtask/README.md` for details about DataFusion's `xtask` tooling.
	#
	# It is being rolled out incrementally across all workflows.
	# Track progress at https://github.com/apache/datafusion/issues/24487.
	#
	name: Rust

	concurrency:
	group: ${{ github.repository }}-${{ github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	on:
	push:
	branches-ignore:
	- 'gh-readonly-queue/**'
	- 'dependabot/**'
	paths-ignore:
	- "docs/**"
	- "**.md"
	- ".github/ISSUE_TEMPLATE/**"
	- ".github/pull_request_template.md"
	pull_request:
	# manual trigger
	# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
	workflow_dispatch:
	merge_group:


	permissions:
	contents: read

	jobs:
	# Check crate compiles and base cargo check passes
	linux-build-lib:
	name: linux build test
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs
	save-if: ${{ github.ref_name == 'main' }}
	- name: Prepare cargo build
	# `--locked` asserts that Cargo.lock is up to date with the manifest.
	run: cargo xtask ci step check workspace

	# Check datafusion-common features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-datafusion-common-features:
	name: cargo check datafusion-common features
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Check datafusion-common (default features)
	run: cargo xtask ci step check datafusion-common default
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion-common (no-default-features)
	run: cargo xtask ci step check datafusion-common no-default
	# Note: don't check other feature flags as datafusion-common is not typically used standalone

	# Check datafusion-substrait features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-datafusion-substrait-features:
	name: cargo check datafusion-substrait features
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: false # set in linux-test
	shared-key: "amd-ci"
	- name: Check datafusion-substrait (default features)
	run: cargo xtask ci step check datafusion-substrait default
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion-substrait (no-default-features)
	run: cargo xtask ci step check datafusion-substrait no-default
	- name: Check datafusion-substrait (physical)
	run: cargo xtask ci step check datafusion-substrait physical
	- name: Install cmake
	run: \|
	# note the builder setup runs apt-get update / installs protobuf compiler
	apt-get install -y cmake
	- name: Check datafusion-substrait (protoc)
	run: cargo xtask ci step check datafusion-substrait protoc

	# Check datafusion-proto features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-datafusion-proto-features:
	name: cargo check datafusion-proto features
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Check datafusion-proto (default features)
	run: cargo xtask ci step check datafusion-proto default
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion-proto (no-default-features)
	run: cargo xtask ci step check datafusion-proto no-default
	- name: Check datafusion-proto (json)
	run: cargo xtask ci step check datafusion-proto json
	- name: Check datafusion-proto (parquet)
	run: cargo xtask ci step check datafusion-proto parquet
	- name: Check datafusion-proto (avro)
	run: cargo xtask ci step check datafusion-proto avro

	# Check datafusion-ffi features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-datafusion-ffi-features:
	name: cargo check datafusion-ffi features
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Check datafusion-ffi (no-default-features)
	run: cargo xtask ci step check datafusion-ffi no-default


	# Check datafusion crate features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-cargo-check-datafusion:
	name: cargo check datafusion features
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: false # set in linux-test
	shared-key: "amd-ci"
	- name: Check datafusion (default features)
	run: cargo xtask ci step check datafusion default
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion (no-default-features)
	run: cargo xtask ci step check datafusion no-default
	- name: Check datafusion (nested_expressions)
	run: cargo xtask ci step check datafusion nested_expressions
	- name: Check datafusion (array_expressions)
	run: cargo xtask ci step check datafusion array_expressions
	- name: Check datafusion (avro)
	run: cargo xtask ci step check datafusion avro
	- name: Check datafusion (backtrace)
	run: cargo xtask ci step check datafusion backtrace
	- name: Check datafusion (compression)
	run: cargo xtask ci step check datafusion compression
	- name: Check datafusion (crypto_expressions)
	run: cargo xtask ci step check datafusion crypto_expressions
	- name: Check datafusion (datetime_expressions)
	run: cargo xtask ci step check datafusion datetime_expressions
	- name: Check datafusion (encoding_expressions)
	run: cargo xtask ci step check datafusion encoding_expressions
	- name: Check datafusion (force_hash_collisions)
	run: cargo xtask ci step check datafusion force_hash_collisions
	- name: Check datafusion (math_expressions)
	run: cargo xtask ci step check datafusion math_expressions
	- name: Check datafusion (parquet)
	run: cargo xtask ci step check datafusion parquet
	- name: Check datafusion (regex_expressions)
	run: cargo xtask ci step check datafusion regex_expressions
	- name: Check datafusion (recursive_protection)
	run: cargo xtask ci step check datafusion recursive_protection
	- name: Check datafusion (serde)
	run: cargo xtask ci step check datafusion serde
	- name: Check datafusion (sql)
	run: cargo xtask ci step check datafusion sql
	- name: Check datafusion (string_expressions)
	run: cargo xtask ci step check datafusion string_expressions
	- name: Check datafusion (unicode_expressions)
	run: cargo xtask ci step check datafusion unicode_expressions
	- name: Check parquet encryption (parquet_encryption)
	run: cargo xtask ci step check datafusion parquet_encryption

	# Check datafusion-functions crate features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	linux-cargo-check-datafusion-functions:
	name: cargo check datafusion-functions features
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Check datafusion-functions (default features)
	run: cargo xtask ci step check datafusion-functions default
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion-functions (no-default-features)
	run: cargo xtask ci step check datafusion-functions no-default
	- name: Check datafusion-functions (crypto_expressions)
	run: cargo xtask ci step check datafusion-functions crypto_expressions
	- name: Check datafusion-functions (datetime_expressions)
	run: cargo xtask ci step check datafusion-functions datetime_expressions
	- name: Check datafusion-functions (encoding_expressions)
	run: cargo xtask ci step check datafusion-functions encoding_expressions
	- name: Check datafusion-functions (math_expressions)
	run: cargo xtask ci step check datafusion-functions math_expressions
	- name: Check datafusion-functions (regex_expressions)
	run: cargo xtask ci step check datafusion-functions regex_expressions
	- name: Check datafusion-functions (string_expressions)
	run: cargo xtask ci step check datafusion-functions string_expressions
	- name: Check datafusion-functions (unicode_expressions)
	run: cargo xtask ci step check datafusion-functions unicode_expressions

	# Check datafusion-spark crate features
	#
	# Ensure via `cargo check` that the crate can be built with a
	# subset of the features packages enabled.
	# See https://github.com/apache/datafusion/issues/24474
	linux-cargo-check-datafusion-spark:
	name: cargo check datafusion-spark features
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Check datafusion-spark (default features)
	run: cargo check --profile ci --all-targets -p datafusion-spark
	#
	# Note: Only check libraries (not --all-targets) to cover end user APIs
	#
	- name: Check datafusion-spark (no-default-features)
	run: cargo check --profile ci --no-default-features -p datafusion-spark
	- name: Check datafusion-spark (core)
	run: cargo check --profile ci --no-default-features -p datafusion-spark --features=core

	# Library and integration tests
	linux-test:
	name: cargo test (amd64)
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	volumes:
	- /usr/local:/host/usr/local
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Install llvm-tools-preview
	run: rustup component add llvm-tools-preview
	- name: Install cargo-llvm-cov
	uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
	with:
	tool: cargo-llvm-cov
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: ${{ github.ref_name == 'main' }}
	shared-key: "amd-ci"
	- name: Run tests (excluding doctests and datafusion-cli)
	env:
	RUST_BACKTRACE: 1
	run: cargo xtask ci step test workspace
	- parallel:
	- name: Verify Working Directory Clean
	run: git diff --exit-code
	# Check no temporary directories created during test.
	# `false/` folder is excuded for rust cache.
	- name: Verify Working Directory Clean (No Untracked Files)
	run: \|
	STATUS="$(git status --porcelain \| sed -e '/^?? false\/$/d' -e '/^?? false$/d')"
	if [ -n "$STATUS" ]; then
	echo "$STATUS"
	exit 1
	fi
	- name: Upload coverage to codecov.io
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	files: target/codecov.json
	fail_ci_if_error: false
	token: ${{ secrets.CODECOV_TOKEN }}

	# datafusion-cli tests
	linux-test-datafusion-cli:
	name: cargo test datafusion-cli (amd64)
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	run: rustup toolchain install stable
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: false # set in linux-test
	shared-key: "amd-ci"
	- name: Run tests (excluding doctests)
	env:
	RUST_BACKTRACE: 1
	AWS_ENDPOINT: http://127.0.0.1:9000
	AWS_ACCESS_KEY_ID: TEST-DataFusionLogin
	AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
	TEST_STORAGE_INTEGRATION: 1
	AWS_ALLOW_HTTP: true
	run: cargo xtask ci step test cli
	- name: Verify Working Directory Clean
	run: git diff --exit-code


	linux-test-example:
	name: cargo examples (amd64)
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: ${{ github.ref_name == 'main' }}
	shared-key: "amd-ci-linux-test-example"
	- name: Run examples
	run: \|
	# test datafusion-sql examples
	cargo run --profile ci --example sql
	# test datafusion-examples
	ci/scripts/rust_example.sh
	- name: Verify Working Directory Clean
	run: git diff --exit-code

	# Run `cargo test doc` (test documentation examples)
	linux-test-doc:
	name: cargo test doc (amd64)
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run doctests
	run: cargo xtask ci step test doctest
	- name: Verify Working Directory Clean
	run: git diff --exit-code

	# Run `cargo doc` to ensure the rustdoc is clean
	linux-rustdoc:
	name: cargo doc
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run cargo doc
	run: ci/scripts/rust_docs.sh

	linux-wasm-pack:
	name: build and run with wasm-pack
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- parallel:
	- name: Setup for wasm32
	run: \|
	rustup target add wasm32-unknown-unknown
	- name: Install dependencies
	run: \|
	sudo apt-get update -qq
	sudo apt-get install -y -qq clang
	- name: Setup wasm-pack
	uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
	with:
	tool: wasm-pack
	- name: Run tests with headless mode
	working-directory: ./datafusion/wasmtest
	run: \|
	# debuginfo=none because CI tests weren't completing successfully after this upstream PR:
	# https://github.com/wasm-bindgen/wasm-bindgen/pull/4635
	RUSTFLAGS='--cfg getrandom_backend="wasm_js" -C debuginfo=none' wasm-pack test --headless --firefox
	RUSTFLAGS='--cfg getrandom_backend="wasm_js" -C debuginfo=none' wasm-pack test --headless --chrome --chromedriver $CHROMEWEBDRIVER/chromedriver

	# verify that the benchmark queries return the correct results
	verify-benchmark-results:
	name: verify benchmark results (amd64)
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- parallel:
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Generate benchmark data and expected query results
	run: \|
	mkdir -p datafusion/sqllogictest/test_files/tpch/data
	git clone https://github.com/databricks/tpch-dbgen.git
	cd tpch-dbgen
	make
	./dbgen -f -s 0.1
	mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data
	- name: Verify that benchmark queries return expected results
	run: \|
	cargo xtask ci step test benchmark-plan
	cargo xtask ci step test benchmark-sqllogic
	- name: Verify Working Directory Clean
	run: git diff --exit-code

	sqllogictest-postgres:
	name: "Run sqllogictest with Postgres runner"
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	services:
	postgres:
	image: postgres:15
	env:
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: db_test
	POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C
	ports:
	- 5432:5432
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run sqllogictest
	run: cargo xtask ci step test postgres
	env:
	# use postgres for the host here because we have specified a container for the job
	POSTGRES_HOST: postgres
	POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}

	sqllogictest-substrait:
	name: "Run sqllogictest in Substrait round-trip mode"
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run sqllogictest
	# TODO: Right now several tests are failing in Substrait round-trip mode, so this
	# command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
	# until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
	# and this command can be run without filters.
	run: cargo xtask ci step test substrait

	# Temporarily commenting out the Windows flow, the reason is enormously slow running build
	# Waiting for new Windows 2025 github runner
	# Details: https://github.com/apache/datafusion/issues/13726
	#
	# windows:
	# name: cargo test (win64)
	# runs-on: windows-latest
	# steps:
	# - uses: actions/checkout@v4
	# with:
	# submodules: true
	# - name: Setup Rust toolchain
	# uses: ./.github/actions/setup-windows-builder
	# - name: Run tests (excluding doctests)
	# shell: bash
	# run: \|
	# export PATH=$PATH:$HOME/d/protoc/bin
	# cargo test --lib --tests --bins --features avro,json,backtrace

	# macOS scope is narrowed to `datafusion-ffi`: the only bug class amd64
	# cannot reproduce is FFI cdylib loading (`.dylib` vs `.so` resolution in
	# datafusion/ffi/src/tests/utils.rs). All other macos-only failures
	# historically came from datafusion-benchmarks (now covered on amd64) or
	# flaky sqllogictest metrics.
	macos-aarch64:
	name: cargo test (macos-aarch64)
	runs-on: macos-15
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-macos-aarch64-builder
	- name: Run datafusion-ffi tests
	shell: bash
	run: cargo xtask ci step test ffi

	vendor:
	name: Verify Vendored Code
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run gen
	run: ./regen.sh
	working-directory: ./datafusion/proto
	- name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results)
	run: git diff --exit-code

	check-fmt:
	name: Check cargo fmt
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run
	run: \|
	ci/scripts/rust_fmt.sh

	check-workflow-tool-installs:
	name: Check GitHub Actions install tooling
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Check workflow tool installs
	run: ci/scripts/check_no_cargo_install_in_workflows.sh


	clippy:
	name: clippy
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- parallel:
	- name: Install Clippy
	run: rustup component add clippy
	- name: Rust Dependency Cache
	uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
	with:
	save-if: ${{ github.ref_name == 'main' }}
	shared-key: "amd-ci-clippy"
	- name: Run clippy
	run: ci/scripts/rust_clippy.sh

	cargo-toml-formatting-checks:
	name: check Cargo.toml formatting
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Install taplo
	uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
	with:
	tool: taplo-cli@0.9
	# if you encounter an error, try running 'taplo format' to fix the formatting automatically.
	- name: Check Cargo.toml formatting
	run: taplo format --check

	config-docs-check:
	name: check configs.md and ***_functions.md is up-to-date
	needs: linux-build-lib
	runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) \|\| 'ubuntu-latest' }}
	container:
	image: amd64/rust
	steps:
	- uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
	with:
	node-version: "20"
	- name: Check if configs.md has been modified
	run: \|
	# If you encounter an error, run './dev/update_config_docs.sh' and commit
	./dev/update_config_docs.sh
	git diff --exit-code
	- name: Check if any of the ***_functions.md has been modified
	run: \|
	# If you encounter an error, run './dev/update_function_docs.sh' and commit
	./dev/update_function_docs.sh
	git diff --exit-code

	# This job ensures `datafusion-examples/README.md` stays in sync with the source code:
	# 1. Generates README automatically using the Rust examples docs generator
	# (parsing documentation from `examples/<group>/main.rs`)
	# 2. Formats the generated Markdown using DataFusion's standard Prettier setup
	# 3. Compares the result against the committed README.md and fails if out-of-date
	examples-docs-check:
	name: check example README is up-to-date
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust

	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	submodules: true
	fetch-depth: 1

	- parallel:
	- name: Mark repository as safe for git
	# Required for git commands inside container (avoids "dubious ownership" error)
	run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

	- name: Set up Node.js (required for prettier)
	# doc_prettier_check.sh uses npx to run prettier for Markdown formatting
	uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
	with:
	node-version: '18'

	- name: Run examples docs check script
	run: \|
	bash ci/scripts/check_examples_docs.sh

	# Verify MSRV for the crates which are directly used by other projects:
	# - datafusion
	# - datafusion-substrait
	# - datafusion-proto
	# - datafusion-cli
	msrv:
	name: Verify MSRV (Min Supported Rust Version)
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Install cargo-msrv
	uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
	with:
	tool: cargo-msrv

	- name: Check datafusion
	working-directory: datafusion/core
	run: \|
	# If you encounter an error with any of the commands below it means
	# your code or some crate in the dependency tree has a higher MSRV
	# (Min Supported Rust Version) than the one specified in the
	# `rust-version` key of `Cargo.toml`.
	#
	# To reproduce:
	# 1. Install the version of Rust that is failing.
	# 2. Run the command that failed with that version.
	#
	# Example:
	# # MSRV looks like "1.80.0" and is specified in Cargo.toml. We can read the value with the following command:
	# msrv="$(cargo metadata --format-version=1 \| jq '.packages[] \| select( .name == "datafusion" ) \| .rust_version' -r)"
	# echo "MSRV: ${msrv}"
	# rustup install "${msrv}"
	# cargo "+${msrv}" check
	#
	# To resolve, either:
	# 1. Change your code to use older Rust features,
	# 2. Revert dependency update
	# 3. Update the MSRV version in `Cargo.toml`
	#
	# Please see the DataFusion Rust Version Compatibility Policy before
	# updating Cargo.toml. You may have to update the code instead.
	# https://github.com/apache/datafusion/blob/main/README.md#rust-version-compatibility-policy
	cargo msrv --output-format json --log-target stdout verify
	- name: Check datafusion-substrait
	working-directory: datafusion/substrait
	run: cargo msrv --output-format json --log-target stdout verify
	- name: Check datafusion-proto
	working-directory: datafusion/proto
	run: cargo msrv --output-format json --log-target stdout verify

Navigation Menu

Uh oh!

test: skip count distinct spill memory test under forced hash collisions #75607

Workflow file

test: skip count distinct spill memory test under forced hash collisions #75607

Uh oh!

Workflow file for this run