From 3f366c8ceb7d302083029b5caa25507df93ff03b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 26 Oct 2026 07:12:26 -0600 Subject: [PATCH 1/3] ci: label the TPC-DS jobs as adaptive planner and update the docs Adaptive query planning became the default in #2315, so the TPC-DS workflow has been running AQE while its job names and comments still said static planner. Set the planner flag explicitly and update the benchmarks README, which also still described a non-empty skip list and the fixed #2047. --- .github/workflows/tpcds.yml | 18 +++++++++--------- benchmarks/README.md | 18 +++++++----------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index b503d0e705..4bdb082381 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -56,7 +56,7 @@ on: jobs: tpcds-sf1: - name: TPC-DS SF1 (static planner, ${{ matrix.label }}) + name: TPC-DS SF1 (adaptive planner, ${{ matrix.label }}) runs-on: ubuntu-latest container: image: amd64/rust @@ -119,20 +119,20 @@ jobs: run: | set -euo pipefail - # This matrix leg runs the suite under the default (static) planner at - # one task-packing setting. The tpcds binary internally loops all - # non-skipped queries and exits non-zero on any failure, so a single - # invocation covers the whole suite. The other leg runs in parallel. - # - # Coverage for the adaptive planner (AQE on) is being added - # separately; it currently fails on pre-existing bugs. - echo "::group::[static planner, ${{ matrix.label }}] TPC-DS suite" + # This matrix leg runs the suite under the adaptive planner (AQE) at + # one task-packing setting. AQE is the default, but it is set + # explicitly so the job label stays true if the default changes. The + # tpcds binary internally loops all non-skipped queries and exits + # non-zero on any failure, so a single invocation covers the whole + # suite. The other leg runs in parallel. + echo "::group::[adaptive planner, ${{ matrix.label }}] TPC-DS suite" ./target/tpch-ci/tpcds \ --host 127.0.0.1 --port 50050 \ --path "$DATA_DIR" \ --partitions 16 \ --verify \ -c datafusion.optimizer.prefer_hash_join=false \ + -c ballista.planner.adaptive.enabled=true \ ${{ matrix.task_args }} echo "::endgroup::" diff --git a/benchmarks/README.md b/benchmarks/README.md index 4b5a50dac6..114ae6e54a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -459,21 +459,17 @@ queries run unchanged. ### Skip list -Some queries are excluded from the gate. These are listed in the `SKIP` const -near the top of `benchmarks/src/bin/tpcds.rs`, as `(query_id, reason)` pairs, -grouped by cause: queries whose distributed result diverges from DataFusion -(tracked as bugs), queries that are non-deterministic under `LIMIT`/`ORDER BY` -ties, and queries whose `tpcgen-cli` schema column names differ from the -DataFusion query text. A default run (no `--query`) executes `1..=99` minus -`SKIP`. +Queries can be excluded from the gate by listing them in the `SKIP` const near +the top of `benchmarks/src/bin/tpcds.rs`, as `(query_id, reason)` pairs. The +list is currently empty, so a default run (no `--query`) executes all 99 +queries. Results that differ only in the order of rows tied under `ORDER BY` +are accepted rather than skipped. ### CI `.github/workflows/tpcds.yml` runs the SF1 suite against a local scheduler + executor on every push/PR touching `ballista/**` or -`benchmarks/**`, under the default (static) planner. The adaptive planner -(AQE on) is not gated yet — it currently fails many TPC-DS queries with an -`EmptyExec invalid partition` assertion (issue #2047); re-enable an AQE-on run -once that is fixed. +`benchmarks/**`, under the adaptive planner (AQE), with one leg at one +partition per task and one at up to four partitions per task. [1]: http://www.tpc.org/tpch/ From ea8478ad0f2dea774a739d26e6f01978cbe26bd9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 26 Oct 2026 07:14:44 -0600 Subject: [PATCH 2/3] ci: run the TPC-H "AQE off" leg with the static planner The leg passed no planner flag, so once adaptive planning became the default it duplicated the "AQE on" leg and the static planner lost TPC-H coverage. --- .github/workflows/tpch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index cf1de6e575..47ee370546 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -68,7 +68,7 @@ jobs: matrix: include: - label: "AQE off" - planner_args: "" + planner_args: "-c ballista.planner.adaptive.enabled=false" slug: "aqe-off" - label: "AQE on" planner_args: "-c ballista.planner.adaptive.enabled=true" From b054c3cb29071bd1c8909a98639906864716fdbb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 26 Oct 2026 07:16:08 -0600 Subject: [PATCH 3/3] ci: run TPC-DS under both the static and adaptive planners Add static planner legs alongside the adaptive ones, each at one and four partitions per task, so both planners are gated on all 99 queries. --- .github/workflows/tpcds.yml | 56 ++++++++++++++++++++++--------------- benchmarks/README.md | 4 +-- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index 4bdb082381..daebe3c1fa 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -56,7 +56,7 @@ on: jobs: tpcds-sf1: - name: TPC-DS SF1 (adaptive planner, ${{ matrix.label }}) + name: TPC-DS SF1 (${{ matrix.planner }} planner, ${{ matrix.label }}) runs-on: ubuntu-latest container: image: amd64/rust @@ -65,21 +65,36 @@ jobs: # sitting for hours with no useful signal. timeout-minutes: 45 strategy: - # One red leg must not cancel the other; the two task-packing modes are - # independent signals. + # One red leg must not cancel the others; each planner and task-packing + # combination is an independent signal. fail-fast: false matrix: + # Both planners, each at both task-packing settings. The planner is + # always set explicitly so the job label stays true if the default + # changes. + # + # 4 partitions per task rather than a larger cap because the scheduler + # clamps the slice to the executor's free vcores + # (`budget.vcores.min(cap)`), and the executor runs with `--vcores 4`. + # A higher cap would be unreachable and the label would overstate what + # is covered. include: - - label: "1 partition per task" - task_args: "" - slug: "mpt1" - # 4 rather than a larger cap because the scheduler clamps the slice to - # the executor's free vcores (`budget.vcores.min(cap)`), and the - # executor runs with `--vcores 4`. A higher cap would be unreachable - # and the label would overstate what is covered. - - label: "4 partitions per task" - task_args: "-c ballista.scheduler.max_partitions_per_task=4" - slug: "mpt4" + - planner: "static" + label: "1 partition per task" + extra_args: "-c ballista.planner.adaptive.enabled=false" + slug: "static-mpt1" + - planner: "static" + label: "4 partitions per task" + extra_args: "-c ballista.planner.adaptive.enabled=false -c ballista.scheduler.max_partitions_per_task=4" + slug: "static-mpt4" + - planner: "adaptive" + label: "1 partition per task" + extra_args: "-c ballista.planner.adaptive.enabled=true" + slug: "aqe-mpt1" + - planner: "adaptive" + label: "4 partitions per task" + extra_args: "-c ballista.planner.adaptive.enabled=true -c ballista.scheduler.max_partitions_per_task=4" + slug: "aqe-mpt4" steps: - uses: actions/checkout@v7.0.1 with: @@ -119,21 +134,18 @@ jobs: run: | set -euo pipefail - # This matrix leg runs the suite under the adaptive planner (AQE) at - # one task-packing setting. AQE is the default, but it is set - # explicitly so the job label stays true if the default changes. The - # tpcds binary internally loops all non-skipped queries and exits - # non-zero on any failure, so a single invocation covers the whole - # suite. The other leg runs in parallel. - echo "::group::[adaptive planner, ${{ matrix.label }}] TPC-DS suite" + # This matrix leg runs the suite under one planner at one + # task-packing setting. The tpcds binary internally loops all + # non-skipped queries and exits non-zero on any failure, so a single + # invocation covers the whole suite. The other legs run in parallel. + echo "::group::[${{ matrix.planner }} planner, ${{ matrix.label }}] TPC-DS suite" ./target/tpch-ci/tpcds \ --host 127.0.0.1 --port 50050 \ --path "$DATA_DIR" \ --partitions 16 \ --verify \ -c datafusion.optimizer.prefer_hash_join=false \ - -c ballista.planner.adaptive.enabled=true \ - ${{ matrix.task_args }} + ${{ matrix.extra_args }} echo "::endgroup::" - name: Stop Ballista cluster diff --git a/benchmarks/README.md b/benchmarks/README.md index 114ae6e54a..737ec24a35 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -469,7 +469,7 @@ are accepted rather than skipped. `.github/workflows/tpcds.yml` runs the SF1 suite against a local scheduler + executor on every push/PR touching `ballista/**` or -`benchmarks/**`, under the adaptive planner (AQE), with one leg at one -partition per task and one at up to four partitions per task. +`benchmarks/**`. It runs four legs: the static planner and the adaptive planner +(AQE), each at one partition per task and at up to four partitions per task. [1]: http://www.tpc.org/tpch/