diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index b503d0e705..daebe3c1fa 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -56,7 +56,7 @@ on: jobs: tpcds-sf1: - name: TPC-DS SF1 (static planner, ${{ matrix.label }}) + name: TPC-DS SF1 (${{ matrix.planner }} planner, ${{ matrix.label }}) runs-on: ubuntu-latest container: image: amd64/rust @@ -65,21 +65,36 @@ jobs: # sitting for hours with no useful signal. timeout-minutes: 45 strategy: - # One red leg must not cancel the other; the two task-packing modes are - # independent signals. + # One red leg must not cancel the others; each planner and task-packing + # combination is an independent signal. fail-fast: false matrix: + # Both planners, each at both task-packing settings. The planner is + # always set explicitly so the job label stays true if the default + # changes. + # + # 4 partitions per task rather than a larger cap because the scheduler + # clamps the slice to the executor's free vcores + # (`budget.vcores.min(cap)`), and the executor runs with `--vcores 4`. + # A higher cap would be unreachable and the label would overstate what + # is covered. include: - - label: "1 partition per task" - task_args: "" - slug: "mpt1" - # 4 rather than a larger cap because the scheduler clamps the slice to - # the executor's free vcores (`budget.vcores.min(cap)`), and the - # executor runs with `--vcores 4`. A higher cap would be unreachable - # and the label would overstate what is covered. - - label: "4 partitions per task" - task_args: "-c ballista.scheduler.max_partitions_per_task=4" - slug: "mpt4" + - planner: "static" + label: "1 partition per task" + extra_args: "-c ballista.planner.adaptive.enabled=false" + slug: "static-mpt1" + - planner: "static" + label: "4 partitions per task" + extra_args: "-c ballista.planner.adaptive.enabled=false -c ballista.scheduler.max_partitions_per_task=4" + slug: "static-mpt4" + - planner: "adaptive" + label: "1 partition per task" + extra_args: "-c ballista.planner.adaptive.enabled=true" + slug: "aqe-mpt1" + - planner: "adaptive" + label: "4 partitions per task" + extra_args: "-c ballista.planner.adaptive.enabled=true -c ballista.scheduler.max_partitions_per_task=4" + slug: "aqe-mpt4" steps: - uses: actions/checkout@v7.0.1 with: @@ -119,21 +134,18 @@ jobs: run: | set -euo pipefail - # This matrix leg runs the suite under the default (static) planner at - # one task-packing setting. The tpcds binary internally loops all + # This matrix leg runs the suite under one planner at one + # task-packing setting. The tpcds binary internally loops all # non-skipped queries and exits non-zero on any failure, so a single - # invocation covers the whole suite. The other leg runs in parallel. - # - # Coverage for the adaptive planner (AQE on) is being added - # separately; it currently fails on pre-existing bugs. - echo "::group::[static planner, ${{ matrix.label }}] TPC-DS suite" + # invocation covers the whole suite. The other legs run in parallel. + echo "::group::[${{ matrix.planner }} planner, ${{ matrix.label }}] TPC-DS suite" ./target/tpch-ci/tpcds \ --host 127.0.0.1 --port 50050 \ --path "$DATA_DIR" \ --partitions 16 \ --verify \ -c datafusion.optimizer.prefer_hash_join=false \ - ${{ matrix.task_args }} + ${{ matrix.extra_args }} echo "::endgroup::" - name: Stop Ballista cluster diff --git a/.github/workflows/tpch.yml b/.github/workflows/tpch.yml index cf1de6e575..47ee370546 100644 --- a/.github/workflows/tpch.yml +++ b/.github/workflows/tpch.yml @@ -68,7 +68,7 @@ jobs: matrix: include: - label: "AQE off" - planner_args: "" + planner_args: "-c ballista.planner.adaptive.enabled=false" slug: "aqe-off" - label: "AQE on" planner_args: "-c ballista.planner.adaptive.enabled=true" diff --git a/benchmarks/README.md b/benchmarks/README.md index 4b5a50dac6..737ec24a35 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -459,21 +459,17 @@ queries run unchanged. ### Skip list -Some queries are excluded from the gate. These are listed in the `SKIP` const -near the top of `benchmarks/src/bin/tpcds.rs`, as `(query_id, reason)` pairs, -grouped by cause: queries whose distributed result diverges from DataFusion -(tracked as bugs), queries that are non-deterministic under `LIMIT`/`ORDER BY` -ties, and queries whose `tpcgen-cli` schema column names differ from the -DataFusion query text. A default run (no `--query`) executes `1..=99` minus -`SKIP`. +Queries can be excluded from the gate by listing them in the `SKIP` const near +the top of `benchmarks/src/bin/tpcds.rs`, as `(query_id, reason)` pairs. The +list is currently empty, so a default run (no `--query`) executes all 99 +queries. Results that differ only in the order of rows tied under `ORDER BY` +are accepted rather than skipped. ### CI `.github/workflows/tpcds.yml` runs the SF1 suite against a local scheduler + executor on every push/PR touching `ballista/**` or -`benchmarks/**`, under the default (static) planner. The adaptive planner -(AQE on) is not gated yet — it currently fails many TPC-DS queries with an -`EmptyExec invalid partition` assertion (issue #2047); re-enable an AQE-on run -once that is fixed. +`benchmarks/**`. It runs four legs: the static planner and the adaptive planner +(AQE), each at one partition per task and at up to four partitions per task. [1]: http://www.tpc.org/tpch/