ci.yml

name: Linters and Tests

# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
  push:
    branches: [develop]
  pull_request: { }
  workflow_dispatch: { }

permissions:
  actions: read
  contents: read
  checks: write  # audit-check creates checks
  issues: write  # audit-check creates issues

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  NIGHTLY_TOOLCHAIN: nightly-2026-09-10

jobs:
  duckdb-mirror:
    name: "Mirror DuckDB to R2"
    if: github.event_name == 'pull_request'
    uses: ./.github/workflows/duckdb-r2.yml
    secrets: inherit

  duckdb-ready:
    name: "DuckDB libraries available in R2"
    needs: duckdb-mirror
    if: "!cancelled()"
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Verify DuckDB mirror
        if: needs.duckdb-mirror.result == 'failure'
        run: |
          echo "DuckDB mirror failed; downstream builds would 404"
          exit 1

  lint-toml:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6

  validate-workflow-yaml:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - name: Validate YAML file
        run: |
          # Lint the workflows and yamllint's configuration file.
          yamllint \
            --strict \
            -c .yamllint.yaml \
            .github/

  python-lint:
    name: "Python (lint)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-lint', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 10
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
      - name: Python Lint - Format
        run: uvx ruff format --check .
      - name: Python Lint - Ruff
        run: uvx ruff check .
      # ty needs the Python dependencies for type information, but resolves the
      # vortex package itself from the in-tree sources and stubs, so skip the
      # Rust extension build entirely.
      - name: Python Lint - ty
        run: |
          uv sync --all-packages --no-install-workspace
          uvx ty check vortex-python vortex-python-cuda vortex-ffi/cmake/tests scripts/tests

  python-test:
    name: "Python (test)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-test', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    env:
      RUST_LOG: "info,maturin=off,uv=debug"
      MATURIN_PEP517_ARGS: "--profile ci"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"

      - name: Pytest - Vortex
        run: |
          uv run --all-packages pytest --benchmark-disable -n auto test/
        working-directory: vortex-python/

      # The datasets integration relies on private `datasets` APIs, so exercise the oldest
      # supported release in addition to the locked one.
      - name: Pytest - Vortex (datasets 4.x floor)
        run: |
          uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \
            pytest --benchmark-disable test/test_hf_datasets.py
        working-directory: vortex-python/

      # The lockfile tracks Polars 2, so also exercise the oldest supported 1.x release. The
      # Polars integration parses Polars' serialized expression format, which differs across
      # versions.
      - name: Pytest - Vortex (Polars 1.x floor)
        run: |
          uv run --all-packages --with 'polars==1.32.0' \
            pytest --benchmark-disable test/test_polars_.py test/test_dataset.py
        working-directory: vortex-python/

      - name: Check Python documentation
        run: |
          uv run --all-packages make check
        working-directory: docs/

  python-windows:
    name: "Python (windows-x64)"
    timeout-minutes: 60
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id)
          || 'windows-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
        with:
          sccache: s3
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - name: Setup (Windows)
        run: |
          echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7
        with:
          python-version: "3.11"

      # Mirrors the wheel build in package.yml, which spells the Windows interpreter
      # `python` because maturin has no `python3.11` executable to resolve there.
      - name: Build wheel
        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b  # v1
        with:
          command: build
          maturin-version: 1.15.0
          target: x86_64-pc-windows-msvc
          args: >
            --profile ci
            --manifest-path vortex-python/Cargo.toml
            --out dist
            --interpreter python
        env:
          # Keep this constant to avoid pyo3 invalidating itself
          PYO3_ENVIRONMENT_SIGNATURE: "cpython3"
          # Also applies to host proc-macro links when Cargo receives an explicit target.
          LINK: /Brepro

      # rust-test-windows excludes vortex-python, so this is the only check that the
      # win_amd64 wheel imports and can round-trip a file. Runs outside the checkout so
      # the repo-root `vortex/` crate directory cannot shadow the installed package.
      - name: Import and round-trip the wheel
        shell: bash
        run: |
          python -m pip install dist/*.whl
          cd "$RUNNER_TEMP"
          python - <<'PY'
          import pyarrow as pa

          import vortex as vx

          table = pa.table({"a": [1, 2, 3], "b": ["x", "y", "z"]})
          vx.io.write(table, "smoke.vortex")
          # Vortex reads utf8 back as string_view, so compare column-wise rather than by table.
          round_tripped = vx.open("smoke.vortex").to_arrow().read_all()
          assert round_tripped.column_names == table.column_names
          assert round_tripped.column("a").to_pylist() == [1, 2, 3]
          assert round_tripped.column("b").to_pylist() == ["x", "y", "z"]
          print("vortex", vx.__version__, "round-tripped a file on Windows")
          PY

  # Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
  # benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
  # the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
  # would break ON CONFLICT idempotency against the rows already in production.
  measurement-id-golden:
    name: "measurement_id golden vectors"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      # sync: false — the test runs via `uv run --no-project` and needs no workspace
      # packages; the default `uv sync` builds the vortex-data Rust extension (~6 min).
      - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
        with:
          sync: false
      - name: Pytest - measurement_id golden vectors
        run: |
          uv run --no-project --with pytest --with xxhash \
            pytest scripts/tests/test_measurement_id.py

  benchmark-reporting:
    name: "Benchmark reporting"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      # sync: false -- the tests run via `uv run --no-project` and need no workspace packages.
      - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
        with:
          sync: false
      - name: Pytest - benchmark reporting and ingestion
        run: |
          uv run --no-project --with pytest --with pandas --with numpy --with orjson \
            --with tabulate --with xxhash \
            pytest scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py

  python-cuda-test:
    name: "Python CUDA (test)"
    if: github.repository == 'vortex-data/vortex'
    runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=python-cuda-test
    timeout-minutes: 30
    env:
      RUST_LOG: "info,maturin=off,uv=debug"
      MATURIN_PEP517_ARGS: "--profile ci"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - parallel:
          - uses: ./.github/actions/setup-rust
            with:
              repo-token: ${{ secrets.GITHUB_TOKEN }}
              components: cargo
              enable-sccache: "true"
          - name: Install uv
            uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
            with:
              sync: false
      - name: Pin rustup proxy to repository toolchain
        run: |
          TOOLCHAIN="$(grep '^channel' rust-toolchain.toml | cut -d '"' -f 2)"
          echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV"

      - name: Pytest - PyVortex CUDA bridge
        run: |
          uv run --extra cuda \
            pytest --benchmark-disable ../vortex-python-cuda/test/test_native_bridge.py
        working-directory: vortex-python/

  rust-docs:
    name: "Rust (docs)"
    needs: duckdb-ready
    # The PR-only mirror is a skipped ancestor on develop; override implicit success().
    if: ${{ !cancelled() && needs.duckdb-ready.result == 'success' }}
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-docs/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Docs
        env:
          # required to make sure docs build for features
          RUSTDOCFLAGS: "-D warnings --cfg docsrs"
        run: |
          # Build docs for the whole workspace (aside from vortex-python that is having hard time with the python-docs syntax),
          # including all private docs.
          cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
          # nextest doesn't support doc tests, so we run it here
          cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast

  build-rust:
    name: "Rust build (${{matrix.config.name}})"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
          || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: "all-features"
            runner: amd64-ci-large
            args: "--all-features --all-targets"
          - name: "default features"
            runner: amd64-ci-large
            args: "--all-targets"
          - name: "with tokio dispatcher"
            runner: amd64-ci-medium
            # Only build the crates that have the tokio features, not re-building other crates with no-default-features
            args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout"
          - name: "wasm32 with default features"
            runner: amd64-ci-medium
            target: wasm32-unknown-unknown
            env:
              rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'"
            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
          targets: ${{ matrix.config.target }}
      - uses: ./.github/actions/check-rebuild
        with:
          command: "${{matrix.config.env.rustflags}} cargo hack build --profile ci --locked ${{matrix.config.args}} --ignore-private"
      - name: "Make sure no files changed after build"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"
      - name: sccache stats
        if: always() && github.repository == 'vortex-data/vortex'
        run: sccache --show-stats

  check-min-deps:
    name: "Check build with minimal dependencies"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - run: cargo minimal-versions check --direct --workspace --ignore-private

  rust-msrv:
    name: "Rust (MSRV)"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-msrv', github.run_id)
          || 'ubuntu-latest' }}
    env:
      RUSTFLAGS: "-A warnings"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Check publishable crates against the MSRV
        run: cargo hack --rust-version --no-dev-deps --ignore-private check --profile ci --all-features

  rust-lint:
    name: "Rust (lint)"
    needs: duckdb-ready
    if: ${{ !cancelled() && needs.duckdb-ready.result == 'success' }}
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Install nightly for fmt
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt
      # Run one Cargo command at a time so it can use the whole runner and reuse
      # artifacts from the shared target directory in subsequent checks.
      - name: Rust Lint - Format
        id: fmt
        continue-on-error: true
        run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check
      - name: Rustc check (release)
        id: check-release
        continue-on-error: true
        run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release
      - name: Rust Lint - Clippy All Features
        id: clippy-all
        continue-on-error: true
        run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings
      - name: Rust Lint - Clippy Default Features
        id: clippy-default
        continue-on-error: true
        run: cargo clippy --profile ci --locked --all-targets -- -D warnings
      - name: Check lint results
        if: always()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9
        with:
          script: |
            const failed = Object.entries({
              fmt: '${{ steps.fmt.outcome }}',
              'check-release': '${{ steps.check-release.outcome }}',
              'clippy-all': '${{ steps.clippy-all.outcome }}',
              'clippy-default': '${{ steps.clippy-default.outcome }}',
            }).filter(([, o]) => o === 'failure').map(([n]) => n);
            if (!failed.length) return;

            const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({
              ...context.repo, run_id: context.runId,
            });
            const url = jobs.find(j => j.name === context.job)?.check_run_url;
            if (url) {
              await github.rest.checks.update({
                ...context.repo,
                check_run_id: Number(url.split('/').pop()),
                output: { title: `${failed.join(', ')} - failing`, summary: '' },
              });
            }
            core.setFailed(`Lint failed: ${failed.join(', ')}`);

      - name: sccache stats
        if: always() && github.repository == 'vortex-data/vortex'
        run: sccache --show-stats

  rust-lint-no-default:
    name: "Rust (lint, no default)"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Rust Lint - Clippy No Default Features
        shell: bash
        run: |
          cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings

  cpp-lint:
    name: "C/C++ (lint)"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - name: C/C++ Lint - clang-format
        run: |
          git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \
            | grep -E '\.(cpp|hpp|cu|cuh|h)$' \
            | grep -v 'arrow/reference/arrow_c_device\.h$' \
            | grep -v 'kernels/src/bit_unpack_.*\.cu$' \
            | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \
            | xargs clang-format --dry-run --Werror --style=file

  rust-test-windows:
    name: "Rust tests (windows-x64)"
    env:
      # Stable proc-macro DLL contents let dependent crates reuse sccache entries.
      RUSTFLAGS: "-C link-arg=/Brepro"
    needs: duckdb-ready
    if: ${{ !cancelled() && needs.duckdb-ready.result == 'success' }}
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id)
          || 'windows-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - name: Setup (Windows)
        run: |
          echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Rust Tests (Windows)
        run: |
          cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast `
            --exclude vortex-bench `
            --exclude vortex-python --exclude vortex-duckdb `
            --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-cuda-ffi `
            --exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda `
            --exclude duckdb-bench `
            --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench `
            --exclude compress-bench --exclude xtask --exclude vortex-datafusion `
            --exclude gpu-scan-cli --exclude vortex-sqllogictest

      - name: sccache stats
        if: always() && github.repository == 'vortex-data/vortex'
        run: sccache --show-stats

      - name: Alert incident.io
        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
        uses: ./.github/actions/alert-incident-io
        with:
          api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
          alert-title: "Rust tests (windows-x64) failed on develop"
          deduplication-key: ci-rust-test-windows-x64-failure

  rust-test-linux-arm64:
    name: "Rust tests (linux-arm64)"
    needs: duckdb-ready
    if: >-
      !cancelled() && needs.duckdb-ready.result == 'success' &&
      github.repository == 'vortex-data/vortex'
    timeout-minutes: 30
    runs-on: runs-on=${{ github.run_id }}/runner=arm64-ci-test/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64
    steps:
      - parallel:
          - uses: runs-on/action@v2
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Rust Tests
        run: |
          cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest
      - uses: ./.github/actions/check-rebuild
        with:
          command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest"

      - name: Alert incident.io
        if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
        uses: ./.github/actions/alert-incident-io
        with:
          api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
          alert-title: "Rust tests (linux-arm64) failed on develop"
          deduplication-key: ci-rust-test-linux-arm64-failure

  # Runs the crates with hand-written SIMD kernels on EC2 instance families chosen for
  # their CPU features, so that every runtime-dispatched kernel tier is exercised:
  #   c6a (AMD Zen 3): AVX2 and BMI2, no AVX-512
  #   c6i (Intel Ice Lake): AVX-512 F/BW/VBMI2/VPOPCNTDQ
  #   c8g (Graviton4, Neoverse V2): SVE2 with BITPERM
  # `VORTEX_EXPECT_CPU_FEATURES` lists every feature the instance must have; vortex-array-macros'
  # `cpu_features` test fails if any of them is missing or any other known feature (e.g.
  # AVX-512 on the avx2 entry) is detected.
  rust-test-cpu-features:
    name: "Rust tests (cpu-features, ${{ matrix.config.name }})"
    if: github.repository == 'vortex-data/vortex'
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: avx2
            runner: amd64-ci-medium
            family: c6a.2xlarge
            image: ubuntu24-full-x64-pre-v2
            expect: "sse2,popcnt,bmi1,bmi2,avx,avx2"
          - name: avx512
            runner: amd64-ci-medium
            family: c6i.2xlarge
            image: ubuntu24-full-x64-pre-v2
            expect: "sse2,popcnt,bmi1,bmi2,avx,avx2,avx512f,avx512bw,avx512vbmi2,avx512vpopcntdq"
          - name: sve
            runner: arm64-medium
            family: c8g.2xlarge
            image: ubuntu24-full-arm64-pre-v2
            expect: "neon,sve,sve2,sve2-bitperm"
    runs-on: >-
      runs-on=${{ github.run_id }}/runner=${{ matrix.config.runner }}/family=${{ matrix.config.family }}/spot=price-capacity-optimized/image=${{ matrix.config.image }}/extras=s3-cache/tag=rust-test-cpu-features-${{ matrix.config.name }}
    env:
      VORTEX_EXPECT_CPU_FEATURES: ${{ matrix.config.expect }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      # Print the detected features and fail fast if the instance is not the expected tier.
      - name: Check CPU features
        run: |
          cargo nextest run --cargo-profile ci --locked --no-capture \
            -p vortex-array-macros --test cpu_features
      - name: Rust Tests
        run: |
          cargo nextest run --cargo-profile ci --locked --no-fail-fast \
            -p vortex-buffer -p vortex-mask -p vortex-array

  btrblocks-golden:
    name: "Rust (btrblocks golden corpus)"
    timeout-minutes: 10
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      # The golden suite pins compressor decisions for the core-only and explicitly enabled
      # edition configurations. Compact compression additionally needs its optional schemes.
      - name: Golden corpus (edition configurations)
        run: |
          cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden
      - name: Golden corpus (compact, zstd + pco)
        run: |
          cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \
            --features zstd,pco -E 'test(golden_compact)'

  build-java:
    name: "Java"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=java', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - parallel:
          - uses: ./.github/actions/setup-prebuild
            with:
              enable-sccache: "true"
          - uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6  # v6
      - name: Build documentation and run Java checks
        run: ./gradlew --parallel javadoc check
        working-directory: ./java

  license-check-and-audit-check:
    name: License Check and Audit Check
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        checks:
          - advisories
          - bans licenses sources
    # Prevent sudden announcement of a new advisory from failing ci:
    continue-on-error: ${{ matrix.checks == 'advisories' }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25  # v2
        with:
          command: check ${{ matrix.checks }}

  cmake-integration-test:
    name: "CMake integration (test)"
    timeout-minutes: 10
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/tag=cmake-integration-test', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - uses: runs-on/action@v2
        if: github.repository == 'vortex-data/vortex'
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
      - name: Install Ninja for CMake integration tests
        run: |
          sudo apt-get update
          sudo apt-get install -y ninja-build
      # Populate the registry cache for the lockfile-selected offline fixtures.
      - name: Fetch Cargo dependencies for offline fixtures
        run: cargo fetch --locked
      - name: Test CMake integration
        run: python3 -m unittest discover -s vortex-ffi/cmake/tests -v

  cxx-api:
    name: "C++ API"
    # Leave room for cold, per-build-tree Cargo artifacts, including sanitized std.
    timeout-minutes: 15
    env:
      # symbolizing error backtrace on debug .so is slow
      RUST_LIB_BACKTRACE: 0
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
          toolchain: ${{ env.NIGHTLY_TOOLCHAIN }}
          components: "rust-src, rustfmt, clippy, llvm-tools-preview"
      - name: Build C++ tests and examples (asan)
        env:
          RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }}
        run: |
          cmake -S lang/cpp -B build \
            -DCMAKE_BUILD_TYPE=Debug \
            -DCMAKE_LINKER_TYPE=MOLD \
            -DVORTEX_CARGO_PROFILE=ci \
            -DBUILD_SHARED_LIBS=ON \
            -DCMAKE_C_COMPILER=clang \
            -DCMAKE_CXX_COMPILER=clang++ \
            -DVORTEX_SANITIZER=asan \
            -DVORTEX_SANITIZE_RUST_STD=ON \
            -DVORTEX_BUILD_TESTS=ON \
            -DVORTEX_BUILD_EXAMPLES=ON
          cmake --build build --parallel "$(nproc)"
      - name: Show C++ API compiler cache statistics
        if: github.repository == 'vortex-data/vortex'
        run: sccache --show-stats
      - name: Run C++ tests
        run: ctest --test-dir build --output-on-failure
      - name: Run C++ examples
        run: |
          cd build/examples
          ./writer people0.vortex
          ./writer people1.vortex
          ./writer me.vortex

          ./scan '*.vortex'
          ./dtype me.vortex

          ./reader

  sqllogic-test:
    name: "SQL logic tests"
    needs: duckdb-ready
    if: ${{ !cancelled() && needs.duckdb-ready.result == 'success' }}
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-sql-logic-test/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=sql-logic-test', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Run sqllogictest tests
        run: |
          VORTEX_SLT_PROFILE=ci ./vortex-sqllogictest/slt/generate_data.sh
          cargo nextest run --cargo-profile ci -p vortex-sqllogictest
      - uses: ./.github/actions/check-rebuild
        with:
          command: "cargo nextest run --cargo-profile ci -p vortex-sqllogictest --no-run"
          message-format-flag: "--cargo-message-format"

  duckdb-s3-test:
    name: "DuckDB S3 tests"
    needs: duckdb-ready
    if: ${{ !cancelled() && needs.duckdb-ready.result == 'success' }}
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=duckdb-s3-test', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - parallel:
          - name: Build DuckDB tests
            shell: bash
            run: |
              cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run
          - name: Start RustFS
            shell: bash
            run: |
              curl -sSL https://dl.rustfs.com/artifacts/rustfs/release/rustfs-linux-x86_64-musl-latest.zip \
                -o "${RUNNER_TEMP}/rustfs.zip"
              unzip -o -q "${RUNNER_TEMP}/rustfs.zip" rustfs -d "${RUNNER_TEMP}"
              chmod +x "${RUNNER_TEMP}/rustfs"
              mkdir -p "${RUNNER_TEMP}/rustfs-data"
              RUSTFS_ACCESS_KEY=rustfsadmin RUSTFS_SECRET_KEY=rustfsadmin \
                "${RUNNER_TEMP}/rustfs" "${RUNNER_TEMP}/rustfs-data" --address 127.0.0.1:9000 &
              until curl -sf http://127.0.0.1:9000/health; do sleep 1; done
              curl -sf -X PUT "http://127.0.0.1:9000/vortex-test" \
                --user "rustfsadmin:rustfsadmin" --aws-sigv4 "aws:amz:us-east-1:s3"
      - name: Run tests
        shell: bash
        env:
          AWS_ACCESS_KEY_ID: rustfsadmin
          AWS_SECRET_ACCESS_KEY: rustfsadmin
          AWS_ENDPOINT: "http://127.0.0.1:9000"
          AWS_REGION: "us-east-1"
          AWS_ALLOW_HTTP: "true"
          VORTEX_DUCKDB_S3_URI: "s3://vortex-test/duckdb"
        run: |
          cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --run-ignored all s3_

  wasm-integration:
    name: "WASM integration smoke test"
    timeout-minutes: 30
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=wasm-integration', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - parallel:
          - uses: ./.github/actions/setup-prebuild
            with:
              enable-sccache: "true"
              targets: "wasm32-wasip1,wasm32-unknown-unknown"
          - name: Setup Wasmer
            uses: wasmerio/setup-wasmer@ba7631cb2d521612aa7ac5b13cb76fb77e824a1a  # v3
            with:
              version: "v7.4.0"
      - name: Verify Wasmer installation
        run: wasmer --version
      - run: cargo build --profile ci --target wasm32-wasip1
        working-directory: ./wasm-test
      - run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm
        working-directory: ./wasm-test
      - run: RUSTFLAGS='--cfg getrandom_backend="unsupported"' cargo build --profile ci --target wasm32-unknown-unknown
        working-directory: ./wasm-test
      - name: "Check wasm32-unknown-unknown does not need wasm-bindgen"
        # Pure wasm hosts (e.g. wasmtime) cannot provide wasm-bindgen's JS imports.
        shell: bash
        working-directory: ./wasm-test
        run: |
          ! cargo tree --target wasm32-unknown-unknown -e normal --prefix none \
            | grep -E '^(wasm-bindgen|js-sys) '

  generated-files:
    name: "Generated source files and C API"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=generated-files', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
            with:
              # check-editions compares against the merge base, so it needs real history.
              fetch-depth: 0
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      # The only job that needs flatc: builds use the checked-in bindings.
      - uses: ./.github/actions/setup-flatc
      - name: Install nightly for cbindgen macro expansion
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN
      - name: "regenerate the FlatBuffers and Protocol Buffers bindings"
        # Fails on a flatc version mismatch, and the porcelain check below fails on stale bindings.
        run: |
          cargo run --profile ci -p xtask -- generate-flatbuffers
          cargo run --profile ci -p xtask -- generate-proto
      - name: "regenerate the edition records"
        run: |
          cargo run --profile ci -p xtask -- generate-editions
      - name: "check frozen edition records never change"
        # Independent of the regeneration above: a stale record must not mask a frozen one
        # being edited, nor the other way round.
        if: "!cancelled()"
        run: |
          BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}"
          cargo run --profile ci -p xtask -- check-editions --base "$BASE"
      - name: "regenerate FFI header and test the C API"
        env:
          # Nightly regenerates cinclude/vortex.h with cbindgen during the build.
          RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }}
        run: |
          cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" \
            -DCMAKE_LINKER_TYPE=MOLD \
            -DBUILD_SHARED_LIBS=ON \
            -DVORTEX_CARGO_PROFILE=ci \
            -DVORTEX_BUILD_TESTS=ON
          cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)"
          ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" --output-on-failure
      - name: "Make sure no files changed after regenerating"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"

      # Schemas live under `<crate>/flatbuffers/` but resolve includes against those dirs
      # collectively, so flatten each revision into one tree for flatc to compare.
      - name: "Collect flatbuffer schemas from this revision and from develop"
        run: |
          git fetch origin develop --depth 1
          collect() {
            git ls-tree -r --name-only "$1" \
              | grep -E '(^|/)flatbuffers/.*\.fbs$' \
              | while read -r path; do
                  rel="${path#*/flatbuffers/}"
                  mkdir -p "$2/$(dirname "$rel")"
                  git show "$1:$path" > "$2/$rel"
                done
          }
          collect HEAD "$RUNNER_TEMP/fbs.head"
          collect origin/develop "$RUNNER_TEMP/fbs.develop"
      - name: "Verify flatbuffer back-compat"
        working-directory: ${{ runner.temp }}
        run: |
          find fbs.develop/ -type f -name "*.fbs" | sed 's|^fbs.develop/||' \
            | xargs -I{} -n1 flatc -I fbs.head --conform-includes fbs.develop --conform fbs.develop/{} fbs.head/{}