rust-instrumented.yml

name: Rust Instrumented

# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
  push:
    branches: [develop]
  pull_request: { }
  workflow_dispatch: { }

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  NIGHTLY_TOOLCHAIN: nightly-2026-09-10

jobs:
  rust-coverage:
    name: "Rust tests (coverage) (${{ matrix.suite }})"
    # Coverage only runs on develop; PRs get the same tests uninstrumented in ci.yml.
    if: github.event_name != 'pull_request'
    timeout-minutes: 30
    permissions:
      id-token: write
    strategy:
      matrix:
        include:
          - suite: tests
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-large/volume=200gb/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite)
          || 'ubuntu-latest' }}
    env:
      RUSTFLAGS: "-Cinstrument-coverage -A warnings"
      CARGO_INCREMENTAL: 0  # Disable incremental compilation to get accurate coverage
      # %8m merges counters online into a pool of 8 files per binary. A per-process (%p) file for
      # each nextest test process fills the runner disk.
      LLVM_PROFILE_FILE: "target/coverage/vortex-%8m.profraw"
      GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Ensure llvm-tools are installed
        run: |
          rustup component add llvm-tools || \
            rustup component add llvm-tools-preview

          # Verify llvm-profdata is accessible (grcov needs it for coverage merging)
          RUST_SYSROOT="$(rustc --print sysroot)"
          RUST_HOST="$(rustc --print host-tuple)"
          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
          LLVM_PROFDATA="${LLVM_TOOLS_BIN}/llvm-profdata"
          if [ ! -f "$LLVM_PROFDATA" ]; then
            echo "ERROR: llvm-profdata not found at $LLVM_PROFDATA"
            echo "Listing bin directory:"
            ls -la "$LLVM_TOOLS_BIN" || echo "Directory does not exist"
            exit 1
          fi
          echo "Found llvm-profdata at $LLVM_PROFDATA"

      - name: Rust Tests
        if: ${{ matrix.suite == 'tests' }}
        run: |
          cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-python-cuda
      - name: Generate coverage report
        run: |
          RUST_SYSROOT="$(rustc --print sysroot)"
          RUST_HOST="$(rustc --print host-tuple)"
          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
          grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
            --llvm-path "${LLVM_TOOLS_BIN}" \
            --threads $(nproc) \
            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' \
            --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' \
            --ignore benchmarks/* --ignore 'vortex-test/*' \
            -o ${{ env.GRCOV_OUTPUT_FILE }}
          test -s ${{ env.GRCOV_OUTPUT_FILE }}
      - name: Codecov
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5  # v7
        with:
          name: run-${{ matrix.suite }}
          files: ${{ env.GRCOV_OUTPUT_FILE }}
          disable_search: true
          flags: ${{ matrix.suite }}
          use_oidc: true

  cxx-coverage:
    name: "C++ API (coverage)"
    if: github.event_name != 'pull_request'
    timeout-minutes: 15
    env:
      # Symbolizing error backtrace in debug .so is slow
      RUST_LIB_BACKTRACE: 0
    permissions:
      id-token: write
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Install lcov
        run: |
          if ! command -v lcov >/dev/null || ! dpkg-query -W libjson-xs-perl >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y lcov libjson-xs-perl
          fi
      - name: Build and test C++ API with coverage
        working-directory: lang/cpp
        run: ./gcov-report.sh
      - name: Codecov
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5  # v7
        with:
          name: cxx-api
          files: lang/cpp/coverage.info
          disable_search: true
          flags: cxx-api
          use_oidc: true

  rust-test-sanitizer:
    strategy:
      fail-fast: false
      matrix:
        include:
          # We don't run memory sanitizer as it provides many false positives
          # for std
          - sanitizer: asan
            sanitizer_flags: "-Zsanitizer=address,leak"
          - sanitizer: tsan
            sanitizer_flags: "-Zsanitizer=thread"
    name: "Rust tests (${{ matrix.sanitizer }})"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-test-sanitizer', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    env:
      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
      LSAN_OPTIONS: "report_objects=1"
      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      MSAN_OPTIONS: "symbolize=1"
      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt"
      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      VORTEX_SKIP_SLOW_TESTS: "1"
      # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins
      # unset -Zsanitizer flag and we should allow that.
      RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none"
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Install Rust nightly toolchain
        run: |
          rustup toolchain install $NIGHTLY_TOOLCHAIN
          rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
      - name: Build tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN build --tests --locked --all-features \
            --target x86_64-unknown-linux-gnu -Zbuild-std \
            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array

      - name: Run tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features --no-fail-fast \
            --target x86_64-unknown-linux-gnu -Zbuild-std \
            -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array

        # cargo nextest reports less sanitizer issues than cargo test
        # --tests skips doctests: rustdoc ignores RUSTFLAGS (it uses RUSTDOCFLAGS),
        # so doctests would build vortex-ffi without -Zsanitizer and mismatch the
        # sanitizer-built deps.
      - name: Run vortex-ffi tests with sanitizer
        run: |
          RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
          cargo +$NIGHTLY_TOOLCHAIN test --locked \
            --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \
            -p vortex-ffi --tests -- --no-capture

  rust-ffi-test-sanitizer:
    strategy:
      fail-fast: false
      matrix:
        include:
          # We don't run memory sanitizer as it's clang-only and provides many
          # false positives for Catch2
          - sanitizer: asan
          - sanitizer: tsan
    name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})"
    # Allow cold builds of Rust's sanitizer-instrumented standard library.
    timeout-minutes: 15
    env:
      ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
      LSAN_OPTIONS: "report_objects=1"
      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      MSAN_OPTIONS: "symbolize=1"
      MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt"
      TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      VORTEX_SKIP_SLOW_TESTS: "1"
      # Symbolizing error backtrace in debug .so is slow
      RUST_LIB_BACKTRACE: 0
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id)
          || 'ubuntu-latest' }}
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
          toolchain: ${{ env.NIGHTLY_TOOLCHAIN }}
          components: "rust-src, rustfmt, clippy, llvm-tools-preview"
      - name: Build FFI library, tests, and examples
        working-directory: vortex-ffi
        env:
          RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }}
        run: |
          cmake -S . -B build \
            -DCMAKE_BUILD_TYPE=Debug \
            -DCMAKE_LINKER_TYPE=MOLD \
            -DBUILD_SHARED_LIBS=ON \
            -DCMAKE_C_COMPILER=clang \
            -DCMAKE_CXX_COMPILER=clang++ \
            -DVORTEX_SANITIZER=${{ matrix.sanitizer }} \
            -DVORTEX_SANITIZE_RUST_STD=ON \
            -DVORTEX_BUILD_TESTS=ON \
            -DVORTEX_BUILD_EXAMPLES=ON
          cmake --build build --parallel "$(nproc)"
      - name: Run tests
        run: |
          set -o pipefail
          ./vortex-ffi/build/test/vortex_ffi_test
      - name: Run examples
        run: |
          set -o pipefail

          ./vortex-ffi/build/examples/write_sample file.vortex
          ./vortex-ffi/build/examples/write_sample file2.vortex
          ./vortex-ffi/build/examples/dtype '*.vortex'
          ./vortex-ffi/build/examples/scan '*.vortex'
          ./vortex-ffi/build/examples/scan_to_arrow '*.vortex'

  miri:
    name: "Rust tests (miri)"
    runs-on: >-
      ${{ github.repository == 'vortex-data/vortex'
          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id)
          || 'ubuntu-latest' }}
    timeout-minutes: 30
    env:
      MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE
      RUSTFLAGS: "-A warnings"
      RUST_BACKTRACE: full
    steps:
      - parallel:
          - uses: runs-on/action@v2
            if: github.repository == 'vortex-data/vortex'
            with:
              sccache: s3
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
      - uses: ./.github/actions/setup-prebuild
        with:
          enable-sccache: "true"
      - name: Install nightly with miri
        run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri
      - name: Run Miri
        run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi