coverage.yml

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

name: "CI: Coverage"

on:
  schedule:
    - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC
  workflow_dispatch: {}

env:
  PY_VER: "3.14"
  LOCAL_CTK: "1"
  GPU: "a100"
  DRIVER: "latest"

jobs:
  coverage-vars:
    if: ${{ github.repository_owner == 'nvidia' }}
    runs-on: ubuntu-latest
    outputs:
      CUDA_VER: ${{ steps.get-vars.outputs.cuda_ver }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      - name: Get CUDA version
        id: get-vars
        run: |
          cuda_ver=$(yq '.cuda.build.version' ci/versions.yml)
          echo "cuda_ver=$cuda_ver" >> $GITHUB_OUTPUT

  coverage-linux:
    name: Coverage (Linux)
    needs: [coverage-vars]
    runs-on: "linux-amd64-gpu-a100-latest-1"
    permissions:
      id-token: write
      contents: write
    defaults:
      run:
        shell: bash --noprofile --norc -xeuo pipefail {0}
    env:
      HOST_PLATFORM: "linux-64"
      ARCH: "x86_64"
      CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }}
    # Our self-hosted runners require a container
    # TODO: use a different (nvidia?) container
    container:
      options: -u root --security-opt seccomp=unconfined --shm-size 16g
      image: ubuntu:24.04
      env:
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
        PIP_CACHE_DIR: "/tmp/pip-cache"
    steps:
      - name: Ensure GPU is working
        run: nvidia-smi

      # We have to install git before checking out the repo (so that we can
      # deploy the docs at the end). This means we can't use install_unix_deps
      # action so install the git package.
      - name: Install git
        run: |
          apt-get update
          apt-get install -y git

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          # Treeless clone: setuptools-scm needs the commit graph (`git describe`)
          # but not historical blobs.
          fetch-depth: 0
          filter: blob:none
          # Fetch CCCL separately: checkout's filter does not apply to submodules.
          submodules: false

      - name: Checkout CCCL submodule
        run: |
          git submodule sync --recursive
          git submodule update --init --force --depth=1 --filter=blob:none --recursive

      - name: Fix workspace ownership
        run: |
          chown -R $(id -u):$(id -g) "$GITHUB_WORKSPACE"

      - name: Install dependencies
        uses: ./.github/actions/install_unix_deps
        continue-on-error: false
        with:
          dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq"
          dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq"

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true

      - name: Set environment variables
        env:
          BUILD_CUDA_VER: ${{ env.CUDA_VER }}
          CUDA_VER: ${{ env.CUDA_VER }}
          HOST_PLATFORM: ${{ env.HOST_PLATFORM }}
          LOCAL_CTK: ${{ env.LOCAL_CTK }}
          PY_VER: ${{ env.PY_VER }}
          SHA: ${{ github.sha }}
        run: |
          ./ci/tools/env-vars test
          echo "CUDA_PYTHON_COVERAGE=1" >> $GITHUB_ENV

      - name: Set up Python
        uses: ./.github/actions/setup-hardened-python
        with:
          python-version: ${{ env.PY_VER }}
          venv: .venv

      - name: Set up mini CTK
        if: ${{ env.LOCAL_CTK == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ env.HOST_PLATFORM }}
          cuda-version: ${{ env.CUDA_VER }}

      - name: Build and install cuda-pathfinder wheel
        run: |
          .venv/bin/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/
          .venv/bin/pip install -v ./wheels/cuda_pathfinder*.whl --group ./cuda_pathfinder/pyproject.toml:test

      - name: Constrain builds to the local cuda-pathfinder wheel
        run: |
          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          mkdir -p wheel-constraints
          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt

      - name: Build and install cuda-bindings wheel
        run: |
          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt"
          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
          .venv/bin/pip wheel -v --no-deps ./cuda_bindings_13 -w ./wheels/
          .venv/bin/pip install -v ./wheels/cuda_bindings*.whl --group ./cuda_bindings_13/pyproject.toml:test

      - name: Constrain cuda-core to the local cuda-bindings wheel
        run: |
          CUDA_MAJOR="${CUDA_VER%%.*}"
          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
          bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test "${#bindings_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          test -f "${bindings_wheels[0]}"
          mkdir -p wheel-constraints
          pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
          bindings_uri="file://$(realpath "${bindings_wheels[0]}")"
          {
            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
          } | tee wheel-constraints/cuda-core.txt

      - name: Build and install cuda-core
        run: |
          export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt"
          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
          cd cuda_core
          ../.venv/bin/pip install -v . --group test

      - name: Install coverage tools
        run: |
          .venv/bin/pip install coverage pytest-cov Cython

      - name: Set cuda package install root
        run: |
          echo "INSTALL_ROOT=$(.venv/bin/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))')" >> $GITHUB_ENV
          echo "REPO_ROOT=$(pwd)" >> $GITHUB_ENV

      - name: Run cuda.pathfinder tests
        continue-on-error: true
        run: |
          cd $INSTALL_ROOT
          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_pathfinder/tests

      - name: Run cuda.bindings tests
        continue-on-error: true
        run: |
          cd $INSTALL_ROOT
          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_bindings_13/tests

      - name: Run cuda.core tests
        continue-on-error: true
        run: |
          cd $INSTALL_ROOT
          $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_core/tests

      - name: Copy Linux coverage file to workspace
        run: |
          cp $INSTALL_ROOT/.coverage $REPO_ROOT/.coverage.linux
          echo "Copied .coverage.linux to $REPO_ROOT"
          ls -lh $REPO_ROOT/.coverage.linux

      - name: Upload Linux coverage data
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: coverage-data-linux
          path: .coverage.linux
          retention-days: 7
          include-hidden-files: true
          if-no-files-found: error

      - name: Upload cuda source code for coverage mapping
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: cuda-source-linux
          path: ${{ env.INSTALL_ROOT }}/cuda/
          retention-days: 7
          if-no-files-found: error

  # Build Windows wheels on GitHub-hosted runner (has VS, no GPU)
  build-wheel-windows:
    name: Build Wheels (Windows)
    needs: [coverage-vars]
    runs-on: windows-2022
    permissions:
      contents: read
    defaults:
      run:
        shell: bash --noprofile --norc -xeuo pipefail {0}
    env:
      HOST_PLATFORM: "win-64"
      CUDA_PYTHON_COVERAGE: "1"
      CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }}
    steps:
      - name: Harden OS certificate trust (eMudhra/emSign CA stopgap)
        uses: $/.github/actions/upgrade-ca-certificates

      - name: Enable Git symlinks (Windows, must precede checkout)
        run: git config --global core.symlinks true

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          # Treeless clone: setuptools-scm needs the commit graph (`git describe`)
          # but not historical blobs.
          fetch-depth: 0
          filter: blob:none
          # Fetch CCCL separately: checkout's filter does not apply to submodules.
          submodules: false

      - name: Checkout CCCL submodule
        run: |
          git submodule sync --recursive
          git submodule update --init --force --depth=1 --filter=blob:none --recursive

      - name: Set up Python
        uses: $/.github/actions/setup-hardened-python
        with:
          python-version: ${{ env.PY_VER }}
          venv: .venv
          stack-reserve: 8388608

      - name: Set up MSVC
        uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4  # v1.13.1

      - name: Set up mini CTK
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: win-64
          cuda-version: ${{ env.CUDA_VER }}

      - name: Build cuda.pathfinder wheel
        run: |
          .venv/Scripts/python -m pip install wheel setuptools Cython
          .venv/Scripts/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/

      - name: Constrain builds to the local cuda.pathfinder wheel
        run: |
          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          mkdir -p wheel-constraints
          pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt

      - name: Build cuda.bindings wheel
        run: |
          export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")"
          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
          cd cuda_bindings_13
          ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/

      - name: Constrain cuda.core to the local cuda.bindings wheel
        run: |
          CUDA_MAJOR="${CUDA_VER%%.*}"
          pathfinder_wheels=(wheels/cuda_pathfinder-*.whl)
          bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test "${#bindings_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          test -f "${bindings_wheels[0]}"
          mkdir -p wheel-constraints
          pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
          bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
          {
            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
          } | tee wheel-constraints/cuda-core.txt

      - name: Build cuda.core wheel
        run: |
          export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")"
          export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}"
          cd cuda_core
          ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/

      # Vendor the DLLs these wheels were built against, the way cibuildwheel
      # does for every other Windows build.  --namespace-pkg is needed because
      # `cuda` is a namespace package.
      - name: Repair the Windows wheels
        run: |
          .venv/Scripts/pip install delvewheel
          mkdir -p wheels-repaired
          for whl in ./wheels/cuda_bindings-*.whl ./wheels/cuda_core-*.whl; do
            .venv/Scripts/delvewheel repair --namespace-pkg cuda \
              --exclude "torch_cpu.dll;torch_python.dll" \
              -w ./wheels-repaired "$whl"
          done
          mv -f ./wheels-repaired/*.whl ./wheels/

      - name: List wheel artifacts
        run: |
          echo "=== Windows wheel artifacts ==="
          ls -lahR ./wheels/

      - name: Upload Windows wheel artifacts
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: coverage-windows-wheels
          path: ./wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  # Run coverage tests on self-hosted GPU runner (no VS needed, installs pre-built wheels)
  coverage-windows:
    name: Coverage (Windows)
    needs: [coverage-vars, build-wheel-windows]
    runs-on: "windows-amd64-gpu-a100-latest-1"
    permissions:
      id-token: write
      contents: write
    env:
      HOST_PLATFORM: "win-64"
      ARCH: "amd64"
      CUDA_PYTHON_COVERAGE: "1"
      CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }}
    defaults:
      run:
        shell: bash --noprofile --norc -xeuo pipefail {0}
    steps:
      - name: Enable Git symlinks (Windows, must precede checkout)
        run: git config --global core.symlinks true

      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          fetch-depth: 1

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true

      # DRIVER above is 'latest' so install_gpu_driver.ps1 is intentionally
      # skipped (it errors on latest/earliest); configure_driver_mode.ps1
      # still runs to put the pre-installed driver into TCC mode.
      - name: Configure driver mode
        shell: powershell
        env:
          DRIVER_MODE: "TCC"
        run: |
          ci/tools/configure_driver_mode.ps1

      - name: Ensure GPU is working
        run: |
          nvidia-smi

      - name: Set up Python
        uses: $/.github/actions/setup-hardened-python
        with:
          python-version: ${{ env.PY_VER }}
          venv: .venv
          stack-reserve: 8388608

      - name: Set up mini CTK
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: win-64
          cuda-version: ${{ env.CUDA_VER }}

      - name: Download Windows wheel artifacts
        uses: $/.github/actions/download-artifact-retry
        with:
          name: coverage-windows-wheels
          path: ./wheels/

      - name: List downloaded wheel artifacts
        run: |
          echo "=== Downloaded wheel artifacts ==="
          ls -lahR ./wheels/

      - name: Install wheels from build job
        run: |
          .venv/Scripts/pip install ./wheels/cuda_pathfinder*.whl
          echo "Installed cuda.pathfinder"

          .venv/Scripts/pip install ./wheels/cuda_bindings*.whl
          echo "Installed cuda.bindings"

          .venv/Scripts/pip install ./wheels/cuda_core*.whl
          echo "Installed cuda.core"

      - name: Install test dependencies and coverage tools
        run: |
          .venv/Scripts/pip install coverage pytest-cov Cython
          .venv/Scripts/pip install --group ./cuda_pathfinder/pyproject.toml:test
          .venv/Scripts/pip install --group ./cuda_bindings_13/pyproject.toml:test
          .venv/Scripts/pip install --group ./cuda_core/pyproject.toml:test

      - name: Get install root
        id: install-root
        run: |
          INSTALL_ROOT=$(.venv/Scripts/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))')
          echo "INSTALL_ROOT=$INSTALL_ROOT" >> $GITHUB_OUTPUT
          echo "Install root: $INSTALL_ROOT"

      - name: Run cuda.pathfinder tests
        continue-on-error: true
        run: |
          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
          "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests"

      # The patched interpreter provides the larger stack on the main thread
      # and in child interpreters. Deferred cleanup needs pytest on the main thread.
      - name: Run cuda.bindings tests
        continue-on-error: true
        run: |
          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
          "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \
            -v --cov=./cuda --cov-append --cov-context=test \
            --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
            "$GITHUB_WORKSPACE/cuda_bindings_13/tests"

      - name: Run cuda.core tests
        continue-on-error: true
        run: |
          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
          "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \
            -v --cov=./cuda --cov-append --cov-context=test \
            --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
            "$GITHUB_WORKSPACE/cuda_core/tests"

      - name: Copy Windows coverage file to workspace
        run: |
          cp "${{ steps.install-root.outputs.INSTALL_ROOT }}/.coverage" "$GITHUB_WORKSPACE/.coverage.windows"
          echo "Copied .coverage.windows to $GITHUB_WORKSPACE"
          ls -lh "$GITHUB_WORKSPACE/.coverage.windows"

      - name: Upload Windows coverage data
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: coverage-data-windows
          path: .coverage.windows
          retention-days: 7
          include-hidden-files: true
          if-no-files-found: error

  combine-and-deploy:
    name: Combine Coverage
    needs: [coverage-linux, coverage-windows]
    runs-on: ubuntu-latest
    if: ${{ always() && github.repository_owner == 'nvidia' }}
    permissions:
      id-token: write
      contents: write
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Set up Python
        uses: ./.github/actions/setup-hardened-python
        with:
          python-version: ${{ env.PY_VER }}

      - name: Install coverage
        run: |
          # .coveragerc enables Cython.Coverage plugin; ensure it's available here too.
          pip install coverage[toml] Cython

      - name: Download Linux coverage data
        uses: $/.github/actions/download-artifact-retry
        with:
          name: coverage-data-linux
          path: ./

      - name: Download cuda source code
        uses: $/.github/actions/download-artifact-retry
        with:
          name: cuda-source-linux
          path: ./cuda/

      - name: Download Windows coverage data
        uses: $/.github/actions/download-artifact-retry
        with:
          name: coverage-data-windows
          path: ./

      - name: Combine coverage data
        run: |
          echo "=== Working directory ==="
          pwd
          echo ""
          echo "=== All files in root ==="
          ls -la
          echo ""

          # Verify both Linux and Windows coverage files exist
          echo "=== Checking coverage files ==="

          if [ ! -f ".coverage.linux" ]; then
            echo "[ERROR] .coverage.linux not found!"
            exit 1
          fi
          echo "[OK] Found .coverage.linux"

          if [ ! -f ".coverage.windows" ]; then
            echo "[ERROR] .coverage.windows not found!"
            exit 1
          fi
          echo "[OK] Found .coverage.windows"

          echo "=== Available coverage files ==="
          ls -lh .coverage.*

          echo "=== Cuda source structure ==="
          find ./cuda -type d || {
            echo "[ERROR] No cuda source directory!"
            exit 1
          }

          # Combine all .coverage.* files
          echo ""
          echo "=== Combining coverage data ==="
          coverage combine --rcfile=./.coveragerc --keep .coverage.*

      - name: Generate coverage reports
        run: |
          echo ""
          echo "=== Generating HTML, XML, and text reports ==="
          coverage html --rcfile=./.coveragerc
          coverage xml --rcfile=./.coveragerc -o coverage.xml
          coverage report --rcfile=./.coveragerc

          # Prepare for upload
          mkdir -p ./docs/coverage
          mv htmlcov/* ./docs/coverage/
          mv coverage.xml ./docs/coverage/

          echo ""
          echo "[SUCCESS] Coverage reports generated successfully"

      - name: Archive combined coverage results
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: coverage-combined
          path: docs/coverage/
          retention-days: 7
          include-hidden-files: true

  deploy:
    name: Deploy Coverage
    # GitHub discards prerequisite artifacts on a new run attempt. If only this
    # job fails, use "Re-run all jobs" so coverage recreates its payload.
    needs: combine-and-deploy
    if: ${{ needs.combine-and-deploy.result == 'success' && github.repository_owner == 'nvidia' }}
    runs-on: ubuntu-latest
    permissions:
      contents: write
    # Must match every job that writes to gh-pages.
    concurrency:
      group: cuda-python-gh-pages-publish
      queue: max
      cancel-in-progress: false
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Download combined coverage results
        uses: $/.github/actions/download-artifact-retry
        with:
          name: coverage-combined
          path: docs/coverage/

      - name: Deploy to gh-pages
        uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445  # v4.9.0
        with:
          git-config-name: cuda-python-bot
          git-config-email: cuda-python-bot@users.noreply.github.com
          folder: docs/
          target-folder: docs/
          commit-message: "Deploy combined coverage (Linux + Windows): ${{ github.sha }}"
          clean: false
          # Rebase onto concurrent docs deployments instead of force-pushing,
          # which the gh-pages ruleset rejects.
          force: false