test-wheel-linux.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

name: "CI: Test wheels"

on:
  workflow_call:
    inputs:
      build-type:
        type: string
        required: true
      host-platform:
        type: string
        required: true
      build-ctk-ver:
        type: string
        required: true
      matrix_filter:
        type: string
        default: "."
      nruns:
        type: number
        default: 1
      workplan:
        description: JSON workplan. An empty value tests everything.
        type: string
        default: ""
      run-id:
        description: >
          Workflow run ID to download artifacts from.
          Defaults to the current run when empty.
        type: string
        default: ''
      test-mode:
        description: >
          Test mode: 'standard' (default), 'nightly-pytorch',
          'nightly-numba-cuda', 'nightly-numba-cuda-mlir', or
          'nightly-cuda-core'.
        type: string
        default: 'standard'
      sha:
        description: >
          Commit SHA used to construct artifact names.
          Defaults to github.sha (current run) when empty.
        type: string
        default: ''

defaults:
  run:
    shell: bash --noprofile --norc -xeuo pipefail {0}

jobs:
  compute-matrix:
    runs-on: ubuntu-latest
    env:
      BUILD_TYPE: ${{ inputs.build-type }}
      ARCH: ${{ (inputs.host-platform == 'linux-64' && 'amd64') ||
                (inputs.host-platform == 'linux-aarch64' && 'arm64') }}
    outputs:
      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
      OLD_BRANCH: ${{ steps.compute-matrix.outputs.OLD_BRANCH }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Validate Test Type
        run: |
          if [[ "$BUILD_TYPE" != "pull-request" ]] && [[ "$BUILD_TYPE" != "nightly" ]] && [[ "$BUILD_TYPE" != "branch" ]]; then
              echo "Invalid build type! Must be one of 'nightly', 'pull-request', or 'branch'."
              exit 1
          fi

      - name: Compute Python Test Matrix
        id: compute-matrix
        run: |
          # Use the nightly matrix for branch tests
          MATRIX_TYPE="${BUILD_TYPE}"
          if [[ "${MATRIX_TYPE}" == "branch" ]]; then
            MATRIX_TYPE="nightly"
          fi

          # Read base matrix from YAML file for the specific architecture
          TEST_MATRIX=$(yq -o json ".linux[\"${MATRIX_TYPE}\"] | map(select(.ARCH == \"${ARCH}\"))" ci/test-matrix.yml)

          # Apply matrix filter; reject custom DRIVER + FLAVOR=wsl (the
          # in-container driver swap doesn't work under WSL); add a
          # RUNNER_DRIVER field that maps any custom version back to
          # 'latest' (the install script swaps the driver itself, so we
          # need to land on the runner that ships with the most recent
          # pre-installed driver); wrap in include structure.
          MATRIX=$(echo "$TEST_MATRIX" | jq -c '${{ inputs.matrix_filter }} | if any(.[]; .DRIVER != "latest" and .DRIVER != "earliest" and .FLAVOR == "wsl") then "Error: custom DRIVER is not supported with FLAVOR=wsl\n" | halt_error(1) else . end | map(. + {RUNNER_DRIVER: (if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)}) | if (. | length) > 0 then {include: .} else "Error: Empty matrix\n" | halt_error(1) end')

          echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}"

          # This job has yq already installed, so let's do it here
          OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml)
          echo "OLD_BRANCH=${OLD_BRANCH}" >> "$GITHUB_OUTPUT"

  test:
    env:
      TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }}
      TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }}
      TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }}
      TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }}
    name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }}
    timeout-minutes: 60
    needs: compute-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
    runs-on: "${{ matrix.FLAVOR || 'linux' }}-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.RUNNER_DRIVER }}-${{ matrix.GPU_COUNT }}"
    # The build stage could fail but we want the CI to keep moving.
    if: ${{ github.repository_owner == 'nvidia' && !cancelled() }}
    # Our self-hosted runners require a container
    # TODO: use a different (nvidia?) container
    container:
      # Custom-DRIVER rows need --privileged --pid=host so install_gpu_driver.sh
      # can nsenter to the host for the install + refresh the toolkit bind mounts
      # back inside the container. Stock options for latest/earliest rows.
      options: ${{ ((matrix.DRIVER == 'latest' || matrix.DRIVER == 'earliest') && '-u root --security-opt seccomp=unconfined --shm-size 16g') || '-u root --security-opt seccomp=unconfined --shm-size 16g --privileged --pid=host' }}
      image: ubuntu:24.04
      env:
        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
        PIP_CACHE_DIR: "/tmp/pip-cache"
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          # Test sources must match the wheels under test (nightly runs
          # reuse wheels built from an older commit of main).
          ref: ${{ inputs.sha || github.sha }}

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true

      - name: Install dependencies
        uses: ./.github/actions/install_unix_deps
        continue-on-error: false
        with:
          # for artifact fetching, graphics libs, g++ required for cffi in
          # example; util-linux for `nsenter` (custom-DRIVER rows re-exec
          # install_gpu_driver.sh onto the host through nsenter)
          # yq: ci/tools/env-vars reads ci/versions.yml with it
          dependencies: "jq wget libgl1 libegl1 g++ util-linux yq"
          dependent_exes: "jq wget yq"

      - name: Checkout driver install scripts (workflow revision)
        if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ github.sha }}
          path: .driver-tools
          sparse-checkout: ci/tools
          persist-credentials: false

      - name: Install GPU driver
        if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
        env:
          DRIVER: ${{ matrix.DRIVER }}
          DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
          GPU_TYPE: ${{ matrix.GPU }}
        run: ./.driver-tools/ci/tools/install_gpu_driver.sh

      - name: Ensure GPU is working
        run: nvidia-smi

      - name: Set environment variables
        env:
          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
          CUDA_VER: ${{ matrix.CUDA_VER }}
          HOST_PLATFORM: ${{ inputs.host-platform }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
          PY_VER: ${{ matrix.PY_VER }}
          SHA: ${{ inputs.sha || github.sha }}
          SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }}
        run: ./ci/tools/env-vars test

      - name: Apply extra matrix environment variables
        if: ${{ matrix.ENV }}
        env:
          MATRIX_ENV: ${{ toJSON(matrix.ENV) }}
        run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV"

      - name: Download cuda-pathfinder build artifacts
        if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: cuda-pathfinder-wheel
          path: ./cuda_pathfinder
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Download cuda-python build artifacts
        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: cuda-python-wheel
          path: .
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Download cuda.bindings build artifacts
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                env.BINDINGS_SOURCE == 'main' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Look up prior-branch artifacts run ID
        id: prior
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                env.BINDINGS_SOURCE == 'backport' }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
          # gh is needed for lookup-run-id, which calls gh api.
          mkdir -p -m 755 /etc/apt/keyrings \
                && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
          && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
          && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
          && apt update \
          && apt install gh -y

          OLD_BRANCH=${{ needs.compute-matrix.outputs.OLD_BRANCH }}
          OLD_ARTIFACT_PATTERN="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}-*[0-9a-f]"
          LOOKUP_ARGS=(
            --branch "${OLD_BRANCH}"
            --artifact "${OLD_ARTIFACT_PATTERN}"
          )
          if ${{ env.TEST_PYTHON == 'true' }}; then
            LOOKUP_ARGS+=(--artifact cuda-python-wheel)
          fi
          LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id \
            "${LOOKUP_ARGS[@]}" NVIDIA/cuda-python "CI")
          echo "run-id=${LATEST_PRIOR_RUN_ID}" >> "${GITHUB_OUTPUT}"
          echo "artifact-pattern=${OLD_ARTIFACT_PATTERN}" >> "${GITHUB_OUTPUT}"

      - name: Download prior-branch cuda.bindings build artifacts
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                env.BINDINGS_SOURCE == 'backport' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          pattern: ${{ steps.prior.outputs.artifact-pattern }}
          merge-multiple: "true"
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
          run-id: ${{ steps.prior.outputs.run-id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Download prior-branch cuda-python wheel
        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'backport' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: cuda-python-wheel
          path: .
          run-id: ${{ steps.prior.outputs.run-id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Verify prior-branch artifact layout
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                env.BINDINGS_SOURCE == 'backport' }}
        run: |
          bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl)
          if [[ "${#bindings_wheels[@]}" -ne 1 ]] || [[ ! -f "${bindings_wheels[0]}" ]]; then
            echo "::error::Expected exactly one cuda_bindings wheel in ${CUDA_BINDINGS_ARTIFACTS_DIR}; found: ${bindings_wheels[*]}"
            exit 1
          fi
          ls -al "${CUDA_BINDINGS_ARTIFACTS_DIR}"

      - name: Display structure of downloaded cuda-python artifacts
        if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'floor' }}
        run: |
          pwd
          ls -lah cuda_python*.whl cuda_pathfinder/

      - name: Display structure of downloaded cuda.bindings artifacts
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') &&
                env.BINDINGS_SOURCE != 'floor' }}
        run: |
          pwd
          ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR

      - name: Download cuda.bindings Cython tests
        if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Display structure of downloaded cuda.bindings Cython tests
        if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }}
        run: |
          pwd
          ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR

      - name: Download cuda.core build artifacts
        if: ${{ env.TEST_CORE == 'true' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Display structure of downloaded cuda.core build artifacts
        if: ${{ env.TEST_CORE == 'true' }}
        run: |
          pwd
          ls -lahR $CUDA_CORE_ARTIFACTS_DIR

      - name: Download cuda.core Cython tests
        if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Display structure of downloaded cuda.core Cython tests
        if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }}
        run: |
          pwd
          ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR

      - name: Download cuda.core test binaries
        if: ${{ env.TEST_CORE == 'true' }}
        uses: $/.github/actions/download-artifact-retry
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries
          path: ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}
          run-id: ${{ inputs.run-id || github.run_id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Display structure of downloaded cuda.core test binaries
        if: ${{ env.TEST_CORE == 'true' }}
        run: |
          pwd
          ls -lahR $CUDA_CORE_TEST_BINARIES_DIR

      - name: Set up Python ${{ matrix.PY_VER }}
        # $/ resolves at the workflow's commit. ./ would resolve against the
        # inputs.sha checkout, which can predate this action.
        uses: $/.github/actions/setup-hardened-python
        with:
          python-version: ${{ matrix.PY_VER }}

      - name: Set up mini CTK
        if: ${{ matrix.LOCAL_CTK == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ matrix.CUDA_VER }}

      - name: Set up latest cuda_sanitizer_api
        if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && env.SETUP_SANITIZER == '1' }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ env.LATEST_CUDA_VERSION }}
          cuda-components: "cuda_sanitizer_api"

      - name: Set up compute-sanitizer
        if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}
        run: setup-sanitizer

      - name: Set up test repetition on nightly runs
        run: echo "PYTEST_ADDOPTS=\"--count=${{ inputs.nruns }}\"" >> "$GITHUB_ENV"

      - name: Register problem matchers
        # compute-sanitizer findings in the test output become annotations
        # on the job page. Sanitizer runs are Linux-only.
        run: echo "::add-matcher::.github/matchers/compute-sanitizer.json"

      # ── Standard test steps (skipped for nightly modes) ──
      - name: Run cuda.pathfinder tests with see_what_works
        if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }}
        env:
          CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works
          CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works
          CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: see_what_works
        run: run-tests pathfinder

      - name: Run cuda.bindings tests
        if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests bindings

      - name: Run cuda.bindings benchmarks (smoke test)
        if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }}
        run: |
          pip install pyperf
          pushd benchmarks/cuda_bindings
          python run_pyperf.py --debug-single-value
          popd

      - name: Run cuda.core tests
        if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests core

      - name: Ensure cuda-python installable
        if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }}
        run: |
          # Package suites install their own dependencies. A metapackage-only
          # run has no preceding suite, so install the exact local internal
          # wheels in one transaction while resolving released dependencies
          # such as cuda-core from the package index.
          if ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}; then
            dependency_args=(--no-deps)
          else
            dependency_args=(
              ./cuda_pathfinder/cuda_pathfinder-*.whl
              "${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-*.whl
            )
          fi
          python_requirements=(cuda_python*.whl)
          if [[ "${{ matrix.LOCAL_CTK }}" != 1 ]]; then
            python_requirements=("${python_requirements[@]/%/[all]}")
          fi
          pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}"

      - name: Install cuda.pathfinder extra wheels for testing
        if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }}
        run: |
          set -euo pipefail
          pushd cuda_pathfinder
          pip install --only-binary=:all: -v ./*.whl --group "test-cu${TEST_CUDA_MAJOR}"
          pip list
          popd

      - name: Run cuda.pathfinder tests with all_must_work
        if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }}
        env:
          CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work
          CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work
          CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS: all_must_work
        run: run-tests pathfinder

      # ── Nightly: install wheels + optional dep together ──
      - name: Install cuda-python wheels + PyTorch
        if: ${{ inputs.test-mode == 'nightly-pytorch' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests nightly-pytorch

      - name: Install cuda-python wheels + numba-cuda
        if: ${{ inputs.test-mode == 'nightly-numba-cuda' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests nightly-numba-cuda

      - name: Install cuda-python wheels + numba-cuda-mlir
        if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests nightly-numba-cuda-mlir

      - name: Install main pathfinder/bindings + released cuda-core
        if: ${{ inputs.test-mode == 'nightly-cuda-core' }}
        env:
          CUDA_VER: ${{ matrix.CUDA_VER }}
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: run-tests nightly-cuda-core

      # ── Nightly: run tests ──
      - name: Run PyTorch interop tests
        if: ${{ inputs.test-mode == 'nightly-pytorch' }}
        run: |
          pushd cuda_core
          pytest -rsxXfE -v --exclude-warning-annotations --junitxml "${GITHUB_WORKSPACE}/test-results/junit-pytorch-interop.xml" tests/test_utils.py tests/example_tests/ | python "${GITHUB_WORKSPACE}/ci/tools/fold_pytest_sections.py"
          popd

      - name: Checkout numba-cuda tests at matching tag
        if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          repository: NVIDIA/numba-cuda
          ref: v${{ env.NUMBA_CUDA_VER }}
          path: numba-cuda-released

      - name: Run numba-cuda tests
        if: ${{ inputs.test-mode == 'nightly-numba-cuda' && env.NUMBA_CUDA_VER != '' }}
        env:
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: |
          if [[ "${LOCAL_CTK}" != 1 ]]; then
            # Let numba-cuda skip inputs that require toolkit executables.
            export NUMBA_CUDA_TEST_WHEEL_ONLY=1
          fi
          # Run from numba-cuda's testing/ dir so its pytest.ini kicks in
          # (consider_namespace_packages=true + --pyargs numba.cuda.tests).
          # -k "not TestIpc": numba-cuda v0.30.4 accesses CUipcMemHandle.reserved,
          # which cuda-bindings intentionally removed. numba-cuda is EOL upstream,
          # so no fix is coming. See cuda-python#2748. The affected tests all
          # live in the two TestIpcMemory/TestIpcStaged classes in test_ipc.py.
          pushd numba-cuda-released/testing
          pytest -v -k "not TestIpc"
          popd

      - name: Checkout numba-cuda-mlir tests at matching tag
        if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          repository: NVIDIA/numba-cuda-mlir
          ref: v${{ env.NUMBA_CUDA_MLIR_VER }}
          path: numba-cuda-mlir-released

      - name: Run numba-cuda-mlir tests
        if: ${{ inputs.test-mode == 'nightly-numba-cuda-mlir' && env.NUMBA_CUDA_MLIR_VER != '' }}
        env:
          LOCAL_CTK: ${{ matrix.LOCAL_CTK }}
        run: |
          if [[ "${LOCAL_CTK}" != 1 ]]; then
            # Let numba-cuda-mlir skip inputs that require toolkit executables.
            export NUMBA_CUDA_MLIR_TEST_WHEEL_ONLY=1
          fi
          pushd numba-cuda-mlir-released
          # Install this tag's test deps (pytest + plugins + ml-dtypes + ...).
          pip install --upgrade "pip>=25.1"
          pip install --group test
          # Skip tests/benchmarks/ and tests/doc_examples/ — they import the
          # numba package at collection time, which cuSIMT intentionally does
          # not depend on. See NVIDIA/numba-cuda-mlir#136.
          # test_device_record_copy (WithRecord variant): flaky NaN vs NaN
          # compare from uninitialized np.recarray memory. See
          # NVIDIA/numba-cuda-mlir#341.
          pytest -rsxXfE -v --exclude-warning-annotations \
            --junitxml "${GITHUB_WORKSPACE}/test-results/junit-numba-cuda-mlir.xml" \
            --ignore=tests/benchmarks \
            --ignore=tests/doc_examples \
            --deselect 'tests/numba_cuda_tests/cudadrv/test_cuda_devicerecord.py::TestCudaDeviceRecordWithRecord::test_device_record_copy' \
            tests/ | python "${GITHUB_WORKSPACE}/ci/tools/fold_pytest_sections.py"
          popd

      - name: Checkout released cuda-core tests at matching tag
        if: ${{ inputs.test-mode == 'nightly-cuda-core' && env.CUDA_CORE_RELEASED_VER != '' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: cuda-core-v${{ env.CUDA_CORE_RELEASED_VER }}
          path: cuda-core-released

      - name: Run released cuda-core tests
        if: ${{ inputs.test-mode == 'nightly-cuda-core' && env.CUDA_CORE_RELEASED_VER != '' }}
        run: |
          pushd cuda-core-released/cuda_core
          # Install the released tag's test group so we exercise the exact deps
          # that cuda-core version shipped with.
          pip install --upgrade "pip>=25.1"
          pip install --group "${CUDA_CORE_TEST_GROUP}"
          # Version-gated deselects: drop automatically when a newer
          # cuda-core release ships that removes each source.
          DESELECTS=()
          if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${CUDA_CORE_RELEASED_VER}') <= Version('1.0.1') else 1)"; then
            # NvlinkVersion: v1.0.1's wrapper mapping predates
            # NvlinkVersion.VERSION_6_0 which main cuda-bindings adds.
            # Expected drift on this mode until released cuda-core catches up.
            DESELECTS+=(
              --deselect 'tests/test_enum_coverage.py::test_wrapper_covers_all_binding_members[NvlinkVersion]'
            )
          fi
          if python -c "from packaging.version import Version; import sys; sys.exit(0 if Version('${CUDA_CORE_RELEASED_VER}') <= Version('1.2.1') else 1)"; then
            # test_frozen_driver_table_covers_all_curesult_members: v1.2.1's
            # frozen table (capped at CTK 13.1.1) predates 3 CUresult members
            # added by main cuda-bindings 13.4.1. Reverted on main by #2792;
            # drop this deselect once a cuda-core release with the revert ships.
            DESELECTS+=(
              --deselect 'tests/test_utils_enum_explanations_helpers.py::test_frozen_driver_table_covers_all_curesult_members'
            )
          fi
          pytest -rsxXfE -v --randomly-dont-reorganize \
            --junitxml "${GITHUB_WORKSPACE}/test-results/junit-cuda-core-released.xml" \
            "${DESELECTS[@]}" \
            tests/ | python "${GITHUB_WORKSPACE}/ci/tools/fold_pytest_sections.py"
          popd

      - name: Upload test results
        # JUnit XML written by run-tests, one file per suite: a machine-readable
        # record of the run, kept for 14 days. The name must be unique per job
        # in the run.
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: test-results-${{ inputs.test-mode }}-${{ inputs.host-platform }}-py${{ matrix.PY_VER }}-cu${{ matrix.CUDA_VER }}-${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}-${{ matrix.GPU }}-x${{ matrix.GPU_COUNT }}-${{ matrix.DRIVER }}${{ matrix.FLAVOR && format('-{0}', matrix.FLAVOR) || '' }}
          path: test-results/
          if-no-files-found: ignore
          retention-days: 14