ci: build and test local CUDA 12 bindings

.github/workflows/build-wheel.yml

@@ -83,14 +83,14 @@ jobs:
         run: echo "::add-matcher::.github/matchers/compiler.json"
 
       - name: Install rapidsai/sccache
-        if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }}
+        if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true') }}
         uses: ./.github/actions/install-sccache
 
       # The runner only exposes these values to actions, not to `run:` steps or
       # the cibuildwheel container, so re-export them for sccache's GHA backend.
       # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867
       - name: Adding addtional GHA cache-related env vars
-        if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }}
+        if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' }}
         uses: actions/github-script@v9
         with:
           script: |
@@ -144,6 +144,9 @@ jobs:
           $env:Path += ";$((Get-Location).Path)\\$env:YQ_DIR"
           yq --version
 
+      - name: Install CI tool dependencies
+        run: python -m pip install -e ./ci
+
       - name: Set environment variables
         env:
           CUDA_VER: ${{ inputs.cuda-version }}
@@ -152,6 +155,12 @@ jobs:
           SHA: ${{ github.sha }}
         run: ./ci/tools/env-vars build
 
+      - name: Prepare local maintenance SCM milestone
+        if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }}
+        run: |
+          python -m ci.tools.bindings_scm_version \
+            --package-root "${MAINTENANCE_BINDINGS_ROOT}" --sha "${{ github.sha }}"
+
       - name: Dump environment
         run: |
           env
@@ -195,7 +204,7 @@ jobs:
           twine check --strict cuda_pathfinder/*.whl
 
       - name: Constrain builds to the local cuda.pathfinder wheel
-        if: ${{ env.BUILD_BINDINGS == 'true' }}
+        if: ${{ env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true' }}
         run: |
           pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
           test "${#pathfinder_wheels[@]}" -eq 1
@@ -232,7 +241,7 @@ jobs:
           output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
           extras: uv
           python-path: ${{ steps.host-python.outputs.python-path }}
-        env:
+        env: &bindings-wheel-env
           CIBW_BUILD: ${{ env.CIBW_BUILD }}
           # build[uv]: same PEP 517 orchestration, with uv instead of pip.
           # extras: uv puts uv on PATH (required on Windows).
@@ -525,7 +534,7 @@ jobs:
 
       # Note: This overwrites CUDA_PATH etc
       - name: Set up mini CTK
-        if: ${{ !inputs.single-cuda-major && (env.BUILD_CORE == 'true' || env.TEST_CORE == 'true') }}
+        if: ${{ !inputs.single-cuda-major && (env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' || env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }}
         uses: ./.github/actions/fetch_ctk
         continue-on-error: false
         with:
@@ -550,6 +559,47 @@ jobs:
             ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.lib
           if-no-files-found: error
 
+      - name: Build maintenance cuda.bindings wheel
+        if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }}
+        uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3  # v4.3.4
+        with:
+          package-dir: ./${{ env.MAINTENANCE_BINDINGS_ROOT }}/
+          output-dir: ${{ env.MAINTENANCE_BINDINGS_ARTIFACTS_DIR }}
+          extras: uv
+          python-path: ${{ steps.host-python.outputs.python-path }}
+        env: *bindings-wheel-env
+
+      - name: Check maintenance cuda.bindings wheel
+        if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }}
+        run: twine check --strict "${MAINTENANCE_BINDINGS_ARTIFACTS_DIR}"/*.whl
+
+      - name: Upload maintenance cuda.bindings wheel
+        if: ${{ !inputs.single-cuda-major && (env.BUILD_BINDINGS == 'true' || env.TEST_BINDINGS == 'true') }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: ${{ env.MAINTENANCE_BINDINGS_ARTIFACT_NAME }}
+          path: ${{ env.MAINTENANCE_BINDINGS_ARTIFACTS_DIR }}/*.whl
+          if-no-files-found: error
+
+      - name: Build maintenance cuda.bindings Cython tests
+        if: ${{ !inputs.single-cuda-major && env.TEST_BINDINGS == 'true' }}
+        run: |
+          pip uninstall -y cuda-bindings
+          bindings_wheels=("${MAINTENANCE_BINDINGS_ARTIFACTS_DIR}"/*.whl)
+          test "${#bindings_wheels[@]}" -eq 1
+          python -m pip install "${bindings_wheels[0]}" --group "${MAINTENANCE_BINDINGS_ROOT}/pyproject.toml:test"
+          pushd "${MAINTENANCE_BINDINGS_CYTHON_TESTS_DIR}"
+          bash build_tests.sh
+          popd
+
+      - name: Upload maintenance cuda.bindings Cython tests
+        if: ${{ !inputs.single-cuda-major && env.TEST_BINDINGS == 'true' }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: ${{ env.MAINTENANCE_BINDINGS_ARTIFACT_NAME }}-tests
+          path: ${{ env.MAINTENANCE_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }}
+          if-no-files-found: error
+
       - name: Look up prior-branch cuda.bindings run ID
         id: prior-bindings
         if: ${{ !inputs.single-cuda-major && env.BUILD_CORE == 'true' }}

.github/workflows/ci.yml

@@ -36,14 +36,23 @@ jobs:
       WINDOWS_ARM64_SUPPORTED: ${{ steps.get-vars.outputs.windows_arm64_supported }}
       WINDOWS_ARM64_SINGLE_CUDA_MAJOR: ${{ steps.get-vars.outputs.windows_arm64_single_cuda_major }}
       PYTHON_SUPPORTED: ${{ steps.get-vars.outputs.python_supported }}
+      BINDINGS_CONFIG: ${{ steps.get-vars.outputs.bindings_config }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 1
+      - name: Set up Python for CI tools
+        uses: ./.github/actions/setup-hardened-python
+        with:
+          python-version: "3.11"
+      - name: Install CI tool dependencies
+        run: python -m pip install -e ./ci
       - name: Get CUDA build versions
         id: get-vars
         run: |
+          bindings_config=$(python -m ci.tools.bindings_config)
+          echo "bindings_config=$bindings_config" >> "$GITHUB_OUTPUT"
           cuda_build_ver=$(yq '.cuda.build.version' ci/versions.yml)
           echo "cuda_build_ver=$cuda_build_ver" >> $GITHUB_OUTPUT
 
@@ -642,6 +651,86 @@ jobs:
       nruns: ${{ (github.event_name == 'schedule' && 5) || 1}}
       workplan: ${{ needs.detect-changes.outputs.workplan }}
 
+  test-maintenance-bindings-linux-64:
+    name: Maintenance bindings (linux-64)
+    if: ${{ github.repository_owner == 'nvidia' &&
+            !fromJSON(needs.should-skip.outputs.skip) &&
+            !fromJSON(needs.should-skip.outputs.doc-only) &&
+            fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test &&
+            fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }}
+    permissions:
+      actions: read
+      contents: read
+    needs:
+      - ci-vars
+      - should-skip
+      - detect-changes
+      - build-linux-64
+    secrets: inherit
+    uses: ./.github/workflows/test-wheel-linux.yml
+    with:
+      build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }}
+      host-platform: linux-64
+      build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }}
+      bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }}
+      maintenance-bindings: true
+      nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }}
+      workplan: ${{ needs.detect-changes.outputs.workplan }}
+
+  test-maintenance-bindings-linux-aarch64:
+    name: Maintenance bindings (linux-aarch64)
+    if: ${{ github.repository_owner == 'nvidia' &&
+            !fromJSON(needs.should-skip.outputs.skip) &&
+            !fromJSON(needs.should-skip.outputs.doc-only) &&
+            fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test &&
+            fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.linux }}
+    permissions:
+      actions: read
+      contents: read
+    needs:
+      - ci-vars
+      - should-skip
+      - detect-changes
+      - build-linux-64
+      - build-linux-aarch64
+    secrets: inherit
+    uses: ./.github/workflows/test-wheel-linux.yml
+    with:
+      build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }}
+      host-platform: linux-aarch64
+      build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }}
+      bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }}
+      maintenance-bindings: true
+      nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }}
+      workplan: ${{ needs.detect-changes.outputs.workplan }}
+
+  test-maintenance-bindings-windows:
+    name: Maintenance bindings (win-64)
+    if: ${{ github.repository_owner == 'nvidia' &&
+            !fromJSON(needs.should-skip.outputs.skip) &&
+            !fromJSON(needs.should-skip.outputs.doc-only) &&
+            fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test &&
+            fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows }}
+    permissions:
+      actions: read
+      contents: read
+    needs:
+      - ci-vars
+      - should-skip
+      - detect-changes
+      - build-linux-64
+      - build-windows
+    secrets: inherit
+    uses: ./.github/workflows/test-wheel-windows.yml
+    with:
+      build-type: ${{ startsWith(github.ref_name, 'pull-request/') && 'pull-request' || (github.event_name == 'schedule' && 'nightly' || 'branch') }}
+      host-platform: win-64
+      build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }}
+      bindings-config: ${{ needs.ci-vars.outputs.BINDINGS_CONFIG }}
+      maintenance-bindings: true
+      nruns: ${{ (github.event_name == 'schedule' && 5) || 1 }}
+      workplan: ${{ needs.detect-changes.outputs.workplan }}
+
   doc:
     name: Docs
     if: ${{ github.repository_owner == 'nvidia' }}
@@ -676,6 +765,9 @@ jobs:
       - test-linux-64
       - test-linux-aarch64
       - test-windows
+      - test-maintenance-bindings-linux-64
+      - test-maintenance-bindings-linux-aarch64
+      - test-maintenance-bindings-windows
       - api-check-core-vs-release
       - api-check-core-vs-base
       - doc
@@ -703,6 +795,7 @@ jobs:
           windows_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platforms.windows || false }}"
           windows_arm64_supported="${{ needs.ci-vars.outputs.WINDOWS_ARM64_SUPPORTED }}"
           build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}"
+          maintenance_bindings_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).modules.bindings.needs_test || false }}"
           run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}"
           is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}"
           status="success"
@@ -753,6 +846,16 @@ jobs:
           check_result "test-linux-aarch64" "$linux_expected"
           check_result "test-windows"       "$windows_expected"
 
+          maintenance_linux_expected="skipped"
+          maintenance_windows_expected="skipped"
+          if [[ "$maintenance_bindings_selected" == "true" ]]; then
+            maintenance_linux_expected="$linux_expected"
+            maintenance_windows_expected="$windows_expected"
+          fi
+          check_result "test-maintenance-bindings-linux-64" "$maintenance_linux_expected"
+          check_result "test-maintenance-bindings-linux-aarch64" "$maintenance_linux_expected"
+          check_result "test-maintenance-bindings-windows" "$maintenance_windows_expected"
+
           # API compatibility checks run for cuda_core source changes and for
           # conservative full runs when reusable base artifacts are unavailable.
           expected="skipped"

.github/workflows/test-wheel-linux.yml

@@ -16,6 +16,14 @@ on:
       build-ctk-ver:
         type: string
         required: true
+      bindings-config:
+        description: Normalized bindings registry for maintenance-only tests.
+        type: string
+        default: ''
+      maintenance-bindings:
+        description: Test this run's maintenance bindings wheel and Cython extensions.
+        type: boolean
+        default: false
       matrix_filter:
         type: string
         default: "."
@@ -55,6 +63,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       BUILD_TYPE: ${{ inputs.build-type }}
+      BINDINGS_CONFIG: ${{ inputs.bindings-config }}
+      BINDINGS_ONLY: ${{ inputs.maintenance-bindings }}
       ARCH: ${{ (inputs.host-platform == 'linux-64' && 'amd64') ||
                 (inputs.host-platform == 'linux-aarch64' && 'arm64') }}
     outputs:
@@ -83,6 +93,14 @@ jobs:
           # Read base matrix from YAML file for the specific architecture
           TEST_MATRIX=$(yq -o json ".linux[\"${MATRIX_TYPE}\"] | map(select(.ARCH == \"${ARCH}\"))" ci/test-matrix.yml)
 
+          if [[ "$BINDINGS_ONLY" == "true" ]]; then
+            TEST_MATRIX=$(jq -c --argjson config "$BINDINGS_CONFIG" '
+              ($config.package_roots[] | select(.release_status == "maintenance")) as $package
+              | map(select(.CUDA_VER == $package.toolkit_version)
+                | . + {BINDINGS_ROOT: $package.package_root})
+            ' <<< "$TEST_MATRIX")
+          fi
+
           # Apply matrix filter; reject custom DRIVER + FLAVOR=wsl (the
           # in-container driver swap doesn't work under WSL); add a
           # RUNNER_DRIVER field that maps any custom version back to
@@ -99,10 +117,10 @@ jobs:
 
   test:
     env:
-      TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }}
-      TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }}
-      TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }}
-      TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }}
+      TEST_PATHFINDER: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test) }}
+      TEST_BINDINGS: ${{ inputs.maintenance-bindings || inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }}
+      TEST_CORE: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test) }}
+      TEST_PYTHON: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test) }}
     name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }}
     timeout-minutes: 60
     needs: compute-matrix
@@ -168,7 +186,9 @@ jobs:
 
       - name: Set environment variables
         env:
-          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          BUILD_CUDA_VER: ${{ inputs.maintenance-bindings && matrix.CUDA_VER || inputs.build-ctk-ver }}
+          CUDA_BINDINGS_ROOT: ${{ matrix.BINDINGS_ROOT || 'cuda_bindings_13' }}
+          BINDINGS_ARTIFACT_PREFIX: ${{ inputs.maintenance-bindings && 'maintenance-bindings' || 'cuda-bindings' }}
           CUDA_VER: ${{ matrix.CUDA_VER }}
           HOST_PLATFORM: ${{ inputs.host-platform }}
           LOCAL_CTK: ${{ matrix.LOCAL_CTK }}

.github/workflows/test-wheel-windows.yml

@@ -16,6 +16,14 @@ on:
       build-ctk-ver:
         type: string
         required: true
+      bindings-config:
+        description: Normalized bindings registry for maintenance-only tests.
+        type: string
+        default: ''
+      maintenance-bindings:
+        description: Test this run's maintenance bindings wheel and Cython extensions.
+        type: boolean
+        default: false
       matrix_filter:
         type: string
         default: "."
@@ -54,6 +62,8 @@ jobs:
         shell: bash --noprofile --norc -xeuo pipefail {0}
     env:
       BUILD_TYPE: ${{ inputs.build-type }}
+      BINDINGS_CONFIG: ${{ inputs.bindings-config }}
+      BINDINGS_ONLY: ${{ inputs.maintenance-bindings }}
       ARCH: ${{ (inputs.host-platform == 'win-64' && 'amd64') }}
     outputs:
       MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
@@ -79,6 +89,14 @@ jobs:
           # Read base matrix from YAML file for the specific architecture
           TEST_MATRIX=$(yq -o json ".windows[\"${MATRIX_TYPE}\"] | map(select(.ARCH == \"${ARCH}\"))" ci/test-matrix.yml)
 
+          if [[ "$BINDINGS_ONLY" == "true" ]]; then
+            TEST_MATRIX=$(jq -c --argjson config "$BINDINGS_CONFIG" '
+              ($config.package_roots[] | select(.release_status == "maintenance")) as $package
+              | map(select(.CUDA_VER == $package.toolkit_version)
+                | . + {BINDINGS_ROOT: $package.package_root})
+            ' <<< "$TEST_MATRIX")
+          fi
+
           # Apply matrix filter; add a RUNNER_DRIVER field that maps any
           # custom DRIVER version back to 'latest' (install_gpu_driver.ps1
           # swaps the driver itself, so the runner must be the one that
@@ -89,10 +107,10 @@ jobs:
 
   test:
     env:
-      TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }}
-      TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }}
-      TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }}
-      TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }}
+      TEST_PATHFINDER: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test) }}
+      TEST_BINDINGS: ${{ inputs.maintenance-bindings || inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }}
+      TEST_CORE: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test) }}
+      TEST_PYTHON: ${{ !inputs.maintenance-bindings && (inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test) }}
     name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }}
     timeout-minutes: 60
     # The build stage could fail but we want the CI to keep moving.
@@ -157,7 +175,9 @@ jobs:
 
       - name: Set environment variables
         env:
-          BUILD_CUDA_VER: ${{ inputs.build-ctk-ver }}
+          BUILD_CUDA_VER: ${{ inputs.maintenance-bindings && matrix.CUDA_VER || inputs.build-ctk-ver }}
+          CUDA_BINDINGS_ROOT: ${{ matrix.BINDINGS_ROOT || 'cuda_bindings_13' }}
+          BINDINGS_ARTIFACT_PREFIX: ${{ inputs.maintenance-bindings && 'maintenance-bindings' || 'cuda-bindings' }}
           CUDA_VER: ${{ matrix.CUDA_VER }}
           HOST_PLATFORM: ${{ inputs.host-platform }}
           LOCAL_CTK: ${{ matrix.LOCAL_CTK }}