CI: add pyyaml to ci-nightly tests; integrate #3342 stack-reserve fix

Finding 1 (ci-nightly.yml): The driver-hash test added by the eMudhra
backport imports `yaml`, which is not in stdlib. Add `pyyaml==6.0.3
pathspec==0.12.1` to the pip install in the CI-tools unit test step, matching
what `main` does. Without this, `ci-nightly` would fail at test collection
with ModuleNotFoundError.

Finding 2 (coverage.yml): Integrate #3342 ("Fix Windows coverage test stack
overflows") onto this branch. Add a `stack-reserve` input to
setup-hardened-python; both Windows coverage jobs (build wheel + test wheel)
request 8 MiB, letting the action patch the base interpreter with `pefile`
and verify the reserve at runtime. Switch the Windows pytest steps from
`run_pytest_with_stack.py` to direct `python -m pytest`, since the patched
interpreter now provides a larger stack on the main thread and in child
processes. The old MSVC `editbin` step is implicitly replaced by the action.

Identified in review by rwgk.

Author: juenglin

.github/actions/setup-hardened-python/action.yml

@@ -31,6 +31,11 @@ inputs:
   venv:
     description: Directory for the venv. Defaults to a new directory under RUNNER_TEMP.
     default: ""
+  stack-reserve:
+    description: >-
+      Windows only. Raise the base interpreter's main-thread stack reserve to at
+      least this many bytes. Empty (default) leaves the interpreter unchanged.
+    default: ""
 
 outputs:
   python-path:
@@ -148,3 +153,41 @@ runs:
           "${PYTHON}" -m pip install \
             "pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl#sha256=fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70"
         fi
+
+    - name: Raise the interpreter's stack reserve
+      if: runner.os == 'Windows' && inputs.stack-reserve != ''
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      env:
+        PYTHON: ${{ steps.venv.outputs.python-path }}
+        STACK_RESERVE: ${{ inputs.stack-reserve }}
+      run: |
+        # sys.stdout.write avoids a trailing CR from Windows Python's CRLF.
+        base="$("${PYTHON}" -c 'import sys; sys.stdout.write(sys._base_executable)')"
+        # Windows cannot overwrite a running executable. Prepare a copy and
+        # replace the original after the patching interpreter exits.
+        uv run --no-project --isolated --with pefile==2024.8.26 --python "${base}" python - "${base}" "${STACK_RESERVE}" <<'EOF'
+        import sys
+
+        import pefile
+
+        path, reserve = sys.argv[1], int(sys.argv[2])
+        if reserve <= 0:
+            raise ValueError("stack-reserve must be a positive number of bytes")
+        with pefile.PE(path) as pe:
+            pe.OPTIONAL_HEADER.SizeOfStackReserve = max(pe.OPTIONAL_HEADER.SizeOfStackReserve, reserve)
+            pe.write(path + ".stack-reserve")
+        EOF
+        mv -f -- "${base}.stack-reserve" "${base}"
+        # Verify the running interpreter's stack, rather than just its header.
+        "${base}" - "${STACK_RESERVE}" <<'EOF'
+        import ctypes
+        import sys
+
+        lo, hi = ctypes.c_size_t(), ctypes.c_size_t()
+        get_limits = ctypes.windll.kernel32.GetCurrentThreadStackLimits
+        get_limits.argtypes = [ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t)]
+        get_limits.restype = None
+        get_limits(ctypes.byref(lo), ctypes.byref(hi))
+        print("main-thread stack reserve:", hi.value - lo.value)
+        sys.exit(hi.value - lo.value < int(sys.argv[1]))
+        EOF

.github/workflows/ci-nightly.yml

@@ -49,7 +49,7 @@ jobs:
 
       - name: Run CI tools unit tests
         run: |
-          python -m pip install pytest
+          python -m pip install pytest pyyaml==6.0.3 pathspec==0.12.1
           # Standalone CI tool tests; skip repo-root conftest.py (imports cuda.pathfinder).
           python -m pytest -v --noconftest ci/tools/tests
 

.github/workflows/coverage.yml

@@ -235,10 +235,11 @@ jobs:
           filter: blob:none
 
       - name: Set up Python
-        uses: ./.github/actions/setup-hardened-python
+        uses: $/.github/actions/setup-hardened-python
         with:
           python-version: ${{ env.PY_VER }}
           venv: .venv
+          stack-reserve: 8388608
 
       - name: Set up MSVC
         uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4  # v1.13.1
@@ -363,10 +364,11 @@ jobs:
           nvidia-smi
 
       - name: Set up Python
-        uses: ./.github/actions/setup-hardened-python
+        uses: $/.github/actions/setup-hardened-python
         with:
           python-version: ${{ env.PY_VER }}
           venv: .venv
+          stack-reserve: 8388608
 
       - name: Set up mini CTK
         uses: ./.github/actions/fetch_ctk
@@ -417,22 +419,22 @@ jobs:
           cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
           "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests"
 
-      # Cython linetrace under coverage on Windows needs more stack than the
-      # default 1 MB thread size.  The helper runs pytest on an 8 MB thread.
-      - name: Run cuda.bindings tests (with 8MB stack)
+      # The patched interpreter provides the larger stack on the main thread
+      # and in child interpreters. Deferred cleanup needs pytest on the main thread.
+      - name: Run cuda.bindings tests
         continue-on-error: true
         run: |
-          "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \
-            --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \
+          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
+          "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \
             -v --cov=./cuda --cov-append --cov-context=test \
             --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
             "$GITHUB_WORKSPACE/cuda_bindings/tests"
 
-      - name: Run cuda.core tests (with 8MB stack)
+      - name: Run cuda.core tests
         continue-on-error: true
         run: |
-          "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \
-            --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \
+          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
+          "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \
             -v --cov=./cuda --cov-append --cov-context=test \
             --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
             "$GITHUB_WORKSPACE/cuda_core/tests"