diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2e998453e5c..475701ff2d8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -386,6 +386,15 @@ jobs: python-version: ${{ env.PY_VER }} venv: .venv + # This job installs a separate Python from the wheel-build job. Give its + # main thread and test subprocesses enough stack for coverage-built bindings. + - name: Raise the test interpreter's stack reserve to 8 MiB + shell: powershell + run: | + $basePy = & .venv/Scripts/python.exe -c "import sys; print(sys._base_executable)" + if ($LASTEXITCODE -ne 0) { throw "Could not locate base Python executable" } + & ./ci/tools/set_windows_python_stack.ps1 -Executable $basePy -ReserveBytes 8388608 + - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false @@ -435,8 +444,8 @@ jobs: cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests" - # Cython linetrace under coverage on Windows needs more stack than the - # default 1 MB thread size. The helper runs pytest on an 8 MB thread. + # The patched interpreter gives pytest's main thread and subprocesses + # 8 MiB; the helper gives threads created during pytest the same stack. - name: Run cuda.bindings tests (with 8MB stack) continue-on-error: true run: | @@ -577,7 +586,11 @@ jobs: # GitHub discards prerequisite artifacts on a new run attempt. If only this # job fails, use "Re-run all jobs" so coverage recreates its payload. needs: combine-and-deploy - if: ${{ needs.combine-and-deploy.result == 'success' && github.repository_owner == 'nvidia' }} + # Manual runs on PR mirror branches should not publish over main's report. + if: >- + needs.combine-and-deploy.result == 'success' && + github.repository_owner == 'nvidia' && + github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: contents: write diff --git a/ci/tools/run_pytest_with_stack.py b/ci/tools/run_pytest_with_stack.py index e1e1f782ba1..a1fb2ef53ce 100644 --- a/ci/tools/run_pytest_with_stack.py +++ b/ci/tools/run_pytest_with_stack.py @@ -4,24 +4,46 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Run pytest on a thread with a larger stack size. +"""Run pytest on Python's main thread with a larger stack. -Cython linetrace instrumentation under coverage on Windows can exceed the -default 1 MB thread stack. This helper spawns a single worker thread with -a configurable stack (default 8 MB) so the rest of the CI workflow stays -readable. +The Windows coverage workflow first raises the base interpreter's PE stack +reserve to 8 MB, covering the main thread and test subprocesses. Set the +stack size for threads started during pytest to match. Running pytest on the +main thread lets CPython process pending calls used by CUDA graph cleanup. Usage: python run_pytest_with_stack.py [--stack-mb N] [--cwd DIR] [pytest args ...] """ import argparse -import concurrent.futures import os import sys import threading -import pytest + +def _ensure_main_stack(stack_mb): + stack_bytes = stack_mb * 1024 * 1024 + if sys.platform == "win32": + import ctypes + + low, high = ctypes.c_size_t(), ctypes.c_size_t() + get_limits = ctypes.WinDLL("kernel32", use_last_error=True).GetCurrentThreadStackLimits + get_limits.argtypes = [ + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] + get_limits.restype = None + get_limits(ctypes.byref(low), ctypes.byref(high)) + actual = high.value - low.value + if actual < stack_bytes: + raise RuntimeError(f"Main thread stack is {actual} bytes, requested {stack_bytes}") + print(f"[stack-wrapper] main thread stack: {actual} bytes", flush=True) + else: + import resource + + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + if soft != resource.RLIM_INFINITY and soft < stack_bytes: + resource.setrlimit(resource.RLIMIT_STACK, (stack_bytes, hard)) def main(): @@ -30,7 +52,7 @@ def main(): "--stack-mb", type=int, default=8, - help="Thread stack size in megabytes (default: 8)", + help="Main thread stack size in megabytes (default: 8)", ) parser.add_argument( "--cwd", @@ -38,16 +60,16 @@ def main(): help="Working directory for the test run", ) args, pytest_args = parser.parse_known_args() + if args.stack_mb <= 0: + parser.error("--stack-mb must be positive") + _ensure_main_stack(args.stack_mb) + threading.stack_size(args.stack_mb * 1024 * 1024) if args.cwd: os.chdir(args.cwd) + import pytest - threading.stack_size(args.stack_mb * 1024 * 1024) - - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - code = pool.submit(pytest.main, pytest_args).result() - - sys.exit(code) + sys.exit(pytest.main(pytest_args)) if __name__ == "__main__": diff --git a/ci/tools/set_windows_python_stack.ps1 b/ci/tools/set_windows_python_stack.ps1 new file mode 100644 index 00000000000..dd6b9a90379 --- /dev/null +++ b/ci/tools/set_windows_python_stack.ps1 @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +param( + [Parameter(Mandatory = $true)] + [string]$Executable, + + [Parameter(Mandatory = $true)] + [UInt64]$ReserveBytes +) + +$ErrorActionPreference = "Stop" + +if (-not $env:RUNNER_TEMP) { + throw "RUNNER_TEMP is required" +} +$path = [IO.Path]::GetFullPath($Executable) +$installRoot = [IO.Path]::GetFullPath((Join-Path $env:RUNNER_TEMP "setup-hardened-python/interpreters")) +if (-not $path.StartsWith($installRoot + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { + throw "Refusing to modify Python outside $installRoot`: $path" +} + +$image = [IO.File]::ReadAllBytes($path) +if ($image.Length -lt 64 -or $image[0] -ne 0x4d -or $image[1] -ne 0x5a) { + throw "Not a PE executable: $path" +} +$peOffset = [BitConverter]::ToInt32($image, 0x3c) +if ($peOffset -lt 64 -or $peOffset -gt $image.Length - 24 -or + $image[$peOffset] -ne 0x50 -or $image[$peOffset + 1] -ne 0x45 -or + $image[$peOffset + 2] -ne 0 -or $image[$peOffset + 3] -ne 0) { + throw "Invalid PE header: $path" +} +$optionalOffset = $peOffset + 24 +$optionalSize = [BitConverter]::ToUInt16($image, $peOffset + 20) +if ($optionalSize -lt 2 -or $optionalOffset + $optionalSize -gt $image.Length) { + throw "Truncated PE optional header: $path" +} +$magic = [BitConverter]::ToUInt16($image, $optionalOffset) +if ($magic -eq 0x20b) { + $width = 8 +} elseif ($magic -eq 0x10b) { + $width = 4 + if ($ReserveBytes -gt [UInt32]::MaxValue) { + throw "Stack reserve is too large for a 32-bit PE executable" + } +} else { + throw "Unsupported PE optional header: $magic" +} +if ($optionalSize -lt 72 + $width) { + throw "PE optional header does not contain a stack reserve: $path" +} + +$reserveOffset = $optionalOffset + 72 +$current = if ($width -eq 8) { + [BitConverter]::ToUInt64($image, $reserveOffset) +} else { + [BitConverter]::ToUInt32($image, $reserveOffset) +} +if ($current -lt $ReserveBytes) { + $replacement = if ($width -eq 8) { + [BitConverter]::GetBytes($ReserveBytes) + } else { + [BitConverter]::GetBytes([UInt32]$ReserveBytes) + } + $stream = [IO.File]::Open($path, [IO.FileMode]::Open, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None) + try { + $stream.Position = $reserveOffset + $stream.Write($replacement, 0, $width) + $stream.Flush() + } finally { + $stream.Dispose() + } +} + +$updated = [IO.File]::ReadAllBytes($path) +$actual = if ($width -eq 8) { + [BitConverter]::ToUInt64($updated, $reserveOffset) +} else { + [BitConverter]::ToUInt32($updated, $reserveOffset) +} +if ($actual -lt $ReserveBytes) { + throw "Stack reserve verification failed: $actual < $ReserveBytes" +} +Write-Output "[stack-reserve] $path`: $current -> $actual bytes"