From d093d5473f19488fc3367210fb265bb4f414dac9 Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Tue, 3 Nov 2026 09:41:28 -0800 Subject: [PATCH 1/3] Fix Windows coverage test stack overflows Raise the GPU runner's base Python executable stack reserve to at least 8 MiB using a standard-library PE-header helper. Write a patched copy and replace the executable after Python exits, avoiding the need for editbin on the GPU runner. Preserve the existing editbin build-job step. run_pytest_with_stack.py enlarges only the worker thread running pytest. Samples, subprocess tests, and multiprocessing workers start fresh Python interpreters with the executable's original stack reserve, so they still overflow. The wrapper also leaves Python's main thread waiting for pytest, which can delay deferred cleanup scheduled through Py_AddPendingCall. Run pytest directly on the main thread with the patched interpreter and remove the worker-thread wrapper. Keep continue-on-error enabled. Validation: 22 tests passed; Ruff, formatting, and workflow syntax checks passed. Verified patched PE32 and PE32+ executables with objdump. Windows GPU CI confirmation remains pending. --- .github/workflows/coverage.yml | 24 ++- ci/tools/prepare_windows_python_stack.py | 70 +++++++++ ci/tools/run_pytest_with_stack.py | 54 ------- .../test_prepare_windows_python_stack.py | 140 ++++++++++++++++++ 4 files changed, 226 insertions(+), 62 deletions(-) create mode 100644 ci/tools/prepare_windows_python_stack.py delete mode 100644 ci/tools/run_pytest_with_stack.py create mode 100644 ci/tools/tests/test_prepare_windows_python_stack.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2e998453e5c..149d8234925 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -386,6 +386,14 @@ jobs: python-version: ${{ env.PY_VER }} venv: .venv + - name: Raise the interpreter's stack reserve to 8 MiB + # This runner has no MSVC tools. Patch a copy, then replace the base + # interpreter after Python exits so subprocesses also get 8 MiB stacks. + run: | + base_py="$(cygpath -u "$(.venv/Scripts/python -c 'import sys; print(sys._base_executable)')")" + .venv/Scripts/python ci/tools/prepare_windows_python_stack.py "$base_py" "$base_py.stack-reserve" + mv -f -- "$base_py.stack-reserve" "$base_py" + - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false @@ -435,22 +443,22 @@ jobs: cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests" - # Cython linetrace under coverage on Windows needs more stack than the - # default 1 MB thread size. The helper runs pytest on an 8 MB thread. - - name: Run cuda.bindings tests (with 8MB stack) + # The patched interpreter provides the larger stack on the main thread + # and in child interpreters. Deferred cleanup needs pytest on the main thread. + - name: Run cuda.bindings tests continue-on-error: true run: | - "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ - --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ + cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" + "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \ -v --cov=./cuda --cov-append --cov-context=test \ --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ "$GITHUB_WORKSPACE/cuda_bindings_13/tests" - - name: Run cuda.core tests (with 8MB stack) + - name: Run cuda.core tests continue-on-error: true run: | - "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ - --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ + cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" + "$GITHUB_WORKSPACE/.venv/Scripts/python" -m pytest \ -v --cov=./cuda --cov-append --cov-context=test \ --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ "$GITHUB_WORKSPACE/cuda_core/tests" diff --git a/ci/tools/prepare_windows_python_stack.py b/ci/tools/prepare_windows_python_stack.py new file mode 100644 index 00000000000..8c5a74399ef --- /dev/null +++ b/ci/tools/prepare_windows_python_stack.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare a copy of a Windows Python executable with at least an 8 MiB stack. + +Windows prevents writing to a running executable. Write a patched copy, then +have the calling shell replace the original after this interpreter exits. +Patch sys._base_executable rather than the virtual environment's launcher. +""" + +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +STACK_RESERVE = 8 * 1024 * 1024 + + +def prepare_executable(source: Path, destination: Path) -> tuple[int, int]: + """Validate a PE image and write a new copy with a larger stack reserve.""" + image = bytearray(source.read_bytes()) + if len(image) < 64 or image[:2] != b"MZ": + raise ValueError("Not a Windows executable: missing DOS header") + pe_offset = struct.unpack_from(" len(image) or image[pe_offset : pe_offset + 4] != b"PE\0\0": + raise ValueError("Missing or truncated PE header") + + optional_offset = pe_offset + 24 + optional_size = struct.unpack_from(" len(image): + raise ValueError("Truncated PE optional header") + magic = struct.unpack_from(" int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path, help="Base Python executable (sys._base_executable)") + parser.add_argument("destination", type=Path, help="New patched copy; must not already exist") + args = parser.parse_args(argv) + previous, reserve = prepare_executable(args.source, args.destination) + print(f"{args.source}: stack reserve {previous} -> {reserve} bytes; prepared {args.destination}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/tools/run_pytest_with_stack.py b/ci/tools/run_pytest_with_stack.py deleted file mode 100644 index e1e1f782ba1..00000000000 --- a/ci/tools/run_pytest_with_stack.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Run pytest on a thread with a larger stack size. - -Cython linetrace instrumentation under coverage on Windows can exceed the -default 1 MB thread stack. This helper spawns a single worker thread with -a configurable stack (default 8 MB) so the rest of the CI workflow stays -readable. - -Usage: - python run_pytest_with_stack.py [--stack-mb N] [--cwd DIR] [pytest args ...] -""" - -import argparse -import concurrent.futures -import os -import sys -import threading - -import pytest - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--stack-mb", - type=int, - default=8, - help="Thread stack size in megabytes (default: 8)", - ) - parser.add_argument( - "--cwd", - default=None, - help="Working directory for the test run", - ) - args, pytest_args = parser.parse_known_args() - - if args.cwd: - os.chdir(args.cwd) - - threading.stack_size(args.stack_mb * 1024 * 1024) - - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - code = pool.submit(pytest.main, pytest_args).result() - - sys.exit(code) - - -if __name__ == "__main__": - main() diff --git a/ci/tools/tests/test_prepare_windows_python_stack.py b/ci/tools/tests/test_prepare_windows_python_stack.py new file mode 100644 index 00000000000..feeb28df1e1 --- /dev/null +++ b/ci/tools/tests/test_prepare_windows_python_stack.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Test executable patching without requiring Windows or MSVC.""" + +from __future__ import annotations + +import importlib.util +import struct +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "prepare_windows_python_stack.py" +spec = importlib.util.spec_from_file_location("prepare_windows_python_stack", SCRIPT) +tool = importlib.util.module_from_spec(spec) +spec.loader.exec_module(tool) + + +def _pe_image(magic=0x20B, reserve=2_000_000): + # DOS stub, COFF header, optional header, and sentinel payload. Keep the + # commit/heap values nonzero to detect writes beyond the reserve field. + image = bytearray(512) + image[:2] = b"MZ" + struct.pack_into(" 8388608 bytes" in result.stdout + assert result.stderr == "" + # The caller can replace the executable once the patching process exits. + destination.replace(source) + assert int.from_bytes(source.read_bytes()[0xE0:0xE8], "little") == 8_388_608 From 4f28c6604db67b841ea87a098ce4af3b83f33fdc Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Tue, 3 Nov 2026 10:07:27 -0800 Subject: [PATCH 2/3] Strip CR from path output. --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 149d8234925..e4509d9c6fa 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -390,7 +390,7 @@ jobs: # This runner has no MSVC tools. Patch a copy, then replace the base # interpreter after Python exits so subprocesses also get 8 MiB stacks. run: | - base_py="$(cygpath -u "$(.venv/Scripts/python -c 'import sys; print(sys._base_executable)')")" + base_py="$(cygpath -u "$(.venv/Scripts/python -c 'import sys; sys.stdout.write(sys._base_executable)')")" .venv/Scripts/python ci/tools/prepare_windows_python_stack.py "$base_py" "$base_py.stack-reserve" mv -f -- "$base_py.stack-reserve" "$base_py" From 7af7f9b266a42198d87d34b6d98472b0f8641a13 Mon Sep 17 00:00:00 2001 From: Jason Pascucci Date: Tue, 3 Nov 2026 10:31:08 -0800 Subject: [PATCH 3/3] Implement Ralf J's suggestions. --- .../actions/setup-hardened-python/action.yml | 43 ++++++ .github/workflows/coverage.yml | 22 +-- ci/tools/prepare_windows_python_stack.py | 70 --------- .../test_prepare_windows_python_stack.py | 140 ------------------ 4 files changed, 47 insertions(+), 228 deletions(-) delete mode 100644 ci/tools/prepare_windows_python_stack.py delete mode 100644 ci/tools/tests/test_prepare_windows_python_stack.py diff --git a/.github/actions/setup-hardened-python/action.yml b/.github/actions/setup-hardened-python/action.yml index 8afb6feef2b..b5b909c9043 100644 --- a/.github/actions/setup-hardened-python/action.yml +++ b/.github/actions/setup-hardened-python/action.yml @@ -26,6 +26,11 @@ inputs: venv: description: Directory for the venv. Defaults to a new directory under RUNNER_TEMP. default: "" + stack-reserve: + description: >- + Windows only. Raise the base interpreter's main-thread stack reserve to at + least this many bytes. Empty (default) leaves the interpreter unchanged. + default: "" outputs: python-path: @@ -123,3 +128,41 @@ runs: PYTHON: ${{ steps.venv.outputs.python-path }} run: | "${PYTHON}" -m pip install -U pip + + - name: Raise the interpreter's stack reserve + if: runner.os == 'Windows' && inputs.stack-reserve != '' + shell: bash --noprofile --norc -xeuo pipefail {0} + env: + PYTHON: ${{ steps.venv.outputs.python-path }} + STACK_RESERVE: ${{ inputs.stack-reserve }} + run: | + # sys.stdout.write avoids a trailing CR from Windows Python's CRLF. + base="$("${PYTHON}" -c 'import sys; sys.stdout.write(sys._base_executable)')" + # Windows cannot overwrite a running executable. Prepare a copy and + # replace the original after the patching interpreter exits. + uv run --no-project --isolated --with pefile==2024.8.26 --python "${base}" python - "${base}" "${STACK_RESERVE}" <<'EOF' + import sys + + import pefile + + path, reserve = sys.argv[1], int(sys.argv[2]) + if reserve <= 0: + raise ValueError("stack-reserve must be a positive number of bytes") + with pefile.PE(path) as pe: + pe.OPTIONAL_HEADER.SizeOfStackReserve = max(pe.OPTIONAL_HEADER.SizeOfStackReserve, reserve) + pe.write(path + ".stack-reserve") + EOF + mv -f -- "${base}.stack-reserve" "${base}" + # Verify the running interpreter's stack, rather than just its header. + "${base}" - "${STACK_RESERVE}" <<'EOF' + import ctypes + import sys + + lo, hi = ctypes.c_size_t(), ctypes.c_size_t() + get_limits = ctypes.windll.kernel32.GetCurrentThreadStackLimits + get_limits.argtypes = [ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t)] + get_limits.restype = None + get_limits(ctypes.byref(lo), ctypes.byref(hi)) + print("main-thread stack reserve:", hi.value - lo.value) + sys.exit(hi.value - lo.value < int(sys.argv[1])) + EOF diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e4509d9c6fa..f75ac200b06 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -242,22 +242,15 @@ jobs: submodules: true - name: Set up Python - uses: ./.github/actions/setup-hardened-python + uses: $/.github/actions/setup-hardened-python with: python-version: ${{ env.PY_VER }} venv: .venv + stack-reserve: 8388608 - name: Set up MSVC uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - - name: Raise the interpreter's stack reserve to 8 MiB - # Importing the coverage-built cuda.bindings.driver overflows the - # 2,000,000-byte main-thread stack of python-build-standalone (#3330). - run: | - base_py="$(.venv/Scripts/python -c 'import sys; print(sys._base_executable)')" - editbin -nologo -stack:8388608 "$base_py" - dumpbin -nologo -headers "$base_py" | grep -i "size of stack reserve" - - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false @@ -381,18 +374,11 @@ jobs: nvidia-smi - name: Set up Python - uses: ./.github/actions/setup-hardened-python + uses: $/.github/actions/setup-hardened-python with: python-version: ${{ env.PY_VER }} venv: .venv - - - name: Raise the interpreter's stack reserve to 8 MiB - # This runner has no MSVC tools. Patch a copy, then replace the base - # interpreter after Python exits so subprocesses also get 8 MiB stacks. - run: | - base_py="$(cygpath -u "$(.venv/Scripts/python -c 'import sys; sys.stdout.write(sys._base_executable)')")" - .venv/Scripts/python ci/tools/prepare_windows_python_stack.py "$base_py" "$base_py.stack-reserve" - mv -f -- "$base_py.stack-reserve" "$base_py" + stack-reserve: 8388608 - name: Set up mini CTK uses: ./.github/actions/fetch_ctk diff --git a/ci/tools/prepare_windows_python_stack.py b/ci/tools/prepare_windows_python_stack.py deleted file mode 100644 index 8c5a74399ef..00000000000 --- a/ci/tools/prepare_windows_python_stack.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Prepare a copy of a Windows Python executable with at least an 8 MiB stack. - -Windows prevents writing to a running executable. Write a patched copy, then -have the calling shell replace the original after this interpreter exits. -Patch sys._base_executable rather than the virtual environment's launcher. -""" - -from __future__ import annotations - -import argparse -import struct -from pathlib import Path - -STACK_RESERVE = 8 * 1024 * 1024 - - -def prepare_executable(source: Path, destination: Path) -> tuple[int, int]: - """Validate a PE image and write a new copy with a larger stack reserve.""" - image = bytearray(source.read_bytes()) - if len(image) < 64 or image[:2] != b"MZ": - raise ValueError("Not a Windows executable: missing DOS header") - pe_offset = struct.unpack_from(" len(image) or image[pe_offset : pe_offset + 4] != b"PE\0\0": - raise ValueError("Missing or truncated PE header") - - optional_offset = pe_offset + 24 - optional_size = struct.unpack_from(" len(image): - raise ValueError("Truncated PE optional header") - magic = struct.unpack_from(" int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("source", type=Path, help="Base Python executable (sys._base_executable)") - parser.add_argument("destination", type=Path, help="New patched copy; must not already exist") - args = parser.parse_args(argv) - previous, reserve = prepare_executable(args.source, args.destination) - print(f"{args.source}: stack reserve {previous} -> {reserve} bytes; prepared {args.destination}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/ci/tools/tests/test_prepare_windows_python_stack.py b/ci/tools/tests/test_prepare_windows_python_stack.py deleted file mode 100644 index feeb28df1e1..00000000000 --- a/ci/tools/tests/test_prepare_windows_python_stack.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Test executable patching without requiring Windows or MSVC.""" - -from __future__ import annotations - -import importlib.util -import struct -import subprocess -import sys -from pathlib import Path - -import pytest - -SCRIPT = Path(__file__).resolve().parents[1] / "prepare_windows_python_stack.py" -spec = importlib.util.spec_from_file_location("prepare_windows_python_stack", SCRIPT) -tool = importlib.util.module_from_spec(spec) -spec.loader.exec_module(tool) - - -def _pe_image(magic=0x20B, reserve=2_000_000): - # DOS stub, COFF header, optional header, and sentinel payload. Keep the - # commit/heap values nonzero to detect writes beyond the reserve field. - image = bytearray(512) - image[:2] = b"MZ" - struct.pack_into(" 8388608 bytes" in result.stdout - assert result.stderr == "" - # The caller can replace the executable once the patching process exits. - destination.replace(source) - assert int.from_bytes(source.read_bytes()[0xE0:0xE8], "little") == 8_388_608