diff --git a/.ci/scripts/test_model_e2e_windows.ps1 b/.ci/scripts/test_model_e2e_windows.ps1 index 8f7d6c2456e..fd01194fa51 100644 --- a/.ci/scripts/test_model_e2e_windows.ps1 +++ b/.ci/scripts/test_model_e2e_windows.ps1 @@ -133,6 +133,8 @@ function Download-IfNeeded { Push-Location $executorchRoot try { Write-Host "::group::Check CUDA toolchain" + # The GPU runners come without a driver, and every CUDA call fails without one. + & (Join-Path -Path $scriptDir -ChildPath "install_nvidia_driver_windows.ps1") $nvccOutput = nvcc --version | Out-String Write-Host $nvccOutput $nvidiaSmiCmd = Get-Command nvidia-smi -ErrorAction SilentlyContinue @@ -299,7 +301,8 @@ try { -RedirectStandardOutput $stdoutFile ` -RedirectStandardError $stderrFile - $stdout = if (Test-Path -Path $stdoutFile -PathType Leaf) { Get-Content -Path $stdoutFile -Raw } else { "" } + # [string] because Get-Content returns nothing for an empty file, and matching against nothing never fails. + $stdout = if (Test-Path -Path $stdoutFile -PathType Leaf) { [string](Get-Content -Path $stdoutFile -Raw) } else { "" } $stderr = if (Test-Path -Path $stderrFile -PathType Leaf) { Get-Content -Path $stderrFile -Raw } else { "" } $exitCode = $proc.ExitCode } @@ -313,7 +316,7 @@ try { Write-Host $stderr if ($exitCode -ne 0) { - Write-Warning "Runner exited with code $exitCode (may be benign)" + throw "Runner exited with code $exitCode" } if ($expectedOutput -ne "") { diff --git a/.github/workflows/build-wheels-cuda-windows.yml b/.github/workflows/build-wheels-cuda-windows.yml index 728740f649a..4a5887b9431 100644 --- a/.github/workflows/build-wheels-cuda-windows.yml +++ b/.github/workflows/build-wheels-cuda-windows.yml @@ -94,9 +94,11 @@ jobs: repository: ${{ matrix.repository }} ref: "" test-infra-repository: pytorch/test-infra - # Avoid the unsatisfiable conda upgrade in the Windows runner's base env. + # Avoid the unsatisfiable conda upgrade in the Windows runner's base env, and the + # Python 3.15 release candidate download that fails on the current runner image. # https://github.com/pytorch/test-infra/pull/9179 - test-infra-ref: e3d344d886a7281f6a919115d332df2a080da8ad + # https://github.com/pytorch/test-infra/pull/9265 + test-infra-ref: b46669025788e540ed4ecc7c56cc52db343e284f build-matrix: ${{ needs.filter-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} env-script: ${{ matrix.env-script }} diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index e06d72451f8..95e7ec48c70 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -17,8 +17,8 @@ on: - backends/cuda/** - backends/aoti/** - extension/cuda/** - # Read only by unittest-cuda-runtime-windows; the other jobs filter on - # their own paths and stay skipped. + # Only unittest-cuda-runtime-windows runs for these alone; the other jobs + # filter on their own paths and stay skipped. - .ci/scripts/test_cuda_runtime_windows.ps1 - .ci/scripts/install_nvidia_driver_windows.ps1 - tools/cmake/Test.cmake @@ -202,7 +202,10 @@ jobs: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "13.0" + # The Windows runner image carries only the toolkits the CUDA wheels are built for, + # 13.2 and 13.4, so this cannot match the export's 13.0. The exported program loads + # cudart64_13.dll, which every CUDA 13 toolkit provides. + gpu-arch-version: "13.2" download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | @@ -214,7 +217,7 @@ jobs: \$ErrorActionPreference = 'Stop' \$PSNativeCommandUseErrorActionPreference = \$true - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.0' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' \$env:CUDA_PATH = \$env:CUDA_HOME \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\" nvcc --version @@ -225,7 +228,7 @@ jobs: throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.' } - .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.0' + .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.2' }" # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners @@ -248,7 +251,8 @@ jobs: timeout: 120 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: "13.0" + # The same toolkit as test-model-cuda-windows-e2e, for the same reason. + gpu-arch-version: "13.2" ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -257,7 +261,7 @@ jobs: powershell -Command "& { Set-PSDebug -Trace 1 \$ErrorActionPreference = 'Stop' - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.0' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' \$env:CUDA_PATH = \$env:CUDA_HOME .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 diff --git a/examples/models/dinov2/CMakeLists.txt b/examples/models/dinov2/CMakeLists.txt index 8e83ee44873..d20837854d9 100644 --- a/examples/models/dinov2/CMakeLists.txt +++ b/examples/models/dinov2/CMakeLists.txt @@ -90,8 +90,10 @@ if(MSVC) add_custom_command( TARGET dinov2_runner POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ - COMMENT "Copying aoti_cuda_shims.dll to dinov2_runner directory" + COMMAND + ${CMAKE_COMMAND} -E copy_if_different $ + $ + COMMENT "Copying runtime DLLs to dinov2_runner directory" + COMMAND_EXPAND_LISTS ) endif() diff --git a/examples/models/nemotron3_diarization/CMakeLists.txt b/examples/models/nemotron3_diarization/CMakeLists.txt index 0316f73eab2..3e69cff4a03 100644 --- a/examples/models/nemotron3_diarization/CMakeLists.txt +++ b/examples/models/nemotron3_diarization/CMakeLists.txt @@ -80,8 +80,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) TARGET nemotron3_diarization_runner POST_BUILD COMMAND - ${CMAKE_COMMAND} -E copy_if_different $ + ${CMAKE_COMMAND} -E copy_if_different + $ $ + COMMAND_EXPAND_LISTS ) endif() if(EXECUTORCH_BUILD_VULKAN) diff --git a/examples/models/parakeet/CMakeLists.txt b/examples/models/parakeet/CMakeLists.txt index 9449583c613..c15cf01b026 100644 --- a/examples/models/parakeet/CMakeLists.txt +++ b/examples/models/parakeet/CMakeLists.txt @@ -151,9 +151,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) TARGET ${parakeet_target} POST_BUILD COMMAND - ${CMAKE_COMMAND} -E copy_if_different $ + ${CMAKE_COMMAND} -E copy_if_different + $ $ - COMMENT "Copying aoti_cuda_shims.dll to ${parakeet_target} directory" + COMMENT "Copying runtime DLLs to ${parakeet_target} directory" + COMMAND_EXPAND_LISTS ) endforeach() endif() diff --git a/examples/models/sortformer/CMakeLists.txt b/examples/models/sortformer/CMakeLists.txt index 171eb726ed2..8b17e645741 100644 --- a/examples/models/sortformer/CMakeLists.txt +++ b/examples/models/sortformer/CMakeLists.txt @@ -111,8 +111,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) add_custom_command( TARGET sortformer_runner POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ - COMMENT "Copying aoti_cuda_shims.dll to sortformer_runner directory" + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Copying runtime DLLs to sortformer_runner directory" + COMMAND_EXPAND_LISTS ) endif() diff --git a/examples/models/voxtral/CMakeLists.txt b/examples/models/voxtral/CMakeLists.txt index 2e8ebb5c5e9..58555e97ce8 100644 --- a/examples/models/voxtral/CMakeLists.txt +++ b/examples/models/voxtral/CMakeLists.txt @@ -141,8 +141,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) add_custom_command( TARGET voxtral_runner POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ - COMMENT "Copying aoti_cuda_shims.dll to voxtral_runner directory" + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + $ $ + COMMENT "Copying runtime DLLs to voxtral_runner directory" + COMMAND_EXPAND_LISTS ) endif() diff --git a/examples/models/voxtral_realtime/CMakeLists.txt b/examples/models/voxtral_realtime/CMakeLists.txt index 2710d86ed83..2442102271c 100644 --- a/examples/models/voxtral_realtime/CMakeLists.txt +++ b/examples/models/voxtral_realtime/CMakeLists.txt @@ -129,8 +129,11 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) add_custom_command( TARGET voxtral_realtime_runner POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ - COMMENT "Copying aoti_cuda_shims.dll to voxtral_realtime_runner directory" + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMENT "Copying runtime DLLs to voxtral_realtime_runner directory" + COMMAND_EXPAND_LISTS ) endif() diff --git a/examples/models/whisper/CMakeLists.txt b/examples/models/whisper/CMakeLists.txt index 08baf3b825f..57c3403ae7e 100644 --- a/examples/models/whisper/CMakeLists.txt +++ b/examples/models/whisper/CMakeLists.txt @@ -114,8 +114,10 @@ if(MSVC AND EXECUTORCH_BUILD_CUDA) add_custom_command( TARGET whisper_runner POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ - COMMENT "Copying aoti_cuda_shims.dll to whisper_runner directory" + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + $ $ + COMMENT "Copying runtime DLLs to whisper_runner directory" + COMMAND_EXPAND_LISTS ) endif() diff --git a/setup.py b/setup.py index c3d73287242..ec9ba0aa4ea 100644 --- a/setup.py +++ b/setup.py @@ -2725,7 +2725,7 @@ def run(self): # noqa C901 ] else: if windows_cuda_home: - log.warning( + log.warn( "building the CUDA backend with the Visual Studio generator, " "because ninja, clang-cl and cl are not all on PATH; run from " "a Visual Studio developer prompt with ninja for the Ninja "