From 3f4c7928906d0e957114fcd6eae3f35f5b8ae162 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 08:54:28 -0700 Subject: [PATCH 1/5] test: consolidate duplicated memory-pool test helpers Follow-up to #3290. The same memory-pool helpers were copied across the cuda_bindings_12, cuda_bindings_13, and cuda_core test suites. Move them into one shared module and delete the copies. No test outcome changes: the conditions behind every skipif and skip are the same. New cuda_python_test_helpers/memory.py: * POOL_SIZE (9 copies, including the cuda_core constant, which now re-exports it). * supports_memory_pools() and supports_managed_memory() (13 copies, including the two Cython test modules). * xfail_graph_mempool_oom() (3 copies of the xfail_on_graph_mempool_oom fixture). Each call site keeps its own out-of-memory check and cleanup. cuda_bindings_12/tests/test_cuda.py also uses the shared driver_version_less_than() instead of its own copy, which behaves the same. cuda_bindings_12/tests/test_cudart.py keeps its local driverVersionLessThan: it does not call cuInit, and the shared one would turn collection-time skips into errors on machines with a driver but no device. cuda_core/tests/helpers/memory.py gains skip_if_mempool_unsupported(), skip_if_managed_mempool_unsupported(), and is_pool_resource(), which replace the per-module _skip_if_no_mempool (6 copies), _skip_without_mempools, _gpu_with_mempool_or_skip, _skip_if_no_managed_mempool (2 copies), and _is_pool (3 copies). Two skip reason strings change wording; their conditions do not. --- .../cython/test_interoperability_cython.pyx | 8 +- cuda_bindings_12/tests/test_cuda.py | 96 ++++++------------- cuda_bindings_12/tests/test_cudart.py | 14 +-- .../tests/test_interoperability.py | 8 +- .../cython/test_interoperability_cython.pyx | 15 +-- .../tests/legacy_api/test_legacy_cuda.py | 54 +++-------- .../test_legacy_interoperability.py | 15 +-- cuda_bindings_13/tests/test_cuda.py | 61 ++++-------- cuda_bindings_13/tests/test_cudart.py | 13 +-- .../tests/test_interoperability.py | 15 +-- cuda_core/tests/AGENTS.md | 10 +- .../tests/graph/test_graph_definition.py | 52 ++++------ .../graph/test_graph_definition_errors.py | 18 ++-- .../test_graph_definition_integration.py | 12 +-- .../graph/test_graph_definition_lifetime.py | 63 +++++------- cuda_core/tests/helpers/constants.py | 14 +-- cuda_core/tests/helpers/memory.py | 29 +++++- .../tests/memory/test_resource_live_count.py | 15 +-- cuda_core/tests/memory_ipc/test_errors.py | 22 ++--- cuda_core/tests/memory_ipc/test_memory_ipc.py | 14 +-- cuda_core/tests/memory_ipc/test_serialize.py | 14 +-- cuda_core/tests/test_object_protocols.py | 27 +++--- cuda_core/tests/test_rlcompleter_patch.py | 5 +- .../cuda_python_test_helpers/memory.py | 64 +++++++++++++ 24 files changed, 274 insertions(+), 384 deletions(-) create mode 100644 cuda_python_test_helpers/cuda_python_test_helpers/memory.py diff --git a/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx b/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx index f98660725f2..54dee295f58 100644 --- a/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx +++ b/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx @@ -7,6 +7,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart import numpy as np import pytest +from cuda_python_test_helpers.memory import supports_memory_pools # TODO: update to new module once the old ones are removed, we use the # tests to cover backward compatibility. @@ -14,11 +15,6 @@ cimport cuda.ccuda as ccuda cimport cuda.ccudart as ccudart -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - def test_interop_stream(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) @@ -140,7 +136,7 @@ def test_interop_graphNode(): assert(err_dr == cuda.CUresult.CUDA_SUCCESS) -@pytest.mark.skipif(not supportsMemoryPool(), reason='Requires mempool operations') +@pytest.mark.skipif(not supports_memory_pools(), reason='Requires mempool operations') def test_interop_memPool(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) diff --git a/cuda_bindings_12/tests/test_cuda.py b/cuda_bindings_12/tests/test_cuda.py index 402f4eb9a5a..7c4388bd37b 100644 --- a/cuda_bindings_12/tests/test_cuda.py +++ b/cuda_bindings_12/tests/test_cuda.py @@ -10,32 +10,14 @@ import pytest import cuda.cuda as cuda -import cuda.cudart as cudart from cuda.bindings import driver - - -def driverVersionLessThan(target): - (err,) = cuda.cuInit(0) - assert err == cuda.CUresult.CUDA_SUCCESS - err, version = cuda.cuDriverGetVersion() - assert err == cuda.CUresult.CUDA_SUCCESS - return version < target - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB +from cuda_python_test_helpers import driver_version_less_than +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) def supportsCudaAPI(name): @@ -325,7 +307,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): (err,) = cuda.cuInit(0) @@ -397,7 +379,7 @@ def test_cuda_memPool_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): (err,) = cuda.cuInit(0) @@ -456,7 +438,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): attributes = [ @@ -482,7 +464,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): assert helper.pyObj() == expected -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(): (err,) = cuda.cuInit(0) assert err == cuda.CUresult.CUDA_SUCCESS @@ -546,27 +528,11 @@ def test_cuda_mem_range_attr(): assert err == cuda.CUresult.CUDA_SUCCESS -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(err): - if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - -@pytest.mark.skipif(driverVersionLessThan(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported") +@pytest.mark.skipif( + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" +) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(): (err,) = cuda.cuInit(0) assert err == cuda.CUresult.CUDA_SUCCESS err, device = cuda.cuDeviceGet(0) @@ -594,7 +560,7 @@ def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS - xfail_on_graph_mempool_oom(err) + xfail_graph_mempool_oom() assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) assert err == cuda.CUresult.CUDA_SUCCESS @@ -632,7 +598,7 @@ def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): @pytest.mark.skipif( - driverVersionLessThan(12010) + driver_version_less_than(12010) or not supportsCudaAPI("cuCoredumpSetAttributeGlobal") or not supportsCudaAPI("cuCoredumpGetAttributeGlobal"), reason="Coredump API not present", @@ -808,7 +774,7 @@ def test_device_get_name(): # TODO: cuStreamGetCaptureInfo_v2 -@pytest.mark.skipif(driverVersionLessThan(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") +@pytest.mark.skipif(driver_version_less_than(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") def test_stream_capture(): pass @@ -941,7 +907,7 @@ def test_invalid_repr_attribute(): @pytest.mark.skipif( - driverVersionLessThan(12020) + driver_version_less_than(12020) or not supportsCudaAPI("cuGraphAddNode") or not supportsCudaAPI("cuGraphNodeSetParams") or not supportsCudaAPI("cuGraphExecNodeSetParams"), @@ -1061,7 +1027,7 @@ def test_graph_poly(): @pytest.mark.skipif( - driverVersionLessThan(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), + driver_version_less_than(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), reason="Polymorphic graph APIs required", ) def test_cuDeviceGetDevResource(): @@ -1090,7 +1056,7 @@ def test_cuDeviceGetDevResource(): @pytest.mark.skipif( - driverVersionLessThan(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), + driver_version_less_than(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), reason="Conditional graph APIs required", ) def test_conditional(): @@ -1159,14 +1125,14 @@ def test_all_CUresult_codes(): assert num_good >= 76 # CTK 11.0.3_450.51.06 -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuKernelGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuKernelGetName") def test_cuKernelGetName_failure(): err, name = cuda.cuKernelGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE assert name is None -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuFuncGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuFuncGetName") def test_cuFuncGetName_failure(): err, name = cuda.cuFuncGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE @@ -1174,7 +1140,7 @@ def test_cuFuncGetName_failure(): @pytest.mark.skipif( - driverVersionLessThan(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), + driver_version_less_than(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), reason="When API was introduced", ) def test_cuCheckpointProcessGetState_failure(): @@ -1216,7 +1182,7 @@ def test_struct_pointer_comparison(target): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphGetId(device, ctx): @@ -1243,7 +1209,7 @@ def test_cuGraphGetId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphExecGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphExecGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphExecGetId(device, ctx): @@ -1290,7 +1256,7 @@ def test_cuGraphExecGetId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetLocalId(device, ctx): @@ -1332,7 +1298,7 @@ def test_cuGraphNodeGetLocalId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetToolsId(device, ctx): @@ -1361,7 +1327,7 @@ def test_cuGraphNodeGetToolsId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetContainingGraph(device, ctx): @@ -1408,7 +1374,7 @@ def test_cuGraphNodeGetContainingGraph(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuStreamGetDevResource"), + driver_version_less_than(13010) or not supportsCudaAPI("cuStreamGetDevResource"), reason="Requires CUDA 13.1+", ) def test_cuStreamGetDevResource(device, ctx): @@ -1427,7 +1393,7 @@ def test_cuStreamGetDevResource(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), + driver_version_less_than(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), reason="Requires CUDA 13.1+", ) def test_cuDevSmResourceSplit(device, ctx): diff --git a/cuda_bindings_12/tests/test_cudart.py b/cuda_bindings_12/tests/test_cudart.py index 5c130fbe4bd..a4c75fe8c46 100644 --- a/cuda_bindings_12/tests/test_cudart.py +++ b/cuda_bindings_12/tests/test_cudart.py @@ -10,6 +10,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart from cuda.bindings import runtime +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools def isSuccess(err): @@ -26,17 +27,6 @@ def driverVersionLessThan(target): return version < target -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return isSuccess(err) and isSupported - - -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - def supportsSparseTexturesDeviceFilter(): err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrSparseCudaArraySupported, 0) return isSuccess(err) and isSupported @@ -424,7 +414,7 @@ def test_cudart_cudaGetDeviceProperties(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driverVersionLessThan(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cudart_MemPool_attr(): poolProps = cudart.cudaMemPoolProps() diff --git a/cuda_bindings_12/tests/test_interoperability.py b/cuda_bindings_12/tests/test_interoperability.py index 5262524d9a2..de6e536bedc 100644 --- a/cuda_bindings_12/tests/test_interoperability.py +++ b/cuda_bindings_12/tests/test_interoperability.py @@ -6,11 +6,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported +from cuda_python_test_helpers.memory import supports_memory_pools def test_interop_stream(): @@ -144,7 +140,7 @@ def test_interop_function(): assert err_dr == cuda.CUresult.CUDA_SUCCESS -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): (err_dr,) = cuda.cuInit(0) assert err_dr == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx b/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx index 62ef4abb312..944c0c21ae8 100644 --- a/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx +++ b/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx @@ -8,22 +8,11 @@ import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools cimport cuda.bindings.cydriver as ccuda cimport cuda.bindings.cyruntime as ccudart -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space (e.g. riscv64 Sv39). -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def test_interop_stream(): err_dr, = cuda.cuInit(0) @@ -147,7 +136,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") -@pytest.mark.skipif(not supportsMemoryPool(), reason='Requires mempool operations') +@pytest.mark.skipif(not supports_memory_pools(), reason='Requires mempool operations') def test_interop_memPool(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) diff --git a/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py b/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py index 8282f21673b..47043cf74f6 100644 --- a/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py +++ b/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py @@ -9,29 +9,19 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) from cuda_python_test_helpers.subprocess_runner import run_python_snippet import cuda.bindings.driver as cuda -import cuda.bindings.runtime as cudart from cuda.bindings import driver from cuda.bindings._internal.utils import FunctionNotFoundError from cuda_python_test_helpers import driver_version_less_than -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def supportsCudaAPI(name): return name in dir(cuda) @@ -263,7 +253,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): poolProps = cuda.CUmemPoolProps() @@ -326,7 +316,7 @@ def test_cuda_memPool_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): err, ptr = cuda.cuMemAllocManaged(0x1000, cuda.CUmemAttach_flags.CU_MEM_ATTACH_GLOBAL.value) @@ -377,7 +367,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): attributes = [ @@ -403,7 +393,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): assert helper.pyObj() == expected -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(device): size = 0x1000 location_device = cuda.CUmemLocation() @@ -466,29 +456,11 @@ def test_cuda_mem_range_attr(device): assert err == cuda.CUresult.CUDA_SUCCESS -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(err): - if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - @pytest.mark.skipif( - driver_version_less_than(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported" + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" ) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(device): err, stream = cuda.cuStreamCreate(0) assert err == cuda.CUresult.CUDA_SUCCESS @@ -509,7 +481,7 @@ def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS - xfail_on_graph_mempool_oom(err) + xfail_graph_mempool_oom() assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) assert err == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py b/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py index b8774af13fe..b268a3db842 100644 --- a/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py +++ b/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py @@ -3,22 +3,11 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def test_interop_stream(): # DRV to RT @@ -92,7 +81,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") @pytest.mark.agent_authored(model="claude-sonnet-5.5") -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): # Use a small, capped pool instead of the device's default pool. The pool # must be set before it is queried below because the getters create (and diff --git a/cuda_bindings_13/tests/test_cuda.py b/cuda_bindings_13/tests/test_cuda.py index 93a86a03986..c8fa0262a04 100644 --- a/cuda_bindings_13/tests/test_cuda.py +++ b/cuda_bindings_13/tests/test_cuda.py @@ -10,8 +10,13 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) -import cuda.bindings.runtime as cudart from cuda.bindings._internal.utils import FunctionNotFoundError from cuda.bindings._v2 import driver from cuda_python_test_helpers import driver_version_less_than @@ -19,21 +24,6 @@ # NULL dependency array: the node has no dependencies. NO_DEPS = 0 -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def supportsCudaAPI(name): return name in dir(driver) @@ -279,7 +269,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): poolProps = driver.MemPoolProps_v1() @@ -339,7 +329,7 @@ def get_attr(attr): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): ptr = driver.mem_alloc_managed(0x1000, driver.MemAttachFlags.GLOBAL.value) @@ -381,7 +371,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): (value,) = driver.pointer_get_attributes([driver.PointerAttribute.DEVICE_ORDINAL], 0) @@ -393,7 +383,7 @@ def test_pointer_get_attributes_device_ordinal(): @pytest.mark.agent_authored(model="gpt-5.6-sol") @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_allowed_handle_types_preserves_64_bits(): ptr = driver.mem_alloc_managed(0x1000, driver.MemAttachFlags.GLOBAL.value) @@ -403,7 +393,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): driver.mem_free_v2(ptr) -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(device): size = 0x1000 location_device = driver.MemLocation_v1() @@ -452,29 +442,11 @@ def test_cuda_mem_range_attr(device): driver.mem_free_v2(ptr) -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(exc_type): - if exc_type is driver.OutOfMemoryError: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - @pytest.mark.skipif( - driver_version_less_than(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported" + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" ) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(device): stream = driver.stream_create(0) graph = driver.graph_create(0) @@ -492,8 +464,7 @@ def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): except driver.OutOfMemoryError: driver.graph_destroy(graph) driver.stream_destroy_v2(stream) - xfail_on_graph_mempool_oom(driver.OutOfMemoryError) - raise + xfail_graph_mempool_oom() deps = node_array([allocNode]) freeNode = driver.graph_add_mem_free_node(graph, ctypes.addressof(deps), 1, params.dptr) @@ -594,7 +565,7 @@ def test_ffi_coverage_mapping_base_addr(): @pytest.mark.agent_authored(model="claude-sonnet-4-6") @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_ffi_coverage_new_mempool_attributes(): """Smoke-test all newly added MemPoolAttribute queries (driver errors for @@ -622,7 +593,7 @@ def test_ffi_coverage_new_mempool_attributes(): @pytest.mark.agent_authored(model="claude-sonnet-4-6") -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_ffi_coverage_new_mem_range_attributes(): """Smoke-test all newly added MemRangeAttribute queries (scalar int).""" size = 0x1000 diff --git a/cuda_bindings_13/tests/test_cudart.py b/cuda_bindings_13/tests/test_cudart.py index f1c263f7c13..98aaa9dd501 100644 --- a/cuda_bindings_13/tests/test_cudart.py +++ b/cuda_bindings_13/tests/test_cudart.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart @@ -13,11 +14,6 @@ from cuda.bindings import runtime from cuda_python_test_helpers import driver_version_less_than -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - def isSuccess(err): return err == cudart.cudaError_t.cudaSuccess @@ -27,11 +23,6 @@ def assertSuccess(err): assert isSuccess(err) -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return isSuccess(err) and isSupported - - def supportsSparseTexturesDeviceFilter(): err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrSparseCudaArraySupported, 0) return isSuccess(err) and isSupported @@ -518,7 +509,7 @@ def test_cudart_cudaGetDeviceProperties(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cudart_MemPool_attr(): poolProps = cudart.cudaMemPoolProps() diff --git a/cuda_bindings_13/tests/test_interoperability.py b/cuda_bindings_13/tests/test_interoperability.py index 20503e8310e..e6fbc8e7d10 100644 --- a/cuda_bindings_13/tests/test_interoperability.py +++ b/cuda_bindings_13/tests/test_interoperability.py @@ -3,26 +3,15 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.runtime as cudart from cuda.bindings._v2 import driver -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - # NULL dependency array: the node has no dependencies. NO_DEPS = 0 -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - def test_interop_stream(): # DRV to RT stream = driver.stream_create(0) @@ -86,7 +75,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") @pytest.mark.agent_authored(model="claude-sonnet-5.5") -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): # Use a small, capped pool instead of the device's default pool. The pool # must be set before it is queried below because the getters create (and diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md index 8bf0a6c7162..732ad1baad5 100644 --- a/cuda_core/tests/AGENTS.md +++ b/cuda_core/tests/AGENTS.md @@ -22,8 +22,10 @@ from helpers.constants import POOL_SIZE mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) ``` -Use a larger value only if a test genuinely requires it, and prefer adding a -shared constant to `helpers/constants.py` over redefining one per module. +`helpers/constants.py` re-exports `POOL_SIZE` from +`cuda_python_test_helpers.memory`, where the `cuda.bindings` tests import it +too. Use a larger value only if a test genuinely requires it, and prefer adding +a shared constant over redefining one per module. ### Passing no options is different from passing empty options @@ -105,6 +107,10 @@ Follow these rules when adding or moving shared test code: when the test needs to communicate with a running child or inspect its lifetime; use `kill_subprocesses` before assertions and `track_child_processes` for fixture cleanup so a stuck child cannot block IPC resource teardown. +- Use the skip guards in `helpers/memory.py` instead of per-module copies: + `skip_if_mempool_unsupported`, `skip_if_managed_mempool_unsupported` (a + property check only; it never constructs a managed resource), and + `is_pool_resource` to tell pool-backed resources from VMM ones. - Search `tests/helpers/` and `cuda_python_test_helpers/` for prior art before adding a new helper; consolidate duplicates across diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 16d44fb6034..041790a25eb 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -12,6 +12,7 @@ import pytest from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_managed_mempool_unsupported, skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -41,17 +42,6 @@ ALLOC_SIZE = 1024 -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - -def _skip_if_no_managed_mempool(): - _skip_if_no_mempool() - if not Device(0).properties.concurrent_managed_access: - pytest.skip("Device does not support managed memory pool operations") - - def _has_node_get_params(): from cuda.core._utils.version import BUILD_CUDA_MAJOR, driver_version @@ -204,13 +194,13 @@ def _build_disconnected(): @pytest.fixture(params=_ALL_BUILDERS) def graph_spec(request, init_cuda): if request.param is not _build_empty: - _skip_if_no_mempool() + skip_if_mempool_unsupported() return request.param() @pytest.fixture(params=_NONEMPTY_BUILDERS) def nonempty_graph_spec(request, init_cuda): - _skip_if_no_mempool() + skip_if_mempool_unsupported() return request.param() @@ -276,7 +266,7 @@ def _build_alloc_node(g): def _build_alloc_managed_node(g): - _skip_if_no_managed_mempool() + skip_if_managed_mempool_unsupported() device_id = Device().device_id entry = g.allocate(ALLOC_SIZE) node = entry.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.MANAGED) @@ -564,7 +554,7 @@ def _build_switch_node(g): def node_spec(request, init_cuda): spec = request.param if spec.needs_mempool: - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() node, expected_attrs = spec.builder(g) return spec, g, node, expected_attrs @@ -839,7 +829,7 @@ def test_graphdef_entry_is_virtual(init_cuda): def test_alloc_zero_size_fails(init_cuda): """Alloc with zero size raises error (CUDA limitation).""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError with pytest.raises(CUDAError): @@ -849,7 +839,7 @@ def test_alloc_zero_size_fails(init_cuda): def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) free = alloc.deallocate(alloc.dptr) assert alloc in free.pred @@ -858,7 +848,7 @@ def test_free_creates_dependency(init_cuda): def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() a1 = sample_graphdef.allocate(ALLOC_SIZE) a2 = a1.allocate(ALLOC_SIZE) f2 = a2.deallocate(a2.dptr) @@ -890,7 +880,7 @@ def test_alloc_memory_type_invalid(init_cuda): def test_alloc_device_option(init_cuda, device_spec): """Device can be specified as int or Device object.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() device = Device() node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device)) assert node.dptr != 0 @@ -907,7 +897,7 @@ def test_alloc_peer_access(mempool_device_x2): @pytest.mark.agent_authored(model="gpt-5.6-sol") def test_alloc_memory_type_host(init_cuda): """HOST graph alloc nodes reconstruct memory_type from the driver, not the Python argument.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError g = GraphDefinition() @@ -940,7 +930,7 @@ def test_alloc_memory_type_host(init_cuda): def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] joined = sample_graphdef.join(*branches) assert isinstance(joined, EmptyNode) @@ -1037,7 +1027,7 @@ def test_instantiate_empty_graph(init_cuda, inst_kwargs): def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) sample_graphdef.allocate(ALLOC_SIZE) graph = _instantiate(sample_graphdef, inst_kwargs) @@ -1081,7 +1071,7 @@ def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs): def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) alloc.deallocate(alloc.dptr) @@ -1095,7 +1085,7 @@ def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE) ms.deallocate(alloc.dptr) @@ -1110,7 +1100,7 @@ def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): """Graph with alloc/memset/memcpy/free can be executed and data is copied.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes src_alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1336,7 +1326,7 @@ def test_instantiate_and_execute_if_then(init_cuda): """If-conditional node: body executes only when condition is non-zero.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1369,7 +1359,7 @@ def test_instantiate_and_execute_if_else(init_cuda): """If-else node: then or else branch executes based on condition.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1404,7 +1394,7 @@ def test_instantiate_and_execute_switch(init_cuda): """Switch node: selected branch executes based on condition value.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1456,7 +1446,7 @@ def test_debug_dot_print_creates_file(init_cuda, tmp_path): """debug_dot_print writes a DOT file.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) sample_graphdef.debug_dot_print(str(dot_file)) assert dot_file.exists() @@ -1468,7 +1458,7 @@ def test_debug_dot_print_with_options(init_cuda, tmp_path): """debug_dot_print accepts GraphDebugPrintOptions.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) options = GraphDebugPrintOptions(verbose=True, handles=True) sample_graphdef.debug_dot_print(str(dot_file), options) @@ -1479,7 +1469,7 @@ def test_debug_dot_print_invalid_options(init_cuda, tmp_path): """debug_dot_print rejects invalid options type.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) with pytest.raises(TypeError, match="options must be a GraphDebugPrintOptions"): sample_graphdef.debug_dot_print(str(dot_file), "invalid") diff --git a/cuda_core/tests/graph/test_graph_definition_errors.py b/cuda_core/tests/graph/test_graph_definition_errors.py index 5069dacf4b8..46ad533741d 100644 --- a/cuda_core/tests/graph/test_graph_definition_errors.py +++ b/cuda_core/tests/graph/test_graph_definition_errors.py @@ -7,6 +7,7 @@ import pytest from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -20,11 +21,6 @@ SIZEOF_INT = ctypes.sizeof(ctypes.c_int) -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # ============================================================================= # Type validation — wrong types for conditional node methods # ============================================================================= @@ -85,7 +81,7 @@ def test_free_null_pointer(init_cuda): def test_memset_invalid_value_size(init_cuda): """memset with 3-byte value (not 1, 2, or 4) raises ValueError.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() alloc = g.allocate(1024) with pytest.raises(ValueError): @@ -129,7 +125,7 @@ def test_empty_node(init_cuda): def test_join_single_predecessor(init_cuda): """node.join() with no extra args creates a single-dep empty node.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() a = g.allocate(1024) joined = a.join() @@ -152,7 +148,7 @@ def test_multiple_instantiation(init_cuda): def test_unmatched_alloc_succeeds(init_cuda): """Alloc without corresponding free is valid (graph-scoped lifetime).""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() g.allocate(1024) graph = g.instantiate() @@ -184,7 +180,7 @@ def _skip_unless_cc_90(): def test_while_loop_zero_iterations(init_cuda): """While loop with default_value=0 never executes its body.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") @@ -212,7 +208,7 @@ def test_while_loop_zero_iterations(init_cuda): def test_if_then_false_skips_body(init_cuda): """If conditional with default_value=0 does not execute its body.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") @@ -240,7 +236,7 @@ def test_if_then_false_skips_body(init_cuda): def test_switch_oob_skips_all_branches(init_cuda): """Switch with out-of-range condition value does not execute any branch.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") diff --git a/cuda_core/tests/graph/test_graph_definition_integration.py b/cuda_core/tests/graph/test_graph_definition_integration.py index a352ffba2de..5a74b739679 100644 --- a/cuda_core/tests/graph/test_graph_definition_integration.py +++ b/cuda_core/tests/graph/test_graph_definition_integration.py @@ -8,6 +8,7 @@ import numpy as np import pytest from helpers.graph_kernels import skip_if_nvrtc_lacks_conditional_handle +from helpers.memory import skip_if_mempool_unsupported from cuda.core import Device, EventOptions, LaunchConfig, Program, ProgramOptions from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return @@ -17,11 +18,6 @@ SIZEOF_INT = 4 -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # =================================================================== # Kernel sources # =================================================================== @@ -200,7 +196,7 @@ def test_heat_diffusion(init_cuda): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_heat, k_countdown = _compile_heat_kernels() @@ -310,7 +306,7 @@ def test_bisection_root(init_cuda): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_eval, k_hi, k_lo, k_cd, k_check, k_newton = _compile_bisect_kernels() @@ -412,7 +408,7 @@ def test_switch_dispatch(init_cuda, mode, expected): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_negate, k_double, k_square = _compile_switch_kernels() diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index e049d76f500..5a3a99b65ea 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -13,7 +13,7 @@ import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet from helpers.graph_kernels import compile_common_kernels -from helpers.memory import VMM_HANDLE_TYPE +from helpers.memory import VMM_HANDLE_TYPE, skip_if_managed_mempool_unsupported, skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda_python_test_helpers import under_compute_sanitizer @@ -102,13 +102,8 @@ def _wait_until(predicate, timeout=None, interval=0.02): from cuda.core.typing import GraphMemoryType -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - def _device_memory_resource(dev): - _skip_if_no_mempool() + skip_if_mempool_unsupported() return DeviceMemoryResource(dev) @@ -126,12 +121,6 @@ def _virtual_memory_resource(dev): ] -def _skip_if_no_managed_mempool(): - _skip_if_no_mempool() - if not Device(0).properties.concurrent_managed_access: - pytest.skip("Device does not support managed memory pool operations") - - # ============================================================================= # Conditional body graph lifetime # ============================================================================= @@ -592,7 +581,7 @@ def callback(): def test_event_record_node_keeps_event_alive(init_cuda): """EventRecordNode should keep the Event alive after original is deleted.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() g = GraphDefinition() alloc = g.allocate(1024) @@ -609,7 +598,7 @@ def test_event_record_node_keeps_event_alive(init_cuda): def test_event_wait_node_keeps_event_alive(init_cuda): """EventWaitNode should keep the Event alive after original is deleted.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() g = GraphDefinition() alloc = g.allocate(1024) @@ -1305,7 +1294,7 @@ def _pred_chain_memcpy_host(g, bufs): def _pred_chain_memcpy_device(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = g.allocate(8) dst = g.allocate(8) return _make_pred_chain_memcpy(g.join(src, dst), src.dptr, dst.dptr) @@ -1336,7 +1325,7 @@ def check(reconstructed): def _pred_chain_alloc_peer_access(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() device = Device() node = g.allocate(64, device=device, peer_access=[device]) expected_peer_access = (device.device_id,) @@ -1350,7 +1339,7 @@ def check(reconstructed): def _pred_chain_alloc_managed(g, bufs): - _skip_if_no_managed_mempool() + skip_if_managed_mempool_unsupported() node = g.allocate(64, memory_type=GraphMemoryType.MANAGED) expected_dptr = node.dptr succ = node.record(Device().create_event()) @@ -1365,7 +1354,7 @@ def check(reconstructed): def _pred_chain_free(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = g.allocate(64) node = alloc.deallocate(alloc.dptr) free_dptr = node.dptr @@ -1435,7 +1424,7 @@ def test_kernel_args_buffer_lifetime(init_cuda): """ from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) add_one = compile_common_kernels().get_kernel("add_one") @@ -1472,7 +1461,7 @@ def test_kernel_args_survive_graph_clone(init_cuda): """ from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) add_one = compile_common_kernels().get_kernel("add_one") @@ -1510,7 +1499,7 @@ def test_memset_buffer_lifetime(init_cuda): """Memset retains the Buffer allocation after the wrapper is collected.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1537,7 +1526,7 @@ def test_memset_buffer_survives_close(init_cuda): """Memset retains the allocation when the Buffer wrapper is closed.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1562,7 +1551,7 @@ def test_memcpy_buffer_lifetime(init_cuda): """Memcpy retains operand allocations after the Buffer wrappers are collected.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1591,7 +1580,7 @@ def test_memcpy_buffer_survives_close(init_cuda): """Memcpy retains allocations when Buffer wrappers are closed.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1627,7 +1616,7 @@ def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda): """ from cuda.core._utils._weak_handles import weak_handle - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1654,7 +1643,7 @@ def test_memcpy_buffers_survive_graph_clone(init_cuda): """Cloned graph keeps memcpy operand allocations alive via CUDA user objects.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1692,7 +1681,7 @@ def test_memset_raw_ptr_with_dst_owner(init_cuda): """Raw dst plus Buffer dst_owner retains the allocation after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1722,7 +1711,7 @@ def test_attachment_owners_released_after_graph_destroyed(init_cuda): This exercises the same teardown that releases Buffer device-pointer handles retained for ``dst`` and ``src``. """ - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(8, stream=dev.default_stream) @@ -1751,7 +1740,7 @@ def test_attachment_owners_released_after_graph_destroyed(init_cuda): @pytest.mark.agent_authored(model="gpt-5.6") def test_deleted_memcpy_bundle_survives_in_existing_exec(init_cuda): """Both memcpy owners remain together until an existing exec releases them.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(8, stream=dev.default_stream) @@ -1782,7 +1771,7 @@ def test_memcpy_raw_ptrs_with_owners(init_cuda): """Raw src/dst plus Buffer owners retain allocations after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1811,7 +1800,7 @@ def test_memcpy_mixed_buffer_and_raw_owner(init_cuda): """Buffer dst and raw src with src_owner retain allocations after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1838,7 +1827,7 @@ def test_memcpy_mixed_buffer_and_raw_owner(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memset_closed_buffer_rejected(init_cuda): """Memset rejects a closed Buffer.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1853,7 +1842,7 @@ def test_memset_closed_buffer_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memset_closed_buffer_dst_owner_rejected(init_cuda): """Memset rejects a closed Buffer passed as dst_owner.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1869,7 +1858,7 @@ def test_memset_closed_buffer_dst_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_closed_buffer_src_owner_rejected(init_cuda): """Memcpy rejects a closed Buffer passed as src_owner.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1885,7 +1874,7 @@ def test_memcpy_closed_buffer_src_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_and_dst_owner_rejected(init_cuda): """dst_owner cannot be combined with a Buffer dst operand.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1899,7 +1888,7 @@ def test_memcpy_buffer_and_dst_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_and_src_owner_rejected(init_cuda): """src_owner cannot be combined with a Buffer src operand.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) diff --git a/cuda_core/tests/helpers/constants.py b/cuda_core/tests/helpers/constants.py index f4ea61b1938..6a9459ec587 100644 --- a/cuda_core/tests/helpers/constants.py +++ b/cuda_core/tests/helpers/constants.py @@ -3,12 +3,8 @@ """Constants shared across the cuda_core test suite.""" -# Cap for memory pools created by tests. A pool created without an explicit -# max_size instead reserves a system-dependent window that scales with -# installed device memory -- hundreds of GiB on large-memory GPUs. The -# per-process virtual address budget is bounded (~1 TB on Windows MCDM), and a -# reservation is not returned until the pool is torn down and its -# stream-ordered frees retire, so oversized windows accumulate across a session -# and eventually starve later pool creations with CUDA_ERROR_OUT_OF_MEMORY -# (issue #2381). See AGENTS.md in the tests directory. -POOL_SIZE = 2097152 # 2 MiB +# Cap for memory pools created by tests; see cuda_python_test_helpers.memory +# for why it matters (issue #2381) and AGENTS.md in the tests directory. +from cuda_python_test_helpers.memory import POOL_SIZE + +__all__ = ["POOL_SIZE"] diff --git a/cuda_core/tests/helpers/memory.py b/cuda_core/tests/helpers/memory.py index 9f7de044bf5..77a879189a9 100644 --- a/cuda_core/tests/helpers/memory.py +++ b/cuda_core/tests/helpers/memory.py @@ -5,7 +5,7 @@ import pytest -from cuda.core import Device, ManagedMemoryResource, PinnedMemoryResource +from cuda.core import Device, DeviceMemoryResource, ManagedMemoryResource, PinnedMemoryResource from cuda.core._utils.cuda_utils import CUDAError from cuda_python_test_helpers import IS_WINDOWS @@ -21,6 +21,33 @@ class BaselineMemoryResourceReleasedWarning(Warning): """An ``init_cuda`` test released an owned memory pool or VA reservation that existed before it started.""" +def skip_if_mempool_unsupported(device=None): + """Skip unless ``device`` (default: device 0) supports memory pools.""" + if device is None: + device = Device(0) + if not device.properties.memory_pools_supported: + pytest.skip("Device does not support mempool operations") + + +def skip_if_managed_mempool_unsupported(device=None): + """Skip unless ``device`` (default: device 0) supports managed memory pools. + + This is a property check only. Unlike ``skip_if_managed_memory_unsupported`` + it never constructs a ``ManagedMemoryResource``, so it is also usable by + tests that allocate managed memory through graph nodes. + """ + if device is None: + device = Device(0) + skip_if_mempool_unsupported(device) + if not device.properties.concurrent_managed_access: + pytest.skip("Device does not support managed memory pool operations") + + +def is_pool_resource(mr): + """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, uuid).""" + return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) + + def skip_if_pinned_memory_unsupported(device): try: if not device.properties.host_memory_pools_supported: diff --git a/cuda_core/tests/memory/test_resource_live_count.py b/cuda_core/tests/memory/test_resource_live_count.py index 75cc6d956ca..892f11395c7 100644 --- a/cuda_core/tests/memory/test_resource_live_count.py +++ b/cuda_core/tests/memory/test_resource_live_count.py @@ -21,7 +21,7 @@ import pytest from helpers.constants import POOL_SIZE -from helpers.memory import VMM_HANDLE_TYPE +from helpers.memory import VMM_HANDLE_TYPE, skip_if_mempool_unsupported from cuda.core import ( DeviceMemoryResource, @@ -34,11 +34,6 @@ _THREAD_UNSAFE_MARKER = pytest.mark.thread_unsafe(reason="asserts on the process-global owned-resource counts") -def _skip_without_mempools(device): - if not device.properties.memory_pools_supported: - pytest.skip("Device does not support memory pools") - - def _owned_pool(device): """An owning DeviceMemoryResource: passing options creates its own pool.""" return DeviceMemoryResource(device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) @@ -47,7 +42,7 @@ def _owned_pool(device): @_THREAD_UNSAFE_MARKER @pytest.mark.agent_authored(model="claude-opus-4.8") def test_owned_pool_is_counted_until_freed(init_cuda): - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() mr = _owned_pool(init_cuda) assert _live_owned_mempool_count() == before + 1 @@ -61,7 +56,7 @@ def test_borrowed_default_pool_is_not_counted(init_cuda): # A no-options DeviceMemoryResource borrows the device default pool # (cuDeviceGetMemPool), which this layer does not own or destroy, so it must # not move the owned-pool counter. - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() mr = DeviceMemoryResource(init_cuda) assert _live_owned_mempool_count() == before @@ -72,7 +67,7 @@ def test_borrowed_default_pool_is_not_counted(init_cuda): @_THREAD_UNSAFE_MARKER @pytest.mark.agent_authored(model="claude-opus-4.8") def test_owned_pool_in_cycle_stays_counted_until_collect(init_cuda): - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) # Disable automatic collection so a background gc cannot free the cycle # before the still-counted assertion is checked. enabled = gc.isenabled() @@ -116,7 +111,7 @@ def test_concurrent_owned_pool_construct_and_drop_returns_to_start(init_cuda): # On a free-threaded build this guards against a non-atomic counter; on a # GIL build it passes trivially. Every thread sets the device current, then # creates and drops owning pools; after join the count is back at the start. - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() n_threads = 4 n_per_thread = 10 diff --git a/cuda_core/tests/memory_ipc/test_errors.py b/cuda_core/tests/memory_ipc/test_errors.py index dd03c59d773..dffb410134f 100644 --- a/cuda_core/tests/memory_ipc/test_errors.py +++ b/cuda_core/tests/memory_ipc/test_errors.py @@ -11,6 +11,7 @@ import pytest from helpers.child_processes import child_timeout_sec, kill_subprocesses from helpers.constants import POOL_SIZE +from helpers.memory import is_pool_resource from cuda.core import ( Buffer, @@ -32,11 +33,6 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, is_mapped).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - def test_outer_timeout_marker_is_applied(request): """Verify that memory_ipc/conftest.py applies the outer pytest-timeout marker. @@ -64,7 +60,7 @@ def test_ipc_types_cannot_be_constructed_directly(): def test_import_truncated_buffer_descriptor(ipc_device, ipc_memory_resource): """Truncated IPC buffer descriptor payload is rejected before driver import.""" desc = IPCBufferDescriptor._init(b"\x00" * 8, NBYTES) - if _is_pool(ipc_memory_resource): + if is_pool_resource(ipc_memory_resource): expected = pytest.raises(ValueError, match=r"payload is 8 bytes; expected at least 64") else: # A pool descriptor cannot be imported with a VirtualMemoryResource at all. @@ -114,7 +110,7 @@ def test_ipc_allocation_handle_state_tracks_close(): @pytest.mark.agent_authored(model="gpt-5.6") def test_closed_ipc_allocation_handle_rejected_before_registry_hit(ipc_device, ipc_memory_resource): mr = ipc_memory_resource - if not _is_pool(mr): + if not is_pool_resource(mr): pytest.skip("allocation handles and the registry are pool-only") handle = IPCAllocationHandle._init(os.dup(int(mr.allocation_handle)), mr.uuid) assert mr.register(mr.uuid) is mr @@ -149,7 +145,7 @@ def test_main(self, ipc_device, ipc_memory_resource): self.device = ipc_device self.mr = ipc_memory_resource self._extra_mrs = [] - if self.POOL_ONLY and not _is_pool(self.mr): + if self.POOL_ONLY and not is_pool_resource(self.mr): pytest.skip("this scenario needs the pool half of the IPC protocol") # Clear stale 'buffer' left by a previous flaky-rerun attempt. # Its backing DeviceMemoryResource may already be closed, which causes @@ -210,7 +206,7 @@ def PARENT_ACTION(self, queue): stream = self.device.default_stream self.buffer = self.mr.allocate(NBYTES, stream=stream) desc = self.buffer.ipc_descriptor - if _is_pool(self.mr): + if is_pool_resource(self.mr): payload, _ = desc.__reduce__()[1] oversized = IPCBufferDescriptor._init(payload, NBYTES * 100) else: @@ -227,7 +223,7 @@ def CHILD_ACTION(self, queue): def ASSERT(self, exc_type, exc_msg): assert exc_type is ValueError assert "exceeds" in exc_msg - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert "mapped allocation extent" in exc_msg else: assert "exported allocations" in exc_msg @@ -245,7 +241,7 @@ def CHILD_ACTION(self, queue): buffer.close() def ASSERT(self, exc_type, exc_msg): - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert exc_type is TypeError assert exc_msg == "Cannot allocate from a mapped IPC-enabled memory resource" else: @@ -258,7 +254,7 @@ class TestImportWrongMR(ChildErrorHarness): """Importing a buffer with a resource other than the exporter's: an error for pools, fine for VMM.""" def PARENT_ACTION(self, queue): - if _is_pool(self.mr): + if is_pool_resource(self.mr): options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr2 = DeviceMemoryResource(self.device, options=options) self._extra_mrs.append(mr2) @@ -274,7 +270,7 @@ def CHILD_ACTION(self, queue): Buffer.from_ipc_descriptor(mr, buffer_desc, stream=self.device.default_stream).close() def ASSERT(self, exc_type, exc_msg): - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert exc_type is CUDAError assert "CUDA_ERROR_INVALID_VALUE" in exc_msg else: diff --git a/cuda_core/tests/memory_ipc/test_memory_ipc.py b/cuda_core/tests/memory_ipc/test_memory_ipc.py index 477abc93da9..34b9de73405 100644 --- a/cuda_core/tests/memory_ipc/test_memory_ipc.py +++ b/cuda_core/tests/memory_ipc/test_memory_ipc.py @@ -6,8 +6,9 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.memory import is_pool_resource -from cuda.core import Buffer, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource +from cuda.core import Buffer, DeviceMemoryResource, VirtualMemoryResource CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 @@ -19,14 +20,9 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, is_mapped).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - def _share_token(mr): """What a child needs to rebuild ``mr``: a pool's allocation handle, or the VMM resource itself.""" - return mr.allocation_handle if _is_pool(mr) else mr + return mr.allocation_handle if is_pool_resource(mr) else mr def _import_token(device, token): @@ -43,7 +39,7 @@ def test_main(self, ipc_device, ipc_memory_resource): # Set up the IPC-enabled memory pool and share it. device = ipc_device mr = ipc_memory_resource - if _is_pool(mr): + if is_pool_resource(mr): assert not mr.is_mapped stream = device.default_stream pgen = PatternGen(device, NBYTES, stream=stream) @@ -75,7 +71,7 @@ def test_main(self, ipc_device, ipc_memory_resource): def child_main(self, device, mr, queue): device.set_current() - if _is_pool(mr): + if is_pool_resource(mr): assert mr.is_mapped buffer = queue.get(timeout=CHILD_TIMEOUT_SEC) assert buffer.is_mapped diff --git a/cuda_core/tests/memory_ipc/test_serialize.py b/cuda_core/tests/memory_ipc/test_serialize.py index a44f480a8df..46858df1339 100644 --- a/cuda_core/tests/memory_ipc/test_serialize.py +++ b/cuda_core/tests/memory_ipc/test_serialize.py @@ -8,6 +8,7 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.memory import is_pool_resource from cuda.core import Buffer, Device, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource @@ -19,11 +20,6 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, uuid).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - class TestObjectSerializationDirect: """ Test the low-level interface for sharing memory resources. @@ -46,8 +42,8 @@ def test_main(self, ipc_device, ipc_memory_resource): # Send a memory resource: a pool by its allocation handle over the raw # fd channel, a VirtualMemoryResource as the object itself (it holds no # driver object; a VMM descriptor carries its fds per allocation). - parent_conn.send(_is_pool(mr)) - if _is_pool(mr): + parent_conn.send(is_pool_resource(mr)) + if is_pool_resource(mr): alloc_handle = mr.allocation_handle mp.reduction.send_handle(parent_conn, alloc_handle.handle, process.pid) else: @@ -151,7 +147,7 @@ def child_main(self, pipe, _): # Buffer. buffer = pipe[0].get(timeout=CHILD_TIMEOUT_SEC) - if _is_pool(mr): + if is_pool_resource(mr): assert buffer.memory_resource.handle == mr.handle else: # A VMM resource pickles as (device, options); the buffer's rebuilt @@ -178,7 +174,7 @@ def test_main(self, ipc_device, ipc_memory_resource): # Define the objects. device = ipc_device mr = ipc_memory_resource - alloc_handle = mr.allocation_handle if _is_pool(mr) else None + alloc_handle = mr.allocation_handle if is_pool_resource(mr) else None stream = device.default_stream buffer = mr.allocate(NBYTES, stream=stream) buffer_desc = buffer.ipc_descriptor diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index 022ee456fd9..1e1e410f807 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -15,6 +15,7 @@ import pytest from helpers.constants import POOL_SIZE from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import ( @@ -31,12 +32,6 @@ from cuda.core._program import _can_load_generated_ptx from cuda.core.graph import GraphDefinition - -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # ============================================================================= # Fixtures - Primary samples # ============================================================================= @@ -280,7 +275,7 @@ def sample_root_node_alt(sample_graphdef_alt): @pytest.fixture def sample_empty_node(sample_graphdef): """An EmptyNode created by merging two branches.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() a = sample_graphdef.allocate(ALLOC_SIZE) b = sample_graphdef.allocate(ALLOC_SIZE) return sample_graphdef.join(a, b) @@ -289,7 +284,7 @@ def sample_empty_node(sample_graphdef): @pytest.fixture def sample_empty_node_alt(sample_graphdef): """An alternate EmptyNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() c = sample_graphdef.allocate(ALLOC_SIZE) d = sample_graphdef.allocate(ALLOC_SIZE) return sample_graphdef.join(c, d) @@ -298,14 +293,14 @@ def sample_empty_node_alt(sample_graphdef): @pytest.fixture def sample_alloc_node(sample_graphdef): """An AllocNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() return sample_graphdef.allocate(ALLOC_SIZE) @pytest.fixture def sample_alloc_node_alt(sample_graphdef): """An alternate AllocNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() return sample_graphdef.allocate(ALLOC_SIZE) @@ -330,7 +325,7 @@ def sample_kernel_node_alt(sample_graphdef, init_cuda): @pytest.fixture def sample_free_node(sample_graphdef): """A FreeNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.deallocate(alloc.dptr) @@ -338,7 +333,7 @@ def sample_free_node(sample_graphdef): @pytest.fixture def sample_free_node_alt(sample_graphdef): """An alternate FreeNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.deallocate(alloc.dptr) @@ -346,7 +341,7 @@ def sample_free_node_alt(sample_graphdef): @pytest.fixture def sample_memset_node(sample_graphdef): """A MemsetNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @@ -354,7 +349,7 @@ def sample_memset_node(sample_graphdef): @pytest.fixture def sample_memset_node_alt(sample_graphdef): """An alternate MemsetNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @@ -362,7 +357,7 @@ def sample_memset_node_alt(sample_graphdef): @pytest.fixture def sample_memcpy_node(sample_graphdef): """A MemcpyNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = sample_graphdef.allocate(ALLOC_SIZE) dst = sample_graphdef.allocate(ALLOC_SIZE) dep = sample_graphdef.join(src, dst) @@ -372,7 +367,7 @@ def sample_memcpy_node(sample_graphdef): @pytest.fixture def sample_memcpy_node_alt(sample_graphdef): """An alternate MemcpyNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = sample_graphdef.allocate(ALLOC_SIZE) dst = sample_graphdef.allocate(ALLOC_SIZE) dep = sample_graphdef.join(src, dst) diff --git a/cuda_core/tests/test_rlcompleter_patch.py b/cuda_core/tests/test_rlcompleter_patch.py index 92a77efee8e..de7306a4b10 100644 --- a/cuda_core/tests/test_rlcompleter_patch.py +++ b/cuda_core/tests/test_rlcompleter_patch.py @@ -22,6 +22,7 @@ import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet +from helpers.memory import skip_if_mempool_unsupported from cuda.core import Device @@ -30,9 +31,7 @@ def _gpu_with_mempool_or_skip(): """Skip when no GPU or no mempool support — test mirrors the bug repro.""" if len(Device.get_all_devices()) == 0: pytest.skip("Test requires a CUDA device") - dev = Device(0) - if not dev.properties.memory_pools_supported: - pytest.skip("Device 0 does not support mempool operations") + skip_if_mempool_unsupported() # Probe script: reproduces the bug-report repro literally, then runs diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/memory.py b/cuda_python_test_helpers/cuda_python_test_helpers/memory.py new file mode 100644 index 00000000000..cb4ca161f12 --- /dev/null +++ b/cuda_python_test_helpers/cuda_python_test_helpers/memory.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Memory-pool helpers shared by the cuda.bindings and cuda.core test suites.""" + +import pytest + +from cuda.bindings import runtime as cudart + +__all__ = [ + "POOL_SIZE", + "supports_managed_memory", + "supports_memory_pools", + "xfail_graph_mempool_oom", +] + +# Cap for memory pools created by tests. A pool created without an explicit +# maximum size instead reserves a system-dependent window that scales with +# installed device memory: about twice the device memory, which is hundreds of +# GiB on large-memory GPUs. That reservation can fail where the virtual address +# space is limited (e.g. riscv64 Sv39, a 39-bit address space). The per-process +# virtual address budget is also bounded (~1 TB on Windows MCDM), and a +# reservation is not returned until the pool is torn down and its +# stream-ordered frees retire, so oversized windows accumulate across a session +# and eventually starve later pool creations with CUDA_ERROR_OUT_OF_MEMORY +# (issue #2381). +POOL_SIZE = 2 * 1024 * 1024 # 2 MiB + + +def _device_attribute_is_set(attribute, device): + err, value = cudart.cudaDeviceGetAttribute(attribute, device) + return err == cudart.cudaError_t.cudaSuccess and bool(value) + + +def supports_memory_pools(device=0): + """Return True if ``device`` supports stream-ordered memory pools. + + A failing attribute query (e.g. no device) counts as unsupported, so this + is safe to evaluate in a ``skipif`` condition at collection time. + """ + return _device_attribute_is_set(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, device) + + +def supports_managed_memory(device=0): + """Return True if ``device`` supports managed memory. + + A failing attribute query (e.g. no device) counts as unsupported. + """ + return _device_attribute_is_set(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, device) + + +def xfail_graph_mempool_oom(): + """Mark the calling test as an expected failure; never returns. + + Call this after the graph memory pool failed with an out-of-memory error. + The graph memory pool reserves virtual address space of about twice the + device memory, and unlike pools from ``cuMemPoolCreate`` it cannot be + capped. On systems with a limited virtual address space this reservation + can fail with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped + pool already exists in the process. + + The caller does its own out-of-memory check and resource cleanup first. + """ + pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") From e954937b7c0b186da0eb536a2fe1d192fad68417 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 08:54:43 -0700 Subject: [PATCH 2/5] test(bindings): destroy the CUDA context on the graph-mempool OOM path cuda_bindings_12/tests/test_cuda.py::test_cuda_graphMem_attr creates its own context with cuCtxCreate. When cuGraphAddMemAllocNode fails with CUDA_ERROR_OUT_OF_MEMORY, the test destroys the graph and stream and then xfails, but never destroyed the context, so the context stayed current for the rest of the session. Destroy the context before xfailing. The test still xfails on this path; later tests no longer inherit the leaked context, which can change their outcomes. The bindings 13 v2 test and legacy_api/test_legacy_cuda.py use the device fixture and create no context of their own, so they are not affected. --- cuda_bindings_12/tests/test_cuda.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cuda_bindings_12/tests/test_cuda.py b/cuda_bindings_12/tests/test_cuda.py index 7c4388bd37b..09a77646b6f 100644 --- a/cuda_bindings_12/tests/test_cuda.py +++ b/cuda_bindings_12/tests/test_cuda.py @@ -560,6 +560,10 @@ def test_cuda_graphMem_attr(): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS + # xfail raises, so the context must be destroyed first or it leaks + # into later tests as the current context. + (destroy_err,) = cuda.cuCtxDestroy(ctx) + assert destroy_err == cuda.CUresult.CUDA_SUCCESS xfail_graph_mempool_oom() assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) From 0035e30f2c9d5ace625b86fde7d682af4bb86d46 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 08:54:51 -0700 Subject: [PATCH 3/5] ci: extend the uncapped-pool lint to the cuda.bindings tests check_mempool_hygiene.py only covered cuda_core/tests, so the uncapped pools fixed in #3140 and #3290 in the bindings tests could have been reintroduced unnoticed. * DEFAULT_TREES now covers cuda_core/tests, cuda_bindings_12/tests, and cuda_bindings_13/tests. * A new pool-properties rule, with its own error message, reports a MemPoolProps_v1, CUmemPoolProps, CUmemPoolProps_v1, CUmemPoolProps_st, or cudaMemPoolProps assigned to a name when the same function (or module, for module-level code) never assigns .max_size or .maxSize. The existing cuda_core rule and message are unchanged. The uncapped-pool-ok marker opts a construction out of either rule. * Graph alloc-node params.poolProps fields are not constructor calls and are not checked; that pool cannot be capped anyway. * .pyx tests are out of scope because ast cannot parse Cython; the module docstring and cuda_bindings_13/AGENTS.md say so. * The check-mempool-hygiene pre-commit hook now also runs on the bindings tests. All current .py tests pass the new rule. --- .pre-commit-config.yaml | 2 +- ci/tools/check_mempool_hygiene.py | 162 ++++++++++++++++--- ci/tools/tests/test_check_mempool_hygiene.py | 84 +++++++++- cuda_bindings_13/AGENTS.md | 10 ++ 4 files changed, 235 insertions(+), 23 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6f794ef58f..86bd18bed93 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,7 @@ repos: name: Check tests do not create uncapped memory pools entry: python ./ci/tools/check_mempool_hygiene.py language: python - files: '^cuda_core/tests/.*\.py$' + files: '^(cuda_core|cuda_bindings_1[23])/tests/.*\.py$' - id: no-markdown-in-docs-source name: Prevent markdown files in docs/source directories diff --git a/ci/tools/check_mempool_hygiene.py b/ci/tools/check_mempool_hygiene.py index b200aba1ebb..aa30924a42b 100644 --- a/ci/tools/check_mempool_hygiene.py +++ b/ci/tools/check_mempool_hygiene.py @@ -9,7 +9,22 @@ address space, after which the rest of the session fails with CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory. -See cuda_core/tests/AGENTS.md for the rule this enforces. +Two rules are enforced: + +* cuda_core: ``DeviceMemoryResource`` / ``PinnedMemoryResource`` and their + options classes must be given ``max_size``. +* cuda.bindings: a pool properties object (``MemPoolProps_v1``, + ``CUmemPoolProps``, ``cudaMemPoolProps``) assigned to a name must have + ``max_size`` / ``maxSize`` set in the same function (or at module level, for + module-level code). Graph alloc-node ``params.poolProps`` is a field, not a + constructor call, so it is not checked; that pool cannot be capped anyway. + +Only ``.py`` files are checked. ``ast`` cannot parse Cython, so ``.pyx`` tests +(e.g. ``cuda_bindings_*/tests/cython``) are out of scope and must be capped by +hand. + +See cuda_core/tests/AGENTS.md and cuda_bindings_13/AGENTS.md for the rules this +enforces. """ from __future__ import annotations @@ -17,18 +32,39 @@ import argparse import ast import sys +from collections.abc import Iterator from pathlib import Path +from typing import NamedTuple ROOT = Path(__file__).resolve().parents[2] -DEFAULT_TREE = ROOT / "cuda_core" / "tests" +DEFAULT_TREES = ( + ROOT / "cuda_core" / "tests", + ROOT / "cuda_bindings_12" / "tests", + ROOT / "cuda_bindings_13" / "tests", +) # Managed pools cannot be right-sized: cuMemPoolCreate requires maxSize == 0 for # managed pools, so ManagedMemoryResourceOptions has no max_size to set. CAPPABLE_OPTIONS = frozenset({"DeviceMemoryResourceOptions", "PinnedMemoryResourceOptions"}) CAPPABLE_RESOURCES = frozenset({"DeviceMemoryResource", "PinnedMemoryResource"}) +# Pool properties types of the cuda.bindings APIs; the size cap is a plain +# attribute, spelled ``max_size`` on the v2 types and ``maxSize`` on the others. +POOL_PROPS_TYPES = frozenset( + {"MemPoolProps_v1", "CUmemPoolProps", "CUmemPoolProps_v1", "CUmemPoolProps_st", "cudaMemPoolProps"} +) +POOL_PROPS_CAP_ATTRIBUTES = frozenset({"max_size", "maxSize"}) + OPT_OUT_MARKER = "uncapped-pool-ok" +RULE_RESOURCE = "resource" +RULE_PROPS = "props" + + +class Violation(NamedTuple): + rule: str + message: str + def _callee_name(node: ast.Call) -> str: func = node.func @@ -60,12 +96,71 @@ def _opted_out(lines: list[str], node: ast.AST) -> bool: return any(OPT_OUT_MARKER in line for line in lines[start:end]) -def violations_in(path: Path) -> list[str]: - """Return one message per uncapped pool construction in ``path``.""" - source = path.read_text(encoding="utf-8") - lines = source.splitlines() +def _walk_scope(scope: ast.AST) -> Iterator[ast.AST]: + """Yield the nodes of ``scope`` without entering nested functions, lambdas, or classes.""" + pending = list(ast.iter_child_nodes(scope)) + while pending: + node = pending.pop() + yield node + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)): + pending.extend(ast.iter_child_nodes(node)) + + +def _assigned_pool_props(node: ast.AST) -> tuple[str, str] | None: + """Return ``(variable, type name)`` for ``variable = (...)``.""" + if not (isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.value, ast.Call)): + return None + target = node.targets[0] + callee = _callee_name(node.value) + if isinstance(target, ast.Name) and callee in POOL_PROPS_TYPES: + return target.id, callee + return None + + +def _capped_variables(scope: ast.AST) -> set[str]: + """Names with ``name.max_size`` or ``name.maxSize`` assigned directly in ``scope``.""" + capped = set() + for node in _walk_scope(scope): + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + targets = [node.target] + else: + continue + capped.update( + target.value.id + for target in targets + if isinstance(target, ast.Attribute) + and target.attr in POOL_PROPS_CAP_ATTRIBUTES + and isinstance(target.value, ast.Name) + ) + return capped + + +def _pool_props_violations(path: Path, tree: ast.AST, lines: list[str]) -> list[Violation]: found = [] - for node in ast.walk(ast.parse(source, filename=str(path))): + scopes = [tree, *(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)))] + for scope in scopes: + capped = _capped_variables(scope) + for node in _walk_scope(scope): + assigned = _assigned_pool_props(node) + if assigned is None: + continue + variable, type_name = assigned + if variable not in capped and not _opted_out(lines, node): + found.append( + Violation( + RULE_PROPS, + f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{variable}' " + f"without {variable}.max_size / {variable}.maxSize", + ) + ) + return found + + +def _resource_violations(path: Path, tree: ast.AST, lines: list[str]) -> list[Violation]: + found = [] + for node in ast.walk(tree): if not isinstance(node, ast.Call): continue name = _callee_name(node) @@ -78,34 +173,61 @@ def violations_in(path: Path) -> list[str]: else: continue if uncapped and not _opted_out(lines, node): - found.append(f"{path.as_posix()}:{node.lineno}: {name} without max_size") + found.append(Violation(RULE_RESOURCE, f"{path.as_posix()}:{node.lineno}: {name} without max_size")) return found +def _violations(path: Path) -> list[Violation]: + source = path.read_text(encoding="utf-8") + lines = source.splitlines() + tree = ast.parse(source, filename=str(path)) + return [*_resource_violations(path, tree, lines), *_pool_props_violations(path, tree, lines)] + + +def violations_in(path: Path) -> list[str]: + """Return one message per uncapped pool construction in ``path``.""" + return [violation.message for violation in _violations(path)] + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "paths", nargs="*", type=Path, - help=f"Files to check. Defaults to every .py under {DEFAULT_TREE.relative_to(ROOT).as_posix()}.", + help="Files to check. Defaults to every .py under " + + ", ".join(tree.relative_to(ROOT).as_posix() for tree in DEFAULT_TREES) + + ".", ) args = parser.parse_args(argv) - paths = args.paths or sorted(DEFAULT_TREE.rglob("*.py")) - violations = sorted(v for path in paths if path.suffix == ".py" for v in violations_in(path)) + paths = args.paths or sorted(path for tree in DEFAULT_TREES for path in tree.rglob("*.py")) + violations = sorted(v for path in paths if path.suffix == ".py" for v in _violations(path)) if not violations: return 0 - print("error: memory pools created by tests must set max_size:", file=sys.stderr) - for violation in violations: - print(f" - {violation}", file=sys.stderr) - print( - f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" - f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" - f"See cuda_core/tests/AGENTS.md.", - file=sys.stderr, - ) + resource_messages = [v.message for v in violations if v.rule == RULE_RESOURCE] + props_messages = [v.message for v in violations if v.rule == RULE_PROPS] + if resource_messages: + print("error: memory pools created by tests must set max_size:", file=sys.stderr) + for message in resource_messages: + print(f" - {message}", file=sys.stderr) + print( + f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" + f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" + f"See cuda_core/tests/AGENTS.md.", + file=sys.stderr, + ) + if props_messages: + print("error: pool properties created by tests must set max_size / maxSize:", file=sys.stderr) + for message in props_messages: + print(f" - {message}", file=sys.stderr) + print( + f"Set max_size (v2 bindings) or maxSize (other bindings) on the props object you constructed,\n" + f"using POOL_SIZE from cuda_python_test_helpers.memory, or annotate a deliberate exception\n" + f"with a '# {OPT_OUT_MARKER}: ' comment. See cuda_bindings_13/AGENTS.md.", + file=sys.stderr, + ) return 1 diff --git a/ci/tools/tests/test_check_mempool_hygiene.py b/ci/tools/tests/test_check_mempool_hygiene.py index 5ff3562059b..14f0a1db51d 100644 --- a/ci/tools/tests/test_check_mempool_hygiene.py +++ b/ci/tools/tests/test_check_mempool_hygiene.py @@ -9,7 +9,7 @@ import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from check_mempool_hygiene import DEFAULT_TREE, main, violations_in +from check_mempool_hygiene import DEFAULT_TREES, main, violations_in def write(tmp_path, source): @@ -92,5 +92,85 @@ def test_main_ignores_non_python_files(tmp_path): def test_the_live_test_suite_is_clean(): # Without a default the hook would only ever see changed files, so a # violation could ride in on a rename or a merge. - assert DEFAULT_TREE.is_dir() + assert all(tree.is_dir() for tree in DEFAULT_TREES) assert main([]) == 0 + + +POOL_PROPS_CAPPED = [ + pytest.param("props = driver.MemPoolProps_v1()\nprops.max_size = POOL_SIZE\n", id="v2-max_size"), + pytest.param("props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", id="driver-maxSize"), + pytest.param("props = cudart.cudaMemPoolProps()\nprops.maxSize = POOL_SIZE\n", id="runtime-maxSize"), + pytest.param("def f():\n props = cuda.CUmemPoolProps_v1()\n props.maxSize = POOL_SIZE\n", id="in-function"), + # A field of an alloc-node params object is not a constructor call, so it is not checked. + pytest.param("params.poolProps.allocType = TYPE\nparams.pool_props = other\n", id="graph-params-fields"), +] + +POOL_PROPS_UNCAPPED = [ + pytest.param("props = driver.MemPoolProps_v1()\nprops.location = loc\n", id="v2"), + pytest.param("props = cuda.CUmemPoolProps()\n", id="driver"), + pytest.param("props = cudart.cudaMemPoolProps()\n", id="runtime"), + pytest.param( + "def f():\n props = cuda.CUmemPoolProps()\n other.maxSize = POOL_SIZE\n", + id="other-object", + ), + # The cap must be set in the function that constructed the props. + pytest.param( + "def make():\n return cuda.CUmemPoolProps()\n\n\ndef f():\n props = cuda.CUmemPoolProps()\n g(props)\n\n\ndef g(props):\n props.maxSize = POOL_SIZE\n", + id="capped-in-another-function", + ), +] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +@pytest.mark.parametrize("source", POOL_PROPS_CAPPED) +def test_capped_pool_props_are_not_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +@pytest.mark.parametrize("source", POOL_PROPS_UNCAPPED) +def test_uncapped_pool_props_are_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_pool_props_marker_opts_out(tmp_path): + source = "# uncapped-pool-ok: pool is never created\nprops = cuda.CUmemPoolProps()\n" + + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_pool_props_message_names_file_line_and_variable(tmp_path): + path = write(tmp_path, "x = 1\nprops = driver.MemPoolProps_v1()\n") + + (violation,) = violations_in(path) + + assert violation.startswith(path.as_posix()) + assert ":2:" in violation + assert "MemPoolProps_v1 assigned to 'props' without props.max_size / props.maxSize" in violation + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_main_reports_pool_props_with_their_own_message(tmp_path, capsys): + path = write(tmp_path, "props = cuda.CUmemPoolProps()\n") + + assert main([str(path)]) == 1 + + err = capsys.readouterr().err + assert "pool properties created by tests must set max_size / maxSize" in err + assert "must set max_size:" not in err + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_main_reports_both_rules_separately(tmp_path, capsys): + path = write( + tmp_path, + "DeviceMemoryResource(dev, DeviceMemoryResourceOptions())\nprops = cuda.CUmemPoolProps()\n", + ) + + assert main([str(path)]) == 1 + + err = capsys.readouterr().err + assert "memory pools created by tests must set max_size:" in err + assert "pool properties created by tests must set max_size / maxSize" in err diff --git a/cuda_bindings_13/AGENTS.md b/cuda_bindings_13/AGENTS.md index a5c92098ced..6b3a3d442b7 100644 --- a/cuda_bindings_13/AGENTS.md +++ b/cuda_bindings_13/AGENTS.md @@ -54,6 +54,16 @@ and `nvrtc`. These test files should not be added to, only updated when necessary to fix test failures. The canonical set of tests are those outside of the `tests/legacy_api` subdirectory. +Tests that create memory pools must cap them: set `max_size` (v2 types) or +`maxSize` (other types) on the pool properties object to `POOL_SIZE` from +`cuda_python_test_helpers.memory`. A pool created with a size of 0 reserves +virtual address space of about twice the device memory, which fails on systems +with a limited virtual address space. `ci/tools/check_mempool_hygiene.py` +(a pre-commit hook) enforces this for `.py` tests. It cannot parse Cython, so +cap pools in `tests/cython/*.pyx` by hand. The same module holds the shared +`supports_memory_pools`, `supports_managed_memory`, and +`xfail_graph_mempool_oom` helpers; reuse them rather than defining copies. + ## Build and environment notes - `CUDA_HOME` or `CUDA_PATH` must point to a valid CUDA Toolkit for source From 069e168af692869bd698ec4965385dc71bfefc37 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 09:09:21 -0700 Subject: [PATCH 4/5] ci: simplify the pool-props lint without changing behavior Fold the pool-props check into one recursive _pool_props_violations that walks each scope once: it collects the caps and the constructions in the same pass and recurses into nested scopes, replacing the separate _capped_variables and _assigned_pool_props helpers and the double walk. Add a _report helper so the two rule outputs no longer duplicate the print/bullet/hint block. Same rules, same messages; all lint tests pass. --- ci/tools/check_mempool_hygiene.py | 130 +++++++++++++++--------------- 1 file changed, 63 insertions(+), 67 deletions(-) diff --git a/ci/tools/check_mempool_hygiene.py b/ci/tools/check_mempool_hygiene.py index aa30924a42b..322c2bca9f2 100644 --- a/ci/tools/check_mempool_hygiene.py +++ b/ci/tools/check_mempool_hygiene.py @@ -97,7 +97,7 @@ def _opted_out(lines: list[str], node: ast.AST) -> bool: def _walk_scope(scope: ast.AST) -> Iterator[ast.AST]: - """Yield the nodes of ``scope`` without entering nested functions, lambdas, or classes.""" + """Yield ``scope``'s descendants, yielding but not entering nested function/lambda/class bodies.""" pending = list(ast.iter_child_nodes(scope)) while pending: node = pending.pop() @@ -106,55 +106,49 @@ def _walk_scope(scope: ast.AST) -> Iterator[ast.AST]: pending.extend(ast.iter_child_nodes(node)) -def _assigned_pool_props(node: ast.AST) -> tuple[str, str] | None: - """Return ``(variable, type name)`` for ``variable = (...)``.""" - if not (isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.value, ast.Call)): - return None - target = node.targets[0] - callee = _callee_name(node.value) - if isinstance(target, ast.Name) and callee in POOL_PROPS_TYPES: - return target.id, callee - return None +def _assign_targets(node: ast.AST) -> list[ast.expr]: + """The assignment targets of ``node``, or ``[]`` if it is not an assignment.""" + if isinstance(node, ast.Assign): + return node.targets + if isinstance(node, (ast.AugAssign, ast.AnnAssign)): + return [node.target] + return [] -def _capped_variables(scope: ast.AST) -> set[str]: - """Names with ``name.max_size`` or ``name.maxSize`` assigned directly in ``scope``.""" - capped = set() +def _pool_props_violations(path: Path, scope: ast.AST, lines: list[str]) -> list[Violation]: + """Report each ``var = PoolPropsType(...)`` with no ``var.max_size`` set in the same scope. + + Recurses so a cap only counts in the scope that built the props; a cap on a + same-named variable elsewhere does not excuse it. + """ + found: list[Violation] = [] + constructed: list[tuple[ast.Assign, str, str]] = [] # (node, variable, type name) + capped: set[str] = set() for node in _walk_scope(scope): - if isinstance(node, ast.Assign): - targets = node.targets - elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): - targets = [node.target] - else: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + found += _pool_props_violations(path, node, lines) continue - capped.update( - target.value.id - for target in targets - if isinstance(target, ast.Attribute) - and target.attr in POOL_PROPS_CAP_ATTRIBUTES - and isinstance(target.value, ast.Name) - ) - return capped - - -def _pool_props_violations(path: Path, tree: ast.AST, lines: list[str]) -> list[Violation]: - found = [] - scopes = [tree, *(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)))] - for scope in scopes: - capped = _capped_variables(scope) - for node in _walk_scope(scope): - assigned = _assigned_pool_props(node) - if assigned is None: - continue - variable, type_name = assigned - if variable not in capped and not _opted_out(lines, node): - found.append( - Violation( - RULE_PROPS, - f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{variable}' " - f"without {variable}.max_size / {variable}.maxSize", - ) + for target in _assign_targets(node): + if ( + isinstance(target, ast.Attribute) + and target.attr in POOL_PROPS_CAP_ATTRIBUTES + and isinstance(target.value, ast.Name) + ): + capped.add(target.value.id) + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.value, ast.Call): + (target,) = node.targets + type_name = _callee_name(node.value) + if isinstance(target, ast.Name) and type_name in POOL_PROPS_TYPES: + constructed.append((node, target.id, type_name)) + for node, variable, type_name in constructed: + if variable not in capped and not _opted_out(lines, node): + found.append( + Violation( + RULE_PROPS, + f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{variable}' " + f"without {variable}.max_size / {variable}.maxSize", ) + ) return found @@ -189,6 +183,16 @@ def violations_in(path: Path) -> list[str]: return [violation.message for violation in _violations(path)] +def _report(header: str, messages: list[str], hint: str) -> None: + """Print ``header``, the bulleted ``messages``, and ``hint`` to stderr; a no-op if empty.""" + if not messages: + return + print(header, file=sys.stderr) + for message in messages: + print(f" - {message}", file=sys.stderr) + print(hint, file=sys.stderr) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -206,28 +210,20 @@ def main(argv: list[str] | None = None) -> int: if not violations: return 0 - resource_messages = [v.message for v in violations if v.rule == RULE_RESOURCE] - props_messages = [v.message for v in violations if v.rule == RULE_PROPS] - if resource_messages: - print("error: memory pools created by tests must set max_size:", file=sys.stderr) - for message in resource_messages: - print(f" - {message}", file=sys.stderr) - print( - f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" - f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" - f"See cuda_core/tests/AGENTS.md.", - file=sys.stderr, - ) - if props_messages: - print("error: pool properties created by tests must set max_size / maxSize:", file=sys.stderr) - for message in props_messages: - print(f" - {message}", file=sys.stderr) - print( - f"Set max_size (v2 bindings) or maxSize (other bindings) on the props object you constructed,\n" - f"using POOL_SIZE from cuda_python_test_helpers.memory, or annotate a deliberate exception\n" - f"with a '# {OPT_OUT_MARKER}: ' comment. See cuda_bindings_13/AGENTS.md.", - file=sys.stderr, - ) + _report( + "error: memory pools created by tests must set max_size:", + [v.message for v in violations if v.rule == RULE_RESOURCE], + f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" + f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" + f"See cuda_core/tests/AGENTS.md.", + ) + _report( + "error: pool properties created by tests must set max_size / maxSize:", + [v.message for v in violations if v.rule == RULE_PROPS], + f"Set max_size (v2 bindings) or maxSize (other bindings) on the props object you constructed,\n" + f"using POOL_SIZE from cuda_python_test_helpers.memory, or annotate a deliberate exception\n" + f"with a '# {OPT_OUT_MARKER}: ' comment. See cuda_bindings_13/AGENTS.md.", + ) return 1 From 1c7ec398c6959dde1e5ba197776ea9b864a76933 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 09:46:51 -0700 Subject: [PATCH 5/5] ci: fix two pool-props lint gaps found in CodeRabbit review Finding 1 (minor): a type-annotated construction ``props: cuda.CUmemPoolProps = cuda.CUmemPoolProps()`` produces an ast.AnnAssign, not ast.Assign, so the rule never saw the constructor. Extend the construction detection to cover AnnAssign. Finding 2 (major): when the same variable is assigned twice, ``props = CUmemPoolProps()`` # capped ``props = CUmemPoolProps()`` # uncapped, but lint was silent the old cap remained in the shared `capped` set and excused the second construction. Fix: process nodes in source order and track the latest uncapped construction per variable name. When the variable is reassigned, the old cap is reset; the new construction must have its own cap. Any construction still pending at end-of-scope is reported. Also change _walk_scope to yield nodes in source order (previously reversed) so the forward-pass cap-state tracking is order-correct. Adds targeted tests for both findings. --- ci/tools/check_mempool_hygiene.py | 64 ++++++++++++++------ ci/tools/tests/test_check_mempool_hygiene.py | 43 +++++++++++++ 2 files changed, 90 insertions(+), 17 deletions(-) diff --git a/ci/tools/check_mempool_hygiene.py b/ci/tools/check_mempool_hygiene.py index 322c2bca9f2..f1837f2a919 100644 --- a/ci/tools/check_mempool_hygiene.py +++ b/ci/tools/check_mempool_hygiene.py @@ -97,13 +97,14 @@ def _opted_out(lines: list[str], node: ast.AST) -> bool: def _walk_scope(scope: ast.AST) -> Iterator[ast.AST]: - """Yield ``scope``'s descendants, yielding but not entering nested function/lambda/class bodies.""" - pending = list(ast.iter_child_nodes(scope)) + """Yield ``scope``'s descendants in source order, not entering nested function/lambda/class bodies.""" + # Push children reversed so the leftmost (earliest) child pops first. + pending = list(reversed(list(ast.iter_child_nodes(scope)))) while pending: node = pending.pop() yield node if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)): - pending.extend(ast.iter_child_nodes(node)) + pending.extend(reversed(list(ast.iter_child_nodes(node)))) def _assign_targets(node: ast.AST) -> list[ast.expr]: @@ -118,35 +119,64 @@ def _assign_targets(node: ast.AST) -> list[ast.expr]: def _pool_props_violations(path: Path, scope: ast.AST, lines: list[str]) -> list[Violation]: """Report each ``var = PoolPropsType(...)`` with no ``var.max_size`` set in the same scope. - Recurses so a cap only counts in the scope that built the props; a cap on a - same-named variable elsewhere does not excuse it. + Processes nodes in source order. When ``var`` is reassigned to a new + construction, any cap set on the previous object is reset; only a cap that + follows the new construction (in the same scope) excuses it. + + Recurses into nested scopes so a cap set in another function does not count. + + Handles both plain and type-annotated assignments (``ast.Assign`` and + ``ast.AnnAssign``). """ found: list[Violation] = [] - constructed: list[tuple[ast.Assign, str, str]] = [] # (node, variable, type name) - capped: set[str] = set() + # Maps variable name -> (construction node, type name) for the latest + # uncapped construction. Removed when the variable is capped; overwritten + # (with a report of the old entry if it was still pending) on reassignment. + pending: dict[str, tuple[ast.AST, str]] = {} + for node in _walk_scope(scope): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): found += _pool_props_violations(path, node, lines) continue + # Cap: props.max_size = ... or props.maxSize = ... for target in _assign_targets(node): if ( isinstance(target, ast.Attribute) and target.attr in POOL_PROPS_CAP_ATTRIBUTES and isinstance(target.value, ast.Name) ): - capped.add(target.value.id) - if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.value, ast.Call): - (target,) = node.targets - type_name = _callee_name(node.value) - if isinstance(target, ast.Name) and type_name in POOL_PROPS_TYPES: - constructed.append((node, target.id, type_name)) - for node, variable, type_name in constructed: - if variable not in capped and not _opted_out(lines, node): + pending.pop(target.value.id, None) + # Construction: props = PoolPropsType(...) or props: T = PoolPropsType(...) + con_target = con_value = None + if isinstance(node, ast.Assign) and len(node.targets) == 1: + con_target, con_value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + con_target, con_value = node.target, node.value + if ( + isinstance(con_target, ast.Name) + and isinstance(con_value, ast.Call) + and _callee_name(con_value) in POOL_PROPS_TYPES + ): + var, type_name = con_target.id, _callee_name(con_value) + if var in pending: + prev_node, prev_type = pending.pop(var) + if not _opted_out(lines, prev_node): + found.append( + Violation( + RULE_PROPS, + f"{path.as_posix()}:{prev_node.lineno}: {prev_type} assigned to '{var}' " + f"without {var}.max_size / {var}.maxSize", + ) + ) + pending[var] = (node, type_name) + + for var, (node, type_name) in pending.items(): + if not _opted_out(lines, node): found.append( Violation( RULE_PROPS, - f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{variable}' " - f"without {variable}.max_size / {variable}.maxSize", + f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{var}' " + f"without {var}.max_size / {var}.maxSize", ) ) return found diff --git a/ci/tools/tests/test_check_mempool_hygiene.py b/ci/tools/tests/test_check_mempool_hygiene.py index 14f0a1db51d..54d2b2344b1 100644 --- a/ci/tools/tests/test_check_mempool_hygiene.py +++ b/ci/tools/tests/test_check_mempool_hygiene.py @@ -103,6 +103,16 @@ def test_the_live_test_suite_is_clean(): pytest.param("def f():\n props = cuda.CUmemPoolProps_v1()\n props.maxSize = POOL_SIZE\n", id="in-function"), # A field of an alloc-node params object is not a constructor call, so it is not checked. pytest.param("params.poolProps.allocType = TYPE\nparams.pool_props = other\n", id="graph-params-fields"), + # Type-annotated assignment (ast.AnnAssign) must also be detected and treated as capped. + pytest.param( + "props: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", + id="annotated-capped", + ), + # Second construction caps correctly; the first was already capped. + pytest.param( + "props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\nprops = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", + id="two-constructions-both-capped", + ), ] POOL_PROPS_UNCAPPED = [ @@ -118,6 +128,13 @@ def test_the_live_test_suite_is_clean(): "def make():\n return cuda.CUmemPoolProps()\n\n\ndef f():\n props = cuda.CUmemPoolProps()\n g(props)\n\n\ndef g(props):\n props.maxSize = POOL_SIZE\n", id="capped-in-another-function", ), + # Type-annotated assignment without a cap must also be reported (ast.AnnAssign, Finding 1). + pytest.param("props: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\n", id="annotated-uncapped"), + # A cap on the first object does not cover a second construction of the same name (Finding 2). + pytest.param( + "props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\nprops = cuda.CUmemPoolProps()\n", + id="reassignment-second-uncapped", + ), ] @@ -174,3 +191,29 @@ def test_main_reports_both_rules_separately(tmp_path, capsys): err = capsys.readouterr().err assert "memory pools created by tests must set max_size:" in err assert "pool properties created by tests must set max_size / maxSize" in err + + +@pytest.mark.agent_authored(model="claude-sonnet-4.6") +def test_annotated_pool_props_assignment_reports_correct_line(tmp_path): + # Finding 1: props: T = CUmemPoolProps() is ast.AnnAssign; it must be detected. + path = write(tmp_path, "x = 1\nprops: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\n") + + (violation,) = violations_in(path) + + assert ":2:" in violation + + +@pytest.mark.agent_authored(model="claude-sonnet-4.6") +def test_reassignment_reports_only_the_uncapped_construction(tmp_path): + # Finding 2: a cap on the first object must not excuse a second construction + # of the same variable. Only the second (uncapped) object should be reported. + source = ( + "props = cuda.CUmemPoolProps()\n" # line 1 + "props.maxSize = POOL_SIZE\n" # line 2 — caps line 1 + "props = cuda.CUmemPoolProps()\n" # line 3 — new object, no cap + ) + path = write(tmp_path, source) + + (violation,) = violations_in(path) + + assert ":3:" in violation