From 9569417c8a71e5140ae3ef924f3797bb4f6a9424 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 30 Oct 2026 18:16:15 -0700 Subject: [PATCH 1/2] test(cuda.core): keep the cycle collector out of the latch window in test_error_timing_incomplete The test failed with "DID NOT RAISE" after about 60 s, the latch timeout, on one CI row. The likely cause is a finalizer that the cycle collector runs while the latch holds the stream: a free of memory synchronizes the context and waits for the latch. Collect before the launch and disable the collector until the latch is released. Create the end event before the launch. Use a 10 s latch and fail with the blocked time when the record call stalls. Closes #3296. Co-Authored-By: Claude Opus 5.5 --- cuda_core/tests/test_event.py | 41 +++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index 95d1d8f4107..d529cb6847a 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -2,7 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 +import gc import math +import time import pytest from helpers.latch import LatchKernel @@ -15,6 +17,10 @@ EventOptions, ) +# Short enough that a stalled host call fails fast; long enough that the +# latch cannot time out on its own during the few calls it guards. +_LATCH_TIMEOUT_SEC = 10 + def test_event_init_disabled(): with pytest.raises(RuntimeError, match=r"^Event objects cannot be instantiated directly\."): @@ -120,22 +126,43 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") @pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") +@pytest.mark.agent_authored(model="claude-opus-5-5") def test_error_timing_incomplete(): device = Device() device.set_current() - latch = LatchKernel(device) + latch = LatchKernel(device, timeout_sec=_LATCH_TIMEOUT_SEC) enabled = EventOptions(timing_enabled=True) stream = device.create_stream() event1 = stream.record(options=enabled) - latch.launch(stream) - event3 = stream.record(options=enabled) + # Create event3 before the latch holds the stream, so that no event + # creation runs while the stream is held. + event3 = device.create_event(options=enabled) - # event3 will never complete because the latch has not been released - with pytest.raises(RuntimeError, match="^One or both events have not completed."): - event3 - event1 + # A finalizer that frees memory synchronizes the context. If the cycle + # collector runs one while the latch holds the stream, the host blocks + # until the latch times out, and event3 completes (#3296). Collect now + # and keep the collector off until the latch is released. + gc.collect() + gc_enabled = gc.isenabled() + gc.disable() + try: + latch.launch(stream) + start = time.monotonic() + stream.record(event3) + elapsed = time.monotonic() - start + assert elapsed < _LATCH_TIMEOUT_SEC / 2, ( + f"a host call blocked for {elapsed:.1f} s while the latch held the stream " + f"(latch timeout {_LATCH_TIMEOUT_SEC} s)" + ) + # event3 cannot complete while the latch holds the stream + with pytest.raises(RuntimeError, match="^One or both events have not completed."): + event3 - event1 + finally: + latch.release() + if gc_enabled: + gc.enable() - latch.release() event3.sync() event3 - event1 # this should work From 55565e4c0c7b7836eb1e85959dd61a853f6df3f1 Mon Sep 17 00:00:00 2001 From: Andy Jost Date: Fri, 30 Oct 2026 19:32:38 -0700 Subject: [PATCH 2/2] test(cuda.core): time the whole latch window; request init_cuda Start the blocked-time check before the latch launch, so that a stall in the launch or the record reports the time. Re-enable the collector in its own finally block. Request init_cuda, as the test builds a pinned memory resource through the latch. Co-Authored-By: Claude Opus 5.5 --- cuda_core/tests/test_event.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cuda_core/tests/test_event.py b/cuda_core/tests/test_event.py index d529cb6847a..4be5f8b94c5 100644 --- a/cuda_core/tests/test_event.py +++ b/cuda_core/tests/test_event.py @@ -127,9 +127,8 @@ def test_error_timing_recorded(): @pytest.mark.skipif(Device().compute_capability.major < 7, reason="__nanosleep is only available starting Volta (sm70)") @pytest.mark.thread_unsafe(reason="requires a barrier wait to avoid overlapping pinned latch allocations") @pytest.mark.agent_authored(model="claude-opus-5-5") -def test_error_timing_incomplete(): - device = Device() - device.set_current() +def test_error_timing_incomplete(init_cuda): + device = init_cuda latch = LatchKernel(device, timeout_sec=_LATCH_TIMEOUT_SEC) enabled = EventOptions(timing_enabled=True) stream = device.create_stream() @@ -147,8 +146,8 @@ def test_error_timing_incomplete(): gc_enabled = gc.isenabled() gc.disable() try: - latch.launch(stream) start = time.monotonic() + latch.launch(stream) stream.record(event3) elapsed = time.monotonic() - start assert elapsed < _LATCH_TIMEOUT_SEC / 2, ( @@ -159,9 +158,11 @@ def test_error_timing_incomplete(): with pytest.raises(RuntimeError, match="^One or both events have not completed."): event3 - event1 finally: - latch.release() - if gc_enabled: - gc.enable() + try: + latch.release() + finally: + if gc_enabled: + gc.enable() event3.sync() event3 - event1 # this should work