From 8b992bccefa78df3f78c2f6fd1689e49860d2bb3 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 11:04:13 -0700 Subject: [PATCH 1/2] test(cuda.core): make the event-wait update case deterministic Make the wait target pending with a stream-ordered record instead of a record node behind a blocking host callback, and observe the consumer through a probe event instead of a host callback. The old check raced the driver's processing of the record node and relied on host callbacks on different streams being serialized. Also assert that the exec does not block on the other event, so a wrong wait target is detected. Signed-off-by: Ralf Juengling --- .../tests/graph/test_graph_node_update.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/cuda_core/tests/graph/test_graph_node_update.py b/cuda_core/tests/graph/test_graph_node_update.py index 85345019bc0..0f8e46e9587 100644 --- a/cuda_core/tests/graph/test_graph_node_update.py +++ b/cuda_core/tests/graph/test_graph_node_update.py @@ -133,18 +133,16 @@ def assert_exec_uses(graph, expected): def _event_wait_case(device): - """Keep the selected event pending to identify each exec's wait target.""" + """Keep one event pending at a time to identify each exec's wait target.""" original = device.create_event() replacement = device.create_event() invalid_replacement = device.create_event() invalid_replacement.close() - callback_called = threading.Event() graph_def = GraphDefinition() node = graph_def.wait(original) - node.callback(callback_called.set) - def assert_exec_uses(graph, expected): + def is_blocked_by(graph, pending): producer_started = threading.Event() producer_release = threading.Event() @@ -153,22 +151,35 @@ def blocking_callback(): producer_release.wait(timeout=30) producer_def = GraphDefinition() - producer_def.callback(blocking_callback).record(expected) + producer_def.callback(blocking_callback) producer_graph = producer_def.instantiate() producer_stream = device.create_stream() consumer_stream = device.create_stream() - callback_called.clear() producer_graph.launch(producer_stream) + # A stream-ordered record marks the event pending before it returns, + # unlike a record node, which does so only when the driver reaches it. + producer_stream.record(pending) try: assert producer_started.wait(timeout=5) + assert pending.is_done is False graph.launch(consumer_stream) - assert not callback_called.wait(timeout=0.1) + # Probe with an event rather than a host callback: host callbacks + # on different streams may or may not be serialized. + probe = consumer_stream.record() + time.sleep(0.1) + blocked = not probe.is_done finally: producer_release.set() producer_stream.sync() consumer_stream.sync() - assert callback_called.is_set() + assert probe.is_done + return blocked + + def assert_exec_uses(graph, expected): + unexpected = replacement if expected is original else original + assert is_blocked_by(graph, expected) + assert not is_blocked_by(graph, unexpected) return _DefinitionUpdateCase( graph_def=graph_def, From c9df97164cae792c3b83af19d2709d0cdfdb3ee5 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 30 Oct 2026 14:28:44 -0700 Subject: [PATCH 2/2] test(cuda.core): poll the probe when the event-wait exec must not block Wait for the probe with a bounded poll in the not-blocked direction, so a slow consumer cannot fail the check after the fixed 100 ms observation. Record the pending event inside the try block, so the blocking callback is released if the record fails. Signed-off-by: Ralf Juengling --- .../tests/graph/test_graph_node_update.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cuda_core/tests/graph/test_graph_node_update.py b/cuda_core/tests/graph/test_graph_node_update.py index 0f8e46e9587..21d1a6549e0 100644 --- a/cuda_core/tests/graph/test_graph_node_update.py +++ b/cuda_core/tests/graph/test_graph_node_update.py @@ -142,7 +142,7 @@ def _event_wait_case(device): graph_def = GraphDefinition() node = graph_def.wait(original) - def is_blocked_by(graph, pending): + def assert_blocked_by(graph, pending, expect_blocked): producer_started = threading.Event() producer_release = threading.Event() @@ -157,29 +157,31 @@ def blocking_callback(): consumer_stream = device.create_stream() producer_graph.launch(producer_stream) - # A stream-ordered record marks the event pending before it returns, - # unlike a record node, which does so only when the driver reaches it. - producer_stream.record(pending) try: + # A stream-ordered record marks the event pending before it returns, + # unlike a record node, which does so only when the driver reaches it. + producer_stream.record(pending) assert producer_started.wait(timeout=5) assert pending.is_done is False graph.launch(consumer_stream) # Probe with an event rather than a host callback: host callbacks # on different streams may or may not be serialized. probe = consumer_stream.record() - time.sleep(0.1) - blocked = not probe.is_done + if expect_blocked: + time.sleep(0.1) + assert not probe.is_done + else: + _wait_until(lambda: probe.is_done) finally: producer_release.set() producer_stream.sync() consumer_stream.sync() assert probe.is_done - return blocked def assert_exec_uses(graph, expected): unexpected = replacement if expected is original else original - assert is_blocked_by(graph, expected) - assert not is_blocked_by(graph, unexpected) + assert_blocked_by(graph, expected, expect_blocked=True) + assert_blocked_by(graph, unexpected, expect_blocked=False) return _DefinitionUpdateCase( graph_def=graph_def,