From 74fd886c8dcfab49ca473dbe4c19cc3f852fa2d4 Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Wed, 16 Sep 2026 22:52:35 -0400 Subject: [PATCH 1/5] Narrow validation settlement payload --- src/execution.jl | 16 +++------ src/execution/stage_program.jl | 4 ++- .../stage_program_kernelabstractions.jl | 7 ++-- src/execution/validation_support.jl | 36 ++++++++++--------- test/test_execution_receipts.jl | 4 +-- 5 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/execution.jl b/src/execution.jl index b519ba2..31d9a15 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -114,16 +114,7 @@ function _success_gate(prepared::PreparedPlan, lease_index::Int32, parent) expected = prepared.owner, actual = current_task(), )) - statuses = map( - status -> status.device, _prepared_validation_statuses(prepared) - ) - isempty(statuses) && throw(LocalMathValidationError( - "success_gate requires a source law with device validation status"; - stage = :prepare, - contract = :validation_status, - expected = :device_validation_status, - actual = :none, - )) + statuses = (prepared.runtime.execution_gate,) return _SuccessfulLawGate( parent, statuses, lease_index ) @@ -430,8 +421,9 @@ end function _transfer_receipt_statuses!(receipt::ExecutionReceipt, seen::Base.IdSet{Any}) prepared = receipt.prepared if !(prepared in seen) - _transfer_settled_validation_statuses!(prepared.lane, - _prepared_validation_statuses(prepared)) + runtime = prepared.runtime + _transfer_settled_validation_status!(prepared.lane, + runtime.execution_gate, runtime.validation_host) push!(seen, prepared) end for dependency in receipt.dependencies diff --git a/src/execution/stage_program.jl b/src/execution/stage_program.jl index 1ce7466..07d784e 100644 --- a/src/execution/stage_program.jl +++ b/src/execution/stage_program.jl @@ -69,6 +69,7 @@ end struct _PreparedStageProgram{E} launches::Vector{_AbstractPreparedStageLaunch} execution_gate::E + validation_host::Matrix{UInt32} end @@ -962,7 +963,8 @@ Base.@nospecializeinfer Base.@noinline function _prepare_stage_program( annotated === error ? rethrow() : throw(annotated) end end - return _PreparedStageProgram(launches, workspace.execution_gate) + return _PreparedStageProgram( + launches, workspace.execution_gate, program_host) end # `_BoundLaw` already proves that the scientific bindings are mutually legal. diff --git a/src/execution/stage_program_kernelabstractions.jl b/src/execution/stage_program_kernelabstractions.jl index 7e7c56c..961dba0 100644 --- a/src/execution/stage_program_kernelabstractions.jl +++ b/src/execution/stage_program_kernelabstractions.jl @@ -233,11 +233,10 @@ function _settle_lane_tail!(lane::_KernelAbstractionsLane, statuses::Tuple) return nothing end -function _transfer_settled_validation_statuses!( - lane::_KernelAbstractionsLane, statuses::Tuple) - isempty(statuses) && return nothing +function _transfer_settled_validation_status!( + lane::_KernelAbstractionsLane, device, host) try - _transfer_validation_statuses!(statuses) + _transfer_validation_status!(device, host) catch error _poison_lane!(lane, error) rethrow() diff --git a/src/execution/validation_support.jl b/src/execution/validation_support.jl index ed8f954..0464d29 100644 --- a/src/execution/validation_support.jl +++ b/src/execution/validation_support.jl @@ -94,32 +94,34 @@ _is_publication_validation_error(error) = error isa LocalMathValidationError && :runtime_ordered_fold_validation, ) -_prepared_validation_statuses(prepared) = () -@inline function _prepared_validation_statuses(prepared::PreparedPlan) - # Every contextual Stage status views the same program-level device/host - # buffer. Settlement and success gating therefore need one representative, - # not a flattened tuple whose type grows with total program length. - return (first(prepared.runtime.launches).status,) -end - -@inline _prepared_validation_status_groups(prepared::PreparedPlan) = - (map(launch -> launch.status, prepared.runtime.launches),) - function _transfer_validation_statuses!(statuses::Tuple) isempty(statuses) && return nothing # Every Stage status is a contextual view of this same program-level # buffer. One host-visible copy is therefore the complete settlement. status = first(statuses) - copyto!(status.host, status.device) + _transfer_validation_status!(status.device, status.host) + return nothing +end + +@inline function _transfer_validation_status!(device, host) + copyto!(host, device) return nothing end function _prepared_validation_error_at(prepared, lease_index::Int32) - for statuses in _prepared_validation_status_groups(prepared) - for status in statuses - error = _validated_publication_error(status, Int(lease_index)) - error === nothing || return error - end + runtime = prepared.runtime + host = runtime.validation_host + @inbounds host[_VALIDATION_FAILURE_CLASS, lease_index] == UInt32(0) && + return nothing + recorded_stage = _validation_decode_int32( + @inbounds host[_VALIDATION_STAGE_INDEX, lease_index]) + # Stage zero is publication suppression propagated from a failed dependency. + # Exact receipt traversal remains the diagnostic and ordering authority. + recorded_stage == 0 && return nothing + for launch in runtime.launches + status = launch.status + status.stage == recorded_stage || continue + return _validated_publication_error(status, Int(lease_index)) end return nothing end diff --git a/test/test_execution_receipts.jl b/test/test_execution_receipts.jl index d7f8a47..50066cf 100644 --- a/test/test_execution_receipts.jl +++ b/test/test_execution_receipts.jl @@ -134,8 +134,8 @@ end four_dependencies = (event_a, event_b, event_1, event_2) four_bytes = _warm_receipt_bookkeeping_bytes( warm_four, four_dependencies) - @test zero_bytes <= 4096 - @test four_bytes <= 4096 + @test zero_bytes <= 1024 + @test four_bytes <= 1024 end @testset "receipt failures are exact, cached, and dependency-local" begin From a6798a9da84b041ef63518f2766198045d0524d7 Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Thu, 17 Sep 2026 00:29:38 -0400 Subject: [PATCH 2/5] Reject unknown validation settlement stages --- src/execution/validation_support.jl | 18 ++++++++++++--- test/test_execution_receipts.jl | 35 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/execution/validation_support.jl b/src/execution/validation_support.jl index 0464d29..fec003f 100644 --- a/src/execution/validation_support.jl +++ b/src/execution/validation_support.jl @@ -111,10 +111,12 @@ end function _prepared_validation_error_at(prepared, lease_index::Int32) runtime = prepared.runtime host = runtime.validation_host - @inbounds host[_VALIDATION_FAILURE_CLASS, lease_index] == UInt32(0) && + failure_class = @inbounds host[_VALIDATION_FAILURE_CLASS, lease_index] + failure_class == UInt32(0) && return nothing recorded_stage = _validation_decode_int32( - @inbounds host[_VALIDATION_STAGE_INDEX, lease_index]) + @inbounds host[_VALIDATION_STAGE_INDEX, lease_index] + ) # Stage zero is publication suppression propagated from a failed dependency. # Exact receipt traversal remains the diagnostic and ordering authority. recorded_stage == 0 && return nothing @@ -123,7 +125,17 @@ function _prepared_validation_error_at(prepared, lease_index::Int32) status.stage == recorded_stage || continue return _validated_publication_error(status, Int(lease_index)) end - return nothing + return LocalMathValidationError( + "runtime validation status references an unprepared stage"; + stage = :wait, + contract = :validation_status_stage, + expected = :prepared_stage, + actual = ( + recorded_stage, + failure_class = _validation_decode_int32(failure_class), + ), + hint = "discard the prepared plan and report the invalid validation status", + ) end function _exact_host_int(value, purpose; stage::Symbol = :plan) diff --git a/test/test_execution_receipts.jl b/test/test_execution_receipts.jl index 50066cf..b609d4d 100644 --- a/test/test_execution_receipts.jl +++ b/test/test_execution_receipts.jl @@ -190,6 +190,41 @@ end @test healthy_storage == Int32[81, 82] end +@testset "validation settlement rejects an unprepared stage reference" begin + prepared, _ = _receipt_test_preparation(Int32(90)) + receipt = LWER.execute!(prepared) + status = prepared.runtime.execution_gate + lease_index = receipt.lease_index + @inbounds begin + status[LWER._VALIDATION_FAILURE_CLASS, lease_index] = UInt32(1) + status[LWER._VALIDATION_STAGE_INDEX, lease_index] = + reinterpret(UInt32, Int32(typemax(Int32))) + end + + failure = try + wait(receipt) + nothing + catch error + error + end + @test failure isa LWER.LocalMathValidationError + @test failure.contract === :validation_status_stage + @test failure.stage === :wait + @test failure.actual == ( + recorded_stage = typemax(Int32), + failure_class = Int32(1), + ) + @test !LWER.ispending(receipt) + @test LWER.submission_capacity(prepared).outstanding == 0 + cached_failure = try + wait(receipt) + nothing + catch error + error + end + @test cached_failure === failure +end + @testset "provider failures poison only their provider scope" begin facts = fetch(@async begin failing, _ = _receipt_test_preparation(Int32(0); From 99ee958c8fa1150cf579af00729794767b823746 Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Thu, 17 Sep 2026 01:52:01 -0400 Subject: [PATCH 3/5] Settle provider scopes through validation copies --- src/execution.jl | 73 +++++++++----- src/execution/program_inspection.jl | 8 +- .../stage_program_kernelabstractions.jl | 40 ++++---- src/execution/validation_support.jl | 9 -- src/model.jl | 8 +- test/metal/execution_receipts.jl | 94 ++++++++++++++++++- test/test_execution_receipts.jl | 87 ++++++++++++++++- 7 files changed, 251 insertions(+), 68 deletions(-) diff --git a/src/execution.jl b/src/execution.jl index 31d9a15..81fc11d 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -402,19 +402,14 @@ function _cache_receipt_result!(receipt::ExecutionReceipt) return failure end -function _synchronize_receipt_scope!(receipt::ExecutionReceipt) - lane = receipt.prepared.lane - receipt.scope_ordinal <= _lane_settled_ordinal(lane) && return nothing - target = _lane_scope_ordinal(lane) - try - _wait_lane!(lane) - _mark_lane_settled!(lane, target) - catch error - annotated = _provider_execution_error(error, :wait) - receipt.state = _EXECUTION_RECEIPT_PROVIDER_FAILURE - receipt.failure = annotated - throw(annotated) - end +function _settle_prepared_scope!(prepared::PreparedPlan, target::UInt64, + seen::Base.IdSet{Any}) + lane = prepared.lane + target <= _lane_settled_ordinal(lane) && return nothing + runtime = prepared.runtime + _settle_lane_tail!(lane, runtime.execution_gate, runtime.validation_host) + push!(seen, prepared) + _mark_lane_settled!(lane, target) return nothing end @@ -427,19 +422,39 @@ function _transfer_receipt_statuses!(receipt::ExecutionReceipt, seen::Base.IdSet push!(seen, prepared) end for dependency in receipt.dependencies + # Admission permits unresolved dependencies only within this provider + # scope; settled cross-scope dependencies already own host-visible status. _receipt_settled(dependency) || _transfer_receipt_statuses!(dependency, seen) end return nothing end +function _settle_receipt_statuses!(receipt::ExecutionReceipt, + seen::Base.IdSet{Any}) + try + lane = receipt.prepared.lane + receipt.scope_ordinal <= _lane_settled_ordinal(lane) || + _settle_prepared_scope!(receipt.prepared, + _lane_scope_ordinal(lane), seen) + _transfer_receipt_statuses!(receipt, seen) + catch error + annotated = _provider_execution_error(error, :wait) + receipt.state = _EXECUTION_RECEIPT_PROVIDER_FAILURE + receipt.failure = annotated + _release_receipt_lease!(receipt) + throw(annotated) + end + return nothing +end + function Base.wait(receipt::ExecutionReceipt) current_task() === receipt.prepared.owner || throw(LocalMathValidationError( "this ExecutionReceipt belongs to another owner task"; stage = :wait, contract = :receipt_owner)) if !_receipt_settled(receipt) - _synchronize_receipt_scope!(receipt) - _transfer_receipt_statuses!(receipt, Base.IdSet{Any}()) + seen = Base.IdSet{Any}() + _settle_receipt_statuses!(receipt, seen) _cache_receipt_result!(receipt) end receipt.failure === nothing || throw(receipt.failure) @@ -449,8 +464,8 @@ end """ waitall(receipts::ExecutionReceipt...) -Settle several logical receipts, synchronizing each represented provider scope -at most once. Cached failures are reported deterministically in argument order. +Settle several logical receipts, completing each represented provider scope at +most once. Cached failures are reported deterministically in argument order. Inspection is not required before waiting and waiting remains idempotent. """ function waitall(receipts::Tuple{Vararg{ExecutionReceipt}}) @@ -458,7 +473,7 @@ function waitall(receipts::Tuple{Vararg{ExecutionReceipt}}) "waitall requires at least one ExecutionReceipt"; stage = :wait, contract = :receipt_group_arity, expected = :nonempty, actual = 0)) - lanes = Any[] + settlement_prepared = Any[] targets = UInt64[] for receipt in receipts current_task() === receipt.prepared.owner || throw( @@ -467,29 +482,36 @@ function waitall(receipts::Tuple{Vararg{ExecutionReceipt}}) stage = :wait, contract = :receipt_owner)) lane = receipt.prepared.lane index = findfirst(candidate -> - _lane_same_wait_scope(candidate, lane), lanes) + _lane_same_wait_scope(candidate.lane, lane), settlement_prepared) if index === nothing - push!(lanes, lane) + push!(settlement_prepared, receipt.prepared) push!(targets, _lane_scope_ordinal(lane)) else targets[index] = max(targets[index], _lane_scope_ordinal(lane)) end end provider_failures = IdDict{Any,Any}() - for (lane, target) in zip(lanes, targets) + seen = Base.IdSet{Any}() + for (prepared, target) in zip(settlement_prepared, targets) + lane = prepared.lane target <= _lane_settled_ordinal(lane) && continue try - _wait_lane!(lane) - _mark_lane_settled!(lane, target) + _settle_prepared_scope!(prepared, target, seen) catch error provider_failures[lane.scope] = _provider_execution_error(error, :wait) end end - seen = Base.IdSet{Any}() for receipt in receipts haskey(provider_failures, receipt.prepared.lane.scope) && continue - _receipt_settled(receipt) || _transfer_receipt_statuses!(receipt, seen) + if !_receipt_settled(receipt) + try + _transfer_receipt_statuses!(receipt, seen) + catch error + provider_failures[receipt.prepared.lane.scope] = + _provider_execution_error(error, :wait) + end + end end first_failure = nothing for receipt in receipts @@ -501,6 +523,7 @@ function waitall(receipts::Tuple{Vararg{ExecutionReceipt}}) else receipt.state = _EXECUTION_RECEIPT_PROVIDER_FAILURE receipt.failure = provider_failure + _release_receipt_lease!(receipt) end end first_failure === nothing && receipt.failure !== nothing && diff --git a/src/execution/program_inspection.jl b/src/execution/program_inspection.jl index c54985b..428890f 100644 --- a/src/execution/program_inspection.jl +++ b/src/execution/program_inspection.jl @@ -690,9 +690,9 @@ function inspect(prepared::PreparedPlan; level = nothing) drained = prepared.drained, outstanding = prepared.outstanding, poisoned = prepared.poisoned, - provider_completions = _lane_wait_count(prepared.lane), + provider_completions = _lane_completion_count(prepared.lane), provider_scope_completions = - _lane_scope_wait_count(prepared.lane), + _lane_scope_completion_count(prepared.lane), validation_transfers = _lane_transfer_count(prepared.lane), ), ) @@ -772,8 +772,8 @@ function execution_contract(prepared::PreparedPlan) receipt_scope = _lane_wait_scope(lane), receipt_cumulative = _lane_cumulative(lane), receipt_selective = _lane_selective(lane), - observed_provider_completions = _lane_wait_count(lane), - observed_scope_completions = _lane_scope_wait_count(lane), + observed_provider_completions = _lane_completion_count(lane), + observed_scope_completions = _lane_scope_completion_count(lane), observed_validation_transfers = _lane_transfer_count(lane), ) end diff --git a/src/execution/stage_program_kernelabstractions.jl b/src/execution/stage_program_kernelabstractions.jl index 961dba0..c6b153e 100644 --- a/src/execution/stage_program_kernelabstractions.jl +++ b/src/execution/stage_program_kernelabstractions.jl @@ -1,6 +1,7 @@ # Hardware-agnostic KernelAbstractions provider. Kernel launches are implicitly -# ordered by the backend; one KernelAbstractions.synchronize call is the only -# execution-visibility boundary. No native stream, queue, or event is exposed. +# ordered by the backend. Provider completion is observed either by an explicit +# synchronize or by a nonempty blocking device-to-host validation copy. No +# native stream, queue, or event is exposed. struct _KernelAbstractionsDeviceGetter end @inline (::_KernelAbstractionsDeviceGetter)(backend) = @@ -14,7 +15,7 @@ mutable struct _KernelAbstractionsScope{B, D, T, E, G} const device_getter::G poisoned::Bool poison_reason::Any - synchronizations::Int + completions::Int transfers::Int submitted_ordinal::UInt64 settled_ordinal::UInt64 @@ -22,7 +23,7 @@ end mutable struct _KernelAbstractionsLane{S} <: _AbstractProviderLane const scope::S - waits::Int + completions::Int transfers::Int end @@ -117,13 +118,13 @@ _lane_transfer_law(::_KernelAbstractionsLane) = :same_owner_task_only _lane_cumulative(::_KernelAbstractionsLane) = true _lane_selective(::_KernelAbstractionsLane) = false _lane_error_observation(::_KernelAbstractionsLane) = ( - synchronization = :kernelabstractions_backend_contract, + completion = :blocking_validation_transfer_or_backend_synchronize, asynchronous_failures = :backend_defined, failure_scope = :backend_owner_task, ) -_lane_wait_count(lane::_KernelAbstractionsLane) = lane.waits -_lane_scope_wait_count(lane::_KernelAbstractionsLane) = - lane.scope.synchronizations +_lane_completion_count(lane::_KernelAbstractionsLane) = lane.completions +_lane_scope_completion_count(lane::_KernelAbstractionsLane) = + lane.scope.completions _lane_transfer_count(lane::_KernelAbstractionsLane) = lane.transfers _lane_same_wait_scope( first::_KernelAbstractionsLane, second::_KernelAbstractionsLane @@ -191,12 +192,12 @@ _validate_provider_capacity(::_KernelAbstractionsLane, evidence, capacity) = nothing function _synchronize_lane_tail!(lane::_KernelAbstractionsLane) - lane.waits += 1 + lane.completions += 1 scope = lane.scope current_task() === scope.owner || throw(LocalMathValidationError( "KernelAbstractions tail drain must use the preparing owner task" )) - scope.synchronizations += 1 + scope.completions += 1 try KernelAbstractions.synchronize(scope.backend) catch error @@ -209,20 +210,22 @@ function _synchronize_lane_tail!(lane::_KernelAbstractionsLane) return nothing end -function _settle_lane_tail!(lane::_KernelAbstractionsLane, statuses::Tuple) +function _settle_lane_tail!(lane::_KernelAbstractionsLane, device, host) _validate_lane_current!(lane) - lane.waits += 1 + lane.completions += 1 scope = lane.scope - scope.synchronizations += 1 + scope.completions += 1 try - if isempty(statuses) + if isempty(host) KernelAbstractions.synchronize(scope.backend) else # A host-visible device-to-host copy is the provider completion # operation. GPU providers must complete their queued prefix before # returning host data; issuing a second explicit synchronize would - # duplicate that boundary (Metal and CUDA copies are blocking). - _transfer_validation_statuses!(statuses) + # duplicate that boundary. If a provider ever aliases the two + # representations, the copy is not a distinct completion operation. + device === host && KernelAbstractions.synchronize(scope.backend) + _transfer_validation_status!(device, host) scope.transfers += 1 lane.transfers += 1 end @@ -249,11 +252,6 @@ end _drain_lane_tail!(lane::_KernelAbstractionsLane) = _synchronize_lane_tail!(lane) -function _wait_lane!(lane::_KernelAbstractionsLane) - _validate_lane_current!(lane) - return _synchronize_lane_tail!(lane) -end - function _atomic_capability( backend::KernelAbstractions.Backend, type::Type, diff --git a/src/execution/validation_support.jl b/src/execution/validation_support.jl index fec003f..0f9d593 100644 --- a/src/execution/validation_support.jl +++ b/src/execution/validation_support.jl @@ -94,15 +94,6 @@ _is_publication_validation_error(error) = error isa LocalMathValidationError && :runtime_ordered_fold_validation, ) -function _transfer_validation_statuses!(statuses::Tuple) - isempty(statuses) && return nothing - # Every Stage status is a contextual view of this same program-level - # buffer. One host-visible copy is therefore the complete settlement. - status = first(statuses) - _transfer_validation_status!(status.device, status.host) - return nothing -end - @inline function _transfer_validation_status!(device, host) copyto!(host, device) return nothing diff --git a/src/model.jl b/src/model.jl index 602d815..57a6f9d 100644 --- a/src/model.jl +++ b/src/model.jl @@ -301,11 +301,11 @@ end """ ExecutionReceipt -Logical execution receipt returned by [`execute!`](@ref). Physical waiting remains -cumulative within a KernelAbstractions provider scope, while settlement, +Logical execution receipt returned by [`execute!`](@ref). Provider completion +remains cumulative within a KernelAbstractions provider scope, while settlement, failure caching, and lease release belong to the exact requested receipt. -[`waitall`](@ref) groups receipts by provider scope and synchronizes each scope -at most once. +[`waitall`](@ref) groups receipts by provider scope and completes each scope at +most once. """ const _EXECUTION_RECEIPT_PENDING = UInt8(0) const _EXECUTION_RECEIPT_SUCCESS = UInt8(1) diff --git a/test/metal/execution_receipts.jl b/test/metal/execution_receipts.jl index 0887887..302faf9 100644 --- a/test/metal/execution_receipts.jl +++ b/test/metal/execution_receipts.jl @@ -11,6 +11,11 @@ end @inline (evaluator::LMEReceiptEvaluator)(item::Int32, reads, parameters) = (value = LMER.UniqueValue(evaluator.value + item),) +function execution_receipt_provider_fault_kernel(storage) + storage[2] = Int32(1) + return +end + function execution_receipt_receipt_preparation(array_type, backend, value::Int32; dependency_arity::Int = 0, lease_capacity::Int = 1) space = LMER.Space(LMEReceiptNode, 2) @@ -82,12 +87,12 @@ end @test Array(quaternary[2]) == Int32[51, 52] @test LMER.ispending(event_a) - synchronizations = LMER.inspect( + completions = LMER.inspect( quaternary[1]).realized.state.provider_scope_completions LMER.waitall(event_a, event_b, event_1, event_2) @test LMER.inspect( quaternary[1]).realized.state.provider_scope_completions == - synchronizations + completions cpu_root, _ = execution_receipt_receipt_preparation(identity, LMER.KernelAbstractions.CPU(), Int32(60)) @@ -125,6 +130,16 @@ end @test child_error.contract === :execution_dependency @test Array(failing_storage) == Int32[91] @test Array(dependent_storage) == fill(Int32(-1), 2) + semantic_error = try + wait(failed) + nothing + catch error + error + end + @test semantic_error isa LMER.LocalMathValidationError + @test semantic_error.contract === :runtime_stage_validation + @test LMER.submission_capacity(failing).outstanding == 0 + @test LMER.submission_capacity(dependent).outstanding == 0 warm, warm_storage = execution_receipt_receipt_preparation(Metal.MtlArray, backend, Int32(90); lease_capacity = 1) @@ -133,4 +148,79 @@ end @test warm_result.compile_time == 0.0 @test warm_result.recompile_time == 0.0 @test Array(warm_storage) == Int32[91, 92] + + provider_fault = fetch(@async begin + isolated_backend = Metal.MetalBackend() + prepared, _ = execution_receipt_receipt_preparation( + Metal.MtlArray, isolated_backend, Int32(100)) + fault_storage = Metal.MtlArray(Int32[0]) + receipt = LMER.execute!(prepared) + Metal.@metal threads=1 execution_receipt_provider_fault_kernel( + fault_storage) + first_error = try + wait(receipt) + nothing + catch error + error + end + cached_error = try + wait(receipt) + nothing + catch error + error + end + (; first_error, cached_error, + pending = LMER.ispending(receipt), + capacity = LMER.submission_capacity(prepared)) + end) + @test provider_fault.first_error isa LMER.LocalMathValidationError + @test provider_fault.first_error.contract === :provider_execution + @test provider_fault.first_error.actual isa Metal.KernelException + @test provider_fault.cached_error === provider_fault.first_error + @test !provider_fault.pending + @test provider_fault.capacity.outstanding == 0 + @test provider_fault.capacity.drained == UInt64(1) + + grouped_provider_fault = fetch(@async begin + isolated_backend = Metal.MetalBackend() + first, _ = execution_receipt_receipt_preparation( + Metal.MtlArray, isolated_backend, Int32(110)) + second, _ = execution_receipt_receipt_preparation( + Metal.MtlArray, isolated_backend, Int32(120)) + fault_storage = Metal.MtlArray(Int32[0]) + first_receipt = LMER.execute!(first) + second_receipt = LMER.execute!(second) + Metal.@metal threads=1 execution_receipt_provider_fault_kernel( + fault_storage) + failure = try + LMER.waitall(second_receipt, first_receipt) + nothing + catch error + error + end + first_cached = try + wait(first_receipt) + nothing + catch error + error + end + second_cached = try + wait(second_receipt) + nothing + catch error + error + end + (; failure, first_cached, second_cached, + first_capacity = LMER.submission_capacity(first), + second_capacity = LMER.submission_capacity(second)) + end) + @test grouped_provider_fault.failure isa LMER.LocalMathValidationError + @test grouped_provider_fault.failure.contract === :provider_execution + @test grouped_provider_fault.failure.actual isa Metal.KernelException + @test grouped_provider_fault.first_cached === grouped_provider_fault.failure + @test grouped_provider_fault.second_cached === grouped_provider_fault.failure + @test grouped_provider_fault.first_capacity.outstanding == 0 + @test grouped_provider_fault.first_capacity.drained == UInt64(1) + @test grouped_provider_fault.second_capacity.outstanding == 0 + @test grouped_provider_fault.second_capacity.drained == UInt64(1) end diff --git a/test/test_execution_receipts.jl b/test/test_execution_receipts.jl index b609d4d..6fcd700 100644 --- a/test/test_execution_receipts.jl +++ b/test/test_execution_receipts.jl @@ -82,16 +82,16 @@ end @test !LWER.ispending(event_4) @test LWER.ispending(event_a) - synchronizations = + completions = LWER.inspect(quaternary).realized.state.provider_scope_completions LWER.waitall(event_a, event_b, event_1, event_2) @test LWER.inspect(quaternary).realized.state.provider_scope_completions == - synchronizations + completions @test all(prepared -> LWER.submission_capacity(prepared).outstanding == 0, (root_a, root_b, unary, binary, quaternary, wide)) wait(event_a) @test LWER.inspect(quaternary).realized.state.provider_scope_completions == - synchronizations + completions compatible, _ = _receipt_test_preparation(Int32(35); dependency_arity = 1) @@ -138,6 +138,85 @@ end @test four_bytes <= 1024 end +@testset "status copies complete one shared provider scope" begin + root, root_storage = _receipt_test_preparation(Int32(110)) + dependent, dependent_storage = _receipt_test_preparation(Int32(120); + dependency_arity = 4) + root_event = LWER.execute!(root) + dependent_event = LWER.execute!(dependent; + dependencies = ntuple(_ -> root_event, 4)) + root_before = LWER.inspect(root).realized.state + dependent_before = LWER.inspect(dependent).realized.state + + wait(dependent_event) + + root_after = LWER.inspect(root).realized.state + dependent_after = LWER.inspect(dependent).realized.state + @test root_storage == Int32[111, 112] + @test dependent_storage == Int32[121, 122] + @test root_after.validation_transfers == + root_before.validation_transfers + 1 + @test dependent_after.validation_transfers == + dependent_before.validation_transfers + 1 + @test dependent_after.provider_scope_completions == + dependent_before.provider_scope_completions + 1 + + first, first_storage = _receipt_test_preparation(Int32(130)) + second, second_storage = _receipt_test_preparation(Int32(140)) + first_event = LWER.execute!(first) + second_event = LWER.execute!(second) + @test first_event.scope_ordinal < second_event.scope_ordinal + first_before = LWER.inspect(first).realized.state + second_before = LWER.inspect(second).realized.state + + LWER.waitall(second_event, first_event) + + first_after = LWER.inspect(first).realized.state + second_after = LWER.inspect(second).realized.state + @test first_storage == Int32[131, 132] + @test second_storage == Int32[141, 142] + @test first_after.validation_transfers == + first_before.validation_transfers + 1 + @test second_after.validation_transfers == + second_before.validation_transfers + 1 + @test second_after.provider_scope_completions == + second_before.provider_scope_completions + 1 +end + +@testset "only same-scope unresolved dependencies reach recursive settlement" begin + published = Channel{Any}(1) + release = Channel{Nothing}(1) + producer_task = @async begin + producer, producer_storage = _receipt_test_preparation(Int32(150)) + producer_event = LWER.execute!(producer) + put!(published, producer_event) + take!(release) + wait(producer_event) + producer_storage + end + producer_event = take!(published) + consumer, consumer_storage = _receipt_test_preparation(Int32(160); + dependency_arity = 1) + + rejection = try + LWER.execute!(consumer; dependencies = (producer_event,)) + nothing + catch error + error + end + @test rejection isa LWER.LocalMathValidationError + @test rejection.contract === :execution_dependency_scope + @test rejection.actual === :unresolved_cross_scope + @test consumer.submitted == 0 + + put!(release, nothing) + @test fetch(producer_task) == Int32[151, 152] + consumer_event = LWER.execute!(consumer; + dependencies = (producer_event,)) + wait(consumer_event) + @test consumer_storage == Int32[161, 162] +end + @testset "receipt failures are exact, cached, and dependency-local" begin failing, failing_storage = _receipt_test_conflict_preparation() dependent, dependent_storage = _receipt_test_preparation(Int32(60); @@ -175,6 +254,8 @@ end error end @test cached_failure === failure + @test LWER.submission_capacity(failing).outstanding == 0 + @test LWER.submission_capacity(dependent).outstanding == 0 producer_summary = only(LWER.inspect(child).dependencies) @test producer_summary.state === :semantic_failure @test producer_summary.failure === failure From e727eaef4ce06a32bfda30810e4600de8d56788e Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Wed, 16 Sep 2026 23:52:36 -0400 Subject: [PATCH 4/5] Use host copy completion for fixed relation validation --- src/structural_binding.jl | 2 -- test/metal/localmath_authoring.jl | 20 +++++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/structural_binding.jl b/src/structural_binding.jl index 235b915..ceb0b8c 100644 --- a/src/structural_binding.jl +++ b/src/structural_binding.jl @@ -736,10 +736,8 @@ function _validate_device_fixed_relation_content(relation, storage) status, storage.endpoints, counts, Int32(degree_bound(relation)), Int32(length(domain(relation))), Int32(length(codomain(relation))), Val(counts !== nothing); ndrange = max(length(domain(relation)), 1)) - KernelAbstractions.synchronize(backend) host_status = zeros(Int32, 1) copyto!(host_status, status) - KernelAbstractions.synchronize(backend) iszero(only(host_status)) || throw(LocalMathValidationError( "fixed relation device content violates its count or endpoint bounds"; stage = :bind, contract = :fixed_relation_content, diff --git a/test/metal/localmath_authoring.jl b/test/metal/localmath_authoring.jl index 5761f98..6daa4f2 100644 --- a/test/metal/localmath_authoring.jl +++ b/test/metal/localmath_authoring.jl @@ -80,11 +80,21 @@ struct LocalMathMetalNode end backend) @test_throws LocalMath.LocalMathValidationError LocalMath.plan( mixed_device_bound; backend) - @test_throws LocalMath.LocalMathValidationError LocalMath.prepare(reduction, - assembled => LocalMath.Allocate(Int32(0)), - scatter => LocalMath.Allocate((; - endpoints=reshape(Int32[1, 3, 1], 1, 3))); - backend) + invalid_fixed_relation = try + LocalMath.prepare(reduction, + assembled => LocalMath.Allocate(Int32(0)), + scatter => LocalMath.Allocate((; + endpoints=reshape(Int32[1, 3, 1], 1, 3))); + backend) + nothing + catch error + error + end + @test invalid_fixed_relation isa LocalMath.LocalMathValidationError + @test invalid_fixed_relation.stage == :bind + @test invalid_fixed_relation.contract == :fixed_relation_content + @test invalid_fixed_relation.expected == (count = 0:1, endpoint = 1:2) + @test invalid_fixed_relation.actual == :invalid_device_content resolved = LocalMath.Field(destination, Int32) resolution = LocalMath.@localmath item ∈ source begin From a21856f40f872208eb95aefa4ef942c8b4d62c1b Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Thu, 17 Sep 2026 01:57:37 -0400 Subject: [PATCH 5/5] Reuse validation status transfer for fixed relations --- src/structural_binding.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/structural_binding.jl b/src/structural_binding.jl index ceb0b8c..5a5dbec 100644 --- a/src/structural_binding.jl +++ b/src/structural_binding.jl @@ -737,7 +737,7 @@ function _validate_device_fixed_relation_content(relation, storage) Int32(length(domain(relation))), Int32(length(codomain(relation))), Val(counts !== nothing); ndrange = max(length(domain(relation)), 1)) host_status = zeros(Int32, 1) - copyto!(host_status, status) + _transfer_validation_status!(status, host_status) iszero(only(host_status)) || throw(LocalMathValidationError( "fixed relation device content violates its count or endpoint bounds"; stage = :bind, contract = :fixed_relation_content,