Cancel sub-workflow child runs that no parent waits for

backend/src/CodeSpace.Core/Handlers/CommandHandlers/Workflows/ReconcileStuckRunsCommandHandler.cs

@@ -28,6 +28,7 @@ public async Task<ReconcileStuckRunsResponse> Handle(ReconcileStuckRunsCommand r
             RecoveredStrandedTimerWait = summary.RecoveredStrandedTimerWait,
             RecoveredStrandedSupervisorInfraParkWait = summary.RecoveredStrandedSupervisorInfraParkWait,
             RecoveredStrandedSubworkflowParent = summary.RecoveredStrandedSubworkflowParent,
+            CancelledUnwaitedChildRun = summary.CancelledUnwaitedChildRun,
         };
     }
 }

backend/src/CodeSpace.Core/Services/Workflows/Dispatch/IWorkflowRunDispatcher.cs

@@ -28,6 +28,9 @@ public interface IWorkflowRunDispatcher
     ///   <item><c>false</c> if the row was not in <c>Pending</c> state (another caller won
     ///         the race, OR the run is already terminal). This is NOT an error; the loser
     ///         silently returns. Reconciler relies on this idempotence.</item>
+    ///   <item><c>false</c>, too, for a sub-workflow child with a finished ancestor (reached through
+    ///         sub-workflow children only — a rerun's lineage never counts). Nothing waits for it; the
+    ///         row is left as it is, for the stuck-run reconciler to cancel.</item>
     /// </list>
     ///
     /// Throws when the background-job client itself fails to enqueue (e.g. Hangfire storage

backend/src/CodeSpace.Core/Services/Workflows/Dispatch/RunAncestry.cs

@@ -0,0 +1,41 @@
+using CodeSpace.Core.Services.Workflows.Engine;
+using CodeSpace.Messages.Constants;
+using CodeSpace.Messages.Enums;
+
+namespace CodeSpace.Core.Services.Workflows.Dispatch;
+
+/// <summary>
+/// The sub-workflow ancestry walk the dispatcher refuses a run by, the same walk <c>ExecutionAuthorityService</c> denies an
+/// agent's authority by: up the parent chain while each run is a sub-workflow child, refusing at the first ancestor that has
+/// finished. Only a child's parent can be waiting for it; a rerun's or replay's parent is its lineage, finished by definition,
+/// so the walk ends at the first run that is not a child.
+/// </summary>
+internal static class RunAncestry
+{
+    /// <summary>One run on the walk: what kind of run it is, the run it hangs under, and how far it has got.</summary>
+    internal sealed record Hop(string SourceType, Guid? ParentRunId, WorkflowRunStatus Status);
+
+    /// <summary>The most child hops a walk takes. Nesting is capped at <see cref="SubworkflowService.MaxDepth"/>, so a longer chain of children is not one the engine staged.</summary>
+    internal const int MaxChildHops = SubworkflowService.MaxDepth;
+
+    /// <summary>
+    /// Whether some ancestor of <paramref name="runId"/>, reached through sub-workflow children only, has finished — or is missing,
+    /// loops back, or lies deeper than nesting allows, none of which the engine stages. <paramref name="load"/> reads one run, null when absent.
+    /// </summary>
+    internal static async Task<bool> HasFinishedAncestorAsync(Guid runId, Func<Guid, Task<Hop?>> load)
+    {
+        var lineage = new HashSet<Guid> { runId };
+        var run = await load(runId).ConfigureAwait(false);
+
+        for (var hops = 0; run is { SourceType: WorkflowRunSourceTypes.ChildWorkflow, ParentRunId: { } parentId }; hops++)
+        {
+            if (hops == MaxChildHops || !lineage.Add(parentId)) return true;
+
+            run = await load(parentId).ConfigureAwait(false);
+
+            if (run is null || WorkflowRunState.IsTerminal(run.Status)) return true;
+        }
+
+        return false;
+    }
+}

backend/src/CodeSpace.Core/Services/Workflows/Dispatch/WorkflowRunDispatcher.cs

@@ -10,7 +10,7 @@ namespace CodeSpace.Core.Services.Workflows.Dispatch;
 
 /// <summary>
 /// Atomic CAS + revert-on-throw isolated from the three call sites (manual run, replay run,
-/// webhook dispatcher).
+/// webhook dispatcher), behind a refusal of any sub-workflow child whose parent run has finished.
 /// </summary>
 public sealed class WorkflowRunDispatcher : IWorkflowRunDispatcher, IScopedDependency
 {
@@ -27,6 +27,8 @@ public WorkflowRunDispatcher(CodeSpaceDbContext db, ICodeSpaceBackgroundJobClien
 
     public async Task<bool> DispatchAsync(Guid runId, CancellationToken cancellationToken)
     {
+        if (await HasFinishedAncestorAsync(runId, cancellationToken).ConfigureAwait(false)) return RefuseUnderFinishedAncestor(runId);
+
         // Atomic CAS — Pending → Enqueued.
         // ExecuteUpdateAsync emits a single UPDATE ... WHERE Status = Pending statement.
         // Two concurrent callers (e.g. RunManuallyAsync + a reconciler that found the same
@@ -89,4 +91,22 @@ await _db.WorkflowRun
             throw;
         }
     }
+
+    /// <summary>
+    /// Whether the run is a sub-workflow child whose ancestry has finished. Every resume, revive and reconciler re-dispatch comes
+    /// through here, so this is the last door before a worker walks the run: a child whose parent has ended — a timer still firing
+    /// after the parent failed, say — would run toward a wait nothing holds open any more, spending for no one.
+    /// </summary>
+    private async Task<bool> HasFinishedAncestorAsync(Guid runId, CancellationToken cancellationToken) =>
+        await RunAncestry.HasFinishedAncestorAsync(runId, id => LoadHopAsync(id, cancellationToken)).ConfigureAwait(false);
+
+    private async Task<RunAncestry.Hop?> LoadHopAsync(Guid runId, CancellationToken cancellationToken) =>
+        await _db.WorkflowRun.AsNoTracking().Where(r => r.Id == runId).Select(r => new RunAncestry.Hop(r.SourceType, r.ParentRunId, r.Status)).SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+    /// <summary>Leave the run as it is — Pending, if a resume flipped it — for the stuck-run reconciler, which cancels a sub-workflow child no live parent waits for.</summary>
+    private bool RefuseUnderFinishedAncestor(Guid runId)
+    {
+        _logger.LogInformation("WorkflowRunDispatcher: run {RunId} not dispatched — it is a sub-workflow whose parent run has finished, so nothing is waiting for it", runId);
+        return false;
+    }
 }

backend/src/CodeSpace.Core/Services/Workflows/IWorkflowService.cs

@@ -151,8 +151,8 @@ public interface IWorkflowService
     /// teardown kills its agents and cancels its own children) — and emit a <c>run.cancelled</c>
     /// ledger record. The teardown is best-effort, so one agent's failed kill never strands the other agents, and one
     /// child's failed cancel never strands its siblings or the run's remaining waits. The agent-run reconciler's
-    /// parent-run-terminal guard re-cleans an agent run the teardown missed, and the stuck-run reconciler never dispatches
-    /// a child still staged under a finished parent; nothing stops a child that had already started.
+    /// parent-run-terminal guard re-cleans an agent run the teardown missed, and a child the teardown left live — staged or
+    /// started — is no longer waited for: the dispatcher will not queue it and the stuck-run reconciler cancels it.
     /// The teardown runs after the terminal commits and does not stop when the request that asked for it goes away — and
     /// the response waits for it across the whole sub-tree: every descendant's agents, children and waits are torn down
     /// before this returns. Returning at the commit instead is a follow-up, not built: a <c>TearDownStoppedRunAsync</c>
@@ -171,17 +171,19 @@ public interface IWorkflowService
     /// Cancelled run, whose halting / interrupted steps re-run where they stopped. CAS → Pending then post-commit
     /// dispatch. TEAM-SCOPED: a foreign run throws <see cref="KeyNotFoundException"/> (404). Returns <c>true</c> if this
     /// call drove it; <c>false</c> when the run is Success / Running, still parked on a pending wait (use
-    /// <c>/resume</c>), has nothing to re-run in place, or the CAS lost to a concurrent continue. Idempotent + race-safe
+    /// <c>/resume</c>), has nothing to re-run in place, is a sub-workflow child with a finished ancestor (continue the parent
+    /// instead; nothing waits for the child), or the CAS lost to a concurrent continue. Idempotent + race-safe
     /// (the dispatcher's Pending → Enqueued CAS is the double-dispatch guard).
     ///
     /// <para>Reviving a terminal run starts a new run generation: a walk still winding down from the stop or failure,
     /// and a stop's teardown still in flight, carry the old one and touch nothing of the revived run. The revive also
     /// closes, in the same transaction, what the ended attempt left pending — its waits Discarded, its queued and
-    /// running agents cancelled, its staged child runs cancelled — so the revived run never parks on, reuses or folds
-    /// as its own work a late teardown then ends. A running agent gets only its row flipped there; its process kill,
-    /// credential revoke and spend settlement run once that commit is visible, and a stop's teardown landing later
-    /// loses every CAS on what the revive already ended. A Continue after a Failure, which has no teardown, gets its
-    /// kills from the revive alone.</para>
+    /// running agents cancelled — so the revived run never parks on, reuses or folds as its own work a late teardown
+    /// then ends. A running agent gets only its row flipped there; its process kill, credential revoke and spend
+    /// settlement run once that commit is visible, and so does the full cancel of every sub-workflow child the ended
+    /// attempt left live, staged or started: the revived walk stages its own, so nothing waits for the old one. A stop's
+    /// teardown landing later loses every CAS on what the revive already ended. A Continue after a Failure, which has no
+    /// teardown, gets its kills from the revive alone.</para>
     ///
     /// <para>A step the continue re-runs starts over, because a wait the stop or failure closed carries no answer. So a
     /// <c>flow.sleep</c> parks a fresh timer for its FULL delay — the time it already slept is not credited — and a

backend/src/CodeSpace.Core/Services/Workflows/Reconciliation/IStuckRunReconcilerService.cs

@@ -52,6 +52,12 @@ namespace CodeSpace.Core.Services.Workflows.Reconciliation;
 ///       this wait kind had NO backstop until now — a lost deadline would strand the run on the park ladder
 ///       FOREVER, even once the model plane recovered. Re-fire the same deadline resume; idempotent against the
 ///       real (late) job via the resume's still-Pending gate.</item>
+///   <item><b>Unwaited sub-workflow child</b> (a live <c>workflow.child</c> run past a 2-min staging grace that no live
+///       run holds a Pending <c>Subworkflow</c> wait for) — its parent ended without reaching it (the engine's failure
+///       cleanup cancels only a child still Pending or Enqueued; a stop's cancel of a child can fail), or moved on without
+///       it (a Continue's re-walk stages a fresh child). Nothing will ever read its result, yet a timer or an agent can
+///       keep it spending. Cancelled through the full run cancel, FIRST in the tick, so no sweep after it revives one;
+///       every wake sweep also leaves out a child whose parent has finished, and the dispatcher refuses one.</item>
 /// </list>
 ///
 /// <para>Idempotent + safe to call concurrently from multiple replicas because every state
@@ -88,5 +94,8 @@ public sealed record StuckRunReconcileSummary
     /// <summary>Stranded Subworkflow parents re-fired because the child's inline on-completion resume was lost (a crash between the child's terminal commit and the resume). 0 when no parent is parked on a terminal child.</summary>
     public int RecoveredStrandedSubworkflowParent { get; init; }
 
-    public int Total => RedispatchedFromPending + RevertedFromEnqueued + MarkedAbandonedFromRunning + RedispatchedFromStrandedSuspended + RecoveredSupervisorAdvance + RecoveredAbandonedSupervisorRun + ReleasedRerunLeases + RecoveredStrandedTimerWait + RecoveredStrandedSupervisorInfraParkWait + RecoveredStrandedSubworkflowParent;
+    /// <summary>Sub-workflow children cancelled because no live run holds a Pending Subworkflow wait naming them — a parent that ended without reaching them, or moved on without them. 0 when every live child is waited for.</summary>
+    public int CancelledUnwaitedChildRun { get; init; }
+
+    public int Total => RedispatchedFromPending + RevertedFromEnqueued + MarkedAbandonedFromRunning + RedispatchedFromStrandedSuspended + RecoveredSupervisorAdvance + RecoveredAbandonedSupervisorRun + ReleasedRerunLeases + RecoveredStrandedTimerWait + RecoveredStrandedSupervisorInfraParkWait + RecoveredStrandedSubworkflowParent + CancelledUnwaitedChildRun;
 }

backend/src/CodeSpace.Core/Services/Workflows/Reconciliation/StuckRunReconcilerService.UnwaitedChildren.cs

@@ -0,0 +1,120 @@
+using CodeSpace.Core.Persistence.Entities;
+using CodeSpace.Messages.Constants;
+using CodeSpace.Messages.Enums;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace CodeSpace.Core.Services.Workflows.Reconciliation;
+
+/// <summary>
+/// Sub-workflow children no parent is waiting for: the sweep that cancels them, and the exclusion that keeps every wake sweep
+/// off a child whose parent has finished in the meantime.
+/// </summary>
+public sealed partial class StuckRunReconcilerService
+{
+    /// <summary>
+    /// How old a sub-workflow child must be before the sweep may judge it unwaited. The engine commits a staged child a moment
+    /// before it parks the parent's <c>Subworkflow</c> wait naming it, so for that moment a wanted child has no wait. Matches
+    /// <see cref="PendingStuckAfter"/>, so a child staged by a walk that died before parking is cancelled on the tick that
+    /// would otherwise re-dispatch it. Committed, not configured.
+    /// </summary>
+    public static readonly TimeSpan UnwaitedChildStagingGrace = TimeSpan.FromMinutes(2);
+
+    /// <summary>
+    /// Cancel the sub-workflow children that no live run is waiting for — least recently attempted first, at most <see cref="BatchSize"/> a tick,
+    /// each through the full <see cref="IWorkflowService.CancelRunAsync"/> so its own teardown kills its agents and cancels the
+    /// children it waits for in turn. A child is left this way when its parent ends without reaching it — the engine's failure
+    /// cleanup cancels only a child still Pending or Enqueued, a stop's cancel of a child can fail — or when its parent moves
+    /// on without it: a Continue closes the ended attempt's waits and its re-walk stages a fresh child. Each cancel stands
+    /// alone: a failure is logged and retried next tick, and the rest of the batch still goes. An attempt that fails or cancels
+    /// nothing stamps the child's <c>LastModifiedDate</c>, so it queues behind the rest instead of holding a batch slot forever. A level below a cancelled child
+    /// that its teardown did not reach is unwaited from then on, and a later tick takes it.
+    /// </summary>
+    internal async Task<int> CancelUnwaitedChildRunsAsync(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var children = await FindUnwaitedChildRunsAsync(cancellationToken).ConfigureAwait(false);
+
+            var cancelled = 0;
+
+            foreach (var child in children)
+                if (await TryCancelUnwaitedChildAsync(child, cancellationToken).ConfigureAwait(false)) cancelled++;
+
+            return cancelled;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            // A failed find stops this sweep only; the sweeps before it have already run and the next tick tries again.
+            _logger.LogWarning(ex, "StuckRunReconciler: the sweep for sub-workflow children no parent is waiting for failed; will retry next tick");
+            return 0;
+        }
+    }
+
+    /// <summary>
+    /// The live sub-workflow children past <see cref="UnwaitedChildStagingGrace"/> that no live run holds a Pending
+    /// <c>Subworkflow</c> wait for. The match is sound because only <c>SubworkflowService.StageChildRunAsync</c> writes this
+    /// source type, and only the engine's park writes a <c>Subworkflow</c> wait, its token the id of the child it just staged —
+    /// so every child some live run will resume is named by such a wait. A rerun or replay fork carries its own source type
+    /// and is never a candidate; a child a live fork waits for is named by the fork's wait like any other.
+    /// </summary>
+    private async Task<List<UnwaitedChild>> FindUnwaitedChildRunsAsync(CancellationToken cancellationToken)
+    {
+        var threshold = DateTimeOffset.UtcNow - UnwaitedChildStagingGrace;
+
+        return await _db.WorkflowRun.AsNoTracking()
+            .Where(r => r.SourceType == WorkflowRunSourceTypes.ChildWorkflow && NonTerminalRunStatuses.Contains(r.Status) && r.CreatedDate < threshold)
+            .Where(r => !_db.WorkflowRunWait.Any(w => w.Token == r.Id.ToString() && w.WaitKind == WorkflowWaitKinds.Subworkflow && w.Status == WorkflowWaitStatuses.Pending && _db.WorkflowRun.Any(p => p.Id == w.RunId && NonTerminalRunStatuses.Contains(p.Status))))
+            .OrderBy(r => r.LastModifiedDate).ThenBy(r => r.CreatedDate)
+            .Take(BatchSize)
+            .Select(r => new UnwaitedChild(r.Id, r.TeamId))
+            .ToListAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private async Task<bool> TryCancelUnwaitedChildAsync(UnwaitedChild child, CancellationToken cancellationToken)
+    {
+        try
+        {
+            var outcome = await _workflowService.CancelRunAsync(child.Id, child.TeamId, cancellationToken).ConfigureAwait(false);
+
+            if (outcome?.Cancelled == true) return true;
+
+            await RotateToBackAsync(child.Id, cancellationToken).ConfigureAwait(false);
+            return false;
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            // Per-row resilience: one child's failed cancel doesn't abort the sweep.
+            _logger.LogWarning(ex, "StuckRunReconciler: cancelling sub-workflow child run {RunId}, which no parent is waiting for, failed; will retry next tick", child.Id);
+            await RotateToBackAsync(child.Id, cancellationToken).ConfigureAwait(false);
+            return false;
+        }
+    }
+
+    /// <summary>Stamps a live child's <c>LastModifiedDate</c> (<c>ExecuteUpdateAsync</c> skips the audit hook) so the oldest-attempted-first batch moves past it. Best effort.</summary>
+    private async Task RotateToBackAsync(Guid runId, CancellationToken cancellationToken)
+    {
+        try
+        {
+            var now = DateTimeOffset.UtcNow;
+
+            await _db.WorkflowRun.Where(r => r.Id == runId && NonTerminalRunStatuses.Contains(r.Status)).ExecuteUpdateAsync(s => s.SetProperty(r => r.LastModifiedDate, now), cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is not OperationCanceledException)
+        {
+            _logger.LogWarning(ex, "StuckRunReconciler: moving sub-workflow child run {RunId} to the back of the unwaited-child batch failed", runId);
+        }
+    }
+
+    /// <summary>
+    /// Every run a wake or re-dispatch sweep may revive: all but a sub-workflow child whose parent has finished. No one waits for
+    /// such a child — its parent's waits closed when it ended — so reviving it would spend on work nothing will read;
+    /// <see cref="CancelUnwaitedChildRunsAsync"/> cancels it instead, and until that lands the dispatcher refuses it. The guard is
+    /// by source type because <c>ParentRunId</c> is also a rerun's or replay's lineage, finished by definition.
+    /// </summary>
+    private IQueryable<WorkflowRun> RunsWithNoFinishedParent => _db.WorkflowRun.Where(r => !(r.SourceType == WorkflowRunSourceTypes.ChildWorkflow && _db.WorkflowRun.Any(p => p.Id == r.ParentRunId && TerminalRunStatuses.Contains(p.Status))));
+
+    private static WorkflowRunStatus[] NonTerminalRunStatuses => WorkflowRunState.NonTerminal;
+
+    private sealed record UnwaitedChild(Guid Id, Guid TeamId);
+}