From 71655b2cee01ac01c23ed8550b6da74a015c72b1 Mon Sep 17 00:00:00 2001 From: FarinM Date: Tue, 27 Oct 2026 15:06:51 +0100 Subject: [PATCH 01/20] Resume a child workflow whose queued approval its parent never relayed A childWorkflow step relays its child's parks to the parent one at a time and holds the rest in memory. When the process stopped while an approval waited there, resume found the child parked on an approval the parent had never relayed, with its snapshot in neither log, and failed the run. The snapshot was durable all along: the parked step stores it with its pending operation. Resume now reads it from there through the env's new `loadParkedApproval` and relays the approval, and the sidecar wires its step-store read into the top-level and nested run envs. The lookup that follows a relay down to the step that parked moves into the runtime, so re-registering parked correlations and resume share it. --- docs/IMPLEMENTATION.md | 5 +- packages/workflow-host/src/child/index.ts | 2 +- .../src/child/parked-correlations.test.ts | 6 +- .../src/child/parked-correlations.ts | 82 +------- packages/workflow-host/src/child/run-child.ts | 32 +-- .../src/child/substrate-factory.ts | 184 +++++++++--------- ...-substrate-factory-parked-approval.test.ts | 132 +++++++++++++ packages/workflow/README.md | 5 +- packages/workflow/src/index.ts | 2 + .../src/runtime/child-workflow-relay.test.ts | 122 ++++++++++++ packages/workflow/src/runtime/env.ts | 22 ++- packages/workflow/src/runtime/index.ts | 3 + .../workflow/src/runtime/parked-approval.ts | 73 +++++++ packages/workflow/src/runtime/run.ts | 141 ++++++++++++-- 14 files changed, 602 insertions(+), 209 deletions(-) create mode 100644 packages/workflow/src/runtime/parked-approval.ts diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index aec479f41..ab9a74b19 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -933,7 +933,10 @@ A timed gate is exempt. Its scheduler timer resolves it in process, so it needs nothing upstream. A parked child survives a process restart. Resume reattaches the same -child run from its log and reestablishes the relay on the spawn step. The +child run from its log and reestablishes the relay on the spawn step. A +park still queued at the restart is relayed then too; for an approval, the +snapshot comes from the parked step's stored state, through the env's +`loadParkedApproval`, and a resume without one fails the spawn step. The child stays unaddressable; the parent is the resume root. #### Recycle diff --git a/packages/workflow-host/src/child/index.ts b/packages/workflow-host/src/child/index.ts index 8c4c64735..23eec56c6 100644 --- a/packages/workflow-host/src/child/index.ts +++ b/packages/workflow-host/src/child/index.ts @@ -72,7 +72,7 @@ export { type DiscoveredRun, } from "./self-discovery"; -export type { LoadParkedApproval } from "./parked-correlations"; +export type { LoadParkedApproval } from "@intx/workflow"; export { createWarmAgentCache, diff --git a/packages/workflow-host/src/child/parked-correlations.test.ts b/packages/workflow-host/src/child/parked-correlations.test.ts index 49657adaa..cfd40db2f 100644 --- a/packages/workflow-host/src/child/parked-correlations.test.ts +++ b/packages/workflow-host/src/child/parked-correlations.test.ts @@ -11,14 +11,12 @@ import type { } from "@intx/hub-sessions/substrate"; import { createInMemoryRepoStore, + type LoadParkedApproval, type RepoStore as RuntimeRepoStore, type WorkflowEvent, } from "@intx/workflow"; -import { - collectParkedApprovalCorrelations, - type LoadParkedApproval, -} from "./parked-correlations"; +import { collectParkedApprovalCorrelations } from "./parked-correlations"; const at = new Date().toISOString(); const repoId: RepoId = { kind: "workflow-run", id: "dep-1" }; diff --git a/packages/workflow-host/src/child/parked-correlations.ts b/packages/workflow-host/src/child/parked-correlations.ts index cec521cbd..3cf89cded 100644 --- a/packages/workflow-host/src/child/parked-correlations.ts +++ b/packages/workflow-host/src/child/parked-correlations.ts @@ -24,29 +24,15 @@ import type { } from "@intx/hub-sessions/substrate"; import { controlParkKindOf, - resumeFromLog, + findParkedApprovalSnapshot, + type LoadParkedApproval, type RepoStore as RuntimeRepoStore, - type RunState, } from "@intx/workflow"; import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime"; -import { correlationIdFromSignalName, signalName } from "@intx/types/signals"; +import { correlationIdFromSignalName } from "@intx/types/signals"; import { discoverInFlightRuns } from "./self-discovery"; -/** - * Recover the durable approval snapshot for one parked control-plane - * correlation. The child owns enumeration; the host owns the per-step - * on-disk layout (cold vs warm), so the snapshot read is a host binding. - * Returns `undefined` when no pending operation for the correlation carries - * a snapshot. - */ -export type LoadParkedApproval = (args: { - runId: string; - stepId: string; - attempt: number; - correlationId: string; -}) => Promise; - /** * One parked control-plane correlation: the child-supplied half of a * suspension registration. `parkKind` discriminates approval parks @@ -74,53 +60,6 @@ export interface CollectParkedApprovalCorrelationsOpts { * disagreements between the reduced state and the durable store that must not * silently drop a correlation the hub is waiting to register. */ -/** - * The snapshot for a relayed approval. The container's store has none; the - * body step that is still awaiting the same correlation has it. A nested - * relay is followed the same way. A run is read once. - */ -async function loadRelayedApprovalSnapshot( - opts: CollectParkedApprovalCorrelationsOpts, - load: LoadParkedApproval, - state: RunState, - stepId: string, - correlationId: string, - seen: Set, -): Promise { - const name = signalName(correlationId); - for (const child of state.children.values()) { - if (child.spawnedBy !== stepId || child.terminalStatus !== undefined) { - continue; - } - if (seen.has(child.childRunId)) continue; - seen.add(child.childRunId); - const events = [...(await opts.runtimeRepoStore.read(child.childRunId))]; - if (events.length === 0) continue; - const childState = resumeFromLog(child.childRunId, events); - for (const childStep of childState.steps.values()) { - if (childStep.phase !== "awaiting-signal") continue; - if (childStep.awaitingSignal?.name !== name) continue; - const direct = await load({ - runId: child.childRunId, - stepId: childStep.stepId, - attempt: childStep.currentAttempt, - correlationId, - }); - if (direct !== undefined) return direct; - const nested = await loadRelayedApprovalSnapshot( - opts, - load, - childState, - childStep.stepId, - correlationId, - seen, - ); - if (nested !== undefined) return nested; - } - } - return undefined; -} - export async function collectParkedApprovalCorrelations( opts: CollectParkedApprovalCorrelationsOpts, ): Promise { @@ -159,22 +98,15 @@ export async function collectParkedApprovalCorrelations( `workflow-child parked-correlations: run ${run.runId} step ${step.stepId} is parked on control-plane correlation ${correlationId}, but no loadParkedApproval binding is wired to recover its snapshot`, ); } - const direct = await opts.loadParkedApproval({ + const snapshot = await findParkedApprovalSnapshot({ + repoStore: opts.runtimeRepoStore, + load: opts.loadParkedApproval, runId: run.runId, + state: run.resumedState, stepId: step.stepId, attempt: step.currentAttempt, correlationId, }); - const snapshot = - direct ?? - (await loadRelayedApprovalSnapshot( - opts, - opts.loadParkedApproval, - run.resumedState, - step.stepId, - correlationId, - new Set(), - )); if (snapshot === undefined) { throw new Error( `workflow-child parked-correlations: reduced state shows run ${run.runId} step ${step.stepId} awaiting control-plane correlation ${correlationId} (attempt ${String(step.currentAttempt)}), but durable storage carries no approval snapshot for it; the run log and the step store disagree`, diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index 67b5f6a11..d9f4c05bf 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -72,6 +72,7 @@ import { import type { ActionCredentialsFactory, ActionHandler, + LoadParkedApproval, RunResult, RuntimeWorkflowRun, Scheduler, @@ -148,10 +149,7 @@ import { loadWorkflowLoopFnsFromClosure, } from "../workflow-definition-loader"; import { discoverInFlightRuns } from "./self-discovery"; -import { - collectParkedApprovalCorrelations, - type LoadParkedApproval, -} from "./parked-correlations"; +import { collectParkedApprovalCorrelations } from "./parked-correlations"; import type { ChildOutboundMailBridge } from "./outbound-mail-bridge"; import type { ChildMailboxCallBridge } from "./mailbox-call-bridge"; import { createSupervisorBackedMailPartReader } from "./supervisor-backed-mail-part-reader"; @@ -362,17 +360,17 @@ export interface RunWorkflowChildBindings { cleanupRunStorage?: (runId: string) => Promise; /** * Recover the durable approval snapshot for a parked control-plane - * correlation, so the child can answer a supervisor - * `parked-correlations.request` (the supervisor's re-registration path - * after a re-establishment). The child owns enumeration -- it walks its - * own reduced run state for `awaiting-signal` steps on control-plane - * channels -- but the snapshot lives in per-step durable storage whose - * on-disk layout (cold vs warm) the host owns, so the read is a host - * binding next to `cleanupRunStorage`. Optional so tests inject a stub and - * the recursive child-workflow adapter (which roots no per-step approval - * storage) can omit it; a production child that enumerates a parked - * control-plane step with no binding wired throws rather than silently - * dropping the correlation the hub is waiting to register. + * correlation. The child answers a supervisor + * `parked-correlations.request` with it (the supervisor's re-registration + * path after a re-establishment), and a resumed container relays from it a + * body's approval the container never relayed. The child owns enumeration + * -- it walks its own reduced run state for `awaiting-signal` steps on + * control-plane channels -- but the snapshot lives in per-step durable + * storage whose on-disk layout (cold vs warm) the host owns, so the read is + * a host binding next to `cleanupRunStorage`. Optional so tests inject a + * stub; a production child that enumerates a parked control-plane step with + * no binding wired throws rather than silently dropping the correlation the + * hub is waiting to register. */ loadParkedApproval?: LoadParkedApproval; /** @@ -1816,6 +1814,10 @@ function buildRuntimeEnv(args: { ...(args.bindings.persistRecoveredPark !== undefined ? { persistRecoveredPark: args.bindings.persistRecoveredPark } : {}), + // Let a resume relay a body's approval its container never did. + ...(args.bindings.loadParkedApproval !== undefined + ? { loadParkedApproval: args.bindings.loadParkedApproval } + : {}), }; // The suspendable-loop executor runs each iteration's body under THIS run's // inherited env (its real tool-bearing invokeStep, invokeAction, credentials diff --git a/packages/workflow-host/src/child/substrate-factory.ts b/packages/workflow-host/src/child/substrate-factory.ts index 83d73a8b7..fbd84cdbf 100644 --- a/packages/workflow-host/src/child/substrate-factory.ts +++ b/packages/workflow-host/src/child/substrate-factory.ts @@ -87,6 +87,7 @@ import { walkWorkflowSteps, LOOP_BODY_DESCENT, type ActionHandler, + type LoadParkedApproval, type LoopFnRegistry, type ParkedApprovalOp, type PersistRecoveredPark, @@ -145,7 +146,6 @@ import { import type { SubstrateFactory, SubstrateFactoryEnv } from "./from-process-env"; import { createMailboxWatchRegistry } from "./mailbox-watch-registry"; import type { ChildOutboundMailBridge } from "./outbound-mail-bridge"; -import type { LoadParkedApproval } from "./parked-correlations"; import { createProxyWorkflowRunRepoStore } from "./proxy-repo-store"; import { createCredentialsBackedAuthorize, @@ -776,6 +776,71 @@ export function toParkedApprovalOps( })); } +export interface ParkedApprovalHooks { + loadParkedApproval: LoadParkedApproval; + readParkedApprovalOps: ReadParkedApprovalOps; + persistRecoveredPark: PersistRecoveredPark; +} + +/** + * The durable park reads, and the recovered-park write, for every run of a + * deployment. Each run's steps are stored by the invoker that ran them. A + * warm-kept deployment's warm invoker runs its top-level run, `warmRunId`, and + * files each step under it by step id. The cold invoker runs every spawned + * run, a `childWorkflow` child or an onTrigger body, and stores each step per + * attempt. No loop iteration, which would inherit the top-level env, runs + * warm: a loop names another step at its level to run once exhausted, so a + * warm-kept deployment's one step is never a loop. + */ +export function createParkedApprovalHooks(args: { + dataDir: string; + substrate: RepoStore; + workflowRunRepoId: RepoId; + workflowRunRef: string; + principal: Principal; + signer: (payload: string) => Promise; + warmRunId?: string; +}): ParkedApprovalHooks { + const { warmRunId, signer, ...store } = args; + const cold: ParkedApprovalHooks = { + loadParkedApproval: (park) => + readColdParkedApprovalSnapshot({ ...store, ...park }), + readParkedApprovalOps: async (step) => + toParkedApprovalOps( + await readColdParkedPendingOperations({ ...store, ...step }), + ), + persistRecoveredPark: (park) => + persistColdRecoveredPark({ ...store, signer, ...park }), + }; + if (warmRunId === undefined) return cold; + return { + loadParkedApproval: (park) => + park.runId === warmRunId + ? readWarmParkedApprovalSnapshot({ + ...store, + runId: warmRunId, + stepId: park.stepId, + correlationId: park.correlationId, + }) + : cold.loadParkedApproval(park), + readParkedApprovalOps: async (step) => + step.runId === warmRunId + ? toParkedApprovalOps( + await readWarmParkedPendingOperations({ + ...store, + runId: warmRunId, + stepId: step.stepId, + }), + ) + : cold.readParkedApprovalOps(step), + // A warm read answers only from committed state, so a warm park never + // needs committing before the runtime records it. + persistRecoveredPark: async (park) => { + if (park.runId !== warmRunId) await cold.persistRecoveredPark(park); + }, + }; +} + export interface SidecarStepBuildEnvDeps { dataDir: string; workflowRunRepoId: RepoId; @@ -1233,6 +1298,11 @@ export type SidecarChildStepInvoker = ( interface SidecarRunChildDeps { /** Wrapped workflow-run substrate (the factory's `substrate`). */ substrate: RepoStore; + /** + * The step store's approval snapshot read, so a nested run's resume can + * relay an approval its own body parked and it never relayed. + */ + loadParkedApproval?: LoadParkedApproval; /** Workflow-run repo identifying the parent's deployment. */ workflowRunRepoId: RepoId; /** Workflow-run ref the child reads/writes against. */ @@ -2123,6 +2193,9 @@ async function buildChildRunEnv(args: { ...(effects !== undefined && invokeAction !== undefined ? { effects, invokeAction } : {}), + ...(args.deps.loadParkedApproval !== undefined + ? { loadParkedApproval: args.deps.loadParkedApproval } + : {}), }; // Wire loop-iteration spawning for a `loop` nested in this body. Assigned // AFTER the env literal because the iteration host closes over `env`: a loop @@ -2687,6 +2760,21 @@ export function createSidecarSubstrateFactory( const evaluateGrantsAdapter: GrantEvaluator = createGrantEvaluator(); + // Recover a parked correlation's approval snapshot, for the child's + // re-registration enumeration and for a resume that relays a body's + // approval its container never did, and recover a crashed step's park. + // One set serves every env, since a container reads its bodies' parks + // through its own. + const parkHooks = createParkedApprovalHooks({ + dataDir: validated.SIDECAR_DATA_DIR, + substrate, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + principal, + signer: conversationSigner, + ...(warmRunId !== undefined ? { warmRunId } : {}), + }); + const childRunDeps: SidecarRunChildDeps = { substrate, workflowRunRepoId, @@ -2712,6 +2800,7 @@ export function createSidecarSubstrateFactory( // A spawned child's steps resolve `director` refs against the same // closure-composed registry the parent's steps use. directors: stepDirectors, + loadParkedApproval: parkHooks.loadParkedApproval, }; // Terminal childWorkflow executor. `run-child` builds the in-memory // resolver from this plus the lifted-body map it extracts after loading @@ -2751,95 +2840,6 @@ export function createSidecarSubstrateFactory( }; } - // Recover a parked correlation's approval snapshot for the child's - // re-registration enumeration. Wired unconditionally (unlike - // `cleanupRunStorage`, which is cold-only): a warm agent parks on approval - // just as a cold one does, and the branch on `warmKeep` selects the durable - // read -- cold reads the attempt's local store while it is current and its - // committed step state otherwise, warm reconstructs the agent's durable - // conversation state from the substrate. - const loadParkedApproval: LoadParkedApproval = ({ - runId, - stepId, - attempt, - correlationId, - }) => - warmRunId !== undefined - ? readWarmParkedApprovalSnapshot({ - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - runId: warmRunId, - stepId, - correlationId, - }) - : readColdParkedApprovalSnapshot({ - dataDir: validated.SIDECAR_DATA_DIR, - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - runId, - stepId, - attempt, - correlationId, - }); - - // Enumerate a crashed step's durable pending approval operations for the - // resume classifier, off the same cold/warm durable read as - // `loadParkedApproval`. Where that binding is a lookup by a known - // correlationId (answering the supervisor's re-registration), this is the - // enumeration the classifier needs when the correlationId never reached the - // log -- the crash-across-park case: read the pending operations, project - // to the minimal approval records the runtime reconstructs `SignalAwaited` - // from. - const readParkedApprovalOps: ReadParkedApprovalOps = async ({ - runId, - stepId, - attempt, - }) => - toParkedApprovalOps( - warmRunId !== undefined - ? await readWarmParkedPendingOperations({ - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - runId: warmRunId, - stepId, - }) - : await readColdParkedPendingOperations({ - dataDir: validated.SIDECAR_DATA_DIR, - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - runId, - stepId, - attempt, - }), - ); - - // A warm read answers only from committed state, so only a cold park can - // need committing before the runtime records it. - const persistRecoveredPark: PersistRecoveredPark | undefined = - warmRunId !== undefined - ? undefined - : ({ runId, stepId, attempt, correlationId }) => - persistColdRecoveredPark({ - dataDir: validated.SIDECAR_DATA_DIR, - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - signer: conversationSigner, - runId, - stepId, - attempt, - correlationId, - }); - const bindings: RunWorkflowChildBindings = { substrate, workflowRunRepoId, @@ -2870,9 +2870,7 @@ export function createSidecarSubstrateFactory( }, scheduler, evaluateGrants: evaluateGrantsAdapter, - loadParkedApproval, - readParkedApprovalOps, - ...(persistRecoveredPark !== undefined ? { persistRecoveredPark } : {}), + ...parkHooks, // The same registry the warm agent's transport registers `watch` // callbacks into; `runWorkflowChild` routes each `mailbox.notify` to it. mailboxWatchRegistry, diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts index 6ab69b4a1..aa2554d84 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts @@ -32,13 +32,23 @@ import { type RepoStore, type WorkflowRunWorkflowProcessPrincipal, } from "@intx/hub-sessions"; +import { defineAgent } from "@intx/agent"; import { createIsogitStore } from "@intx/storage-isogit/node"; +import { + defineWorkflow, + loop, + loopBodyRunId, + sectionBodyRunId, + step, + type WorkflowDefinition, +} from "@intx/workflow"; import { createDurableConversationStore, readStepState, } from "@intx/workflow-host"; import { + createParkedApprovalHooks, persistColdRecoveredPark, readColdParkedApprovalSnapshot, readColdParkedPendingOperations, @@ -699,6 +709,128 @@ describe("readWarmParkedPendingOperations", () => { }); }); +// A warm-kept deployment's warm invoker runs its top-level run, and the cold +// invoker every run it spawns. +describe("createParkedApprovalHooks", () => { + const agent = defineAgent({ + id: "a", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }); + const agentBody = (id: string): WorkflowDefinition => + defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { s: step({ agent }) }, + }); + + // A warm deployment whose agent of step "s" is parked on `corr-warm`. + async function warmDeployment(): Promise<{ + substrate: RepoStore; + hooks: ReturnType; + }> { + const dataDir = await makeTempDir(); + const substrate = await createSubstrate(dataDir); + const store = await createDurableConversationStore({ + localStoreDir: await makeTempDir(), + signer: testSigner, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + runId: RUN_ID, + stepId: "s", + lifetime: { kind: "deployment" }, + }); + await store.storage.writeMetadata({ + pendingOperations: [pendingApproval("corr-warm", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + await store.mirrorToSubstrate(); + const hooks = createParkedApprovalHooks({ + dataDir, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + ...COMMITTED_READS, + signer: testSigner, + warmRunId: RUN_ID, + }); + return { substrate, hooks }; + } + + test("the top-level run reads its parks from the warm store", async () => { + const { hooks } = await warmDeployment(); + const at = { runId: RUN_ID, stepId: "s", attempt: 1 }; + + expect( + await hooks.loadParkedApproval({ ...at, correlationId: "corr-warm" }), + ).toEqual(snapshot); + expect(await hooks.readParkedApprovalOps(at)).toEqual([ + { correlationId: "corr-warm" }, + ]); + }); + + test.each([ + ["a childWorkflow child", "run-child"], + ["an onTrigger body", sectionBodyRunId(RUN_ID, "section", 0)], + ["a loop iteration inside a child", loopBodyRunId("run-child", "loop", 0)], + ])("%s reads its parks from its cold step state", async (_name, runId) => { + const { substrate, hooks } = await warmDeployment(); + await parkColdStep(substrate, runId, 1, [ + pendingApproval("corr-cold", snapshot), + ]); + const at = { runId, stepId: "s", attempt: 1 }; + + expect( + await hooks.loadParkedApproval({ ...at, correlationId: "corr-cold" }), + ).toEqual(snapshot); + expect(await hooks.readParkedApprovalOps(at)).toEqual([ + { correlationId: "corr-cold" }, + ]); + }); + + // The top-level run's park is read from committed state, so it has nothing + // to commit; any other run's goes to its cold step state, which here holds + // no park to commit. + test("only a run other than the top-level one commits a recovered park", async () => { + const { hooks } = await warmDeployment(); + const park = { stepId: "s", attempt: 1, correlationId: "corr-lost" }; + + await hooks.persistRecoveredPark({ runId: RUN_ID, ...park }); + await expect( + hooks.persistRecoveredPark({ runId: "run-child", ...park }), + ).rejects.toThrow( + "neither the committed step state nor a current local store holds its pending operation", + ); + }); + + // The hooks read only the top-level run warm, since a loop iteration would + // inherit its env. + test.each(["loop", "done"])( + "a deployment's one step cannot be a loop, whose onExhausted names %s", + (onExhausted) => { + expect(() => + defineWorkflow({ + id: "looped", + trigger: { type: "manual" }, + steps: { + loop: loop({ + body: agentBody("loop-body"), + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted, + }), + }, + }), + ).toThrow("onExhausted"); + }, + ); +}); + describe("toParkedApprovalOps", () => { test("projects approval ops to correlationId plus the epoch-ms deadline", async () => { const withDeadline: PendingOperation = { diff --git a/packages/workflow/README.md b/packages/workflow/README.md index 29c457fe4..37ce591ad 100644 --- a/packages/workflow/README.md +++ b/packages/workflow/README.md @@ -233,7 +233,10 @@ down. A queued park whose gate has already moved on is not given a signal that arrived while it waited. A parked child survives a process restart. Resume reattaches the same -child run from its log and reestablishes the relay on the spawn step. The +child run from its log and reestablishes the relay on the spawn step. A +park still queued at the restart is relayed then too; for an approval, the +snapshot comes from the parked step's stored state, through the env's +`loadParkedApproval`, and a resume without one fails the spawn step. The child stays unaddressable; the parent is the resume root. On drain, the relay sits on the spawn step. That step defaults to diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts index 7cf2659a7..94b51b6ab 100644 --- a/packages/workflow/src/index.ts +++ b/packages/workflow/src/index.ts @@ -52,6 +52,7 @@ export { createSuspendableChildHandle, createTerminalChildHandle, createLoopIterationHandle, + findParkedApprovalSnapshot, MAX_CHILD_SPAWN_DEPTH, ChildSpawnDepthExceededError, resolveMaxChildSpawnDepth, @@ -74,6 +75,7 @@ export { type EffectContext, type EffectContextConfig, type EffectLedger, + type LoadParkedApproval, type LoopFn, type LoopFnRegistry, type ParkedApprovalOp, diff --git a/packages/workflow/src/runtime/child-workflow-relay.test.ts b/packages/workflow/src/runtime/child-workflow-relay.test.ts index 757a5f051..9943caa04 100644 --- a/packages/workflow/src/runtime/child-workflow-relay.test.ts +++ b/packages/workflow/src/runtime/child-workflow-relay.test.ts @@ -50,6 +50,7 @@ import { type WorkflowAuthorizeFn, type WorkflowDefinition, type WorkflowEvent, + type LoadParkedApproval, type WorkflowPark, type WorkflowRuntimeEnv, } from "@intx/workflow"; @@ -201,6 +202,7 @@ function startRelayed(args: { resumeFromEvents?: readonly WorkflowEvent[]; hasUpstreamSignalResolver?: boolean; clock?: () => Date; + loadParkedApproval?: LoadParkedApproval; onSignalPark?: (park: { runId: string; name: string; @@ -264,6 +266,9 @@ function startRelayed(args: { ? { onSignalPark: args.onSignalPark } : {}), ...(args.loopFns !== undefined ? { loopFns: args.loopFns } : {}), + ...(args.loadParkedApproval !== undefined + ? { loadParkedApproval: args.loadParkedApproval } + : {}), }; env.spawnLoopIteration = createSpawnLoopIteration(env, loopBodies); return { @@ -2022,6 +2027,123 @@ describe("a parked childWorkflow survives a process restart", () => { expect(spawnOutput(result.outputs.spawn).childRunId).toBe(childId); }); + test("relays an approval the parent had not reached from the snapshot its step stored", async () => { + const parent = defineWorkflow({ + id: "resume-queued-approval-parent", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "resume-queued-approval-child", + trigger: { type: "manual" }, + steps: { alpha: step({ agent }), beta: step({ agent }) }, + }), + }), + }, + }); + const runId = "run-resume-queued-approval"; + // What each suspended step durably stored, as the host's step store does. + const stored = new Map(); + const key = (args: { + runId?: string; + stepId?: string; + attempt?: number; + correlationId: string; + }): string => + `${String(args.runId)}/${String(args.stepId)}/${String(args.attempt)}/${args.correlationId}`; + const first = startRelayed({ + definition: parent, + runId, + invokeStep: async (req) => { + if (req.resume !== undefined) return { output: null }; + const correlationId = `corr-${String(req.authzContext.stepId)}`; + const approvalSnapshot: ApprovalSnapshot = { + ...snapshot, + arguments: { step: req.authzContext.stepId }, + }; + stored.set( + key({ ...req.authzContext, correlationId }), + approvalSnapshot, + ); + return { + suspend: { correlationId, kind: "approval", approvalSnapshot }, + }; + }, + }); + void first.run.complete.catch(() => undefined); + await waitForEvent( + first.repoStore, + runId, + (event) => + event.kind === "SignalAwaited" && + event.stepId === "spawn" && + event.parkKind === "approval", + ); + const parentLog = await copyLog(first.repoStore, runId); + const childId = spawnedChildId(parentLog, "spawn"); + for (const stepId of ["alpha", "beta"]) { + await waitForEvent( + first.repoStore, + childId, + (event) => + event.kind === "SignalAwaited" && + event.signalName === signalName(`corr-${stepId}`), + ); + } + const childLog = await copyLog(first.repoStore, childId); + // The parent relays one park at a time, so the other is still queued. + const shown = parentLog.flatMap((event) => + event.kind === "SignalAwaited" && event.parkKind === "approval" + ? [correlationIdFromSignalName(event.signalName)] + : [], + ); + expect(shown).toHaveLength(1); + const queued = shown[0] === "corr-alpha" ? "corr-beta" : "corr-alpha"; + + const parks = parkSink(); + const repoStore = createInMemoryRepoStore(); + await seedLog(repoStore, childId, childLog); + const { run } = startRelayed({ + definition: parent, + runId, + invokeStep: async () => ({ output: null }), + onPark: parks.onPark, + repoStore, + resumeFromEvents: parentLog, + loadParkedApproval: async (args) => stored.get(key(args)), + }); + // A run that ends instead of relaying says why. + const ended = run.complete.then((result) => { + const failed = result.events.find((event) => event.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await run.signal( + signalName(String(shown[0])), + { outcome: "approved" }, + "sig-shown", + ); + const relayedPark = await Promise.race([parks.next(), ended]); + expect(relayedPark.correlationId).toBe(queued); + expect(relayedPark.approvalSnapshot).toEqual( + stored.get( + key({ + runId: childId, + stepId: queued.slice("corr-".length), + attempt: 1, + correlationId: queued, + }), + ), + ); + await run.signal(signalName(queued), { outcome: "approved" }, "sig-queued"); + const result = await run.complete; + + expect(result.terminalStatus).toBe("completed"); + expect(countEvents(result.events, "ChildSpawned")).toBe(1); + }); + test("an unproxied approval fails before spawn and writes no await", async () => { const parent = defineWorkflow({ id: "resume-unproxied-parent", diff --git a/packages/workflow/src/runtime/env.ts b/packages/workflow/src/runtime/env.ts index 86920548e..f07113755 100644 --- a/packages/workflow/src/runtime/env.ts +++ b/packages/workflow/src/runtime/env.ts @@ -628,7 +628,8 @@ export type WorkflowSignalPark = { * `SignalAwaited`. Deliberately narrow: the runtime reconstructs the missing * `SignalAwaited` from the correlationId (and optional timeout) alone and must * not learn the reactor's pending-operation internals (`gateId`, - * `suspendedCall`, the snapshot), which stay the host's concern. + * `suspendedCall`), which stay the host's concern. A relay that needs the + * snapshot reads it through `LoadParkedApproval`. */ export type ParkedApprovalOp = { correlationId: string; @@ -651,6 +652,17 @@ export type ReadParkedApprovalOps = (args: { attempt: number; }) => Promise; +/** + * Read the approval snapshot a parked step stored with its pending operation. + * See `WorkflowRuntimeEnv.loadParkedApproval`. + */ +export type LoadParkedApproval = (args: { + runId: string; + stepId: string; + attempt: number; + correlationId: string; +}) => Promise; + /** * Recovery hook the resume classifier awaits before it commits the * `SignalAwaited` of a park it recovered through `ReadParkedApprovalOps`: make @@ -829,6 +841,14 @@ export interface WorkflowRuntimeEnv { * A host whose read answers only from durable state leaves it unset. */ persistRecoveredPark?: PersistRecoveredPark; + /** + * Optional read of the approval snapshot a parked step stored with its + * pending operation. A restart can find a container's body parked on an + * approval the container never relayed; resume relays it from this + * snapshot. Production wires it to the sidecar's step store. Without it, + * such a resume fails the container step. + */ + loadParkedApproval?: LoadParkedApproval; } /** diff --git a/packages/workflow/src/runtime/index.ts b/packages/workflow/src/runtime/index.ts index 73a0e2ee9..c46c11066 100644 --- a/packages/workflow/src/runtime/index.ts +++ b/packages/workflow/src/runtime/index.ts @@ -7,6 +7,7 @@ export type { EffectContext, EffectLedger, LoopFn, + LoadParkedApproval, LoopFnRegistry, ParkedApprovalOp, PerformScope, @@ -60,6 +61,8 @@ export { export { createLoopIterationHandle } from "./loop-iteration-handle"; +export { findParkedApprovalSnapshot } from "./parked-approval"; + export { MAX_CHILD_SPAWN_DEPTH, ChildSpawnDepthExceededError, diff --git a/packages/workflow/src/runtime/parked-approval.ts b/packages/workflow/src/runtime/parked-approval.ts new file mode 100644 index 000000000..e05f33e20 --- /dev/null +++ b/packages/workflow/src/runtime/parked-approval.ts @@ -0,0 +1,73 @@ +import type { ApprovalSnapshot } from "@intx/types/runtime"; +import { signalName } from "@intx/types/signals"; + +import { resumeFromLog, type RunState } from "../state-machine/index"; +import type { LoadParkedApproval, RepoStore } from "./env"; + +/** + * The approval snapshot for a step parked on `correlationId`. A step that + * suspended stores it with its pending operation. A container that relays a + * body's park stores none, so the body run still awaiting the same + * correlation is followed instead, through nested relays too. + */ +export async function findParkedApprovalSnapshot(args: { + repoStore: Pick; + load: LoadParkedApproval; + runId: string; + state: RunState; + stepId: string; + attempt: number; + correlationId: string; +}): Promise { + const direct = await args.load({ + runId: args.runId, + stepId: args.stepId, + attempt: args.attempt, + correlationId: args.correlationId, + }); + if (direct !== undefined) return direct; + return relayedSnapshot(args, args.state, args.stepId, new Set()); +} + +// A run is read once. +async function relayedSnapshot( + args: { + repoStore: Pick; + load: LoadParkedApproval; + correlationId: string; + }, + state: RunState, + stepId: string, + seen: Set, +): Promise { + const name = signalName(args.correlationId); + for (const child of state.children.values()) { + if (child.spawnedBy !== stepId || child.terminalStatus !== undefined) { + continue; + } + if (seen.has(child.childRunId)) continue; + seen.add(child.childRunId); + const events = [...(await args.repoStore.read(child.childRunId))]; + if (events.length === 0) continue; + const childState = resumeFromLog(child.childRunId, events); + for (const childStep of childState.steps.values()) { + if (childStep.phase !== "awaiting-signal") continue; + if (childStep.awaitingSignal?.name !== name) continue; + const direct = await args.load({ + runId: child.childRunId, + stepId: childStep.stepId, + attempt: childStep.currentAttempt, + correlationId: args.correlationId, + }); + if (direct !== undefined) return direct; + const nested = await relayedSnapshot( + args, + childState, + childStep.stepId, + seen, + ); + if (nested !== undefined) return nested; + } + } + return undefined; +} diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index 7b91856b2..ff3ad6650 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -82,6 +82,7 @@ import { } from "./errors"; import { loopBodyRunId, scopedStepId, sectionBodyRunId } from "./step-scope"; import { inlineBodyRef } from "../inline-body-ref"; +import { findParkedApprovalSnapshot } from "./parked-approval"; import { applyEvent, controlParkKindOf, @@ -6795,6 +6796,8 @@ type OpenChildPark = name: string; correlationId: string; awaitSeq: number; + stepId: string; + attempt: number; }; /** @@ -6825,7 +6828,26 @@ type ChildRelay = correlationId: string; decision: unknown; } - | { kind: "input-fresh"; correlationId: string }; + | { kind: "input-fresh"; correlationId: string } + | { + kind: "approval-fresh"; + correlationId: string; + approvalSnapshot: ApprovalSnapshot; + }; + +/** + * A relay as the resume planner sees it. A fresh approval names the step that + * parked on it; its snapshot is read from that step's stored state before the + * drive, so planning needs only the logs. + */ +type PlannedChildRelay = + | Exclude + | { + kind: "approval-fresh"; + correlationId: string; + stepId: string; + attempt: number; + }; type ChildWorkflowResumePlan = | { kind: "fresh"; stepStarted: boolean } @@ -6955,6 +6977,8 @@ function openChildParks( name: event.signalName, correlationId, awaitSeq: event.seq, + stepId: event.stepId, + attempt: step.currentAttempt, }); } return parks; @@ -7027,7 +7051,7 @@ function deliveredForAwait( return boundSignalForContainerAwait(parentLog, name, stepId, awaitSeq); } -function relayName(relay: ChildRelay): string { +function relayName(relay: ChildRelay | PlannedChildRelay): string { switch (relay.kind) { case "signal-abandon": case "signal-reestablish": @@ -7037,6 +7061,7 @@ function relayName(relay: ChildRelay): string { case "control-reestablish": case "control-deliver": case "input-fresh": + case "approval-fresh": return signalName(relay.correlationId); default: { const unexpected: never = relay; @@ -7046,12 +7071,12 @@ function relayName(relay: ChildRelay): string { } function orderChildRelays( - relays: readonly ChildRelay[], + relays: readonly PlannedChildRelay[], parentAwaitName: string | undefined, -): ChildRelay[] { - const abandon: ChildRelay[] = []; - const current: ChildRelay[] = []; - const rest: ChildRelay[] = []; +): PlannedChildRelay[] { + const abandon: PlannedChildRelay[] = []; + const current: PlannedChildRelay[] = []; + const rest: PlannedChildRelay[] = []; for (const relay of relays) { if (relay.kind === "signal-abandon") { abandon.push(relay); @@ -7081,6 +7106,12 @@ function parentSignalAwait( return awaiting; } +/** + * What a container still has to do for the parks of one body run it relays: + * re-establish the relay it awaits, hand on input delivered to it but not yet + * to the body, and relay every park the body holds that it never relayed, of + * which a restart can leave several. + */ function planChildRelays( stepId: string, childRunId: string, @@ -7089,11 +7120,11 @@ function planChildRelays( childLog: readonly WorkflowEvent[], childState: RunState, nowMs: number, -): ChildRelay[] { +): PlannedChildRelay[] { const parks = openChildParks(childLog, childState); const parentAwait = parentSignalAwait(parentState, stepId); const childTerminal = isTerminalRunPhase(childState.phase); - const relays: ChildRelay[] = []; + const relays: PlannedChildRelay[] = []; for (const park of parks) { if (parentAwait !== undefined && parentAwait.name === park.name) { const parentKind = controlParkKindOf(parentAwait); @@ -7147,9 +7178,13 @@ function planChildRelays( continue; } if (park.kind === "approval") { - throw new Error( - `childWorkflow ${stepId} cannot resume approval park of child run ${childRunId}: the approval snapshot is not in the log`, - ); + relays.push({ + kind: "approval-fresh", + correlationId: park.correlationId, + stepId: park.stepId, + attempt: park.attempt, + }); + continue; } if (park.kind === "author") { const timeout = remainingSignalTimeout(park.timeoutAt, nowMs); @@ -7182,6 +7217,50 @@ function planChildRelays( return orderChildRelays(relays, parentAwait?.name); } +/** + * Read the snapshot of each approval the plan relays fresh from the stored + * state of the step that parked on it, before anything is driven. + */ +async function loadPlannedRelays( + env: WorkflowRuntimeEnv, + container: string, + childRunId: string, + childState: RunState, + planned: readonly PlannedChildRelay[], +): Promise { + const relays: ChildRelay[] = []; + for (const relay of planned) { + if (relay.kind !== "approval-fresh") { + relays.push(relay); + continue; + } + const load = env.loadParkedApproval; + const approvalSnapshot = + load === undefined + ? undefined + : await findParkedApprovalSnapshot({ + repoStore: env.repoStore, + load, + runId: childRunId, + state: childState, + stepId: relay.stepId, + attempt: relay.attempt, + correlationId: relay.correlationId, + }); + if (approvalSnapshot === undefined) { + throw new Error( + `${container} cannot resume approval park ${relay.correlationId} of child run ${childRunId}: no step state holds its approval snapshot`, + ); + } + relays.push({ + kind: "approval-fresh", + correlationId: relay.correlationId, + approvalSnapshot, + }); + } + return relays; +} + async function planChildWorkflowResume( env: WorkflowRuntimeEnv, stepId: string, @@ -7249,14 +7328,20 @@ async function planChildWorkflowResume( return { kind: "reattach", childRunId: recorded.childRunId, - relays: planChildRelays( - stepId, + relays: await loadPlannedRelays( + env, + `childWorkflow ${stepId}`, recorded.childRunId, - parentLog, - prior, - childLog, childState, - env.clock().getTime(), + planChildRelays( + stepId, + recorded.childRunId, + parentLog, + prior, + childLog, + childState, + env.clock().getTime(), + ), ), seed: true, }; @@ -7371,6 +7456,26 @@ async function drivePlannedChildRelay( "hold", ); } + case "approval-fresh": { + const name = signalName(relay.correlationId); + const snapshot = await readChildPark(handle, stepId, name); + if (snapshot.kind === "terminal") { + return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; + } + if (snapshot.kind === "released") return { kind: "held" }; + return awaitChildPark( + env, + parentRunId, + stepId, + handle, + relay.correlationId, + "approval", + relay.approvalSnapshot, + abort, + sectionSteps, + "hold", + ); + } default: { const unexpected: never = relay; throw new Error(`unexpected child relay ${String(unexpected)}`); From 64407e8423a7f58bba0b61384f79505ba0238ef7 Mon Sep 17 00:00:00 2001 From: FarinM Date: Sat, 24 Oct 2026 18:11:05 +0200 Subject: [PATCH 02/20] Report when a workflow run's trigger is durable A host that stops a workflow child has to know whether the trigger it handed the child survives the stop. A run's started promise settles once its RunStarted is durable, or reports that the run settled without one or resumed from a log that had already ended. --- packages/workflow/src/runtime/commit-chain.ts | 35 +++++ packages/workflow/src/runtime/env.ts | 7 + packages/workflow/src/runtime/run.ts | 27 ++++ packages/workflow/src/runtime/started.test.ts | 137 ++++++++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 packages/workflow/src/runtime/started.test.ts diff --git a/packages/workflow/src/runtime/commit-chain.ts b/packages/workflow/src/runtime/commit-chain.ts index 5a79f9542..b848de58c 100644 --- a/packages/workflow/src/runtime/commit-chain.ts +++ b/packages/workflow/src/runtime/commit-chain.ts @@ -185,6 +185,37 @@ export function whenRunFlushFails(runId: string): Promise { }); } +// The highest seq this process has written durably for a run, and who waits +// for one, behind `whenRunDurable`. +const durableSeqs = new Map(); +const durableWaiters = new Map< + string, + { seq: number; resolve: () => void }[] +>(); + +/** Resolves once this run's events up to `seq` are durable. */ +export function whenRunDurable(runId: string, seq: number): Promise { + if ((durableSeqs.get(runId) ?? -1) >= seq) return Promise.resolve(); + return new Promise((resolve) => { + const list = durableWaiters.get(runId) ?? []; + list.push({ seq, resolve }); + durableWaiters.set(runId, list); + }); +} + +function noteDurable(runId: string, seq: number): void { + durableSeqs.set(runId, Math.max(durableSeqs.get(runId) ?? -1, seq)); + const list = durableWaiters.get(runId); + if (list === undefined) return; + const waiting = list.filter((waiter) => { + if (waiter.seq > seq) return true; + waiter.resolve(); + return false; + }); + if (waiting.length === 0) durableWaiters.delete(runId); + else durableWaiters.set(runId, waiting); +} + function throwIfFlushFailed(runId: string): void { const cause = failedFlushes.get(runId); if (cause !== undefined) throw cause; @@ -231,6 +262,8 @@ async function flushBuffer(env: CommitEnv, runId: string): Promise { } throw cause; } + const last = events.at(-1); + if (last !== undefined) noteDurable(runId, last.seq); } /** @@ -433,4 +466,6 @@ export function dropChain(runId: string): void { failedFlushes.delete(runId); flushWaiters.delete(runId); bufferWaiters.delete(runId); + durableSeqs.delete(runId); + durableWaiters.delete(runId); } diff --git a/packages/workflow/src/runtime/env.ts b/packages/workflow/src/runtime/env.ts index f07113755..9c79ff57d 100644 --- a/packages/workflow/src/runtime/env.ts +++ b/packages/workflow/src/runtime/env.ts @@ -869,6 +869,13 @@ export interface WorkflowRun { * the runtime act on it. */ export interface RuntimeWorkflowRun extends WorkflowRun { + /** + * Resolves `true` once the run's `RunStarted` is durable, at once for a live + * run whose log already holds one, and `false` when the run settles without + * one or resumes from a log that has already ended. A host uses it to tell + * when the trigger it delivered can no longer be lost. + */ + started: Promise; /** * Stop the run for a `CancelRequested` its log already holds. The request * is durable, so the run aborts even if the child cancellation cascade diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index ff3ad6650..33aba99c1 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -58,6 +58,7 @@ import { isRunFlushFailure, markRepoWriteFailure, whenRunFlushFails, + whenRunDurable, reloadState as reloadStateInChain, runAfterDiscardingBuffer, } from "./commit-chain"; @@ -219,15 +220,25 @@ export function runtimeRun( cancelController.abort(); }); } + const started = Promise.withResolvers(); + const markStarted = (committed: boolean) => { + started.resolve(committed); + }; const completePromise = executeRun( definition, env, runId, cancelController, options, + markStarted, ); + const settled = () => { + markStarted(false); + }; + void completePromise.then(settled, settled); return { runId, + started: started.promise, complete: completePromise, async cancel(origin, reason) { // Route through `commit` so cancel races against in-flight @@ -304,6 +315,7 @@ async function executeRun( runId: string, cancelController: AbortController, options: RuntimeRunOptions, + onStarted: (committed: boolean) => void, ): Promise { try { return await executeRunBody( @@ -312,6 +324,7 @@ async function executeRun( runId, cancelController, options, + onStarted, ); } catch (cause) { // The body escaped. A nested `commitDurable` would wait on this @@ -609,6 +622,7 @@ async function executeRunBody( runId: string, cancelController: AbortController, options: RuntimeRunOptions, + onStarted: (committed: boolean) => void, ): Promise { const initialEvents = options.resumeFromEvents ?? []; @@ -948,6 +962,9 @@ async function executeRunBody( ]); } + // A log that already holds RunStarted is durable; one issued here becomes + // durable with the next flush. + let issuedStartSeq: number | undefined; // Issue RunStarted only if the state machine has not seen it. if (state.phase === "pending") { const event: WorkflowEvent = { @@ -963,6 +980,7 @@ async function executeRunBody( }; try { state = await commit(env, runId, event); + issuedStartSeq = state.lastSeq; } catch (cause) { // This block is reached only when canonical state was `pending` // (an empty or RunStarted-less log), so seedless recovery @@ -983,6 +1001,15 @@ async function executeRunBody( } } } + if (issuedStartSeq !== undefined) { + void whenRunDurable(runId, issuedStartSeq).then(() => { + onStarted(true); + }); + } else { + // A log that held RunStarted already, unless a cancellation beat this + // one and the run settles cancelled without it. + onStarted(state.definitionHash !== undefined); + } const inFlight = new Set(); const stepOutputs: Record = {}; diff --git a/packages/workflow/src/runtime/started.test.ts b/packages/workflow/src/runtime/started.test.ts new file mode 100644 index 000000000..211d8edab --- /dev/null +++ b/packages/workflow/src/runtime/started.test.ts @@ -0,0 +1,137 @@ +// A host learns from `started` that the trigger it delivered is durable, so a +// run stopped after that point keeps its trigger. + +import { describe, test, expect } from "bun:test"; + +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; + +import { + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, + createNoopDrainController, + defineWorkflow, + runtimeRun, + step, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { createTerminalChildHandle } from "./suspendable-child-handle"; + +const definition = defineWorkflow({ + id: "started", + trigger: { type: "manual" }, + steps: { + only: step({ + agent: defineAgent({ + id: "only", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }), + }), + }, +}); + +function createEnv(invoke: () => Promise) { + const clock = (): Date => new Date(); + const repoStore = createInMemoryRepoStore(); + const env: WorkflowRuntimeEnv = { + repoStore, + scheduler: createInMemoryScheduler({ repoStore, clock }), + signalChannel: createInMemorySignalChannel(), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: async () => { + await invoke(); + return { output: null }; + }, + spawnChild: async () => createTerminalChildHandle("completed"), + clock, + newId: (prefix) => `${prefix}-${Math.random().toString(36).slice(2, 8)}`, + drain: createNoopDrainController(definition), + hasUpstreamSignalResolver: true, + }; + return { env, repoStore }; +} + +describe("a run's started promise", () => { + test("settles once RunStarted is durable, before the run completes", async () => { + const finish = Promise.withResolvers(); + const { env, repoStore } = createEnv(() => finish.promise); + + const run = runtimeRun(definition, env, { + runId: "r", + consumedMessageId: "mail-1", + triggerPayload: null, + }); + expect(await run.started).toBe(true); + const log = await repoStore.read("r"); + expect(log[0]).toMatchObject({ + kind: "RunStarted", + consumedMessageId: "mail-1", + }); + + finish.resolve(undefined); + expect((await run.complete).terminalStatus).toBe("completed"); + }); + + test("resolves false for a run that settles without a durable RunStarted", async () => { + const { env } = createEnv(async () => undefined); + env.repoStore = { + ...env.repoStore, + appendBatch: () => Promise.reject(new Error("the disk is gone")), + }; + const run = runtimeRun(definition, env, { + runId: "r", + consumedMessageId: "mail-1", + triggerPayload: null, + }); + expect(await run.started).toBe(false); + await run.complete.catch(() => undefined); + }); + + test("settles for a resumed run whose log already holds RunStarted", async () => { + const first = createEnv(async () => undefined); + const done = runtimeRun(definition, first.env, { + runId: "r", + triggerPayload: null, + }); + await done.complete; + const log = await first.repoStore.read("r"); + + const finish = Promise.withResolvers(); + const second = createEnv(() => finish.promise); + const resumed = runtimeRun(definition, second.env, { + runId: "r", + resumeFromEvents: log.filter((event) => event.kind === "RunStarted"), + }); + expect(await resumed.started).toBe(true); + finish.resolve(undefined); + await resumed.complete; + }); + + test("answers false for a run resumed from a log that has already ended", async () => { + const first = createEnv(async () => undefined); + const done = runtimeRun(definition, first.env, { + runId: "r", + triggerPayload: null, + }); + await done.complete; + const log = await first.repoStore.read("r"); + + const second = createEnv(async () => undefined); + const resumed = runtimeRun(definition, second.env, { + runId: "r", + resumeFromEvents: log, + }); + expect(await resumed.started).toBe(false); + await resumed.complete; + }); +}); From 6087145217c3725e8080702a44adcd451212cee1 Mon Sep 17 00:00:00 2001 From: FarinM Date: Thu, 29 Oct 2026 23:08:50 +0100 Subject: [PATCH 03/20] Read what a run took from committed history, not the checkout A write puts its files in the checkout before it commits, so one that failed in between left an event there that the resumed run never reads. Boot recovery read the checkout, so it could hold back the mail that started such a run, and nothing delivered it again. The dispatch's check of whether a run started read the checkout too, so it then waited on a run that never resumed. Both now read committed history. Boot recovery reads one committed snapshot, which parses each tree once, since a walk of runs/ would otherwise parse that tree again for every run. --- .../hub-api/src/run-grant-materialization.ts | 2 +- packages/hub-sessions/src/index.ts | 1 - packages/hub-sessions/src/repo-store/store.ts | 18 +- packages/hub-sessions/src/substrate.ts | 1 - .../src/workflow-run-kind.test.ts | 194 ++++++++---------- .../hub-sessions/src/workflow-run-kind.ts | 192 +++++------------ .../src/supervisor/crash-respawn.test.ts | 2 + .../supervisor/stale-cohort-routing.test.ts | 2 + .../src/supervisor/supervisor.test.ts | 94 +++++++-- .../src/supervisor/supervisor.ts | 21 +- .../src/testing/stub-repo-store.ts | 40 +++- 11 files changed, 281 insertions(+), 286 deletions(-) diff --git a/packages/hub-api/src/run-grant-materialization.ts b/packages/hub-api/src/run-grant-materialization.ts index 47d8f46a0..f6e64f343 100644 --- a/packages/hub-api/src/run-grant-materialization.ts +++ b/packages/hub-api/src/run-grant-materialization.ts @@ -1382,7 +1382,7 @@ export function createMailTriggeredRunGrantsMaterializer( // This preflight inspects the anchor row's status, cancellation, and // deadline, and omits the durable-lifecycle terminal check that the HTTP // trigger route in workflow-run-trigger.ts applies. The supervisor's durable run-ref guard - // (rejectTerminalRun / readWorkflowRunLifecycle in supervisor.ts) is the + // (rejectTerminalRun / readCommittedWorkflowRunLifecycle in supervisor.ts) is the // fired/not-fired authority and never re-fires a durably-settled run, so // this status check is a lagging fast-fail only. The two preflights diverge // inside the status-flip lag window; that asymmetry is tolerable and is diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 6623f4ccb..6f972764f 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -206,7 +206,6 @@ export { classifyTerminalEvent, scanRunsForBoot, readCommittedWorkflowRunLifecycle, - readWorkflowRunLifecycle, workflowRunRepoIdForAddress, WORKFLOW_RUN_REF, replayProcessingToInbox, diff --git a/packages/hub-sessions/src/repo-store/store.ts b/packages/hub-sessions/src/repo-store/store.ts index 9e94c8504..de60348f0 100644 --- a/packages/hub-sessions/src/repo-store/store.ts +++ b/packages/hub-sessions/src/repo-store/store.ts @@ -2756,7 +2756,15 @@ export function createRepoStore(config: CreateRepoStoreConfig): LocalRepoStore { // (`openCommittedReadsAtCommit`) entrypoints, which differ only in how // they obtain and validate the commit SHA. function committedReadsAt(dir: string, commitSha: string): CommittedReads { - const reads = dirReads(dir); + // Every path walk starts at the root, so a walk of `runs/` would parse + // that tree again for each run. A snapshot's objects never change, so + // one window holds each parsed commit and tree for its whole life. + const raw = dirReads(dir); + const reads: ObjectReads = { + ...raw, + readCommit: windowedMemo(raw.readCommit).get, + readTree: windowedMemo(raw.readTree).get, + }; const { readBlobByOid } = buildPriorTreeClosures(reads, commitSha); const treeOid = async (relPath: string): Promise => { // "." and "" both name the root tree; resolveTreeEntry treats the @@ -2767,13 +2775,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): LocalRepoStore { const listDir = async (relPath: string): Promise => { const oid = await treeOid(relPath); if (oid === null) return []; - const { tree } = await git.readTree({ - fs, - dir, - cache: cacheFor(dir), - oid, - }); - return tree.map((e) => ({ name: e.path, oid: e.oid, type: e.type })); + return listingOf((await reads.readTree(oid)).entries); }; return { listDir, readBlobByOid, treeOid }; } diff --git a/packages/hub-sessions/src/substrate.ts b/packages/hub-sessions/src/substrate.ts index 2685dc737..3fe151b72 100644 --- a/packages/hub-sessions/src/substrate.ts +++ b/packages/hub-sessions/src/substrate.ts @@ -24,7 +24,6 @@ export { parseEventSeq, scanRunsForBoot, readCommittedWorkflowRunLifecycle, - readWorkflowRunLifecycle, readProcessingEntry, replayProcessingToInbox, requireEventSeq, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index 9158fb551..ab91acbea 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -16,7 +16,6 @@ import { parseEventSeq, readCommittedWorkflowRunLifecycle, scanRunsForBoot, - readWorkflowRunLifecycle, replayProcessingToInbox, WORKFLOW_RUN_GITIGNORE_PATH, WORKFLOW_RUN_RUNS_PREFIX, @@ -3371,8 +3370,9 @@ describe("claim-check substrate FIFO invariant", () => { // replay leaves them in `processing/` rather than re-admitting them to // `inbox/` and dispatching a colliding second run on the same runId. The // run logs live on the run-event ref (`refs/heads/main`); the claim-check -// ref (`refs/heads/events`) cannot see them, so the caller reads them via -// the working tree and hands the owned set to `replayProcessingToInbox`. +// ref (`refs/heads/events`) cannot see them, so the caller reads them from +// that ref's committed history and hands the owned set to +// `replayProcessingToInbox`. describe("claim-check API — resume-owned processing entries survive replay", () => { function runStartedBody(runId: string): string { return JSON.stringify({ @@ -3408,11 +3408,50 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }, message: "seed one live and one terminal run", }); - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned.has("live")).toBe(true); expect(owned.has("done")).toBe(false); }); + test("scanRunsForBoot ignores events the checkout holds that never committed", async () => { + const { store, repoId, principal } = await makeClaimCheckStore( + "cc-owned-uncommitted-", + ); + await store.writeTree(principal, repoId, "refs/heads/main", { + files: { + [`${WORKFLOW_RUN_RUNS_PREFIX}/live/events/0.json`]: + runStartedBody("live"), + }, + message: "seed a live run", + }); + // A write lays its files in the checkout before it commits, so one that + // failed in between leaves these behind. + const runsDir = path.join( + store.getRepoDir(repoId), + WORKFLOW_RUN_RUNS_PREFIX, + ); + await fs.promises.mkdir(path.join(runsDir, "lost", "events"), { + recursive: true, + }); + await fs.promises.writeFile( + path.join(runsDir, "lost", "events", "0.json"), + runStartedBody("lost"), + ); + + const { ownedMessageIds } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); + expect(ownedMessageIds).toEqual(new Set(["live"])); + }); + test("scanRunsForBoot proposes a terminal per-event run for sealing and drops it from owned", async () => { const { store, repoId, principal } = await makeClaimCheckStore("cc-scan-"); // `live` is parked (RunStarted only). `done` reached RunCompleted but is @@ -3432,7 +3471,9 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }); const { ownedMessageIds, pendingSealRunIds } = await scanRunsForBoot( store, + principal, repoId, + "refs/heads/main", ); expect(pendingSealRunIds).toEqual(["done"]); expect(ownedMessageIds.has("done")).toBe(false); @@ -3452,39 +3493,13 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }, message: "seed a sealed (terminated, compacted) run", }); - const { pendingSealRunIds } = await scanRunsForBoot(store, repoId); - expect(pendingSealRunIds).toEqual([]); - }); - - test("readWorkflowRunLifecycle distinguishes grants-only, live, and terminal logs", async () => { - const { store, repoId, principal } = - await makeClaimCheckStore("cc-lifecycle-"); - await store.writeTree(principal, repoId, "refs/heads/main", { - files: { - [`${WORKFLOW_RUN_RUNS_PREFIX}/staged/grants.json`]: JSON.stringify({ - stepGrants: [], - }), - [`${WORKFLOW_RUN_RUNS_PREFIX}/live/events/0.json`]: - runStartedBody("live-message"), - [`${WORKFLOW_RUN_RUNS_PREFIX}/done/events/0.json`]: - runStartedBody("done-message"), - [`${WORKFLOW_RUN_RUNS_PREFIX}/done/events/1.json`]: runCompletedBody(1), - }, - message: "seed lifecycle states", - }); - - await expect( - readWorkflowRunLifecycle(store, repoId, "missing"), - ).resolves.toBe("absent"); - await expect( - readWorkflowRunLifecycle(store, repoId, "staged"), - ).resolves.toBe("absent"); - await expect(readWorkflowRunLifecycle(store, repoId, "live")).resolves.toBe( - "live", - ); - await expect(readWorkflowRunLifecycle(store, repoId, "done")).resolves.toBe( - "terminal", + const { pendingSealRunIds } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", ); + expect(pendingSealRunIds).toEqual([]); }); test("readCommittedWorkflowRunLifecycle reads the Git ref without a checkout", async () => { @@ -3523,10 +3538,7 @@ describe("claim-check API — resume-owned processing entries survive replay", ( ).resolves.toBe("absent"); }); - test("readWorkflowRunLifecycle and readCommittedWorkflowRunLifecycle agree across states", async () => { - // Both readers run one shared classification core over different read - // surfaces (working-tree fs vs committed git objects). Seed one tree and - // read every run BOTH ways to prove the two surfaces classify identically. + test("readCommittedWorkflowRunLifecycle distinguishes grants-only, live, terminal, and sealed logs", async () => { const { store, repoId, principal } = await makeClaimCheckStore("cc-equiv-"); const sealedLog = [runStartedBody("sealed"), runCompletedBody(1)].join("\n") + "\n"; @@ -3542,7 +3554,7 @@ describe("claim-check API — resume-owned processing entries survive replay", ( [`${WORKFLOW_RUN_RUNS_PREFIX}/done/events/1.json`]: runCompletedBody(1), [`${WORKFLOW_RUN_RUNS_PREFIX}/sealed/events.jsonl`]: sealedLog, }, - message: "seed lifecycle states for the equivalence check", + message: "seed lifecycle states", }); const reads = await store.openCommittedReads( @@ -3560,45 +3572,12 @@ describe("claim-check API — resume-owned processing entries survive replay", ( ["sealed", "terminal"], ] as const; for (const [runId, expected] of cases) { - const fsResult = await readWorkflowRunLifecycle(store, repoId, runId); - const committedResult = await readCommittedWorkflowRunLifecycle( - reads, - runId, - ); - expect(fsResult).toBe(expected); - expect(committedResult).toBe(expected); - expect(committedResult).toBe(fsResult); + await expect( + readCommittedWorkflowRunLifecycle(reads, runId), + ).resolves.toBe(expected); } }); - test("readWorkflowRunLifecycle wraps an unreadable latest event", async () => { - // A validated write cannot store a corrupt event, so seed a valid live run - // and then corrupt its latest event on disk -- the storage-corruption case - // the reader must surface as `workflow_run_event_unreadable` rather than - // misclassify. - const { store, repoId, principal } = - await makeClaimCheckStore("cc-corrupt-fs-"); - await store.writeTree(principal, repoId, "refs/heads/main", { - files: { - [`${WORKFLOW_RUN_RUNS_PREFIX}/corrupt/events/0.json`]: - runStartedBody("corrupt"), - }, - message: "seed a valid live run", - }); - const eventPath = path.join( - store.getRepoDir(repoId), - WORKFLOW_RUN_RUNS_PREFIX, - "corrupt", - "events", - "0.json", - ); - await fs.promises.writeFile(eventPath, "not valid json{"); - - await expect( - readWorkflowRunLifecycle(store, repoId, "corrupt"), - ).rejects.toThrow(/workflow_run_event_unreadable/); - }); - test("readCommittedWorkflowRunLifecycle wraps an unreadable latest event", async () => { // The committed reader reads git objects, so inject the corruption through // a reads stub whose latest blob will not parse. @@ -3622,24 +3601,6 @@ describe("claim-check API — resume-owned processing entries survive replay", ( ).rejects.toThrow(/workflow_run_event_unreadable/); }); - test("readWorkflowRunLifecycle surfaces a non-ENOENT events-directory read error", async () => { - // An absent events directory (ENOENT) classifies as absent, but a - // non-ENOENT read error must surface rather than be swallowed as absent. - // Seed the per-event `events` path as a file so readdir fails with ENOTDIR. - const { store, repoId } = await makeClaimCheckStore("cc-enotdir-"); - const runDir = path.join( - store.getRepoDir(repoId), - WORKFLOW_RUN_RUNS_PREFIX, - "notdir", - ); - await fs.promises.mkdir(runDir, { recursive: true }); - await fs.promises.writeFile(path.join(runDir, "events"), "not a directory"); - - await expect( - readWorkflowRunLifecycle(store, repoId, "notdir"), - ).rejects.toThrow(); - }); - test("a CancelRequested-without-finalizer run is still owned (its message stays suppressed)", async () => { const { store, repoId, principal } = await makeClaimCheckStore( "cc-owned-cancelling-", @@ -3664,7 +3625,12 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }, message: "seed a run with a pending cancel but no finalizer", }); - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned.has("cancelling")).toBe(true); }); @@ -3688,7 +3654,12 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }, message: "seed a live child run with no consumedMessageId", }); - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned.size).toBe(0); }); @@ -3709,18 +3680,28 @@ describe("claim-check API — resume-owned processing entries survive replay", ( }, message: "seed a sealed (terminated, compacted) run", }); - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned.has("sealed")).toBe(false); expect(owned.size).toBe(0); }); test("an absent runs directory yields an empty owned set", async () => { - const { store, repoId } = await makeClaimCheckStore("cc-owned-no-runs-"); + const { store, repoId, principal } = + await makeClaimCheckStore("cc-owned-no-runs-"); // A freshly-initialised repo has only a `.gitignore` genesis tree and - // no `runs/` directory. readdir on the missing directory surfaces - // ENOENT, which scanRunsForBoot treats as an empty owned set - // rather than throwing. - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + // no `runs/` directory, which scanRunsForBoot treats as an empty owned + // set rather than throwing. + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned.size).toBe(0); }); @@ -3751,7 +3732,12 @@ describe("claim-check API — resume-owned processing entries survive replay", ( message: "seed the live run for the owned entry", }); - const { ownedMessageIds: owned } = await scanRunsForBoot(store, repoId); + const { ownedMessageIds: owned } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); expect(owned).toEqual(new Set(["live"])); const replay = await replayProcessingToInbox( diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 4661a876f..d4795fac0 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -4248,7 +4248,7 @@ export type ScanRunsForBootResult = { /** * Walk `runs/` once and return the two boot-recovery inputs the supervisor's - * spawn needs, from a single traversal of the working tree via `getRepoDir`: + * spawn needs, from one snapshot of the committed history at `ref`: * * - `ownedMessageIds`: the `consumedMessageId`s of NON-terminal runs -- the * messages a live run still owns. Spawn feeds this into @@ -4264,99 +4264,75 @@ export type ScanRunsForBootResult = { * re-checks the run's max-seq event and no-ops a run that is not actually * terminal, so this scan may be loose. * - * The working tree tracks the run-event ref (`refs/heads/main`); the - * claim-check ref (`refs/heads/events`) cannot see it, which is why this lives - * at the caller rather than inside `replayProcessingToInbox`'s single-ref - * delta. A run whose log is sealed (combined `events.jsonl`, only permitted for - * a terminated run) contributes to neither set; an absent `runs/` directory - * yields empty results. + * Only committed history shows what a run took: a write lays its files in the + * checkout before it commits, so a failed write leaves an event there that the + * resumed run never reads, and mail held back for it would never be handled. + * The run events live on the run-event ref; the claim-check ref + * (`refs/heads/events`) cannot see them, which is why this lives at the caller + * rather than inside `replayProcessingToInbox`'s single-ref delta. A run whose + * log is sealed (combined `events.jsonl`, only permitted for a terminated run) + * contributes to neither set; an absent `runs/` directory yields empty + * results. */ export async function scanRunsForBoot( store: RepoStore, + principal: Principal, repoId: RepoId, + ref: string, ): Promise { - const fs = await import("node:fs/promises"); - const path = await import("node:path"); - const repoDir = store.getRepoDir(repoId); - const runsDir = path.join(repoDir, WORKFLOW_RUN_RUNS_PREFIX); - let runIds: string[]; - try { - runIds = await fs.readdir(runsDir); - } catch (cause) { - if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { - return { ownedMessageIds: new Set(), pendingSealRunIds: [] }; - } - throw cause; - } const owned = new Set(); const pendingSealRunIds: string[] = []; + const reads = await store.openCommittedReads(principal, repoId, ref); + if (reads === null) return { ownedMessageIds: owned, pendingSealRunIds }; + const runIds = (await reads.listDir(WORKFLOW_RUN_RUNS_PREFIX)).flatMap( + (entry) => (entry.type === "tree" ? [entry.name] : []), + ); + const decoder = new TextDecoder(); for (const runId of runIds) { - const runDir = path.join(runsDir, runId); - // A sealed run (combined events file) is terminal by the handler's - // own invariant -- only a terminated run is sealed -- so it owns - // nothing and is already folded. Its presence also means the per-event - // directory is absent. - let sealed = false; + const runPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`; + let eventEntries: CommittedTreeEntry[]; try { - await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE)); - sealed = true; - } catch (cause) { - // ENOENT is the normal "not sealed" case. Any other stat error leaves - // the run to fall through to the events-dir read below, which resolves - // it, so this catch is benign; warn so the anomaly is still visible. + // A sealed run (combined events file) is terminal by the handler's + // own invariant -- only a terminated run is sealed -- so it owns + // nothing and is already folded. + const runChildren = await reads.listDir(runPath); if ( - !(cause instanceof Error) || - !("code" in cause) || - cause.code !== "ENOENT" + runChildren.some( + (entry) => + entry.type === "blob" && entry.name === WORKFLOW_RUN_EVENTS_FILE, + ) ) { - logger.warn`scanRunsForBoot: stat of the sealed-log file for run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`; + continue; } - sealed = false; - } - if (sealed) continue; - const eventsDir = path.join(runDir, WORKFLOW_RUN_EVENTS_DIR); - let files: string[]; - try { - files = await fs.readdir(eventsDir); + // Empty when the run has no events yet (grants may be staged before + // the first event). + eventEntries = await reads.listDir( + `${runPath}/${WORKFLOW_RUN_EVENTS_DIR}`, + ); } catch (cause) { - // ENOENT means the run has neither a sealed log nor a per-event - // directory (grants may be staged before the first event); skip it. - // A non-ENOENT error drops the run from BOTH result sets, and a live - // run dropped from ownedMessageIds gets its message re-admitted and + // A failed read drops the run from BOTH result sets, and a live run + // dropped from ownedMessageIds gets its message re-admitted and // dispatched a second time on the same runId -- the double-driver // corruption this scan exists to prevent. Surface it, but still skip: // aborting the whole boot scan over one run is worse. - if ( - !(cause instanceof Error) || - !("code" in cause) || - cause.code !== "ENOENT" - ) { - logger.error`scanRunsForBoot: reading events for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`; - } + logger.error`scanRunsForBoot: reading events for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`; continue; } let terminal = false; let consumedMessageId: string | undefined; - for (const file of files) { - if (!file.endsWith(".json")) continue; + for (const entry of eventEntries) { + if (entry.type !== "blob" || !entry.name.endsWith(".json")) continue; let parsed: unknown; try { parsed = JSON.parse( - await fs.readFile(path.join(eventsDir, file), "utf8"), + decoder.decode(await reads.readBlobByOid(entry.oid)), ); } catch (cause) { - // A corrupt or unreadable event file drops this run's - // classification: a missed RunStarted re-admits its message (a - // second run on the same runId), a missed terminal event skips a - // needed seal. Surface it, but skip the file rather than abort the - // scan. ENOENT here is a benign race (the file vanished mid-scan). - if ( - !(cause instanceof Error) || - !("code" in cause) || - cause.code !== "ENOENT" - ) { - logger.error`scanRunsForBoot: reading event ${file} for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`; - } + // A corrupt or unreadable event drops this run's classification: a + // missed RunStarted re-admits its message (a second run on the same + // runId), a missed terminal event skips a needed seal. Surface it, + // but skip the event rather than abort the scan. + logger.error`scanRunsForBoot: reading event ${entry.name} for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`; continue; } if ( @@ -4389,11 +4365,10 @@ export async function scanRunsForBoot( export type WorkflowRunLifecycle = "absent" | "live" | "terminal"; /** - * Classify a run's lifecycle from a read surface. The committed (git-object) - * and working-tree (node:fs) readers share this core: a sealed combined log is + * Classify a run's lifecycle from a read surface: a sealed combined log is * terminal; otherwise the latest per-event file decides terminal-vs-live, and a * run with no events is absent. The surface owns every read detail -- the - * absent/ENOENT discrimination, the per-surface entry filter, and wrapping an + * absent/ENOENT discrimination, the entry filter, and wrapping an * unreadable latest event as `workflow_run_event_unreadable` -- so this core * never sees a raw read error. */ @@ -4465,79 +4440,6 @@ export async function readCommittedWorkflowRunLifecycle( }); } -/** - * Read the durable lifecycle of one run from the workflow-run working tree. - * `grants.json` alone is still an absent run: grants are staged before the - * first trigger, while the first event is the durable proof that the run was - * fired. A sealed event log is terminal by the kind handler's compaction - * invariant. - * - * The supervisor uses this when in-memory cohort membership is empty. That - * happens both for a genuinely new deployment and briefly during recovery, - * so treating both states as "fire" would start a second driver for a live - * log or reuse a terminal run. - */ -export async function readWorkflowRunLifecycle( - store: RepoStore, - repoId: RepoId, - runId: string, -): Promise { - const fs = await import("node:fs/promises"); - const path = await import("node:path"); - const runDir = path.join( - store.getRepoDir(repoId), - WORKFLOW_RUN_RUNS_PREFIX, - runId, - ); - const eventsDir = path.join(runDir, WORKFLOW_RUN_EVENTS_DIR); - return classifyRunLifecycle<{ seq: number; name: string }>({ - async sealedLogPresent() { - try { - await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE)); - return true; - } catch (cause) { - if ( - !(cause instanceof Error) || - !("code" in cause) || - cause.code !== "ENOENT" - ) { - throw cause; - } - return false; - } - }, - async listEventEntries() { - let files: string[]; - try { - files = await fs.readdir(eventsDir); - } catch (cause) { - if ( - cause instanceof Error && - "code" in cause && - cause.code === "ENOENT" - ) { - return []; - } - throw cause; - } - return files.flatMap((file) => { - const seq = parseEventSeq(file); - return seq === null ? [] : [{ seq, name: file }]; - }); - }, - async readEvent(entry) { - const eventPath = path.join(eventsDir, entry.name); - try { - return JSON.parse(await fs.readFile(eventPath, "utf8")); - } catch (cause) { - throw new Error(`workflow_run_event_unreadable: ${eventPath}`, { - cause, - }); - } - }, - }); -} - export type ReplayProcessingToInboxOpts = { /** * MessageIds whose run is still LIVE (non-terminal) and therefore owns diff --git a/packages/workflow-host/src/supervisor/crash-respawn.test.ts b/packages/workflow-host/src/supervisor/crash-respawn.test.ts index 60736fb51..b2ce0c013 100644 --- a/packages/workflow-host/src/supervisor/crash-respawn.test.ts +++ b/packages/workflow-host/src/supervisor/crash-respawn.test.ts @@ -99,6 +99,8 @@ function createStubRepoStore( writeRecorder?.record(args.preservePrefix); return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; }, + // Its writes land nowhere, so it holds no history. + openCommittedReads: async () => null, }; // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; missing methods surface as a precise failure via the proxy return new Proxy(stub as RepoStore, { diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 2491d7601..213b27320 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -69,6 +69,8 @@ function createStubRepoStore(baseDir: string): RepoStore { async writeTreePreservingPrefix() { return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; }, + // Its writes land nowhere, so it holds no history. + openCommittedReads: async () => null, }; // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; missing methods surface as a precise failure via the proxy return new Proxy(stub as RepoStore, { diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index e7b003b59..5cab97a92 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -576,6 +576,12 @@ function createStubRepoStore(opts: { * tests that assert per-call shape from racing across calls. */ statefulWrites?: boolean; + /** History the repo already holds, as a restart finds it. */ + committedFiles?: { + repoId: RepoId; + ref: string; + files: Record; + }; }): RepoStore { const committed = new Map>(); // Tracks the latest commit sha per (repoId, ref) so `resolveRef` answers the @@ -590,6 +596,18 @@ function createStubRepoStore(opts: { function keyFor(repoId: RepoId, ref: string, preservePrefix: string): string { return `${repoId.kind}/${repoId.id}\x00${ref}\x00${preservePrefix}`; } + if (opts.committedFiles !== undefined) { + const { repoId, ref, files } = opts.committedFiles; + committed.set( + keyFor(repoId, ref, ""), + new Map( + Object.entries(files).map(([filePath, content]) => [ + filePath, + new TextEncoder().encode(content), + ]), + ), + ); + } // Merge every committed submap for a (repoId, ref) across preserve-prefixes // into one repo-root-relative path -> bytes map, the coherent tree snapshot // `openCommittedReads` serves. @@ -969,6 +987,8 @@ async function buildBindings(opts: { message: string; }) => void | Promise; statefulWrites?: boolean; + // Committed files of the deployment's workflow-run repo. + committedRunFiles?: Record; inboxPrimitives?: InboxPrimitives; }): Promise { const repoStore = createStubRepoStore({ @@ -978,6 +998,15 @@ async function buildBindings(opts: { ? { beforeWrite: opts.beforeWrite } : {}), ...(opts.statefulWrites === true ? { statefulWrites: true } : {}), + ...(opts.committedRunFiles !== undefined + ? { + committedFiles: { + repoId: { kind: "workflow-run", id: "run_deployment-x" }, + ref: "refs/heads/main", + files: opts.committedRunFiles, + }, + } + : {}), }); return { repoStore, @@ -1208,6 +1237,7 @@ describe("createWorkflowSupervisor", () => { inboxPrimitives?: MemoryInboxPrimitives; mailBus?: ReturnType; statefulWrites?: boolean; + committedRunFiles?: Record; failControlWrite?: () => boolean; }) { const supervisorIpcKeyPair = await generateKeyPair(); @@ -1264,6 +1294,9 @@ describe("createWorkflowSupervisor", () => { ? { beforeWrite: opts.beforeWrite } : {}), ...(opts.statefulWrites === true ? { statefulWrites: true } : {}), + ...(opts.committedRunFiles !== undefined + ? { committedRunFiles: opts.committedRunFiles } + : {}), }); const bindings: WorkflowSupervisorBindings = { ...baseBindings, @@ -5054,26 +5087,18 @@ describe("createWorkflowSupervisor", () => { { resource: "thing", action: "read" }, ]); - const runEventsDir = path.join( - baseDir, - "workflow-run", - "run_deployment-x", - "runs", - "run_deployment-x", - "events", - ); - await fs.mkdir(runEventsDir, { recursive: true }); - await fs.writeFile( - path.join(runEventsDir, "0.json"), - JSON.stringify({ type: "RunStarted", seq: 0 }), - ); - await fs.writeFile( - path.join(runEventsDir, "1.json"), - JSON.stringify({ type: "RunCompleted", seq: 1 }), - ); - const wired = await spawnWithRunStart({ baseDir, + committedRunFiles: { + "runs/run_deployment-x/events/0.json": JSON.stringify({ + type: "RunStarted", + seq: 0, + }), + "runs/run_deployment-x/events/1.json": JSON.stringify({ + type: "RunCompleted", + seq: 1, + }), + }, onRunStart: async () => { return seededSnapshot(baseDir, { stepOrder: ["step-1"], @@ -5099,6 +5124,39 @@ describe("createWorkflowSupervisor", () => { await wired.supervisor.shutdown(); }); + test("a run whose start only the checkout holds is fired, not waited on", async () => { + const baseDir = await makeTempDir("supervisor-uncommitted-start-"); + await seedStepGrants(baseDir, "step-1", [ + { resource: "thing", action: "read" }, + ]); + // A write that failed before it committed left the start in the checkout. + const runEventsDir = path.join( + baseDir, + "workflow-run", + "run_deployment-x", + "runs", + "run_deployment-x", + "events", + ); + await fs.mkdir(runEventsDir, { recursive: true }); + await fs.writeFile( + path.join(runEventsDir, "0.json"), + JSON.stringify({ type: "RunStarted", seq: 0 }), + ); + + const wired = await spawnWithRunStart({ baseDir }); + + wired.mailBus.deliver( + "run_deployment-x@example.com", + new TextEncoder().encode("msg-1"), + ); + expect(await waitForTriggerFireRunIds(wired.supervisorToChild, 1)).toEqual([ + "run_deployment-x", + ]); + + await wired.supervisor.shutdown(); + }); + test("an instant park during the pre-wait window still advances the dispatch loop", async () => { const baseDir = await makeTempDir("supervisor-instant-park-"); await seedStepGrants(baseDir, "step-1", [ diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index ba77bdc9c..790a804c2 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -56,7 +56,7 @@ import { dequeueToProcessing as defaultDequeueToProcessing, markConsumed as defaultMarkConsumed, scanRunsForBoot, - readWorkflowRunLifecycle, + readCommittedWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, DEFAULT_CONSUMED_RETENTION_MS, @@ -1007,12 +1007,12 @@ export function createWorkflowSupervisor( markConsumed: defaultMarkConsumed, replayProcessingToInbox: defaultReplayProcessingToInbox, }; - const defaultInboxWritePrincipal: WorkflowRunSupervisorPrincipal = { + const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { kind: "supervisor", anchorRunId: bindings.anchorRunId, }; const inboxWritePrincipal: Principal = - bindings.inboxWritePrincipal ?? defaultInboxWritePrincipal; + bindings.inboxWritePrincipal ?? supervisorPrincipal; // Resolve the consumed-dedup retention horizon once at the bindings // edge (the layer that owns the operator config); every markConsumed // is threaded the concrete value. See `WorkflowSupervisorBindings. @@ -3103,7 +3103,9 @@ export function createWorkflowSupervisor( // scan. const scanDone = scanRunsForBoot( bindings.repoStore, + supervisorPrincipal, bindings.workflowRunRepoId, + bindings.workflowRunRef, ); const replayDone = scanDone .then(({ ownedMessageIds }) => @@ -3679,13 +3681,18 @@ export function createWorkflowSupervisor( // a child is rediscovering a live run after restart. Consult the durable // log before deciding that "no cohort" means "fire". A live log is added // back to cohort tracking so the existing terminal/park wait handles the - // recovery window; a terminal log is rejected permanently. + // recovery window; a terminal log is rejected permanently. The log is the + // committed one, which boot recovery and the child read too: a failed + // write can leave a run's start in the checkout alone. if (!cohortRunIds.has(runId)) { const lifecycle = terminalRunIds.has(runId) ? "terminal" - : await readWorkflowRunLifecycle( - bindings.repoStore, - bindings.workflowRunRepoId, + : await readCommittedWorkflowRunLifecycle( + await bindings.repoStore.openCommittedReads( + supervisorPrincipal, + bindings.workflowRunRepoId, + bindings.workflowRunRef, + ), runId, ); if (lifecycle === "terminal") { diff --git a/packages/workflow-host/src/testing/stub-repo-store.ts b/packages/workflow-host/src/testing/stub-repo-store.ts index 7612e1025..a59e5354e 100644 --- a/packages/workflow-host/src/testing/stub-repo-store.ts +++ b/packages/workflow-host/src/testing/stub-repo-store.ts @@ -6,14 +6,19 @@ // an incomplete version of the broad ones, and substituting a working store // would silently permit paths that currently fail loudly. // +// Every stub serves its repo from a directory under `baseDir`: `getRepoDir` +// names it, and `openCommittedReads` reads it as the committed tree, since a +// stub keeps no git history. +// // Only the two capability sets that were byte-identical across several files // live here. The rest of this repo's stub stores are genuinely different // fidelities, up to a 218-line in-memory implementation, and merging those // would trade a real assertion for a smaller diff. +import fs from "node:fs/promises"; import path from "node:path"; -import type { RepoId, RepoStore } from "@intx/hub-sessions"; +import type { CommittedReads, RepoId, RepoStore } from "@intx/hub-sessions"; export type StubRepoStoreOpts = { /** @@ -32,6 +37,9 @@ export function createStubRepoStore( getRepoDir(repoId: RepoId): string { return path.join(baseDir, repoId.kind, repoId.id); }, + async openCommittedReads(_principal, repoId) { + return directoryReads(path.join(baseDir, repoId.kind, repoId.id)); + }, ...(opts.writeTree === true ? { writeTreePreservingPrefix: async () => ({ @@ -54,3 +62,33 @@ export function createStubRepoStore( }, }); } + +// A blob's oid is its repo-relative path. +function directoryReads(repoDir: string): CommittedReads { + return { + async listDir(relPath) { + let dirents; + try { + dirents = await fs.readdir(path.join(repoDir, relPath), { + withFileTypes: true, + }); + } catch (cause) { + if ( + cause instanceof Error && + "code" in cause && + cause.code === "ENOENT" + ) { + return []; + } + throw cause; + } + return dirents.map((dirent) => ({ + name: dirent.name, + oid: path.join(relPath, dirent.name), + type: dirent.isDirectory() ? ("tree" as const) : ("blob" as const), + })); + }, + readBlobByOid: (oid) => fs.readFile(path.join(repoDir, oid)), + treeOid: async () => null, + }; +} From de8153d8513a8db0a9469eead7d23def26d4c9d5 Mon Sep 17 00:00:00 2001 From: FarinM Date: Sat, 24 Oct 2026 18:17:56 +0200 Subject: [PATCH 04/20] Keep mail a resumed run took across a workflow child respawn A recycle or crash respawn moved every mail still in processing back to the inbox, so mail a live run had already taken reached the new child again: a turn the run already had stalled the dispatch on a re-park that never came, and the start mail came back as a new turn. A respawn now leaves that mail for the resumed run and consumes it once the run next parks or ends. Mail of a run that had already ended goes back to the inbox, where the dispatch refuses it. A recycle replayed that mail before it killed the old child, which could still take mail in between. The replay now waits for the old child to be gone, for the writes it asked for, and for a boot replay still running, which would put back mail the replacement takes. A write waiting on a merge from a child that exited fails instead of holding the repository lock those replays need. --- packages/hub-sessions/src/index.ts | 1 + packages/hub-sessions/src/substrate.ts | 1 + .../src/workflow-run-kind.test.ts | 87 +- .../hub-sessions/src/workflow-run-kind.ts | 101 +- packages/workflow-host/README.md | 10 +- .../src/supervisor/crash-respawn.test.ts | 2 +- .../src/supervisor/lifecycle-races.test.ts | 2 +- .../supervisor/outbound-signed-send.test.ts | 2 +- .../src/supervisor/park-notify.test.ts | 2 +- .../src/supervisor/race-abort.test.ts | 40 + .../src/supervisor/race-abort.ts | 22 + .../re-emit-parked-correlations.test.ts | 2 +- .../src/supervisor/recycle.test.ts | 987 +++++++++++++++++- .../workflow-host/src/supervisor/recycle.ts | 59 +- .../src/supervisor/spawn-replay-fifo.test.ts | 2 +- .../supervisor/stale-cohort-routing.test.ts | 2 +- .../src/supervisor/substrate-write.test.ts | 2 +- .../src/supervisor/supervisor.test.ts | 6 +- .../src/supervisor/supervisor.ts | 231 +++- .../workflow-host/src/supervisor/types.ts | 12 + 20 files changed, 1450 insertions(+), 123 deletions(-) create mode 100644 packages/workflow-host/src/supervisor/race-abort.test.ts create mode 100644 packages/workflow-host/src/supervisor/race-abort.ts diff --git a/packages/hub-sessions/src/index.ts b/packages/hub-sessions/src/index.ts index 6f972764f..d930593e2 100644 --- a/packages/hub-sessions/src/index.ts +++ b/packages/hub-sessions/src/index.ts @@ -203,6 +203,7 @@ export { dequeueToProcessing, readProcessingEntry, markConsumed, + NotInProcessingError, classifyTerminalEvent, scanRunsForBoot, readCommittedWorkflowRunLifecycle, diff --git a/packages/hub-sessions/src/substrate.ts b/packages/hub-sessions/src/substrate.ts index 3fe151b72..79dbc03b4 100644 --- a/packages/hub-sessions/src/substrate.ts +++ b/packages/hub-sessions/src/substrate.ts @@ -21,6 +21,7 @@ export { dequeueToProcessing, enqueueInbox, markConsumed, + NotInProcessingError, parseEventSeq, scanRunsForBoot, readCommittedWorkflowRunLifecycle, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index ab91acbea..55b227733 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -13,6 +13,7 @@ import { enqueueInbox, dequeueToProcessing, markConsumed, + NotInProcessingError, parseEventSeq, readCommittedWorkflowRunLifecycle, scanRunsForBoot, @@ -3225,7 +3226,28 @@ describe("claim-check API — markConsumed", () => { runId: "run-X", consumedAt: 500, }), - ).rejects.toThrow(/claim_check_processing_not_found/); + ).rejects.toThrow(NotInProcessingError); + }); + + test("rejects a second consume of the same message", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-mark-twice-"); + await enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-1", + receivedAt: 100, + }); + await dequeueToProcessing(store, principal, repoId, ADDRESS); + const args = { + address: ADDRESS, + messageId: "msg-1", + runId: "run-X", + consumedAt: 500, + }; + await markConsumed(store, principal, repoId, args); + await expect(markConsumed(store, principal, repoId, args)).rejects.toThrow( + NotInProcessingError, + ); }); }); @@ -3418,6 +3440,57 @@ describe("claim-check API — resume-owned processing entries survive replay", ( expect(owned.has("done")).toBe(false); }); + test("scanRunsForBoot limited to one run reads only that run", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-owned-one-"); + await store.writeTree(principal, repoId, "refs/heads/main", { + files: { + [`${WORKFLOW_RUN_RUNS_PREFIX}/live/events/0.json`]: + runStartedBody("live"), + [`${WORKFLOW_RUN_RUNS_PREFIX}/other/events/0.json`]: + runStartedBody("other"), + }, + message: "seed two live runs", + }); + const { ownedMessageIds } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + { + runId: "live", + }, + ); + expect(ownedMessageIds).toEqual(new Set(["live"])); + }); + + test("scanRunsForBoot counts each turn a live run received as owned", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-owned-turns-"); + await store.writeTree(principal, repoId, "refs/heads/main", { + files: { + [`${WORKFLOW_RUN_RUNS_PREFIX}/live/events/0.json`]: + runStartedBody("live"), + [`${WORKFLOW_RUN_RUNS_PREFIX}/live/events/1.json`]: JSON.stringify({ + type: "SignalReceived", + seq: 1, + at: "2026-01-01T00:00:01.000Z", + signalName: "input", + signalId: "follow-up", + payload: null, + }), + }, + message: "seed a live run that received a follow-up turn", + }); + const { ownedMessageIds } = await scanRunsForBoot( + store, + principal, + repoId, + "refs/heads/main", + ); + expect(ownedMessageIds).toEqual(new Set(["live", "follow-up"])); + }); + test("scanRunsForBoot ignores events the checkout holds that never committed", async () => { const { store, repoId, principal } = await makeClaimCheckStore( "cc-owned-uncommitted-", @@ -3435,6 +3508,17 @@ describe("claim-check API — resume-owned processing entries survive replay", ( store.getRepoDir(repoId), WORKFLOW_RUN_RUNS_PREFIX, ); + await fs.promises.writeFile( + path.join(runsDir, "live", "events", "1.json"), + JSON.stringify({ + type: "SignalReceived", + seq: 1, + at: "2026-01-01T00:00:01.000Z", + signalName: "input", + signalId: "follow-up", + payload: null, + }), + ); await fs.promises.mkdir(path.join(runsDir, "lost", "events"), { recursive: true, }); @@ -3749,6 +3833,7 @@ describe("claim-check API — resume-owned processing entries survive replay", ( ); // Only the orphan is re-admitted; the owned entry stays in processing. expect(replay.replayedKeys).toEqual(["200-orphan"]); + expect(replay.retainedMessageIds).toEqual(["live"]); const repoDir = store.getRepoDir(repoId); const processingDir = path.join( diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index d4795fac0..4079c8aa4 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -3557,7 +3557,8 @@ export const workflowRunAuthorize: AuthorizeFn = ( // consumed/.json dedup index // entry. // replayProcessingToInbox — recovery path that moves every -// processing entry back to inbox preserving +// processing entry no live run owns back to +// inbox preserving // its `-` filename // key so FIFO ordering survives a crash. // @@ -4112,13 +4113,25 @@ export type MarkConsumedResult = { prunedMessageIds: string[]; }; +/** + * `markConsumed` found no processing entry for the message: it is in the + * consumed index already, or was never dequeued. Retrying cannot succeed. + */ +export class NotInProcessingError extends Error { + constructor(message: string) { + super(message); + this.name = "NotInProcessingError"; + } +} + /** * Atomically remove the processing entry for `messageId` at `address`, * write the canonical `consumed/.json` dedup index entry, * advance the per-address retention watermark, and prune consumed * entries the watermark has passed. The caller is expected to have * called `dequeueToProcessing` for this messageId; calling - * `markConsumed` without a matching processing entry throws. + * `markConsumed` without a matching processing entry throws + * `NotInProcessingError`. * * The consumed envelope preserves the original `receivedAt` from the * processing entry so the dedup index doubles as an audit record. @@ -4153,7 +4166,7 @@ export async function markConsumed( const consumedFull = consumedPath(addressSegment, args.messageId); const consumedFname = `${args.messageId}.json`; if (listing.consumed.some((e) => e.name === consumedFname)) { - throw new Error( + throw new NotInProcessingError( `claim_check_already_consumed: ${consumedFull} already in the dedup index`, ); } @@ -4162,7 +4175,7 @@ export async function markConsumed( e.name.endsWith(`-${args.messageId}.json`), ); if (processingEntry === undefined) { - throw new Error( + throw new NotInProcessingError( `claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`, ); } @@ -4239,6 +4252,8 @@ export async function markConsumed( export type ReplayProcessingToInboxResult = { commitSha: string; replayedKeys: string[]; + /** Owned entries left in processing/ for their live run to consume. */ + retainedMessageIds: string[]; }; export type ScanRunsForBootResult = { @@ -4250,13 +4265,16 @@ export type ScanRunsForBootResult = { * Walk `runs/` once and return the two boot-recovery inputs the supervisor's * spawn needs, from one snapshot of the committed history at `ref`: * - * - `ownedMessageIds`: the `consumedMessageId`s of NON-terminal runs -- the - * messages a live run still owns. Spawn feeds this into - * `replayProcessingToInbox`'s `ownedMessageIds` so a parked run's message is - * not re-admitted to inbox and dispatched a second time while the run is - * recovered by re-driving its durable log. Without this, the re-drive AND the - * re-triggered fresh run both re-park the same awaitSignal gate on the same - * runId, and the two concurrent runtime bodies race to a corrupt terminal. + * - `ownedMessageIds`: the `consumedMessageId`s and `SignalReceived` signal + * ids of NON-terminal runs -- the messages a live run still owns, the one + * that started it and each follow-up turn's. Spawn and every respawn feed + * this into `replayProcessingToInbox`'s `ownedMessageIds` so a parked run's + * message is not re-admitted to inbox and dispatched a second time while the + * run is recovered by re-driving its durable log. Without this, the re-drive + * AND the re-triggered fresh run both re-park the same awaitSignal gate on + * the same runId, and the two concurrent runtime bodies race to a corrupt + * terminal; a follow-up turn redelivered to a run that already received it is + * ignored, and its dispatch waits for a turn that never comes. * - `pendingSealRunIds`: runs that are terminal but still in per-event form -- * an interrupted fold left them unsealed. Spawn hands these to the recovery * sweep, which re-runs the idempotent fold. A terminal event is a *proposal*: @@ -4272,21 +4290,27 @@ export type ScanRunsForBootResult = { * rather than inside `replayProcessingToInbox`'s single-ref delta. A run whose * log is sealed (combined `events.jsonl`, only permitted for a terminated run) * contributes to neither set; an absent `runs/` directory yields empty - * results. + * results. `runId` limits the walk to that one run: a respawn needs only the + * top-level run, the one mail reaches. */ export async function scanRunsForBoot( store: RepoStore, principal: Principal, repoId: RepoId, ref: string, + opts: { readonly runId?: string } = {}, ): Promise { const owned = new Set(); const pendingSealRunIds: string[] = []; const reads = await store.openCommittedReads(principal, repoId, ref); if (reads === null) return { ownedMessageIds: owned, pendingSealRunIds }; - const runIds = (await reads.listDir(WORKFLOW_RUN_RUNS_PREFIX)).flatMap( + let runIds = (await reads.listDir(WORKFLOW_RUN_RUNS_PREFIX)).flatMap( (entry) => (entry.type === "tree" ? [entry.name] : []), ); + if (opts.runId !== undefined) { + const only = opts.runId; + runIds = runIds.filter((runId) => runId === only); + } const decoder = new TextDecoder(); for (const runId of runIds) { const runPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`; @@ -4319,7 +4343,10 @@ export async function scanRunsForBoot( continue; } let terminal = false; - let consumedMessageId: string | undefined; + // The message that started the run, and the mail of each turn it took + // since: a follow-up message is delivered with its message id as the + // signal id. + const taken: string[] = []; for (const entry of eventEntries) { if (entry.type !== "blob" || !entry.name.endsWith(".json")) continue; let parsed: unknown; @@ -4350,14 +4377,18 @@ export async function scanRunsForBoot( } if (type === "RunStarted" && "consumedMessageId" in parsed) { const mid = parsed.consumedMessageId; - if (typeof mid === "string") consumedMessageId = mid; + if (typeof mid === "string") taken.push(mid); + } + if (type === "SignalReceived" && "signalId" in parsed) { + const signalId = parsed.signalId; + if (typeof signalId === "string") taken.push(signalId); } } if (terminal) { pendingSealRunIds.push(runId); continue; } - if (consumedMessageId !== undefined) owned.add(consumedMessageId); + for (const messageId of taken) owned.add(messageId); } return { ownedMessageIds: owned, pendingSealRunIds }; } @@ -4442,25 +4473,27 @@ export async function readCommittedWorkflowRunLifecycle( export type ReplayProcessingToInboxOpts = { /** - * MessageIds whose run is still LIVE (non-terminal) and therefore owns - * its inbound message: recovery re-drives that run against the durable - * log, so re-admitting the message to `inbox/` would dispatch a SECOND - * run for it, colliding with the re-drive on the shared runId. Entries - * in this set are left in `processing/` untouched; the run's eventual - * `markConsumed` clears them. The caller computes this by reading the - * run event logs, which live on a DIFFERENT ref (`refs/heads/main`) - * than the claim-check subtree this operation commits to - * (`refs/heads/events`). Empty/absent means replay every processing - * entry (the pre-existing behaviour: recover all orphans). + * MessageIds a LIVE (non-terminal) run already took: the mail that started + * it, each follow-up turn's, and the signals it received. Recovery + * re-drives that run against the durable log, so re-admitting one to + * `inbox/` would dispatch a SECOND run on the shared runId, or a turn the + * run already has. Entries in this set are left in `processing/` and + * returned as `retainedMessageIds`, which the supervisor consumes once the + * run reaches a new park or ends. The caller computes this by reading the + * run event logs, which live on a DIFFERENT ref (`refs/heads/main`) than + * the claim-check subtree this operation commits to (`refs/heads/events`). + * Empty/absent means replay every processing entry (the pre-existing + * behaviour: recover all orphans). */ ownedMessageIds?: ReadonlySet; }; /** - * Recovery path: move every processing entry at `address` back to - * inbox preserving the original `-` filename - * key so FIFO ordering survives a workflow-process crash. Returns - * the set of keys that were moved; when nothing was in processing + * Recovery path: move every processing entry at `address` that no live + * run owns back to inbox preserving the original + * `-` filename key so FIFO ordering survives a + * workflow-process crash. Returns the set of keys that were moved and + * the ids of the owned entries left in place; when nothing was moved * the returned `replayedKeys` is empty (and the commit is a no-op * rewrite of the same tree). * @@ -4490,6 +4523,7 @@ export async function replayProcessingToInbox( const ref = claimCheckCommitRef(); const ownedMessageIds = opts.ownedMessageIds ?? new Set(); const replayedKeys: string[] = []; + const retainedMessageIds: string[] = []; const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, { changedPathPrefixes: new Set([addressPrefix(addressSegment)]), message: `replay processing ${address}`, @@ -4506,8 +4540,8 @@ export async function replayProcessingToInbox( // log) is owned by the recovery re-drive of that same run. Re- // admitting the message to inbox would dispatch a SECOND run for // it, colliding with the re-drive on the shared runId. Leave such - // an entry in processing untouched; the run's eventual - // `markConsumed` clears it. Only genuinely orphaned entries (no + // an entry in processing; the supervisor consumes it once the run + // reaches a new park or ends. Only genuinely orphaned entries (no // run, or a terminal run) are replayed. The run logs live on a // different ref, so the caller precomputes the owned set. const envelope = decodeQueueEnvelopeOrThrow( @@ -4515,6 +4549,7 @@ export async function replayProcessingToInbox( `${processingDir}${entry.name}`, ); if (ownedMessageIds.has(envelope.messageId)) { + retainedMessageIds.push(envelope.messageId); continue; } const inboxFull = `${inboxDir}${entry.name}`; @@ -4535,5 +4570,5 @@ export async function replayProcessingToInbox( return { puts, deletes }; }, }); - return { commitSha, replayedKeys }; + return { commitSha, replayedKeys, retainedMessageIds }; } diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index 9765bb11c..370668dfa 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -225,7 +225,15 @@ On an unexpected exit the supervisor, with no external intervention: back into `inbox/` under their original `-` keys. Those keys sort ahead of any mail that arrived during the kill/respawn gap, so the stranded entry is re-dispatched first and - FIFO ordering holds across the respawn boundary. + FIFO ordering holds across the respawn boundary. Mail a live run + already took, its start mail or a turn it received, stays in + `processing/`: the resumed run carries on with it, and the supervisor + consumes it once the run reaches a new park or ends, retrying a failed + consume a few times. A park the run only re-enters from its log does + not count, so until then the mail stays in `processing/`, never + delivered again. A recycle replays the same way, once it has killed the old + child, so the child cannot take mail between the check of what its runs + took and the replay. 2. Spawns a fresh workflow-process child against the same definition (reusing the recycle path's respawn machinery) and resumes dispatch. diff --git a/packages/workflow-host/src/supervisor/crash-respawn.test.ts b/packages/workflow-host/src/supervisor/crash-respawn.test.ts index b2ce0c013..52dba384f 100644 --- a/packages/workflow-host/src/supervisor/crash-respawn.test.ts +++ b/packages/workflow-host/src/supervisor/crash-respawn.test.ts @@ -337,7 +337,7 @@ function createMemoryInboxPrimitives(): InboxPrimitives & { replayedKeys.push(k); } s.processing.clear(); - return { commitSha: "memory", replayedKeys }; + return { commitSha: "memory", replayedKeys, retainedMessageIds: [] }; }, replayCalls() { return replayCalls; diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index d830c1aac..59eed1e58 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -152,7 +152,7 @@ function createMemoryInboxPrimitives(): InboxPrimitives { replayedKeys.push(k); } s.processing.clear(); - return { commitSha: "memory", replayedKeys }; + return { commitSha: "memory", replayedKeys, retainedMessageIds: [] }; }, }; } diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index 61c529340..d0cde9d09 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -71,7 +71,7 @@ function createNoopInboxPrimitives(): InboxPrimitives { throw new Error("markConsumed not exercised in the outbound test"); }, async replayProcessingToInbox() { - return { commitSha: "noop", replayedKeys: [] }; + return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, }; } diff --git a/packages/workflow-host/src/supervisor/park-notify.test.ts b/packages/workflow-host/src/supervisor/park-notify.test.ts index c04319a19..bc7444d9c 100644 --- a/packages/workflow-host/src/supervisor/park-notify.test.ts +++ b/packages/workflow-host/src/supervisor/park-notify.test.ts @@ -53,7 +53,7 @@ function createNoopInboxPrimitives(): InboxPrimitives { throw new Error("markConsumed not exercised in the park test"); }, async replayProcessingToInbox() { - return { commitSha: "noop", replayedKeys: [] }; + return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, }; } diff --git a/packages/workflow-host/src/supervisor/race-abort.test.ts b/packages/workflow-host/src/supervisor/race-abort.test.ts new file mode 100644 index 000000000..be07704c6 --- /dev/null +++ b/packages/workflow-host/src/supervisor/race-abort.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { getEventListeners } from "node:events"; + +import { raceAbort } from "./race-abort"; + +describe("raceAbort", () => { + test("settles with each wait and leaves no abort listener behind", async () => { + const controller = new AbortController(); + for (const value of [1, 2, 3]) { + expect(await raceAbort(controller.signal, Promise.resolve(value))).toBe( + value, + ); + } + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + test("settles with undefined once the signal aborts", async () => { + const controller = new AbortController(); + const raced = raceAbort( + controller.signal, + Promise.withResolvers().promise, + ); + controller.abort(); + expect(await raced).toBeUndefined(); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + test("a wait that rejects leaves no abort listener behind", async () => { + const controller = new AbortController(); + const cause = await raceAbort( + controller.signal, + Promise.reject(new Error("the wait failed")), + ).then( + () => undefined, + (rejected: unknown) => rejected, + ); + expect(cause).toBeInstanceOf(Error); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); +}); diff --git a/packages/workflow-host/src/supervisor/race-abort.ts b/packages/workflow-host/src/supervisor/race-abort.ts new file mode 100644 index 000000000..a4a338d06 --- /dev/null +++ b/packages/workflow-host/src/supervisor/race-abort.ts @@ -0,0 +1,22 @@ +/** + * Settle with `wait`, or with `undefined` once `signal` aborts, and drop the + * abort listener either way. `{ once: true }` removes a listener only when + * the abort fires, so a loop that races each wake against a long-lived + * signal would otherwise keep one listener per wake. + */ +export async function raceAbort( + signal: AbortSignal, + wait: Promise, +): Promise { + if (signal.aborted) return undefined; + const aborted = Promise.withResolvers(); + const onAbort = (): void => { + aborted.resolve(undefined); + }; + signal.addEventListener("abort", onAbort, { once: true }); + try { + return await Promise.race([wait, aborted.promise]); + } finally { + signal.removeEventListener("abort", onAbort); + } +} diff --git a/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts b/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts index 1fc1c6f67..04e7e644b 100644 --- a/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts +++ b/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts @@ -83,7 +83,7 @@ function createNoopInboxPrimitives(): InboxPrimitives { throw new Error("markConsumed not exercised in the re-emit test"); }, async replayProcessingToInbox() { - return { commitSha: "noop", replayedKeys: [] }; + return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, }; } diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 3c945a6e6..154471448 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -10,7 +10,14 @@ // re-seeds the workflow definition or mutates step credentials in a // way that would conflate the recycle path with a redeploy. -import { describe, test, expect } from "bun:test"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + test, + expect, +} from "bun:test"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -18,10 +25,13 @@ import path from "node:path"; import { type } from "arktype"; import { generateKeyPair } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; +import { base64Encode, hexEncode } from "@intx/types"; import type { CredentialDelivery } from "@intx/types/credential-delivery"; import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; +import type { RepoStore } from "@intx/hub-sessions"; +import { NotInProcessingError } from "@intx/hub-sessions"; import { + createLogCapture, createMemoryFrameStream, createMemoryNdjsonStream, createMockMailBus, @@ -227,7 +237,13 @@ async function driveReady( return childSender; } -function createMemoryInboxPrimitives(): InboxPrimitives { +function createMemoryInboxPrimitives(): InboxPrimitives & { + messageIds(address: string): { + inbox: string[]; + processing: string[]; + consumed: string[]; + }; +} { type Entry = { messageId: string; receivedAt: number; @@ -316,8 +332,15 @@ function createMemoryInboxPrimitives(): InboxPrimitives { break; } } + if (s.consumed.has(args.messageId)) { + throw new NotInProcessingError( + `claim_check_already_consumed: ${args.messageId}`, + ); + } if (foundKey === null || envelope === null) { - throw new Error("processing entry not found"); + throw new NotInProcessingError( + `claim_check_processing_not_found: ${args.messageId}`, + ); } s.processing.delete(foundKey); s.consumed.set(args.messageId, envelope); @@ -334,15 +357,30 @@ function createMemoryInboxPrimitives(): InboxPrimitives { prunedMessageIds: [], }; }, - async replayProcessingToInbox(_store, _principal, _repoId, address) { + async replayProcessingToInbox(_store, _principal, _repoId, address, opts) { const s = getOrCreate(address); const replayedKeys: string[] = []; + const retainedMessageIds: string[] = []; for (const [k, value] of s.processing) { + if (opts?.ownedMessageIds?.has(value.messageId) === true) { + retainedMessageIds.push(value.messageId); + continue; + } + s.processing.delete(k); s.inbox.set(k, value); replayedKeys.push(k); } - s.processing.clear(); - return { commitSha: "memory", replayedKeys }; + return { commitSha: "memory", replayedKeys, retainedMessageIds }; + }, + messageIds(address) { + const s = getOrCreate(address); + const ids = (entries: Map) => + [...entries.values()].map((entry) => entry.messageId); + return { + inbox: ids(s.inbox), + processing: ids(s.processing), + consumed: [...s.consumed.keys()], + }; }, }; } @@ -403,6 +441,7 @@ async function spawnSupervisor(opts: { recyclePolicy?: RecyclePolicyBounds; onRunStart?: WorkflowSupervisorBindings["onRunStart"]; credentialDelivery?: WorkflowSupervisorBindings["credentialDelivery"]; + overrides?: Partial; }) { await seedStepGrants(opts.baseDir, "step-1", [ { resource: "thing", action: "read" }, @@ -420,7 +459,10 @@ async function spawnSupervisor(opts: { ? { credentialDelivery: opts.credentialDelivery } : {}), }); - const supervisor = createWorkflowSupervisor(bindings); + const supervisor = createWorkflowSupervisor({ + ...bindings, + ...opts.overrides, + }); const spawnPromise = supervisor.spawn({ definitionHash: TEST_DEFINITION_HASH, @@ -1350,21 +1392,27 @@ describe("supervisor recycle: terminal-event broadcaster cohort", () => { }); describe("supervisor recycle: drain-side processing replay", () => { - test("triggerRecycle invokes replayProcessingToInbox between drain and kill", async () => { + test("triggerRecycle invokes replayProcessingToInbox after the kill", async () => { // This is a focused unit test against `triggerRecycle`'s ordering - // contract: the `replayProcessingToInbox` callback must fire after - // `drain` returns and before `killChildHandle` lands. The test - // observes the order via call recording. + // contract: the `replayProcessingToInbox` callback must fire once + // `killChildHandle` has landed, when the old child can no longer take + // mail. The test observes the order via call recording. const baseDir = await makeTempDir("recycle-replay-order-"); const ipcKeypair = await generateKeyPair(); const mailBus = createMockMailBus(); const tracker = createSpawnTracker({}); const calls: string[] = []; + const spawnReplayed = Promise.withResolvers(); const recordingInbox: InboxPrimitives = { ...createMemoryInboxPrimitives(), async replayProcessingToInbox(_store, _principal, _repoId, _address) { calls.push("replayProcessingToInbox"); - return { commitSha: "memory", replayedKeys: [] }; + spawnReplayed.resolve(undefined); + return { + commitSha: "memory", + replayedKeys: [], + retainedMessageIds: [], + }; }, }; await seedStepGrants(baseDir, "step-1", [ @@ -1404,8 +1452,9 @@ describe("supervisor recycle: drain-side processing replay", () => { if (first === undefined) throw new Error("first child missing"); await driveReady(first, ipcKeypair); await spawnPromise; - // Clear out the spawn-time replay call so we can isolate the - // recycle-time one. + // Clear out the spawn-time replay call, which can land after the spawn + // resolves, so we can isolate the recycle-time one. + await spawnReplayed.promise; calls.length = 0; const recyclePromise = supervisor.recycle({ reason: "ordering-test" }); @@ -1414,13 +1463,167 @@ describe("supervisor recycle: drain-side processing replay", () => { if (second === undefined) throw new Error("second child missing"); await driveReady(second, ipcKeypair); await recyclePromise; - // The recycle path invokes `replayProcessingToInbox` BEFORE the - // kill. The order recorded in `calls` reflects that. + // The recycle path invokes `replayProcessingToInbox` AFTER the kill. + // The order recorded in `calls` reflects that. const replayIdx = calls.indexOf("replayProcessingToInbox"); const killIdx = calls.indexOf("kill"); expect(replayIdx).toBeGreaterThanOrEqual(0); expect(killIdx).toBeGreaterThanOrEqual(0); - expect(replayIdx).toBeLessThan(killIdx); + expect(killIdx).toBeLessThan(replayIdx); + await supervisor.shutdown(); + }); +}); + +async function sendChildWrite( + childSender: ReturnType, + requestId: string, +): Promise { + await childSender.send({ + type: "substrate.write.request", + data: { + requestId, + repoId: { kind: "workflow-run", id: "run_deployment-x" }, + ref: "refs/heads/main", + preservePrefix: "runs/", + message: `child write ${requestId}`, + }, + }); +} + +describe("supervisor recycle: the old child's last writes", () => { + test.each([1, 2])( + "%d write(s) the old child left waiting on a merge do not hold up the replay", + async (writes) => { + const baseDir = await makeTempDir("recycle-merge-left-waiting-"); + const ipcKeypair = await generateKeyPair(); + const tracker = createSpawnTracker({}); + await seedStepGrants(baseDir, "step-1", [ + { resource: "thing", action: "read" }, + ]); + const bindings = await buildBindings({ + baseDir, + spawner: tracker.spawner, + mailBus: createMockMailBus(), + ipcKeypair, + }); + // Writes run their merge, which the child computes, one at a time as + // under the repo lock. + let lock: Promise = Promise.resolve(); + const repoStore = new Proxy(bindings.repoStore, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return ( + ...args: Parameters + ) => { + const write = lock.then(async () => { + await args[3].merge(new Map()); + return target.writeTreePreservingPrefix(...args); + }); + lock = write.catch(() => undefined); + return write; + }; + }, + }); + const supervisor = createWorkflowSupervisor({ ...bindings, repoStore }); + const spawned = supervisor.spawn({ + definitionHash: TEST_DEFINITION_HASH, + onInferenceEvent: () => undefined, + }); + await tracker.awaitChildren(1); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + const firstSender = await driveReady(first, ipcKeypair); + await spawned; + for (let write = 0; write < writes; write++) { + await sendChildWrite(firstSender, `left-waiting-${String(write)}`); + } + await waitForUpstreamPayload( + first.supervisorToChild, + "substrate.merge.request", + ); + + const recycled = supervisor.recycle({ reason: "merge left waiting" }); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await recycled; + // A write that reached its merge after the old child was cut off never + // asked that child. + expect( + parseControlFrames(first.supervisorToChild.flushed()).filter( + (frame) => frame.type === "substrate.merge.request", + ), + ).toHaveLength(1); + await supervisor.shutdown(); + }, + ); + + test("a crash respawn is not held up by a boot replay queued behind the dead child's merge", async () => { + const baseDir = await makeTempDir("respawn-boot-lock-"); + const inbox = createMemoryInboxPrimitives(); + const tracker = createSpawnTracker({}); + const ipcKeypair = await generateKeyPair(); + // The boot replay's claim-check write and a child's write take the same + // repo lock, and the child's write holds it through its merge. + let lock: Promise = Promise.resolve(); + function withLock(fn: () => Promise): Promise { + const run = lock.then(fn); + lock = run.catch(() => undefined); + return run; + } + const bootReplayHeld = Promise.withResolvers(); + let replays = 0; + const repoStore = createStubRepoStore(baseDir, { writeTree: true }); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: { + respawnBackoffInitialMs: 0, + repoStore: new Proxy(repoStore, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return ( + ...args: Parameters + ) => + withLock(async () => { + await args[3].merge(new Map()); + return target.writeTreePreservingPrefix(...args); + }); + }, + }), + inboxPrimitives: { + ...inbox, + async replayProcessingToInbox(...args) { + replays += 1; + if (replays === 1) await bootReplayHeld.promise; + return withLock(() => inbox.replayProcessingToInbox(...args)); + }, + }, + }, + }); + await sendChildWrite(firstSender, "dies-in-merge"); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await waitForUpstreamPayload( + first.supervisorToChild, + "substrate.merge.request", + ); + bootReplayHeld.resolve(undefined); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); await supervisor.shutdown(); }); }); @@ -2531,3 +2734,751 @@ describe("supervisor recycle: the new child's authority", () => { } }); }); + +const RUN = "run_deployment-x"; +const ADDRESS = "run_deployment-x@example.com"; + +// The run's log in the stub store's directory, which the stub serves as its +// committed history. +async function writeWorkingRunLog( + baseDir: string, + events: readonly Record[], +): Promise { + const dir = path.join(baseDir, "workflow-run", RUN, "runs", RUN, "events"); + await fs.mkdir(dir, { recursive: true }); + for (const [index, event] of events.entries()) { + const seq = index + 1; + await fs.writeFile( + path.join(dir, `${String(seq)}.json`), + JSON.stringify({ seq, at: "2026-10-01T00:00:00.000Z", ...event }), + ); + } +} + +function runStarted(): Record { + return { + type: "RunStarted", + runId: RUN, + definitionHash: "def-hash-abc", + trigger: { type: "manual", payload: null }, + }; +} + +// A respawned child resumes the run from its log, so mail the run already +// took stays in processing/ with it instead of being delivered again. +describe("supervisor respawn: mail a resumed run took", () => { + // Move a mail to processing/, as a dispatch that took it would. + async function takeMail( + inbox: ReturnType, + baseDir: string, + messageId: string, + receivedAt: number, + ): Promise { + const store = createStubRepoStore(baseDir); + const repoId = { kind: "workflow-run", id: RUN } as const; + const principal = { kind: "supervisor", anchorRunId: RUN } as const; + await inbox.enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId, + receivedAt, + }); + await inbox.dequeueToProcessing(store, principal, repoId, ADDRESS); + } + + async function setupOwned(opts: { + inbox: ReturnType; + inboxPrimitives?: Partial; + setTimer?: WorkflowSupervisorBindings["setTimer"]; + onSuspensionRegister?: WorkflowSupervisorBindings["onSuspensionRegister"]; + mailBus?: MailBusBindings; + sigtermExits?: boolean; + }) { + const baseDir = await makeTempDir("owned-"); + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + ]); + await takeMail(opts.inbox, baseDir, "mail-owned", 1); + const replays: Awaited< + ReturnType + >[] = []; + const replayed = createChangeNotifier(); + const tracker = createSpawnTracker( + opts.sigtermExits !== undefined + ? { sigtermExits: opts.sigtermExits } + : {}, + ); + const ipcKeypair = await generateKeyPair(); + const harness = await spawnSupervisor({ + baseDir, + tracker, + mailBus: opts.mailBus ?? createMockMailBus(), + ipcKeypair, + overrides: { + inboxPrimitives: { + ...opts.inbox, + async replayProcessingToInbox(...args) { + const result = await opts.inbox.replayProcessingToInbox(...args); + replays.push(result); + replayed.notify(); + return result; + }, + ...opts.inboxPrimitives, + }, + ...(opts.setTimer !== undefined ? { setTimer: opts.setTimer } : {}), + ...(opts.onSuspensionRegister !== undefined + ? { onSuspensionRegister: opts.onSuspensionRegister } + : {}), + }, + }); + await replayed.until(() => replays.length === 1); + return { ...harness, baseDir, tracker, ipcKeypair, replays }; + } + + test("a respawn during the boot replay waits for it, so mail the replacement takes stays taken", async () => { + const baseDir = await makeTempDir("boot-replay-"); + const inbox = createMemoryInboxPrimitives(); + const tracker = createSpawnTracker({}); + const ipcKeypair = await generateKeyPair(); + const bootScanning = Promise.withResolvers(); + const releaseBoot = Promise.withResolvers(); + const respawnWaits = Promise.withResolvers(); + let opened = 0; + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: { + inboxPrimitives: inbox, + onRespawnWait: (waiting) => { + if (waiting === "replay") respawnWaits.resolve(undefined); + }, + // The boot scan reads the history from before any run started, and + // stalls on it. + repoStore: new Proxy( + createStubRepoStore(baseDir, { writeTree: true }), + { + get(target, prop, receiver) { + if (prop !== "openCommittedReads") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + const reads = await target.openCommittedReads(...args); + opened += 1; + if (opened !== 1 || reads === null) return reads; + return { + ...reads, + async listDir(relPath: string) { + const entries = await reads.listDir(relPath); + bootScanning.resolve(undefined); + await releaseBoot.promise; + return entries; + }, + }; + }; + }, + }, + ), + }, + }); + await bootScanning.promise; + const principal = { kind: "supervisor", anchorRunId: RUN } as const; + await inbox.enqueueInbox( + createStubRepoStore(baseDir), + principal, + { kind: "workflow-run", id: RUN }, + { + address: ADDRESS, + messageId: "mail-during-boot", + receivedAt: 1, + rawMessage: base64Encode( + new TextEncoder().encode("Subject: during boot\r\n\r\ninput"), + ), + }, + ); + const recycling = supervisor.recycle({ reason: "during the boot replay" }); + // A respawn that does not wait spawns the replacement first, which then + // takes mail the boot replay puts back. + expect( + await Promise.race([ + respawnWaits.promise.then(() => "waited"), + tracker.awaitChildren(2).then(() => "spawned"), + ]), + ).toBe("waited"); + + releaseBoot.resolve(undefined); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await recycling; + await waitForUpstreamPayload(second.supervisorToChild, "trigger.fire"); + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: ["mail-during-boot"], + consumed: [], + }); + + await supervisor.shutdown(); + }); + + test("stays in processing/ and is consumed when the run next parks", async () => { + const inbox = createMemoryInboxPrimitives(); + const consumed = Promise.withResolvers(); + const { supervisor, firstSender } = await setupOwned({ + inbox, + inboxPrimitives: { + async markConsumed(...args) { + const result = await inbox.markConsumed(...args); + consumed.resolve(undefined); + return result; + }, + }, + }); + expect(inbox.messageIds(ADDRESS).processing).toEqual(["mail-owned"]); + + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + await consumed.promise; + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: [], + consumed: ["mail-owned"], + }); + + await supervisor.shutdown(); + }); + + test("a park read once shutdown has waited for its writes starts no consume", async () => { + const inbox = createMemoryInboxPrimitives(); + let consumes = 0; + const markerRegistered = Promise.withResolvers(); + // Shutdown unregisters the address once it has waited for the last write. + const bus = createMockMailBus(); + const unregistered = Promise.withResolvers(); + const { supervisor, firstSender, tracker } = await setupOwned({ + inbox, + inboxPrimitives: { + async markConsumed(...args) { + consumes += 1; + return inbox.markConsumed(...args); + }, + }, + onSuspensionRegister: ({ correlationId }) => { + if (correlationId === "corr-marker") + markerRegistered.resolve(undefined); + }, + mailBus: { + ...bus, + unregisterAddress(address) { + bus.unregisterAddress(address); + unregistered.resolve(undefined); + }, + }, + // A child that ignores SIGTERM keeps sending until shutdown's SIGKILL. + sigtermExits: false, + }); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + + const shuttingDown = supervisor.shutdown(); + await unregistered.promise; + // The supervisor takes frames in order, so once it registers the park sent + // next, it has read the run's own. + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + await firstSender.send({ + type: "park.notify", + data: { + runId: "run-marker", + correlationId: "corr-marker", + parkKind: "approval", + }, + }); + await markerRegistered.promise; + expect(consumes).toBe(0); + + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + await shuttingDown; + expect(inbox.messageIds(ADDRESS).processing).toEqual(["mail-owned"]); + }); + + test("is consumed when the run ended before the replay kept it", async () => { + const inbox = createMemoryInboxPrimitives(); + const baseDir = await makeTempDir("owned-ended-"); + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + ]); + await takeMail(inbox, baseDir, "mail-owned", 1); + const replaying = Promise.withResolvers(); + const release = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const consumed = Promise.withResolvers(); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker: createSpawnTracker({}), + mailBus: createMockMailBus(), + ipcKeypair: await generateKeyPair(), + overrides: { + onSuspensionRegister: () => registered.resolve(undefined), + inboxPrimitives: { + ...inbox, + async replayProcessingToInbox(...args) { + replaying.resolve(undefined); + await release.promise; + return inbox.replayProcessingToInbox(...args); + }, + async markConsumed(...args) { + const result = await inbox.markConsumed(...args); + consumed.resolve(undefined); + return result; + }, + }, + }, + }); + await replaying.promise; + + // The run ends while the replay still decides what it took. The park sent + // next shows the supervisor has taken the end. + await firstSender.send({ + type: "terminal.event", + data: { runId: RUN, seq: 1, kind: "RunCompleted", at: "test" }, + }); + await firstSender.send({ + type: "park.notify", + data: { + runId: "run-marker", + correlationId: "corr-marker", + parkKind: "approval", + }, + }); + await registered.promise; + release.resolve(undefined); + await consumed.promise; + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: [], + consumed: ["mail-owned"], + }); + + await supervisor.shutdown(); + }); + + test("is consumed when the run parked before the replay kept it", async () => { + const inbox = createMemoryInboxPrimitives(); + const baseDir = await makeTempDir("owned-parked-"); + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + ]); + await takeMail(inbox, baseDir, "mail-owned", 1); + const replaying = Promise.withResolvers(); + const release = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const consumed = Promise.withResolvers(); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker: createSpawnTracker({}), + mailBus: createMockMailBus(), + ipcKeypair: await generateKeyPair(), + overrides: { + onSuspensionRegister: () => registered.resolve(undefined), + inboxPrimitives: { + ...inbox, + async replayProcessingToInbox(...args) { + replaying.resolve(undefined); + await release.promise; + return inbox.replayProcessingToInbox(...args); + }, + async markConsumed(...args) { + const result = await inbox.markConsumed(...args); + consumed.resolve(undefined); + return result; + }, + }, + }, + }); + await replaying.promise; + + // The run parks while the replay still decides what it took, and the + // approval park sent next shows the supervisor has taken it. + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + await firstSender.send({ + type: "park.notify", + data: { + runId: "run-marker", + correlationId: "corr-marker", + parkKind: "approval", + }, + }); + await registered.promise; + release.resolve(undefined); + await consumed.promise; + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: [], + consumed: ["mail-owned"], + }); + + await supervisor.shutdown(); + }); + + test("a shutdown waits for the consume of mail a replay it waited for kept", async () => { + const inbox = createMemoryInboxPrimitives(); + const baseDir = await makeTempDir("owned-shutdown-"); + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + ]); + await takeMail(inbox, baseDir, "mail-owned", 1); + const replaying = Promise.withResolvers(); + const release = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const writesWaited = Promise.withResolvers(); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker: createSpawnTracker({}), + mailBus: createMockMailBus(), + ipcKeypair: await generateKeyPair(), + overrides: { + onSuspensionRegister: () => registered.resolve(undefined), + onShutdownWait: (waiting) => { + if (waiting === "writes") writesWaited.resolve(undefined); + }, + inboxPrimitives: { + ...inbox, + async replayProcessingToInbox(...args) { + replaying.resolve(undefined); + await release.promise; + return inbox.replayProcessingToInbox(...args); + }, + async markConsumed(...args) { + // The consume lands only once the shutdown waits for it. + await writesWaited.promise; + return inbox.markConsumed(...args); + }, + }, + }, + }); + await replaying.promise; + await firstSender.send({ + type: "terminal.event", + data: { runId: RUN, seq: 1, kind: "RunCompleted", at: "test" }, + }); + await firstSender.send({ + type: "park.notify", + data: { + runId: "run-marker", + correlationId: "corr-marker", + parkKind: "approval", + }, + }); + await registered.promise; + + const shutdown = supervisor.shutdown(); + release.resolve(undefined); + await shutdown; + expect(inbox.messageIds(ADDRESS).consumed).toEqual(["mail-owned"]); + }); + + test("a turn the run took stays in processing/ across a recycle", async () => { + const inbox = createMemoryInboxPrimitives(); + const { supervisor, baseDir, tracker, ipcKeypair, replays } = + await setupOwned({ inbox }); + // The run took a follow-up mail as its next turn. + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + { + type: "SignalReceived", + signalName: "turn", + signalId: "mail-turn", + payload: null, + }, + ]); + await takeMail(inbox, baseDir, "mail-turn", 2); + + const recycled = supervisor.recycle({ reason: "owned-mail" }); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await recycled; + expect(replays.at(-1)).toMatchObject({ + replayedKeys: [], + retainedMessageIds: ["mail-owned", "mail-turn"], + }); + + await supervisor.shutdown(); + }); + + const respawns: [ + string, + ( + supervisor: WorkflowSupervisor, + first: FakeChild, + ) => Promise | undefined, + ][] = [ + ["recycle", (supervisor) => supervisor.recycle({ reason: "owned-mail" })], + [ + "crash", + (_supervisor, first) => { + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + return undefined; + }, + ], + ]; + test.each(respawns)( + "a turn kept across a %s is consumed when the run parks in the new child", + async (_case, respawn) => { + const inbox = createMemoryInboxPrimitives(); + let consumed = 0; + const consumes = createChangeNotifier(); + const { supervisor, baseDir, tracker, ipcKeypair } = await setupOwned({ + inbox, + inboxPrimitives: { + async markConsumed(...args) { + const result = await inbox.markConsumed(...args); + consumed += 1; + consumes.notify(); + return result; + }, + }, + }); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + // Only the respawn's replay finds the turn the run took. + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + { + type: "SignalReceived", + signalName: "turn", + signalId: "mail-turn", + payload: null, + }, + ]); + await takeMail(inbox, baseDir, "mail-turn", 2); + + const respawned = respawn(supervisor, first); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await respawned; + await secondSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + await consumes.until(() => consumed === 2); + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: [], + consumed: ["mail-owned", "mail-turn"], + }); + + await supervisor.shutdown(); + }, + ); + + test("a turn the crashed child's last write records stays in processing/", async () => { + const inbox = createMemoryInboxPrimitives(); + const baseDir = await makeTempDir("owned-late-write-"); + const writing = Promise.withResolvers(); + const land = Promise.withResolvers(); + const respawnWaits = Promise.withResolvers(); + const replays: Awaited< + ReturnType + >[] = []; + const replayed = createChangeNotifier(); + const tracker = createSpawnTracker({}); + const ipcKeypair = await generateKeyPair(); + const base = createStubRepoStore(baseDir, { writeTree: true }); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: { + onRespawnWait: (waiting) => { + if (waiting === "writes") respawnWaits.resolve(undefined); + }, + // The child's write that records the run took its next turn. + repoStore: new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") + return Reflect.get(target, prop, receiver); + return async () => { + writing.resolve(undefined); + await land.promise; + await writeWorkingRunLog(baseDir, [ + runStarted(), + { + type: "SignalReceived", + signalName: "turn", + signalId: "mail-turn", + payload: null, + }, + ]); + return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; + }; + }, + }), + inboxPrimitives: { + ...inbox, + async replayProcessingToInbox(...args) { + const result = await inbox.replayProcessingToInbox(...args); + replays.push(result); + replayed.notify(); + return result; + }, + }, + }, + }); + await replayed.until(() => replays.length === 1); + await takeMail(inbox, baseDir, "mail-turn", 1); + await sendChildWrite(firstSender, "turn-write"); + await writing.promise; + + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + await respawnWaits.promise; + land.resolve(undefined); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await replayed.until(() => replays.length === 2); + expect(replays[1]).toMatchObject({ + replayedKeys: [], + retainedMessageIds: ["mail-turn"], + }); + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: ["mail-turn"], + consumed: [], + }); + + await supervisor.shutdown(); + }); + + // A run that has ended never parks again, so a failed consume is retried + // on a timer; one already consumed is dropped. + describe("a failed consume", () => { + const logs = createLogCapture(); + beforeAll(() => { + logs.install(); + }); + afterAll(() => { + logs.restore(); + }); + beforeEach(() => { + logs.reset(); + }); + + async function setupFailing( + consumeWith: ( + inbox: ReturnType, + ) => InboxPrimitives["markConsumed"], + ) { + const inbox = createMemoryInboxPrimitives(); + const markConsumed = consumeWith(inbox); + const retries: (() => void)[] = []; + const retried = createChangeNotifier(); + const harness = await setupOwned({ + inbox, + inboxPrimitives: { markConsumed: (...args) => markConsumed(...args) }, + // The retry backoffs are a second and then four; other timers stay + // real. + setTimer: (callback, ms) => { + if (ms !== 1_000 && ms !== 4_000) return setTimeout(callback, ms); + retries.push(callback); + retried.notify(); + return callback; + }, + }); + await harness.firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + return { ...harness, inbox, retries, retried }; + } + + test("is retried without the run parking again", async () => { + let calls = 0; + const consumed = Promise.withResolvers(); + const owned = await setupFailing((inbox) => async (...args) => { + calls += 1; + if (calls === 1) throw new Error("the claim-check write failed"); + const result = await inbox.markConsumed(...args); + consumed.resolve(undefined); + return result; + }); + await owned.retried.until(() => owned.retries.length === 1); + owned.retries[0]?.(); + await consumed.promise; + expect(owned.inbox.messageIds(ADDRESS).consumed).toEqual(["mail-owned"]); + + await owned.supervisor.shutdown(); + }); + + test("gives up after its last retry and leaves the mail in processing/", async () => { + const owned = await setupFailing(() => async () => { + throw new Error("the claim-check write keeps failing"); + }); + await owned.retried.until(() => owned.retries.length === 1); + owned.retries[0]?.(); + await owned.retried.until(() => owned.retries.length === 2); + owned.retries[1]?.(); + await logs.waitForError("gave up consuming mail-owned"); + expect(owned.retries).toHaveLength(2); + expect(owned.inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: ["mail-owned"], + consumed: [], + }); + + await owned.supervisor.shutdown(); + }); + + test("is not retried for mail something else already consumed", async () => { + let calls = 0; + const called = Promise.withResolvers(); + const owned = await setupFailing((inbox) => async (...args) => { + calls += 1; + // Something else consumes it first. + await inbox.markConsumed(...args); + try { + return await inbox.markConsumed(...args); + } finally { + called.resolve(undefined); + } + }); + await called.promise; + // A respawn replays, and so starts its child, only once the old child's + // writes, this consume among them, have settled. + const recycled = owned.supervisor.recycle({ reason: "consume settled" }); + await owned.tracker.awaitChildren(2); + const second = owned.tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + expect(calls).toBe(1); + expect(owned.retries).toEqual([]); + await driveReady(second, owned.ipcKeypair); + await recycled; + + await owned.supervisor.shutdown(); + }); + }); +}); diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 78d2a07e5..4a03b9eff 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -58,13 +58,15 @@ // loop dequeues in arrival order (the envelope's `receivedAt` prefix // preserves FIFO discipline across the gap). // -// Before the kill lands the recycle path calls +// Once the old child is gone the recycle path calls // `ctx.replayProcessingToInbox()` so any in-flight `processing/` // entries that the dying cohort's dispatch loop did not reach // `markConsumed` for get moved back to `inbox/` under their original // `-` keys. Without this step the dying child // would leave an orphaned processing entry that no live dispatch -// loop owns. +// loop owns. Mail a live run already took stays in `processing/` +// instead: the resumed run carries on with it, and the supervisor +// consumes it once the run next parks or ends. // // Three trigger origins funnel through `triggerRecycle(reason, ctx)`: // @@ -182,11 +184,12 @@ export interface RecycleContext { /** Supervisor-side drain primitive; sends the existing drain mail. */ readonly drain: (deadlineMs: number) => Promise; /** - * Replay any `processing/` claim-check entries for the deployment's - * mail address back to `inbox/` so the FIFO ordering survives the - * recycle. Invoked AFTER the drain step settles and BEFORE the kill - * step lands, which eliminates the race window where a processing - * entry would exist with no owner. The supervisor closes this + * Replay the `processing/` claim-check entries for the deployment's + * mail address that no live run took back to `inbox/` so the FIFO + * ordering survives the recycle, and leave the ones a live run took + * for it. Invoked AFTER the kill, once the old child can no longer + * take mail, and BEFORE the new child is spawned, so its dispatch + * loop finds every entry back in place. The supervisor closes this * callback over its `inboxPrimitives.replayProcessingToInbox` plus * the deployment's substrate principal and repo identity. */ @@ -194,9 +197,9 @@ export interface RecycleContext { /** * Abort the prior cohort's terminal source and wake the dispatch * loop so it exits before the kill step lands. Invoked AFTER drain - * and replay settle and BEFORE the kill -- earlier would starve the - * drain step's accumulators of live terminal events; later would - * race the kill against the dispatch loop's next iteration. + * settles and BEFORE the kill -- earlier would starve the drain + * step's accumulators of live terminal events; later would race the + * kill against the dispatch loop's next iteration. */ readonly abortPriorCohort: () => void; /** @@ -294,23 +297,8 @@ export async function triggerRecycle( // through the existing supervisor-drain origin. await ctx.drain(drainDeadlineMs); - // After drain settles and BEFORE the kill lands, replay any - // in-flight `processing/` entries back to `inbox/`. Without this - // replay, a child whose run was mid-dispatch when drain expired - // would leave its `processing/` entry orphaned -- the dispatch - // loop for the dying cohort already aborted on cohort teardown - // and will not reach its `markConsumed` step. The new child's - // dispatch loop dequeues the recovered entries in arrival order - // (the envelope's `receivedAt` prefix preserves FIFO discipline). - try { - await ctx.replayProcessingToInbox(); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`recycle: replayProcessingToInbox before kill failed: ${message}`; - } - - // Abort the prior cohort here -- after drain and replay have run - // against a live cohort, before the kill drops the child. Aborting + // Abort the prior cohort here -- after drain has run against a live + // cohort, before the kill drops the child. Aborting // up front (in the supervisor wrapper) would starve drain // accumulators of live terminal events and force every recycle to // pay the full drainTimeout budget. Aborting after the kill would @@ -328,6 +316,23 @@ export async function triggerRecycle( ...(ctx.clearTimer !== undefined ? { clearTimer: ctx.clearTimer } : {}), }); + // Replay the in-flight `processing/` entries no live run took back to + // `inbox/`. Without this replay, a child whose run was mid-dispatch + // when drain expired would leave its `processing/` entry orphaned -- + // the dispatch loop for the dying cohort already aborted on cohort + // teardown and will not reach its `markConsumed` step. It runs after + // the kill: a live child could still take mail between the scan of + // what its runs took and the replay, and that mail would come back as + // new. The new child's dispatch loop dequeues the recovered entries in + // arrival order (the envelope's `receivedAt` prefix preserves FIFO + // discipline). + try { + await ctx.replayProcessingToInbox(); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`recycle: replayProcessingToInbox after kill failed: ${message}`; + } + // Step 3: respawn. Fresh channelId, fresh HMAC key, fresh Ed25519 // IPC keypair. The new child gets the run's current grants and // credential material in its first frame, before it resumes any run. diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index b3599bb86..b47dbe62d 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -209,7 +209,7 @@ function createGatedInboxPrimitives(): { resolveReplaySettled(); resolveReplaySettled = null; } - return { commitSha: "memory", replayedKeys }; + return { commitSha: "memory", replayedKeys, retainedMessageIds: [] }; }, }; return { diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 213b27320..57fd39df7 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -189,7 +189,7 @@ function createMemoryInbox(): InboxPrimitives & { s.inbox.set(key, value); } s.processing.clear(); - return { commitSha: "memory", replayedKeys: [] }; + return { commitSha: "memory", replayedKeys: [], retainedMessageIds: [] }; }, seedProcessing(address, messageId, receivedAt) { const s = get(address); diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index 3408cebe7..c0b434a64 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -309,7 +309,7 @@ function createMemoryInboxPrimitives(opts: { replayedKeys.push(k); } s.processing.clear(); - return { commitSha: "memory", replayedKeys }; + return { commitSha: "memory", replayedKeys, retainedMessageIds: [] }; }, }; } diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 5cab97a92..bc249f6b1 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -964,7 +964,11 @@ function createMemoryInboxPrimitives(): MemoryInboxPrimitives { // await landing in between would not strand a waiter on `processing` // emptying. changes.notify(); - return { commitSha: "memory-inbox", replayedKeys }; + return { + commitSha: "memory-inbox", + replayedKeys, + retainedMessageIds: [], + }; }, }; } diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 790a804c2..f9830601c 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -59,6 +59,7 @@ import { readCommittedWorkflowRunLifecycle, replayProcessingToInbox as defaultReplayProcessingToInbox, StaleInboxEnqueueError, + NotInProcessingError, DEFAULT_CONSUMED_RETENTION_MS, type Principal, type WorkflowRunSupervisorPrincipal, @@ -168,6 +169,7 @@ import { killChildHandle, waitDeadline, } from "./child-termination"; +import { raceAbort } from "./race-abort"; const logger = getLogger(["workflow-host", "supervisor"]); @@ -244,6 +246,10 @@ export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000; */ export const TERMINAL_OR_PARK_BACKSTOP_MS = 300_000; +// Attempts to consume mail a resumed run took, a second and then four apart. +const OWNED_CONSUME_ATTEMPTS = 3; +const OWNED_CONSUME_BACKOFF_MS = 1_000; + /** * Public surface returned by `createWorkflowSupervisor`. Each method * advances the supervisor through one lifecycle transition; the @@ -1414,8 +1420,8 @@ export function createWorkflowSupervisor( priorContext, // The child is already dead: there is nothing to drain, and // `runRespawn`'s kill step is a no-op on a dead handle. The replay - // step still runs, moving any mail stranded mid-flight back to the - // inbox tail before dispatch resumes. + // step still runs, moving mail stranded mid-flight that no live run + // took back to the inbox tail before dispatch resumes. drain: async () => undefined, }); // The respawned child is now the running cohort. Arm the stable-run @@ -1828,6 +1834,7 @@ export function createWorkflowSupervisor( const event = terminalEventFromPayload(payload.data); cohortBroadcaster.notify(payload.data.runId, event); terminalRunIds.add(payload.data.runId); + consumeOwnedMessages(payload.data.runId); // Clean up cohort tracking for the terminated run. Self-discovered // runs have no dispatch-loop entry, so their cleanup happens here. cohortRunIds.delete(payload.data.runId); @@ -1892,6 +1899,7 @@ export function createWorkflowSupervisor( ); // Wake any dispatch loop waiting for this run to park. resolveParkNotifyWaiter(payload.data.runId); + consumeOwnedMessages(payload.data.runId); continue; } if (payload.type === "parked-correlations.response") { @@ -1931,6 +1939,22 @@ export function createWorkflowSupervisor( ) => void; }; const pendingMerges = new Map(); + // Aborted once a child exits, so a wait on it ends without each one + // subscribing to `exited` and staying subscribed after it is answered. + const childExitSignals = new WeakMap(); + function childExitSignal(handle: SubprocessHandle): AbortSignal { + let signal = childExitSignals.get(handle); + if (signal === undefined) { + const exited = new AbortController(); + const abort = (): void => { + exited.abort(); + }; + void handle.exited.then(abort, abort); + signal = exited.signal; + childExitSignals.set(handle, signal); + } + return signal; + } const pendingCancellations = new Map< string, { @@ -1940,9 +1964,21 @@ export function createWorkflowSupervisor( >(); let cancellationSeq = 0; const cancellationCommits = new Set>(); + // Mail a resumed run took before its child was replaced. The dispatch that + // took it is gone, so it is consumed when the run next parks or ends. + const ownedMessages = new Set(); + // Retries of a failed consume, which a run that has ended would not + // trigger again. + const ownedConsumeRetries = new Set(); + // The boot replay. It read history from before any respawn, so a respawn + // waits for it. + let bootReplay: Promise | null = null; // Repository writes started off the control pump. Each promise settles // without rejecting; its handler logs any failure. const detachedWrites = new Set>(); + // Set once shutdown has waited for the last of them. A write started after + // that would land after the host may have removed the run's repository. + let detachedWritesClosed = false; function ownDetachedWrite(write: Promise): void { detachedWrites.add(write); @@ -2610,8 +2646,8 @@ export function createWorkflowSupervisor( async function handleSubstrateWriteRequest( data: Extract["data"], ): Promise { - const controlSender = activeControlSender(); - if (controlSender === null) { + const cohort = activeCohort(); + if (cohort === null) { // The request arrived after `activeControlSender()` was // cleared (the supervisor is in `recycling` mid-swap, or // `draining`/`stopping`/`stopped`). The child's read end of @@ -2626,6 +2662,7 @@ export function createWorkflowSupervisor( logger.warn`substrate.write.request received outside running phase; requestId=${data.requestId} dropped (child waiter will fail on pipe close)`; return; } + const controlSender = cohort.controlSender; const validatedRepoId = RepoId(data.repoId); if (validatedRepoId instanceof type.errors) { onChildCrash( @@ -2702,13 +2739,21 @@ export function createWorkflowSupervisor( preservePrefix: data.preservePrefix, message: data.message, merge: async (existing) => { - const sender = activeControlSender(); - if (sender === null) { - throw new Error( - "supervisor substrate.write.request: control channel unavailable for merge round-trip", - ); + // Only the child that asked for the write can compute its merge. + // One cut off from its cohort may already be dead, and a send to + // a dead child's pipe succeeds without an answer ever coming. + // The wait holds the repo lock, so it also ends when the child + // exits. + const childGone = + "supervisor substrate.write.request: the workflow child that asked for this write is gone"; + const sender = controlSender; + if ( + cohort.terminalCohortAbort.signal.aborted || + activeControlSender() !== sender + ) { + throw new Error(childGone); } - const result = await new Promise< + const answered = new Promise< | { ok: true; files: Record } | { ok: false; reason: string } >((resolve) => { @@ -2741,6 +2786,12 @@ export function createWorkflowSupervisor( }); }); }); + const result = await raceAbort( + childExitSignal(cohort.handle), + answered, + ); + pendingMerges.delete(data.requestId); + if (result === undefined) throw new Error(childGone); if (!result.ok) { throw new Error( `supervisor substrate.write.request: child merge failed: ${result.reason}`, @@ -2807,17 +2858,24 @@ export function createWorkflowSupervisor( } } - function activeControlSender(): ControlChannelSender | null { + function activeCohort(): Extract< + SupervisorState, + { phase: "starting" | "running" | "recycling" } + > | null { if ( state.phase === "starting" || state.phase === "running" || state.phase === "recycling" ) { - return state.controlSender; + return state; } return null; } + function activeControlSender(): ControlChannelSender | null { + return activeCohort()?.controlSender ?? null; + } + async function wireChild(args: { channelId: string; hmacKey: Uint8Array; @@ -3117,7 +3175,8 @@ export function createWorkflowSupervisor( { ownedMessageIds }, ), ) - .then(() => { + .then(({ retainedMessageIds }) => { + keepOwnedMessages(retainedMessageIds, { childRunning: true }); wakeDispatch(); }) .catch((cause) => { @@ -3143,6 +3202,7 @@ export function createWorkflowSupervisor( // flight would otherwise leave the substrate write pending past // the supervisor's exit. state.replayDone = replayDone; + bootReplay = replayDone; // Re-seal runs a crash left terminal-but-per-event when their // fire-and-forget fold never ran. Unlike the replay above, this must @@ -3809,10 +3869,11 @@ export function createWorkflowSupervisor( // commit lands -- so the substrate subscription IS the ack, and a // failed deliver commit is observed by nothing, never re-parks, and // never releases markConsumed (the mail stays reclaimable). A crash - // before the re-park/terminal leaves the claim-check entry in - // processing/, so replayProcessingToInbox re-delivers the signal on - // restart. On cohort abort the wait returns and the post-loop guard - // skips markConsumed. + // before the child commits the signal leaves the claim-check entry + // in processing/ for replayProcessingToInbox to re-deliver; after + // it, the resumed run owns the mail and the supervisor consumes it + // once the run next parks or ends. On cohort abort the wait returns + // and the post-loop guard skips markConsumed. waitEntered = true; await waitForRunTerminalOrPark( iter, @@ -4277,6 +4338,8 @@ export function createWorkflowSupervisor( async function performShutdown(opts: ShutdownOptions): Promise { const prior = state; state = { phase: "stopping" }; + for (const retry of ownedConsumeRetries) readyClearTimer(retry); + ownedConsumeRetries.clear(); // shutdownInternal is designed to be TOTAL: when a child is up it must // always kill it and always reach `stopped`, no matter which teardown // step throws. Rather than depend on every step being individually @@ -4371,11 +4434,6 @@ export function createWorkflowSupervisor( // An already-started signed append may finish after the child exits. // Own that write through teardown before the host releases its bindings. await Promise.allSettled([...cancellationCommits]); - // Own the pump's detached writes the same way, so every write the child - // requested lands before shutdown resolves. New merges fail once - // stopping, so what remains is local I/O. A finishing write can start a - // fold. - while (detachedWrites.size > 0) await Promise.all([...detachedWrites]); // Await every accumulator's `disposed()` so a pending escalation // commit or terminal-event watcher coroutine cannot outlive the // supervisor and fire against torn-down bindings. @@ -4425,6 +4483,15 @@ export function createWorkflowSupervisor( the in-flight fold's substrate commit to settle. */ }); } + // Own the detached writes last, so every write lands before shutdown + // resolves: the child's, and the consumes the waits above can start. New + // merges fail once stopping, so what remains is local I/O. A finishing + // write can start a fold. + while (detachedWrites.size > 0) { + bindings.onShutdownWait?.("writes"); + await Promise.all([...detachedWrites]); + } + detachedWritesClosed = true; if (recyclePolicy !== null) { try { recyclePolicy.stop(); @@ -4672,12 +4739,12 @@ export function createWorkflowSupervisor( respawnInProgress = true; const { origin, reason, prior, priorContext, drain } = args; // The cohort abort no longer fires up-front. triggerRecycle drives - // the drain and replay steps against a LIVE cohort first, then - // invokes `abortPriorCohort` (the callback below) between replay - // and the kill step. Aborting up-front would starve drain - // accumulators of live terminal events; aborting after the kill - // would race the dispatch loop's next iteration against the - // controlSender that's about to disappear. + // the drain step against a LIVE cohort first, then invokes + // `abortPriorCohort` (the callback below) between the drain and the + // kill, and replays the processing mail after the kill. Aborting + // up-front would starve drain accumulators of live terminal events; + // aborting after the kill would race the dispatch loop's next + // iteration against the controlSender that's about to disappear. const priorDispatchLoop = prior.dispatchLoop; // Transition to `recycling`. Inbound mail continues to flow through // `enqueueInbox` unchanged; the prior dispatch loop is still alive @@ -4730,15 +4797,42 @@ export function createWorkflowSupervisor( }, drain, replayProcessingToInbox: async () => { - await inboxPrimitives.replayProcessingToInbox( + // A boot replay still running would put back mail the replacement + // takes. + if (bootReplay !== null) { + bindings.onRespawnWait?.("replay"); + await bootReplay; + } + // Mail a live run already took stays in processing/, as on spawn: + // the resumed run carries on with it. Redelivered, its start mail + // would reach it again as a new turn, and a turn it already has + // would stall the dispatch on a re-park that never comes. The old + // child is gone: its round-trips end, and the writes it asked for + // settle before the scan. + rejectCohortAwaiters("recycle"); + while (detachedWrites.size > 0) { + bindings.onRespawnWait?.("writes"); + await Promise.all([...detachedWrites]); + } + const { ownedMessageIds } = await scanRunsForBoot( bindings.repoStore, - inboxWritePrincipal, + supervisorPrincipal, bindings.workflowRunRepoId, - bindings.deploymentMailAddress, + bindings.workflowRunRef, + { runId: deriveWorkflowRunId(bindings.deploymentMailAddress) }, ); + const { retainedMessageIds } = + await inboxPrimitives.replayProcessingToInbox( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + { ownedMessageIds }, + ); + keepOwnedMessages(retainedMessageIds, { childRunning: false }); }, abortPriorCohort: () => { - // Fired by triggerRecycle between drain/replay and kill. + // Fired by triggerRecycle between the drain and the kill. // The prior dispatch loop notices the abort on its next // wake and exits before the kill drops the child. prior.terminalCohortAbort.abort(); @@ -4780,8 +4874,8 @@ export function createWorkflowSupervisor( return; } // The previous cohort was aborted inside triggerRecycle - // by the `abortPriorCohort` callback (after drain and - // replay, before kill) so the prior dispatch loop did not + // by the `abortPriorCohort` callback (after drain, before + // kill) so the prior dispatch loop did not // race the kill/respawn gap. Stop every armed accumulator // (they were tracking runs that lived inside the killed // child); the resumed child re-discovers any survivors @@ -5018,6 +5112,75 @@ export function createWorkflowSupervisor( }); } + // Mail a replay left with the run that took it. A run that parked or ended + // while the replay ran is already past that mail, so it is consumed now. Only + // a first spawn's child runs during the replay; a respawn replays before its + // child starts, while the park generations are still the old child's. A run + // that had ended before the scan owns no mail: the replay puts its mail back, + // and the dispatch refuses it. + function keepOwnedMessages( + messageIds: readonly string[], + opts: { childRunning: boolean }, + ): void { + for (const messageId of messageIds) ownedMessages.add(messageId); + const runId = deriveWorkflowRunId(bindings.deploymentMailAddress); + const parked = opts.childRunning && parkGenerations.has(runId); + if (parked || terminalRunIds.has(runId)) consumeOwnedMessages(runId); + } + + function consumeOwnedMessages(runId: string): void { + if (runId !== deriveWorkflowRunId(bindings.deploymentMailAddress)) return; + // The mail stays in processing/ for the next boot's scan, which keeps it + // for a run still live. + if (detachedWritesClosed) return; + for (const messageId of [...ownedMessages]) { + ownedMessages.delete(messageId); + consumeOwnedMessage(runId, messageId, 1); + } + } + + function consumeOwnedMessage( + runId: string, + messageId: string, + attempt: number, + ): void { + ownDetachedWrite( + inboxPrimitives + .markConsumed( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + { + address: bindings.deploymentMailAddress, + messageId, + runId, + consumedAt: Date.now(), + retentionHorizonMs: consumedRetentionMs, + }, + ) + .catch((cause: unknown) => { + // Something else consumed it already; a retry cannot find it either. + if (cause instanceof NotInProcessingError) return; + if (state.phase === "stopping" || state.phase === "stopped") return; + const message = + cause instanceof Error ? cause.message : String(cause); + if (attempt >= OWNED_CONSUME_ATTEMPTS) { + logger.error`gave up consuming ${messageId}, which the resumed run ${runId} took, after ${String(attempt)} attempts; it stays in processing/ until the next respawn: ${message}`; + return; + } + logger.warn`consuming ${messageId}, which the resumed run ${runId} took, failed; retrying: ${message}`; + const retry = readySetTimer( + () => { + ownedConsumeRetries.delete(retry); + consumeOwnedMessage(runId, messageId, attempt + 1); + }, + OWNED_CONSUME_BACKOFF_MS * 4 ** (attempt - 1), + ); + ownedConsumeRetries.add(retry); + }), + ); + } + async function deliverSources(opts: DeliverSourcesOpts): Promise { // The supervisor is the single producer of `sources-updated` control // frames. `recycling` is rejected for the same reason as diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index b4c7a3c5e..8a4cf89c6 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -319,6 +319,18 @@ export interface WorkflowSupervisorBindings { phase: "stopped" | "crash-looping"; reason: string; }) => void; + /** + * Test seam, unset in production. Called each time a shutdown waits for + * work it owns before it resolves: `writes` for repository writes started + * off the control pump. + */ + onShutdownWait?: (waiting: "writes") => void; + /** + * Test seam, unset in production. Called each time a respawn, before it + * scans `runs/`, waits for the boot replay (`replay`) or for repository + * writes started off the control pump (`writes`). + */ + onRespawnWait?: (waiting: "replay" | "writes") => void; /** * Per-run grants source. The dispatch loop consults it before it forwards a * `trigger.fire`, and every new child (spawn, crash-respawn, recycle) gets From 104b59f0ea85eff42ffdd5a3e3f8037330378608 Mon Sep 17 00:00:00 2001 From: FarinM Date: Sun, 25 Oct 2026 20:54:01 +0100 Subject: [PATCH 05/20] Let a workflow child report when every run only waits on outside input A host that stops a workflow child loses whatever the child was doing, so it may stop the child only while no run works and no committed input is still on its way to the run that waits for it. On `quiet.request` the child answers that the run has ended, or whether it is quiet together with the earliest timer its scheduler has queued, so the host can wake it for that timer. Waits on signals and child handles take a required `onWake`, so whatever wakes a waiting step counts it as working before the step stops counting itself. Scheduler recovery no longer queues a timer whose step settled after it was set, or whose run ended, since the answer would report a wake for it that does nothing. --- docs/IMPLEMENTATION.md | 15 +- packages/workflow-host/README.md | 3 +- .../src/adapters/spawn-child.test.ts | 8 +- .../src/child/proxy-repo-store.ts | 27 + .../src/child/quiet-handoff.test.ts | 764 ++++++++++++++++++ .../src/child/run-child-source-ref.test.ts | 1 + .../workflow-host/src/child/run-child.test.ts | 589 ++++++++++++++ packages/workflow-host/src/child/run-child.ts | 104 +++ .../src/child/substrate-factory.ts | 1 + .../workflow-substrate-factory-abort.test.ts | 8 +- ...low-substrate-factory-child-action.test.ts | 20 +- ...flow-substrate-factory-child-depth.test.ts | 8 +- ...low-substrate-factory-child-grants.test.ts | 8 +- ...ubstrate-factory-suspendable-child.test.ts | 8 +- .../src/ipc/control-payloads.test.ts | 14 + .../workflow-host/src/ipc/control-payloads.ts | 31 + .../src/seams/loop-signal-delivery.test.ts | 4 +- .../src/seams/quiet-scheduler.test.ts | 252 ++++++ .../workflow-host/src/seams/scheduler.test.ts | 129 ++- packages/workflow-host/src/seams/scheduler.ts | 126 ++- .../src/seams/signal-channel.test.ts | 63 +- .../workflow-host/src/seams/signal-channel.ts | 40 +- .../src/runlocal/signal-channel.test.ts | 61 +- .../workflow/src/runlocal/signal-channel.ts | 3 +- packages/workflow/src/runtime/activity.ts | 165 ++++ .../src/runtime/child-workflow-relay.test.ts | 12 +- packages/workflow/src/runtime/commit-chain.ts | 35 +- packages/workflow/src/runtime/env.ts | 16 +- packages/workflow/src/runtime/index.ts | 9 + .../src/runtime/loop-iteration-handle.test.ts | 4 +- .../workflow/src/runtime/parked-work.test.ts | 151 ++++ .../runtime/resume-awaiting-signal.test.ts | 4 +- packages/workflow/src/runtime/run.ts | 262 ++++-- .../runtime/suspendable-child-handle.test.ts | 182 ++++- .../src/runtime/suspendable-child-handle.ts | 151 ++-- .../testing/signal-channel-observer.test.ts | 26 +- .../src/testing/signal-channel-observer.ts | 6 +- ...ingle-step-conversation-durability.test.ts | 1 + 38 files changed, 3019 insertions(+), 292 deletions(-) create mode 100644 packages/workflow-host/src/child/quiet-handoff.test.ts create mode 100644 packages/workflow-host/src/seams/quiet-scheduler.test.ts create mode 100644 packages/workflow/src/runtime/activity.ts create mode 100644 packages/workflow/src/runtime/parked-work.test.ts diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index ab9a74b19..0a34c9f98 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -488,7 +488,7 @@ The workflow-process runs code the operator deploys but the sidecar does not ful Each deployment gets two channels: -- **Control channel.** NDJSON over stdio, Ed25519-signed per direction. Carries the child's first `authority.set`, trigger fires, signal deliveries, drain, shutdown, grants-updated, credentials-updated, sources-updated, and back from the child its `definition` report and its `ready` signal. Low rate, high authority. Sending and receiving live in `packages/workflow-host/src/ipc/control-sender.ts` and `control-receiver.ts`; payload validation lives in `control-payloads.ts`. +- **Control channel.** NDJSON over stdio, Ed25519-signed per direction. Carries the child's first `authority.set`, trigger fires, signal deliveries, drain, quiet checks, shutdown, grants-updated, credentials-updated, sources-updated, and back from the child its `definition` report and its `ready` signal. Low rate, high authority. Sending and receiving live in `packages/workflow-host/src/ipc/control-sender.ts` and `control-receiver.ts`; payload validation lives in `control-payloads.ts`. - **Event channel.** A UNIX socketpair, HMAC-SHA256-authenticated with a 32-byte symmetric key derived at spawn time. Carries `InferenceEvent`s from the reactor (including the per-message `message.run.started` / `message.run.ended` brackets). High rate. Sending lives in `packages/workflow-host/src/ipc/event-sender.ts`; receiving and payload validation live in `event-channel.ts`. Each side signs control frames with its own private key and verifies incoming frames against its peer's public key, so the child cannot forge supervisor commands. Symmetric HMAC authenticates the event stream at reactor cadence — Ed25519 per frame at that rate would dominate runtime cost. The control receiver validates each payload against its message kind's schema. Those schemas and the `InferenceEvent` schema applied by `parseInferenceEvent` accept disjoint payload shapes: control schemas reject inference-event shapes, and the event schema rejects control-plane shapes, so a confused-deputy attack that smuggles a `drain` or `recycle` over the event channel fails at validation. @@ -655,6 +655,13 @@ ChannelSender` writes against; the event channel is the - `grants-updated` -> swap the active credentialsSnapshot. - `credentials-updated` -> merge into the material cell. - `drain` -> flip the drain controller's signal. + - `quiet.request` -> make every buffered event durable and answer + whether every run only waits on input from outside, with the + earliest timer the scheduler has queued; with `wait`, answer once + that holds. A run that has ended answers `ended`, and a check that + fails answers with its error. + A check still waiting when the next one arrives is dropped + unanswered, since the supervisor gave up on it. - `shutdown` -> stop accepting new triggers and exit the loop. #### Trust Anchors @@ -713,9 +720,9 @@ grants until the next send, and it can ask again. - **Control channel.** Supervisor -> child. Carries every `ControlPayload` variant (`authority.set`, `trigger.fire`, - `signal.deliver`, `drain`, `shutdown`, `grants-updated`, - `credentials-updated`, `sources-updated`) plus the child's upstream - `ready` ride. Every + `signal.deliver`, `drain`, `quiet.request`, `shutdown`, + `grants-updated`, `credentials-updated`, `sources-updated`) plus the + child's upstream `ready` ride. Every inbound frame is verified against `HOST_PUBKEY`; the receiver iterator crashes the loop on any signature/channelId/seq violation. diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index 370668dfa..65a8a9822 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -318,7 +318,8 @@ forking a process. 6. Emit `ready` and then `resumed.runs` on the control channel. Every other upstream control frame waits until both are out. 7. Loop on `trigger.fire`, `authority.set`, `grants-updated`, - `credentials-updated`, `drain`, `shutdown`, and `signal.deliver` frames. + `credentials-updated`, `drain`, `shutdown`, `signal.deliver`, and + `quiet.request` frames. ### Authorize Closure diff --git a/packages/workflow-host/src/adapters/spawn-child.test.ts b/packages/workflow-host/src/adapters/spawn-child.test.ts index b933b9bbe..c04b3548d 100644 --- a/packages/workflow-host/src/adapters/spawn-child.test.ts +++ b/packages/workflow-host/src/adapters/spawn-child.test.ts @@ -26,6 +26,8 @@ import { type RunChildWorkflow, } from "./spawn-child"; +const noWake = (): void => undefined; + const tempDirs: string[] = []; // These tests exercise the adapter's resolution/abort/forwarding, not event @@ -112,7 +114,7 @@ describe("workflow-host SpawnChildWorkflow adapter - in-memory resolution", () = noopOnEvent, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); @@ -174,7 +176,7 @@ describe("workflow-host SpawnChildWorkflow adapter - child execution", () => { }, noopOnEvent, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("failed"); @@ -367,7 +369,7 @@ describe("workflow-host SpawnChildWorkflow adapter - sub-namespace scoping", () }, noopOnEvent, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); diff --git a/packages/workflow-host/src/child/proxy-repo-store.ts b/packages/workflow-host/src/child/proxy-repo-store.ts index 88b36e0e0..12434aa64 100644 --- a/packages/workflow-host/src/child/proxy-repo-store.ts +++ b/packages/workflow-host/src/child/proxy-repo-store.ts @@ -36,6 +36,8 @@ import type { WriteResult, } from "@intx/hub-sessions/substrate"; +import { beginDelivery } from "@intx/workflow/runtime"; + import type { ChildSubstrateWriteBridge } from "./substrate-write-bridge"; type WriteTreeArgs = Parameters[3]; @@ -79,8 +81,22 @@ type SubscriberState = { | null; closed: boolean; signal: AbortSignal; + // A commit is on its way to this subscriber's consumer while an entry is + // buffered, or handed out and the consumer has not come back for the next. + handedOut: boolean; + endDelivery: (() => void) | null; }; +function noteDelivery(sub: SubscriberState): void { + const delivering = !sub.closed && (sub.buffer.length > 0 || sub.handedOut); + if (delivering && sub.endDelivery === null) { + sub.endDelivery = beginDelivery(); + } else if (!delivering && sub.endDelivery !== null) { + sub.endDelivery(); + sub.endDelivery = null; + } +} + export function createProxyWorkflowRunRepoStore( opts: CreateProxyWorkflowRunRepoStoreOpts, ): RepoStore { @@ -132,10 +148,13 @@ export function createProxyWorkflowRunRepoStore( if (sub.waiter !== null) { const w = sub.waiter; sub.waiter = null; + sub.handedOut = true; + noteDelivery(sub); w({ value: entry, done: false }); continue; } sub.buffer.push(entry); + noteDelivery(sub); } } @@ -252,10 +271,13 @@ export function createProxyWorkflowRunRepoStore( waiter: null, closed: false, signal: subOpts.signal, + handedOut: false, + endDelivery: null, }; set.add(sub); const cleanup = (): void => { sub.closed = true; + noteDelivery(sub); const current = subscribers.get(key); if (current !== undefined) { current.delete(sub); @@ -280,16 +302,21 @@ export function createProxyWorkflowRunRepoStore( return this; }, next(): Promise> { + sub.handedOut = false; if (sub.closed) { return Promise.resolve({ value: undefined, done: true }); } if (sub.buffer.length > 0) { const next = sub.buffer.shift(); if (next === undefined) { + noteDelivery(sub); return Promise.resolve({ value: undefined, done: true }); } + sub.handedOut = true; + noteDelivery(sub); return Promise.resolve({ value: next, done: false }); } + noteDelivery(sub); return new Promise((resolve) => { sub.waiter = resolve; }); diff --git a/packages/workflow-host/src/child/quiet-handoff.test.ts b/packages/workflow-host/src/child/quiet-handoff.test.ts new file mode 100644 index 000000000..d3f14308f --- /dev/null +++ b/packages/workflow-host/src/child/quiet-handoff.test.ts @@ -0,0 +1,764 @@ +// A child reports quiet only when nothing works and no committed input is on +// its way to a run. Whoever ends a wait must count the waiter as working +// before it stops counting itself, or a quiet check can land in between and +// stop a child that is about to act. Each test drives one way a wait ends and +// looks, at the first moment the process looks quiet, at what the woken run +// has done by then. +// +// The runs use the child's own stack: the proxy store over a real repo store, +// the host signal channel, and the runtime store adapter, so a subscription +// counts a commit as on its way until its consumer is done with it, as in a +// live child. + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + expect, + test, +} from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; +import { generateKeyPair } from "@intx/crypto"; +import { createRepoStore } from "@intx/hub-sessions"; +import type { + AuthorizeFn, + KindHandler, + Principal, + RepoId, + ValidatePushResult, +} from "@intx/hub-sessions"; +import type { KeyPair } from "@intx/types/runtime"; +import { + action, + awaitSignal, + childWorkflow, + createInMemoryBlobSubstrate, + createNoopDrainController, + createSuspendableChildHandle, + defineWorkflow, + emptyState, + enumerateInlineLoopBodies, + escalation, + loop, + onTrigger, + resumeFromLog, + rewriteInlineChildWorkflowBodies, + rewriteInlineOnTriggerBodies, + runtimeRun, + sleep, + step, + type RepoStore as RuntimeRepoStore, + type SpawnSuspendableChild, + type RuntimeWorkflowRun, + type StepInvoker, + type WorkflowDefinition, + type WorkflowEvent, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { + beginDelivery, + bufferedEvents, + isQuiet, + isWorking, + whenQuiet, + withRunCommitBarrier, +} from "@intx/workflow/runtime"; + +import { createWorkflowRunRepoStore } from "../adapters/repo-store"; +import { + createWorkflowHostSignalChannel, + type SignalChannelHandle, +} from "../seams/signal-channel"; + +import { createProxyWorkflowRunRepoStore } from "./proxy-repo-store"; + +const REF = "refs/heads/main"; +const AT = "2026-01-01T00:00:00.000Z"; +const principal: Principal = { kind: "test" }; +const allowAll: AuthorizeFn = () => ({ allowed: true }); + +const tempDirs: string[] = []; +let signingKey: KeyPair; +const teardowns: (() => Promise)[] = []; + +beforeAll(async () => { + signingKey = await generateKeyPair(); +}); + +// The counts are process-wide. Work an earlier test file left running in +// this worker would keep `whenQuiet` from ever resolving. +beforeEach(() => { + if (!isQuiet()) { + throw new Error("work or a delivery from an earlier test is still running"); + } +}); + +afterEach(async () => { + for (const teardown of teardowns.splice(0).reverse()) await teardown(); +}); + +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }).catch(() => { + /* best effort */ + }); + } +}); + +function permissiveHandler(directoryPrefix: string): KindHandler { + return { + kind: "skill", + directoryPrefix, + validatePush(): ValidatePushResult { + return { ok: true }; + }, + onRefUpdated() { + /* no-op */ + }, + }; +} + +type Timer = { runId: string; timerId: string }; + +// The adapter closes the store tail a few microtasks after its consumer +// returns. Closing it at once ends that delivery as early as it can end, so +// a handoff cannot lean on those microtasks. +function closingOnReturn( + store: RuntimeRepoStore, + runId: string, + opts: Parameters[1], +): ReturnType { + const done = new AbortController(); + const inner = store.subscribe(runId, { + ...opts, + signal: AbortSignal.any([opts.signal, done.signal]), + }); + return { + [Symbol.asyncIterator]() { + return this; + }, + next: () => inner.next(), + async return() { + done.abort(); + return (await inner.return?.()) ?? { done: true, value: undefined }; + }, + }; +} + +async function childStack( + prefix: string, + definition: WorkflowDefinition, + invokeStep: StepInvoker = () => { + throw new Error("quiet handoff: no agent step runs here"); + }, +) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dataDir); + const bare = createRepoStore({ + dataDir, + signingKey, + handlers: { skill: permissiveHandler(prefix) }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: prefix }; + const proxy = createProxyWorkflowRunRepoStore({ + bareStore: bare, + bridge: { + async submit(req) { + const result = await bare.writeTreePreservingPrefix( + principal, + repoId, + req.ref, + { + preservePrefix: req.preservePrefix, + message: req.message, + merge: req.merge, + }, + ); + return { commitSha: result.commitSha }; + }, + handleMergeRequest: () => undefined, + handleWriteResponse: () => undefined, + cancelAll: () => undefined, + pendingCount: 0, + }, + workflowRunRepoId: repoId, + }); + const adapter = createWorkflowRunRepoStore({ + substrate: proxy, + repoId, + principal, + ref: REF, + }); + // What each run has made durable, readable at once. + const durable = new Map(); + const record = (runId: string, events: readonly WorkflowEvent[]): void => { + durable.set(runId, [...(durable.get(runId) ?? []), ...events]); + }; + const runtimeStore: RuntimeRepoStore = { + read: (runId) => adapter.read(runId), + subscribe: (runId, opts) => closingOnReturn(adapter, runId, opts), + async append(runId, event) { + await adapter.append(runId, event); + record(runId, [event]); + }, + async appendBatch(runId, events) { + await adapter.appendBatch(runId, events); + record(runId, events); + }, + }; + + const channels: SignalChannelHandle[] = []; + const hostChannel = (runId: string): SignalChannelHandle => { + const channel = createWorkflowHostSignalChannel({ + repoStore: proxy, + runtimeStore, + principal, + repoId, + ref: REF, + runId, + readState: () => emptyState(runId), + newId: () => "generated-signal", + clock: () => new Date(AT), + }); + channels.push(channel); + return channel; + }; + + const timers: Timer[] = []; + const armed: (() => void)[] = []; + const sections = rewriteInlineOnTriggerBodies(definition); + const rewrite = rewriteInlineChildWorkflowBodies(sections.workflow); + const bodies = new Map( + [ + ...sections.bodies, + ...rewrite.bodies, + ...enumerateInlineLoopBodies(rewrite.workflow), + ].map((b) => [b.ref, b.definition]), + ); + const spawnBody = async ( + spawn: Parameters[0], + surfaceInputPark: boolean, + ) => { + const body = bodies.get(spawn.definitionRef); + if (body === undefined) { + throw new Error(`no child workflow ${spawn.definitionRef}`); + } + return createSuspendableChildHandle(envFor(spawn.childRunId, body), { + definition: body, + childRunId: spawn.childRunId, + input: spawn.input, + depth: spawn.depth, + maxChildSpawnDepth: spawn.maxChildSpawnDepth, + ...(spawn.resumeFromEvents !== undefined + ? { resumeFromEvents: spawn.resumeFromEvents } + : {}), + signal: spawn.signal, + surfaceInputPark, + }); + }; + + const envFor = ( + runId: string, + runDefinition: WorkflowDefinition, + ): WorkflowRuntimeEnv => ({ + repoStore: runtimeStore, + scheduler: { + scheduleIn(timerRunId, timerId) { + timers.push({ runId: timerRunId, timerId }); + for (const wake of armed.splice(0)) wake(); + return () => undefined; + }, + }, + signalChannel: hostChannel(runId), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep, + invokeAction: async () => ({ output: "done" }), + spawnChild: (spawn) => spawnBody(spawn, true), + spawnSuspendableChild: (spawn) => spawnBody(spawn, false), + spawnLoopIteration: (spawn) => spawnBody(spawn, false), + loopFns: (ref) => (ref === "again" ? () => true : () => null), + clock: () => new Date(AT), + newId: (idPrefix) => `${idPrefix}-id`, + drain: createNoopDrainController(runDefinition), + hasUpstreamSignalResolver: true, + }); + + const stops: (() => Promise)[] = []; + const stop = async (): Promise => { + for (const stopRun of stops.splice(0).reverse()) await stopRun(); + }; + teardowns.push(stop); + + const start = (runId: string): RuntimeWorkflowRun => { + const teardown = new AbortController(); + const run = runtimeRun(rewrite.workflow, envFor(runId, rewrite.workflow), { + runId, + triggerPayload: null, + localAbort: teardown.signal, + }); + stops.push(async () => { + teardown.abort(); + await run.complete.catch(() => undefined); + for (const channel of channels.splice(0)) await channel.stop(); + }); + return run; + }; + + // Events a run has committed or buffered so far. + const events = (runId: string): WorkflowEvent[] => [ + ...(durable.get(runId) ?? []), + ...bufferedEvents(runId), + ]; + + const nextTimer = (): Promise => + new Promise((resolve) => { + const take = (): void => { + const timer = timers.shift(); + if (timer === undefined) armed.push(take); + else resolve(timer); + }; + take(); + }); + + // Fire a timer as the host scheduler does: the commit is input on its way + // to the run from the moment the timer leaves the queue. `seen` reads the + // first quiet moment after the fire began. + async function fireTimer(timer: Timer, look: () => T): Promise { + const endDelivery = beginDelivery(); + const seen = whenQuiet().then(look); + try { + await withRunCommitBarrier( + { repoStore: runtimeStore }, + timer.runId, + async () => { + const state = resumeFromLog( + timer.runId, + await adapter.read(timer.runId), + ); + await runtimeStore.append(timer.runId, { + kind: "TimerFired", + seq: state.lastSeq + 1, + at: AT, + timerId: timer.timerId, + }); + }, + ); + } finally { + endDelivery(); + } + return seen; + } + + // Deliver a signal as the child's control loop does. + async function deliverSignal( + runId: string, + name: string, + look: () => T, + ): Promise { + const endDelivery = beginDelivery(); + const seen = whenQuiet().then(look); + try { + const transient = createWorkflowHostSignalChannel({ + repoStore: proxy, + runtimeStore, + principal, + repoId, + ref: REF, + runId, + readState: () => emptyState(runId), + newId: () => "delivered-signal", + clock: () => new Date(AT), + }); + await transient.deliver(name, { approved: true }, `sig-${name}`); + await transient.stop(); + } finally { + endDelivery(); + } + return seen; + } + + // A log written as if an earlier child had left it. + const seed = (runId: string, log: readonly WorkflowEvent[]): Promise => + runtimeStore.appendBatch(runId, log); + + return { + start, + stop, + events, + durable: (runId: string) => durable.get(runId) ?? [], + seed, + nextTimer, + fireTimer, + deliverSignal, + }; +} + +function kinds(events: readonly WorkflowEvent[]): string[] { + return events.map((event) => event.kind); +} + +function awaitedBy(events: readonly WorkflowEvent[], stepId: string): boolean { + return events.some( + (event) => event.kind === "SignalAwaited" && event.stepId === stepId, + ); +} + +// A fresh child does not continue a step in retry backoff, so the child may +// not stop there. +test("a step waiting out a retry backoff counts as working", async () => { + const definition = defineWorkflow({ + id: "quiet-backoff", + trigger: { type: "manual" }, + steps: { + flaky: step({ + agent: defineAgent({ + id: "flaky", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock" }] }, + }), + input: { literal: null }, + retry: { maxAttempts: 2, initialBackoffMs: 60_000 }, + }), + }, + }); + let attempts = 0; + const stack = await childStack("quiet-backoff-", definition, async () => { + attempts += 1; + if (attempts === 1) throw new Error("the first attempt fails"); + return { output: "done" }; + }); + stack.start("run-backoff"); + const timer = await stack.nextTimer(); + expect(isWorking()).toBe(true); + + const seen = await stack.fireTimer(timer, () => + kinds(stack.events("run-backoff")), + ); + expect(attempts).toBe(2); + expect(seen).toContain("RunCompleted"); +}); + +test("a step woken by its timer counts as working before the timer's delivery ends", async () => { + const definition = defineWorkflow({ + id: "quiet-timer", + trigger: { type: "manual" }, + steps: { nap: sleep({ duration: 60_000 }) }, + }); + const stack = await childStack("quiet-timer-", definition); + stack.start("run-timer"); + const timer = await stack.nextTimer(); + await whenQuiet(); + + const seen = await stack.fireTimer(timer, () => + kinds(stack.events("run-timer")), + ); + expect(seen).toContain("RunCompleted"); +}); + +test("a run woken by a signal counts as working before the signal's delivery ends", async () => { + const definition = defineWorkflow({ + id: "quiet-signal", + trigger: { type: "manual" }, + steps: { gate: awaitSignal({ name: "go" }) }, + }); + const stack = await childStack("quiet-signal-", definition); + stack.start("run-signal"); + await whenQuiet(); + expect(awaitedBy(stack.events("run-signal"), "gate")).toBe(true); + + const seen = await stack.deliverSignal("run-signal", "go", () => + kinds(stack.events("run-signal")), + ); + expect(seen).toContain("RunCompleted"); +}); + +// The nested run reads its log between the park and going idle, so the +// parent wakes first even without the park's handoff. What can fail here is +// the parent going idle before it records the relay. +test("a parent relays its nested run's park before the child looks quiet", async () => { + const definition = defineWorkflow({ + id: "quiet-nested-park", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "quiet-nested-park-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner" }) }, + }), + }), + }, + }); + const stack = await childStack("quiet-nested-park-", definition); + stack.start("run-nested-park"); + const relayed = await whenQuiet().then(() => + awaitedBy(stack.events("run-nested-park"), "spawn"), + ); + expect(relayed).toBe(true); +}); + +test("a loop relays its body's park before the child looks quiet", async () => { + const definition = defineWorkflow({ + id: "quiet-loop-park", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: defineWorkflow({ + id: "quiet-loop-park-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner" }) }, + }), + while: "again", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "exhausted", + }), + exhausted: escalation({ to: "operator", after: ["rework"] }), + }, + }); + const stack = await childStack("quiet-loop-park-", definition); + stack.start("run-loop-park"); + const relayed = await whenQuiet().then(() => + awaitedBy(stack.events("run-loop-park"), "rework"), + ); + expect(relayed).toBe(true); +}); + +test("an onTrigger section relays its body's park before the child looks quiet", async () => { + const definition = defineWorkflow({ + id: "quiet-section-park", + trigger: { type: "manual" }, + steps: { + inbox: onTrigger({ + on: { type: "mail", to: "run_quiet@t.example" }, + body: defineWorkflow({ + id: "quiet-section-park-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner" }) }, + }), + }), + }, + }); + const stack = await childStack("quiet-section-park-", definition); + stack.start("run-section-park"); + const relayed = await whenQuiet().then(() => + awaitedBy(stack.events("run-section-park"), "inbox"), + ); + expect(relayed).toBe(true); +}); + +test("a parent counts as working once its nested run ends, before the nested run stops counting", async () => { + const definition = defineWorkflow({ + id: "quiet-nested-end", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "quiet-nested-end-body", + trigger: { type: "manual" }, + steps: { work: action({ handler: "work" }) }, + }), + }), + }, + }); + const stack = await childStack("quiet-nested-end-", definition); + stack.start("run-nested-end"); + const seen = await whenQuiet().then(() => + kinds(stack.events("run-nested-end")), + ); + expect(seen).toContain("RunCompleted"); +}); + +test("a parent holding a nested gate's signal counts as working once the resumed gate re-arms", async () => { + const definition = defineWorkflow({ + id: "quiet-rearm", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "quiet-rearm-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner" }) }, + }), + }), + }, + }); + // A child left the parent relaying the nested gate. + const before = await childStack("quiet-rearm-before-", definition); + before.start("run-rearm"); + await whenQuiet(); + const parentLog = before.durable("run-rearm"); + const spawned = parentLog.find((event) => event.kind === "ChildSpawned"); + const relay = parentLog.find( + (event) => event.kind === "SignalAwaited" && event.stepId === "spawn", + ); + if (spawned?.kind !== "ChildSpawned" || relay?.kind !== "SignalAwaited") { + throw new Error("the parent did not relay the nested gate"); + } + const childLog = before.durable(spawned.childRunId); + // Runs share the commit chain by run id, so the first child goes first. + await before.stop(); + + // The parent took the signal; the child never got it before it stopped. + const after = await childStack("quiet-rearm-after-", definition); + await after.seed(spawned.childRunId, childLog); + await after.seed("run-rearm", [ + ...parentLog, + { + kind: "SignalReceived", + seq: (parentLog.at(-1)?.seq ?? 0) + 1, + at: AT, + signalName: relay.signalName, + signalId: "sig-held", + payload: { approved: true }, + }, + ]); + after.start("run-rearm"); + // The channel writes the delivered signal itself, so the child's own + // events show that its gate took it. + const child = await whenQuiet().then(() => + kinds(after.events(spawned.childRunId)), + ); + expect(child).toContain("StepCompleted"); +}); + +// The timer that releases the gate wakes the nested run too, and it works +// through to its end, so the release leaves no gap. What can fail here is +// the parent's wait for that end once it has abandoned the relay. +test("a parent that abandons a released nested relay counts as working once the nested run ends", async () => { + const definition = defineWorkflow({ + id: "quiet-release", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "quiet-release-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner", timeout: 60_000 }) }, + }), + }), + }, + }); + const stack = await childStack("quiet-release-", definition); + stack.start("run-release"); + const timer = await stack.nextTimer(); + await whenQuiet(); + expect(awaitedBy(stack.events("run-release"), "spawn")).toBe(true); + + const seen = await stack.fireTimer(timer, () => + kinds(stack.events("run-release")), + ); + expect(seen).toContain("ChildCompleted"); +}); + +test("a run loop counts as working once a step ends, before the step stops counting", async () => { + const definition = defineWorkflow({ + id: "quiet-loop", + trigger: { type: "manual" }, + steps: { + first: action({ handler: "first" }), + gate: awaitSignal({ name: "go", after: ["first"] }), + }, + }); + const stack = await childStack("quiet-loop-", definition); + stack.start("run-loop"); + const parked = await whenQuiet().then(() => + awaitedBy(stack.events("run-loop"), "gate"), + ); + expect(parked).toBe(true); +}); + +test("a gate whose timeout aborts its signal wait counts as working before the timer's delivery ends", async () => { + const definition = defineWorkflow({ + id: "quiet-timeout", + trigger: { type: "manual" }, + steps: { gate: awaitSignal({ name: "go", timeout: 60_000 }) }, + }); + const stack = await childStack("quiet-timeout-", definition); + stack.start("run-timeout"); + const timer = await stack.nextTimer(); + await whenQuiet(); + + const seen = await stack.fireTimer(timer, () => + kinds(stack.events("run-timeout")), + ); + expect( + seen.some((kind) => kind === "RunFailed" || kind === "RunCompleted"), + ).toBe(true); +}); + +test("a loop counts as working once its body ends, before the body stops counting", async () => { + const definition = defineWorkflow({ + id: "quiet-loop-end", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: defineWorkflow({ + id: "quiet-loop-end-body", + trigger: { type: "manual" }, + steps: { work: action({ handler: "work" }) }, + }), + while: "stop", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "exhausted", + }), + exhausted: escalation({ to: "operator", after: ["rework"] }), + }, + }); + const stack = await childStack("quiet-loop-end-", definition); + stack.start("run-loop-end"); + const seen = await whenQuiet().then(() => + kinds(stack.events("run-loop-end")), + ); + expect(seen).toContain("RunCompleted"); +}); + +test("a loop that hands its body a signal counts as working until the next iteration parks", async () => { + const definition = defineWorkflow({ + id: "quiet-loop-handed", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: defineWorkflow({ + id: "quiet-loop-handed-body", + trigger: { type: "manual" }, + steps: { inner: awaitSignal({ name: "inner" }) }, + }), + while: "again", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "exhausted", + }), + exhausted: escalation({ to: "operator", after: ["rework"] }), + }, + }); + const stack = await childStack("quiet-loop-handed-", definition); + stack.start("run-loop-handed"); + await whenQuiet(); + const relays = (): number => + stack + .events("run-loop-handed") + .filter( + (event) => event.kind === "SignalAwaited" && event.stepId === "rework", + ).length; + expect(relays()).toBe(1); + expect(await stack.deliverSignal("run-loop-handed", "inner", relays)).toBe(2); +}); diff --git a/packages/workflow-host/src/child/run-child-source-ref.test.ts b/packages/workflow-host/src/child/run-child-source-ref.test.ts index 939b3f36b..d7f121413 100644 --- a/packages/workflow-host/src/child/run-child-source-ref.test.ts +++ b/packages/workflow-host/src/child/run-child-source-ref.test.ts @@ -509,6 +509,7 @@ function buildBindings(baseDir: string): RunWorkflowChildBindings { actionCredentials: () => unavailableActionCredentials, spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, + earliestQueuedTimer: () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index f50a2dd9b..b24b28e29 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -8,9 +8,11 @@ import { base64Encode, hexEncode } from "@intx/types"; import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; import { computeLiveDefinitionHash, + createInMemoryRepoStore, createTerminalChildHandle, unavailableActionCredentials, } from "@intx/workflow"; +import { beginDelivery, commitBuffered } from "@intx/workflow/runtime"; import { loadWorkflowDefinitionFromClosure } from "@intx/workflow-host"; import type { ApprovalSnapshot, KeyPair } from "@intx/types/runtime"; import type { @@ -65,6 +67,7 @@ import { createMemoryFrameStream, createMemoryNdjsonStream, waitForUpstreamPayload, + waitForUpstreamPayloads, type ChangeNotifier, sendAuthoritySet, } from "@intx/workflow-host/testing"; @@ -409,6 +412,7 @@ function buildBindings(opts: { scheduler: { scheduleIn: () => () => undefined, }, + earliestQueuedTimer: () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], @@ -602,6 +606,583 @@ describe("runWorkflowChild", () => { expect(result.finalCredentialsSnapshot?.steps).toHaveLength(1); }); + test("a later quiet check supersedes one still waiting, which is not answered", async () => { + const baseDir = await makeTempDir("child-quiet-superseded-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings: buildBindings({ baseDir, childKeyPair }), + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + // Input on its way to a run keeps the child from being quiet. + const delivered = beginDelivery(); + try { + for (const requestId of ["quiet-abandoned", "quiet-current"]) { + await supervisorSender.send({ + type: "quiet.request", + data: { requestId, runId: "run-1", wait: true }, + }); + } + // The child takes frames in order, so both checks wait once this + // answers. + await supervisorSender.send({ + type: "parked-correlations.request", + data: { requestId: "parked-1" }, + }); + await waitForUpstreamPayload( + childToSupervisor, + "parked-correlations.response", + ); + } finally { + delivered(); + } + await waitForUpstreamPayload(childToSupervisor, "quiet.response"); + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-after", runId: "run-1", wait: false }, + }); + const answers = await waitForUpstreamPayloads( + childToSupervisor, + "quiet.response", + 2, + ); + expect(answers.map((answer) => answer.data.requestId)).toEqual([ + "quiet-current", + "quiet-after", + ]); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + + test.each([true, false])( + "a quiet check a run ends during answers it as ended (wait: %p)", + async (wait) => { + const baseDir = await makeTempDir("child-quiet-ends-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + await seedProcessingEntry( + baseDir, + { kind: "workflow-run", id: "deployment-x" }, + { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "hello from the inbox", + }, + ); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { + snapshot: AUTHORITY_SNAPSHOT, + }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings: buildBindings({ baseDir, childKeyPair }), + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + // The check starts before the run exists and waits past its end. + const delivered = beginDelivery(); + try { + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait }, + }); + await supervisorSender.send({ + type: "trigger.fire", + data: { + runId: "run-1", + messageId: "msg-1", + receivedAt: 1, + payload: textMail("hello from the inbox"), + }, + }); + await waitForUpstreamPayload(childToSupervisor, "terminal.event"); + } finally { + delivered(); + } + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")) + .data, + ).toEqual({ requestId: "quiet-1", ended: true }); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }, + ); + + test("answers a quiet check with the earliest queued timer, and one for an ended run as ended", async () => { + const baseDir = await makeTempDir("child-quiet-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + await seedProcessingEntry( + baseDir, + { kind: "workflow-run", id: "deployment-x" }, + { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "hello from the inbox", + }, + ); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const wakeAt = "2026-10-02T00:00:00.000Z"; + const bindings: RunWorkflowChildBindings = { + ...buildBindings({ baseDir, childKeyPair }), + earliestQueuedTimer: () => new Date(wakeAt), + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait: false }, + }); + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")).data, + ).toEqual({ requestId: "quiet-1", quiet: true, wakeAt }); + + await supervisorSender.send({ + type: "trigger.fire", + data: { + runId: "run-1", + messageId: "msg-1", + receivedAt: 1, + payload: textMail("hello from the inbox"), + }, + }); + await waitForUpstreamPayload(childToSupervisor, "terminal.event"); + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-2", runId: "run-1", wait: true }, + }); + const answers = await waitForUpstreamPayloads( + childToSupervisor, + "quiet.response", + 2, + ); + // An ended run decides nothing about quiet, so it reports no timer. + expect(answers[1]?.data).toEqual({ requestId: "quiet-2", ended: true }); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + + test("answers a quiet check as not quiet while a step is in flight", async () => { + const baseDir = await makeTempDir("child-quiet-step-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-quiet-step-def-", { + id: "test-workflow", + triggers: [], + steps: { "step-1": agentStep("step-1") }, + stepOrder: ["step-1"], + }); + await seedProcessingEntry( + baseDir, + { kind: "workflow-run", id: "deployment-x" }, + { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "hello from the inbox", + }, + ); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const stepStarted = Promise.withResolvers(); + const finishStep = Promise.withResolvers(); + const bindings: RunWorkflowChildBindings = { + ...buildBindings({ baseDir, childKeyPair }), + invokeStep: async () => { + stepStarted.resolve(undefined); + await finishStep.promise; + return { output: null }; + }, + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + await supervisorSender.send({ + type: "trigger.fire", + data: { + runId: "run-1", + messageId: "msg-1", + receivedAt: 1, + payload: textMail("hello from the inbox"), + }, + }); + await stepStarted.promise; + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait: false }, + }); + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")).data, + ).toEqual({ requestId: "quiet-1", quiet: false }); + + finishStep.resolve(undefined); + await waitForUpstreamPayload(childToSupervisor, "terminal.event"); + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + + test("answers a quiet check as not quiet when work starts while it reads the run's history", async () => { + const baseDir = await makeTempDir("child-quiet-read-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const base = buildBindings({ baseDir, childKeyPair }); + // The check reads the run's log before it decides and again after; the + // second read is held. + let reads: number | null = null; + const held = Promise.withResolvers(); + const release = Promise.withResolvers(); + const bindings: RunWorkflowChildBindings = { + ...base, + substrate: { + ...base.substrate, + openCommittedReads: async (...args) => { + if (reads !== null && (reads += 1) === 2) { + held.resolve(undefined); + await release.promise; + } + return base.substrate.openCommittedReads(...args); + }, + }, + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + reads = 0; + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait: false }, + }); + await held.promise; + // Activity that starts and ends during the read, as a step a timer began + // can; nothing is working by the time the answer goes out. + beginDelivery()(); + release.resolve(undefined); + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")).data, + ).toEqual({ requestId: "quiet-1", quiet: false }); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + + test("a quiet check counts activity that starts and ends during its flush", async () => { + const baseDir = await makeTempDir("child-quiet-flush-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings: buildBindings({ baseDir, childKeyPair }), + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + // Work that starts and ends while the check flushes may have buffered an + // event that flush did not see. + const inner = createInMemoryRepoStore(); + const repoStore: typeof inner = { + read: (id) => inner.read(id), + append: (id, event) => inner.append(id, event), + appendBatch: async (id, events) => { + beginDelivery()(); + await inner.appendBatch(id, events); + }, + subscribe: (id, opts) => inner.subscribe(id, opts), + }; + await commitBuffered({ repoStore }, "run-buffered", { + kind: "RunStarted", + seq: 1, + at: "2026-01-01T00:00:00.000Z", + runId: "run-buffered", + definitionHash: "definition-hash-abc", + trigger: { type: "manual", payload: null }, + }); + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait: false }, + }); + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")).data, + ).toEqual({ requestId: "quiet-1", quiet: false }); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + + test("answers a quiet check it cannot carry out with the error, as not quiet", async () => { + const baseDir = await makeTempDir("child-quiet-error-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const base = buildBindings({ baseDir, childKeyPair }); + let failReads = false; + const bindings: RunWorkflowChildBindings = { + ...base, + substrate: { + ...base.substrate, + openCommittedReads: (...args) => + failReads + ? Promise.reject(new Error("the run log could not be read")) + : base.substrate.openCommittedReads(...args), + }, + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + failReads = true; + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "quiet-1", runId: "run-1", wait: false }, + }); + expect( + (await waitForUpstreamPayload(childToSupervisor, "quiet.response")).data, + ).toEqual({ + requestId: "quiet-1", + quiet: false, + error: "the run log could not be read", + }); + failReads = false; + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + test("routes a mailbox.notify to the watch registry as an exists event", async () => { const baseDir = await makeTempDir("child-mailbox-notify-"); const supervisorKeyPair = await generateKeyPair(); @@ -2178,6 +2759,7 @@ describe("warm-agent round-trip (Phase 4.4)", () => { actionCredentials: () => unavailableActionCredentials, spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, + earliestQueuedTimer: () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], @@ -2277,6 +2859,9 @@ describe("warm-agent round-trip (Phase 4.4)", () => { expect(spy.replies[1]).toContain("alpha body"); expect(spy.replies[1]).toContain("bravo body"); + // Both runs end before the undeploy, so none is left counted as work + // in this process. + await waitForUpstreamPayloads(childToSupervisor, "terminal.event", 2); // Undeploy -> shutdown. Eviction runs the wrapped close exactly once // and the LSP subprocess dies. await supervisorSender.send({ @@ -2372,6 +2957,7 @@ describe("warm-agent round-trip (Phase 4.4)", () => { actionCredentials: () => unavailableActionCredentials, spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, + earliestQueuedTimer: () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], @@ -2459,6 +3045,9 @@ describe("warm-agent round-trip (Phase 4.4)", () => { expect(spy.sourceRotations[0]?.defaultSource).toBe("rotated"); expect(factoryCalls).toBe(1); + // The run ends before the undeploy, so it is not left counted as work in + // this process. + await waitForUpstreamPayload(childToSupervisor, "terminal.event"); await supervisorSender.send({ type: "shutdown", data: { reason: "undeploy" }, diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index d9f4c05bf..af40adcf6 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -32,6 +32,8 @@ // - `authority.set` -> replace the snapshot and the material. // - `grants-updated` -> replace the credentialsSnapshot. // - `drain` -> forward to the drain controller (no-op here). +// - `quiet.request` -> answer whether every run only waits on +// input from outside, and the earliest queued timer. // - `shutdown` -> stop accepting new triggers and exit the // loop. // @@ -94,8 +96,18 @@ import { createInMemoryEffectLedger, createLoopIterationHandle, emptyState, + isTerminalRunPhase, + resumeFromLog, runtimeRun, } from "@intx/workflow"; +import { + activityStarts, + beginDelivery, + flushAllBuffers, + isQuiet, + whenDelivered, + whenQuiet, +} from "@intx/workflow/runtime"; import { createWorkflowHostDrainController, @@ -341,6 +353,11 @@ export interface RunWorkflowChildBindings { }) => Promise; /** Host-process scheduler singleton. The child consumes the same instance. */ scheduler: Scheduler; + /** + * The earliest timer the host scheduler has queued, which a stopped child + * must be woken for. + */ + earliestQueuedTimer: () => Date | undefined; /** Grant evaluator wired against the host's grant-rule grammar. */ evaluateGrants: GrantEvaluator; /** @@ -983,6 +1000,7 @@ export async function runWorkflowChild( upstreamSender.open(); const triggeredRunIds: string[] = []; + const quietCheck: { current: AbortController | null } = { current: null }; // Resolve the mailbox watch registry the control loop routes `mailbox.notify` // frames to. Production wires it on the bindings (the substrate factory builds @@ -1017,6 +1035,7 @@ export async function runWorkflowChild( triggeredRunIds, runsInFlight, cancellationBarrier, + quietCheck, warmCache, sourcesRef, credentialMaterialRef, @@ -1106,6 +1125,75 @@ export async function runWorkflowChild( }; } +// Whether every run only waits on input from outside, for a supervisor that +// stops the child once it is. Buffered events are made durable and input on +// its way to a run is delivered first, so the answer holds for the log the +// stopped child would leave. A check that fails is reported, not retried. +async function answerQuiet( + ctx: { + bindings: RunWorkflowChildBindings; + runtimeRepoStore: ReturnType; + upstreamSender: ControlChannelSender; + quietCheck: { current: AbortController | null }; + }, + data: Extract["data"], +): Promise { + // The supervisor runs one standby attempt at a time, so a check still + // waiting when the next arrives belongs to one it gave up on. + ctx.quietCheck.current?.abort(); + const superseded = new AbortController(); + ctx.quietCheck.current = superseded; + let quiet = false; + let ended = false; + let wakeAt: Date | undefined; + let error: string | undefined; + try { + const runEnded = async (): Promise => + isTerminalRunPhase( + resumeFromLog(data.runId, await ctx.runtimeRepoStore.read(data.runId)) + .phase, + ); + ended = await runEnded(); + while (!ended) { + if (data.wait) await whenQuiet(superseded.signal); + // Taken before the flush: work that starts and ends during these waits + // can leave an event buffered that the flush did not see. + const startsAtDecision = activityStarts(); + await flushAllBuffers(); + await whenDelivered(); + quiet = isQuiet(); + // Read with `quiet`: a timer that leaves the queue after the decision + // is still reported, and wakes the deployment at once. + wakeAt = ctx.bindings.earliestQueuedTimer(); + // The run can end while the check waits or makes its events durable. + ended = await runEnded(); + // Work that started meanwhile, such as a step a timer began, may + // already have written history, so the decision no longer holds. + if (activityStarts() !== startsAtDecision) quiet = false; + if (quiet || !data.wait) break; + } + } catch (cause) { + if (superseded.signal.aborted) return; + error = cause instanceof Error ? cause.message : String(cause); + logger.warn`workflow-child quiet check failed: ${error}`; + quiet = false; + } finally { + if (ctx.quietCheck.current === superseded) ctx.quietCheck.current = null; + } + if (superseded.signal.aborted) return; + await ctx.upstreamSender.send({ + type: "quiet.response", + data: ended + ? { requestId: data.requestId, ended: true } + : { + requestId: data.requestId, + quiet, + ...(wakeAt !== undefined ? { wakeAt: wakeAt.toISOString() } : {}), + ...(error !== undefined ? { error } : {}), + }, + }); +} + /** * Handle a single control-channel payload. Returns `true` when the * payload signals shutdown so the caller exits the loop; otherwise @@ -1134,6 +1222,7 @@ async function handleControlPayload( triggeredRunIds: string[]; runsInFlight: Map; cancellationBarrier: ReturnType; + quietCheck: { current: AbortController | null }; warmCache: WarmAgentCache | undefined; sourcesRef: SourcesSnapshotRef; credentialMaterialRef: CredentialMaterialRef; @@ -1319,6 +1408,7 @@ async function handleControlPayload( // runtime body's `signalChannel.awaitNext` peer either resolves // (deliver landed) or remains pending until a subsequent // delivery. + const endDelivery = beginDelivery(); const transientSignalChannel = createWorkflowHostSignalChannel({ repoStore: ctx.bindings.substrate, runtimeStore: ctx.runtimeRepoStore, @@ -1345,6 +1435,7 @@ async function handleControlPayload( logger.warn`signal.deliver commit failed runId=${payload.data.runId} signalName=${payload.data.signalName}: ${reason}`; } finally { await transientSignalChannel.stop(); + endDelivery(); } })(); return false; @@ -1587,6 +1678,19 @@ async function handleControlPayload( ctx.substrateWriteBridge.handleWriteResponse(payload.data); return false; } + case "quiet.request": { + // Answered off the loop: a wait for quiet can outlast many frames, and + // the flush it starts with needs the write responses this loop takes. + void answerQuiet(ctx, payload.data).catch((cause: unknown) => { + logger.error`workflow-child quiet.response send failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + return false; + } + case "quiet.response": { + throw new Error( + "workflow-child received a `quiet.response` frame on its inbound control channel; this is a child-only upstream payload", + ); + } case "parked-correlations.request": { // Answer the supervisor's re-registration enumeration from durable // state. Awaiting inline is safe -- unlike `signal.deliver`, this diff --git a/packages/workflow-host/src/child/substrate-factory.ts b/packages/workflow-host/src/child/substrate-factory.ts index fbd84cdbf..a09be8731 100644 --- a/packages/workflow-host/src/child/substrate-factory.ts +++ b/packages/workflow-host/src/child/substrate-factory.ts @@ -2869,6 +2869,7 @@ export function createSidecarSubstrateFactory( }); }, scheduler, + earliestQueuedTimer: () => hostScheduler.earliestQueuedTimer(), evaluateGrants: evaluateGrantsAdapter, ...parkHooks, // The same registry the warm agent's transport registers `watch` diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-abort.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-abort.test.ts index 2ceecb0f4..eeeb51030 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-abort.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-abort.test.ts @@ -54,6 +54,8 @@ import { type SidecarChildStepInvoker, } from "./substrate-factory"; +const noWake = (): void => undefined; + const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-abort"; const WORKFLOW_RUN_REPO_ID: RepoId = { @@ -313,7 +315,7 @@ describe("in-process child parent-abort", () => { abort.abort(); const result = await settled; - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("failed"); @@ -347,11 +349,11 @@ describe("in-process child parent-abort", () => { // The body parked on an approval and the handle surfaced it: the child is // live. Abort the parent now. - const parked = await handle.next(); + const parked = await handle.next(noWake); expect(parked.kind).toBe("park"); abort.abort(); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); expect(terminal.kind).toBe("terminal"); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("failed"); diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-child-action.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-child-action.test.ts index d8a99e865..7848ad9a8 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-child-action.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-child-action.test.ts @@ -61,6 +61,8 @@ import { type SidecarChildStepInvoker, } from "./substrate-factory"; +const noWake = (): void => undefined; + const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-child-action"; const WORKFLOW_RUN_REPO_ID: RepoId = { @@ -348,7 +350,7 @@ describe("spawned body action steps", () => { () => undefined, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); expect(terminal.kind).toBe("terminal"); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("completed"); @@ -386,7 +388,7 @@ describe("spawned body action steps", () => { () => undefined, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); expect(terminal.kind).toBe("terminal"); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("failed"); @@ -425,7 +427,7 @@ describe("spawned body action steps", () => { () => undefined, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); @@ -635,7 +637,7 @@ describe("spawned body action credentials", () => { }, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("completed"); expect(await fs.promises.readFile(marker, "utf8")).toBe("http"); @@ -661,7 +663,7 @@ describe("spawned body action credentials", () => { }, ); - const event = await result.next(); + const event = await result.next(noWake); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); expect(await fs.promises.readFile(marker, "utf8")).toBe("http"); @@ -688,7 +690,7 @@ describe("spawned body action credentials", () => { }, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("completed"); expect(await fs.promises.readFile(marker, "utf8")).toBe("http"); @@ -726,7 +728,7 @@ describe("spawned body action credentials", () => { }, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("completed"); expect(await fs.promises.readFile(marker, "utf8")).toBe("http"); @@ -754,7 +756,7 @@ describe("spawned body action credentials", () => { }, ); - const terminal = await handle.next(); + const terminal = await handle.next(noWake); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("failed"); const events: readonly WorkflowEvent[] = @@ -800,7 +802,7 @@ describe("spawned body action credentials", () => { () => undefined, { material: credentialCell(), parentGrants: rootGrantsOf(root) }, ); - return result.next(); + return result.next(noWake); } test("an action after a step resolves its credential while the top-level run holds the use grant", async () => { diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-child-depth.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-child-depth.test.ts index 6aa17cde9..bb4774149 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-child-depth.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-child-depth.test.ts @@ -57,6 +57,8 @@ import { type SidecarChildStepInvoker, } from "./substrate-factory"; +const noWake = (): void => undefined; + const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-child-depth"; const WORKFLOW_RUN_REPO_ID: RepoId = { @@ -294,7 +296,7 @@ describe("createSidecarRunChild spawn-depth ceiling", () => { // The top run settled failed: the deep spawn was rejected and the failure // propagated up every rung's spawn step. - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("failed"); @@ -385,7 +387,7 @@ describe("createSidecarRunChild root grants at depth", () => { parentGrants: createSnapshotGrantsReader(rootGrants, "test root"), }, ); - return result.next(); + return result.next(noWake); } beforeAll(async () => { @@ -523,7 +525,7 @@ describe("createSidecarRunChild per-rung cap", () => { parentGrants: createSnapshotGrantsReader(rootGrants, "test root"), }, ); - const event = await handle.next(); + const event = await handle.next(noWake); expect(event.kind).toBe("terminal"); expect(decisions).toEqual([null]); }); diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-child-grants.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-child-grants.test.ts index b1bd8bb52..d29198a21 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-child-grants.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-child-grants.test.ts @@ -56,6 +56,8 @@ import { type SidecarChildStepInvoker, } from "./substrate-factory"; +const noWake = (): void => undefined; + const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-child-grants"; const WORKFLOW_RUN_REPO_ID: RepoId = { @@ -322,7 +324,7 @@ describe("createSidecarRunChild grant capping", () => { noopOnEvent, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); @@ -386,7 +388,7 @@ describe("createSidecarRunChild grant capping", () => { noopOnEvent, ); - const event = await result.next(); + const event = await result.next(noWake); expect(event.kind).toBe("terminal"); if (event.kind !== "terminal") throw new Error("expected a terminal"); expect(event.terminalStatus).toBe("completed"); @@ -543,7 +545,7 @@ describe("createSidecarRunChild root grants", () => { parentGrants: createSnapshotGrantsReader(rootGrants, "test root"), }, ); - return handle.next(); + return handle.next(noWake); } test("a live child reads an always decision on its next step, and its grants file is unchanged", async () => { diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts index 0c7f80bef..286960e90 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts @@ -54,6 +54,8 @@ import { type SidecarChildStepInvoker, } from "./substrate-factory"; +const noWake = (): void => undefined; + const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-suspendable-child"; const WORKFLOW_RUN_REPO_ID: RepoId = { @@ -273,7 +275,7 @@ describe("createSidecarSpawnSuspendableChild", () => { // The body step suspended -> the handle surfaces the approval park on the // reserved correlation, carrying the step's snapshot. - const parked = await handle.next(); + const parked = await handle.next(noWake); expect(parked.kind).toBe("park"); if (parked.kind !== "park") throw new Error("expected a park"); expect(parked.park.correlationId).toBe(CORRELATION_ID); @@ -286,7 +288,7 @@ describe("createSidecarSpawnSuspendableChild", () => { // The re-invoked step ran with the delivered decision and the run // completed. - const terminal = await handle.next(); + const terminal = await handle.next(noWake); expect(terminal.kind).toBe("terminal"); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("completed"); @@ -328,7 +330,7 @@ describe("createSidecarSpawnSuspendableChild", () => { ); // The body's childWorkflow step failed loud, so the body run settles failed. - const terminal = await handle.next(); + const terminal = await handle.next(noWake); expect(terminal.kind).toBe("terminal"); if (terminal.kind !== "terminal") throw new Error("expected a terminal"); expect(terminal.terminalStatus).toBe("failed"); diff --git a/packages/workflow-host/src/ipc/control-payloads.test.ts b/packages/workflow-host/src/ipc/control-payloads.test.ts index 462a16ca3..984e9c3ad 100644 --- a/packages/workflow-host/src/ipc/control-payloads.test.ts +++ b/packages/workflow-host/src/ipc/control-payloads.test.ts @@ -114,3 +114,17 @@ test("dispatching control payloads preserves validation and parsed values", () = expect(dispatched).toEqual(combined); } }); + +test("a quiet answer names its earliest timer as an ISO time", () => { + const answer = (wakeAt: string) => ({ + type: "quiet.response", + data: { requestId: "quiet-1", quiet: true, wakeAt }, + }); + expect( + parseControlPayload(answer("2026-10-02T00:00:00.000Z")) instanceof + type.errors, + ).toBe(false); + expect(parseControlPayload(answer("tomorrow")) instanceof type.errors).toBe( + true, + ); +}); diff --git a/packages/workflow-host/src/ipc/control-payloads.ts b/packages/workflow-host/src/ipc/control-payloads.ts index ae16f5fbd..c164d8c45 100644 --- a/packages/workflow-host/src/ipc/control-payloads.ts +++ b/packages/workflow-host/src/ipc/control-payloads.ts @@ -957,6 +957,37 @@ const controlPayloadFactories = defineControlPayloadFactories({ }).array(), }, }), + "quiet.request": () => + type({ + // Supervisor asks whether every run only waits on input from outside, + // so it can stop the child. With `wait` the child answers once it is + // quiet; without, it answers at once. While the run lives it first + // makes every buffered event durable. `runId` is the deployment's run. + // One check is out at a time: a new request supersedes one still + // waiting, which gets no answer. + type: "'quiet.request'", + data: { + requestId: "string > 0", + runId: "string > 0", + wait: "boolean", + }, + }), + "quiet.response": () => + type({ + // `ended` alone when the deployment's run has ended, so there is + // nothing to stop it for. Otherwise `wakeAt` is the earliest timer the + // child has queued, and `error` says the child could not check. + type: "'quiet.response'", + data: type({ + requestId: "string > 0", + ended: "true", + }).or({ + requestId: "string > 0", + quiet: "boolean", + "wakeAt?": "string.date.iso", + "error?": "string", + }), + }), "resumed.runs": () => type({ // Child reports self-discovered runs after reconnect or recycle. diff --git a/packages/workflow-host/src/seams/loop-signal-delivery.test.ts b/packages/workflow-host/src/seams/loop-signal-delivery.test.ts index 60c880340..ca69aeebc 100644 --- a/packages/workflow-host/src/seams/loop-signal-delivery.test.ts +++ b/packages/workflow-host/src/seams/loop-signal-delivery.test.ts @@ -162,8 +162,8 @@ function deliverOnceSubscribed(inner: SignalChannelHandle): { const channel: SignalChannel = { deliver: (name, payload, signalId) => inner.deliver(name, payload, signalId), - async awaitNext(name, signal) { - const pending = inner.awaitNext(name, signal); + async awaitNext(name, onWake, signal) { + const pending = inner.awaitNext(name, onWake, signal); if (name === "go") resolveSubscribed(); return pending; }, diff --git a/packages/workflow-host/src/seams/quiet-scheduler.test.ts b/packages/workflow-host/src/seams/quiet-scheduler.test.ts new file mode 100644 index 000000000..444b9ff1e --- /dev/null +++ b/packages/workflow-host/src/seams/quiet-scheduler.test.ts @@ -0,0 +1,252 @@ +// A child in standby is woken for its earliest queued timer, so a sleeping +// run may look quiet only once the host scheduler has queued its timer, and a +// fired timer must count as delivery until the woken run counts as working. +// These drive the real host scheduler and its runtime adapter: in a child, +// `scheduleIn` queues nothing, and the timer is queued by the scheduler's own +// TimerSet subscription over the proxy store. + +import { afterAll, beforeEach, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { createDefaultDirectorRegistry } from "@intx/agent"; +import { generateKeyPair } from "@intx/crypto"; +import { createRepoStore } from "@intx/hub-sessions"; +import type { + AuthorizeFn, + KindHandler, + Principal, + RepoId, + ValidatePushResult, +} from "@intx/hub-sessions"; +import { + awaitSignal, + createInMemoryBlobSubstrate, + createNoopDrainController, + defineWorkflow, + emptyState, + runtimeRun, + sleep, + type WorkflowDefinition, +} from "@intx/workflow"; +import { bufferedEvents, isQuiet, whenQuiet } from "@intx/workflow/runtime"; + +import { createWorkflowRunRepoStore } from "../adapters/repo-store"; +import { createProxyWorkflowRunRepoStore } from "../child/proxy-repo-store"; +import { createWorkflowHostScheduler } from "./scheduler"; +import { adaptHostScheduler } from "./scheduler-adapter"; +import { createWorkflowHostSignalChannel } from "./signal-channel"; + +const REF = "refs/heads/main"; +const principal: Principal = { kind: "test" }; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const tempDirs: string[] = []; + +beforeEach(() => { + if (!isQuiet()) + throw new Error("work or a delivery from an earlier test is still running"); +}); + +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }); + } +}); + +function permissiveHandler(directoryPrefix: string): KindHandler { + return { + kind: "skill", + directoryPrefix, + validatePush(): ValidatePushResult { + return { ok: true }; + }, + onRefUpdated() { + /* no-op */ + }, + }; +} + +async function childWithScheduler( + prefix: string, + definition: WorkflowDefinition, +) { + const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dataDir); + const bare = createRepoStore({ + dataDir, + signingKey: await generateKeyPair(), + handlers: { skill: permissiveHandler(prefix) }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: prefix }; + const proxy = createProxyWorkflowRunRepoStore({ + bareStore: bare, + bridge: { + async submit(req) { + const result = await bare.writeTreePreservingPrefix( + principal, + repoId, + req.ref, + { + preservePrefix: req.preservePrefix, + message: req.message, + merge: req.merge, + }, + ); + return { commitSha: result.commitSha }; + }, + handleMergeRequest: () => undefined, + handleWriteResponse: () => undefined, + cancelAll: () => undefined, + pendingCount: 0, + }, + workflowRunRepoId: repoId, + }); + const adapter = createWorkflowRunRepoStore({ + substrate: proxy, + repoId, + principal, + ref: REF, + }); + // What each run has made durable, readable at once. + const durable = new Map(); + const record = (runId: string, kinds: readonly string[]): void => { + durable.set(runId, [...(durable.get(runId) ?? []), ...kinds]); + }; + const runtimeStore: typeof adapter = { + ...adapter, + read: (runId) => adapter.read(runId), + subscribe: (runId, opts) => adapter.subscribe(runId, opts), + async append(runId, event) { + await adapter.append(runId, event); + record(runId, [event.kind]); + }, + async appendBatch(runId, events) { + await adapter.appendBatch(runId, events); + record( + runId, + events.map((event) => event.kind), + ); + }, + }; + // Armed timers fire on demand instead of after their delay. + const armed: (() => void)[] = []; + const scheduler = createWorkflowHostScheduler({ + repoStore: proxy, + runtimeStore, + principal, + listActiveDeployments: () => [repoId], + ref: REF, + clock: () => new Date(), + scheduleTimeout: (handler) => { + armed.push(handler); + return () => undefined; + }, + }); + await scheduler.start(); + + const start = (runId: string) => { + const teardown = new AbortController(); + const channel = createWorkflowHostSignalChannel({ + repoStore: proxy, + runtimeStore, + principal, + repoId, + ref: REF, + runId, + readState: () => emptyState(runId), + newId: () => "generated-signal", + clock: () => new Date(), + }); + const run = runtimeRun( + definition, + { + repoStore: runtimeStore, + scheduler: adaptHostScheduler(scheduler), + signalChannel: channel, + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: () => { + throw new Error("no agent step runs here"); + }, + invokeAction: async () => ({ output: "done" }), + spawnChild: () => { + throw new Error("no nested run here"); + }, + clock: () => new Date(), + newId: (idPrefix) => `${idPrefix}-id`, + drain: createNoopDrainController(definition), + hasUpstreamSignalResolver: true, + }, + { runId, triggerPayload: null, localAbort: teardown.signal }, + ); + return { + run, + async stop(): Promise { + teardown.abort(); + await run.complete.catch(() => undefined); + await channel.stop(); + await scheduler.stop(); + }, + }; + }; + const events = (runId: string): string[] => [ + ...(durable.get(runId) ?? []), + ...bufferedEvents(runId).map((event) => event.kind), + ]; + return { scheduler, armed, start, events }; +} + +test("a sleeping run looks quiet only once the host scheduler has queued its timer", async () => { + const definition = defineWorkflow({ + id: "real-scheduler-sleep", + trigger: { type: "manual" }, + steps: { nap: sleep({ duration: 3_600_000 }) }, + }); + const child = await childWithScheduler("real-scheduler-sleep-", definition); + const { stop } = child.start("run-sleep"); + const queued = await whenQuiet().then(() => + child.scheduler.queuedTimers().map((timer) => timer.runId), + ); + await stop(); + expect(queued).toEqual(["run-sleep"]); +}); + +test("a timed gate looks quiet only once the host scheduler has queued its timeout", async () => { + const definition = defineWorkflow({ + id: "real-scheduler-gate", + trigger: { type: "manual" }, + steps: { gate: awaitSignal({ name: "go", timeout: 3_600_000 }) }, + }); + const child = await childWithScheduler("real-scheduler-gate-", definition); + const { stop } = child.start("run-gate"); + const queued = await whenQuiet().then(() => + child.scheduler.queuedTimers().map((timer) => timer.runId), + ); + await stop(); + expect(queued).toEqual(["run-gate"]); +}); + +test("a timer the host scheduler fires counts as delivery until the woken run works", async () => { + const definition = defineWorkflow({ + id: "real-scheduler-fire", + trigger: { type: "manual" }, + steps: { nap: sleep({ duration: 3_600_000 }) }, + }); + const child = await childWithScheduler("real-scheduler-fire-", definition); + const { run, stop } = child.start("run-fire"); + await whenQuiet(); + const fire = child.armed.shift(); + if (fire === undefined) throw new Error("the scheduler armed no timer"); + fire(); + const seen = await whenQuiet().then(() => child.events("run-fire")); + await run.complete; + await stop(); + expect(seen).toContain("RunCompleted"); +}); diff --git a/packages/workflow-host/src/seams/scheduler.test.ts b/packages/workflow-host/src/seams/scheduler.test.ts index 70a96ac44..8f9a9dfa9 100644 --- a/packages/workflow-host/src/seams/scheduler.test.ts +++ b/packages/workflow-host/src/seams/scheduler.test.ts @@ -279,6 +279,40 @@ describe("workflow-host scheduler", () => { { timeout: 5000 }, ); + test("reports when the earliest queued timer fires, whatever order they were queued in", async () => { + const dataDir = await makeTempDir("scheduler-earliest-"); + const store = createRepoStore({ + dataDir, + signingKey, + handlers: { + skill: permissiveHandler("workflow-runs-earliest"), + }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: "deployment-earliest" }; + const earliest = Date.now() + 60_000; + await seedTimerSet(store, repoId, "r1", 0, "t-late", earliest + 60_000); + await seedTimerSet(store, repoId, "r2", 0, "t-early", earliest); + await seedTimerSet(store, repoId, "r3", 0, "t-middle", earliest + 30_000); + const scheduler = createWorkflowHostScheduler({ + repoStore: store, + runtimeStore: runtimeStoreFor(store, repoId), + principal, + listActiveDeployments: () => [repoId], + ref: REF, + clock: () => new Date(), + scheduleTimeout: createManualTimeouts().scheduleTimeout, + }); + try { + expect(scheduler.earliestQueuedTimer()).toBeUndefined(); + await scheduler.start(); + expect(scheduler.queuedTimers()).toHaveLength(3); + expect(scheduler.earliestQueuedTimer()).toEqual(new Date(earliest)); + } finally { + await scheduler.stop(); + } + }); + test( "recovery enumerates unfired timers; matched TimerFired skips re-queue", async () => { @@ -1013,12 +1047,6 @@ describe("workflow-host scheduler", () => { timerId: "t-terminal", fireAt: new Date(Date.now() + 60_000).toISOString(), }); - await runtimeStore.append(runId, { - kind: "RunFailed", - seq: 2, - at, - error: { message: "already terminal" }, - }); const timeouts = createManualTimeouts(); const commits = observeSchedulerCommits(store); @@ -1034,6 +1062,13 @@ describe("workflow-host scheduler", () => { try { await scheduler.start(); expect(scheduler.queuedTimers()).toHaveLength(1); + // The run ends after recovery queued its timer. + await runtimeStore.append(runId, { + kind: "RunFailed", + seq: 2, + at, + error: { message: "already terminal" }, + }); timeouts.fireAll(); await commits.whenCommitted(); @@ -1050,3 +1085,85 @@ describe("workflow-host scheduler", () => { } }); }); + +describe("workflow-host scheduler recovery", () => { + const at = "2026-01-01T00:00:00.000Z"; + const fireAt = "2026-01-02T00:00:00.000Z"; + const timerSet = (seq: number, stepId?: string) => ({ + seq, + type: "TimerSet", + at, + timerId: "t", + fireAt, + ...(stepId !== undefined ? { stepId } : {}), + }); + const cases: [string, Record[], string[]][] = [ + [ + "drops a timer whose step settled after it was set", + [ + { seq: 1, type: "StepStarted", at, stepId: "gate", attempt: 1 }, + timerSet(2, "gate"), + { seq: 3, type: "StepCompleted", at, stepId: "gate", attempt: 1 }, + ], + [], + ], + [ + "drops a timer of a run that ended", + [timerSet(1), { seq: 2, type: "RunCompleted", at }], + [], + ], + [ + "keeps a retry timer set after its step failed", + [ + { seq: 1, type: "StepStarted", at, stepId: "gate", attempt: 1 }, + { seq: 2, type: "StepFailed", at, stepId: "gate", attempt: 1 }, + timerSet(3, "gate"), + ], + ["t"], + ], + [ + "drops a fired timer whose TimerFired sorts before its TimerSet", + [timerSet(9), { seq: 10, type: "TimerFired", at, timerId: "t" }], + [], + ], + ]; + test.each(cases)("%s", async (_name, events, queued) => { + const dataDir = await makeTempDir("scheduler-recovery-"); + const store = createRepoStore({ + dataDir, + signingKey, + handlers: { + skill: permissiveHandler("workflow-runs-recovery"), + }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: "deployment-recovery" }; + const runId = "r-recovery"; + await store.writeTree(principal, repoId, REF, { + files: Object.fromEntries( + events.map((event) => [ + `runs/${runId}/events/${String(event.seq)}.json`, + JSON.stringify(event), + ]), + ), + message: "seed the run's events", + }); + const scheduler = createWorkflowHostScheduler({ + repoStore: store, + runtimeStore: runtimeStoreFor(store, repoId), + principal, + listActiveDeployments: () => [repoId], + ref: REF, + clock: () => new Date(at), + scheduleTimeout: createManualTimeouts().scheduleTimeout, + }); + try { + await scheduler.start(); + expect(scheduler.queuedTimers().map((timer) => timer.timerId)).toEqual( + queued, + ); + } finally { + await scheduler.stop(); + } + }); +}); diff --git a/packages/workflow-host/src/seams/scheduler.ts b/packages/workflow-host/src/seams/scheduler.ts index 40b200389..b9238cb2d 100644 --- a/packages/workflow-host/src/seams/scheduler.ts +++ b/packages/workflow-host/src/seams/scheduler.ts @@ -8,9 +8,10 @@ // the host -- workflow-process child, supervisor, sidecar handler -- // commits `TimerFired`. This single-writer invariant is what makes // the recovery story work: at startup the scheduler walks the -// persisted log for unfired `TimerSet` events and re-queues them, -// confident that any `TimerFired` it does not find was never -// committed and is its responsibility to commit now. +// persisted log for unfired `TimerSet` events and re-queues those +// something still waits on, confident that any `TimerFired` it does +// not find was never committed and is its responsibility to commit +// now. A timer whose step has settled or whose run has ended is left. // // Recovery semantics: // - One-shot timers (`TimerSet` without a `cron` discriminator) on @@ -58,6 +59,7 @@ import { WORKFLOW_RUN_RUNS_PREFIX, } from "@intx/hub-sessions/substrate"; import { + beginDelivery, withRunCommitBarrier, type RepoStore as RuntimeRepoStore, } from "@intx/workflow/runtime"; @@ -65,7 +67,7 @@ import { /** * Substrate-shape envelope for the workflow-event blob committed to * `runs//events/.json`. The validator covers the two - * event types the scheduler reads (TimerSet, TimerFired) and the + * event types it needs for timers (TimerSet, TimerFired) and the * single type it writes (TimerFired). Non-timer blobs at the same * path prefix do not match this validator and are skipped silently * by the recovery walk. `seq` mirrors the integer in the filename, @@ -87,6 +89,17 @@ export const TimerEventEnvelope = type({ }); export type TimerEventEnvelope = typeof TimerEventEnvelope.infer; +// The events that settle a step or end its run. The runtime disposes a timer +// whose step settled on a sibling event, such as a signal that beat its +// timeout, without committing anything, so recovery reads these too. +const SettlingEventEnvelope = type({ + seq: "number >= 0", + type: "'StepCompleted' | 'StepFailed' | 'RunCompleted' | 'RunFailed' | 'RunCancelled'", + "stepId?": "string", + "+": "ignore", +}); +type SettlingEventEnvelope = typeof SettlingEventEnvelope.infer; + export type SchedulerOpts = { /** * Substrate handle the scheduler reads from and writes to. The @@ -163,8 +176,9 @@ type QueuedTimer = { export type SchedulerHandle = { /** * Run start-time recovery against every active deployment. After - * the recovery walk completes, every unfired one-shot timer is - * queued and every missed cron tick has been skipped per the spec. + * the recovery walk completes, every unfired one-shot timer that + * something still waits on is queued and every missed cron tick has + * been skipped per the spec. * Idempotent: a second `start()` is a no-op while the first is * still pending. */ @@ -186,15 +200,16 @@ export type SchedulerHandle = { */ cancelQueued(runId: string, timerId: string): void; /** - * Test-visible view of currently-queued timers. The shape is - * intentionally narrow: the runtime body never inspects the - * scheduler's queue; tests assert on it directly. + * The currently-queued timers. The runtime body never inspects the + * queue; tests assert on it directly. */ queuedTimers(): readonly { runId: string; timerId: string; fireAtMs: number; }[]; + /** When the earliest queued timer fires, which a child going quiet reports. */ + earliestQueuedTimer(): Date | undefined; }; export function createWorkflowHostScheduler( @@ -226,9 +241,17 @@ export function createWorkflowHostScheduler( const key = queueKey(runId, timerId); const entry = queues.get(key); if (entry === undefined) return; + // A fired timer is input on its way to the run that waits for it, from + // the moment it leaves the queue until its commit reaches the + // subscriptions that wait for it. + const endDelivery = beginDelivery(); queues.delete(key); - if (stopped) return; - await commitTimerFired(opts, runId, timerId); + try { + if (stopped) return; + await commitTimerFired(opts, runId, timerId); + } finally { + endDelivery(); + } } function enqueue( @@ -302,40 +325,67 @@ export function createWorkflowHostScheduler( async function recoverDeployment(repoId: RepoId): Promise { const events = await readAllEvents(opts, repoId); - // Build per-(runId, timerId) ledger: a TimerSet without a - // matching TimerFired is unfired. The walk is order-insensitive - // because the second pass deletes matched entries. - const unfired = new Map< + // Per-(runId, timerId) ledger. Blobs are listed by name, not by seq, so + // the walk collects everything first and compares seqs after. + const set = new Map< string, - { runId: string; timerId: string; fireAtMs: number; cron: boolean } + { + runId: string; + timerId: string; + fireAtMs: number; + cron: boolean; + seq: number; + stepId: string | undefined; + } >(); + const fired = new Set(); + const stepSettledAt = new Map(); + const endedRuns = new Set(); for (const e of events) { - if (e.envelope.type === "TimerSet") { - const fireAt = e.envelope.fireAt; + const envelope = e.envelope; + if (envelope.type === "TimerSet") { + const fireAt = envelope.fireAt; if (fireAt === undefined) { throw new Error( - `scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} missing fireAt`, + `scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${envelope.timerId} missing fireAt`, ); } const fireAtMs = Date.parse(fireAt); if (Number.isNaN(fireAtMs)) { throw new Error( - `scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} fireAt unparseable: ${fireAt}`, + `scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${envelope.timerId} fireAt unparseable: ${fireAt}`, ); } - const cron = e.envelope.cron !== undefined && e.envelope.cron !== null; - unfired.set(`${e.runId} ${e.envelope.timerId}`, { + set.set(`${e.runId} ${envelope.timerId}`, { runId: e.runId, - timerId: e.envelope.timerId, + timerId: envelope.timerId, fireAtMs, - cron, + cron: envelope.cron !== undefined && envelope.cron !== null, + seq: envelope.seq, + stepId: envelope.stepId ?? undefined, }); - } else { - unfired.delete(`${e.runId} ${e.envelope.timerId}`); + } else if (envelope.type === "TimerFired") { + fired.add(`${e.runId} ${envelope.timerId}`); + } else if (envelope.type.startsWith("Run")) { + endedRuns.add(e.runId); + } else if (envelope.stepId !== undefined) { + const key = `${e.runId} ${envelope.stepId}`; + stepSettledAt.set( + key, + Math.max(stepSettledAt.get(key) ?? -1, envelope.seq), + ); } } const now = opts.clock().getTime(); - for (const entry of unfired.values()) { + for (const [key, entry] of set) { + if (fired.has(key) || endedRuns.has(entry.runId)) continue; + // A step that settled after the timer was set no longer waits on it. + if ( + entry.stepId !== undefined && + (stepSettledAt.get(`${entry.runId} ${entry.stepId}`) ?? -1) > entry.seq + ) { + continue; + } if (entry.cron && entry.fireAtMs < now) { // Spec: missed cron ticks are skipped on resume. The next // cron tick will be committed by whoever owns the cron @@ -398,17 +448,26 @@ export function createWorkflowHostScheduler( fireAtMs: t.fireAtMs, })); }, + earliestQueuedTimer() { + let earliest: number | undefined; + for (const timer of queues.values()) { + if (earliest === undefined || timer.fireAtMs < earliest) { + earliest = timer.fireAtMs; + } + } + return earliest === undefined ? undefined : new Date(earliest); + }, }; } type EventReadEntry = { runId: string; - envelope: TimerEventEnvelope; + envelope: TimerEventEnvelope | SettlingEventEnvelope; }; /** - * Read every timer-event blob across every run under the given - * workflow-run repo. Recovery reconstructs its ledger from committed + * Read every timer-event and settling-event blob across every run under the + * given workflow-run repo. Recovery reconstructs its ledger from committed * state, so `enumerateEventBlobs` reads the git object store at the * events ref tip via `openCommittedReads` rather than the materialized * working tree, whose non-atomic post-commit checkout can lag the @@ -425,9 +484,14 @@ async function readAllEvents( const entries: EventReadEntry[] = []; const records = await enumerateEventBlobs(opts, repoId); for (const r of records) { + const settling = SettlingEventEnvelope(r.payload); + if (!(settling instanceof type.errors)) { + entries.push({ runId: r.runId, envelope: settling }); + continue; + } const env = TimerEventEnvelope(r.payload); if (env instanceof type.errors) { - // A non-timer event blob (StepStarted, RunStarted, ...) at the + // Any other event blob (StepStarted, RunStarted, ...) at the // same path prefix is expected -- the scheduler skips it // silently. A timer-shaped blob whose narrow fails is a // substrate-level invariant violation; the kind handler's diff --git a/packages/workflow-host/src/seams/signal-channel.test.ts b/packages/workflow-host/src/seams/signal-channel.test.ts index 2fbfec103..45900c455 100644 --- a/packages/workflow-host/src/seams/signal-channel.test.ts +++ b/packages/workflow-host/src/seams/signal-channel.test.ts @@ -20,6 +20,8 @@ import { commitBuffered, dropChain, reloadState } from "@intx/workflow/runtime"; import { createWorkflowRunRepoStore } from "../adapters/repo-store"; import { createWorkflowHostSignalChannel } from "./signal-channel"; +const noWake = (): void => undefined; + const tempDirs: string[] = []; async function makeTempDir(prefix: string): Promise { @@ -116,7 +118,7 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const received = channel.awaitNext("approve"); + const received = channel.awaitNext("approve", noWake); // Give the subscription a moment to install before the commit // lands so the `from: "head"` watcher sees the post-subscribe // ref-update. @@ -169,7 +171,12 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const out = await channel.awaitNext("approve"); + let woken = 0; + const out = await channel.awaitNext("approve", () => { + woken += 1; + }); + // The caller counts as working again before it continues. + expect(woken).toBe(1); expect(out.signalId).toBe("sig-queued"); expect(out.payload).toEqual({ resumed: true }); } finally { @@ -206,7 +213,7 @@ describe("workflow-host signal channel", () => { }); try { const ac = new AbortController(); - const pending = channel.awaitNext("approve", ac.signal); + const pending = channel.awaitNext("approve", noWake, ac.signal); await new Promise((resolve) => { setTimeout(resolve, 25); }); @@ -215,7 +222,7 @@ describe("workflow-host signal channel", () => { // After the abort, a deliver for that name must not crash the // channel and a fresh awaiter must still resolve normally. - const second = channel.awaitNext("approve"); + const second = channel.awaitNext("approve", noWake); await new Promise((resolve) => { setTimeout(resolve, 50); }); @@ -261,8 +268,8 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const a = channel.awaitNext("approve"); - const b = channel.awaitNext("reject"); + const a = channel.awaitNext("approve", noWake); + const b = channel.awaitNext("reject", noWake); await new Promise((resolve) => { setTimeout(resolve, 50); }); @@ -308,7 +315,7 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const first = channel.awaitNext("approve"); + const first = channel.awaitNext("approve", noWake); await new Promise((resolve) => { setTimeout(resolve, 50); }); @@ -330,7 +337,7 @@ describe("workflow-host signal channel", () => { // path leaks the per-name entry in the subscriptions Map, // this awaitNext joins the awaiter queue against a dead // subscribeKind loop and the deliver below never wakes it. - const second = channel.awaitNext("approve"); + const second = channel.awaitNext("approve", noWake); await new Promise((resolve) => { setTimeout(resolve, 50); }); @@ -384,8 +391,8 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const first = channel.awaitNext("approve"); - const second = channel.awaitNext("approve"); + const first = channel.awaitNext("approve", noWake); + const second = channel.awaitNext("approve", noWake); await new Promise((resolve) => { setTimeout(resolve, 50); }); @@ -547,7 +554,7 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const received = channel.awaitNext("approve"); + const received = channel.awaitNext("approve", noWake); await channel.deliver("approve", { ok: true }, "sig-empty"); expect(await received).toEqual({ signalId: "sig-empty", @@ -646,11 +653,16 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const named = await channel.awaitNext("go"); + const named = await channel.awaitNext("go", noWake); expect(named).toEqual({ payload: { n: true }, signalId: "sig-name" }); expect(named).not.toHaveProperty("correlationId"); - const correlated = await channel.awaitNext("go", undefined, "corr-1"); + const correlated = await channel.awaitNext( + "go", + noWake, + undefined, + "corr-1", + ); expect(correlated).toEqual({ payload: { c: true }, signalId: "sig-c", @@ -659,6 +671,7 @@ describe("workflow-host signal channel", () => { const otherName = await channel.awaitNext( "stop", + noWake, undefined, "corr-other", ); @@ -666,7 +679,12 @@ describe("workflow-host signal channel", () => { // The non-matching name stayed at the head of corr-1. Peek // reads past it instead of handing it to the "go" awaiter. - const skipped = await channel.awaitNext("stop", undefined, "corr-1"); + const skipped = await channel.awaitNext( + "stop", + noWake, + undefined, + "corr-1", + ); expect(skipped.signalId).toBe("sig-stop"); } finally { await channel.stop(); @@ -692,7 +710,7 @@ describe("workflow-host signal channel", () => { }); try { let nameStole = false; - const pending = isolated.awaitNext("go").then( + const pending = isolated.awaitNext("go", noWake).then( () => { nameStole = true; }, @@ -700,7 +718,12 @@ describe("workflow-host signal channel", () => { // stop() rejects the name awaiter that peek did not satisfy. }, ); - const correlatedHit = await isolated.awaitNext("go", undefined, "corr-1"); + const correlatedHit = await isolated.awaitNext( + "go", + noWake, + undefined, + "corr-1", + ); expect(correlatedHit.signalId).toBe("sig-c2"); expect(nameStole).toBe(false); void pending; @@ -732,8 +755,8 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const named = channel.awaitNext("go"); - const correlated = channel.awaitNext("go", undefined, "corr-1"); + const named = channel.awaitNext("go", noWake); + const correlated = channel.awaitNext("go", noWake, undefined, "corr-1"); await channel.deliver("go", { c: true }, "sig-c", "corr-1"); expect(await correlated).toEqual({ @@ -782,8 +805,8 @@ describe("workflow-host signal channel", () => { clock: () => new Date(), }); try { - const correlated = channel.awaitNext("go", undefined, "corr-1"); - const named = channel.awaitNext("go"); + const correlated = channel.awaitNext("go", noWake, undefined, "corr-1"); + const named = channel.awaitNext("go", noWake); await channel.deliver("go", { n: true }, "sig-name"); const namedResult = await named; diff --git a/packages/workflow-host/src/seams/signal-channel.ts b/packages/workflow-host/src/seams/signal-channel.ts index 9770d664a..6ffded61a 100644 --- a/packages/workflow-host/src/seams/signal-channel.ts +++ b/packages/workflow-host/src/seams/signal-channel.ts @@ -126,6 +126,9 @@ type Awaiter = { correlationId?: string; }) => void; reject: (cause: unknown) => void; + // The caller's `onWake`, called before this channel stops counting the + // entry that settles the wait. + wake: () => void; signal?: AbortSignal; onAbort?: () => void; }; @@ -259,28 +262,25 @@ export function createWorkflowHostSignalChannel( // the same slot picks it up via the state-reader path. break; } + const received = { + payload: entry.event.payload, + signalId: entry.event.signalId, + ...(entry.event.correlationId !== undefined + ? { correlationId: entry.event.correlationId } + : {}), + }; + // Both the teardown and the next pull end this entry's delivery. + next.wake(); if (awaiterCount(slot) === 0) { // Resolving the last awaiter on this subscription. Tear // down before the resolve so any awaitNext call inside // the awaiter's continuation installs a fresh // subscription instead of joining a dead one. teardown(); - next.resolve({ - payload: entry.event.payload, - signalId: entry.event.signalId, - ...(entry.event.correlationId !== undefined - ? { correlationId: entry.event.correlationId } - : {}), - }); + next.resolve(received); return; } - next.resolve({ - payload: entry.event.payload, - signalId: entry.event.signalId, - ...(entry.event.correlationId !== undefined - ? { correlationId: entry.event.correlationId } - : {}), - }); + next.resolve(received); } } finally { // Catches the queue-drained natural break, the `stopped` @@ -381,18 +381,23 @@ export function createWorkflowHostSignalChannel( ), ); }, - async awaitNext(name, signal, correlationId) { + async awaitNext(name, onWake, signal, correlationId) { if (stopped) { + onWake(); throw new Error("signal channel: awaitNext after stop"); } if (signal !== undefined && signal.aborted) { + onWake(); throw new Error("aborted"); } // Drain the state-machine queue first. A signal that was // committed before the awaiter subscribed lives in the name // queue, or under its correlation, after log replay. const queued = peekState(name, correlationId); - if (queued !== null) return queued; + if (queued !== null) { + onWake(); + return queued; + } const slot = slotOf(name, correlationId); return new Promise<{ @@ -403,10 +408,12 @@ export function createWorkflowHostSignalChannel( const awaiter: Awaiter = { resolve, reject, + wake: onWake, ...(signal !== undefined ? { signal } : {}), }; if (signal !== undefined) { const onAbort = (): void => { + onWake(); const list = awaiters.get(slot); if (list !== undefined) { const idx = list.indexOf(awaiter); @@ -438,6 +445,7 @@ export function createWorkflowHostSignalChannel( if (awaiter.signal !== undefined && awaiter.onAbort !== undefined) { awaiter.signal.removeEventListener("abort", awaiter.onAbort); } + awaiter.wake(); awaiter.reject(new Error("signal channel stopped")); } } diff --git a/packages/workflow/src/runlocal/signal-channel.test.ts b/packages/workflow/src/runlocal/signal-channel.test.ts index 104ef431b..76a4d5fdc 100644 --- a/packages/workflow/src/runlocal/signal-channel.test.ts +++ b/packages/workflow/src/runlocal/signal-channel.test.ts @@ -2,28 +2,35 @@ import { describe, expect, test } from "bun:test"; import { createInMemorySignalChannel } from "./signal-channel"; +const noWake = (): void => undefined; + describe("in-memory signal channel correlation", () => { test("a queued name delivery is not taken by a correlated awaiter", async () => { const channel = createInMemorySignalChannel(); await channel.deliver("go", { n: 1 }, "sig-name"); await channel.deliver("go", { c: 1 }, "sig-c", "corr-1"); - const correlated = await channel.awaitNext("go", undefined, "corr-1"); + const correlated = await channel.awaitNext( + "go", + noWake, + undefined, + "corr-1", + ); expect(correlated).toEqual({ payload: { c: 1 }, signalId: "sig-c", correlationId: "corr-1", }); - const named = await channel.awaitNext("go"); + const named = await channel.awaitNext("go", noWake); expect(named).toEqual({ payload: { n: 1 }, signalId: "sig-name" }); expect(named).not.toHaveProperty("correlationId"); }); test("a correlated awaiter registered first does not consume a name delivery", async () => { const channel = createInMemorySignalChannel(); - const correlated = channel.awaitNext("go", undefined, "corr-1"); - const named = channel.awaitNext("go"); + const correlated = channel.awaitNext("go", noWake, undefined, "corr-1"); + const named = channel.awaitNext("go", noWake); await channel.deliver("go", { n: 1 }, "sig-name"); const namedResult = await named; @@ -47,9 +54,9 @@ describe("in-memory signal channel correlation", () => { test("same-correlation deliveries stay ordered around a different correlation", async () => { const channel = createInMemorySignalChannel(); - const first = channel.awaitNext("go", undefined, "corr-a"); - const other = channel.awaitNext("go", undefined, "corr-b"); - const second = channel.awaitNext("go", undefined, "corr-a"); + const first = channel.awaitNext("go", noWake, undefined, "corr-a"); + const other = channel.awaitNext("go", noWake, undefined, "corr-b"); + const second = channel.awaitNext("go", noWake, undefined, "corr-a"); await channel.deliver("go", { n: 1 }, "a1", "corr-a"); await channel.deliver("go", { n: 2 }, "b1", "corr-b"); @@ -66,13 +73,13 @@ describe("in-memory signal channel correlation", () => { await channel.deliver("go", { n: 1 }, "sig-name"); await channel.deliver("go", { c: 2 }, "sig-c2", "corr-1"); - expect((await channel.awaitNext("go", undefined, "corr-1")).signalId).toBe( - "sig-c1", - ); - expect((await channel.awaitNext("go", undefined, "corr-1")).signalId).toBe( - "sig-c2", - ); - expect((await channel.awaitNext("go")).signalId).toBe("sig-name"); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-1")).signalId, + ).toBe("sig-c1"); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-1")).signalId, + ).toBe("sig-c2"); + expect((await channel.awaitNext("go", noWake)).signalId).toBe("sig-name"); }); test("same-id deliveries stay ordered when a different id is queued between them", async () => { @@ -81,15 +88,15 @@ describe("in-memory signal channel correlation", () => { await channel.deliver("go", { n: 2 }, "b1", "corr-b"); await channel.deliver("go", { n: 3 }, "a2", "corr-a"); - expect((await channel.awaitNext("go", undefined, "corr-a")).signalId).toBe( - "a1", - ); - expect((await channel.awaitNext("go", undefined, "corr-a")).signalId).toBe( - "a2", - ); - expect((await channel.awaitNext("go", undefined, "corr-b")).signalId).toBe( - "b1", - ); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-a")).signalId, + ).toBe("a1"); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-a")).signalId, + ).toBe("a2"); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-b")).signalId, + ).toBe("b1"); }); test("a name awaiter does not take a queued correlated delivery", async () => { @@ -97,11 +104,11 @@ describe("in-memory signal channel correlation", () => { await channel.deliver("go", { c: 1 }, "sig-c", "corr-1"); await channel.deliver("go", { n: 1 }, "sig-name"); - const named = await channel.awaitNext("go"); + const named = await channel.awaitNext("go", noWake); expect(named).toEqual({ payload: { n: 1 }, signalId: "sig-name" }); expect(named).not.toHaveProperty("correlationId"); - expect((await channel.awaitNext("go", undefined, "corr-1")).signalId).toBe( - "sig-c", - ); + expect( + (await channel.awaitNext("go", noWake, undefined, "corr-1")).signalId, + ).toBe("sig-c"); }); }); diff --git a/packages/workflow/src/runlocal/signal-channel.ts b/packages/workflow/src/runlocal/signal-channel.ts index 0773e2332..da5a6ca85 100644 --- a/packages/workflow/src/runlocal/signal-channel.ts +++ b/packages/workflow/src/runlocal/signal-channel.ts @@ -71,7 +71,8 @@ export function createInMemorySignalChannel( } q.push(delivery); }, - async awaitNext(name, signal, correlationId) { + // Nothing in a local run asks whether it is quiet. + async awaitNext(name, _onWake, signal, correlationId) { const q = queued.get(name); const queuedMatch = q?.findIndex((item) => item.correlationId === correlationId) ?? -1; diff --git a/packages/workflow/src/runtime/activity.ts b/packages/workflow/src/runtime/activity.ts new file mode 100644 index 000000000..0442fc226 --- /dev/null +++ b/packages/workflow/src/runtime/activity.ts @@ -0,0 +1,165 @@ +// What the runtime is doing in this process. A host that stops the process +// while every run waits must know that nothing here is still working, and +// that nothing committed is still on its way to the run that waits for it. +// +// Work is a run loop or a step in progress. A step that waits on input from +// outside -- a signal, a timer, a nested run's next event -- is not working +// while it waits. Delivery is the host handing a committed input to the run +// that waits for it. Both counts are process-wide, like the commit chain, +// since every run in a process shares one stop. +// +// Whatever wakes a waiter hands its own count over: it resumes the waiter +// before it stops counting itself. A waiter that only counted again once its +// await continued would leave a gap in which nothing counts. + +let working = 0; +let delivering = 0; +// Every start of work or of a delivery, so a check can tell that something +// began since it last looked, even work that has already ended again. +let starts = 0; +const waiters = new Set<() => void>(); + +function lowered(): void { + for (const wake of [...waiters]) wake(); +} + +function until(settled: () => boolean, signal?: AbortSignal): Promise { + if (settled()) return Promise.resolve(); + if (signal?.aborted === true) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const wake = (): void => { + if (!settled()) return; + waiters.delete(wake); + signal?.removeEventListener("abort", stop); + resolve(); + }; + const stop = (): void => { + waiters.delete(wake); + reject(signal?.reason); + }; + waiters.add(wake); + signal?.addEventListener("abort", stop, { once: true }); + }); +} + +/** Count work in progress until the returned release is called. */ +export function beginWork(): () => void { + working += 1; + starts += 1; + let released = false; + return () => { + if (released) return; + released = true; + working -= 1; + lowered(); + }; +} + +/** + * Wait without counting the caller as working meanwhile. The caller is + * counted work, and `wait` starts the one thing it waits for: input from + * outside, or work that is counted on its own. Whatever settles that wait + * calls `resume` first, while it still counts itself; `resume` is + * idempotent, and the wait resumes on its own as a fallback. + */ +export async function waitIdle( + wait: (resume: () => void) => Promise, +): Promise { + working -= 1; + let resumed = false; + const resume = (): void => { + if (resumed) return; + resumed = true; + working += 1; + starts += 1; + }; + try { + const waiting = wait(resume); + lowered(); + return await waiting; + } finally { + resume(); + } +} + +/** Waits whose inputs a caller starts before it stops working. */ +export type IdleWait = { + /** A wake hook for one input; pass it to whatever settles that input. */ + waker(): () => void; + /** + * Wait idle on `waiting`, which settles with one of `wakers`. A waker + * that already fired settled its input before this wait began, so the + * caller keeps counting. + */ + wait(waiting: Promise, wakers: readonly (() => void)[]): Promise; +}; + +export function createIdleWait(): IdleWait { + const fired = new Set<() => void>(); + let current: + | { resume: () => void; wakers: readonly (() => void)[] } + | undefined; + return { + waker() { + const wake = (): void => { + fired.add(wake); + if (current !== undefined && current.wakers.includes(wake)) { + current.resume(); + } + }; + return wake; + }, + async wait(waiting, wakers) { + try { + return await waitIdle((resume) => { + current = { resume, wakers }; + if (wakers.some((wake) => fired.has(wake))) resume(); + return waiting; + }); + } finally { + current = undefined; + } + }, + }; +} + +/** Count a delivery of committed input until the returned release is called. */ +export function beginDelivery(): () => void { + delivering += 1; + starts += 1; + let released = false; + return () => { + if (released) return; + released = true; + delivering -= 1; + lowered(); + }; +} + +/** How many times work or a delivery has started in this process. */ +export function activityStarts(): number { + return starts; +} + +/** Whether nothing in this process works or delivers input. */ +export function isQuiet(): boolean { + return working === 0 && delivering === 0; +} + +/** Whether a run loop or a step in this process is working. */ +export function isWorking(): boolean { + return working > 0; +} + +/** Resolves once no committed input is on its way to a run. */ +export function whenDelivered(): Promise { + return until(() => delivering === 0); +} + +/** + * Resolves once nothing in this process works or delivers input, or rejects + * when `signal` aborts first. + */ +export function whenQuiet(signal?: AbortSignal): Promise { + return until(isQuiet, signal); +} diff --git a/packages/workflow/src/runtime/child-workflow-relay.test.ts b/packages/workflow/src/runtime/child-workflow-relay.test.ts index 9943caa04..8dec5cda4 100644 --- a/packages/workflow/src/runtime/child-workflow-relay.test.ts +++ b/packages/workflow/src/runtime/child-workflow-relay.test.ts @@ -375,7 +375,7 @@ describe("childWorkflow signal relay", () => { repoStore, signalChannel: { deliver: () => Promise.reject(new Error("deliver is not used")), - awaitNext: (name, signal) => { + awaitNext: (name, _onWake, signal) => { if (!injected && name === "go") { injected = true; const last = parentTip; @@ -1347,6 +1347,7 @@ describe("park-release watch", () => { const watch = handle.waitForParkRelease?.( "go", new AbortController().signal, + () => undefined, ); if (watch === undefined) { throw new Error("handle does not watch a park release"); @@ -1467,7 +1468,7 @@ async function raceLoggedSignalAgainstRelease( repoStore, signalChannel: { deliver: () => Promise.reject(new Error("deliver is not used")), - awaitNext: (_name, signal) => pendingSignal(signal), + awaitNext: (_name, _onWake, signal) => pendingSignal(signal), }, spawnChild: () => Promise.resolve({ @@ -1985,11 +1986,12 @@ describe("a parked childWorkflow survives a process restart", () => { // open keeps that crash window: the parent log names a child run whose // own log was never written. Letting the child start would share this // process's commit chain with the resume. + const spawner = Promise.withResolvers(); const first = startRelayed({ definition: parent, runId, invokeStep: async () => ({ output: null }), - spawnChild: () => new Promise(() => undefined), + spawnChild: () => spawner.promise, }); void first.run.complete; await waitForEvent( @@ -2020,6 +2022,10 @@ describe("a parked childWorkflow survives a process restart", () => { }); await run.signal("go", { done: true }, "sig-keep"); const result = await run.complete; + // The first run would otherwise keep working for the rest of the + // process, and other test files share it. + spawner.reject(new Error("the crash window is over")); + await first.run.complete.catch(() => undefined); expect(result.terminalStatus).toBe("completed"); expect(seen).toEqual([{ token: "keep" }]); diff --git a/packages/workflow/src/runtime/commit-chain.ts b/packages/workflow/src/runtime/commit-chain.ts index b848de58c..aff818c89 100644 --- a/packages/workflow/src/runtime/commit-chain.ts +++ b/packages/workflow/src/runtime/commit-chain.ts @@ -59,6 +59,9 @@ const commitChains = new Map>(); * share one buffer, serialized through that runId's chain. */ const pendingBuffers = new Map(); +// The env each run last buffered through, so a host that stops the process +// can make every run's buffer durable first. +const bufferEnvs = new Map(); export type CommitEnv = { repoStore: RepoStore; @@ -174,13 +177,23 @@ export function flushFailureCause(runId: string): unknown { * Resolves when this run's buffer flush throws. Already resolved when * the failure was recorded before the caller started waiting. A parked * step never rejects, so the drive loop races this with its step - * promises. + * promises. `onFail` runs as the failure is recorded, before the flush + * that threw rejects to its writer. */ -export function whenRunFlushFails(runId: string): Promise { - if (failedFlushes.has(runId)) return Promise.resolve(); +export function whenRunFlushFails( + runId: string, + onFail: () => void, +): Promise { + if (failedFlushes.has(runId)) { + onFail(); + return Promise.resolve(); + } return new Promise((resolve) => { const list = flushWaiters.get(runId) ?? []; - list.push(resolve); + list.push(() => { + onFail(); + resolve(); + }); flushWaiters.set(runId, list); }); } @@ -333,6 +346,7 @@ export async function commitBufferedFrom( const { state, events } = applyBuilt(fresh, build); if (events.length === 0) return state; getBuffer(runId).push(...events); + bufferEnvs.set(runId, env); wake(bufferWaiters, runId); return state; }); @@ -463,9 +477,22 @@ export async function reloadState( export function dropChain(runId: string): void { commitChains.delete(runId); pendingBuffers.delete(runId); + bufferEnvs.delete(runId); failedFlushes.delete(runId); flushWaiters.delete(runId); bufferWaiters.delete(runId); durableSeqs.delete(runId); durableWaiters.delete(runId); } + +/** Make every run's buffered events durable. */ +export async function flushAllBuffers(): Promise { + await Promise.all( + [...pendingBuffers].flatMap(([runId, buffer]) => { + const env = bufferEnvs.get(runId); + return buffer.length === 0 || env === undefined + ? [] + : [flushChain(env, runId)]; + }), + ); +} diff --git a/packages/workflow/src/runtime/env.ts b/packages/workflow/src/runtime/env.ts index 9c79ff57d..3f2147728 100644 --- a/packages/workflow/src/runtime/env.ts +++ b/packages/workflow/src/runtime/env.ts @@ -146,10 +146,13 @@ export interface SignalChannel { * delivery resolves. Resolves with the payload and the assigned * `signalId`. Rejects on `signal.abort()` if `signal` is supplied. * Must see every signal delivered after the call, since the runtime - * only checks the log for ones delivered before it. + * only checks the log for ones delivered before it. A channel whose + * process answers a quiet check calls `onWake` before it settles the + * wait, while whatever woke it still counts as delivery. */ awaitNext( name: string, + onWake: () => void, signal?: AbortSignal, correlationId?: string, ): Promise<{ payload: unknown; signalId: string; correlationId?: string }>; @@ -441,9 +444,15 @@ export type SuspendableChildPark = { * signal at a time reads `readParkRelease` before it accepts a signal that * is already waiting, and uses `waitForParkRelease` to learn that the open * park ended without consuming a sibling. + * + * Each wait takes an `onWake`, called before the wait settles while + * whatever settles it still counts as work, so a caller that waits idle + * counts again first. */ export type SuspendableChildHandle = { - next(): Promise< + next( + onWake: () => void, + ): Promise< | { kind: "park"; park: SuspendableChildPark } | { kind: "signal-park"; name: string; timeout?: number } | { kind: "input-park"; correlationId: string } @@ -491,6 +500,7 @@ export type SuspendableChildHandle = { waitForParkRelease?( name: string, abort: AbortSignal, + onWake: () => void, ): Promise< | { kind: "released" } | { @@ -507,7 +517,7 @@ export type SuspendableChildHandle = { * Present on the live suspendable handle. A terminal handle has no * channel and omits it. */ - whenSignalAwaiter?(name: string): Promise; + whenSignalAwaiter?(name: string, onWake: () => void): Promise; }; /** diff --git a/packages/workflow/src/runtime/index.ts b/packages/workflow/src/runtime/index.ts index c46c11066..d128cd6fb 100644 --- a/packages/workflow/src/runtime/index.ts +++ b/packages/workflow/src/runtime/index.ts @@ -49,10 +49,19 @@ export { bufferedEvents, commitBuffered, dropChain, + flushAllBuffers, reloadState, whenRunNextBuffers, withRunCommitBarrier, } from "./commit-chain"; +export { + activityStarts, + beginDelivery, + isQuiet, + isWorking, + whenDelivered, + whenQuiet, +} from "./activity"; export { createSuspendableChildHandle, diff --git a/packages/workflow/src/runtime/loop-iteration-handle.test.ts b/packages/workflow/src/runtime/loop-iteration-handle.test.ts index 551d81ad7..a638f679c 100644 --- a/packages/workflow/src/runtime/loop-iteration-handle.test.ts +++ b/packages/workflow/src/runtime/loop-iteration-handle.test.ts @@ -32,6 +32,8 @@ import { createLoopIterationHandle } from "./loop-iteration-handle"; import { runtimeRun } from "./run"; import { createTerminalChildHandle } from "./suspendable-child-handle"; +const noWake = (): void => undefined; + const body = defineWorkflow({ id: "handle-body", trigger: { type: "manual" }, @@ -104,7 +106,7 @@ describe("createLoopIterationHandle", () => { signalChannel: createInMemorySignalChannel(), }); - const event = await handle.next(); + const event = await handle.next(noWake); expect(event).toEqual({ kind: "terminal", terminalStatus: "completed" }); // The body short-circuited on the terminal log; the action did NOT re-run. expect(runs).toBe(1); diff --git a/packages/workflow/src/runtime/parked-work.test.ts b/packages/workflow/src/runtime/parked-work.test.ts new file mode 100644 index 000000000..01905759f --- /dev/null +++ b/packages/workflow/src/runtime/parked-work.test.ts @@ -0,0 +1,151 @@ +// What a host may stop while a step is parked: a parked step waits idle, but +// a map around it or a timeout on it lives only in the process, so the +// process stays counted as working. + +import { afterEach, expect, test } from "bun:test"; + +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; +import { signalName } from "@intx/types"; +import type { ApprovalSnapshot, ConversationTurn } from "@intx/types/runtime"; + +import { + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, + createNoopDrainController, + defineWorkflow, + map, + runtimeRun, + step, + type StepInvoker, + type WorkflowDefinition, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { waitForEvent } from "@intx/workflow/testing"; +import { isQuiet, isWorking, whenQuiet } from "./activity"; +import { createTerminalChildHandle } from "./suspendable-child-handle"; + +const agent = defineAgent({ + id: "a", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, +}); + +const snapshot: ApprovalSnapshot = { + name: "charge_card", + description: "Charge the customer's card", + inputSchema: { type: "object" }, + arguments: { amount: 100 }, +}; + +const reply: ConversationTurn = { + role: "assistant", + content: [{ type: "text", text: "done" }], + timestamp: 0, +}; + +// The first call parks for approval; the call after the decision replies. +const parkOnce: StepInvoker = async (req) => + req.resume === undefined + ? { + suspend: { + correlationId: "corr-park", + kind: "approval", + approvalSnapshot: snapshot, + }, + } + : { output: { reply: "done", turn: reply } }; + +const finishing: (() => Promise)[] = []; + +afterEach(async () => { + for (const finish of finishing.splice(0)) await finish(); +}); + +// Runs `definition` until its step parks. +async function park( + definition: WorkflowDefinition, + parkedStepId: string, +): Promise { + if (!isQuiet()) + throw new Error("an earlier test left work or a delivery running"); + const repoStore = createInMemoryRepoStore(); + const channel = createInMemorySignalChannel(); + const env: WorkflowRuntimeEnv = { + repoStore, + scheduler: createInMemoryScheduler({ repoStore, clock: () => new Date() }), + signalChannel: channel, + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: parkOnce, + spawnChild: async () => createTerminalChildHandle("completed"), + clock: () => new Date(), + newId: (prefix) => `${prefix}-id`, + drain: createNoopDrainController(definition), + hasUpstreamSignalResolver: true, + }; + const run = runtimeRun(definition, env, { runId: "run-parked" }); + finishing.push(async () => { + await channel.deliver( + signalName("corr-park"), + { outcome: "approved" }, + "sig-park", + ); + await run.complete; + }); + await waitForEvent( + repoStore, + "run-parked", + (event) => event.kind === "SignalAwaited" && event.stepId === parkedStepId, + ); +} + +// Whether the process still counts as working once it settled into the park. +// Everything in memory after the park is microtasks, so once they drain the +// step waits on its signal, and only a count held across that wait is left. +async function workingWhileParked( + definition: WorkflowDefinition, + parkedStepId: string, +): Promise { + await park(definition, parkedStepId); + await new Promise((resolve) => setImmediate(resolve)); + return isWorking(); +} + +test("a parked step waits idle", async () => { + const definition = defineWorkflow({ + id: "parked-step", + trigger: { type: "manual" }, + steps: { s: step({ agent }) }, + }); + await park(definition, "s"); + await expect(whenQuiet()).resolves.toBeUndefined(); +}); + +test("a map whose item is parked counts as working, since resume cannot continue it", async () => { + const definition = defineWorkflow({ + id: "parked-map", + trigger: { type: "manual" }, + steps: { + items: map({ over: { literal: ["a"] }, step: step({ agent }) }), + }, + }); + expect(await workingWhileParked(definition, "items[0]")).toBe(true); +}); + +test("a parked step with a timeout counts as working, since its deadline lives in the process", async () => { + const definition = defineWorkflow({ + id: "parked-timed-step", + trigger: { type: "manual" }, + steps: { s: step({ agent, timeout: 300_000 }) }, + }); + expect(await workingWhileParked(definition, "s")).toBe(true); +}); diff --git a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts index 361f416f1..17e22f71b 100644 --- a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts +++ b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts @@ -777,7 +777,7 @@ describe("signal committed before the wait starts", () => { const channel: SignalChannel = { deliver: (name, payload, signalId) => inner.deliver(name, payload, signalId), - awaitNext: (name, signal) => { + awaitNext: (name, onWake, signal) => { waits += 1; landed ??= base.repoStore.append(runId, { kind: "SignalReceived", @@ -787,7 +787,7 @@ describe("signal committed before the wait starts", () => { signalId: "sig-gap", payload: { gap: true }, }); - return inner.awaitNext(name, signal); + return inner.awaitNext(name, onWake, signal); }, }; const env = { ...base, signalChannel: channel }; diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index 33aba99c1..8a8e85653 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -45,6 +45,7 @@ import { isRunDone, nextSchedulable, resumeResidualOf, + type StepKindLookup, } from "./dag"; import { commit as commitDurableToChain, @@ -72,6 +73,7 @@ import type { } from "./env"; import { shouldAbortForDrain } from "./drain"; import { bridgeAbort } from "./abort-bridge"; +import { beginWork, createIdleWait, type IdleWait, waitIdle } from "./activity"; import { assertSpawnDepthWithinLimit, resolveMaxChildSpawnDepth, @@ -161,6 +163,11 @@ export interface RuntimeRunOptions { * sign the control-plane `CancelRequested` a cascade cancel would require. */ localAbort?: AbortSignal; + /** + * Called as the run settles, before it stops counting as work. A caller + * that waits idle on this run resumes here. + */ + onSettle?: () => void; } /** @@ -317,6 +324,7 @@ async function executeRun( options: RuntimeRunOptions, onStarted: (committed: boolean) => void, ): Promise { + const endWork = beginWork(); try { return await executeRunBody( definition, @@ -338,6 +346,8 @@ async function executeRun( // not hold dead promise chains for runs that crashed during // resume seeding or a stall guard. dropChain(runId); + options.onSettle?.(); + endWork(); } } @@ -1094,9 +1104,14 @@ async function executeRunBody( // error and rethrow it; the loop stops every in-flight step before // the error leaves it. let repoWriteFailure: unknown; + // Whatever ends the loop's wait calls this first, while it still counts + // itself. Outside a wait it is a spent resume and does nothing. + let resumeLoop = (): void => undefined; // One waiter for the whole loop. A flush that already failed resolves // it immediately, so the next race observes it without a new registration. - const flushFailed = whenRunFlushFails(runId); + const flushFailed = whenRunFlushFails(runId, () => { + resumeLoop(); + }); function throwRememberedWriteFailure(): void { if (repoWriteFailure === undefined) { repoWriteFailure = flushFailureCause(runId); @@ -1176,6 +1191,7 @@ async function executeRunBody( env.drain, ); stepAborts.set(primitive.id, stepLocalAbort); + const endStepWork = beginWork(); const promise = runPrimitiveSafe( definition, env, @@ -1202,6 +1218,8 @@ async function executeRunBody( inFlight.delete(primitive.id); justSettled.add(primitive.id); stepAborts.delete(primitive.id); + resumeLoop(); + endStepWork(); }); stepPromises.set(primitive.id, promise); } @@ -1252,13 +1270,21 @@ async function executeRunBody( // Wait for at least one in-flight primitive to settle. Step // failures are already `StepFailed`. A parked step never settles, // so the flush waiter is what wakes the loop when an external - // writer drops the buffer. - await Promise.race([ - ...Array.from(stepPromises.values()).map((p) => - p.catch(() => undefined), - ), - flushFailed, - ]); + // writer drops the buffer. The loop only waits here: each step counts + // as work on its own. A step that settled since the last wait, or a + // flush that failed, ends this wait at once. + await waitIdle((resume) => { + resumeLoop = resume; + if (justSettled.size > 0 || flushFailureCause(runId) !== undefined) { + resume(); + } + return Promise.race([ + ...Array.from(stepPromises.values()).map((p) => + p.catch(() => undefined), + ), + flushFailed, + ]); + }); throwRememberedWriteFailure(); state = await reloadState(env, runId); throwRememberedWriteFailure(); @@ -1546,7 +1572,7 @@ async function runPrimitive( case "awaitSignal": return runAwaitSignal(definition, env, runId, primitive, abort); case "sleep": - return runSleep(env, runId, primitive, abort); + return runSleep(definition, env, runId, primitive, abort); case "childWorkflow": return runChildWorkflow( definition, @@ -1890,7 +1916,11 @@ async function runStep( }; bridgeAbort(abort, onOuter); let timer: ReturnType | undefined; + // The deadline lives only in this process, so a timed step counts as + // working while it runs, even parked: stopping the process drops it. + let endTimedWork: (() => void) | undefined; if (step.timeout !== undefined) { + endTimedWork = beginWork(); timer = setTimeout(() => { stepAbort.abort(); }, step.timeout); @@ -2204,6 +2234,7 @@ async function runStep( // into the next attempt. The step itself stays awaiting-timer // through the wait. await waitForTimer( + definition, env, runId, timerId, @@ -2228,6 +2259,7 @@ async function runStep( } finally { abort.removeEventListener("abort", onOuter); if (timer !== undefined) clearTimeout(timer); + endTimedWork?.(); } } } @@ -2810,7 +2842,8 @@ async function driveSuspendableOccurrence( } for (;;) { - const bodyEvent = pending ?? (await child.next()); + const bodyEvent = + pending ?? (await waitIdle((resume) => child.next(resume))); pending = undefined; if (bodyEvent.kind === "terminal") { terminalStatus = bodyEvent.terminalStatus; @@ -3203,7 +3236,7 @@ function finishRelayPull( pull: ChildRelayPull, ): Promise { if (pull === "hold") return Promise.resolve({ kind: "held" }); - return child.next(); + return waitIdle((resume) => child.next(resume)); } async function driveContainerSignalRelay( @@ -3512,10 +3545,13 @@ async function raceChildSignalRelease( const detachRaceAbort = bridgeAbort(abort, () => { raceAbort.abort(); }); + const idle = createIdleWait(); + const onSignal = idle.waker(); + const onRelease = idle.waker(); const signalWait = env.signalChannel - .awaitNext(name, raceAbort.signal) + .awaitNext(name, onSignal, raceAbort.signal) .then((r) => ({ tag: "signal" as const, r })); - const releaseWait = watch(name, raceAbort.signal).then((rel) => ({ + const releaseWait = watch(name, raceAbort.signal, onRelease).then((rel) => ({ tag: "release" as const, rel, })); @@ -3562,7 +3598,10 @@ async function raceChildSignalRelease( | { tag: "signal"; r: { payload: unknown; signalId: string } } | { tag: "release"; rel: ChildParkRelease }; try { - outcome = await Promise.race([signalWait, releaseWait]); + outcome = await idle.wait(Promise.race([signalWait, releaseWait]), [ + onSignal, + onRelease, + ]); } catch (cause) { raceAbort.abort(); await signalWait.catch(() => undefined); @@ -3669,14 +3708,20 @@ async function raceContainerSignalRelay( raceAbort.abort(); }; bridgeAbort(abort, onOuterAbort); + const idle = createIdleWait(); + const onNext = idle.waker(); + const race = { idle, waker: onNext }; const waited = correlationId === undefined - ? awaitSignalFor(env, runId, containerStepId, name, raceAbort.signal) + ? awaitSignalFor(env, runId, containerStepId, name, raceAbort.signal, { + race, + }) : awaitSignalFor(env, runId, containerStepId, name, raceAbort.signal, { correlationId, + race, }); const pSignal = waited.then((r) => ({ tag: "signal" as const, r })); - const pNext = child.next().then((ev) => ({ tag: "next" as const, ev })); + const pNext = child.next(onNext).then((ev) => ({ tag: "next" as const, ev })); let outcome: | { tag: "signal"; @@ -3715,7 +3760,7 @@ async function raceContainerSignalRelay( await commit(env, runId, received); await flush(env, runId); await child.deliverSignal(name, outcome.r.payload, outcome.r.signalId); - return (await pNext).ev; + return (await idle.wait(pNext, [onNext])).ev; } // The body produced its next event before the signal arrived: its gate timed @@ -4981,8 +5026,13 @@ async function skipUnroutedHandlerBranches( * after the caller's last-observed event rather than from `"head"`, * which would miss a TimerFired that landed during the * `await env.repoStore.subscribe(...)` setup. + * + * The wait is idle only where the resume guard continues the step from + * the log. A step in retry backoff is not continued, so its wait counts as + * work and a host waits it out before it stops the process. */ async function waitForTimer( + definition: StepKindLookup, env: WorkflowRuntimeEnv, runId: string, timerId: string, @@ -5002,7 +5052,10 @@ async function waitForTimer( const settled = await reloadState(env, runId); const subscribeFromSeq = settled.lastSeq + 1; const ac = new AbortController(); + const idle = createIdleWait(); + const wake = idle.waker(); const onOuterAbort = (): void => { + wake(); ac.abort(); }; if (abort.aborted) { @@ -5025,20 +5078,33 @@ async function waitForTimer( // controller so the `for await` ends cleanly. const onDrain = (): void => { if (shouldAbortForDrain(drain, stepId)) { + wake(); ac.abort(); } }; drain.signal.addEventListener("abort", onDrain, { once: true }); const dispose = env.scheduler.scheduleIn(runId, timerId, fireAt); try { - for await (const { event } of env.repoStore.subscribe(runId, { - signal: ac.signal, - from: { seq: subscribeFromSeq }, - })) { - if (event.kind === "TimerFired" && event.timerId === timerId) { - return; + const waiting = (async () => { + for await (const { event } of env.repoStore.subscribe(runId, { + signal: ac.signal, + from: { seq: subscribeFromSeq }, + })) { + if (event.kind === "TimerFired" && event.timerId === timerId) { + // The return closes the subscription, which ends this entry's + // delivery. + wake(); + return true; + } } - } + wake(); + return false; + })(); + const fired = + resumeResidualOf(definition, stepId, "awaiting-timer") === "continue" + ? await idle.wait(waiting, [wake]) + : await waiting; + if (fired) return; if (abort.aborted) throw new Error("aborted"); if (shouldAbortForDrain(drain, stepId)) { throw new Error("aborted: drain requested"); @@ -5079,47 +5145,54 @@ async function runMap( if (!Array.isArray(over)) { throw new Error(`map.over for ${primitive.id} did not resolve to an array`); } - await emitStepStartedWithValue(env, runId, primitive.id, over); - // v1 runs the inner steps sequentially. A parallel fan-out would - // need per-item commit serialization against the same run log - // beyond what the existing commit chain offers, plus a parallelism - // bound on the env. The spec does not commit to either semantic; - // sequential keeps the event log readable and the runtime simple. - const inner = primitive.step; - const outputs: unknown[] = []; - for (let i = 0; i < over.length; i += 1) { - const item = over[i]; - const itemCtx: SelectorContext = { - ...selectorCtx, - trigger: { payload: item }, - }; - const scopedStep: StepPrimitive = { - ...inner, - id: scopedStepId(primitive.id, i), - // The outer map's retry policy applies as the fan-out-level - // default when the inner step does not declare its own. The - // inner step's policy already rides in via `...inner`; the - // spread below only fills in from the map when the inner is - // missing one. - ...(inner.retry === undefined && primitive.retry !== undefined - ? { retry: primitive.retry } - : {}), - }; - // Resume cannot continue a map still in flight, so an item's completion - // stays buffered: writing it durably would buy nothing. - const output = await runStep( - definition, - env, - runId, - scopedStep, - itemCtx, - abort, - "buffered", - ); - outputs.push(output); + // Resume cannot continue a map in flight, so the map counts as working + // while an item waits: stopping the process would fail its run. + const endMapWork = beginWork(); + try { + await emitStepStartedWithValue(env, runId, primitive.id, over); + // v1 runs the inner steps sequentially. A parallel fan-out would + // need per-item commit serialization against the same run log + // beyond what the existing commit chain offers, plus a parallelism + // bound on the env. The spec does not commit to either semantic; + // sequential keeps the event log readable and the runtime simple. + const inner = primitive.step; + const outputs: unknown[] = []; + for (let i = 0; i < over.length; i += 1) { + const item = over[i]; + const itemCtx: SelectorContext = { + ...selectorCtx, + trigger: { payload: item }, + }; + const scopedStep: StepPrimitive = { + ...inner, + id: scopedStepId(primitive.id, i), + // The outer map's retry policy applies as the fan-out-level + // default when the inner step does not declare its own. The + // inner step's policy already rides in via `...inner`; the + // spread below only fills in from the map when the inner is + // missing one. + ...(inner.retry === undefined && primitive.retry !== undefined + ? { retry: primitive.retry } + : {}), + }; + // Resume cannot continue a map still in flight, so an item's completion + // stays buffered: writing it durably would buy nothing. + const output = await runStep( + definition, + env, + runId, + scopedStep, + itemCtx, + abort, + "buffered", + ); + outputs.push(output); + } + await emitStepCompletedWithValue(env, runId, primitive.id, outputs); + return outputs; + } finally { + endMapWork(); } - await emitStepCompletedWithValue(env, runId, primitive.id, outputs); - return outputs; } async function runGate( @@ -5838,7 +5911,8 @@ function signalBoundToAwait( * caller read the step as awaiting before that; a signal delivered in between * is already in the log, so it is taken from there. `onWaiting` runs only * when the log has no such signal, after the subscriber exists and before - * the wait blocks. + * the wait blocks. `race` is an input the caller races with this signal; + * its waker ends this wait too. */ async function awaitSignalFor( env: WorkflowRuntimeEnv, @@ -5849,17 +5923,27 @@ async function awaitSignalFor( opts?: { correlationId?: string; onWaiting?: () => void; + race?: { idle: IdleWait; waker: () => void }; }, ): Promise<{ payload: unknown; signalId: string; correlationId?: string }> { const correlationId = opts?.correlationId; + const idle = opts?.race?.idle ?? createIdleWait(); + const onSignal = idle.waker(); + const wakers = + opts?.race === undefined ? [onSignal] : [onSignal, opts.race.waker]; const wait = new AbortController(); const detach = bridgeAbort(abort, () => { wait.abort(); }); const waiting = correlationId === undefined - ? env.signalChannel.awaitNext(signalName, wait.signal) - : env.signalChannel.awaitNext(signalName, wait.signal, correlationId); + ? env.signalChannel.awaitNext(signalName, onSignal, wait.signal) + : env.signalChannel.awaitNext( + signalName, + onSignal, + wait.signal, + correlationId, + ); const waitSettled = waiting.then( () => undefined, () => undefined, @@ -5871,7 +5955,7 @@ async function awaitSignalFor( return correlationId === undefined ? bound : { ...bound, correlationId }; } opts?.onWaiting?.(); - return await waiting; + return await idle.wait(waiting, wakers); } finally { detach(); wait.abort(); @@ -5969,6 +6053,8 @@ async function parkOnSignalResult( claimsRunInputChannel?: { sectionInputStepIds: ReadonlySet; }; + /** An input the caller races with this park; its waker ends the wait too. */ + race?: { idle: IdleWait; waker: () => void }; }, state: ReturnType, abort: AbortSignal, @@ -6244,13 +6330,16 @@ async function parkOnSignalResult( opts.stepId, opts.signalName, combinedAbort.signal, - repark - ? { - onWaiting: () => { - env.onSignalAwaiter?.({ runId, name: opts.signalName }); - }, - } - : undefined, + { + ...(repark + ? { + onWaiting: () => { + env.onSignalAwaiter?.({ runId, name: opts.signalName }); + }, + } + : {}), + ...(opts.race !== undefined ? { race: opts.race } : {}), + }, ); let next = await reloadState(env, runId); const signalReceived: WorkflowEvent = { @@ -6475,6 +6564,7 @@ async function completeAwaitSignalOutcome( } async function runSleep( + definition: WorkflowDefinition, env: WorkflowRuntimeEnv, runId: string, primitive: SleepPrimitive, @@ -6550,6 +6640,7 @@ async function runSleep( } void state; await waitForTimer( + definition, env, runId, timerId, @@ -6593,7 +6684,7 @@ async function closeFailedChild( if (handle !== undefined) { try { for (;;) { - const event = await handle.next(); + const event = await waitIdle((resume) => handle.next(resume)); if (event.kind === "terminal") { drained = event.terminalStatus; break; @@ -6691,6 +6782,8 @@ async function awaitChildPark( watchAbort.abort(); }); const parkRearm = await reloadState(env, parentRunId); + const idle = createIdleWait(); + const onRelease = idle.waker(); const decision = parkOnSignal( env, parentRunId, @@ -6702,11 +6795,12 @@ async function awaitChildPark( ...(parkKind === "input" ? { claimsRunInputChannel: { sectionInputStepIds: sectionSteps } } : {}), + race: { idle, waker: onRelease }, }, parkRearm, parkAbort.signal, ); - const release = watch(name, watchAbort.signal); + const release = watch(name, watchAbort.signal, onRelease); let outcome: | { tag: "decision"; payload: unknown } | { tag: "release"; rel: ChildParkRelease }; @@ -7409,8 +7503,9 @@ async function drivePlannedChildRelay( return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; } if (snapshot.kind === "released") return { kind: "held" }; - if (handle.whenSignalAwaiter !== undefined) { - await handle.whenSignalAwaiter(relay.name); + const whenAwaiter = handle.whenSignalAwaiter; + if (whenAwaiter !== undefined) { + await waitIdle((resume) => whenAwaiter(relay.name, resume)); } await handle.deliverSignal(relay.name, relay.payload, relay.signalId); return { kind: "held" }; @@ -7457,8 +7552,9 @@ async function drivePlannedChildRelay( return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; } if (snapshot.kind === "released") return { kind: "held" }; - if (handle.whenSignalAwaiter !== undefined) { - await handle.whenSignalAwaiter(name); + const whenAwaiter = handle.whenSignalAwaiter; + if (whenAwaiter !== undefined) { + await waitIdle((resume) => whenAwaiter(name, resume)); } await handle.resume(relay.correlationId, relay.decision); return { kind: "held" }; @@ -7660,7 +7756,9 @@ async function runChildWorkflow( if (terminalStatus === undefined) { let pending: ChildDriveResult | undefined; for (;;) { - const bodyEvent = pending ?? (await spawnedHandle.next()); + const bodyEvent = + pending ?? + (await waitIdle((resume) => spawnedHandle.next(resume))); pending = undefined; if (bodyEvent.kind === "terminal") { terminalStatus = bodyEvent.terminalStatus; diff --git a/packages/workflow/src/runtime/suspendable-child-handle.test.ts b/packages/workflow/src/runtime/suspendable-child-handle.test.ts index 6ac3c00a1..fda8bf96d 100644 --- a/packages/workflow/src/runtime/suspendable-child-handle.test.ts +++ b/packages/workflow/src/runtime/suspendable-child-handle.test.ts @@ -1,12 +1,59 @@ import { describe, expect, test } from "bun:test"; -import { createTerminalChildHandle } from "./suspendable-child-handle"; +import { createDefaultDirectorRegistry } from "@intx/agent"; + +import { action, awaitSignal, defineWorkflow } from "../definition/index"; +import { + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, +} from "../runlocal/index"; +import { beginWork, isQuiet, isWorking, waitIdle, whenQuiet } from "./activity"; +import { createNoopDrainController } from "./drain"; +import type { SuspendableChildHandle, WorkflowRuntimeEnv } from "./env"; +import { + createSuspendableChildHandle, + createTerminalChildHandle, +} from "./suspendable-child-handle"; + +const noWake = (): void => undefined; + +const body = defineWorkflow({ + id: "settle-body", + trigger: { type: "manual" }, + steps: { touch: action({ handler: "touch" }) }, +}); + +function bodyEnv(): WorkflowRuntimeEnv { + const repoStore = createInMemoryRepoStore(); + const clock = (): Date => new Date(0); + return { + repoStore, + scheduler: createInMemoryScheduler({ repoStore, clock }), + signalChannel: createInMemorySignalChannel(), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: async () => ({ output: null }), + invokeAction: async () => ({ output: "done" }), + spawnChild: async () => createTerminalChildHandle("completed"), + clock, + newId: (prefix) => `${prefix}-0`, + drain: createNoopDrainController(body), + hasUpstreamSignalResolver: true, + }; +} describe("createTerminalChildHandle", () => { test("next returns the same terminal on every call", async () => { const handle = createTerminalChildHandle("failed"); - const first = await handle.next(); - const second = await handle.next(); + const first = await handle.next(noWake); + const second = await handle.next(noWake); expect(first).toEqual({ kind: "terminal", terminalStatus: "failed" }); expect(second).toBe(first); }); @@ -21,3 +68,132 @@ describe("createTerminalChildHandle", () => { ); }); }); + +describe("createSuspendableChildHandle", () => { + const waits: [ + string, + (handle: SuspendableChildHandle, onWake: () => void) => Promise, + ][] = [ + ["next", (handle, onWake) => handle.next(onWake)], + [ + "re-arm wait", + (handle, onWake) => + handle.whenSignalAwaiter?.("go", onWake) ?? Promise.resolve(), + ], + ]; + for (const [name, wait] of waits) { + // The body has stopped counting as work, and its handle has not seen + // the result yet. + test(`a ${name} that starts as the body ends counts as working`, async () => { + if (!isQuiet()) + throw new Error("an earlier test left work or a delivery running"); + const handle = createSuspendableChildHandle(bodyEnv(), { + definition: body, + childRunId: "settle-child", + input: null, + depth: 0, + maxChildSpawnDepth: 32, + signal: new AbortController().signal, + surfaceInputPark: false, + }); + const working = await whenQuiet().then(() => { + const endParent = beginWork(); + void waitIdle((resume) => wait(handle, resume)) + .catch(() => undefined) + .finally(endParent); + return isWorking(); + }); + expect(working).toBe(true); + }); + } + + const gated = defineWorkflow({ + id: "gated-body", + trigger: { type: "manual" }, + steps: { gate: awaitSignal({ name: "go" }) }, + }); + + // A body parked on `go`, which ends once it gets that signal. + async function parkedBody(env = bodyEnv()) { + if (!isQuiet()) + throw new Error("an earlier test left work or a delivery running"); + const stop = new AbortController(); + const handle = createSuspendableChildHandle(env, { + definition: gated, + childRunId: "gated-child", + input: null, + depth: 0, + maxChildSpawnDepth: 32, + signal: stop.signal, + surfaceInputPark: false, + }); + await handle.next(noWake); + await whenQuiet(); + return { handle, stop }; + } + + const waitsUnderWay: [ + string, + (handle: SuspendableChildHandle, onWake: () => void) => Promise, + ][] = [ + [ + "re-arm wait", + (handle, onWake) => + handle.whenSignalAwaiter?.("later", onWake) ?? Promise.resolve(), + ], + ]; + for (const [name, wait] of waitsUnderWay) { + test(`a ${name} under way counts as working before the body that ended stops counting`, async () => { + const { handle, stop } = await parkedBody(); + let parentDone = false; + const endParent = beginWork(); + void waitIdle((resume) => wait(handle, resume)) + .catch(() => undefined) + .finally(() => { + parentDone = true; + endParent(); + }); + + await handle.deliverSignal("go", null, "sig-1"); + await whenQuiet(); + expect(parentDone).toBe(true); + stop.abort(); + }); + } + + test("a park watch keeps the process from looking quiet while it reads the body's log", async () => { + const env = bodyEnv(); + const inner = env.repoStore; + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + let holding = false; + env.repoStore = { + ...inner, + read: async (runId) => { + if (holding) { + holding = false; + reached.resolve(undefined); + await release.promise; + } + return inner.read(runId); + }, + }; + const { handle, stop } = await parkedBody(env); + const endParent = beginWork(); + holding = true; + const watching = waitIdle( + (resume) => + handle.waitForParkRelease?.("go", stop.signal, resume) ?? + Promise.resolve(), + ) + .catch(() => undefined) + .finally(endParent); + await reached.promise; + expect(isQuiet()).toBe(false); + + stop.abort(); + release.resolve(undefined); + await watching; + await whenQuiet(); + }); +}); diff --git a/packages/workflow/src/runtime/suspendable-child-handle.ts b/packages/workflow/src/runtime/suspendable-child-handle.ts index 87b423bfa..5143239bd 100644 --- a/packages/workflow/src/runtime/suspendable-child-handle.ts +++ b/packages/workflow/src/runtime/suspendable-child-handle.ts @@ -24,6 +24,7 @@ import type { } from "./env"; import { resumeFromLog, type WorkflowEvent } from "../state-machine/index"; import { bridgeAbort } from "./abort-bridge"; +import { beginDelivery } from "./activity"; import { runtimeRun } from "./run"; /** @@ -44,7 +45,10 @@ export function createTerminalChildHandle( new Error(`child handle has already settled ${terminalStatus}`), ); return { - next: () => Promise.resolve(terminal), + next: (onWake) => { + onWake(); + return Promise.resolve(terminal); + }, resume: () => rejectSettled(), deliverSignal: () => rejectSettled(), }; @@ -118,13 +122,13 @@ export function createSuspendableChildHandle( | { kind: "input-park"; correlationId: string } | { kind: "error"; error: Error }; const events: BodyEvent[] = []; - let wake: (() => void) | null = null; + let nextWaiter: { resolve: () => void; resume: () => void } | null = null; const notify = (): void => { - if (wake !== null) { - const resolve = wake; - wake = null; - resolve(); - } + if (nextWaiter === null) return; + const waiter = nextWaiter; + nextWaiter = null; + waiter.resume(); + waiter.resolve(); }; // Names whose re-park has entered `awaitNext`. A host channel does not @@ -134,21 +138,35 @@ export function createSuspendableChildHandle( const awaiterReady = new Set(); const awaiterWaiters = new Map< string, - { resolve: () => void; reject: (error: Error) => void }[] + { + resolve: () => void; + reject: (error: Error) => void; + resume: () => void; + }[] >(); const resolveAwaiter = (name: string): void => { awaiterReady.add(name); const waiters = awaiterWaiters.get(name); if (waiters === undefined) return; awaiterWaiters.delete(name); - for (const waiter of waiters) waiter.resolve(); + for (const waiter of waiters) { + waiter.resume(); + waiter.resolve(); + } }; const rejectAwaiterWaiters = (error: Error): void => { for (const waiters of awaiterWaiters.values()) { - for (const waiter of waiters) waiter.reject(error); + for (const waiter of waiters) { + waiter.resume(); + waiter.reject(error); + } } awaiterWaiters.clear(); }; + let watchResume: (() => void) | null = null; + // The body has stopped counting as work; `settled` follows a few + // microtasks later. A wait that starts in between counts at once. + let bodyEnded = false; const runEnv: WorkflowRuntimeEnv = { ...env, @@ -224,13 +242,23 @@ export function createSuspendableChildHandle( const handle = runtimeRun(definition, runEnv, { ...baseOptions, localAbort: localTeardown.signal, + // The body's settle reaches its waiters only after the body has stopped + // counting as work, so they count again first. + onSettle: () => { + bodyEnded = true; + nextWaiter?.resume(); + watchResume?.(); + for (const waiters of awaiterWaiters.values()) { + for (const waiter of waiters) waiter.resume(); + } + }, }); let settled: { terminalStatus: "completed" | "failed" | "cancelled"; } | null = null; let failure: Error | null = null; - // Separate from `wake`. `next` has one consumer; a park-release watch is + // Separate from `nextWaiter`. `next` has one consumer; a park-release watch is // the other, and it must not take the event-queue slot. const releaseWaiters = new Set<() => void>(); const notifyReleaseWaiters = (): void => { @@ -323,10 +351,12 @@ export function createSuspendableChildHandle( // attaches its tail on the first pull, not when `subscribe` returns, so // the pull happens before the second read. That read sees a release // that landed while the previous snapshot was taken. A release that - // lands during that read wakes the tail. + // lands during that read wakes the tail. The caller waits idle on this + // watch, so its reads count as delivery: one can find the release. const waitForParkRelease = async ( name: string, abort: AbortSignal, + onWake: () => void, ): Promise< | { kind: "released" } | { @@ -335,12 +365,15 @@ export function createSuspendableChildHandle( } > => { if (openParkWatch !== null) { + onWake(); throw new Error( `child ${childRunId} already watches ${openParkWatch} for release; ` + `a drive relays one park at a time`, ); } openParkWatch = name; + watchResume = onWake; + let endRead: (() => void) | undefined = beginDelivery(); try { if (abort.aborted) throw new Error("aborted"); for (;;) { @@ -350,6 +383,7 @@ export function createSuspendableChildHandle( const fromSeq = read.lastSeq + 1; const poke = new AbortController(); const detachPoke = bridgeAbort(abort, () => { + onWake(); poke.abort(); }); const onSettle = (): void => { @@ -384,7 +418,11 @@ export function createSuspendableChildHandle( // does not replay `from` would wait forever on the old cursor. if (again.lastSeq !== read.lastSeq) continue; let sawEvent = false; + // The store counts what reaches the pull from here on. + endRead?.(); + endRead = undefined; const first = await pending; + endRead = beginDelivery(); pending = undefined; if (first.done !== true) sawEvent = true; if (abort.aborted) throw new Error("aborted"); @@ -412,46 +450,56 @@ export function createSuspendableChildHandle( } } } finally { + onWake(); + endRead?.(); + watchResume = null; openParkWatch = null; } }; return { - next: async () => { - for (;;) { - const event = events.shift(); - if (event !== undefined) { - if (event.kind === "error") { - // The body re-armed an input park nothing will resolve. Tear the - // child down locally so its terminal (and the channel teardown tied - // to it) fires, then surface the error: the throw lands the - // container run's terminal via `runPrimitiveSafe`. Local teardown, - // not `handle.cancel`, because an in-process body cannot sign a - // control-plane cancel (see the `localTeardown` note above). - localTeardown.abort(); - throw event.error; + next: async (onWake) => { + try { + for (;;) { + const event = events.shift(); + if (event !== undefined) { + if (event.kind === "error") { + // The body re-armed an input park nothing will resolve. Tear the + // child down locally so its terminal (and the channel teardown + // tied to it) fires, then surface the error: the throw lands the + // container run's terminal via `runPrimitiveSafe`. Local + // teardown, not `handle.cancel`, because an in-process body + // cannot sign a control-plane cancel (see the `localTeardown` + // note above). + localTeardown.abort(); + throw event.error; + } + if (event.kind === "signal-park") { + return { + kind: "signal-park", + name: event.name, + ...(event.timeout !== undefined + ? { timeout: event.timeout } + : {}), + }; + } + if (event.kind === "input-park") { + return { kind: "input-park", correlationId: event.correlationId }; + } + return { kind: "park", park: event.park }; } - if (event.kind === "signal-park") { - return { - kind: "signal-park", - name: event.name, - ...(event.timeout !== undefined - ? { timeout: event.timeout } - : {}), - }; + if (failure !== null) throw failure; + if (settled !== null) { + return { kind: "terminal", terminalStatus: settled.terminalStatus }; } - if (event.kind === "input-park") { - return { kind: "input-park", correlationId: event.correlationId }; - } - return { kind: "park", park: event.park }; - } - if (failure !== null) throw failure; - if (settled !== null) { - return { kind: "terminal", terminalStatus: settled.terminalStatus }; + if (bodyEnded) onWake(); + await new Promise((resolve) => { + nextWaiter = { resolve, resume: onWake }; + }); } - await new Promise((resolve) => { - wake = resolve; - }); + } finally { + // Covers an event that was already queued, where nothing woke this. + onWake(); } }, resume: async (correlationId, decision) => { @@ -462,7 +510,15 @@ export function createSuspendableChildHandle( }, readParkRelease, waitForParkRelease, - whenSignalAwaiter: (name) => { + whenSignalAwaiter: (name, onWake) => { + if ( + awaiterReady.has(name) || + failure !== null || + settled !== null || + bodyEnded + ) { + onWake(); + } if (awaiterReady.has(name)) return Promise.resolve(); if (failure !== null) return Promise.reject(failure); if (settled !== null) { @@ -471,12 +527,13 @@ export function createSuspendableChildHandle( ); } return new Promise((resolve, reject) => { + const waiter = { resolve, reject, resume: onWake }; const waiters = awaiterWaiters.get(name); if (waiters !== undefined) { - waiters.push({ resolve, reject }); + waiters.push(waiter); return; } - awaiterWaiters.set(name, [{ resolve, reject }]); + awaiterWaiters.set(name, [waiter]); }); }, }; diff --git a/packages/workflow/src/testing/signal-channel-observer.test.ts b/packages/workflow/src/testing/signal-channel-observer.test.ts index 0749e28df..cd45e006c 100644 --- a/packages/workflow/src/testing/signal-channel-observer.test.ts +++ b/packages/workflow/src/testing/signal-channel-observer.test.ts @@ -2,6 +2,8 @@ import { describe, test, expect } from "bun:test"; import { createObservedSignalChannel } from "./signal-channel-observer"; +const noWake = (): void => undefined; + const NAME = "__signal__:corr-1"; describe("createObservedSignalChannel counting", () => { @@ -15,9 +17,9 @@ describe("createObservedSignalChannel counting", () => { // Park on two names, one of them twice. Nothing delivers, so the awaits // stay pending; the counts are what this asserts, so the promises are // deliberately left unsettled and are resolved by the deliveries below. - void channel.awaitNext(NAME); - void channel.awaitNext(NAME); - void channel.awaitNext("other"); + void channel.awaitNext(NAME, noWake); + void channel.awaitNext(NAME, noWake); + void channel.awaitNext("other", noWake); expect(channel.awaitedCount(NAME)).toBe(2); expect(channel.awaitedCount("other")).toBe(1); @@ -38,7 +40,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { // the park-first interleaving, which is the common one. test("is satisfied by a park that happened BEFORE the call", async () => { const channel = createObservedSignalChannel(); - const parked = channel.awaitNext(NAME); + const parked = channel.awaitNext(NAME, noWake); expect(channel.awaitedCount(NAME)).toBe(1); await channel.awaitAwaitedCount(NAME, 1); @@ -50,7 +52,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { test("resolves on a park that happens AFTER the call", async () => { const channel = createObservedSignalChannel(); const waited = channel.awaitAwaitedCount(NAME, 1); - const parked = channel.awaitNext(NAME); + const parked = channel.awaitNext(NAME, noWake); await waited; await channel.deliver(NAME, { ok: true }, "s1"); @@ -61,7 +63,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { const channel = createObservedSignalChannel(); const waited = channel.awaitAwaitedCount(NAME, 2); - const first = channel.awaitNext(NAME); + const first = channel.awaitNext(NAME, noWake); // One park is not two. Give the waiter every chance to resolve early: a // macrotask turn drains the microtasks its continuation would run on. let resolved = false; @@ -75,7 +77,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { // what the real re-park sequence does. await channel.deliver(NAME, { n: 1 }, "s1"); await first; - const second = channel.awaitNext(NAME); + const second = channel.awaitNext(NAME, noWake); await waited; expect(channel.awaitedCount(NAME)).toBe(2); @@ -87,7 +89,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { const channel = createObservedSignalChannel(); const waited = channel.awaitAwaitedCount(NAME, 1); - const other = channel.awaitNext("other"); + const other = channel.awaitNext("other", noWake); // A park on another name must not wake this waiter. As above, the yield is // a yield and not a duration bet: it drains the microtasks the waiter's // continuation would run on, so a wrongly-woken waiter is caught here. @@ -98,7 +100,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { await new Promise((r) => setTimeout(r, 0)); expect(resolved).toBe(false); - const wanted = channel.awaitNext(NAME); + const wanted = channel.awaitNext(NAME, noWake); await waited; await channel.deliver("other", { ok: true }, "s1"); @@ -121,7 +123,7 @@ describe("createObservedSignalChannel awaitAwaitedCount", () => { describe("createObservedSignalChannel passthrough", () => { test("delivery reaches a live awaiter, payload and signalId intact", async () => { const channel = createObservedSignalChannel(); - const parked = channel.awaitNext(NAME); + const parked = channel.awaitNext(NAME, noWake); await channel.deliver(NAME, { outcome: "approved" }, "sig-1"); expect(await parked).toEqual({ payload: { outcome: "approved" }, @@ -134,7 +136,7 @@ describe("createObservedSignalChannel passthrough", () => { await channel.deliver(NAME, { queued: true }, "sig-q"); // The wrapper must not change the queueing semantics the runtime relies // on: a signal that arrives before the park is not lost. - expect(await channel.awaitNext(NAME)).toEqual({ + expect(await channel.awaitNext(NAME, noWake)).toEqual({ payload: { queued: true }, signalId: "sig-q", }); @@ -145,7 +147,7 @@ describe("createObservedSignalChannel passthrough", () => { test("an aborted await rejects and still counts as a park", async () => { const channel = createObservedSignalChannel(); const abort = new AbortController(); - const parked = channel.awaitNext(NAME, abort.signal); + const parked = channel.awaitNext(NAME, noWake, abort.signal); expect(channel.awaitedCount(NAME)).toBe(1); abort.abort(); await expect(parked).rejects.toThrow(/aborted/); diff --git a/packages/workflow/src/testing/signal-channel-observer.ts b/packages/workflow/src/testing/signal-channel-observer.ts index 8255bac65..e8911bec2 100644 --- a/packages/workflow/src/testing/signal-channel-observer.ts +++ b/packages/workflow/src/testing/signal-channel-observer.ts @@ -78,13 +78,13 @@ export function createObservedSignalChannel(): ObservedSignalChannel { } return inner.deliver(name, payload, signalId, correlationId); }, - awaitNext(name, signal, correlationId) { + awaitNext(name, onWake, signal, correlationId) { counts.set(name, countOf(name) + 1); announce(); if (correlationId === undefined) { - return inner.awaitNext(name, signal); + return inner.awaitNext(name, onWake, signal); } - return inner.awaitNext(name, signal, correlationId); + return inner.awaitNext(name, onWake, signal, correlationId); }, awaitedCount: countOf, async awaitAwaitedCount(name, count) { diff --git a/tests/workflow-deploy/single-step-conversation-durability.test.ts b/tests/workflow-deploy/single-step-conversation-durability.test.ts index 6bac45e46..b598a55d2 100644 --- a/tests/workflow-deploy/single-step-conversation-durability.test.ts +++ b/tests/workflow-deploy/single-step-conversation-durability.test.ts @@ -569,6 +569,7 @@ describe("single-step conversation durability across respawn (Phase 4.5)", () => actionCredentials: () => unavailableActionCredentials, spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, + earliestQueuedTimer: () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], From 364d9866ad6bb3eb593f30f458aa47a30e43e960 Mon Sep 17 00:00:00 2001 From: FarinM Date: Thu, 29 Oct 2026 01:28:36 +0100 Subject: [PATCH 06/20] Remove the temp dirs the workflow host tests create The supervisor, recycle, wiring, run-child and parked-correlation tests made their data dirs under the system temp dir and left them there, so each run of those files added more. They now remove them once their tests are done. --- .../src/child/parked-correlations.test.ts | 24 +++++++++++++++---- .../workflow-host/src/child/run-child.test.ts | 14 +++++++++-- .../src/deploy/workflow-host-wiring.test.ts | 16 ++++++++++--- .../src/supervisor/recycle.test.ts | 12 +++++++++- .../src/supervisor/supervisor.test.ts | 9 +++++++ 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/packages/workflow-host/src/child/parked-correlations.test.ts b/packages/workflow-host/src/child/parked-correlations.test.ts index cfd40db2f..621c2813a 100644 --- a/packages/workflow-host/src/child/parked-correlations.test.ts +++ b/packages/workflow-host/src/child/parked-correlations.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect } from "bun:test"; +import { afterAll, describe, test, expect } from "bun:test"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -21,6 +21,20 @@ import { collectParkedApprovalCorrelations } from "./parked-correlations"; const at = new Date().toISOString(); const repoId: RepoId = { kind: "workflow-run", id: "dep-1" }; +const tempDirs: string[] = []; + +async function makeTempDir(): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "parked-corr-")); + tempDirs.push(dir); + return dir; +} + +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + const snapshot: ApprovalSnapshot = { name: "charge_card", description: "Charge the customer's card", @@ -120,7 +134,7 @@ async function setup( substrate: SubstrateRepoStore; runtimeRepoStore: RuntimeRepoStore; }> { - const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "parked-corr-")); + const baseDir = await makeTempDir(); const runsDir = path.join(baseDir, repoId.kind, repoId.id, "runs"); const runtimeRepoStore = createInMemoryRepoStore(); for (const run of runs) { @@ -164,7 +178,7 @@ describe("collectParkedApprovalCorrelations", () => { }); test("enumerates parked correlations across multiple runs and steps", async () => { - const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "parked-corr-")); + const baseDir = await makeTempDir(); const runsDir = path.join(baseDir, repoId.kind, repoId.id, "runs"); const runtimeRepoStore = createInMemoryRepoStore(); // run-a parks two steps concurrently; run-b parks one. The enumeration @@ -238,7 +252,7 @@ describe("collectParkedApprovalCorrelations", () => { // loadParkedApproval binding wired. Were it not skipped, its mere presence // would take the whole deployment's approval re-registration down on every // reconnect. - const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "parked-corr-")); + const baseDir = await makeTempDir(); const runsDir = path.join(baseDir, repoId.kind, repoId.id, "runs"); const runtimeRepoStore = createInMemoryRepoStore(); await fs.mkdir(path.join(runsDir, "run-input"), { recursive: true }); @@ -303,7 +317,7 @@ describe("collectParkedApprovalCorrelations", () => { }); test("a relayed approval loads the snapshot from the child step", async () => { - const baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "parked-corr-")); + const baseDir = await makeTempDir(); const runsDir = path.join(baseDir, repoId.kind, repoId.id, "runs"); const runtimeRepoStore = createInMemoryRepoStore(); await fs.mkdir(path.join(runsDir, "parent-run"), { recursive: true }); diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index b24b28e29..f3b8c442c 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect } from "bun:test"; +import { afterAll, describe, test, expect } from "bun:test"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -82,10 +82,20 @@ import { type ControlPayload, } from "../ipc/index"; +const tempDirs: string[] = []; + async function makeTempDir(prefix: string): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), prefix)); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; } +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + // Build the minimal decoded `Mail` for a plain-text body: a single // `text/plain` part whose decoded text is inlined (its `ref` is never read // for text parts). This is the shape a `trigger.fire` / `signal.deliver` diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts index 8cb4bf330..afdb43b07 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, afterEach, spyOn } from "bun:test"; +import { describe, test, expect, afterAll, afterEach, spyOn } from "bun:test"; import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import fs from "node:fs/promises"; @@ -381,10 +381,20 @@ afterEach(() => { } }); -function createTempBaseDir(prefix: string): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), prefix)); +const tempDirs: string[] = []; + +async function createTempBaseDir(prefix: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; } +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + // The supervisor's `parked-correlations.request` control-frame discriminator. // The re-emit-parked-correlations tests assert this reaches the downstream // (supervisor -> child) stream when Trigger B fires. diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 154471448..2d0fa2d03 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -99,10 +99,20 @@ function authoritySetFrames(lines: readonly string[]) { ); } +const tempDirs: string[] = []; + async function makeTempDir(prefix: string): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), prefix)); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; } +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + type FakeChild = { pid: number; channelId: string | undefined; diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index bc249f6b1..04d4657b8 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -465,11 +465,20 @@ function readCancelRequestedBlob( return validated; } +const tempDirs: string[] = []; + async function makeTempDir(prefix: string): Promise { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); return dir; } +afterAll(async () => { + for (const dir of tempDirs.splice(0)) { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + // A mail bus that exposes the durable settlement of a delivery via `settle` // (the subscribed handler's returned promise), so the ack/withhold mapping can // be asserted directly. `settle` reads the `handlers` map below, which nothing From 6da04fed75c09fd46b0e754ed7cf5749552f83b9 Mon Sep 17 00:00:00 2001 From: FarinM Date: Wed, 4 Nov 2026 14:36:46 +0100 Subject: [PATCH 07/20] Give a resumed workflow run the trigger payload it started with Only a fresh run was given its trigger payload, so a run resumed after a restart or respawn read null for trigger.payload in every later step, though its RunStarted records the payload. --- .../runtime/resume-awaiting-signal.test.ts | 51 +++++++++++++++++++ packages/workflow/src/runtime/run.ts | 6 ++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts index 17e22f71b..a2ea3b170 100644 --- a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts +++ b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts @@ -211,6 +211,57 @@ describe("resume awaiting signal", () => { expect(types).toContain("RunCompleted"); }); + test("a resumed run's steps read the trigger payload its RunStarted recorded", async () => { + const runId = "run-resumed-trigger"; + const def = defineWorkflow({ + id: "wait-resume-trigger-input", + trigger: { type: "manual" }, + steps: { + gate: awaitSignal({ name: "go" }), + after: step({ + agent: resumeAgent("after"), + after: ["gate"], + input: { from: "trigger.payload" }, + }), + }, + }); + const inputs: unknown[] = []; + const channel = createObservedSignalChannel(); + const env = buildEnv(def, { + signalChannel: channel, + invokeStep: async (req) => { + inputs.push(req.input); + return { output: null }; + }, + }); + const seed: WorkflowEvent[] = [ + { + kind: "RunStarted", + seq: 1, + at, + runId, + definitionHash: "x", + trigger: { type: "manual", payload: { text: "first mail" } }, + }, + { + kind: "StepStarted", + seq: 2, + at, + stepId: "gate", + attempt: 1, + input: { ref: "inline:null" }, + }, + { kind: "SignalAwaited", seq: 3, at, stepId: "gate", signalName: "go" }, + ]; + + const handle = runtimeRun(def, env, { runId, resumeFromEvents: seed }); + await channel.awaitAwaitedCount("go", 1); + await channel.deliver("go", { resumed: true }, "sig-live"); + + expect((await handle.complete).terminalStatus).toBe("completed"); + expect(inputs).toEqual([{ text: "first mail" }]); + }); + test("resumes a seed that already carries the SignalReceived (gate in-flight), completing it without a live deliver", async () => { const runId = "run-received"; const env = buildEnv(gateOnly); diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index 8a8e85653..1d0e1c419 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -1035,7 +1035,11 @@ async function executeRunBody( // protects these `resolveRef` calls runs up front, before any blob // resolution. const canonicalLog = await env.repoStore.read(runId); + // Selectors read the trigger payload the log's RunStarted recorded, so a + // resume, which is given none, reads what the run started with. + let triggerPayload = options.triggerPayload; for (const event of canonicalLog) { + if (event.kind === "RunStarted") triggerPayload = event.trigger.payload; if (event.kind !== "StepCompleted") continue; stepOutputs[event.stepId] = await env.blobs.resolveRef(event.output.ref); } @@ -1180,7 +1184,7 @@ async function executeRunBody( for (const primitive of ready) { inFlight.add(primitive.id); const ctx: SelectorContext = { - trigger: { payload: options.triggerPayload }, + trigger: { payload: triggerPayload }, steps: Object.fromEntries( Object.entries(stepOutputs).map(([id, output]) => [id, { output }]), ), From e663440ac2920b6f828d4b88ea32badf143a35cc Mon Sep 17 00:00:00 2001 From: FarinM Date: Tue, 27 Oct 2026 17:06:28 +0100 Subject: [PATCH 08/20] Resume a loop or onTrigger body's parks its container never relayed A restart can find a loop or onTrigger body holding parks its container never relayed, as it can a childWorkflow child. Resume handed the container a single token for the park it had relayed, so the others were never relayed and the run waited forever. Both containers now resume through the childWorkflow's relay plan, which relays every park the body holds that the container never did. The plan also counts an approval park a body step stored but crashed before logging, and re-registering parked correlations reads bodies the same way, so both find the same parks. A body whose log is gone starts over: the container re-links the relay it holds for it once, and fails the step when input was delivered after the spawn, which it cannot hand down again. A re-established relay lets a nested container hand down a signal its parent took before the restart. The Hub's parked-run check reads the body's log too, so an onTrigger body parked on a signal its section never relayed counts as parked. A timed step can leave an approval and go on, so a body may no longer hold the approval its container relays. A loop or onTrigger container now retires that relay and carries on, whether the body left before the restart or while the relay waits. A spawn step still fails on it. SignalAwaitAbandoned now accepts the approval park of a step that spawned a child, which changes which logs replay as valid. A plan that only retires a relay holds no park, so the parked-run check does not count it. A loop or onTrigger body its container resumes is given its input, so a body whose log is gone starts over with it. An onTrigger section recovers that input from its own log: the run's trigger payload for its first event, and what its input park delivered for a later one. --- packages/hub-sessions/src/parked-run.test.ts | 265 +++- packages/hub-sessions/src/parked-run.ts | 51 +- .../src/child/parked-correlations.test.ts | 86 ++ .../src/child/parked-correlations.ts | 16 +- packages/workflow-host/src/child/run-child.ts | 3 + .../src/child/substrate-factory.ts | 15 +- ...-substrate-factory-parked-approval.test.ts | 142 +- ...ubstrate-factory-suspendable-child.test.ts | 88 ++ packages/workflow/README.md | 5 +- packages/workflow/src/index.ts | 2 +- .../src/runtime/child-workflow-relay.test.ts | 127 ++ packages/workflow/src/runtime/env.ts | 6 +- packages/workflow/src/runtime/index.ts | 3 +- .../src/runtime/loop-suspend-resume.test.ts | 963 +++++++++++- .../src/runtime/on-trigger-run.test.ts | 1269 ++++++++++++--- packages/workflow/src/runtime/run.ts | 1367 +++++++++-------- .../src/runtime/suspendable-child-handle.ts | 16 +- packages/workflow/src/state-machine/events.ts | 3 +- .../src/state-machine/transition.test.ts | 46 +- .../workflow/src/state-machine/transition.ts | 22 +- ...trigger-body-sleep-restart-resumes.test.ts | 5 +- 21 files changed, 3561 insertions(+), 939 deletions(-) diff --git a/packages/hub-sessions/src/parked-run.test.ts b/packages/hub-sessions/src/parked-run.test.ts index b6a17b705..cb9a68874 100644 --- a/packages/hub-sessions/src/parked-run.test.ts +++ b/packages/hub-sessions/src/parked-run.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import git from "isomorphic-git"; +import { signalName } from "@intx/types/signals"; import { loopBodyRunId, sectionBodyRunId } from "@intx/workflow"; import { createParkedRunClassifier } from "./parked-run"; @@ -101,6 +102,71 @@ function stepStarted(stepId: string): Record { }; } +function approvalAwaited( + stepId: string, + correlationId: string, +): Record { + return { + type: "SignalAwaited", + stepId, + signalName: signalName(correlationId), + parkKind: "approval", + }; +} + +function containerRelaying( + stepId: string, + childRunId: string, + relay: Record, +): Record[] { + return [ + runStarted(ANCHOR), + stepStarted(stepId), + { type: "ChildSpawned", stepId, childRunId, childDefinitionRef: "body" }, + { ...relay, stepId }, + ]; +} + +// A body step that timed out on approval `corr-x` and began its second attempt. +function retriedPastApproval(): Record[] { + return [ + stepStarted("work"), + approvalAwaited("work", "corr-x"), + { + type: "StepFailed", + stepId: "work", + attempt: 1, + error: { message: "timed out" }, + retriesExhausted: false, + }, + { type: "TimerSet", timerId: "retry", fireAt: AT, stepId: "work" }, + { + type: "AttemptScheduled", + stepId: "work", + nextAttempt: 2, + timerId: "retry", + fireAt: AT, + }, + { type: "TimerFired", timerId: "retry" }, + ]; +} + +function thenSleeping( + completedStep: string, + attempt: number, +): Record[] { + return [ + { + type: "StepCompleted", + stepId: completedStep, + attempt, + output: { ref: "inline:null" }, + }, + stepStarted("nap"), + { type: "TimerSet", timerId: "nap", fireAt: AT, stepId: "nap" }, + ]; +} + describe("createParkedRunClassifier", () => { let root: string; let dir: string; @@ -264,6 +330,127 @@ describe("createParkedRunClassifier", () => { expect(verdict.verdict).toBe("parked"); }); + const loopBody = { + again: { + kind: "loop", + maxIterations: 3, + body: { + steps: { + work: { kind: "step" }, + hold: { kind: "awaitSignal" }, + nap: { kind: "sleep" }, + }, + }, + }, + }; + + test("a loop body that left its relayed approval for a sleep is not parked", async () => { + const childRunId = loopBodyRunId(ANCHOR, "again", 0); + await commitFiles(dir, { + ...eventFiles( + ANCHOR, + containerRelaying( + "again", + childRunId, + approvalAwaited("again", "corr-x"), + ), + ), + ...eventFiles(childRunId, [ + runStarted(childRunId), + ...retriedPastApproval(), + ...thenSleeping("work", 2), + ]), + }); + + const verdict = await classify({ steps: loopBody })({ + anchorRunId: ANCHOR, + }); + + expect(verdict.verdict).toBe("not-parked"); + }); + + test("a loop body that left its relayed approval for a crashed step is not parked", async () => { + const childRunId = loopBodyRunId(ANCHOR, "again", 0); + await commitFiles(dir, { + ...eventFiles( + ANCHOR, + containerRelaying( + "again", + childRunId, + approvalAwaited("again", "corr-x"), + ), + ), + ...eventFiles(childRunId, [ + runStarted(childRunId), + ...retriedPastApproval(), + ]), + }); + + const verdict = await classify({ steps: loopBody })({ + anchorRunId: ANCHOR, + }); + + expect(verdict.verdict).toBe("not-parked"); + }); + + test("a loop body that left its relayed approval for another approval is parked", async () => { + const childRunId = loopBodyRunId(ANCHOR, "again", 0); + await commitFiles(dir, { + ...eventFiles( + ANCHOR, + containerRelaying( + "again", + childRunId, + approvalAwaited("again", "corr-x"), + ), + ), + ...eventFiles(childRunId, [ + runStarted(childRunId), + ...retriedPastApproval(), + approvalAwaited("work", "corr-y"), + ]), + }); + + const verdict = await classify({ steps: loopBody })({ + anchorRunId: ANCHOR, + }); + + expect(verdict.verdict).toBe("parked"); + }); + + test("a loop body that left its relayed gate for a sleep is not parked", async () => { + const childRunId = loopBodyRunId(ANCHOR, "again", 0); + await commitFiles(dir, { + ...eventFiles( + ANCHOR, + containerRelaying("again", childRunId, { + type: "SignalAwaited", + signalName: "go", + parkKind: "signal-relay", + }), + ), + ...eventFiles(childRunId, [ + runStarted(childRunId), + stepStarted("hold"), + { type: "TimerSet", timerId: "gate", fireAt: AT, stepId: "hold" }, + { + type: "SignalAwaited", + stepId: "hold", + signalName: "go", + timeoutAt: AT, + }, + { type: "TimerFired", timerId: "gate" }, + ...thenSleeping("hold", 1), + ]), + }); + + const verdict = await classify({ steps: loopBody })({ + anchorRunId: ANCHOR, + }); + + expect(verdict.verdict).toBe("not-parked"); + }); + test("a tolerated iteration followed by one unrelayed author signal is parked", async () => { const failedChild = loopBodyRunId(ANCHOR, "again", 0); const parkedChild = loopBodyRunId(ANCHOR, "again", 1); @@ -310,7 +497,7 @@ describe("createParkedRunClassifier", () => { expect(verdict.verdict).toBe("parked"); }); - test("two drive-fresh loops are parked together", async () => { + test("two loops whose author signals were never relayed are parked together", async () => { const left = loopBodyRunId(ANCHOR, "left", 0); const right = loopBodyRunId(ANCHOR, "right", 0); const child = (runId: string) => @@ -353,7 +540,8 @@ describe("createParkedRunClassifier", () => { expect(verdict.verdict).toBe("parked"); }); - test("an unrelayed onTrigger body is not parked", async () => { + // The section never relayed the body's gate; resume relays it fresh. + test("an unrelayed onTrigger body is parked", async () => { const childRunId = sectionBodyRunId(ANCHOR, "inbox", 0); await commitFiles(dir, { ...eventFiles(ANCHOR, [ @@ -382,9 +570,82 @@ describe("createParkedRunClassifier", () => { }, })({ anchorRunId: ANCHOR }); + expect(verdict.verdict).toBe("parked"); + }); + + test("an onTrigger body that left its relayed approval for a sleep is not parked", async () => { + const childRunId = sectionBodyRunId(ANCHOR, "inbox", 0); + await commitFiles(dir, { + ...eventFiles( + ANCHOR, + containerRelaying( + "inbox", + childRunId, + approvalAwaited("inbox", "corr-x"), + ), + ), + ...eventFiles(childRunId, [ + runStarted(childRunId), + ...retriedPastApproval(), + ...thenSleeping("work", 2), + ]), + }); + + const verdict = await classify({ + steps: { + inbox: { + kind: "onTrigger", + body: { + inline: { + steps: { work: { kind: "step" }, nap: { kind: "sleep" } }, + }, + }, + }, + }, + })({ anchorRunId: ANCHOR }); + expect(verdict.verdict).toBe("not-parked"); }); + test("an onTrigger body whose log cannot be read is unknown", async () => { + const childRunId = sectionBodyRunId(ANCHOR, "inbox", 0); + await commitFiles(dir, { + ...eventFiles(ANCHOR, [ + runStarted(ANCHOR), + stepStarted("inbox"), + { + type: "ChildSpawned", + stepId: "inbox", + childRunId, + childDefinitionRef: "body", + }, + ]), + }); + const runReader = createWorkflowRunReader(repoStoreFor(root)); + + const verdict = await createParkedRunClassifier({ + runReader: { + ...runReader, + readRunEvents: async (repoId, ref, runId) => { + if (runId === childRunId) throw new Error("the store is down"); + return runReader.readRunEvents(repoId, ref, runId); + }, + }, + addressForAnchor: async () => ADDRESS, + projectionForAnchor: async () => ({ + steps: { + inbox: { + kind: "onTrigger", + body: { inline: { steps: { hold: { kind: "awaitSignal" } } } }, + }, + }, + }), + })({ anchorRunId: ANCHOR }); + + expect(verdict.verdict).toBe("unknown"); + expect(verdict.reason).toContain("the store is down"); + }); + test("a legacy onTrigger body id with an unsupported inner step is not parked", async () => { const childRunId = "inbox__0"; await commitFiles(dir, { diff --git a/packages/hub-sessions/src/parked-run.ts b/packages/hub-sessions/src/parked-run.ts index f3193de99..725dac689 100644 --- a/packages/hub-sessions/src/parked-run.ts +++ b/packages/hub-sessions/src/parked-run.ts @@ -43,7 +43,7 @@ export function createParkedRunClassifier(deps: { }): ParkedRunClassifier { return async (allocation) => { try { - return await classifyAnchor(deps, allocation.anchorRunId); + return await classifyAnchor(deps, allocation.anchorRunId, Date.now()); } catch (cause) { return { verdict: "unknown", @@ -60,6 +60,7 @@ async function classifyAnchor( readonly projectionForAnchor: (anchorRunId: string) => Promise; }, anchorRunId: string, + nowMs: number, ): Promise { const address = await deps.addressForAnchor(anchorRunId); if (address === null) { @@ -121,6 +122,7 @@ async function classifyAnchor( raw, state, log: events.map(toWorkflowEvent), + nowMs, }); if (judged.verdict !== undefined) return judged.classification; if (judged.park) hasPark = true; @@ -132,6 +134,7 @@ async function classifyAnchor( raw, state, log: events.map(toWorkflowEvent), + nowMs, }); if (judged.verdict !== undefined) return judged.classification; if (judged.park) hasPark = true; @@ -163,6 +166,7 @@ async function judgeLoop(args: { raw: unknown; state: ReturnType; log: readonly WorkflowEvent[]; + nowMs: number; }): Promise { const maxIterations = maxIterationsOf(args.raw); if (maxIterations === undefined) { @@ -192,17 +196,20 @@ async function judgeLoop(args: { } if (frontier === undefined) return { park: false }; const childRunId = loopBodyRunId(args.anchorRunId, args.stepId, frontier); - const childState = resumeFromLog( + const childLog = await readChild( + args.deps.runReader, + args.repoId, childRunId, - await readChild(args.deps.runReader, args.repoId, childRunId), ); - const reattach = await loopOccurrenceReattach({ + const childState = resumeFromLog(childRunId, childLog); + const reattach = loopOccurrenceReattach({ loopId: args.stepId, runId: args.anchorRunId, iteration: frontier, state: args.state, log: args.log, - childState, + childLog, + nowMs: args.nowMs, }); return finishReattach( reattach, @@ -219,6 +226,7 @@ async function judgeOnTrigger(args: { raw: unknown; state: ReturnType; log: readonly WorkflowEvent[]; + nowMs: number; }): Promise { const policy = failurePolicyOf(args.raw, "onBodyFailure"); if (policy === "invalid") { @@ -227,7 +235,7 @@ async function judgeOnTrigger(args: { `onTrigger ${args.stepId} has an invalid onBodyFailure`, ); } - const childState = resumeFromLog(args.stepId, []); + let readFailure: unknown; const reattach = await onTriggerOccurrenceReattach({ sectionId: args.stepId, ...(policy === "end" || policy === "tolerate" @@ -235,11 +243,23 @@ async function judgeOnTrigger(args: { : {}), state: args.state, log: args.log, - childState, + loadChild: (childRunId) => + readChild(args.deps.runReader, args.repoId, childRunId).catch( + (cause: unknown) => { + readFailure = cause; + throw cause; + }, + ), + nowMs: args.nowMs, }); - // The planner reads the child only when the container is not already - // parked. Load that child when the decision names one, and re-judge so - // the unrelayed-body throw sees the committed child log. + // The reattach reports any throw as a rejected plan, which a read failure + // is not: nothing judged the body. + if (readFailure !== undefined) { + return stop( + "unknown", + `onTrigger ${args.stepId} body could not be read: ${readFailure instanceof Error ? readFailure.message : String(readFailure)}`, + ); + } if (reattach.kind !== "parked" && reattach.kind !== "reenter") { return { park: false }; } @@ -249,17 +269,8 @@ async function judgeOnTrigger(args: { reattach.childRunId, await readChild(args.deps.runReader, args.repoId, reattach.childRunId), ); - const again = await onTriggerOccurrenceReattach({ - sectionId: args.stepId, - ...(policy === "end" || policy === "tolerate" - ? { onBodyFailure: policy } - : {}), - state: args.state, - log: args.log, - childState: loaded, - }); return finishReattach( - again, + reattach, loaded, onTriggerBodyLookup(args.raw), args.stepId, diff --git a/packages/workflow-host/src/child/parked-correlations.test.ts b/packages/workflow-host/src/child/parked-correlations.test.ts index 621c2813a..00806b359 100644 --- a/packages/workflow-host/src/child/parked-correlations.test.ts +++ b/packages/workflow-host/src/child/parked-correlations.test.ts @@ -424,6 +424,92 @@ describe("collectParkedApprovalCorrelations", () => { ]); }); + test("a relayed approval whose body stored the park but has not logged it is found", async () => { + const baseDir = await makeTempDir(); + const runsDir = path.join(baseDir, repoId.kind, repoId.id, "runs"); + const runtimeRepoStore = createInMemoryRepoStore(); + await fs.mkdir(path.join(runsDir, "parent-run"), { recursive: true }); + await fs.mkdir(path.join(runsDir, "child-run"), { recursive: true }); + const parent: WorkflowEvent[] = [ + { + kind: "RunStarted", + seq: 1, + at, + runId: "parent-run", + definitionHash: "x", + trigger: { type: "manual", payload: undefined }, + }, + { + kind: "StepStarted", + seq: 2, + at, + stepId: "spawn", + attempt: 1, + input: { ref: "inline:null" }, + }, + { + kind: "ChildSpawned", + seq: 3, + at, + stepId: "spawn", + childRunId: "child-run", + childDefinitionRef: "child", + }, + { + kind: "SignalAwaited", + seq: 4, + at, + stepId: "spawn", + signalName: signalName("corr-body"), + parkKind: "approval", + }, + ]; + // The body step stored its park and crashed before logging it. + const child: WorkflowEvent[] = [ + { + kind: "RunStarted", + seq: 1, + at, + runId: "child-run", + definitionHash: "y", + trigger: { type: "manual", payload: undefined }, + }, + { + kind: "StepStarted", + seq: 2, + at, + stepId: "hold", + attempt: 1, + input: { ref: "inline:null" }, + }, + ]; + for (const event of parent) + await runtimeRepoStore.append("parent-run", event); + for (const event of child) + await runtimeRepoStore.append("child-run", event); + const storedInBody = (args: { runId: string; stepId: string }): boolean => + args.runId === "child-run" && args.stepId === "hold"; + + const result = await collectParkedApprovalCorrelations({ + substrate: createStubSubstrate(baseDir), + repoId, + runtimeRepoStore, + loadParkedApproval: async (args) => + storedInBody(args) ? snapshot : undefined, + readParkedApprovalOps: async (args) => + storedInBody(args) ? [{ correlationId: "corr-body" }] : [], + }); + + expect(result).toEqual([ + { + runId: "parent-run", + correlationId: "corr-body", + parkKind: "approval", + snapshot, + }, + ]); + }); + test("throws when the binding has no snapshot for an enumerated park", async () => { const { substrate, runtimeRepoStore } = await setup([ { runId: "run-4", stepId: "s", name: signalName("corr-4") }, diff --git a/packages/workflow-host/src/child/parked-correlations.ts b/packages/workflow-host/src/child/parked-correlations.ts index 3cf89cded..a8e10917c 100644 --- a/packages/workflow-host/src/child/parked-correlations.ts +++ b/packages/workflow-host/src/child/parked-correlations.ts @@ -16,16 +16,20 @@ // `childWorkflow` parks the container on the body's correlation, and the // snapshot stays in the body's step store. The container's own store has // none. Enumeration follows the spawned child that is still awaiting that -// correlation, including a child that relays it onward. +// correlation, including a child that relays it onward. It reads a body's log +// as the container's resume plan does, so a park the body stored but has not +// logged yet is found too. import type { RepoId, RepoStore as SubstrateRepoStore, } from "@intx/hub-sessions/substrate"; import { + bodyLogForRelays, controlParkKindOf, findParkedApprovalSnapshot, type LoadParkedApproval, + type ReadParkedApprovalOps, type RepoStore as RuntimeRepoStore, } from "@intx/workflow"; import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime"; @@ -50,6 +54,7 @@ export interface CollectParkedApprovalCorrelationsOpts { repoId: RepoId; runtimeRepoStore: RuntimeRepoStore; loadParkedApproval?: LoadParkedApproval; + readParkedApprovalOps?: ReadParkedApprovalOps; } /** @@ -68,6 +73,13 @@ export async function collectParkedApprovalCorrelations( repoId: opts.repoId, runtimeRepoStore: opts.runtimeRepoStore, }); + const bodyEnv = { + repoStore: opts.runtimeRepoStore, + clock: () => new Date(), + ...(opts.readParkedApprovalOps !== undefined + ? { readParkedApprovalOps: opts.readParkedApprovalOps } + : {}), + }; const out: ParkedApprovalCorrelation[] = []; for (const run of discovered) { for (const step of run.resumedState.steps.values()) { @@ -99,7 +111,7 @@ export async function collectParkedApprovalCorrelations( ); } const snapshot = await findParkedApprovalSnapshot({ - repoStore: opts.runtimeRepoStore, + repoStore: { read: (id) => bodyLogForRelays(bodyEnv, id) }, load: opts.loadParkedApproval, runId: run.runId, state: run.resumedState, diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index af40adcf6..5c7f9df7c 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -1708,6 +1708,9 @@ async function handleControlPayload( ...(ctx.bindings.loadParkedApproval !== undefined ? { loadParkedApproval: ctx.bindings.loadParkedApproval } : {}), + ...(ctx.bindings.readParkedApprovalOps !== undefined + ? { readParkedApprovalOps: ctx.bindings.readParkedApprovalOps } + : {}), }); await ctx.upstreamSender.send({ type: "parked-correlations.response", diff --git a/packages/workflow-host/src/child/substrate-factory.ts b/packages/workflow-host/src/child/substrate-factory.ts index a09be8731..c2cabe349 100644 --- a/packages/workflow-host/src/child/substrate-factory.ts +++ b/packages/workflow-host/src/child/substrate-factory.ts @@ -1303,6 +1303,13 @@ interface SidecarRunChildDeps { * relay an approval its own body parked and it never relayed. */ loadParkedApproval?: LoadParkedApproval; + /** + * The step store's pending approval read and recovered-park write, so a + * nested run's step that crashed after it stored its approval park + * re-commits that park on resume, as a top-level step does. + */ + readParkedApprovalOps?: ReadParkedApprovalOps; + persistRecoveredPark?: PersistRecoveredPark; /** Workflow-run repo identifying the parent's deployment. */ workflowRunRepoId: RepoId; /** Workflow-run ref the child reads/writes against. */ @@ -2196,6 +2203,12 @@ async function buildChildRunEnv(args: { ...(args.deps.loadParkedApproval !== undefined ? { loadParkedApproval: args.deps.loadParkedApproval } : {}), + ...(args.deps.readParkedApprovalOps !== undefined + ? { readParkedApprovalOps: args.deps.readParkedApprovalOps } + : {}), + ...(args.deps.persistRecoveredPark !== undefined + ? { persistRecoveredPark: args.deps.persistRecoveredPark } + : {}), }; // Wire loop-iteration spawning for a `loop` nested in this body. Assigned // AFTER the env literal because the iteration host closes over `env`: a loop @@ -2800,7 +2813,7 @@ export function createSidecarSubstrateFactory( // A spawned child's steps resolve `director` refs against the same // closure-composed registry the parent's steps use. directors: stepDirectors, - loadParkedApproval: parkHooks.loadParkedApproval, + ...parkHooks, }; // Terminal childWorkflow executor. `run-child` builds the in-memory // resolver from this plus the lifted-body map it extracts after loading diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts index aa2554d84..d5848895d 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-parked-approval.test.ts @@ -32,15 +32,29 @@ import { type RepoStore, type WorkflowRunWorkflowProcessPrincipal, } from "@intx/hub-sessions"; -import { defineAgent } from "@intx/agent"; +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; import { createIsogitStore } from "@intx/storage-isogit/node"; +import { signalName } from "@intx/types/signals"; import { + childWorkflow, + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, + createNoopDrainController, + createSuspendableChildHandle, defineWorkflow, loop, loopBodyRunId, + rewriteInlineChildWorkflowBodies, + runtimeRun, sectionBodyRunId, step, + type RepoStore as RuntimeRepoStore, + type StepInvoker, type WorkflowDefinition, + type WorkflowEvent, + type WorkflowRuntimeEnv, } from "@intx/workflow"; import { createDurableConversationStore, @@ -829,6 +843,132 @@ describe("createParkedApprovalHooks", () => { ).toThrow("onExhausted"); }, ); + + // The container plans its relays from the body's parks, read through its + // own env, while the body recovers its park through the body's. + test("a warm-kept deployment's resume relays an approval its spawned body stored but never logged", async () => { + const dataDir = await makeTempDir(); + const substrate = await createSubstrate(dataDir); + const childRunId = "run-body"; + const outer = defineWorkflow({ + id: "outer", + trigger: { type: "manual" }, + steps: { spawn: childWorkflow({ definition: agentBody("body") }) }, + }); + const { workflow: rewritten, bodies } = + rewriteInlineChildWorkflowBodies(outer); + const body = bodies[0]?.definition; + if (body === undefined) throw new Error("the inline body was not lifted"); + const hooks = createParkedApprovalHooks({ + dataDir, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + ...COMMITTED_READS, + signer: testSigner, + warmRunId: RUN_ID, + }); + const invokeStep: StepInvoker = async (req) => + req.resume !== undefined + ? { output: null } + : { + suspend: { + correlationId: "corr-1", + kind: "approval", + approvalSnapshot: snapshot, + }, + }; + const envFor = ( + repoStore: RuntimeRepoStore, + run: WorkflowDefinition, + extra: Partial, + ): WorkflowRuntimeEnv => { + const clock = (): Date => new Date(); + let ids = 0; + return { + repoStore, + scheduler: createInMemoryScheduler({ repoStore, clock }), + signalChannel: createInMemorySignalChannel(), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep, + spawnChild: () => Promise.reject(new Error("the body spawns nothing")), + clock, + newId: (prefix) => + prefix === "run" ? childRunId : `${prefix}-${String((ids += 1))}`, + drain: createNoopDrainController(run), + hasUpstreamSignalResolver: true, + ...extra, + }; + }; + const outerEnv = ( + repoStore: RuntimeRepoStore, + parkHooks: Partial, + onPark: () => void, + ): WorkflowRuntimeEnv => + envFor(repoStore, rewritten, { + ...parkHooks, + onPark, + spawnChild: async (spawn) => + createSuspendableChildHandle(envFor(repoStore, body, parkHooks), { + definition: body, + childRunId: spawn.childRunId, + input: spawn.input, + depth: spawn.depth, + maxChildSpawnDepth: spawn.maxChildSpawnDepth, + signal: spawn.signal, + ...(spawn.resumeFromEvents !== undefined + ? { resumeFromEvents: spawn.resumeFromEvents } + : {}), + surfaceInputPark: true, + }), + }); + + // Run to the relayed park for the logs, then drop both from the body's + // park on: the body stored its approval but crashed before logging it. + const firstRepo = createInMemoryRepoStore(); + const relayedFirst = Promise.withResolvers(); + const stop = new AbortController(); + const first = runtimeRun( + rewritten, + outerEnv(firstRepo, {}, () => relayedFirst.resolve(undefined)), + { runId: RUN_ID, localAbort: stop.signal }, + ); + await relayedFirst.promise; + const beforePark = async (runId: string): Promise => { + const log = [...(await firstRepo.read(runId))]; + const parked = log.findIndex((event) => event.kind === "SignalAwaited"); + expect(parked).toBeGreaterThan(0); + return log.slice(0, parked); + }; + const outerLog = await beforePark(RUN_ID); + const bodyLog = await beforePark(childRunId); + stop.abort(); + await first.complete; + await parkColdStep(substrate, childRunId, 1, [ + pendingApproval("corr-1", snapshot), + ]); + + const repo = createInMemoryRepoStore(); + await repo.appendBatch(childRunId, bodyLog); + const relayed = Promise.withResolvers(); + const resumed = runtimeRun( + rewritten, + outerEnv(repo, hooks, () => relayed.resolve(undefined)), + { runId: RUN_ID, resumeFromEvents: outerLog }, + ); + await relayed.promise; + await resumed.signal( + signalName("corr-1"), + { outcome: "approved" }, + "approval-decision", + ); + expect((await resumed.complete).terminalStatus).toBe("completed"); + }); }); describe("toParkedApprovalOps", () => { diff --git a/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts b/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts index 286960e90..acc171d2b 100644 --- a/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts +++ b/packages/workflow-host/src/child/workflow-substrate-factory-suspendable-child.test.ts @@ -19,6 +19,7 @@ import os from "node:os"; import path from "node:path"; import { generateKeyPair } from "@intx/crypto"; +import { signalName } from "@intx/types/signals"; import type { ApprovalSnapshot, KeyPair } from "@intx/types/runtime"; import { defineAgent, createDefaultDirectorRegistry } from "@intx/agent"; import { @@ -221,6 +222,7 @@ function suspendThenComplete(record: { function makeSpawner( substrate: ReturnType, invokeStep: SidecarChildStepInvoker, + extra: Partial[0]> = {}, ): ReturnType { return createSidecarSpawnSuspendableChild({ substrate, @@ -244,6 +246,7 @@ function makeSpawner( parentGrants, closurePackageDir: "/unused-closure", directors: createDefaultDirectorRegistry(), + ...extra, }); } @@ -295,6 +298,91 @@ describe("createSidecarSpawnSuspendableChild", () => { expect(record.resumeDecision).toEqual(decision); }); + test("re-parks a body step whose stored approval park never reached its log, and resumes on the grant", async () => { + const substrate = await makeSubstrate("suspendable-lost-park-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [ + grant(`tool:${BODY_STEP_AGENT_ID}`, "invoke"), + ]); + const childRunId = "run-body-lost-park"; + const at = "2026-10-01T00:00:00.000Z"; + const persisted: unknown[] = []; + const record: { resumeDecision?: unknown } = {}; + const spawn = makeSpawner(substrate, suspendThenComplete(record), { + readParkedApprovalOps: async (args) => + args.runId === childRunId && args.stepId === "s" + ? [{ correlationId: CORRELATION_ID }] + : [], + persistRecoveredPark: async (args) => { + persisted.push(args); + }, + }); + + // The body crashed after its step stored the park and before the park + // reached the body's log. + const handle = await spawn( + { + definition: bodyDefinition("body-wf"), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "section", + signal: new AbortController().signal, + depth: 0, + maxChildSpawnDepth: 32, + resumeFromEvents: [ + { + kind: "RunStarted", + seq: 1, + at, + runId: childRunId, + definitionHash: "body-hash", + trigger: { type: "manual", payload: null }, + }, + { + kind: "StepStarted", + seq: 2, + at, + stepId: "s", + attempt: 1, + input: { ref: "inline:null" }, + }, + ], + }, + () => undefined, + ); + const watch = new AbortController(); + for await (const { event } of reader(substrate).subscribe(childRunId, { + signal: watch.signal, + from: { seq: 1 }, + })) { + if ( + event.kind === "SignalAwaited" && + event.signalName === signalName(CORRELATION_ID) + ) { + watch.abort(); + break; + } + } + expect(persisted).toEqual([ + { + runId: childRunId, + stepId: "s", + attempt: 1, + correlationId: CORRELATION_ID, + }, + ]); + + const decision = { outcome: "approved" as const, note: "ok" }; + await handle.resume(CORRELATION_ID, decision); + const terminal = await handle.next(noWake); + expect(terminal.kind).toBe("terminal"); + if (terminal.kind !== "terminal") throw new Error("expected a terminal"); + expect(terminal.terminalStatus).toBe("completed"); + expect(record.resumeDecision).toEqual(decision); + }); + test("threads depth so a body's childWorkflow grandchild trips the ceiling", async () => { const substrate = await makeSubstrate("suspendable-depth-"); const parentRunId = "run-parent"; diff --git a/packages/workflow/README.md b/packages/workflow/README.md index 37ce591ad..614c9f03b 100644 --- a/packages/workflow/README.md +++ b/packages/workflow/README.md @@ -215,7 +215,10 @@ container's signal path and delivery resumes the iteration, the same way an onTrigger section body parks. A parked iteration also survives a crash -- a restart re-drives the loop, re-establishes the container's signal relay, and resumes the iteration on the next delivery -- so an `awaitSignal` loop body is a -durable human-in-the-loop pause, not just an in-process one. +durable human-in-the-loop pause, not just an in-process one. A restart relays +the parks of a body with several at once that the container had not reached +yet, one at a time, the same way as for a `childWorkflow`; an approval among +them takes its snapshot from the parked step's stored state. A loop body may also spawn a `childWorkflow` grandchild: the child is lifted to a ref and runs as its own child run, depth-counted against the tree-wide spawn diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts index 94b51b6ab..e02bc1d62 100644 --- a/packages/workflow/src/index.ts +++ b/packages/workflow/src/index.ts @@ -44,7 +44,6 @@ export { } from "./live-inert-projector"; export { runtimeRun, - bodyParkedSignals, loopIterationCursor, loopOccurrenceReattach, onTriggerOccurrenceReattach, @@ -53,6 +52,7 @@ export { createTerminalChildHandle, createLoopIterationHandle, findParkedApprovalSnapshot, + bodyLogForRelays, MAX_CHILD_SPAWN_DEPTH, ChildSpawnDepthExceededError, resolveMaxChildSpawnDepth, diff --git a/packages/workflow/src/runtime/child-workflow-relay.test.ts b/packages/workflow/src/runtime/child-workflow-relay.test.ts index 8dec5cda4..c74b6e2e4 100644 --- a/packages/workflow/src/runtime/child-workflow-relay.test.ts +++ b/packages/workflow/src/runtime/child-workflow-relay.test.ts @@ -1241,6 +1241,71 @@ describe("childWorkflow signal relay", () => { // again, or a release during that read is in neither place and the // parent waits forever. describe("park-release watch", () => { + test("a cancel during the relay's log read leaves no unhandled rejection", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + const parent = defineWorkflow({ + id: "abort-read-parent", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ definition: gatedChild("abort-read-child") }), + }, + }); + const runId = "run-abort-read"; + const inner = createInMemoryRepoStore(); + const readHeld = Promise.withResolvers(); + const releaseRead = Promise.withResolvers(); + let holdNextRead = false; + let channelWait: Promise | undefined; + const repoStore: RepoStore = { + read: async (id) => { + if (holdNextRead && id === runId) { + holdNextRead = false; + readHeld.resolve(undefined); + await releaseRead.promise; + } + return inner.read(id); + }, + append: (id, event) => inner.append(id, event), + appendBatch: (id, events) => inner.appendBatch(id, events), + subscribe: (id, opts) => inner.subscribe(id, opts), + }; + const { run } = startRelayed({ + definition: parent, + runId, + invokeStep: async () => ({ output: null }), + repoStore, + signalChannel: { + deliver: () => Promise.reject(new Error("deliver is not used")), + awaitNext: (name, _onWake, signal) => { + // The relay reads the parent's log right after it starts this wait. + const wait = pendingSignal(signal); + if (name === "go") { + holdNextRead = true; + channelWait = wait.catch(() => undefined); + } + return wait; + }, + }, + }); + await readHeld.promise; + await run.cancel("supervisor-operator", "stop"); + // The waits reject while the read is still open, and the runtime reports + // an unhandled rejection before the next turn of the event loop. + await channelWait; + await new Promise((resolve) => setImmediate(resolve)); + releaseRead.resolve(undefined); + await run.complete.catch(() => undefined); + expect(unhandled).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + test("a release that lands during the read still resolves the watch", async () => { const childRunId = "child-watch-gap"; const inner = createInMemoryRepoStore(); @@ -1696,6 +1761,68 @@ describe("a parked childWorkflow survives a process restart", () => { expect(spawnOutput(result.outputs.spawn).childRunId).toBe(childId); }); + test("fails the spawn step when the child left the approval the parent awaits", async () => { + const parent = defineWorkflow({ + id: "resume-left-approval-parent", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "resume-left-approval-child", + trigger: { type: "manual" }, + steps: { hold: step({ agent }) }, + }), + }), + }, + }); + const runId = "run-resume-left-approval"; + const suspending: StepInvoker = async () => ({ + suspend: { + correlationId: "corr-body", + kind: "approval", + approvalSnapshot: snapshot, + }, + }); + const first = startRelayed({ + definition: parent, + runId, + invokeStep: suspending, + }); + void first.run.complete; + await waitForEvent( + first.repoStore, + runId, + (event) => + event.kind === "SignalAwaited" && + event.stepId === "spawn" && + event.parkKind === "approval", + ); + const parentLog = (await copyLog(first.repoStore, runId)).map((event) => + event.kind === "SignalAwaited" && event.stepId === "spawn" + ? { ...event, signalName: signalName("corr-left") } + : event, + ); + const childId = spawnedChildId(parentLog, "spawn"); + const childLog = await copyLog(first.repoStore, childId); + + const repoStore = createInMemoryRepoStore(); + await seedLog(repoStore, childId, childLog); + const { run } = startRelayed({ + definition: parent, + runId, + invokeStep: suspending, + repoStore, + resumeFromEvents: parentLog, + }); + const result = await run.complete; + + expect(result.terminalStatus).toBe("failed"); + const failed = result.events.find((event) => event.kind === "StepFailed"); + expect(failed?.kind === "StepFailed" ? failed.error.message : "").toContain( + "cannot be retired", + ); + }); + test("reestablishes an input park and delivers the next turn", async () => { const parent = defineWorkflow({ id: "resume-input-parent", diff --git a/packages/workflow/src/runtime/env.ts b/packages/workflow/src/runtime/env.ts index 3f2147728..9d1c5177c 100644 --- a/packages/workflow/src/runtime/env.ts +++ b/packages/workflow/src/runtime/env.ts @@ -493,9 +493,9 @@ export type SuspendableChildHandle = { * or the child run has reached a terminal. A sibling park does not * resolve it and is left queued for a later `next`. Abort rejects. * - * Present on a handle whose caller can have several parks live at once. - * A loop iteration and an onTrigger body are driven one step at a time, - * so their callers keep racing `next` and do not need this. + * Present on a handle whose caller can have several parks live at once: + * a childWorkflow, and a loop iteration or onTrigger body on resume, where + * the parks already in the body's log are relayed one at a time. */ waitForParkRelease?( name: string, diff --git a/packages/workflow/src/runtime/index.ts b/packages/workflow/src/runtime/index.ts index d128cd6fb..5c5ab5208 100644 --- a/packages/workflow/src/runtime/index.ts +++ b/packages/workflow/src/runtime/index.ts @@ -36,9 +36,8 @@ export { type EffectContextConfig, } from "./effect-context"; -export { runtimeRun, type RuntimeRunOptions } from "./run"; +export { bodyLogForRelays, runtimeRun, type RuntimeRunOptions } from "./run"; export { - bodyParkedSignals, loopIterationCursor, loopOccurrenceReattach, onTriggerOccurrenceReattach, diff --git a/packages/workflow/src/runtime/loop-suspend-resume.test.ts b/packages/workflow/src/runtime/loop-suspend-resume.test.ts index 9f89eaff3..fa8915c93 100644 --- a/packages/workflow/src/runtime/loop-suspend-resume.test.ts +++ b/packages/workflow/src/runtime/loop-suspend-resume.test.ts @@ -18,7 +18,7 @@ import { describe, test, expect } from "bun:test"; import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; import { signalName } from "@intx/types"; -import type { ConversationTurn } from "@intx/types/runtime"; +import type { ApprovalSnapshot, ConversationTurn } from "@intx/types/runtime"; import { action, @@ -40,6 +40,8 @@ import { type LoopFn, type RepoStore, type SignalChannel, + type LoadParkedApproval, + type ReadParkedApprovalOps, type StepInvoker, type WorkflowAuthorizeFn, type WorkflowDefinition, @@ -83,6 +85,8 @@ function buildEnv(args: { invokeAction: ActionInvoker; invokeStep?: StepInvoker; drain?: DrainController; + loadParkedApproval?: LoadParkedApproval; + readParkedApprovalOps?: ReadParkedApprovalOps; }): WorkflowRuntimeEnv { const clock = (): Date => new Date(); const authorize: WorkflowAuthorizeFn = async () => ({ @@ -105,6 +109,12 @@ function buildEnv(args: { drain: args.drain ?? createNoopDrainController(args.parentDef), hasUpstreamSignalResolver: true, loopFns, + ...(args.loadParkedApproval !== undefined + ? { loadParkedApproval: args.loadParkedApproval } + : {}), + ...(args.readParkedApprovalOps !== undefined + ? { readParkedApprovalOps: args.readParkedApprovalOps } + : {}), }; const loopBodies = new Map( enumerateInlineLoopBodies(args.parentDef).map((b) => [b.ref, b.definition]), @@ -297,8 +307,30 @@ const nestedAwaitParent = withLoopBody( innerAwaitParent, ); +// The nested chain over a gate that times out to `late`. The timeout is long, +// so only a `TimerFired` the test writes ends the gate. +const timedAwaitBody = defineWorkflow({ + id: "timed-await-body", + trigger: { type: "manual" }, + steps: { + pre: action({ handler: "pre", input: { from: "trigger.payload" } }), + hold: awaitSignal({ + name: "go", + timeout: 600_000, + onTimeout: "late", + after: ["pre"], + }), + late: action({ handler: "late", after: ["hold"] }), + }, +}); +const nestedTimedParent = withLoopBody( + nestedAwaitParent, + "outer", + withLoopBody(innerAwaitParent, "inner", timedAwaitBody), +); + // A loop body with TWO parallel author `awaitSignal` gates, so both park at -// once -- the concurrent-park topology the resume planner does not support. +// once and a resume can find both un-relayed. const twoSignalBody = defineWorkflow({ id: "two-signal-body", trigger: { type: "manual" }, @@ -324,8 +356,34 @@ const twoSignalParent = defineWorkflow({ }, }); +// A body that takes two signals in turn, each relayed by its container. +const sequentialSignalBody = defineWorkflow({ + id: "sequential-signal-body", + trigger: { type: "manual" }, + steps: { + first: awaitSignal({ name: "first" }), + second: awaitSignal({ name: "second", after: ["first"] }), + }, +}); + +const sequentialSignalParent = defineWorkflow({ + id: "sequential-signal-parent", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: sequentialSignalBody, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["rework"] }), + }, +}); + describe("loop iteration suspend crash-resume", () => { - test("refuses to resume a body parked on two concurrent author signals", async () => { + test("resumes a body parked on two concurrent author signals the container had not relayed", async () => { const runId = "two-signal-run"; const repoStore1 = createInMemoryRepoStore(); const channel1 = createObservedSignalChannel(); @@ -370,30 +428,25 @@ describe("loop iteration suspend crash-resume", () => { const repoStore2 = createInMemoryRepoStore(); await seedChildLog(repoStore2, bodyRunId, bodyLog); + const channel2 = createInMemorySignalChannel(); const env2 = buildEnv({ parentDef: twoSignalParent, repoStore: repoStore2, blobs: createInMemoryBlobSubstrate(), - signalChannel: createInMemorySignalChannel(), + signalChannel: channel2, invokeAction: awaitInvokeAction({ n: 0 }), }); - // The resume planner refuses the concurrent-park topology loudly rather than - // mis-recovering one gate and stranding the other. - const result = await runtimeRun(twoSignalParent, env2, { + // Both un-relayed gates are relayed, one at a time. + const run2 = runtimeRun(twoSignalParent, env2, { runId, resumeFromEvents: parentLog, - }).complete; - // The planner refuses the concurrent-park topology loudly: the loop step - // fails rather than mis-recovering one gate and stranding the other. - expect(result.terminalStatus).toBe("failed"); - const resumedLog = await repoStore2.read(runId); - const failed = resumedLog.find( - (e) => e.kind === "StepFailed" && e.stepId === "rework", - ); - expect(failed?.kind === "StepFailed" ? failed.error.message : "").toMatch( - /multiple concurrent author signals/, - ); + }); + await channel2.deliver("goA", { ok: true }, "sig-goA"); + await channel2.deliver("goB", { ok: true }, "sig-goB"); + const result = await run2.complete; + + expect(result.terminalStatus).toBe("completed"); }); test("re-establishes an awaitSignal park and resumes on a signal delivered after restart", async () => { @@ -453,6 +506,186 @@ describe("loop iteration suspend crash-resume", () => { }); }); + test("a body whose log is gone starts over with its iteration's input", async () => { + const runId = "await-run-lost-body"; + const blobs = createInMemoryBlobSubstrate(); + const repoStore1 = createInMemoryRepoStore(); + void runtimeRun( + awaitParent, + buildEnv({ + parentDef: awaitParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction({ n: 0 }), + }), + { runId }, + ).complete.catch(() => undefined); + await waitForContainerPark(repoStore1, runId, "signal-relay", 1); + const parentLog = await repoStore1.read(runId); + + // Only the parent log survives the restart. + const ranPre = Promise.withResolvers(); + const run2 = runtimeRun( + awaitParent, + buildEnv({ + parentDef: awaitParent, + repoStore: createInMemoryRepoStore(), + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: async ({ handler, input }) => { + if (handler === "pre") ranPre.resolve(input); + return { output: { ran: true } }; + }, + }), + { runId, resumeFromEvents: parentLog }, + ); + expect(await ranPre.promise).toBe(0); + + await run2.signal("go", { done: true }, "sig-1"); + expect((await run2.complete).terminalStatus).toBe("completed"); + }); + + test("a body whose log is gone after it took a relayed signal fails instead of waiting for it again", async () => { + const runId = "lost-body-after-input"; + const blobs = createInMemoryBlobSubstrate(); + const repoStore1 = createInMemoryRepoStore(); + const channel1 = createInMemorySignalChannel(); + void runtimeRun( + sequentialSignalParent, + buildEnv({ + parentDef: sequentialSignalParent, + repoStore: repoStore1, + blobs, + signalChannel: channel1, + invokeAction: awaitInvokeAction({ n: 0 }), + }), + { runId }, + ).complete.catch(() => undefined); + await waitForContainerPark(repoStore1, runId, "signal-relay", 1); + await channel1.deliver("first", { ok: true }, "sig-first"); + await waitForContainerPark(repoStore1, runId, "signal-relay", 2); + const parentLog = [...(await repoStore1.read(runId))]; + + // Only the parent log survives the restart. + const result = await runtimeRun( + sequentialSignalParent, + buildEnv({ + parentDef: sequentialSignalParent, + repoStore: createInMemoryRepoStore(), + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction({ n: 0 }), + }), + { runId, resumeFromEvents: parentLog }, + ).complete; + expect(result.terminalStatus).toBe("failed"); + const failed = result.events.find((e) => e.kind === "StepFailed"); + expect(failed?.kind === "StepFailed" ? failed.error.message : "").toContain( + "lost its log after first was delivered", + ); + }); + + // A body whose log is gone parks afresh on resume. The container already + // carries that park out, so relaying it again would leave a relay the next + // iteration's signal goes to. + async function resumeWithoutBodyLog( + runId: string, + beforeCrash: ( + run: ReturnType, + store: RepoStore, + ) => Promise, + ) { + const blobs = createInMemoryBlobSubstrate(); + const repoStore1 = createInMemoryRepoStore(); + const run1 = runtimeRun( + carryParent, + buildEnv({ + parentDef: carryParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction({ n: 0 }), + }), + { runId }, + ); + await waitForContainerPark(repoStore1, runId, "signal-relay", 1); + const lastSeq = await beforeCrash(run1, repoStore1); + const parentLog = (await repoStore1.read(runId)).filter( + (e) => e.seq <= lastSeq, + ); + // The commit chain is per run id, so the first run settles before the + // resumed one writes. + await run1.cancel("supervisor-operator", "stop"); + await run1.complete.catch(() => undefined); + + // Only the parent log survives the restart. + const repoStore2 = createInMemoryRepoStore(); + const run2 = runtimeRun( + carryParent, + buildEnv({ + parentDef: carryParent, + repoStore: repoStore2, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction({ n: 0 }), + }), + { runId, resumeFromEvents: parentLog }, + ); + return { run2, repoStore2 }; + } + + async function expectNextRelayIsSecondIteration( + repoStore: RepoStore, + runId: string, + ): Promise { + const next = await waitForNthEvent( + repoStore, + runId, + (e) => e.kind === "SignalAwaited" && e.parkKind === "signal-relay", + 2, + ); + const log = await repoStore.read(runId); + const secondSpawn = log.find( + (e) => + e.kind === "ChildSpawned" && + e.childRunId === loopBodyRunId(runId, "rework", 1), + ); + expect(secondSpawn?.seq ?? Infinity).toBeLessThan(next.seq); + expect(log.some((e) => e.kind === "SignalAwaitAbandoned")).toBe(false); + } + + test("a body whose log is gone has its park relayed once, so the next signal reaches the next iteration", async () => { + const runId = "lost-body-relayed-once"; + const { run2, repoStore2 } = await resumeWithoutBodyLog( + runId, + async (_run, store) => (await store.read(runId)).length, + ); + await run2.signal("go", { n: 1 }, "sig-1"); + await expectNextRelayIsSecondIteration(repoStore2, runId); + await run2.signal("go", { n: 2 }, "sig-2"); + expect((await run2.complete).terminalStatus).toBe("completed"); + }); + + test("a body whose log is gone after its container took a signal fails instead of waiting for it again", async () => { + const runId = "lost-body-took-signal"; + const { run2 } = await resumeWithoutBodyLog(runId, async (run, store) => { + await run.signal("go", { n: 1 }, "sig-1"); + const received = await waitForEvent( + store, + runId, + (e) => e.kind === "SignalReceived" && e.signalId === "sig-1", + ); + return received.seq; + }); + const result = await run2.complete; + expect(result.terminalStatus).toBe("failed"); + const failed = result.events.find((e) => e.kind === "StepFailed"); + expect(failed?.kind === "StepFailed" ? failed.error.message : "").toContain( + "lost its log after go was delivered", + ); + }); + test("resumes when the crash lands before the container relay await flushed", async () => { // The crash window between the body's leaf `SignalAwaited` flush (on the // body run) and the container's relay `SignalAwaited` flush (on the @@ -636,7 +869,7 @@ describe("loop iteration suspend crash-resume", () => { test("resumes a nested loop when only the inner container relay flushed", async () => { // Partial-park window: the inner container's relay await IS durable but the // outer container's is not. The outer planner's scan of the outer body finds - // the inner container awaiting an author name and classifies it drive-fresh; + // the inner container awaiting an author name and relays it fresh; // re-adopting re-establishes the inner relay. It must resume, once. const runId = "nested-partial-outer-run"; const blobs = createInMemoryBlobSubstrate(); @@ -745,6 +978,156 @@ describe("loop iteration suspend crash-resume", () => { iterations: 1, }); }); + + test("relays a signal the outer container took before the crash into a body whose park is a nested relay", async () => { + const runId = "nested-relay-run"; + const blobs = createInMemoryBlobSubstrate(); + + const repoStore1 = createInMemoryRepoStore(); + const env1 = buildEnv({ + parentDef: nestedAwaitParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction({ n: 0 }), + }); + void runtimeRun(nestedAwaitParent, env1, { runId }).complete.catch( + () => undefined, + ); + const relayName = await waitForContainerPark( + repoStore1, + runId, + "signal-relay", + 1, + ); + const outerBodyRunId = loopBodyRunId(runId, "outer", 0); + const innerBodyRunId = loopBodyRunId(outerBodyRunId, "inner", 0); + const parkedLog = await repoStore1.read(runId); + const outerBodyLog = await repoStore1.read(outerBodyRunId); + const innerBodyLog = await repoStore1.read(innerBodyRunId); + + // The outer container took the signal, and the crash landed before it + // reached the outer body, whose own park is the inner container's relay. + const lastSeq = parkedLog[parkedLog.length - 1]?.seq ?? 0; + const delivered: WorkflowEvent = { + kind: "SignalReceived", + seq: lastSeq + 1, + at: new Date().toISOString(), + signalName: relayName, + signalId: "sig-pre", + payload: { done: true }, + }; + + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog(repoStore2, outerBodyRunId, outerBodyLog); + await seedChildLog(repoStore2, innerBodyRunId, innerBodyLog); + const preRuns2 = { n: 0 }; + const env2 = buildEnv({ + parentDef: nestedAwaitParent, + repoStore: repoStore2, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: awaitInvokeAction(preRuns2), + }); + const result = await runtimeRun(nestedAwaitParent, env2, { + runId, + resumeFromEvents: [...parkedLog, delivered], + }).complete; + + expect(result.terminalStatus).toBe("completed"); + expect(preRuns2.n).toBe(0); + const innerReceived = (await repoStore2.read(innerBodyRunId)).filter( + (e) => e.kind === "SignalReceived", + ); + expect(innerReceived).toEqual([ + expect.objectContaining({ signalName: "go", signalId: "sig-pre" }), + ]); + }); + + test("drops a signal the outer container took when the nested gate times out before the relay waits again", async () => { + const runId = "nested-timed-out-run"; + const blobs = createInMemoryBlobSubstrate(); + const ran: string[] = []; + const invokeAction: ActionInvoker = async ({ handler }) => { + ran.push(handler); + return { output: handler }; + }; + + const repoStore1 = createInMemoryRepoStore(); + const env1 = buildEnv({ + parentDef: nestedTimedParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction, + }); + const crash = new AbortController(); + const first = runtimeRun(nestedTimedParent, env1, { + runId, + localAbort: crash.signal, + }); + const relayName = await waitForContainerPark( + repoStore1, + runId, + "signal-relay", + 1, + ); + const outerBodyRunId = loopBodyRunId(runId, "outer", 0); + const innerBodyRunId = loopBodyRunId(outerBodyRunId, "inner", 0); + // Copies, since stopping the first run appends to its logs. + const parkedLog = [...(await repoStore1.read(runId))]; + const outerBodyLog = [...(await repoStore1.read(outerBodyRunId))]; + const innerBodyLog = [...(await repoStore1.read(innerBodyRunId))]; + // The first run's gate timer would otherwise outlive the test. + crash.abort(); + await first.complete; + + // The outer container took the signal, and the inner gate's timer fired + // and was committed, before the crash. + const timerSet = innerBodyLog.find((e) => e.kind === "TimerSet"); + if (timerSet?.kind !== "TimerSet") throw new Error("the gate set no timer"); + const timerFired: WorkflowEvent = { + kind: "TimerFired", + seq: (innerBodyLog[innerBodyLog.length - 1]?.seq ?? 0) + 1, + at: new Date().toISOString(), + timerId: timerSet.timerId, + }; + const delivered: WorkflowEvent = { + kind: "SignalReceived", + seq: (parkedLog[parkedLog.length - 1]?.seq ?? 0) + 1, + at: new Date().toISOString(), + signalName: relayName, + signalId: "sig-pre", + payload: { done: true }, + }; + + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog(repoStore2, outerBodyRunId, outerBodyLog); + await seedChildLog(repoStore2, innerBodyRunId, [ + ...innerBodyLog, + timerFired, + ]); + ran.length = 0; + const env2 = buildEnv({ + parentDef: nestedTimedParent, + repoStore: repoStore2, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction, + }); + const result = await runtimeRun(nestedTimedParent, env2, { + runId, + resumeFromEvents: [...parkedLog, delivered], + }).complete; + + expect(result.terminalStatus).toBe("completed"); + expect(ran).toEqual(["late"]); + expect( + (await repoStore2.read(innerBodyRunId)).some( + (e) => e.kind === "SignalReceived", + ), + ).toBe(false); + }); }); const replyTurn: ConversationTurn = { @@ -783,7 +1166,313 @@ const approvalParent = defineWorkflow({ }, }); +// The approval loop, one nesting level down. +const nestedApprovalParent = withLoopBody( + defineWorkflow({ + id: "nested-approval-loop-parent", + trigger: { type: "manual" }, + steps: { + outer: loop({ + body: approvalBody, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["outer"] }), + }, + }), + "outer", + defineWorkflow({ + id: "inner-approval-loop", + trigger: { type: "manual" }, + steps: { + inner: loop({ + body: approvalBody, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "iescalate", + }), + iescalate: action({ handler: "escalate", after: ["inner"] }), + }, + }), +); + +// A loop body with two parallel approval steps: the container relays one, and +// the other waits for it in memory. +const twoApprovalBody = defineWorkflow({ + id: "two-approval-body", + trigger: { type: "manual" }, + steps: { + a: step({ agent: approvalAgent }), + b: step({ agent: approvalAgent }), + }, +}); + +const twoApprovalParent = defineWorkflow({ + id: "two-approval-loop-parent", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: twoApprovalBody, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["rework"] }), + }, +}); + +// A loop whose timed approval step retries, so its next attempt parks on a new +// approval once the deadline leaves the first. +const timedApprovalParent = (timeout: number): WorkflowDefinition => + defineWorkflow({ + id: "timed-approval-loop-parent", + trigger: { type: "manual" }, + steps: { + rework: loop({ + body: defineWorkflow({ + id: "timed-approval-body", + trigger: { type: "manual" }, + steps: { + timed: step({ + agent: approvalAgent, + timeout, + retry: { maxAttempts: 2, initialBackoffMs: 1 }, + }), + }, + }), + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 3, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["rework"] }), + }, + }); + describe("loop iteration approval crash-resume", () => { + test("retires an approval the body left before a restart and relays the one it holds", async () => { + const runId = "left-approval-run"; + const blobs = createInMemoryBlobSubstrate(); + const escalateAction: ActionInvoker = async ({ handler }) => { + if (handler === "escalate") return { output: "escalated" }; + throw new Error(`unknown handler ${handler}`); + }; + const stored = new Map(); + const key = (args: { + runId?: string; + stepId?: string; + attempt?: number; + correlationId: string; + }): string => + `${String(args.runId)}/${String(args.stepId)}/${String(args.attempt)}/${args.correlationId}`; + const invokeStep: StepInvoker = async (req) => { + if (req.resume !== undefined) { + return { output: { reply: "done", turn: replyTurn } }; + } + const correlationId = `corr-${String(req.authzContext.attempt)}`; + const approvalSnapshot: ApprovalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: { attempt: req.authzContext.attempt }, + }; + stored.set(key({ ...req.authzContext, correlationId }), approvalSnapshot); + return { + suspend: { correlationId, kind: "approval", approvalSnapshot }, + }; + }; + const left = signalName("corr-1"); + const held = signalName("corr-2"); + + const short = timedApprovalParent(20); + const repoStore1 = createInMemoryRepoStore(); + const run1 = runtimeRun( + short, + buildEnv({ + parentDef: short, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: escalateAction, + invokeStep, + }), + { runId }, + ); + const bodyRunId = loopBodyRunId(runId, "rework", 0); + expect(await waitForContainerPark(repoStore1, runId, "approval", 1)).toBe( + left, + ); + await waitForEvent( + repoStore1, + bodyRunId, + (e) => e.kind === "SignalAwaited" && e.signalName === held, + ); + const parentLog = [...(await repoStore1.read(runId))]; + const childLog = [...(await repoStore1.read(bodyRunId))]; + await run1.cancel("supervisor-operator", "crash"); + await run1.complete.catch(() => undefined); + + // A later deadline, so the attempt it holds stays parked for the decision. + const long = timedApprovalParent(60_000); + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog(repoStore2, bodyRunId, childLog); + const channel2 = createInMemorySignalChannel(); + const run2 = runtimeRun( + long, + buildEnv({ + parentDef: long, + repoStore: repoStore2, + blobs, + signalChannel: channel2, + invokeAction: escalateAction, + invokeStep, + loadParkedApproval: async (args) => stored.get(key(args)), + }), + { runId, resumeFromEvents: parentLog }, + ); + // A run that ends instead of relaying says why. + const ended = run2.complete.then((result) => { + const failed = result.events.find((e) => e.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "rework" && + e.signalName === held, + ), + ended, + ]); + expect( + (await repoStore2.read(runId)).some( + (e) => e.kind === "SignalAwaitAbandoned" && e.signalName === left, + ), + ).toBe(true); + await channel2.deliver(held, { outcome: "approved" }, "g-held"); + const result = await run2.complete; + + expect(result.terminalStatus).toBe("completed"); + }); + + test("relays an approval the container had not reached after a restart", async () => { + const runId = "two-approval-run"; + const blobs = createInMemoryBlobSubstrate(); + const escalateAction: ActionInvoker = async ({ handler }) => { + if (handler === "escalate") return { output: "escalated" }; + throw new Error(`unknown handler ${handler}`); + }; + // What each suspended step durably stored, as the host's step store does. + const stored = new Map(); + const key = (args: { + runId?: string; + stepId?: string; + attempt?: number; + correlationId: string; + }): string => + `${String(args.runId)}/${String(args.stepId)}/${String(args.attempt)}/${args.correlationId}`; + const invokeStep: StepInvoker = async (req) => { + if (req.resume !== undefined) { + return { output: { reply: "done", turn: replyTurn } }; + } + const correlationId = `corr-${String(req.authzContext.stepId)}`; + const approvalSnapshot: ApprovalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: { step: req.authzContext.stepId }, + }; + stored.set(key({ ...req.authzContext, correlationId }), approvalSnapshot); + return { + suspend: { correlationId, kind: "approval", approvalSnapshot }, + }; + }; + + const repoStore1 = createInMemoryRepoStore(); + const env1 = buildEnv({ + parentDef: twoApprovalParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: escalateAction, + invokeStep, + }); + void runtimeRun(twoApprovalParent, env1, { runId }).complete.catch( + () => undefined, + ); + const bodyRunId = loopBodyRunId(runId, "rework", 0); + const shown = await waitForContainerPark(repoStore1, runId, "approval", 1); + for (const stepId of ["a", "b"]) { + await waitForEvent( + repoStore1, + bodyRunId, + (e) => + e.kind === "SignalAwaited" && + e.signalName === signalName(`corr-${stepId}`), + ); + } + const parentLog = await repoStore1.read(runId); + const childLog = await repoStore1.read(bodyRunId); + const queued = + shown === signalName("corr-a") + ? signalName("corr-b") + : signalName("corr-a"); + + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog(repoStore2, bodyRunId, childLog); + const channel2 = createInMemorySignalChannel(); + const env2 = buildEnv({ + parentDef: twoApprovalParent, + repoStore: repoStore2, + blobs, + signalChannel: channel2, + invokeAction: escalateAction, + invokeStep, + loadParkedApproval: async (args) => stored.get(key(args)), + }); + const run2 = runtimeRun(twoApprovalParent, env2, { + runId, + resumeFromEvents: parentLog, + }); + await channel2.deliver(shown, { outcome: "approved" }, "g-shown"); + // A run that ends instead of relaying says why. + const ended = run2.complete.then((result) => { + const failed = result.events.find((e) => e.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "rework" && + e.signalName === queued, + ), + ended, + ]); + await channel2.deliver(queued, { outcome: "approved" }, "g-queued"); + const result = await run2.complete; + + expect(result.terminalStatus).toBe("completed"); + }); + test("re-adopts an approval park and re-invokes the step with a grant delivered after restart", async () => { const runId = "approval-run"; const blobs = createInMemoryBlobSubstrate(); @@ -872,6 +1561,242 @@ describe("loop iteration approval crash-resume", () => { iterations: 1, }); }); + + test.each([ + ["before the container relayed it", false], + ["after the container relayed it", true], + ])( + "relays an approval its body stored but never committed, after a crash %s", + async (_case, relayed) => { + const runId = "lost-approval-run"; + const bodyRunId = loopBodyRunId(runId, "rework", 0); + const blobs = createInMemoryBlobSubstrate(); + const escalateAction: ActionInvoker = async ({ handler }) => { + if (handler === "escalate") return { output: "escalated" }; + throw new Error(`unknown handler ${handler}`); + }; + const approvalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: {}, + }; + const invocations: { resume: unknown }[] = []; + const invokeStep: StepInvoker = async (req) => { + invocations.push({ resume: req.resume }); + if (req.resume === undefined) { + return { + suspend: { + correlationId: "corr-1", + kind: "approval", + approvalSnapshot, + }, + }; + } + return { output: { reply: "done", turn: replyTurn } }; + }; + + const repoStore1 = createInMemoryRepoStore(); + const crash = new AbortController(); + const first = runtimeRun( + approvalParent, + buildEnv({ + parentDef: approvalParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: escalateAction, + invokeStep, + }), + { runId, localAbort: crash.signal }, + ); + await waitForContainerPark(repoStore1, runId, "approval", 1); + // The crash came after the body's step stored its park and before the + // park reached the body's log, with or without the container's relay. + const isPark = (e: WorkflowEvent): boolean => e.kind === "SignalAwaited"; + const parentLog = [...(await repoStore1.read(runId))]; + const childLog = [...(await repoStore1.read(bodyRunId))]; + crash.abort(); + await first.complete; + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog( + repoStore2, + bodyRunId, + childLog.slice(0, childLog.findIndex(isPark)), + ); + const channel2 = createInMemorySignalChannel(); + invocations.length = 0; + const run2 = runtimeRun( + approvalParent, + buildEnv({ + parentDef: approvalParent, + repoStore: repoStore2, + blobs, + signalChannel: channel2, + invokeAction: escalateAction, + invokeStep, + readParkedApprovalOps: async (args) => + args.runId === bodyRunId && args.stepId === "s" + ? [{ correlationId: "corr-1" }] + : [], + loadParkedApproval: async () => approvalSnapshot, + }), + { + runId, + resumeFromEvents: parentLog.slice( + 0, + parentLog.findIndex(isPark) + (relayed ? 1 : 0), + ), + }, + ); + // A run that ends instead of relaying says why. + const ended = run2.complete.then((result) => { + throw new Error(`the run ended ${result.terminalStatus}`); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "rework" && + e.signalName === signalName("corr-1"), + ), + ended, + ]); + await channel2.deliver( + signalName("corr-1"), + { outcome: "approved" }, + "g-1", + ); + const result = await run2.complete; + + expect(result.terminalStatus).toBe("completed"); + // The step re-parked from its stored park, and ran again only with the + // decision. + expect(invocations).toEqual([ + { + resume: { + correlationId: "corr-1", + decision: { outcome: "approved" }, + kind: "approval", + }, + }, + ]); + }, + ); + + test("relays a nested approval its leaf never committed, after a crash that the inner relay outlived", async () => { + const runId = "nested-lost-approval-run"; + const outerBodyRunId = loopBodyRunId(runId, "outer", 0); + const leafRunId = loopBodyRunId(outerBodyRunId, "inner", 0); + const blobs = createInMemoryBlobSubstrate(); + const escalateAction: ActionInvoker = async ({ handler }) => { + if (handler === "escalate") return { output: "escalated" }; + throw new Error(`unknown handler ${handler}`); + }; + const approvalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: {}, + }; + const invocations: { resume: unknown }[] = []; + const invokeStep: StepInvoker = async (req) => { + invocations.push({ resume: req.resume }); + if (req.resume === undefined) { + return { + suspend: { + correlationId: "corr-1", + kind: "approval", + approvalSnapshot, + }, + }; + } + return { output: { reply: "done", turn: replyTurn } }; + }; + + const repoStore1 = createInMemoryRepoStore(); + const crash = new AbortController(); + const first = runtimeRun( + nestedApprovalParent, + buildEnv({ + parentDef: nestedApprovalParent, + repoStore: repoStore1, + blobs, + signalChannel: createInMemorySignalChannel(), + invokeAction: escalateAction, + invokeStep, + }), + { runId, localAbort: crash.signal }, + ); + await waitForContainerPark(repoStore1, runId, "approval", 1); + // The inner relay committed; the leaf's park and the outer relay did not. + const isPark = (e: WorkflowEvent): boolean => e.kind === "SignalAwaited"; + const parentLog = [...(await repoStore1.read(runId))]; + const outerBodyLog = [...(await repoStore1.read(outerBodyRunId))]; + const leafLog = [...(await repoStore1.read(leafRunId))]; + crash.abort(); + await first.complete; + const repoStore2 = createInMemoryRepoStore(); + await seedChildLog(repoStore2, outerBodyRunId, outerBodyLog); + await seedChildLog( + repoStore2, + leafRunId, + leafLog.slice(0, leafLog.findIndex(isPark)), + ); + const channel2 = createInMemorySignalChannel(); + invocations.length = 0; + const run2 = runtimeRun( + nestedApprovalParent, + buildEnv({ + parentDef: nestedApprovalParent, + repoStore: repoStore2, + blobs, + signalChannel: channel2, + invokeAction: escalateAction, + invokeStep, + readParkedApprovalOps: async (args) => + args.runId === leafRunId && args.stepId === "s" + ? [{ correlationId: "corr-1" }] + : [], + // Only the leaf step stores its approval. + loadParkedApproval: async (args) => + args.runId === leafRunId && args.stepId === "s" + ? approvalSnapshot + : undefined, + }), + { + runId, + resumeFromEvents: parentLog.slice(0, parentLog.findIndex(isPark)), + }, + ); + const ended = run2.complete.then((result) => { + throw new Error(`the run ended ${result.terminalStatus}`); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "outer" && + e.signalName === signalName("corr-1"), + ), + ended, + ]); + await channel2.deliver( + signalName("corr-1"), + { outcome: "approved" }, + "g-1", + ); + const result = await run2.complete; + + expect(result.terminalStatus).toBe("completed"); + expect(invocations).toHaveLength(1); + }); }); function createControllableDrain( diff --git a/packages/workflow/src/runtime/on-trigger-run.test.ts b/packages/workflow/src/runtime/on-trigger-run.test.ts index 8f4c62061..b9be92547 100644 --- a/packages/workflow/src/runtime/on-trigger-run.test.ts +++ b/packages/workflow/src/runtime/on-trigger-run.test.ts @@ -12,7 +12,7 @@ import { describe, test, expect } from "bun:test"; -import { createDefaultDirectorRegistry } from "@intx/agent"; +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; import { signalName } from "@intx/types"; import type { ApprovalSnapshot } from "@intx/types/runtime"; @@ -24,11 +24,16 @@ import { createInMemoryScheduler, createInMemorySignalChannel, createNoopDrainController, + createSpawnLoopIteration, defineWorkflow, + enumerateInlineLoopBodies, + loop, loopBodyRunId, runtimeRun, RuntimeResumeUnsupportedError, sectionBodyRunId, + step, + type LoadParkedApproval, type Primitive, type RepoStore, type SignalChannel, @@ -321,8 +326,9 @@ function midSleepSeed( // A body child's OWN durable log, reduced to a single non-terminal step in the // named phase. `planOnTriggerResume` inspects this to decide whether an -// in-flight body is re-adoptable (a bare sleep parked in `awaiting-timer`) or -// must stay refused (an un-relayed author `awaitSignal`, still `awaiting-signal`). +// in-flight body re-adopts with nothing to relay (a bare sleep parked in +// `awaiting-timer`) or holds a park to relay (an author `awaitSignal`, still +// `awaiting-signal`). function bodyChildLog( childRunId: string, tail: @@ -417,6 +423,97 @@ function buildEnv(args: { }; } +// A body re-spawned from its log and parked on `names`. A resume or a delivered +// signal releases a park, and the body completes once none is left: the +// contract of a real handle, reduced to what the section's side needs. Records +// the spawn and what reached the body. +function parkedBody(names: readonly string[]): { + spawn: SpawnSuspendableChild; + childRunId?: string; + resumeFromEvents?: readonly WorkflowEvent[] | undefined; + resumed?: { correlationId: string; decision: unknown }; + delivered?: { name: string; payload: unknown; signalId: string }; +} { + const open = new Set(names); + const waiters = new Set<() => void>(); + const settled = () => + open.size === 0 + ? ({ kind: "terminal", terminalStatus: "completed" } as const) + : ({ kind: "released" } as const); + const until = (done: () => boolean, abort?: AbortSignal) => + new Promise((resolve, reject) => { + if (done()) return resolve(); + const wake = (): void => { + if (!done()) return; + waiters.delete(wake); + resolve(); + }; + waiters.add(wake); + abort?.addEventListener("abort", () => { + waiters.delete(wake); + reject(new Error("aborted")); + }); + }); + const release = (name: string): void => { + open.delete(name); + for (const wake of [...waiters]) wake(); + }; + const body: ReturnType = { + spawn: async ({ childRunId, resumeFromEvents }) => { + body.childRunId = childRunId; + body.resumeFromEvents = resumeFromEvents; + return { + next: async () => { + await until(() => open.size === 0); + return { kind: "terminal", terminalStatus: "completed" }; + }, + resume: async (correlationId, decision) => { + body.resumed = { correlationId, decision }; + release(signalName(correlationId)); + }, + deliverSignal: async (name, payload, signalId) => { + body.delivered = { name, payload, signalId }; + release(name); + }, + readParkRelease: async (name) => + open.has(name) ? { kind: "open" } : settled(), + waitForParkRelease: async (name, abort) => { + await until(() => !open.has(name), abort); + return settled(); + }, + whenSignalAwaiter: (name) => until(() => open.has(name)), + }; + }, + }; + return body; +} + +// A body that runs until the section is torn down, recording the input each +// spawn of it was given. With `endFirst`, the first spawn ends at once. +function recordingBody( + inputs: unknown[], + opts: { endFirst?: boolean } = {}, +): SpawnSuspendableChild { + return async ({ input, signal }) => { + inputs.push(input); + const ends = opts.endFirst === true && inputs.length === 1; + return { + next: () => + ends + ? Promise.resolve({ kind: "terminal", terminalStatus: "completed" }) + : new Promise((resolve) => { + signal.addEventListener("abort", () => { + resolve({ kind: "terminal", terminalStatus: "cancelled" }); + }); + }), + resume: async () => { + throw new Error("resume on a body that does not park"); + }, + deliverSignal: async () => undefined, + }; + }; +} + // The DEPLOYED form of a section carries a body ref (deploy materializes the // inline body to a workflow asset); build it directly since runOnTrigger runs // the body by ref. @@ -628,25 +725,8 @@ describe("runOnTrigger", () => { await repoStore.appendBatch(runId, seed); await repoStore.appendBatch(bodyRunId, bodyLog); const channel = createInMemorySignalChannel(); - let resumedCorr: string | undefined; - let resumedDecision: unknown; - let spawnedId: string | undefined; - let spawnResume: readonly WorkflowEvent[] | undefined; - const spawnSuspendableChild: SpawnSuspendableChild = async ({ - childRunId, - resumeFromEvents, - }) => { - spawnedId = childRunId; - spawnResume = resumeFromEvents; - return { - next: async () => ({ kind: "terminal", terminalStatus: "completed" }), - resume: async (correlationId, decision) => { - resumedCorr = correlationId; - resumedDecision = decision; - }, - deliverSignal: async () => undefined, - }; - }; + const body = parkedBody([signalName(corr)]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -668,10 +748,12 @@ describe("runOnTrigger", () => { await channel.deliver(signalName(corr), { approved: true }, "grant-1"); await waitForPark(repoStore, runId, "input", 1); - expect(spawnedId).toBe(bodyRunId); - expect(spawnResume).toEqual(bodyLog); - expect(resumedCorr).toBe(corr); - expect(resumedDecision).toEqual({ approved: true }); + expect(body.childRunId).toBe(bodyRunId); + expect(body.resumeFromEvents).toEqual(bodyLog); + expect(body.resumed).toEqual({ + correlationId: corr, + decision: { approved: true }, + }); const log = await repoStore.read(runId); expect(log.filter((e) => e.kind === "ChildCompleted").length).toBe(1); @@ -777,14 +859,13 @@ describe("runOnTrigger", () => { await run.complete.catch(() => undefined); }); - test("refuses a resume when the body is parked on an un-relayed author signal", async () => { - const runId = "sec-authorsig-refuse"; + test("relays an author signal the section never relayed after a restart", async () => { + const runId = "sec-authorsig-fresh"; const seed = midSleepSeed(runId); const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); - // The body child is parked on an author `awaitSignal` gate whose container - // relay await never flushed. Re-adopting it would re-park the body on a - // signal the container never relays -- a hang -- so resume must fail loud. + // The body is parked on an author `awaitSignal` gate whose section relay + // await never flushed, so the resumed section relays it fresh. const bodyRunId = sectionEventRunId(runId, 0); await repoStore.appendBatch( bodyRunId, @@ -794,51 +875,56 @@ describe("runOnTrigger", () => { }), ); const channel = createInMemorySignalChannel(); - let spawned = false; - const spawnSuspendableChild: SpawnSuspendableChild = async () => { - spawned = true; - return { - next: async () => ({ kind: "terminal", terminalStatus: "completed" }), - resume: async () => undefined, - deliverSignal: async () => undefined, - }; - }; + const body = parkedBody(["author-go"]); const def = sectionWorkflow(); - const result = await runtimeRun( + const run = runtimeRun( def, buildEnv({ def, repoStore, signalChannel: channel, - spawnSuspendableChild, + spawnSuspendableChild: body.spawn, }), { runId, resumeFromEvents: seed }, - ).complete; + ); - // Fail loud, not hang: the planner refuses before the body is re-spawned, - // and the section never re-arms. - expect(result.terminalStatus).toBe("failed"); - expect(spawned).toBe(false); - const log = await repoStore.read(runId); - expect( - log.some((e) => e.kind === "SignalAwaited" && e.parkKind === "input"), - ).toBe(false); - const failure = log.find((e) => e.kind === "StepFailed"); - if (failure?.kind !== "StepFailed") - throw new Error("expected a StepFailed"); - expect(failure.error.message).toContain("un-relayed signal"); + const relay = await waitForEvent( + repoStore, + runId, + (e) => + e.kind === "SignalAwaited" && + e.parkKind === "signal-relay" && + e.signalName === "author-go", + ); + if (relay.kind !== "SignalAwaited") throw new Error("expected the relay"); + await channel.deliver( + "author-go", + { ok: true }, + "sig-go", + relay.correlationId, + ); + await waitForPark(repoStore, runId, "input", 1); + + expect(body.delivered).toEqual({ + name: "author-go", + payload: { ok: true }, + signalId: "sig-go", + }); + + await run.cancel("supervisor-operator", "test done"); + await run.complete.catch(() => undefined); }); - test("refuses a resume when the body is parked on an un-relayed control-plane approval", async () => { + test("fails a resume whose un-relayed body approval has no stored snapshot", async () => { const runId = "sec-approval-refuse"; const seed = midSleepSeed(runId); const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); // The body's agent step suspended for approval on a reserved control-plane - // channel, and the container's proxy park never flushed. Like the author - // case, re-adopting it re-parks the body silently on a decision the container - // will never relay -- a hang -- so resume must fail loud, not re-adopt. + // channel, and the container's proxy park never flushed. Relaying it fresh + // needs the snapshot the step stored, and this env reads none, so resume + // fails loud before it re-spawns the body. const bodyRunId = sectionEventRunId(runId, 0); await repoStore.appendBatch( bodyRunId, @@ -879,19 +965,19 @@ describe("runOnTrigger", () => { const failure = log.find((e) => e.kind === "StepFailed"); if (failure?.kind !== "StepFailed") throw new Error("expected a StepFailed"); - expect(failure.error.message).toContain("un-relayed signal"); + expect(failure.error.message).toContain( + "no step state holds its approval snapshot", + ); }); - test("a re-adopted in-flight body whose resume is refused fails loud even under tolerate", async () => { + test("a body handle that rejects fails loud even under tolerate", async () => { const runId = "sec-tolerate-refuse"; const seed = midSleepSeed(runId); const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); - // An in-flight body with no author gate is re-adopted, but its own resume - // classifier refuses (a mid-flight childWorkflow raises - // RuntimeResumeUnsupportedError, which REJECTS the body run). The seam - // re-throws that rejection, so it is loud -- `tolerate` re-arms only on a - // RETURNED failed terminal, never on a rejection. + // A real body run settles a refused resume as `RunFailed`, which `tolerate` + // absorbs. A handle that rejects instead is loud: `tolerate` re-arms only on + // a RETURNED failed terminal, never on a rejection. const bodyRunId = sectionEventRunId(runId, 0); await repoStore.appendBatch( bodyRunId, @@ -938,20 +1024,20 @@ describe("runOnTrigger", () => { // the container reduces to in-flight with awaitingSignal stripped, so the // correlation and decision live only in the log. const seed = midApprovalSeed(runId, corr, { grantDelivered: true }); + const bodyRunId = sectionEventRunId(runId, 0); const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); + await repoStore.appendBatch( + bodyRunId, + bodyChildLog(bodyRunId, { + phase: "awaiting-signal", + signalName: signalName(corr), + }), + ); const channel = createInMemorySignalChannel(); - let resumedCorr: string | undefined; - let resumedDecision: unknown; - const spawnSuspendableChild: SpawnSuspendableChild = async () => ({ - next: async () => ({ kind: "terminal", terminalStatus: "completed" }), - resume: async (correlationId, decision) => { - resumedCorr = correlationId; - resumedDecision = decision; - }, - deliverSignal: async () => undefined, - }); + const body = parkedBody([signalName(corr)]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -970,8 +1056,10 @@ describe("runOnTrigger", () => { // completes and re-arms. await waitForPark(repoStore, runId, "input", 1); - expect(resumedCorr).toBe(corr); - expect(resumedDecision).toEqual({ approved: true }); + expect(body.resumed).toEqual({ + correlationId: corr, + decision: { approved: true }, + }); const log = await repoStore.read(runId); expect(log.filter((e) => e.kind === "ChildCompleted").length).toBe(1); @@ -979,6 +1067,159 @@ describe("runOnTrigger", () => { await run.complete.catch(() => undefined); }); + test.each([ + { + input: "approval grant", + name: signalName("body-corr-lost"), + seed: (runId: string) => + midApprovalSeed(runId, "body-corr-lost", { grantDelivered: true }), + }, + { + input: "signal", + name: "go", + seed: (runId: string) => + midSignalRelaySeed(runId, "go", { signalDelivered: true }), + }, + ])( + "a body whose log is gone after the section took its $input fails instead of waiting for it again", + async ({ name, seed: seedOf }) => { + const runId = "sec-lost-body-input"; + const seed = seedOf(runId); + const repoStore = createInMemoryRepoStore(); + await repoStore.appendBatch(runId, seed); + const body = parkedBody([name]); + + const def = sectionWorkflow(); + const result = await runtimeRun( + def, + buildEnv({ + def, + repoStore, + signalChannel: createInMemorySignalChannel(), + spawnSuspendableChild: body.spawn, + }), + { runId, resumeFromEvents: seed }, + ).complete; + + expect(result.terminalStatus).toBe("failed"); + const failed = result.events.find((e) => e.kind === "StepFailed"); + expect( + failed?.kind === "StepFailed" ? failed.error.message : "", + ).toContain(`lost its log after ${name} was delivered`); + expect(body.childRunId).toBeUndefined(); + }, + ); + + test("a resume relay that cannot go on stops the body before the run fails", async () => { + const runId = "sec-relay-unusable"; + const seed = midApprovalSeed(runId, "body-corr-unusable", { + grantDelivered: false, + }); + const repoStore = createInMemoryRepoStore(); + await repoStore.appendBatch(runId, seed); + let bodySignal: AbortSignal | undefined; + const spawnSuspendableChild: SpawnSuspendableChild = async ({ signal }) => { + bodySignal = signal; + return { + next: async () => { + if (!signal.aborted) { + await new Promise((resolve) => { + signal.addEventListener("abort", resolve, { once: true }); + }); + } + return { kind: "terminal", terminalStatus: "failed" }; + }, + resume: async () => undefined, + deliverSignal: async () => undefined, + // No way to watch the park, so the relay cannot start. + }; + }; + + const def = sectionWorkflow(); + const run = runtimeRun( + def, + buildEnv({ + def, + repoStore, + signalChannel: createInMemorySignalChannel(), + spawnSuspendableChild, + }), + { runId, resumeFromEvents: seed }, + ); + await run.complete.catch(() => undefined); + + expect(bodySignal?.aborted).toBe(true); + const log = await repoStore.read(runId); + expect(log.filter((e) => e.kind === "ChildCompleted")).toMatchObject([ + { terminalStatus: "failed" }, + ]); + expect(log.at(-1)?.kind).toBe("RunFailed"); + }); + + test.each([ + { event: 0, payload: { text: "event-0" } }, + { event: 1, payload: { text: "event-1" } }, + ])( + "a body of event $event whose log is gone starts over with that event's input", + async ({ event, payload }) => { + const runId = `sec-lost-body-${String(event)}`; + const def = sectionWorkflow(); + const repoStore1 = createInMemoryRepoStore(); + const channel1 = createInMemorySignalChannel(); + const run1 = runtimeRun( + def, + buildEnv({ + def, + repoStore: repoStore1, + signalChannel: channel1, + spawnSuspendableChild: recordingBody([], { endFirst: event === 1 }), + }), + { runId, triggerPayload: { text: "event-0" } }, + ); + if (event === 1) { + const input = await waitForPark(repoStore1, runId, "input", 1); + await channel1.deliver(input, payload, "sig-1"); + } + await waitForNthEvent( + repoStore1, + runId, + (e) => e.kind === "ChildSpawned", + event + 1, + ); + // A copy: the store hands out its live log, which the cancel below + // extends. + const parentLog = [...(await repoStore1.read(runId))]; + await run1.cancel("supervisor-operator", "crash"); + await run1.complete.catch(() => undefined); + + // Only the parent log survives the restart. + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(runId, parentLog); + const inputs: unknown[] = []; + const spawned = recordingBody(inputs); + const respawned = Promise.withResolvers(); + const run2 = runtimeRun( + def, + buildEnv({ + def, + repoStore: repoStore2, + signalChannel: createInMemorySignalChannel(), + spawnSuspendableChild: async (args) => { + const handle = await spawned(args); + respawned.resolve(undefined); + return handle; + }, + }), + { runId, resumeFromEvents: parentLog }, + ); + await respawned.promise; + expect(inputs).toEqual([payload]); + + await run2.cancel("supervisor-operator", "test done"); + await run2.complete.catch(() => undefined); + }, + ); + test("advances to the next event with a trigger delivered before the crash spawned its body", async () => { const runId = "sec-resume-input"; const event0 = sectionEventRunId(runId, 0); @@ -1686,35 +1927,9 @@ describe("runOnTrigger", () => { const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); const channel = createInMemorySignalChannel(); - let spawnResume: readonly WorkflowEvent[] | undefined; - let delivered: - | { name: string; payload: unknown; signalId: string } - | undefined; - const spawnSuspendableChild: SpawnSuspendableChild = async ({ - resumeFromEvents, - }) => { - spawnResume = resumeFromEvents; - let release: (() => void) | null = null; - return { - next: async () => { - // The re-spawned body re-parked on "go" silently; block until the - // recovered section re-drives the await and relays. - await new Promise((r) => { - release = r; - }); - return { kind: "terminal", terminalStatus: "completed" }; - }, - resume: async () => undefined, - deliverSignal: async (n, payload, signalId) => { - delivered = { name: n, payload, signalId }; - if (release !== null) { - const r = release; - release = null; - r(); - } - }, - }; - }; + // The re-spawned body re-parked on "go" silently. + const body = parkedBody([name]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -1735,8 +1950,8 @@ describe("runOnTrigger", () => { await waitForPark(repoStore, runId, "input", 1); // This seed does not write a body log, so the body re-spawns from an empty log. - expect(spawnResume).toEqual([]); - expect(delivered).toEqual({ + expect(body.resumeFromEvents).toEqual([]); + expect(body.delivered).toEqual({ name, payload: { text: "late" }, signalId: "sig-2", @@ -1755,6 +1970,43 @@ describe("runOnTrigger", () => { await run.complete.catch(() => undefined); }); + test("a resumed relay whose body left the park is abandoned on its correlation", async () => { + const runId = "sec-relay-resume-closed"; + const name = "go"; + const correlationId = "corr-closed"; + const seed = midSignalRelaySeed(runId, name, { + signalDelivered: false, + correlationId, + }); + const repoStore = createInMemoryRepoStore(); + await repoStore.appendBatch(runId, seed); + // The re-spawned body is no longer parked on "go". + const body = parkedBody([]); + + const def = sectionWorkflow(); + const run = runtimeRun( + def, + buildEnv({ + def, + repoStore, + signalChannel: createInMemorySignalChannel(), + spawnSuspendableChild: body.spawn, + }), + { runId, resumeFromEvents: seed }, + ); + await waitForPark(repoStore, runId, "input", 1); + + const abandoned = (await repoStore.read(runId)).filter( + (e) => e.kind === "SignalAwaitAbandoned", + ); + expect(abandoned).toEqual([ + expect.objectContaining({ signalName: name, correlationId }), + ]); + + await run.cancel("supervisor-operator", "test done"); + await run.complete.catch(() => undefined); + }); + test("reuses the logged correlation when a signal-relay park is still open", async () => { const runId = "sec-relay-resume-corr"; const name = "go"; @@ -1767,29 +2019,8 @@ describe("runOnTrigger", () => { const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); const channel = createInMemorySignalChannel(); - let delivered: - | { name: string; payload: unknown; signalId: string } - | undefined; - const spawnSuspendableChild: SpawnSuspendableChild = async () => { - let release: (() => void) | null = null; - return { - next: async () => { - await new Promise((r) => { - release = r; - }); - return { kind: "terminal", terminalStatus: "completed" }; - }, - resume: async () => undefined, - deliverSignal: async (n, payload, signalId) => { - delivered = { name: n, payload, signalId }; - if (release !== null) { - const r = release; - release = null; - r(); - } - }, - }; - }; + const body = parkedBody([name]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -1809,7 +2040,7 @@ describe("runOnTrigger", () => { await channel.deliver(name, { text: "late" }, "sig-2", correlationId); await waitForPark(repoStore, runId, "input", 1); - expect(delivered).toEqual({ + expect(body.delivered).toEqual({ name, payload: { text: "late" }, signalId: "sig-2", @@ -1848,6 +2079,7 @@ describe("runOnTrigger", () => { const openId = "corr-b"; const event0 = sectionEventRunId(runId, 0); const event1 = sectionEventRunId(runId, 1); + const inputChannel = signalName("input-corr-1"); const seed: WorkflowEvent[] = [ { kind: "RunStarted", @@ -1907,16 +2139,32 @@ describe("runOnTrigger", () => { terminalStatus: "completed", }, { - kind: "ChildSpawned", + kind: "SignalAwaited", seq: 8, at, stepId: "section", + signalName: inputChannel, + parkKind: "input", + }, + { + kind: "SignalReceived", + seq: 9, + at, + signalName: inputChannel, + signalId: "trigger-1", + payload: { text: "event-1" }, + }, + { + kind: "ChildSpawned", + seq: 10, + at, + stepId: "section", childRunId: event1, childDefinitionRef: "body-ref", }, { kind: "SignalAwaited", - seq: 9, + seq: 11, at, stepId: "section", signalName: name, @@ -1928,29 +2176,8 @@ describe("runOnTrigger", () => { const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); const channel = createInMemorySignalChannel(); - let delivered: - | { name: string; payload: unknown; signalId: string } - | undefined; - const spawnSuspendableChild: SpawnSuspendableChild = async () => { - let release: (() => void) | null = null; - return { - next: async () => { - await new Promise((r) => { - release = r; - }); - return { kind: "terminal", terminalStatus: "completed" }; - }, - resume: async () => undefined, - deliverSignal: async (n, payload, signalId) => { - delivered = { name: n, payload, signalId }; - if (release !== null) { - const r = release; - release = null; - r(); - } - }, - }; - }; + const body = parkedBody([name]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -1966,9 +2193,10 @@ describe("runOnTrigger", () => { await channel.deliver(name, { text: "turn-0" }, "sig-turn-0", abandonedId); await channel.deliver(name, { text: "turn-1" }, "sig-turn-1", openId); - await waitForPark(repoStore, runId, "input", 1); + // The seed holds event 1's input park; the second is the re-arm. + await waitForPark(repoStore, runId, "input", 2); - expect(delivered).toEqual({ + expect(body.delivered).toEqual({ name, payload: { text: "turn-1" }, signalId: "sig-turn-1", @@ -2000,46 +2228,23 @@ describe("runOnTrigger", () => { await run.complete.catch(() => undefined); }); - test("relays a signal that landed before the crash relayed it (relay-signal-grant)", async () => { + test("relays a signal that landed before the crash relayed it", async () => { const runId = "sec-relay-grant"; const name = "go"; // The signal's SignalReceived is durable but was never relayed: the // container reduced to in-flight, so the payload lives only in the log. const seed = midSignalRelaySeed(runId, name, { signalDelivered: true }); + const bodyRunId = sectionEventRunId(runId, 0); const repoStore = createInMemoryRepoStore(); await repoStore.appendBatch(runId, seed); + await repoStore.appendBatch( + bodyRunId, + bodyChildLog(bodyRunId, { phase: "awaiting-signal", signalName: name }), + ); const channel = createInMemorySignalChannel(); - let delivered: - | { name: string; payload: unknown; signalId: string } - | undefined; - const spawnSuspendableChild: SpawnSuspendableChild = async () => { - // Model the body's signal channel: `deliverSignal` may land before the - // body's `next()` awaits (the relay recovery delivers before the loop - // pulls next()), so `next()` completes if the signal already arrived. - let release: (() => void) | null = null; - let received = false; - return { - next: async () => { - if (!received) { - await new Promise((r) => { - release = r; - }); - } - return { kind: "terminal", terminalStatus: "completed" }; - }, - resume: async () => undefined, - deliverSignal: async (n, payload, signalId) => { - delivered = { name: n, payload, signalId }; - received = true; - if (release !== null) { - const r = release; - release = null; - r(); - } - }, - }; - }; + const body = parkedBody([name]); + const spawnSuspendableChild = body.spawn; const def = sectionWorkflow(); const run = runtimeRun( @@ -2058,7 +2263,7 @@ describe("runOnTrigger", () => { // original id -- into the re-spawned body, which completes and re-arms. await waitForPark(repoStore, runId, "input", 1); - expect(delivered).toEqual({ + expect(body.delivered).toEqual({ name, payload: { text: "delivered" }, signalId: "relay-1", @@ -2070,6 +2275,208 @@ describe("runOnTrigger", () => { await run.complete.catch(() => undefined); }); + test("relays a signal that landed before the crash into a body whose park is a loop's relay", async () => { + const runId = "sec-nested-relay"; + const awaitBody = defineWorkflow({ + id: "sec-await-body", + trigger: { type: "manual" }, + steps: { hold: awaitSignal({ name: "go" }) }, + }); + const loopBody = defineWorkflow({ + id: "sec-loop-body", + trigger: { type: "manual" }, + steps: { + inner: loop({ + body: awaitBody, + while: "cont", + carry: "same", + input: { literal: 0 }, + maxIterations: 1, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["inner"] }), + }, + }); + const def = sectionWorkflow(); + // The in-process loop spawner stands in for the host's section-body + // spawner: both drive the body through `createSuspendableChildHandle`. + const nestedEnv = (repoStore: RepoStore): WorkflowRuntimeEnv => { + const env = buildEnv({ + def, + repoStore, + signalChannel: createInMemorySignalChannel(), + spawnSuspendableChild: async () => { + throw new Error("replaced below"); + }, + }); + env.invokeAction = async () => ({ output: "escalated" }); + env.loopFns = (ref) => + ref === "cont" ? () => false : (_o, currentInput) => currentInput; + env.spawnLoopIteration = createSpawnLoopIteration( + env, + new Map( + enumerateInlineLoopBodies(loopBody).map((b) => [b.ref, b.definition]), + ), + ); + env.spawnSuspendableChild = createSpawnLoopIteration( + env, + new Map([["body-ref", loopBody]]), + ); + return env; + }; + + const repoStore1 = createInMemoryRepoStore(); + void runtimeRun(def, nestedEnv(repoStore1), { + runId, + triggerPayload: { text: "event-0" }, + }).complete.catch(() => undefined); + const relayName = await waitForPark(repoStore1, runId, "signal-relay", 1); + const bodyRunId = sectionEventRunId(runId, 0); + const innerBodyRunId = loopBodyRunId(bodyRunId, "inner", 0); + const parkedLog = await repoStore1.read(runId); + const bodyLog = await repoStore1.read(bodyRunId); + const innerBodyLog = await repoStore1.read(innerBodyRunId); + + // The section took the signal, and the crash landed before it reached the + // body, whose own park is its loop's relay. + const delivered: WorkflowEvent = { + kind: "SignalReceived", + seq: (parkedLog[parkedLog.length - 1]?.seq ?? 0) + 1, + at, + signalName: relayName, + signalId: "sig-pre", + payload: { done: true }, + correlationId: relayCorrelationId(parkedLog), + }; + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(bodyRunId, bodyLog); + await repoStore2.appendBatch(innerBodyRunId, innerBodyLog); + const run = runtimeRun(def, nestedEnv(repoStore2), { + runId, + resumeFromEvents: [...parkedLog, delivered], + }); + + await waitForPark(repoStore2, runId, "input", 1); + const innerReceived = (await repoStore2.read(innerBodyRunId)).filter( + (e) => e.kind === "SignalReceived", + ); + expect(innerReceived).toEqual([ + expect.objectContaining({ signalName: "go", signalId: "sig-pre" }), + ]); + + await run.cancel("supervisor-operator", "test done"); + await run.complete.catch(() => undefined); + }); + + test("a resumed section relays a later event's park fresh, not the signal an earlier event took", async () => { + const runId = "sec-since-spawn"; + const body = defineWorkflow({ + id: "since-spawn-body", + trigger: { type: "manual" }, + steps: { + gate: awaitSignal({ name: "author-go" }), + record: action({ + handler: "record", + after: ["gate"], + input: { from: "steps.gate.output" }, + }), + }, + }); + const def = sectionWorkflow(); + const sectionEnv = ( + repoStore: RepoStore, + signalChannel: SignalChannel, + recorded: { runId: string | undefined; input: unknown }[], + ): WorkflowRuntimeEnv => { + const env = buildEnv({ + def, + repoStore, + signalChannel, + spawnSuspendableChild: async () => { + throw new Error("replaced below"); + }, + }); + env.invokeAction = async (req) => { + recorded.push({ runId: req.authzContext.runId, input: req.input }); + return { output: null }; + }; + env.spawnSuspendableChild = createSpawnLoopIteration( + env, + new Map([["body-ref", body]]), + ); + return env; + }; + const isRelay = (e: WorkflowEvent): boolean => + e.kind === "SignalAwaited" && + e.parkKind === "signal-relay" && + e.signalName === "author-go"; + const isInput = (e: WorkflowEvent): boolean => + e.kind === "SignalAwaited" && e.parkKind === "input"; + const body1 = sectionEventRunId(runId, 1); + + const repoStore1 = createInMemoryRepoStore(); + const channel1 = createInMemorySignalChannel(); + const run1 = runtimeRun(def, sectionEnv(repoStore1, channel1, []), { + runId, + triggerPayload: { text: "event-0" }, + }); + const relay0 = await waitForNthEvent(repoStore1, runId, isRelay, 1); + if (relay0.kind !== "SignalAwaited") throw new Error("expected a relay"); + await channel1.deliver( + "author-go", + { n: 1 }, + "sig-1", + relay0.correlationId, + ); + const input1 = await waitForNthEvent(repoStore1, runId, isInput, 1); + if (input1.kind !== "SignalAwaited") throw new Error("expected an input"); + await channel1.deliver(input1.signalName, { text: "event-1" }, "ev-1"); + await waitForEvent( + repoStore1, + body1, + (e) => e.kind === "SignalAwaited" && e.signalName === "author-go", + ); + const relay1 = await waitForNthEvent(repoStore1, runId, isRelay, 2); + // Event 1's body parked on the name event 0's body took a signal on, and + // the crash landed before the section's relay for event 1 flushed. + const parentLog = (await repoStore1.read(runId)).filter( + (e) => e.seq < relay1.seq, + ); + const bodyLog = await repoStore1.read(body1); + + const recorded2: { runId: string | undefined; input: unknown }[] = []; + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(runId, parentLog); + await repoStore2.appendBatch(body1, bodyLog); + const channel2 = createInMemorySignalChannel(); + const run2 = runtimeRun(def, sectionEnv(repoStore2, channel2, recorded2), { + runId, + resumeFromEvents: parentLog, + }); + // A resume that hands event 1's body event 0's signal re-arms on input + // without relaying again. + const outcome = await Promise.race([ + waitForNthEvent(repoStore2, runId, isRelay, 2), + waitForNthEvent(repoStore2, runId, isInput, 2), + ]); + if (isRelay(outcome) && outcome.kind === "SignalAwaited") { + await channel2.deliver( + "author-go", + { n: 2 }, + "sig-2", + outcome.correlationId, + ); + await waitForNthEvent(repoStore2, runId, isInput, 2); + } + + expect(recorded2).toEqual([{ runId: body1, input: { n: 2 } }]); + + await run2.cancel("supervisor-operator", "test done"); + await run2.complete.catch(() => undefined); + await run1.cancel("supervisor-operator", "test done"); + await run1.complete.catch(() => undefined); + }); + test("two parents on one store run distinct event-0 bodies after the first log is terminal", async () => { const parentA = "run_parent_a"; const parentB = "run_parent_b"; @@ -2422,6 +2829,365 @@ describe("runOnTrigger", () => { }); }); +describe("runOnTrigger queued body parks", () => { + const agent = defineAgent({ + id: "gate-agent", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }); + // A body with two parallel approval steps: the section relays one, and the + // other waits for it in memory. + const body = defineWorkflow({ + id: "two-approval-section-body", + trigger: { type: "manual" }, + steps: { a: step({ agent }), b: step({ agent }) }, + }); + + function realBodyEnv(args: { + def: WorkflowDefinition; + repoStore: RepoStore; + signalChannel: SignalChannel; + invokeStep: StepInvoker; + loadParkedApproval?: LoadParkedApproval; + body?: WorkflowDefinition; + }): WorkflowRuntimeEnv { + const env: WorkflowRuntimeEnv = { + ...buildEnv({ + def: args.def, + repoStore: args.repoStore, + signalChannel: args.signalChannel, + spawnSuspendableChild: async () => { + throw new Error("replaced below"); + }, + }), + invokeStep: args.invokeStep, + ...(args.loadParkedApproval !== undefined + ? { loadParkedApproval: args.loadParkedApproval } + : {}), + }; + env.spawnSuspendableChild = createSpawnLoopIteration( + env, + new Map([["body-ref", args.body ?? body]]), + ); + return env; + } + + // A body whose timed approval step routes to a handler that parks again. + const timedBody = (timeout: number): WorkflowDefinition => + defineWorkflow({ + id: "timed-approval-section-body", + trigger: { type: "manual" }, + steps: { + timed: step({ agent, timeout, onFailure: "handler" }), + handler: step({ agent, after: ["timed"] }), + }, + }); + + function storingApprovals(): { + invokeStep: StepInvoker; + loadParkedApproval: LoadParkedApproval; + } { + const stored = new Map(); + const key = (args: { + runId?: string; + stepId?: string; + attempt?: number; + correlationId: string; + }): string => + `${String(args.runId)}/${String(args.stepId)}/${String(args.attempt)}/${args.correlationId}`; + return { + invokeStep: async (req) => { + if (req.resume !== undefined) { + return { + output: { + reply: "done", + turn: { + role: "assistant", + content: [{ type: "text", text: "done" }], + timestamp: 0, + }, + }, + }; + } + const correlationId = `corr-${String(req.authzContext.stepId)}`; + const approvalSnapshot: ApprovalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: { step: req.authzContext.stepId }, + }; + stored.set( + key({ ...req.authzContext, correlationId }), + approvalSnapshot, + ); + return { + suspend: { correlationId, kind: "approval", approvalSnapshot }, + }; + }, + loadParkedApproval: async (args) => stored.get(key(args)), + }; + } + + test("retires an approval the body left before a restart and relays the one it holds", async () => { + const runId = "sec-left-approval"; + const def = sectionWorkflow(); + const approvals = storingApprovals(); + const left = signalName("corr-timed"); + const held = signalName("corr-handler"); + + const repoStore1 = createInMemoryRepoStore(); + const run1 = runtimeRun( + def, + realBodyEnv({ + def, + repoStore: repoStore1, + signalChannel: createInMemorySignalChannel(), + body: timedBody(20), + ...approvals, + }), + { runId, triggerPayload: { text: "event-0" } }, + ); + const bodyRunId = sectionEventRunId(runId, 0); + expect(await waitForPark(repoStore1, runId, "approval", 1)).toBe(left); + await waitForEvent( + repoStore1, + bodyRunId, + (e) => e.kind === "SignalAwaited" && e.signalName === held, + ); + const parentLog = [...(await repoStore1.read(runId))]; + const childLog = [...(await repoStore1.read(bodyRunId))]; + await run1.cancel("supervisor-operator", "crash"); + await run1.complete.catch(() => undefined); + + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(bodyRunId, childLog); + const channel2 = createInMemorySignalChannel(); + const run2 = runtimeRun( + def, + realBodyEnv({ + def, + repoStore: repoStore2, + signalChannel: channel2, + body: timedBody(20), + ...approvals, + }), + { runId, resumeFromEvents: parentLog }, + ); + // A run that ends instead of relaying says why. + const ended = run2.complete.then((result) => { + const failed = result.events.find((e) => e.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "section" && + e.signalName === held, + ), + ended, + ]); + expect( + (await repoStore2.read(runId)).some( + (e) => e.kind === "SignalAwaitAbandoned" && e.signalName === left, + ), + ).toBe(true); + await channel2.deliver(held, { outcome: "approved" }, "g-held"); + // The body completed, so the section re-arms for the next event. + await Promise.race([waitForPark(repoStore2, runId, "input", 1), ended]); + + await run2.cancel("supervisor-operator", "test done"); + await run2.complete.catch(() => undefined); + }); + + test("retires an approval the body leaves while its relay waits after a restart", async () => { + const runId = "sec-leaves-approval"; + const def = sectionWorkflow(); + const approvals = storingApprovals(); + const left = signalName("corr-timed"); + const held = signalName("corr-handler"); + + const repoStore1 = createInMemoryRepoStore(); + const run1 = runtimeRun( + def, + realBodyEnv({ + def, + repoStore: repoStore1, + signalChannel: createInMemorySignalChannel(), + body: timedBody(60_000), + ...approvals, + }), + { runId, triggerPayload: { text: "event-0" } }, + ); + const bodyRunId = sectionEventRunId(runId, 0); + expect(await waitForPark(repoStore1, runId, "approval", 1)).toBe(left); + const parentLog = [...(await repoStore1.read(runId))]; + const childLog = [...(await repoStore1.read(bodyRunId))]; + await run1.cancel("supervisor-operator", "crash"); + await run1.complete.catch(() => undefined); + + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(bodyRunId, childLog); + const channel2 = createInMemorySignalChannel(); + // The deadline starts over with the process, so this one fires under the + // re-established relay. + const run2 = runtimeRun( + def, + realBodyEnv({ + def, + repoStore: repoStore2, + signalChannel: channel2, + body: timedBody(20), + ...approvals, + }), + { runId, resumeFromEvents: parentLog }, + ); + const ended = run2.complete.then((result) => { + const failed = result.events.find((e) => e.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "section" && + e.signalName === held, + ), + ended, + ]); + expect( + (await repoStore2.read(runId)).some( + (e) => e.kind === "SignalAwaitAbandoned" && e.signalName === left, + ), + ).toBe(true); + await channel2.deliver(held, { outcome: "approved" }, "g-held"); + await Promise.race([waitForPark(repoStore2, runId, "input", 1), ended]); + + await run2.cancel("supervisor-operator", "test done"); + await run2.complete.catch(() => undefined); + }); + + test("relays an approval the section had not reached after a restart", async () => { + const runId = "sec-two-approvals"; + const stored = new Map(); + const key = (args: { + runId?: string; + stepId?: string; + attempt?: number; + correlationId: string; + }): string => + `${String(args.runId)}/${String(args.stepId)}/${String(args.attempt)}/${args.correlationId}`; + const invokeStep: StepInvoker = async (req) => { + if (req.resume !== undefined) { + return { + output: { + reply: "done", + turn: { + role: "assistant", + content: [{ type: "text", text: "done" }], + timestamp: 0, + }, + }, + }; + } + const correlationId = `corr-${String(req.authzContext.stepId)}`; + const approvalSnapshot: ApprovalSnapshot = { + name: "gate", + description: "gate", + inputSchema: { type: "object" }, + arguments: { step: req.authzContext.stepId }, + }; + stored.set(key({ ...req.authzContext, correlationId }), approvalSnapshot); + return { + suspend: { correlationId, kind: "approval", approvalSnapshot }, + }; + }; + const def = sectionWorkflow(); + + const repoStore1 = createInMemoryRepoStore(); + const env1 = realBodyEnv({ + def, + repoStore: repoStore1, + signalChannel: createInMemorySignalChannel(), + invokeStep, + }); + void runtimeRun(def, env1, { + runId, + triggerPayload: { text: "event-0" }, + }).complete.catch(() => undefined); + const bodyRunId = sectionEventRunId(runId, 0); + const shown = await waitForPark(repoStore1, runId, "approval", 1); + for (const stepId of ["a", "b"]) { + await waitForEvent( + repoStore1, + bodyRunId, + (e) => + e.kind === "SignalAwaited" && + e.signalName === signalName(`corr-${stepId}`), + ); + } + const parentLog = await repoStore1.read(runId); + const childLog = await repoStore1.read(bodyRunId); + const queued = + shown === signalName("corr-a") + ? signalName("corr-b") + : signalName("corr-a"); + + const repoStore2 = createInMemoryRepoStore(); + await repoStore2.appendBatch(bodyRunId, childLog); + const channel2 = createInMemorySignalChannel(); + const run2 = runtimeRun( + def, + realBodyEnv({ + def, + repoStore: repoStore2, + signalChannel: channel2, + invokeStep, + loadParkedApproval: async (args) => stored.get(key(args)), + }), + { runId, resumeFromEvents: parentLog }, + ); + await channel2.deliver(shown, { outcome: "approved" }, "g-shown"); + // A run that ends instead of relaying says why. + const ended = run2.complete.then((result) => { + const failed = result.events.find((e) => e.kind === "StepFailed"); + throw new Error( + `the run ended ${result.terminalStatus}: ${failed?.kind === "StepFailed" ? failed.error.message : "no step failed"}`, + ); + }); + void ended.catch(() => undefined); + await Promise.race([ + waitForEvent( + repoStore2, + runId, + (e) => + e.kind === "SignalAwaited" && + e.stepId === "section" && + e.signalName === queued, + ), + ended, + ]); + await channel2.deliver(queued, { outcome: "approved" }, "g-queued"); + // The body completed, so the section re-arms for the next event. + await Promise.race([waitForPark(repoStore2, runId, "input", 1), ended]); + + await run2.cancel("supervisor-operator", "test done"); + await run2.complete.catch(() => undefined); + }); +}); + describe("runOnTrigger onBodyFailure: tolerate", () => { function toleratingSection(): WorkflowDefinition { const section: Primitive = { @@ -2489,6 +3255,125 @@ describe("runOnTrigger onBodyFailure: tolerate", () => { await run.complete.catch(() => undefined); }); + test("a body that ends under a resumed approval relay retires it, so the section can park again", async () => { + const runId = "sec-tolerate-relay-ended"; + const approval = signalName("body-corr-ended"); + const seed = midApprovalSeed(runId, "body-corr-ended", { + grantDelivered: false, + }); + const repoStore = createInMemoryRepoStore(); + await repoStore.appendBatch(runId, seed); + const channel = createInMemorySignalChannel(); + let spawn = 0; + const spawnSuspendableChild: SpawnSuspendableChild = async () => { + const first = spawn++ === 0; + const terminalStatus = first ? "failed" : "completed"; + return { + next: async () => ({ kind: "terminal", terminalStatus }), + resume: async () => undefined, + deliverSignal: async () => undefined, + readParkRelease: async () => + first ? { kind: "terminal", terminalStatus } : { kind: "open" }, + waitForParkRelease: async () => ({ + kind: "terminal", + terminalStatus, + }), + }; + }; + const def = toleratingSection(); + const run = runtimeRun( + def, + buildEnv({ + def, + repoStore, + signalChannel: channel, + spawnSuspendableChild, + }), + { runId, resumeFromEvents: seed }, + ); + + const next = await waitForPark(repoStore, runId, "input", 1); + expect(next).not.toBe(approval); + const log = await repoStore.read(runId); + const retired = log.findIndex( + (e) => e.kind === "SignalAwaitAbandoned" && e.signalName === approval, + ); + const ended = log.findIndex( + (e) => e.kind === "ChildCompleted" && e.terminalStatus === "failed", + ); + expect(retired).toBeGreaterThan(-1); + expect(ended).toBeGreaterThan(retired); + + await channel.deliver(next, { text: "event-1" }, "sig-1"); + await waitForPark(repoStore, runId, "input", 2); + expect( + (await repoStore.read(runId)).filter((e) => e.kind === "ChildSpawned"), + ).toHaveLength(2); + + await run.cancel("supervisor-operator", "test done"); + await run.complete.catch(() => undefined); + }); + + test("a body that ended before a crash retires the approval relay left open, so the section can park again", async () => { + const runId = "sec-tolerate-ended-before-crash"; + const approval = signalName("body-corr-crash"); + const seed = midApprovalSeed(runId, "body-corr-crash", { + grantDelivered: false, + }); + const repoStore = createInMemoryRepoStore(); + await repoStore.appendBatch(runId, seed); + const bodyRunId = sectionEventRunId(runId, 0); + const bodyLog = bodyChildLog(bodyRunId, { phase: "in-flight" }); + await repoStore.appendBatch(bodyRunId, [ + ...bodyLog, + { + kind: "RunFailed", + seq: bodyLog.length + 1, + at, + error: { message: "the body's step timed out" }, + }, + ]); + const channel = createInMemorySignalChannel(); + let spawn = 0; + const spawnSuspendableChild: SpawnSuspendableChild = async () => { + const terminalStatus = spawn++ === 0 ? "failed" : "completed"; + return { + next: async () => ({ kind: "terminal", terminalStatus }), + resume: async () => undefined, + deliverSignal: async () => undefined, + }; + }; + const def = toleratingSection(); + const run = runtimeRun( + def, + buildEnv({ + def, + repoStore, + signalChannel: channel, + spawnSuspendableChild, + }), + { runId, resumeFromEvents: seed }, + ); + + const next = await waitForPark(repoStore, runId, "input", 1); + expect(next).not.toBe(approval); + const log = await repoStore.read(runId); + const retired = log.findIndex( + (e) => e.kind === "SignalAwaitAbandoned" && e.signalName === approval, + ); + const ended = log.findIndex( + (e) => e.kind === "ChildCompleted" && e.terminalStatus === "failed", + ); + expect(retired).toBeGreaterThan(-1); + expect(ended).toBeGreaterThan(retired); + + await channel.deliver(next, { text: "event-1" }, "sig-1"); + await waitForPark(repoStore, runId, "input", 2); + + await run.cancel("supervisor-operator", "test done"); + await run.complete.catch(() => undefined); + }); + test("a cancelled body still ends the section even under tolerate", async () => { const runId = "sec-tolerate-cancel"; const repoStore = createInMemoryRepoStore(); diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index 1d0e1c419..e67e4f7e1 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -64,6 +64,7 @@ import { runAfterDiscardingBuffer, } from "./commit-chain"; import type { + ParkedApprovalOp, RunResult, RuntimeWorkflowRun, SpawnSuspendableChild, @@ -763,6 +764,10 @@ async function executeRunBody( correlationId: string; timeoutAtMs?: number; }[] = []; + // Read once, and only for a step that stored an approval park. + let runLog: Promise | undefined; + const readRunLog = (): Promise => + (runLog ??= env.repoStore.read(runId)); if (state.phase === "running") { for (const [stepId, stepState] of state.steps) { // Resumable carve-outs, each re-offered by `nextSchedulable` on the @@ -807,38 +812,14 @@ async function executeRunBody( // loop -- committing inline would leave `state` stale and merely // relocate the stall to the main loop. if (residual === "crashed-invocation") { - const parkedOps = - env.readParkedApprovalOps !== undefined - ? await env.readParkedApprovalOps({ - runId, - stepId, - attempt: stepState.currentAttempt, - }) - : []; - // A step-attempt parks on at most one control-plane suspension. More - // than one durable pending approval op is a corrupt store, not a state - // the runtime can reconcile by picking one -- fail loud. - if (parkedOps.length > 1) { - throw new Error( - `crashed step ${stepId} (attempt ${String(stepState.currentAttempt)}) has ${String(parkedOps.length)} durable pending approval operations; a step-attempt parks on at most one control-plane suspension`, - ); - } - const parked = parkedOps[0]; - // A durable op whose channel this step already awaited in the log is - // stale, not a lost park: that park flushed and a delivery resumed the - // step, whose reactor consumed the op before the store holding it - // caught up (the committed copy of a step that crashed mid-turn trails - // its local store). Re-parking would wait on a decision already spent, - // so the step is a genuine crash mid-turn. - if ( - parked !== undefined && - !(await stepAwaitedSignal( - env, - runId, - stepId, - signalName(parked.correlationId), - )) - ) { + const parked = await lostApprovalPark( + env, + runId, + stepId, + stepState.currentAttempt, + readRunLog, + ); + if (parked !== undefined) { recoverableParks.push({ stepId, attempt: stepState.currentAttempt, @@ -888,10 +869,7 @@ async function executeRunBody( // channel reusing the original correlationId, and -- because the re-park finds // the step already `awaiting-signal` -- it does not re-fire `onPark`; the // correlation re-registers exactly once through the durable-state re-emit. The - // agent turn is never re-invoked, preserving at-most-once. The reactor stores - // `timeoutAt` as epoch ms; `SignalAwaited` carries it as an ISO string, so - // convert, and omit the field entirely for the indefinite-hold parks that are - // the norm. + // agent turn is never re-invoked, preserving at-most-once. // // This recovery is APPROVAL-only: `recoverableParks` come from the reactor's // durable approval pending-op store, which reconstructs an approval park. An @@ -919,17 +897,14 @@ async function executeRunBody( markRepoWriteFailure(cause); throw cause; } - const awaited: WorkflowEvent = { - kind: "SignalAwaited", - seq: state.lastSeq + 1, - at: env.clock().toISOString(), - stepId, - signalName: signalName(correlationId), - ...(timeoutAtMs !== undefined - ? { timeoutAt: new Date(timeoutAtMs).toISOString() } - : {}), - }; - state = await commitDurable(env, runId, awaited); + state = await commitDurable( + env, + runId, + lostParkAwaited(state.lastSeq + 1, env.clock().toISOString(), stepId, { + correlationId, + ...(timeoutAtMs !== undefined ? { timeoutAtMs } : {}), + }), + ); } // Settle each crashed-mid-invocation step as a terminal `StepFailed` @@ -2430,9 +2405,11 @@ async function runLoop( // Read the log once for cursor re-derivation and input reconstruction. // reloadState reflects durable + this-process buffer, so every "already // emitted?" check below is against everything committed so far; a - // re-emit of anything reflected is refused by the state machine. - const log = await env.repoStore.read(runId); + // re-emit of anything reflected is refused by the state machine. The state + // goes first: input landing between the reads would otherwise read as taken + // in the state and be missing from the log a plan hands it on from. let state = await reloadState(env, runId); + const log = await env.repoStore.read(runId); // The loop container is in state.steps only on resume; a fresh run // emits its StepStarted, a resumed run must not (re-emit throws). @@ -2451,9 +2428,9 @@ async function runLoop( // count the attempt, so the frontier below is the attempt planLoopResume // must re-link. A terminal attempt the policy will not continue throws // here, before that planner runs and before any drive: planLoopResume - // returns undefined for a terminal child, and occurrenceResume is cleared + // returns no relays for a terminal child, and occurrenceRelays is cleared // after the first drive, so driving the terminal attempt would spawn the - // later park with no resume token. + // later park with nothing to re-link it. const iterationZeroInput = primitive.input !== undefined ? evaluate(primitive.input, selectorCtx) @@ -2528,12 +2505,12 @@ async function runLoop( } // A crash-recovered iteration whose body was parked or mid-relay needs an - // active re-link on its FIRST drive below. planLoopResume yields that token - // (or undefined when the forward drive can re-adopt the iteration from the - // child's own durable log). Cleared after the recovered iteration is driven - // so every later iteration spawns fresh. - let occurrenceResume = terminated - ? undefined + // active re-link on its FIRST drive below. planLoopResume yields those relays + // (none when the forward drive can re-adopt the iteration from the child's + // own durable log). Cleared after the recovered iteration is driven so + // every later iteration spawns fresh. + let occurrenceRelays = terminated + ? [] : await planLoopResume(env, primitive, runId, state, log, iteration); let iterations = iteration; @@ -2552,7 +2529,7 @@ async function runLoop( // commits ChildCompleted. A non-suspending body drives straight to // terminal; a body that parks in-process (e.g. an agent step on an approval // gate, or an author `awaitSignal`) is serviced by the park proxy above - // without the body reaching a terminal. `occurrenceResume` is non-undefined + // without the body reaching a terminal. `occurrenceRelays` is non-empty // only on the recovered iteration of a crash resume, where it re-links a // body parked or mid-relay at the crash; it is cleared after this drive so // every later iteration spawns fresh. @@ -2560,14 +2537,14 @@ async function runLoop( childRunId, bodyRef, input: currentInput, - resume: occurrenceResume, + relays: occurrenceRelays, spawnSuspendableChild: spawnLoopIteration, depth, maxChildSpawnDepth, abort, signalRelayAddressing: "name", }); - occurrenceResume = undefined; + occurrenceRelays = []; // Trust the recorded child, not only the drive's return. A child log // that is missing is re-executed, and that fresh terminal can disagree @@ -2654,29 +2631,6 @@ async function runLoop( return output; } -/** - * Re-link state for a crash-recovered occurrence whose body is parked. The - * resume planner yields at most one of these -- an approval park or a - * signal-relay park, never both -- so a single discriminated union encodes the - * mutual exclusion the two former locals maintained by discipline. - */ -type SuspendableOccurrenceResume = - | { kind: "approval"; corr: string; relay: boolean; decision?: unknown } - | { kind: "signal-relay-reestablish"; name: string; awaitSeq: number } - | { - kind: "signal-relay-relay"; - name: string; - payload: unknown; - signalId: string; - } - // The body is parked on an author `awaitSignal` gate, but the container never - // emitted its relay await before the crash (in-flight, no durable relay). The - // re-adopted body re-parks silently, so there is no await to reestablish and - // no `onSignalPark` to surface it -- drive the container relay FRESH over the - // recovered name instead. `timeoutAt` is the body's deadline, when the gate - // is timed, so the fresh relay stays timed. - | { kind: "signal-relay-drive-fresh"; name: string; timeoutAt?: string }; - type ChildRelayPull = "next" | "hold"; type ChildDriveResult = @@ -2716,7 +2670,8 @@ async function driveSuspendableOccurrence( childRunId: string; bodyRef: string; input: unknown; - resume: SuspendableOccurrenceResume | undefined; + // A recovered body's relays, run before its queue is read. + relays: readonly ChildRelay[]; spawnSuspendableChild: SpawnSuspendableChild; depth: number; maxChildSpawnDepth: number; @@ -2728,7 +2683,7 @@ async function driveSuspendableOccurrence( childRunId, bodyRef, input, - resume, + relays, spawnSuspendableChild, depth, maxChildSpawnDepth, @@ -2753,19 +2708,24 @@ async function driveSuspendableOccurrence( // parent log records the spawn ahead of any child-side work. await flush(env, runId); + const bodyLog = + relays.length > 0 ? await env.repoStore.read(childRunId) : undefined; + const bodyTeardown = new AbortController(); const child = await spawnSuspendableChild({ definitionRef: bodyRef, childRunId, input, parentRunId: runId, parentStepId: containerStepId, - signal: abort, + signal: AbortSignal.any([abort, bodyTeardown.signal]), depth, maxChildSpawnDepth, - ...(resume !== undefined - ? { resumeFromEvents: await env.repoStore.read(childRunId) } - : {}), + ...(bodyLog !== undefined ? { resumeFromEvents: bodyLog } : {}), }); + // A body whose log is gone starts over and surfaces its gates afresh, and + // the relays below already carry out the first park on each name. An approval + // comes back under a new id, so its relay matches nothing here. + const carriedOut = bodyLog?.length === 0 ? relays.map(relayName) : []; let terminalStatus: "completed" | "failed" | "cancelled"; // `pending` carries a body event already pulled by a signal-relay drive @@ -2773,79 +2733,59 @@ async function driveSuspendableOccurrence( // than calling `next()` a second time and dropping it. let pending: ChildDriveResult | undefined; - if (resume?.kind === "approval") { - // Re-link the parent to a body re-spawned from its log and parked on the - // shared correlation. A re-park does not re-fire onPark, so the park is - // not surfaced via next(); drive the resume directly from the recovered - // correlation. A grant already delivered to the parent log (its - // SignalReceived consumed the container's park) is relayed as-is; - // otherwise re-park the container and await the grant as the steady-state - // loop would. - let decision: unknown; - if (resume.relay) { - decision = resume.decision; - } else { - const parkRearm = await reloadState(env, runId); - decision = await parkOnSignal( - env, - runId, - { - stepId: containerStepId, - signalName: signalName(resume.corr), - parkKind: "approval", - }, - parkRearm, - abort, - ); + // A resumed body does not surface the parks already in its log, so each is + // relayed here, one at a time, before the queue is read. + let relayedTerminal: "completed" | "failed" | "cancelled" | undefined; + try { + for (const relay of relays) { + let driven: ChildDriveResult; + try { + driven = await drivePlannedChildRelay( + env, + runId, + containerStepId, + child, + relay, + abort, + new Set(), + signalRelayAddressing, + ); + } catch (cause) { + if ( + !(cause instanceof ChildRelayRetiredError) || + cause.parkKind !== "approval" + ) { + throw cause; + } + // The body left this approval and goes on, so the container does too. + await abandonRelayIfOpen(env, runId, containerStepId, cause.parkName); + continue; + } + if (driven.kind === "terminal") { + relayedTerminal = driven.terminalStatus; + break; + } + if (driven.kind !== "held") { + throw new Error( + `suspendable body ${childRunId} resume relay returned ${driven.kind}; a held relay does not read the queue`, + ); + } } - await child.resume(resume.corr, decision); - } else if (resume?.kind === "signal-relay-relay") { - // A signal delivered before the crash but not relayed: deliver it (with - // its original id, so the body's dedup makes it idempotent) to unblock - // the body's gate; the loop then drives the body's next event. - await child.deliverSignal(resume.name, resume.payload, resume.signalId); - } else if (resume?.kind === "signal-relay-reestablish") { - // The container's signal-relay await is durable; the re-spawned body - // re-parks on the name silently, so re-drive the race from the recovered - // await seq (no re-emit) and continue with the body event it yields. - const relayLog = await env.repoStore.read(runId); - pending = await raceContainerSignalRelay( - env, - runId, - containerStepId, - child, - resume.name, - resume.awaitSeq, - correlationIdAt(relayLog, resume.awaitSeq), - abort, - "next", - ); - } else if (resume?.kind === "signal-relay-drive-fresh") { - // The container never emitted its relay await before the crash, so there is - // nothing to reestablish and the re-adopted body re-parks silently (no - // onSignalPark). Drive the container relay FRESH over the recovered name -- - // emit the relay await and race -- so the container ends up awaiting the - // signal that will arrive, and continue with the body event it yields. - // The body's deadline rides along, so the fresh relay stays timed when - // the gate is timed. - const freshTimeout = remainingSignalTimeout( - resume.timeoutAt, - env.clock().getTime(), - ); - pending = await driveContainerSignalRelay( - env, - runId, - containerStepId, - child, - resume.name, - signalRelayAddressing, - abort, - "next", - freshTimeout, - ); + } catch (cause) { + // The run fails, so the body must not go on working. An abort stops it + // already, and this close writes no `abortedTeardown`, so a resume under + // `tolerate` would take the abort for a real failure. + if (!abort.aborted) { + await closeFailedChild(env, runId, childRunId, bodyTeardown, child); + } + throw cause; } for (;;) { + if (relayedTerminal !== undefined) { + terminalStatus = relayedTerminal; + break; + } const bodyEvent = pending ?? (await waitIdle((resume) => child.next(resume))); pending = undefined; @@ -2858,6 +2798,17 @@ async function driveSuspendableOccurrence( `suspendable body ${childRunId} drive held a park; this drive reads the queue`, ); } + if (bodyEvent.kind === "park" || bodyEvent.kind === "signal-park") { + const carried = carriedOut.indexOf( + bodyEvent.kind === "park" + ? signalName(bodyEvent.park.correlationId) + : bodyEvent.name, + ); + if (carried !== -1) { + carriedOut.splice(carried, 1); + continue; + } + } if (bodyEvent.kind === "park") { // A body step parked on an approval. Proxy it up on the SAME // correlation via THIS run's own park machinery, so the whole @@ -2911,6 +2862,20 @@ async function driveSuspendableOccurrence( ); } + // A body can end under a relay of its approval, also before a crash that + // leaves the await open with no relay to plan. Left open, the await would make + // the container's next park read as a re-park of it, and no `SignalAwaited` + // would name what the container then waits on. + if (!abort.aborted) { + const open = parentSignalAwait( + await reloadState(env, runId), + containerStepId, + ); + if (open !== undefined && controlParkKindOf(open) === "approval") { + await abandonRelayIfOpen(env, runId, containerStepId, open.name); + } + } + let after = await reloadState(env, runId); if (after.children.get(childRunId)?.terminalStatus === undefined) { const completed: WorkflowEvent = { @@ -2988,20 +2953,10 @@ async function runOnTrigger( const initial = await reloadState(env, runId); let eventIndex: number; let currentInput: unknown; - // Set only on a crash-recovered iteration whose body is parked mid-approval: - // it re-links the parent to that body on the shared correlation before the - // drive loop, then clears so every later iteration is a fresh spawn. - let resumeApproval: - | { corr: string; relay: boolean; decision?: unknown } - | undefined; - // The signal-relay sibling of `resumeApproval`: set on a crash-recovered - // iteration whose body is parked mid-signal-relay. `reestablish` re-drives the - // durable await's race; `relay` delivers a signal that landed but was not - // relayed before the crash. Cleared after the recovered iteration is re-linked. - let resumeSignalRelay: - | { kind: "reestablish"; name: string; awaitSeq: number } - | { kind: "relay"; name: string; payload: unknown; signalId: string } - | undefined; + // Set only on a crash-recovered event whose body was mid-flight: the relays + // that re-link it on its first drive, cleared so every later event is a + // fresh spawn. + let resumeRelays: ChildRelay[] = []; if (!initial.steps.has(primitive.id)) { await emitStepStartedWithValue(env, runId, primitive.id, { @@ -3029,38 +2984,18 @@ async function runOnTrigger( throw new Error( `onTrigger ${primitive.id} body run ${recordedSectionBodyRunId(runId, primitive.id, plan.eventIndex, initial.children)} ended ${plan.terminalStatus}`, ); - case "reestablish-approval": - eventIndex = plan.eventIndex; - currentInput = undefined; - resumeApproval = { corr: plan.corr, relay: false }; - break; - case "relay-grant": + case "relink-body": + // The event's body was mid-flight. Re-spawn it from its own durable log + // on the same event, whose RunStarted holds its input; a body whose log + // is gone starts over with the event's input. eventIndex = plan.eventIndex; - currentInput = undefined; - resumeApproval = { - corr: plan.corr, - relay: true, - decision: plan.decision, - }; - break; - case "reestablish-signal-relay": - eventIndex = plan.eventIndex; - currentInput = undefined; - resumeSignalRelay = { - kind: "reestablish", - name: plan.name, - awaitSeq: plan.awaitSeq, - }; - break; - case "relay-signal-grant": - eventIndex = plan.eventIndex; - currentInput = undefined; - resumeSignalRelay = { - kind: "relay", - name: plan.name, - payload: plan.payload, - signalId: plan.signalId, - }; + currentInput = + plan.startsOverWith === undefined + ? undefined + : plan.startsOverWith.from === "trigger" + ? evaluate({ from: "trigger.payload" }, selectorCtx) + : plan.startsOverWith.payload; + resumeRelays = plan.relays; break; case "advance-with-input": // The next event's trigger already arrived durably on the input channel @@ -3071,16 +3006,6 @@ async function runOnTrigger( currentInput = plan.input; eventIndex = plan.eventIndex + 1; break; - case "readopt-in-flight-body": - // The body child is in flight with no container park (a bare sleep - // parked entirely inside the child). Re-adopt this SAME event: re-spawn - // the body from its own durable log and await its terminal. `currentInput` - // is unused on a re-adopt (the body has a durable RunStarted), and no - // resume token is staged -- `resume` stays `undefined`, the sentinel for - // "re-adopt from the child log" the loop body path also uses. - eventIndex = plan.eventIndex; - currentInput = undefined; - break; case "reawait-input": { // The current event's body completed; the section is idle on its input // re-arm. Re-adopt the durable input park or mint a fresh one, await @@ -3112,34 +3037,6 @@ async function runOnTrigger( eventIndex, (await reloadState(env, runId)).children, ); - let resume: SuspendableOccurrenceResume | undefined; - if (resumeApproval !== undefined) { - resume = { - kind: "approval", - corr: resumeApproval.corr, - relay: resumeApproval.relay, - ...(resumeApproval.decision !== undefined - ? { decision: resumeApproval.decision } - : {}), - }; - resumeApproval = undefined; - } else if (resumeSignalRelay !== undefined) { - resume = - resumeSignalRelay.kind === "relay" - ? { - kind: "signal-relay-relay", - name: resumeSignalRelay.name, - payload: resumeSignalRelay.payload, - signalId: resumeSignalRelay.signalId, - } - : { - kind: "signal-relay-reestablish", - name: resumeSignalRelay.name, - awaitSeq: resumeSignalRelay.awaitSeq, - }; - resumeSignalRelay = undefined; - } - const { terminalStatus } = await driveSuspendableOccurrence( env, runId, @@ -3148,7 +3045,7 @@ async function runOnTrigger( childRunId, bodyRef, input: currentInput, - resume, + relays: resumeRelays, spawnSuspendableChild, depth, maxChildSpawnDepth, @@ -3156,6 +3053,7 @@ async function runOnTrigger( signalRelayAddressing: "correlation", }, ); + resumeRelays = []; // Terminal-is-final unless the section tolerates a body failure. Two cases // always end the section, `tolerate` or not: a `cancelled` body, and ANY @@ -3208,12 +3106,13 @@ async function runOnTrigger( * event for the caller's loop to continue with. * * `bodyLeg` chooses how "the body moved on" is observed. `"next"` races - * `child.next()` and is the loop and onTrigger leg: those bodies are driven - * one step at a time, so the next event is this gate moving on. `"park-release"` - * is the childWorkflow leg. A child run parks sibling steps at the same time, - * and `next` would take the sibling off the queue and treat it as this park - * ending. That leg watches the named park leave the await instead, and only - * then reads the queue. + * `child.next()` and is the leg of a live loop or onTrigger body: it is driven + * one step at a time, so the next event is this gate moving on. + * `"park-release"` is the leg of a childWorkflow and of every resume relay, + * where the body can hold sibling parks at the same time, and `next` would + * take a sibling off the queue and treat it as this park ending. That leg + * watches the named park leave the await instead, and only then reads the + * queue. * * BUFFER-FIFO (operator ruling): a signal is RELAYED, never dropped, on the * `"next"` leg. Emitting the container's `SignalAwaited` runs the reducer's @@ -3362,9 +3261,9 @@ async function driveContainerSignalRelay( // process restart. Resume re-enters this drive for a park the parent never // recorded, on the park-release leg. A relay the parent already recorded // is raced from that record and does not emit a second `SignalAwaited`. - // Fired on the fresh drive only, after the durable relay await above -- the - // reestablish path re-drives `raceContainerSignalRelay` directly and each - // level re-establishes from its own durable await, so it needs no re-fire. + // Fired on the fresh drive only, after the durable relay await above. Each + // level re-establishes from its own durable await, so a re-established + // relay reports `onSignalAwaiter` instead of a second park. if (env.onSignalPark !== undefined) { env.onSignalPark({ runId, @@ -3390,7 +3289,8 @@ async function driveContainerSignalRelay( /** * The container's relay await is stale: the body left the park, or the child - * run ended, before the signal arrived. Retire it when it is still open. + * run ended, before the signal arrived. Retire it when it is still open, + * naming its correlation so a reader retires that park and not another. * A signal that landed and was consumed during the race is relayed * idempotently instead, and the await is not abandoned. */ @@ -3399,7 +3299,6 @@ async function abandonRelayIfOpen( runId: string, containerStepId: string, name: string, - correlationId?: string, ): Promise { const afterRace = await reloadState(env, runId); const container = afterRace.steps.get(containerStepId); @@ -3409,6 +3308,7 @@ async function abandonRelayIfOpen( ) { return false; } + const correlationId = container.awaitingSignal.correlationId; const retire = await reloadState(env, runId); const abandoned: WorkflowEvent = { kind: "SignalAwaitAbandoned", @@ -3430,11 +3330,8 @@ async function settleStaleSignalRelay( child: SuspendableChildHandle, name: string, awaitSeq: number, - correlationId?: string, ): Promise { - if ( - await abandonRelayIfOpen(env, runId, containerStepId, name, correlationId) - ) { + if (await abandonRelayIfOpen(env, runId, containerStepId, name)) { return; } const log = await env.repoStore.read(runId); @@ -3505,12 +3402,14 @@ type ChildParkRelease = Awaited< >; /** - * Child-workflow arm of a container signal-relay. The spawn step awaits one - * signal, and the child may already have queued a sibling park. Racing - * `next` would consume that sibling and retire this relay while the child - * gate is still parked. Watch the named park instead. A sibling does not - * resolve the watch. The queue is read only after this park has a decision, - * has left the await, or the child run has ended. + * Watched arm of a container signal-relay, for a container whose body can + * hold sibling parks: a childWorkflow, and any container on resume. Racing + * `next` would consume a sibling and retire this relay while the body gate is + * still parked. Watch the named park instead. A sibling does not resolve the + * watch. The queue is read only after this park has a decision, has left the + * await, or the body run has ended. A correlated await takes only a signal + * for its id. `onWaiting` runs as in `awaitSignalFor`: once both waits exist + * and the log holds no signal for this await. */ async function raceChildSignalRelease( env: WorkflowRuntimeEnv, @@ -3518,8 +3417,11 @@ async function raceChildSignalRelease( containerStepId: string, child: SuspendableChildHandle, name: string, + awaitSeq: number, + correlationId: string | undefined, abort: AbortSignal, pull: ChildRelayPull, + onWaiting?: () => void, ): Promise { const watch = child.waitForParkRelease; if (watch === undefined) { @@ -3553,12 +3455,16 @@ async function raceChildSignalRelease( const onSignal = idle.waker(); const onRelease = idle.waker(); const signalWait = env.signalChannel - .awaitNext(name, onSignal, raceAbort.signal) + .awaitNext(name, onSignal, raceAbort.signal, correlationId) .then((r) => ({ tag: "signal" as const, r })); const releaseWait = watch(name, raceAbort.signal, onRelease).then((rel) => ({ tag: "release" as const, rel, })); + // An abort during the read below rejects both waits before anything else + // handles them. + void signalWait.catch(() => undefined); + void releaseWait.catch(() => undefined); // The channel tails from the moment `awaitNext` is called, so a signal // committed before that wait is only in the log. Both waits are already // started: a signal or a release during this read is still observed, and @@ -3568,7 +3474,10 @@ async function raceChildSignalRelease( let logged: { payload: unknown; signalId: string } | undefined; try { const log = await env.repoStore.read(runId); - logged = signalBoundToAwait(runId, log, containerStepId, name); + logged = + correlationId === undefined + ? signalBoundToAwait(runId, log, containerStepId, name) + : boundSignalForContainerAwait(log, name, containerStepId, awaitSeq); } catch (cause) { raceAbort.abort(); detachRaceAbort(); @@ -3601,6 +3510,7 @@ async function raceChildSignalRelease( let outcome: | { tag: "signal"; r: { payload: unknown; signalId: string } } | { tag: "release"; rel: ChildParkRelease }; + onWaiting?.(); try { outcome = await idle.wait(Promise.race([signalWait, releaseWait]), [ onSignal, @@ -3642,6 +3552,7 @@ async function raceChildSignalRelease( signalName: name, signalId: outcome.r.signalId, payload: outcome.r.payload, + ...(correlationId !== undefined ? { correlationId } : {}), }; await commit(env, runId, received); await flush(env, runId); @@ -3670,12 +3581,13 @@ async function raceChildSignalRelease( * The awaiting arm of a container signal-relay: the container's * `SignalAwaited(name, "signal-relay")` at `awaitSeq` is durable and the body * is parked on `name`; race the signal's live arrival against the body - * leg and against abort, then relay or retire. Shared by the fresh drive - * ({@link driveContainerSignalRelay}, which just emitted the await) and by - * crash-recovery (`reestablish-signal-relay`, whose re-spawned body re-parks on - * `name` silently, so the await is re-driven from its durable seq without a - * re-emit). `"next"` is the loop and onTrigger leg and returns the body's - * next event. `"park-release"` is the childWorkflow leg. + * leg and against abort, then relay or retire. The fresh drive + * ({@link driveContainerSignalRelay}, which just emitted the await) enters + * here. A `signal-reestablish` relay on resume, whose re-spawned body re-parks + * on `name` silently, re-drives its durable await on the park-release leg + * directly, without a re-emit. `"next"` is the live loop and onTrigger leg and + * returns the body's next event. `"park-release"` is the leg of every resume + * relay and of a live childWorkflow. */ async function raceContainerSignalRelay( env: WorkflowRuntimeEnv, @@ -3696,6 +3608,8 @@ async function raceContainerSignalRelay( containerStepId, child, name, + awaitSeq, + correlationId, abort, pull, ); @@ -3780,40 +3694,30 @@ async function raceContainerSignalRelay( child, name, awaitSeq, - correlationId, ); return outcome.ev; } /** * Discriminated resume plan for a crash-recovered onTrigger container, derived - * purely from the container's reduced `state` plus its durable `log`. The - * `children` map locates the current event index and the container step's - * phase locates its lifecycle point; the log recovers a delivered-but-unrelayed - * approval grant, whose `SignalReceived` reduces the container step to - * `in-flight` and strips its `awaitingSignal` (so the correlation and decision - * are no longer in the reduced state). + * from the container's reduced `state` plus its durable `log`, and the body's + * own log while it is mid-flight. The `children` map locates the current event + * index and the container step's phase locates its lifecycle point; a body + * mid-flight gets the relays `planChildRelays` plans for it. */ -type OnTriggerResumePlan = +type OnTriggerResumePlan = | { kind: "fresh" } - | { kind: "reestablish-approval"; eventIndex: number; corr: string } - | { kind: "relay-grant"; eventIndex: number; corr: string; decision: unknown } + // The event's body was mid-flight: re-spawn it from its own log and drive + // these relays first, none when nothing waits on the section. A body whose + // log is gone starts over with `startsOverWith`. | { - kind: "reestablish-signal-relay"; + kind: "relink-body"; eventIndex: number; - name: string; - awaitSeq: number; - } - | { - kind: "relay-signal-grant"; - eventIndex: number; - name: string; - payload: unknown; - signalId: string; + relays: R[]; + startsOverWith?: OnTriggerEventInput; } | { kind: "reawait-input"; eventIndex: number; existingSignalName?: string } | { kind: "advance-with-input"; eventIndex: number; input: unknown } - | { kind: "readopt-in-flight-body"; eventIndex: number } | { kind: "terminal-is-final"; eventIndex: number; @@ -3831,36 +3735,6 @@ function bodyFailurePolicyOf(primitive: { return primitive.onBodyFailure ?? "end"; } -/** - * The `awaitSignal` gate names a body child is parked on in its reduced state, - * split by channel: `author` names are author-chosen (`correlationIdFromSignalName` - * undefined); `controlPlane` names are reserved approval/relay channels. Both - * signal a crash window where the body's leaf `SignalAwaited` flushed but the - * container's proxy await did not, so a naive re-adopt re-parks the body on a - * signal the container never relays. The loop planner keys its author-gate - * drive-fresh on `author`; the onTrigger planner refuses on either, since it has - * no fresh-relay drive for a body still awaiting a signal. - */ -export function bodyParkedSignals(childState: RunState): { - author: string[]; - controlPlane: string[]; -} { - const author: string[] = []; - const controlPlane: string[] = []; - for (const step of childState.steps.values()) { - if (step.phase !== "awaiting-signal" || step.awaitingSignal === undefined) { - continue; - } - const name = step.awaitingSignal.name; - if (correlationIdFromSignalName(name) === undefined) { - author.push(name); - } else { - controlPlane.push(name); - } - } - return { author, controlPlane }; -} - /** Decimal event index with no leading zero. `section__01` is not index 1. */ const CANONICAL_EVENT_INDEX = /^(0|[1-9][0-9]*)$/; @@ -3948,10 +3822,32 @@ async function planOnTriggerResume( primitive: OnTriggerPrimitive, state: RunState, log: readonly WorkflowEvent[], -): Promise { - return finishOnTriggerResume(primitive, state, log, (childRunId) => - reloadState(env, childRunId), +): Promise> { + const plan = await finishOnTriggerResume( + primitive, + state, + log, + (childRunId) => bodyLogForRelays(env, childRunId), + env.clock().getTime(), ); + if (plan.kind !== "relink-body") return plan; + const childRunId = recordedSectionBodyRunId( + state.runId, + primitive.id, + plan.eventIndex, + state.children, + ); + const childLog = await bodyLogForRelays(env, childRunId); + return { + ...plan, + relays: await loadPlannedRelays( + env, + `onTrigger ${primitive.id}`, + childRunId, + resumeFromLog(childRunId, childLog), + plan.relays, + ), + }; } async function finishOnTriggerResume( @@ -3961,7 +3857,8 @@ async function finishOnTriggerResume( }, state: RunState, log: readonly WorkflowEvent[], - loadChild: (childRunId: string) => Promise, + loadChild: (childRunId: string) => Promise, + nowMs: number, ): Promise { const eventIndex = highestSectionEventIndex( state.runId, @@ -4058,124 +3955,68 @@ async function finishOnTriggerResume( } if ( container.phase === "awaiting-signal" && - container.awaitingSignal !== undefined + container.awaitingSignal !== undefined && + controlParkKindOf(container.awaitingSignal) === "input" ) { - const parkKind = controlParkKindOf(container.awaitingSignal); - if (parkKind === "signal-relay") { - // The container is proxy-parked on the body's author `awaitSignal` gate. - // Recover the durable await's seq (the FIFO binding key) so the resume - // re-drives the race over the same await without re-emitting it. - const name = container.awaitingSignal.name; - const recovered = lastSignalRelayAwait(primitive.id, log, name); - if (recovered === undefined) { - throw new Error( - `onTrigger ${primitive.id} resume: container awaits signal-relay ${name} but no matching SignalAwaited is in the log`, - ); - } - return { - kind: "reestablish-signal-relay", - eventIndex, - name, - awaitSeq: recovered.seq, - }; - } - if (parkKind !== "approval") { - throw new Error( - `onTrigger ${primitive.id} resume: container is parked on an input channel while body child ${childRunId} is still in flight`, - ); - } - const corr = correlationIdFromSignalName(container.awaitingSignal.name); - if (corr === undefined) { - throw new Error( - `onTrigger ${primitive.id} resume: container awaiting-signal ${container.awaitingSignal.name} is not a reserved control-plane channel`, - ); - } - return { kind: "reestablish-approval", eventIndex, corr }; - } - // The container is not parked but the body is still in flight: the approval - // grant was delivered (its SignalReceived moved the container to in-flight) - // but not yet relayed into the child. Recover the correlation and decision - // from the log. - const grant = recoverDeliveredApprovalGrant(primitive.id, log); - if (grant !== undefined) { - return { - kind: "relay-grant", - eventIndex, - corr: grant.corr, - decision: grant.decision, - }; + throw new Error( + `onTrigger ${primitive.id} resume: container is parked on an input channel while body child ${childRunId} is still in flight`, + ); } - // Signal-relay sibling of the delivered-grant window: a signal delivered to - // the container's last signal-relay await consumed it (moving the container - // to in-flight) but was not relayed into the body before the crash. Recover - // it from the log's FIFO pairing and relay on resume. - const relaySignal = recoverDeliveredSignalRelay(primitive.id, log); - if (relaySignal !== undefined) { + // Re-link the body: re-establish the relay the section awaits, hand on input + // delivered to it but not to the body, and relay every park the body holds + // that the section never relayed. A body that holds none re-adopts from its + // own log, and its own resume classifier decides its terminal: a `sleep` + // resumes via `isResumableSleepStep`, a mid-flight `childWorkflow` + // reattaches, a mid-flight `map` records `RunFailed`, and a crashed agent step + // settles a terminal `StepFailed`. A `tolerate` section absorbs either of the + // last two before it re-arms. + const childLog = [...(await loadChild(childRunId))]; + const sinceSeq = childSpawnSeq(log, childRunId); + if (childLog.length === 0) { return { - kind: "relay-signal-grant", + kind: "relink-body", eventIndex, - name: relaySignal.name, - payload: relaySignal.payload, - signalId: relaySignal.signalId, + relays: containerRelays(primitive.id, log, state, sinceSeq), + startsOverWith: onTriggerEventInput( + primitive.id, + log, + eventIndex, + sinceSeq, + ), }; } - // The body is in flight with nothing delivered to the container. Inspect the - // body's OWN reduced state. A bare `sleep` parks entirely inside the child - // (an `awaiting-timer` step, or an `in-flight` step once its `TimerFired` - // landed) and surfaces no container park, so re-adopt the in-flight body and - // await its terminal: the re-spawned body re-drives its own durable log and - // re-adopts its sleep timer via `isResumableSleepStep`. `resume` stays - // `undefined` in the dispatch below -- the sentinel a loop body already uses - // for this re-adopt. - // - // The shape we must NOT re-adopt is a body still parked on a signal the - // container has not proxied -- author `awaitSignal` OR a reserved - // approval/relay channel -- because its leaf `SignalAwaited` flushed but the - // container's proxy await did not. Re-adopting it re-parks the body silently - // (no `onPark`) on a signal the container never relays, hanging both sides - // forever. onTrigger has no fresh-relay drive (loop's `signal-relay-drive- - // fresh` is out of scope here), so keep failing loud. Any body NOT awaiting a - // signal re-adopts and its OWN resume classifier decides its terminal: a - // `sleep` resumes via `isResumableSleepStep`. A mid-flight `childWorkflow` - // reattaches from its log. A mid-flight `map` REJECTS, and the seam - // re-throws that rejection as a loud section failure -- never a - // silently-tolerated terminal (see the tolerate regression test). A - // crashed agent step instead SETTLES a terminal `StepFailed`: under - // the default `end` policy this fails the section as the pre-branch throw did, - // and under `tolerate` the section absorbs the crashed body and re-arms. That - // last case is the one genuine behavior change here, and it is the intended - // reading of `tolerate` -- a real body failure it is meant to swallow, not a - // hang or a lost run. - const childState = await loadChild(childRunId); - const { author, controlPlane } = bodyParkedSignals(childState); - const parkedSignals = [...author, ...controlPlane]; - if (parkedSignals.length > 0) { - throw new Error( - `onTrigger ${primitive.id} resume: body child ${childRunId} is parked ` + - `on an un-relayed signal (${parkedSignals.join(", ")}) with no ` + - `container proxy await; onTrigger cannot re-establish the relay fresh`, - ); - } - return { kind: "readopt-in-flight-body", eventIndex }; + const relays = planChildRelays( + `onTrigger ${primitive.id}`, + primitive.id, + childRunId, + log, + state, + childLog, + resumeFromLog(childRunId, childLog), + nowMs, + sinceSeq, + "retire", + ); + refuseInputRelays(`onTrigger ${primitive.id}`, childRunId, relays); + return { kind: "relink-body", eventIndex, relays }; } /** - * The resume token for a crash-recovered loop iteration whose body was parked - * or mid-relay at the crash, derived purely from the container's reduced - * `state` plus its durable `log` for the given `iteration`. It yields ONLY the - * active re-link the body needs on its next drive: re-establish the container's - * signal-relay race, relay a grant/signal delivered but not relayed before the - * crash, or re-adopt an approval park. + * The relays a crash-recovered loop iteration needs on its next drive, derived + * from the container's reduced `state` and durable `log` for the given + * `iteration` and the body's own log: re-establish what the container awaits, + * hand on a grant or signal delivered to it but not to the body, and relay + * every park the body holds that the container never relayed. * - * `undefined` means "nothing to re-link", and `runLoop`'s forward drive + * An empty list means "nothing to re-link", and `runLoop`'s forward drive * re-adopts the iteration from the child's own durable log -- a terminal body * short-circuits, a body that crashed mid-invocation settles failed, a body * that had not yet worked re-runs. This is where the planner diverges from - * `planOnTriggerResume`, which throws in the same spot: `runLoop` owns the - * iteration cursor (it passes `iteration` in) rather than an event cursor it - * would have to reset, so there is no `fresh` arm to reset and no - * `terminal-is-final` arm -- the `isIterationDone` replay and the forward - * drive already own the terminal and re-adopt cases. + * `planOnTriggerResume`: `runLoop` owns the iteration cursor (it passes + * `iteration` in) rather than an event cursor it would have to reset, so there + * is no `fresh` arm to reset and no `terminal-is-final` arm -- the + * `isIterationDone` replay and the forward drive already own the terminal and + * re-adopt cases. */ async function planLoopResume( env: WorkflowRuntimeEnv, @@ -4184,29 +4025,39 @@ async function planLoopResume( state: RunState, log: readonly WorkflowEvent[], iteration: number, -): Promise { - return finishLoopResume( - primitive, - runId, - state, - log, - iteration, - (childRunId) => reloadState(env, childRunId), +): Promise { + const childRunId = loopBodyRunId(runId, primitive.id, iteration); + const childLog = await bodyLogForRelays(env, childRunId); + return loadPlannedRelays( + env, + `loop ${primitive.id}`, + childRunId, + resumeFromLog(childRunId, childLog), + finishLoopResume( + primitive, + runId, + state, + log, + iteration, + childLog, + env.clock().getTime(), + ), ); } -async function finishLoopResume( +function finishLoopResume( primitive: { readonly id: string }, runId: string, state: RunState, log: readonly WorkflowEvent[], iteration: number, - loadChild: (childRunId: string) => Promise, -): Promise { + childLog: readonly WorkflowEvent[], + nowMs: number, +): PlannedChildRelay[] { const childRunId = loopBodyRunId(runId, primitive.id, iteration); const child = state.children.get(childRunId); if (child === undefined || child.terminalStatus !== undefined) { - return undefined; + return []; } const container = state.steps.get(primitive.id); if (container === undefined) { @@ -4216,110 +4067,126 @@ async function finishLoopResume( } if ( container.phase === "awaiting-signal" && - container.awaitingSignal !== undefined + container.awaitingSignal !== undefined && + controlParkKindOf(container.awaitingSignal) === "input" ) { - const parkKind = controlParkKindOf(container.awaitingSignal); - if (parkKind === "signal-relay") { - // The container is proxy-parked on the body's author `awaitSignal` gate. - // Recover the durable await's seq (the FIFO binding key) so the resume - // re-drives the race over the same await without re-emitting it. - const name = container.awaitingSignal.name; - const recovered = lastSignalRelayAwait(primitive.id, log, name); - if (recovered === undefined) { - throw new Error( - `loop ${primitive.id} resume: container awaits signal-relay ${name} but no matching SignalAwaited is in the log`, - ); - } - return { - kind: "signal-relay-reestablish", - name, - awaitSeq: recovered.seq, - }; - } - if (parkKind !== "approval") { + throw new Error( + `loop ${primitive.id} resume: container is parked on an input channel while body child ${childRunId} is still in flight`, + ); + } + // On an inconsistent store the body's log is gone: the forward drive re-runs + // the body, and the container re-links what it held itself. + if (childLog.length === 0) { + return containerRelays( + primitive.id, + log, + state, + childSpawnSeq(log, childRunId), + ); + } + const relays = planChildRelays( + `loop ${primitive.id}`, + primitive.id, + childRunId, + log, + state, + childLog, + resumeFromLog(childRunId, childLog), + nowMs, + childSpawnSeq(log, childRunId), + "retire", + ); + refuseInputRelays(`loop ${primitive.id}`, childRunId, relays); + return relays; +} + +// A loop or onTrigger body never parks on an input channel; the live drive +// refuses one, so a log that holds one is not this container's. +function refuseInputRelays( + container: string, + childRunId: string, + relays: readonly PlannedChildRelay[], +): void { + for (const relay of relays) { + const input = + relay.kind === "input-fresh" || + ((relay.kind === "control-reestablish" || + relay.kind === "control-deliver") && + relay.parkKind === "input"); + if (input) { throw new Error( - `loop ${primitive.id} resume: container is parked on an input channel while body child ${childRunId} is still in flight`, + `${container} resume: body child ${childRunId} is parked on an input channel; a loop or onTrigger body cannot re-arm an input park`, ); } - const corr = correlationIdFromSignalName(container.awaitingSignal.name); - if (corr === undefined) { + } +} + +/** + * A body whose log is gone starts over and asks again for every input delivered + * since its spawn at `sinceSeq`, except the open await at `keepSeq`. It fails + * here rather than waiting forever for input that is not coming again: the host + * channel writes a hand-down to the body's log, where one that beats the body's + * `RunStarted` is rejected. + */ +function refuseHandedOnInput( + stepId: string, + log: readonly WorkflowEvent[], + sinceSeq: number, + keepSeq: number | undefined, +): void { + for (const event of log) { + if ( + event.kind === "SignalAwaited" && + event.stepId === stepId && + event.seq > sinceSeq && + event.seq !== keepSeq && + boundSignalForContainerAwait(log, event.signalName, stepId, event.seq) !== + undefined + ) { throw new Error( - `loop ${primitive.id} resume: container awaiting-signal ${container.awaitingSignal.name} is not a reserved control-plane channel`, + `step ${stepId} resume: its body lost its log after ${event.signalName} was delivered, which cannot be handed to it again`, ); } - return { kind: "approval", corr, relay: false }; - } - // The container is not parked but the body is still in flight: a grant or - // signal was DELIVERED (its SignalReceived moved the container to in-flight) - // but not relayed into the body before the crash. Relay it into the - // re-adopted body, else the body's silently re-parked gate would wait forever - // for a signal already consumed. - const grant = recoverDeliveredApprovalGrant(primitive.id, log); - if (grant !== undefined) { - return { - kind: "approval", - corr: grant.corr, - relay: true, - ...(grant.decision !== undefined ? { decision: grant.decision } : {}), - }; - } - const relaySignal = recoverDeliveredSignalRelay(primitive.id, log); - if (relaySignal !== undefined) { - return { - kind: "signal-relay-relay", - name: relaySignal.name, - payload: relaySignal.payload, - signalId: relaySignal.signalId, - }; - } - // Otherwise the body is in flight with nothing delivered to the container. - // On a consistent store the body's own durable log may show it parked on an - // author `awaitSignal` gate the container never relayed -- the crash landed - // after the body's leaf `SignalAwaited` flushed but before the container's - // relay `SignalAwaited` flushed. The re-adopted body re-parks silently, so - // the forward drive would never surface the gate and the container would - // block forever. Recover the parked author name from the body's reduced state - // and drive the container relay FRESH. (On an inconsistent store the child - // log is gone, so no gate is found and the forward drive re-runs the - // iteration -- the existing inconsistent-store behavior.) - const childState = await loadChild(childRunId); - const { author: authorAwaits } = bodyParkedSignals(childState); - if (authorAwaits.length > 1) { - throw new Error( - `loop ${primitive.id} resume: body child ${childRunId} is parked on ` + - `multiple concurrent author signals (${authorAwaits.join(", ")}); ` + - `re-establishing more than one un-relayed gate is not supported`, - ); } - const parkedName = authorAwaits[0]; - if (parkedName !== undefined) { - let timeoutAt: string | undefined; - for (const step of childState.steps.values()) { - if ( - step.phase === "awaiting-signal" && - step.awaitingSignal?.name === parkedName - ) { - timeoutAt = step.awaitingSignal.timeoutAt; - break; - } +} + +/** + * The relay a container awaits for a body whose log is gone. Only its awaits + * after `sinceSeq`, the body's spawn, are this body's. + */ +function containerRelays( + stepId: string, + log: readonly WorkflowEvent[], + state: RunState, + sinceSeq: number, +): PlannedChildRelay[] { + const awaiting = parentSignalAwait(state, stepId); + if (awaiting !== undefined) { + const kind = controlParkKindOf(awaiting); + const awaitSeq = lastAwaitSeq(log, stepId, awaiting.name, sinceSeq); + refuseHandedOnInput(stepId, log, sinceSeq, awaitSeq); + if (kind === "signal-relay") { + return awaitSeq === undefined + ? [] + : [{ kind: "signal-reestablish", name: awaiting.name, awaitSeq }]; + } + const correlationId = correlationIdFromSignalName(awaiting.name); + if (correlationId === undefined) { + throw new Error( + `step ${stepId} awaiting-signal ${awaiting.name} is not a reserved control-plane channel`, + ); } - return { - kind: "signal-relay-drive-fresh", - name: parkedName, - ...(timeoutAt !== undefined ? { timeoutAt } : {}), - }; + return [{ kind: "control-reestablish", parkKind: kind, correlationId }]; } - return undefined; + refuseHandedOnInput(stepId, log, sinceSeq, undefined); + return []; } /** * Whether a crash-recovered container can be re-attached, judged by the same * planners the runtime drives. `rejected` is a planner throw (caught here so * a caller can release instead of letting it escape). `reenter` is a child - * the forward drive re-adopts that is not itself a park. A drive-fresh loop - * park with a control-plane signal beside the author signal is `rejected` - * even though the runtime planner would return the author name: more than - * the author gate is waiting, and that is not re-attached. + * the forward drive re-adopts that is not itself a park. */ export type OccurrenceReattach = | { readonly kind: "parked"; readonly childRunId?: string } @@ -4327,37 +4194,39 @@ export type OccurrenceReattach = | { readonly kind: "open" } | { readonly kind: "rejected" }; -export async function loopOccurrenceReattach(args: { +// Retiring a relay the body has left gives the run nothing to wait on. +function holdsPark(relays: readonly PlannedChildRelay[]): boolean { + return relays.some((relay) => relay.kind !== "signal-abandon"); +} + +export function loopOccurrenceReattach(args: { readonly loopId: string; readonly runId: string; readonly iteration: number; readonly state: RunState; readonly log: readonly WorkflowEvent[]; - readonly childState: RunState; -}): Promise { + readonly childLog: readonly WorkflowEvent[]; + // Bounds a timed gate's remaining wait; a verdict does not depend on it. + readonly nowMs: number; +}): OccurrenceReattach { const childRunId = loopBodyRunId(args.runId, args.loopId, args.iteration); try { - const plan = await finishLoopResume( + const plan = finishLoopResume( { id: args.loopId }, args.runId, args.state, args.log, args.iteration, - async () => args.childState, + args.childLog, + args.nowMs, ); - if (plan === undefined) { + if (!holdsPark(plan)) { const child = args.state.children.get(childRunId); if (child === undefined || child.terminalStatus !== undefined) { return { kind: "open" }; } return { kind: "reenter", childRunId }; } - if (plan.kind === "signal-relay-drive-fresh") { - const { author, controlPlane } = bodyParkedSignals(args.childState); - if (author.length !== 1 || controlPlane.length > 0) { - return { kind: "rejected" }; - } - } return { kind: "parked", childRunId }; } catch { return { kind: "rejected" }; @@ -4369,7 +4238,9 @@ export async function onTriggerOccurrenceReattach(args: { readonly onBodyFailure?: BodyFailurePolicy; readonly state: RunState; readonly log: readonly WorkflowEvent[]; - readonly childState: RunState; + readonly loadChild: (childRunId: string) => Promise; + // Bounds a timed gate's remaining wait; a verdict does not depend on it. + readonly nowMs: number; }): Promise { try { const plan = await finishOnTriggerResume( @@ -4381,7 +4252,8 @@ export async function onTriggerOccurrenceReattach(args: { }, args.state, args.log, - async () => args.childState, + args.loadChild, + args.nowMs, ); // A recorded body keeps the id the log stored. `sectionBodyRunId` // mints the parent-prefixed form, which is an empty log when the @@ -4396,17 +4268,9 @@ export async function onTriggerOccurrenceReattach(args: { switch (plan.kind) { case "reawait-input": return { kind: "parked" }; - case "reestablish-approval": - case "reestablish-signal-relay": - case "relay-grant": - case "relay-signal-grant": - return { - kind: "parked", - childRunId: recordedChild(plan.eventIndex), - }; - case "readopt-in-flight-body": + case "relink-body": return { - kind: "reenter", + kind: holdsPark(plan.relays) ? "parked" : "reenter", childRunId: recordedChild(plan.eventIndex), }; case "fresh": @@ -4419,46 +4283,53 @@ export async function onTriggerOccurrenceReattach(args: { } } +// The input an onTrigger event's body was spawned with: the run's trigger +// payload for event 0, else what the section's input park delivered. +type OnTriggerEventInput = + | { from: "trigger" } + | { from: "input"; payload: unknown }; + /** - * Recover the correlation and delivered decision of the container's most recent - * approval park from the durable log. Used only for the narrow crash window - * where the grant's `SignalReceived` landed before the driver relayed it into - * the body -- the container step reduces to `in-flight` with its - * `awaitingSignal` stripped, so the correlation lives only in the log. + * The input event `eventIndex` of an onTrigger section started with, for a body + * that starts over without its log. A later event took what the section's last + * input park before the body's `ChildSpawned` at `spawnSeq` received. */ -function recoverDeliveredApprovalGrant( +function onTriggerEventInput( stepId: string, log: readonly WorkflowEvent[], -): { corr: string; decision: unknown } | undefined { - let corr: string | undefined; - let parkKind: ControlParkKind | undefined; + eventIndex: number, + spawnSeq: number, +): OnTriggerEventInput { + if (eventIndex === 0) return { from: "trigger" }; + let inputAwait: string | undefined; for (const event of log) { - if (event.kind === "SignalAwaited" && event.stepId === stepId) { - const candidate = correlationIdFromSignalName(event.signalName); - if (candidate !== undefined) { - corr = candidate; - parkKind = controlParkKindOf(event); - } + if ( + event.seq < spawnSeq && + event.kind === "SignalAwaited" && + event.stepId === stepId && + controlParkKindOf(event) === "input" + ) { + inputAwait = event.signalName; } } - if (corr === undefined || parkKind !== "approval") return undefined; - const reserved = signalName(corr); - let decision: unknown; - let found = false; for (const event of log) { - if (event.kind === "SignalReceived" && event.signalName === reserved) { - decision = event.payload; - found = true; + if ( + inputAwait !== undefined && + event.seq < spawnSeq && + event.kind === "SignalReceived" && + event.signalName === inputAwait + ) { + return { from: "input", payload: event.payload }; } } - if (!found) return undefined; - return { corr, decision }; + throw new Error( + `onTrigger ${stepId} resume: the body of event ${String(eventIndex)} lost its log, and the log holds no input that started that event`, + ); } /** * Recover an input re-arm whose next trigger was delivered but whose body spawn - * had not yet committed at the crash -- the input sibling of - * `recoverDeliveredApprovalGrant`. After a body completes, the container + * had not yet committed at the crash. After a body completes, the container * re-arms an `input` park for event N+1; when that park's `SignalReceived` * landed (moving the container to `in-flight` and stripping its * `awaitingSignal`) but `ChildSpawned` N+1 was not written, the delivered @@ -4511,56 +4382,6 @@ function recoverDeliveredInput( return { payload }; } -/** - * The name + seq of the container's last `SignalAwaited(_, "signal-relay")` -- - * optionally filtered to `name` -- or undefined if it has none. The last such - * await is the one a crash-recovery re-drives (`reestablish-signal-relay`) or - * binds a delivered signal against (`recoverDeliveredSignalRelay`). - */ -function lastSignalRelayAwait( - containerStepId: string, - log: readonly WorkflowEvent[], - name?: string, -): { name: string; seq: number } | undefined { - let last: { name: string; seq: number } | undefined; - for (const event of log) { - if ( - event.kind === "SignalAwaited" && - event.stepId === containerStepId && - controlParkKindOf(event) === "signal-relay" && - (name === undefined || event.signalName === name) - ) { - last = { name: event.signalName, seq: event.seq }; - } - } - return last; -} - -/** - * Recover a signal delivered to the container's last signal-relay await but not - * yet relayed into the body -- the signal-relay sibling of - * `recoverDeliveredApprovalGrant`. The delivery's `SignalReceived` consumed the - * container's await (moving it to `in-flight`), so the payload lives only in the - * log; bind it by replaying the reducer's FIFO pairing. Returns undefined when - * the container has no signal-relay await, or its last one carries no paired - * signal (an await that was abandoned rather than consumed). - */ -function recoverDeliveredSignalRelay( - containerStepId: string, - log: readonly WorkflowEvent[], -): { name: string; payload: unknown; signalId: string } | undefined { - const last = lastSignalRelayAwait(containerStepId, log); - if (last === undefined) return undefined; - const bound = boundSignalForContainerAwait( - log, - last.name, - containerStepId, - last.seq, - ); - if (bound === undefined) return undefined; - return { name: last.name, payload: bound.payload, signalId: bound.signalId }; -} - /** * Bind the signal a container `signal-relay` await consumed, by REPLAYING the * reducer's FIFO signal pairing over the container's durable log. The reducer @@ -6733,13 +6554,13 @@ async function flushParkedChildWorkflowFailure( * treat it as this park ending, and the spawn step can await only one * signal. A child that ends while the parent is parked wins the race, and * the caller continues from that terminal without reading the queue. A - * release while the child run is still going fails the spawn: an approval - * or input relay cannot be retired with `SignalAwaitAbandoned`, and parking - * the next signal on a step that is still awaiting this one would wait on - * the old name. A failure of the park itself (nothing upstream can answer - * it, or a second input park) is not a body event. `next` is not started - * on that path: the child is still parked, and the failure path's later - * `next` takes the one wake slot. + * release while the child run is still going throws + * `ChildRelayRetiredError`, since parking the next signal on a step that is + * still awaiting this one would wait on the old name. It fails a spawn step; a + * loop or onTrigger drive retires the approval. A failure of the park itself + * (nothing upstream can answer it, or a second input park) is not a body event. + * `next` is not started on that path: the child is still parked, and the + * failure path's later `next` takes the one wake slot. */ async function awaitChildPark( env: WorkflowRuntimeEnv, @@ -6775,7 +6596,7 @@ async function awaitChildPark( return { kind: "terminal", terminalStatus: before.terminalStatus }; } if (before.kind === "released") { - throw childRelayRetired(stepId, parkKind, name); + throw new ChildRelayRetiredError(stepId, parkKind, name); } const parkAbort = new AbortController(); const detachParkAbort = bridgeAbort(abort, () => { @@ -6832,7 +6653,7 @@ async function awaitChildPark( terminalStatus: outcome.rel.terminalStatus, }; } - throw childRelayRetired(stepId, parkKind, name); + throw new ChildRelayRetiredError(stepId, parkKind, name); } // Stop the watch before resume. The child's own receipt would otherwise // look like the park ending without a decision. Re-read before resume: @@ -6845,21 +6666,24 @@ async function awaitChildPark( return { kind: "terminal", terminalStatus: again.terminalStatus }; } if (again.kind === "released") { - throw childRelayRetired(stepId, parkKind, name); + throw new ChildRelayRetiredError(stepId, parkKind, name); } await handle.resume(correlationId, outcome.payload); return finishRelayPull(handle, pull); } -function childRelayRetired( - stepId: string, - parkKind: "approval" | "input", - name: string, -): Error { - return new Error( - `step ${stepId} ${parkKind} relay on ${name} ended while the child ` + - `run was still going; a control-plane relay cannot be retired`, - ); +class ChildRelayRetiredError extends Error { + constructor( + stepId: string, + readonly parkKind: "approval" | "input", + readonly parkName: string, + ) { + super( + `step ${stepId} ${parkKind} relay on ${parkName} ended while the child ` + + `run was still going; a control-plane relay cannot be retired`, + ); + this.name = "ChildRelayRetiredError"; + } } /** @@ -7147,11 +6971,13 @@ function lastAwaitSeq( log: readonly WorkflowEvent[], stepId: string, name: string, + sinceSeq: number, ): number | undefined { let seq: number | undefined; for (const event of log) { if ( event.kind === "SignalAwaited" && + event.seq > sinceSeq && event.stepId === stepId && event.signalName === name ) { @@ -7162,16 +6988,17 @@ function lastAwaitSeq( } /** - * The signal paired with the parent's last await of `name`, if that await - * already consumed one. The full parent log is required: the pairing - * replays the reducer's queue from seq 1. + * The signal paired with the parent's last await of `name` since `sinceSeq`, + * if that await already consumed one. The full parent log is required: the + * pairing replays the reducer's queue from seq 1. */ function deliveredForAwait( parentLog: readonly WorkflowEvent[], stepId: string, name: string, + sinceSeq: number, ): { payload: unknown; signalId: string } | undefined { - const awaitSeq = lastAwaitSeq(parentLog, stepId, name); + const awaitSeq = lastAwaitSeq(parentLog, stepId, name, sinceSeq); if (awaitSeq === undefined) return undefined; return boundSignalForContainerAwait(parentLog, name, stepId, awaitSeq); } @@ -7235,9 +7062,13 @@ function parentSignalAwait( * What a container still has to do for the parks of one body run it relays: * re-establish the relay it awaits, hand on input delivered to it but not yet * to the body, and relay every park the body holds that it never relayed, of - * which a restart can leave several. + * which a restart can leave several. Only the container's awaits after + * `sinceSeq`, the body's spawn, are this body's. A loop or onTrigger body can + * leave an approval and go on, as a timed step does, so `approvalLeft` retires + * that relay; a spawn step fails on it, with or without a restart. */ function planChildRelays( + container: string, stepId: string, childRunId: string, parentLog: readonly WorkflowEvent[], @@ -7245,6 +7076,8 @@ function planChildRelays( childLog: readonly WorkflowEvent[], childState: RunState, nowMs: number, + sinceSeq: number, + approvalLeft: "retire" | "fail", ): PlannedChildRelay[] { const parks = openChildParks(childLog, childState); const parentAwait = parentSignalAwait(parentState, stepId); @@ -7256,13 +7089,13 @@ function planChildRelays( if (park.kind === "author") { if (parentKind !== "signal-relay") { throw new Error( - `childWorkflow ${stepId} author park ${park.name} is awaited as ${parentKind}`, + `${container} author park ${park.name} is awaited as ${parentKind}`, ); } - const awaitSeq = lastAwaitSeq(parentLog, stepId, park.name); + const awaitSeq = lastAwaitSeq(parentLog, stepId, park.name, sinceSeq); if (awaitSeq === undefined) { throw new Error( - `childWorkflow ${stepId} is awaiting ${park.name} with no SignalAwaited in the log`, + `${container} is awaiting ${park.name} with no SignalAwaited in the log`, ); } relays.push({ kind: "signal-reestablish", name: park.name, awaitSeq }); @@ -7270,7 +7103,7 @@ function planChildRelays( } if (parentKind !== park.kind) { throw new Error( - `childWorkflow ${stepId} ${park.kind} park ${park.name} is awaited as ${parentKind}`, + `${container} ${park.kind} park ${park.name} is awaited as ${parentKind}`, ); } relays.push({ @@ -7280,7 +7113,7 @@ function planChildRelays( }); continue; } - const delivered = deliveredForAwait(parentLog, stepId, park.name); + const delivered = deliveredForAwait(parentLog, stepId, park.name, sinceSeq); if ( delivered !== undefined && !childSawSignal(childLog, park.awaitSeq, delivered.signalId) @@ -7330,7 +7163,12 @@ function planChildRelays( const stillOpen = parks.some((park) => park.name === awaiting.name); if (!stillOpen) { const parentKind = controlParkKindOf(awaiting); - if (parentKind === "signal-relay") { + if ( + parentKind === "signal-relay" || + (parentKind === "approval" && + approvalLeft === "retire" && + !childTerminal) + ) { relays.push({ kind: "signal-abandon", name: awaiting.name }); } else if (!childTerminal) { throw new Error( @@ -7364,7 +7202,8 @@ async function loadPlannedRelays( load === undefined ? undefined : await findParkedApprovalSnapshot({ - repoStore: env.repoStore, + // A relaying body's own body can hold a lost park too. + repoStore: { read: (id) => bodyLogForRelays(env, id) }, load, runId: childRunId, state: childState, @@ -7386,6 +7225,19 @@ async function loadPlannedRelays( return relays; } +// The seq of the parent's record that it spawned `childRunId`, or 0. +function childSpawnSeq( + parentLog: readonly WorkflowEvent[], + childRunId: string, +): number { + for (const event of parentLog) { + if (event.kind === "ChildSpawned" && event.childRunId === childRunId) { + return event.seq; + } + } + return 0; +} + async function planChildWorkflowResume( env: WorkflowRuntimeEnv, stepId: string, @@ -7423,7 +7275,7 @@ async function planChildWorkflowResume( terminalStatus: recorded.terminalStatus, }; } - const childLog = [...(await env.repoStore.read(recorded.childRunId))]; + const childLog = await bodyLogForRelays(env, recorded.childRunId); if (childLog.length === 0) { if (step.phase === "awaiting-signal") { throw new Error( @@ -7459,6 +7311,7 @@ async function planChildWorkflowResume( recorded.childRunId, childState, planChildRelays( + `childWorkflow ${stepId}`, stepId, recorded.childRunId, parentLog, @@ -7466,12 +7319,63 @@ async function planChildWorkflowResume( childLog, childState, env.clock().getTime(), + childSpawnSeq(parentLog, recorded.childRunId), + "fail", ), ), seed: true, }; } +/** + * Wait until the body's park on `name` waits again, so a signal or decision + * this container took before the restart can go down. The park can instead + * end without waiting again, as a nested relay does when its gate timed out + * meanwhile; that read is returned and nothing goes down, as for a park + * already gone before the wait. + */ +async function awaitChildRearm( + handle: SuspendableChildHandle, + stepId: string, + name: string, + whenAwaiter: NonNullable, + abort: AbortSignal, +): Promise<{ kind: "rearmed" } | ChildParkRelease> { + const watch = handle.waitForParkRelease; + if (watch === undefined) { + throw new Error( + `step ${stepId} cannot hand ${name} down: the child handle does not ` + + `watch a park release, so a park that ended would be waited on forever`, + ); + } + const watchAbort = new AbortController(); + const detachWatchAbort = bridgeAbort(abort, () => { + watchAbort.abort(); + }); + const idle = createIdleWait(); + const onRearm = idle.waker(); + const onRelease = idle.waker(); + const released = watch(name, watchAbort.signal, onRelease); + // The awaiter rejects once the body ends, and the watch then reads how. + const rearmed = whenAwaiter(name, onRearm).then( + () => ({ kind: "rearmed" as const }), + () => released, + ); + try { + return await idle.wait(Promise.race([rearmed, released]), [ + onRearm, + onRelease, + ]); + } finally { + detachWatchAbort(); + watchAbort.abort(); + await released.catch(() => undefined); + // The awaiter takes no abort. It settles when the park waits again or + // the body ends. + void rearmed.catch(() => undefined); + } +} + async function drivePlannedChildRelay( env: WorkflowRuntimeEnv, parentRunId: string, @@ -7480,6 +7384,7 @@ async function drivePlannedChildRelay( relay: ChildRelay, abort: AbortSignal, sectionSteps: ReadonlySet, + addressing: SignalRelayAddressing, ): Promise { switch (relay.kind) { case "signal-abandon": { @@ -7488,7 +7393,9 @@ async function drivePlannedChildRelay( } case "signal-reestablish": { const relayLog = await env.repoStore.read(parentRunId); - return raceContainerSignalRelay( + // This run's own parent can hold a signal for this relay that it hands + // down only once the relay waits again, as it would for a re-parked gate. + return raceChildSignalRelease( env, parentRunId, stepId, @@ -7497,8 +7404,10 @@ async function drivePlannedChildRelay( relay.awaitSeq, correlationIdAt(relayLog, relay.awaitSeq), abort, - "park-release", "hold", + () => { + env.onSignalAwaiter?.({ runId: parentRunId, name: relay.name }); + }, ); } case "signal-deliver": { @@ -7509,7 +7418,17 @@ async function drivePlannedChildRelay( if (snapshot.kind === "released") return { kind: "held" }; const whenAwaiter = handle.whenSignalAwaiter; if (whenAwaiter !== undefined) { - await waitIdle((resume) => whenAwaiter(relay.name, resume)); + const rearm = await awaitChildRearm( + handle, + stepId, + relay.name, + whenAwaiter, + abort, + ); + if (rearm.kind === "terminal") { + return { kind: "terminal", terminalStatus: rearm.terminalStatus }; + } + if (rearm.kind === "released") return { kind: "held" }; } await handle.deliverSignal(relay.name, relay.payload, relay.signalId); return { kind: "held" }; @@ -7529,7 +7448,7 @@ async function drivePlannedChildRelay( stepId, handle, relay.name, - "name", + addressing, abort, "park-release", relay.timeout, @@ -7558,7 +7477,17 @@ async function drivePlannedChildRelay( if (snapshot.kind === "released") return { kind: "held" }; const whenAwaiter = handle.whenSignalAwaiter; if (whenAwaiter !== undefined) { - await waitIdle((resume) => whenAwaiter(name, resume)); + const rearm = await awaitChildRearm( + handle, + stepId, + name, + whenAwaiter, + abort, + ); + if (rearm.kind === "terminal") { + return { kind: "terminal", terminalStatus: rearm.terminalStatus }; + } + if (rearm.kind === "released") return { kind: "held" }; } await handle.resume(relay.correlationId, relay.decision); return { kind: "held" }; @@ -7745,6 +7674,7 @@ async function runChildWorkflow( relay, childAbort.signal, sectionSteps, + "name", ); if (driven.kind === "terminal") { terminalStatus = driven.terminalStatus; @@ -7916,20 +7846,107 @@ class ChildWorkflowFailedError extends Error { } } -/** Whether `stepId` has committed a `SignalAwaited` on `name` to the run's log. */ -async function stepAwaitedSignal( - env: WorkflowRuntimeEnv, +/** + * The approval park a step that crashed mid-invocation stored with its pending + * operation but never committed to its run's log, if any. The run re-commits + * it on resume, and a container relaying the run plans from it. `readLog` is + * called only when the step stored one. + */ +async function lostApprovalPark( + env: Pick, runId: string, stepId: string, - name: string, -): Promise { - const events = await env.repoStore.read(runId); - return events.some( + attempt: number, + readLog: () => Promise, +): Promise { + const parkedOps = + env.readParkedApprovalOps !== undefined + ? await env.readParkedApprovalOps({ runId, stepId, attempt }) + : []; + // A step-attempt parks on at most one control-plane suspension. More than + // one durable pending approval op is a corrupt store, not a state the + // runtime can reconcile by picking one -- fail loud. + if (parkedOps.length > 1) { + throw new Error( + `crashed step ${stepId} (attempt ${String(attempt)}) of run ${runId} has ${String(parkedOps.length)} durable pending approval operations; a step-attempt parks on at most one control-plane suspension`, + ); + } + const parked = parkedOps[0]; + if (parked === undefined) return undefined; + // A durable op whose channel this step already awaited in the log is stale, + // not a lost park: that park flushed and a delivery resumed the step, whose + // reactor consumed the op before the store holding it caught up (the + // committed copy of a step that crashed mid-turn trails its local store). + // Re-parking would wait on a decision already spent, so the step is a + // genuine crash mid-turn. + const name = signalName(parked.correlationId); + const awaited = (await readLog()).some( (e) => e.kind === "SignalAwaited" && e.stepId === stepId && e.signalName === name, ); + return awaited ? undefined : parked; +} + +// The `SignalAwaited` a run re-commits for a lost approval park. The +// reactor stores `timeoutAt` as epoch ms; the event carries it as an ISO +// string, and omits it for the indefinite-hold parks that are the norm. +function lostParkAwaited( + seq: number, + at: string, + stepId: string, + park: { correlationId: string; timeoutAtMs?: number }, +): WorkflowEvent { + return { + kind: "SignalAwaited", + seq, + at, + stepId, + signalName: signalName(park.correlationId), + ...(park.timeoutAtMs !== undefined + ? { timeoutAt: new Date(park.timeoutAtMs).toISOString() } + : {}), + }; +} + +/** + * The body log a container plans its relays from: the run's log with the + * `SignalAwaited` the body re-commits on resume for each lost approval park. + * The plan then relays such a park as it does any other, and finds it again + * after a crash that came after the container relayed it but before the body + * re-committed it. Whatever re-registers a container's parks reads its bodies + * through this too, so it finds the same parks. + */ +export async function bodyLogForRelays( + env: Pick< + WorkflowRuntimeEnv, + "repoStore" | "readParkedApprovalOps" | "clock" + >, + childRunId: string, +): Promise { + const log = [...(await env.repoStore.read(childRunId))]; + if (log.length === 0) return log; + const state = resumeFromLog(childRunId, log); + if (isTerminalRunPhase(state.phase)) return log; + const committed = [...log]; + let seq = state.lastSeq; + for (const step of state.steps.values()) { + if (step.phase !== "in-flight") continue; + const parked = await lostApprovalPark( + env, + childRunId, + step.stepId, + step.currentAttempt, + async () => committed, + ); + if (parked === undefined) continue; + seq += 1; + log.push( + lostParkAwaited(seq, env.clock().toISOString(), step.stepId, parked), + ); + } + return log; } // A unit's work succeeded but landing its terminal -- pruning the handler diff --git a/packages/workflow/src/runtime/suspendable-child-handle.ts b/packages/workflow/src/runtime/suspendable-child-handle.ts index 5143239bd..c547ecf60 100644 --- a/packages/workflow/src/runtime/suspendable-child-handle.ts +++ b/packages/workflow/src/runtime/suspendable-child-handle.ts @@ -233,12 +233,16 @@ export function createSuspendableChildHandle( // On resume, drive the run from its durable log; the body step re-parks // silently (a re-park does not re-fire onPark), and the caller relays the - // grant via resume on the correlation it recovered from its own log. On a - // fresh spawn, seed the run with the event's trigger payload. - const baseOptions = - resumeFromEvents !== undefined - ? { runId: childRunId, resumeFromEvents, depth, maxChildSpawnDepth } - : { runId: childRunId, triggerPayload: input, depth, maxChildSpawnDepth }; + // grant via resume on the correlation it recovered from its own log. Either + // way the run is seeded with the event's trigger payload: a log's RunStarted + // overrides it, and a body whose log is gone starts over with it. + const baseOptions = { + runId: childRunId, + triggerPayload: input, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), + depth, + maxChildSpawnDepth, + }; const handle = runtimeRun(definition, runEnv, { ...baseOptions, localAbort: localTeardown.signal, diff --git a/packages/workflow/src/state-machine/events.ts b/packages/workflow/src/state-machine/events.ts index 878dca02c..5ed413a08 100644 --- a/packages/workflow/src/state-machine/events.ts +++ b/packages/workflow/src/state-machine/events.ts @@ -153,7 +153,8 @@ export interface SignalReceived extends EventBase { * another exit -- its own await timeout, or any later body event -- before the * external signal arrives, so a late signal does not resolve a park the body no * longer waits on. Distinct from `SignalReceived`: nothing is delivered, the - * await is simply torn down. + * await is simply torn down. A container also commits this for an approval it + * was relaying when the body ended, so the await cannot swallow its next park. */ export interface SignalAwaitAbandoned extends EventBase { kind: "SignalAwaitAbandoned"; diff --git a/packages/workflow/src/state-machine/transition.test.ts b/packages/workflow/src/state-machine/transition.test.ts index 5e41ed575..60b147146 100644 --- a/packages/workflow/src/state-machine/transition.test.ts +++ b/packages/workflow/src/state-machine/transition.test.ts @@ -785,7 +785,7 @@ describe("applyEvent: correlated signal delivery", () => { correlationId: "corr-other", }, ], - /does not match an active signal-relay park/, + /does not match an active relay park/, ); }); }); @@ -1349,6 +1349,46 @@ describe("applyEvent: SignalAwaitAbandoned", () => { expect(late.unconsumedSignals.get("wake")?.length).toBe(1); }); + test("retires the approval park of a step that relays its child's approval", () => { + const events: WorkflowEvent[] = [ + startRun(), + { + kind: "StepStarted", + seq: 2, + at: T, + stepId: "sec", + attempt: 1, + input: { ref: "i" }, + }, + { + kind: "ChildSpawned", + seq: 3, + at: T, + stepId: "sec", + childRunId: "body", + childDefinitionRef: "body-ref", + }, + { + kind: "SignalAwaited", + seq: 4, + at: T, + stepId: "sec", + signalName: "wake", + parkKind: "approval", + }, + { + kind: "SignalAwaitAbandoned", + seq: 5, + at: T, + stepId: "sec", + signalName: "wake", + }, + ]; + const state = events.reduce(applyEvent, fresh()); + expect(state.steps.get("sec")?.phase).toBe("in-flight"); + expect(state.steps.get("sec")?.awaitingSignal).toBeUndefined(); + }); + test("rejects an abandon that names an approval park", () => { expectThrowsAt( [ @@ -1377,7 +1417,7 @@ describe("applyEvent: SignalAwaitAbandoned", () => { signalName: "wake", }, ], - /does not match an active signal-relay park/, + /does not match an active relay park/, ); }); @@ -1409,7 +1449,7 @@ describe("applyEvent: SignalAwaitAbandoned", () => { signalName: "other", }, ], - /does not match an active signal-relay park/, + /does not match an active relay park/, ); }); diff --git a/packages/workflow/src/state-machine/transition.ts b/packages/workflow/src/state-machine/transition.ts index 355f66a77..3a80cb80f 100644 --- a/packages/workflow/src/state-machine/transition.ts +++ b/packages/workflow/src/state-machine/transition.ts @@ -539,9 +539,9 @@ function consumeAwaiter( } /** - * Retire a step's `signal-relay` await without delivering a payload. Emitted by - * the onTrigger driver when the proxied body progressed by another exit before - * the external signal arrived. + * Retire a step's relay await without delivering a payload. Emitted by the + * onTrigger and loop drivers when the proxied body progressed by another exit + * before the external signal arrived, or ended under its relayed approval. * * Race with `SignalReceived`: the abandon and a delivery can be committed * concurrently for the same await. If the delivery won -- the step is no longer @@ -549,9 +549,11 @@ function consumeAwaiter( * as the signal left it). If the await is still live, retire it: clear the * awaiter and drop the step to `in-flight`. * - * Scope is strict. It touches ONLY a matching `signal-relay` park: an approval - * or input park, a name mismatch, or a correlation-id mismatch is a producer - * bug or a mis-target and is rejected, never silently cleared. It never + * Scope is strict. It touches ONLY a matching `signal-relay` park, or the + * approval park of a step that spawned a child and so relays that child's + * approval: any other approval or input park, a name mismatch, or a + * correlation-id mismatch is a producer bug or a mis-target and is rejected, + * never silently cleared. It never * touches either unconsumed queue, `observedSignalIds`, or * `consumedMessageIds`. A torn-down await does not consume or dedup a signal. * A late name-only delivery still queues for the next name await. A late @@ -572,15 +574,19 @@ function handleSignalAwaitAbandoned( const correlationMismatch = e.correlationId !== undefined && awaiting?.correlationId !== e.correlationId; + const kind = awaiting === undefined ? undefined : controlParkKindOf(awaiting); + const relaysChildApproval = + kind === "approval" && + [...state.children.values()].some((c) => c.spawnedBy === e.stepId); if ( awaiting === undefined || - controlParkKindOf(awaiting) !== "signal-relay" || + (kind !== "signal-relay" && !relaysChildApproval) || awaiting.name !== e.signalName || correlationMismatch ) { throw new TransitionError( "step-phase", - `SignalAwaitAbandoned for step ${e.stepId} on ${e.signalName} does not match an active signal-relay park`, + `SignalAwaitAbandoned for step ${e.stepId} on ${e.signalName} does not match an active relay park`, e, ); } diff --git a/tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts b/tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts index 3fc08cba7..b98f9c574 100644 --- a/tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts +++ b/tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts @@ -20,8 +20,9 @@ // 3. Start a fresh sidecar against the crashed process's SIDECAR_DATA_DIR. // Boot-time restore re-arms the body's unfired timer and re-includes the // container; the container's resume re-adopts the in-flight body -// (`planOnTriggerResume` -> `readopt-in-flight-body`), re-spawns it from its -// log, and the body re-adopts its own sleep timer (`isResumableSleepStep`). +// (`planOnTriggerResume` -> `relink-body` with nothing to relay), re-spawns +// it from its log, and the body re-adopts its own sleep timer +// (`isResumableSleepStep`). // 4. The re-armed timer fires; the body's sleep completes and the container // records `ChildCompleted` for that body -- only AFTER the restart. // From 2c3abd9788e0e5318ec6417f72f8bc90991546f0 Mon Sep 17 00:00:00 2001 From: FarinM Date: Wed, 28 Oct 2026 15:18:45 +0100 Subject: [PATCH 09/20] Measure a workflow child's uptime from its own start The recycle policy kept the first child's start time, so once a max-uptime recycle had run, every later tick recycled the replacement again. --- docs/IMPLEMENTATION.md | 3 +- .../src/supervisor/recycle.test.ts | 89 ++++++++++++++++++- .../src/supervisor/supervisor.ts | 60 +++++++------ 3 files changed, 124 insertions(+), 28 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 0a34c9f98..f5f23d205 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -1032,7 +1032,8 @@ cannot defend against. `maxGrantsAgeMs`. Defaults are unlimited (the bound is disabled unless the operator sets a threshold). On a threshold trip the policy calls `triggerRecycle` with origin `policy` and a reason - naming the tripped bound. + naming the tripped bound. Each child gets its own check, armed + when it starts, so `maxUptimeMs` measures the running child. - **Workflow-process self-initiated.** The child sends a `recycle.request` payload over control IPC when its own self- check decides it needs to be recycled (an internal consistency diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 2d0fa2d03..7fd56d4b5 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -55,7 +55,11 @@ import { type WorkflowSupervisor, type WorkflowSupervisorBindings, } from "./index"; -import { createRecyclePolicy, type RecyclePolicyBounds } from "./recycle"; +import { + createRecyclePolicy, + DEFAULT_POLICY_INTERVAL_MS, + type RecyclePolicyBounds, +} from "./recycle"; import { waitUntil } from "@intx/types/testing"; import { mergeCredentialDelivery } from "../child/credential-cell"; import { parseSpawnTimeEnv } from "../child/env-bootstrap"; @@ -1186,7 +1190,90 @@ describe("supervisor recycle: mail buffered during the kill/respawn gap", () => }); }); +// A supervisor's recycle policy with its clock and ticks driven by hand. Other +// timers on the same bindings run for real. +function manualRecyclePolicy(maxUptimeMs: number) { + let nowMs = 1_700_000_000_000; + const ticks = new Map void>(); + const timers = new Map>(); + let armedNext: PromiseWithResolvers | null = null; + const bindings: Partial = { + recyclePolicy: { maxUptimeMs }, + recyclePolicyNow: () => nowMs, + recyclePolicySetTimer: (cb, ms) => { + const handle = Symbol("timer"); + if (ms === DEFAULT_POLICY_INTERVAL_MS) { + ticks.set(handle, cb); + armedNext?.resolve(undefined); + armedNext = null; + } else { + const timer = setTimeout(() => { + timers.delete(handle); + cb(); + }, ms); + timers.set(handle, timer); + } + return handle; + }, + recyclePolicyClearTimer: (handle) => { + if (typeof handle !== "symbol") return; + ticks.delete(handle); + clearTimeout(timers.get(handle)); + timers.delete(handle); + }, + }; + return { + bindings, + advance(ms: number): void { + nowMs += ms; + }, + armed: () => ticks.size, + // Resolves when a policy next arms its tick. + nextArm(): Promise { + armedNext = Promise.withResolvers(); + return armedNext.promise; + }, + tick(): void { + const due = [...ticks.values()]; + ticks.clear(); + for (const cb of due) cb(); + }, + }; +} + describe("supervisor recycle: policy-initiated (max-uptime trip)", () => { + test("each replacement child's uptime counts from its own start", async () => { + const baseDir = await makeTempDir("recycle-policy-child-"); + const ipcKeypair = await generateKeyPair(); + const tracker = createSpawnTracker({ sigtermExits: true }); + const policy = manualRecyclePolicy(60_000); + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: policy.bindings, + }); + expect(policy.armed()).toBe(1); + + policy.advance(61_000); + const replaced = policy.nextArm(); + policy.tick(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await replaced; + expect(policy.armed()).toBe(1); + + const ticked = policy.nextArm(); + policy.tick(); + await ticked; + expect(tracker.totalSpawns).toBe(2); + + await supervisor.shutdown(); + }); + test("the policy's max-uptime threshold trips recycle with a policy-origin reason", async () => { const baseDir = await makeTempDir("recycle-policy-"); const ipcKeypair = await generateKeyPair(); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index f9830601c..5a75e8e9d 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -3383,30 +3383,7 @@ export function createWorkflowSupervisor( logger.warn`re-emit of parked correlations on re-establishment failed: ${message}`; }); - // Arm the recycle policy. The policy is a no-op when all bounds - // are `undefined`; bounds resolution lives inside `createRecyclePolicy`. - if (bindings.recyclePolicy !== undefined) { - const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; - const clearTimer = - bindings.recyclePolicyClearTimer ?? defaultClearTimer; - recyclePolicy = createRecyclePolicy({ - bounds: bindings.recyclePolicy, - now, - spawnedAt: spawnContext.spawnedAt, - ...(bindings.readRssBytes !== undefined - ? { readRssBytes: bindings.readRssBytes } - : {}), - ...(bindings.readGrantsAgeMs !== undefined - ? { readGrantsAgeMs: bindings.readGrantsAgeMs } - : {}), - setTimer, - clearTimer, - trigger: async (reason) => { - await recycle({ reason, origin: "policy" }); - }, - }); - } - + armRecyclePolicy(spawnContext.spawnedAt); return { pid: childPid, channelId, @@ -3432,6 +3409,38 @@ export function createWorkflowSupervisor( } } + // Arm the recycle policy for the child that just started at `spawnedAt`, + // replacing the previous child's, so its bounds measure the running child. + // The policy is a no-op when all bounds are `undefined`; bounds resolution + // lives inside `createRecyclePolicy`. + function armRecyclePolicy(spawnedAt: number): void { + stopRecyclePolicy(); + if (bindings.recyclePolicy === undefined) return; + const setTimer = bindings.recyclePolicySetTimer ?? defaultSetTimer; + const clearTimer = bindings.recyclePolicyClearTimer ?? defaultClearTimer; + recyclePolicy = createRecyclePolicy({ + bounds: bindings.recyclePolicy, + now: bindings.recyclePolicyNow ?? defaultNow, + spawnedAt, + ...(bindings.readRssBytes !== undefined + ? { readRssBytes: bindings.readRssBytes } + : {}), + ...(bindings.readGrantsAgeMs !== undefined + ? { readGrantsAgeMs: bindings.readGrantsAgeMs } + : {}), + setTimer, + clearTimer, + trigger: async (reason) => { + await recycle({ reason, origin: "policy" }); + }, + }); + } + + function stopRecyclePolicy(): void { + recyclePolicy?.stop(); + recyclePolicy = null; + } + /** * Project the active cohort's terminal broadcaster as a * `TerminalEventSource` the drainTimeout accumulator factory accepts. @@ -4939,14 +4948,13 @@ export function createWorkflowSupervisor( // drives a spurious respawn and a crash of THIS child is // classified against the new generation. armChildForRunning(wiring.handle); - // Cache fresh spawn context with the updated spawnedAt - // so the policy timer's uptime check resets on recycle. const now = bindings.recyclePolicyNow ?? defaultNow; spawnContext = { definitionHash: priorContext.definitionHash, onInferenceEvent: priorContext.onInferenceEvent, spawnedAt: now(), }; + armRecyclePolicy(spawnContext.spawnedAt); // Re-arm the upstream control pump on the new wiring's // iterator. The old wiring's iterator ended when the // recycle path killed the predecessor handle. The new From 4f2e613a88cc21df13aaf9b17600e42da778d60c Mon Sep 17 00:00:00 2001 From: FarinM Date: Wed, 28 Oct 2026 15:22:01 +0100 Subject: [PATCH 10/20] Finish a respawn's mail replay before shutdown resolves A shutdown during a respawn did not wait for its replay of processing/, which could write after the host removed the run repository and recreate it. A replay that has not begun when shutdown starts no longer runs. --- .../src/supervisor/recycle.test.ts | 85 +++++++++++++++++++ .../src/supervisor/supervisor.ts | 45 +++++++--- .../workflow-host/src/supervisor/types.ts | 7 +- 3 files changed, 121 insertions(+), 16 deletions(-) diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 7fd56d4b5..5b4f3e867 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -2028,6 +2028,91 @@ describe("supervisor recycle: external drain phase guard", () => { }); }); +describe("supervisor recycle: shutdown during the replay", () => { + test("a shutdown during a respawn's replay resolves once its write lands", async () => { + const baseDir = await makeTempDir("recycle-replay-shutdown-"); + const ipcKeypair = await generateKeyPair(); + const tracker = createSpawnTracker({ sigtermExits: true }); + const memory = createMemoryInboxPrimitives(); + const replaying = Promise.withResolvers(); + const release = Promise.withResolvers(); + const order: string[] = []; + let replays = 0; + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: { + onShutdownWait: (waiting) => { + if (waiting === "replay") release.resolve(undefined); + }, + inboxPrimitives: { + ...memory, + async replayProcessingToInbox(...args) { + replays += 1; + if (replays === 1) return memory.replayProcessingToInbox(...args); + replaying.resolve(undefined); + await release.promise; + const replayed = await memory.replayProcessingToInbox(...args); + order.push("replayed"); + return replayed; + }, + }, + }, + }); + + const recycled = supervisor + .recycle({ reason: "replay-shutdown-test" }) + .catch(() => undefined); + await replaying.promise; + const stopped = supervisor.shutdown().then(() => order.push("stopped")); + await stopped; + expect(order).toEqual(["replayed", "stopped"]); + await recycled; + }); + + test("a respawn's replay that has not begun when shutdown starts does not run", async () => { + const baseDir = await makeTempDir("recycle-replay-not-begun-"); + const ipcKeypair = await generateKeyPair(); + const tracker = createSpawnTracker({ sigtermExits: true }); + const memory = createMemoryInboxPrimitives(); + const bootReplayHeld = Promise.withResolvers(); + const respawnWaits = Promise.withResolvers(); + let replays = 0; + const { supervisor } = await spawnSupervisor({ + baseDir, + tracker, + mailBus: createMockMailBus(), + ipcKeypair, + overrides: { + onRespawnWait: (waiting) => { + if (waiting === "replay") respawnWaits.resolve(undefined); + }, + inboxPrimitives: { + ...memory, + async replayProcessingToInbox(...args) { + replays += 1; + if (replays === 1) await bootReplayHeld.promise; + return memory.replayProcessingToInbox(...args); + }, + }, + }, + }); + + // The respawn waits for the boot replay, so its own has not begun. + const recycled = supervisor + .recycle({ reason: "replay-not-begun-test" }) + .catch(() => undefined); + await respawnWaits.promise; + const stopped = supervisor.shutdown(); + bootReplayHeld.resolve(undefined); + await stopped; + await recycled; + expect(replays).toBe(1); + }); +}); + describe("supervisor shutdown: teardown robustness", () => { test("shutdown still kills the child and reaches stopped when mail unsubscribe throws", async () => { const baseDir = await makeTempDir("shutdown-unsub-throw-"); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 5a75e8e9d..4056231e7 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -1979,6 +1979,8 @@ export function createWorkflowSupervisor( // Set once shutdown has waited for the last of them. A write started after // that would land after the host may have removed the run's repository. let detachedWritesClosed = false; + // A respawn's replay of `processing/`, which shutdown waits for. + const respawnReplays = new Set>(); function ownDetachedWrite(write: Promise): void { detachedWrites.add(write); @@ -4492,6 +4494,10 @@ export function createWorkflowSupervisor( the in-flight fold's substrate commit to settle. */ }); } + if (respawnReplays.size > 0) { + bindings.onShutdownWait?.("replay"); + await Promise.allSettled([...respawnReplays]); + } // Own the detached writes last, so every write lands before shutdown // resolves: the child's, and the consumes the waits above can start. New // merges fail once stopping, so what remains is local I/O. A finishing @@ -4823,22 +4829,35 @@ export function createWorkflowSupervisor( bindings.onRespawnWait?.("writes"); await Promise.all([...detachedWrites]); } - const { ownedMessageIds } = await scanRunsForBoot( - bindings.repoStore, - supervisorPrincipal, - bindings.workflowRunRepoId, - bindings.workflowRunRef, - { runId: deriveWorkflowRunId(bindings.deploymentMailAddress) }, - ); - const { retainedMessageIds } = - await inboxPrimitives.replayProcessingToInbox( + // Shutdown waits for a replay that began before it; one after it + // would write a repository its host may already have removed. + if (state.phase !== "recycling") { + throw new Error("the workflow supervisor is shutting down"); + } + const replay = (async () => { + const { ownedMessageIds } = await scanRunsForBoot( bindings.repoStore, - inboxWritePrincipal, + supervisorPrincipal, bindings.workflowRunRepoId, - bindings.deploymentMailAddress, - { ownedMessageIds }, + bindings.workflowRunRef, + { runId: deriveWorkflowRunId(bindings.deploymentMailAddress) }, ); - keepOwnedMessages(retainedMessageIds, { childRunning: false }); + const { retainedMessageIds } = + await inboxPrimitives.replayProcessingToInbox( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + { ownedMessageIds }, + ); + keepOwnedMessages(retainedMessageIds, { childRunning: false }); + })(); + respawnReplays.add(replay); + try { + await replay; + } finally { + respawnReplays.delete(replay); + } }, abortPriorCohort: () => { // Fired by triggerRecycle between the drain and the kill. diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index 8a4cf89c6..33d944abf 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -321,10 +321,11 @@ export interface WorkflowSupervisorBindings { }) => void; /** * Test seam, unset in production. Called each time a shutdown waits for - * work it owns before it resolves: `writes` for repository writes started - * off the control pump. + * work it owns before it resolves: `replay` for a respawn's replay of + * `processing/`, `writes` for repository writes started off the control + * pump. */ - onShutdownWait?: (waiting: "writes") => void; + onShutdownWait?: (waiting: "replay" | "writes") => void; /** * Test seam, unset in production. Called each time a respawn, before it * scans `runs/`, waits for the boot replay (`replay`) or for repository From 07b2293bbbb96b0901d661e6db4386532fda450d Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 2 Nov 2026 14:44:01 +0100 Subject: [PATCH 11/20] Move a workflow event's on-disk shape out of the run store adapter The supervisor needs only this shape, but importing it loaded the whole store adapter into the sidecar's boot, along with everything the adapter depends on. --- .../src/adapters/on-disk-event.ts | 21 +++++++++++++++++++ .../workflow-host/src/adapters/repo-store.ts | 18 ++-------------- packages/workflow-host/src/seams/scheduler.ts | 6 +++--- .../src/supervisor/terminal-commit.ts | 2 +- 4 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 packages/workflow-host/src/adapters/on-disk-event.ts diff --git a/packages/workflow-host/src/adapters/on-disk-event.ts b/packages/workflow-host/src/adapters/on-disk-event.ts new file mode 100644 index 000000000..d3b65dfec --- /dev/null +++ b/packages/workflow-host/src/adapters/on-disk-event.ts @@ -0,0 +1,21 @@ +// The flat shape of a workflow event on disk. The supervisor writes events too, +// and the sidecar that loads it must not load the workflow runtime, so this +// stays apart from the runtime's store adapter. + +import type { WorkflowEvent } from "@intx/workflow"; + +/** + * Translate a state-machine `WorkflowEvent` (using `kind` as the + * discriminator) into the on-disk envelope shape (`{seq, type, + * ...rest}`) the workflow-run kind handler validates and the + * substrate's `subscribeKind` helper filters on. Exported so the + * supervisor's terminal-commit path encodes a supervisor-authored + * `RunFailed` through the same single source of the on-disk shape. + */ +export function workflowEventToOnDisk( + event: WorkflowEvent, + seq: number, +): Record { + const { kind, seq: _eventSeq, ...rest } = event; + return { seq, type: kind, ...rest }; +} diff --git a/packages/workflow-host/src/adapters/repo-store.ts b/packages/workflow-host/src/adapters/repo-store.ts index 8396b24e3..723457ade 100644 --- a/packages/workflow-host/src/adapters/repo-store.ts +++ b/packages/workflow-host/src/adapters/repo-store.ts @@ -42,6 +42,8 @@ import { } from "@intx/hub-sessions/substrate"; import type { RepoStore, WorkflowEvent } from "@intx/workflow"; +import { workflowEventToOnDisk } from "./on-disk-event"; + /** * Local handle for the runtime-env subscribe options. The workflow * package's `RepoStore` interface declares the shape inline but does @@ -300,22 +302,6 @@ function onDiskToWorkflowEvent( return built as unknown as WorkflowEvent; } -/** - * Translate a state-machine `WorkflowEvent` (using `kind` as the - * discriminator) into the on-disk envelope shape (`{seq, type, - * ...rest}`) the workflow-run kind handler validates and the - * substrate's `subscribeKind` helper filters on. Exported so the - * supervisor's terminal-commit path encodes a supervisor-authored - * `RunFailed` through the same single source of the on-disk shape. - */ -export function workflowEventToOnDisk( - event: WorkflowEvent, - seq: number, -): Record { - const { kind, seq: _eventSeq, ...rest } = event; - return { seq, type: kind, ...rest }; -} - /** * Append one or more contiguous events to a run's event log in a * SINGLE durable commit. The events must carry strictly-monotonic, diff --git a/packages/workflow-host/src/seams/scheduler.ts b/packages/workflow-host/src/seams/scheduler.ts index b9238cb2d..d42d9e89c 100644 --- a/packages/workflow-host/src/seams/scheduler.ts +++ b/packages/workflow-host/src/seams/scheduler.ts @@ -28,8 +28,8 @@ // The scheduler reads workflow-event blobs at the canonical layout // `runs//events/.json` and writes a fresh blob at the // next-seq slot for each `TimerFired` commit. The blob envelope is -// FLAT -- `{ seq, type, ...eventFields }` -- matching the runtime -// repo-store's `workflowEventToOnDisk` and the workflow-run kind +// FLAT -- `{ seq, type, ...eventFields }` -- matching +// `workflowEventToOnDisk` and the workflow-run kind // handler's `EventEnvelope` contract: `seq` is the integer that also // appears in the filename, `type` is the `subscribeKind` discriminator // the scheduler narrows on, and the timer fields (`timerId`, `fireAt`, @@ -76,7 +76,7 @@ import { export const TimerEventEnvelope = type({ seq: "number >= 0", type: "'TimerSet' | 'TimerFired'", - // FLAT on-disk shape, matching the runtime repo-store's `workflowEventToOnDisk` + // FLAT on-disk shape, matching `workflowEventToOnDisk` // ({ seq, type, ...eventFields }) and the substrate's enforced `EventEnvelope` // contract -- NOT a nested `data` object. The scheduler was the lone component // writing/reading a nested `data` envelope; nothing else parsed it, so a diff --git a/packages/workflow-host/src/supervisor/terminal-commit.ts b/packages/workflow-host/src/supervisor/terminal-commit.ts index 2bc6e2e64..810df534b 100644 --- a/packages/workflow-host/src/supervisor/terminal-commit.ts +++ b/packages/workflow-host/src/supervisor/terminal-commit.ts @@ -27,7 +27,7 @@ import type { WorkflowRunSupervisorPrincipal, } from "@intx/hub-sessions/substrate"; -import { workflowEventToOnDisk } from "../adapters/repo-store"; +import { workflowEventToOnDisk } from "../adapters/on-disk-event"; const logger = getLogger(["workflow-host", "supervisor", "terminal-commit"]); From bfc94d68a12a362eb7293c63ada3c64410e98adb Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 2 Nov 2026 14:10:15 +0100 Subject: [PATCH 12/20] Replay a run's events from its own log to a subscriber The runtime's subscribe contract replays from a seq, but the child's proxy store only tails from head, so a consumer missed whatever landed before it subscribed. A timed gate woken by its due timer waited forever on a TimerFired already in its log. --- .../src/adapters/repo-store.test.ts | 147 ++++++++++++++++++ .../workflow-host/src/adapters/repo-store.ts | 58 +++++-- .../src/child/proxy-repo-store.ts | 10 +- 3 files changed, 197 insertions(+), 18 deletions(-) diff --git a/packages/workflow-host/src/adapters/repo-store.test.ts b/packages/workflow-host/src/adapters/repo-store.test.ts index a9ada7474..a19e7a329 100644 --- a/packages/workflow-host/src/adapters/repo-store.test.ts +++ b/packages/workflow-host/src/adapters/repo-store.test.ts @@ -18,6 +18,7 @@ import type { ValidatePushResult, } from "@intx/hub-sessions"; import type { WorkflowEvent } from "@intx/workflow"; +import { isQuiet, whenDelivered } from "@intx/workflow/runtime"; import { createWorkflowRunRepoStore } from "./repo-store"; import { compactRunEvents } from "../supervisor/run-event-compaction"; @@ -429,6 +430,152 @@ describe("workflow-host RepoStore adapter — first-seq contract against runtime }); }); +describe("workflow-host RepoStore adapter — subscribe", () => { + // The child's proxy store has no replay, so the substrate is only ever asked + // for a tail from head. + async function subscribeHarness(name: string) { + const dataDir = await makeTempDir(`repo-store-adapter-${name}-`); + const repoId: RepoId = { kind: "workflow-run", id: `deployment-${name}` }; + const substrate = createRepoStore({ + dataDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + await substrate.writeTree({ kind: "hub" }, repoId, REF, { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + const froms: unknown[] = []; + const adapter = createWorkflowRunRepoStore({ + substrate: { + ...substrate, + subscribe(principal, id, ref, opts) { + froms.push(opts.from); + return substrate.subscribe(principal, id, ref, opts); + }, + }, + repoId, + principal: WORKFLOW_PROCESS_PRINCIPAL, + ref: REF, + }); + return { adapter, froms }; + } + + test("a seq replays the run's committed events from there, then goes live", async () => { + const { adapter, froms } = await subscribeHarness("replay"); + const runId = "run-replay"; + await adapter.append(runId, freshRunStarted(runId, 1)); + await adapter.append(runId, freshStepStarted(2)); + await adapter.append(runId, freshStepStarted(3)); + await adapter.append("run-other", freshRunStarted("run-other", 1)); + + const abort = new AbortController(); + const iter = adapter.subscribe(runId, { + signal: abort.signal, + from: { seq: 2 }, + }); + expect((await iter.next()).value?.seq).toBe(2); + expect((await iter.next()).value?.seq).toBe(3); + const live = iter.next(); + await adapter.append("run-other", freshStepStarted(2)); + await adapter.append(runId, freshStepStarted(4)); + expect((await live).value?.seq).toBe(4); + expect(froms).toEqual(["head"]); + abort.abort(); + }); + + test("a commit during the replay's read reaches the consumer once", async () => { + const { adapter } = await subscribeHarness("during-read"); + const runId = "run-during-read"; + await adapter.append(runId, freshRunStarted(runId, 1)); + + const abort = new AbortController(); + const iter = adapter.subscribe(runId, { + signal: abort.signal, + from: { seq: 1 }, + }); + const first = iter.next(); + await adapter.append(runId, freshStepStarted(2)); + await adapter.append(runId, freshStepStarted(3)); + + const seqs = [(await first).value?.seq]; + while (seqs.at(-1) !== 3) seqs.push((await iter.next()).value?.seq); + expect(seqs).toEqual([1, 2, 3]); + const next = iter.next(); + abort.abort(); + expect((await next).done).toBe(true); + }); + + test("head replays nothing", async () => { + const { adapter } = await subscribeHarness("head"); + const runId = "run-head"; + await adapter.append(runId, freshRunStarted(runId, 1)); + + const abort = new AbortController(); + const iter = adapter.subscribe(runId, { + signal: abort.signal, + from: "head", + }); + const live = iter.next(); + await adapter.append(runId, freshStepStarted(2)); + expect((await live).value?.seq).toBe(2); + abort.abort(); + }); + + test("a consumer that returns during the replay closes the iterator", async () => { + const { adapter } = await subscribeHarness("return"); + const runId = "run-return"; + await adapter.append(runId, freshRunStarted(runId, 1)); + await adapter.append(runId, freshStepStarted(2)); + + const iter = adapter.subscribe(runId, { + signal: new AbortController().signal, + from: { seq: 1 }, + }); + expect((await iter.next()).value?.seq).toBe(1); + await iter.return?.(); + expect((await iter.next()).done).toBe(true); + }); + + test("an abort during the replay ends the iterator without a throw", async () => { + const { adapter } = await subscribeHarness("abort"); + const runId = "run-abort"; + await adapter.append(runId, freshRunStarted(runId, 1)); + await adapter.append(runId, freshStepStarted(2)); + + const abort = new AbortController(); + const iter = adapter.subscribe(runId, { + signal: abort.signal, + from: { seq: 1 }, + }); + expect((await iter.next()).value?.seq).toBe(1); + abort.abort(); + expect((await iter.next()).done).toBe(true); + }); + + test("a replayed event counts as a delivery until the consumer comes back", async () => { + const { adapter } = await subscribeHarness("delivery"); + const runId = "run-delivery"; + await adapter.append(runId, freshRunStarted(runId, 1)); + + const abort = new AbortController(); + const iter = adapter.subscribe(runId, { + signal: abort.signal, + from: { seq: 1 }, + }); + const first = iter.next(); + expect(isQuiet()).toBe(false); + await first; + expect(isQuiet()).toBe(false); + const live = iter.next(); + await whenDelivered(); + expect(isQuiet()).toBe(true); + abort.abort(); + expect((await live).done).toBe(true); + }); +}); + describe("workflow-host RepoStore adapter — permissive handler smoke", () => { test("write+read round-trip via permissive handler preserves event fields", async () => { const dataDir = await makeTempDir("repo-store-adapter-smoke-"); diff --git a/packages/workflow-host/src/adapters/repo-store.ts b/packages/workflow-host/src/adapters/repo-store.ts index 723457ade..7e1b1620e 100644 --- a/packages/workflow-host/src/adapters/repo-store.ts +++ b/packages/workflow-host/src/adapters/repo-store.ts @@ -41,6 +41,7 @@ import { splitCombinedEventLog, } from "@intx/hub-sessions/substrate"; import type { RepoStore, WorkflowEvent } from "@intx/workflow"; +import { beginDelivery } from "@intx/workflow/runtime"; import { workflowEventToOnDisk } from "./on-disk-event"; @@ -414,19 +415,21 @@ async function* subscribeRun( // workflow-event `type` keeps the runtime body's contract intact (it // wants every event for the run, not a subset). The substrate helper // also surfaces per-run attribution via the entry's `runId`, which - // we use to filter just this run's events. Replay-then-live mode - // mirrors `SubscribeOpts.from`: `"head"` emits only events committed - // strictly after subscription, `{ seq }` enumerates prior events at - // or after the supplied seq before transitioning to live. + // we use to filter just this run's events. + // + // `{ seq }` is a seq in this run's log, which the substrate's replay does + // not count in, and the child's proxy store does not replay at all. So the + // tail always starts at head and the replay comes from the run's log. + const stopTail = new AbortController(); const kindOpts: Parameters[5] = { - signal: subOpts.signal, - from: subOpts.from, + signal: AbortSignal.any([subOpts.signal, stopTail.signal]), + from: "head", kinds: ALL_WORKFLOW_EVENT_TYPES, }; if (subOpts.bufferLimit !== undefined) { kindOpts.bufferLimit = subOpts.bufferLimit; } - const iter = subscribeKind( + const tail = subscribeKind( opts.substrate, opts.principal, opts.repoId, @@ -434,9 +437,42 @@ async function* subscribeRun( OnDiskEnvelope, kindOpts, ); - for await (const entry of iter) { - if (entry.runId !== runId) continue; - const event = onDiskToWorkflowEvent(entry.event); - yield { seq: entry.event.seq, event }; + // This pull attaches the tail. A commit during the read below is then in the + // tail even when the read missed it, and the seq drops it when it did not. + let pending = tail.next(); + void pending.catch(() => undefined); + let replayedThrough = 0; + try { + if (subOpts.from !== "head") { + // Committed input can sit in this read with no subscriber to count it, + // so it counts until the consumer has taken what the read found. + const endRead = beginDelivery(); + try { + for (const event of await readAllEventsForRun(opts, runId)) { + if (event.seq < subOpts.from.seq) continue; + if (subOpts.signal.aborted) return; + replayedThrough = event.seq; + yield { seq: event.seq, event }; + } + } finally { + endRead(); + } + } + for (;;) { + const next = await pending; + if (next.done === true) return; + const entry = next.value; + if (entry.runId === runId && entry.event.seq > replayedThrough) { + const event = onDiskToWorkflowEvent(entry.event); + yield { seq: entry.event.seq, event }; + } + pending = tail.next(); + } + } finally { + // A return closes a generator only between its pulls, so end the pull + // still pending before closing the tail. + stopTail.abort(); + await pending.catch(() => undefined); + await tail.return(); } } diff --git a/packages/workflow-host/src/child/proxy-repo-store.ts b/packages/workflow-host/src/child/proxy-repo-store.ts index 12434aa64..dd6686290 100644 --- a/packages/workflow-host/src/child/proxy-repo-store.ts +++ b/packages/workflow-host/src/child/proxy-repo-store.ts @@ -253,13 +253,9 @@ export function createProxyWorkflowRunRepoStore( // proxy's `writeTreePreservingPrefix` returns successfully. // // `from: { seq }` replay against historical commits is not - // emitted here today: the runtime body's signal-channel and - // similar consumers attach with `from: "head"` so only events - // committed after subscription fire. A `from: { seq }` caller - // (a resume path that wants to replay) would currently miss - // historical commits -- the bare store's `subscribe` is the - // path that supports replay today, and resume code can call it - // directly if needed. + // emitted here. Every consumer attaches with `from: "head"`, and + // the workflow-run adapter replays a run's events from the run's + // own log. const key = refKey(repoId, ref); let set = subscribers.get(key); if (set === undefined) { From 0b21e15332684344a219ddcf929b3ba52daa4caf Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 2 Nov 2026 14:11:13 +0100 Subject: [PATCH 13/20] Let a re-parked timed gate see a timer that already fired A re-park read the run's state twice. A TimerFired between the reads left no timer to adopt, so the gate minted a second TimerSet with a fresh deadline. A gate that lost its timer watch also waited on with no deadline at all, so that now fails the run. A gate its own timer moved also resumes when another gate shares its name. The refusal there guards against binding a signal's payload to the wrong gate, and a timeout has no payload. --- .../runtime/resume-awaiting-signal.test.ts | 267 ++++++++++++++++++ packages/workflow/src/runtime/run.ts | 66 ++++- 2 files changed, 319 insertions(+), 14 deletions(-) diff --git a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts index a2ea3b170..65a34a323 100644 --- a/packages/workflow/src/runtime/resume-awaiting-signal.test.ts +++ b/packages/workflow/src/runtime/resume-awaiting-signal.test.ts @@ -106,6 +106,22 @@ const twoGatesSameName = ((): WorkflowDefinition => { return def; })(); +const sequentialGates = defineWorkflow({ + id: "wait-resume-sequential-gates", + trigger: { type: "manual" }, + steps: { + first: awaitSignal({ name: "go" }), + w: awaitSignal({ + name: "go", + timeout: 60_000, + onTimeout: "recover", + after: ["first"], + }), + recover: step({ agent: resumeAgent("recover"), after: ["w"] }), + normal: step({ agent: resumeAgent("normal"), after: ["w"] }), + }, +}); + const gateThenStep = defineWorkflow({ id: "wait-resume-gate-then-step", trigger: { type: "manual" }, @@ -522,6 +538,146 @@ describe("resume awaiting signal", () => { } }); + describe("a timed gate after a completed gate of the same name", () => { + const signal = (seq: number, signalId: string): WorkflowEvent => ({ + kind: "SignalReceived", + seq, + at, + signalName: "go", + signalId, + payload: { signalId }, + }); + const firstGate = (runId: string): WorkflowEvent[] => [ + runStartedSeed(runId), + { + kind: "StepStarted", + seq: 2, + at, + stepId: "first", + attempt: 1, + input: { ref: "inline:null" }, + }, + { kind: "SignalAwaited", seq: 3, at, stepId: "first", signalName: "go" }, + signal(4, "sig-first"), + { + kind: "StepCompleted", + seq: 5, + at, + stepId: "first", + attempt: 1, + output: { ref: "inline:null" }, + }, + ]; + const started = (seq: number): WorkflowEvent => ({ + kind: "StepStarted", + seq, + at, + stepId: "w", + attempt: 1, + input: { ref: "inline:null" }, + }); + const awaited = (seq: number): WorkflowEvent => ({ + kind: "SignalAwaited", + seq, + at, + stepId: "w", + signalName: "go", + timeoutAt: new Date(Date.now() + 60_000).toISOString(), + }); + const timerSet = (seq: number): WorkflowEvent => ({ + kind: "TimerSet", + seq, + at, + timerId: "timer-1", + fireAt: new Date(Date.now() + 60_000).toISOString(), + stepId: "w", + }); + const timerFired = (seq: number): WorkflowEvent => ({ + kind: "TimerFired", + seq, + at, + timerId: "timer-1", + }); + + test("routes to its handler when its own timer moved it", async () => { + const runId = "run-sequential-timeout"; + const invoked: string[] = []; + const env = buildEnv(sequentialGates, { + invokeStep: async (req) => { + invoked.push(req.agent.id); + return { output: null }; + }, + }); + const result = await runtimeRun(sequentialGates, env, { + runId, + resumeFromEvents: [ + ...firstGate(runId), + started(6), + awaited(7), + timerSet(8), + timerFired(9), + ], + }).complete; + + expect(result.terminalStatus).toBe("completed"); + expect(invoked).toEqual(["recover"]); + const completed = result.events.find( + (e) => e.kind === "StepCompleted" && e.stepId === "w", + ); + if (completed?.kind !== "StepCompleted") + throw new Error("gate not completed"); + expect(await env.blobs.resolveRef(completed.output.ref)).toEqual({ + timedOut: true, + }); + }); + + test.each([ + [ + "a signal moved it before the timer fired", + [ + started(6), + awaited(7), + timerSet(8), + signal(9, "sig-w"), + timerFired(10), + ], + ], + [ + "it drained a queued signal and the timer fired after", + [ + signal(6, "sig-w"), + started(7), + awaited(8), + timerSet(9), + timerFired(10), + ], + ], + ])("is refused when %s", async (_name, tail) => { + const runId = "run-sequential-signal"; + const invoked: string[] = []; + const env = buildEnv(sequentialGates, { + invokeStep: async (req) => { + invoked.push(req.agent.id); + return { output: null }; + }, + }); + const result = await runtimeRun(sequentialGates, env, { + runId, + resumeFromEvents: [...firstGate(runId), ...tail], + }).complete; + + expect(result.terminalStatus).toBe("failed"); + expect(invoked).toEqual([]); + const failed = result.events.find( + (e) => e.kind === "StepFailed" && e.stepId === "w", + ); + if (failed?.kind !== "StepFailed") throw new Error("gate did not fail"); + expect(failed.error.message).toContain( + "another awaitSignal gate for go awaited the signal on a different step", + ); + }); + }); + test("a timed gate resumes on the signal branch when the signal arrived first", async () => { const runId = "run-timed-signal"; const env = buildEnv(gateOnlyTimeout); @@ -815,6 +971,117 @@ describe("resume awaiting signal", () => { }); }); +describe("timer fired while a timed gate re-parks", () => { + function timedGateSeed(runId: string): WorkflowEvent[] { + const fireAt = new Date(Date.now() + 60_000).toISOString(); + return [ + runStartedSeed(runId), + { + kind: "StepStarted", + seq: 2, + at, + stepId: "w", + attempt: 1, + input: { ref: "inline:null" }, + }, + { + kind: "SignalAwaited", + seq: 3, + at, + stepId: "w", + signalName: "go", + timeoutAt: fireAt, + }, + { kind: "TimerSet", seq: 4, at, timerId: "timer-1", fireAt, stepId: "w" }, + ]; + } + + test("the gate times out once wherever the TimerFired lands", async () => { + // Lands the commit after each read the resumed run makes in turn, and last + // once the gate has armed its timer. + let parkedFirst = false; + for (let landAfterRead = 1; !parkedFirst; landAfterRead += 1) { + if (landAfterRead > 50) throw new Error("the gate never armed its timer"); + const runId = `run-timer-window-${String(landAfterRead)}`; + const invoked: string[] = []; + const base = buildEnv(timedGateWithHandler, { + invokeStep: async (req) => { + invoked.push(req.agent.id); + return { output: null }; + }, + }); + let reads = 0; + let landed: Promise | undefined; + const land = (tip: number): Promise => + (landed ??= base.repoStore.append(runId, { + kind: "TimerFired", + seq: tip + 1, + at, + timerId: "timer-1", + })); + const env: WorkflowRuntimeEnv = { + ...base, + repoStore: { + ...base.repoStore, + read: async (id) => { + const log = [...(await base.repoStore.read(id))]; + const tip = log.at(-1); + if (id === runId && tip !== undefined) { + reads += 1; + if (reads === landAfterRead) await land(tip.seq); + } + return log; + }, + }, + scheduler: { + scheduleIn: (id, timerId, fireAt) => { + if (landed === undefined) { + parkedFirst = true; + void land(4); + } + return base.scheduler.scheduleIn(id, timerId, fireAt); + }, + }, + }; + + const result = await runtimeRun(timedGateWithHandler, env, { + runId, + resumeFromEvents: timedGateSeed(runId), + }).complete; + + expect(result.terminalStatus).toBe("completed"); + expect(invoked).toEqual(["recover"]); + expect(result.events.filter((e) => e.kind === "TimerSet")).toHaveLength( + 1, + ); + } + }); + + test("a gate whose timer watch fails fails the run instead of waiting", async () => { + const runId = "run-timer-watch-fails"; + const base = buildEnv(timedGateWithHandler); + const env: WorkflowRuntimeEnv = { + ...base, + repoStore: { + ...base.repoStore, + subscribe: () => ({ + [Symbol.asyncIterator]() { + return this; + }, + next: () => Promise.reject(new Error("the log could not be read")), + }), + }, + }; + + const result = await runtimeRun(timedGateWithHandler, env, { + runId, + resumeFromEvents: timedGateSeed(runId), + }).complete; + + expect(result.terminalStatus).toBe("failed"); + }); +}); + describe("signal committed before the wait starts", () => { test("a re-parked gate takes a signal committed between reading the gate and starting its wait", async () => { const runId = "run-gap"; diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index e67e4f7e1..644d06704 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -5633,9 +5633,9 @@ function gateTimerId( * name-addressed awaiter of the name, so a second same-name awaiter -- even * one that already COMPLETED -- breaks that assumption: the log can no longer * say which gate consumed which delivery. The in-flight short-circuit refuses - * that topology rather than risk binding a payload to the wrong gate; only a - * run where `selfStepId` is the sole name-addressed awaiter of the name is - * provably unambiguous. A completed same-name sibling is the case a + * a signal in that topology rather than risk binding a payload to the wrong + * gate; only a run where `selfStepId` is the sole name-addressed awaiter of the + * name is provably unambiguous. A completed same-name sibling is the case a * phase-scoped in-flight count would miss, so the predicate keys on the * durable `SignalAwaited` marker, which outlives the sibling's completion. */ @@ -5657,6 +5657,27 @@ export function hasForeignSameNameAwaiter( return false; } +/** + * Whether the gate's timer took it off `awaiting-signal`, read from the reducer + * itself, so same-name gates earlier in the log cannot confuse it. A gate that + * drained a queued signal never waited, so no timer moved it. + */ +function timerMovedGate( + log: readonly WorkflowEvent[], + runId: string, + stepId: string, +): boolean { + let state = emptyState(runId); + for (const event of log) { + const waited = state.steps.get(stepId)?.phase === "awaiting-signal"; + state = applyEvent(state, event); + if (waited && state.steps.get(stepId)?.phase === "in-flight") { + return event.kind === "TimerFired"; + } + } + return false; +} + /** * Find an unfired pending timer bound to `stepId` in the reduced state. * On a re-park resume of an awaiting-signal step with a timeout, its @@ -6016,19 +6037,21 @@ async function parkOnSignalResult( let fireAtDate: Date | undefined; let subscribeFromSeq: number | undefined; if (opts.timeout !== undefined) { - let beforeTimer = await reloadState(env, runId); // On a re-park resume the durable log already carries this step's // TimerSet with its original id in `pendingTimers` (unfired -- a fired // timeout would have moved the step to `in-flight` and been refused). // Re-adopt that timer rather than minting a second one: a duplicate // TimerSet would double-count the deadline and leave two scheduler - // entries racing to fire. - const existing = findUnfiredTimerForStep(beforeTimer, opts.stepId); + // entries racing to fire. It comes from the read that found the step + // awaiting: a later read can show it fired, which the subscription below + // replays. + const existing = findUnfiredTimerForStep(state, opts.stepId); if (existing !== undefined) { timerId = existing.timerId; fireAtDate = new Date(existing.fireAt); - subscribeFromSeq = beforeTimer.lastSeq + 1; + subscribeFromSeq = state.lastSeq + 1; } else { + let beforeTimer = await reloadState(env, runId); timerId = env.newId("timer"); fireAtDate = new Date(env.clock().getTime() + opts.timeout); const timerSet: WorkflowEvent = { @@ -6055,9 +6078,9 @@ async function parkOnSignalResult( // state from, and (c) the control-plane suspension is durable in // the log before the host is notified below, so the hub is // never told about a correlation the log cannot reconstruct on resume. - // `subscribeFromSeq` above was computed from the in-memory tip; the - // flush makes the durable tip match, so the timer-watch subscription - // starts exactly past the flushed markers. + // `subscribeFromSeq` above is past the flushed markers on a fresh park. On a + // re-park it comes from the read that found the step awaiting, so the + // timer-watch subscription replays whatever landed since. await flush(env, runId); // Notify the host of the fresh control-plane suspension only now that it @@ -6115,6 +6138,7 @@ async function parkOnSignalResult( env.drain.signal.addEventListener("abort", onDrain, { once: true }); let timerDispose: (() => void) | undefined; let timerFired = false; + const timerWatchFailure: { cause?: unknown } = {}; let timerWaitAbort: AbortController | undefined; let timerWatch: Promise | undefined; if ( @@ -6139,7 +6163,11 @@ async function parkOnSignalResult( return; } } - })(); + })().catch((cause: unknown) => { + // A gate that lost its watch would never see its timer fire. + timerWatchFailure.cause = cause; + combinedAbort.abort(); + }); } try { // `awaitNext` registers its subscriber before this returns. A host @@ -6190,15 +6218,14 @@ async function parkOnSignalResult( if (timerFired) { return { timedOut: true }; } + if ("cause" in timerWatchFailure) throw timerWatchFailure.cause; throw cause; } finally { abort.removeEventListener("abort", onOuterAbort); env.drain.signal.removeEventListener("abort", onDrain); if (timerDispose !== undefined) timerDispose(); if (timerWaitAbort !== undefined) timerWaitAbort.abort(); - if (timerWatch !== undefined) { - await timerWatch.catch(() => undefined); - } + if (timerWatch !== undefined) await timerWatch; } } @@ -6253,6 +6280,17 @@ async function runAwaitSignal( // sibling that already completed -- makes the binding ambiguous, so refuse // rather than risk recovering a wrong payload. if (hasForeignSameNameAwaiter(log, primitive.name, primitive.id)) { + // A timeout has no payload to bind. + if (timerMovedGate(log, runId, primitive.id)) { + return completeAwaitSignalOutcome( + definition, + env, + runId, + primitive, + { timedOut: true }, + abort, + ); + } throw new RuntimeResumeUnsupportedError( primitive.id, "in-flight", From 541451e4da9d665a5f444132af29e2da3f50647f Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 2 Nov 2026 14:12:11 +0100 Subject: [PATCH 14/20] Hand a decision to a resuming body only once it parks again A container handed a decision down as soon as it had it. A body still resuming acts on a park it read before the decision landed, so an agent step looked crashed and failed. --- .../src/runtime/child-workflow-relay.test.ts | 94 ++++++++++++++++++- packages/workflow/src/runtime/run.ts | 33 ++++++- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/packages/workflow/src/runtime/child-workflow-relay.test.ts b/packages/workflow/src/runtime/child-workflow-relay.test.ts index c74b6e2e4..bfa06aa36 100644 --- a/packages/workflow/src/runtime/child-workflow-relay.test.ts +++ b/packages/workflow/src/runtime/child-workflow-relay.test.ts @@ -47,6 +47,7 @@ import { type SpawnChildWorkflow, type SignalChannel, type StepInvoker, + type SuspendableChildHandle, type WorkflowAuthorizeFn, type WorkflowDefinition, type WorkflowEvent, @@ -198,6 +199,7 @@ function startRelayed(args: { holdUntilChildAwaits?: readonly string[]; signalChannel?: SignalChannel; spawnChild?: SpawnChildWorkflow; + wrapChild?: (handle: SuspendableChildHandle) => SuspendableChildHandle; repoStore?: RepoStore; resumeFromEvents?: readonly WorkflowEvent[]; hasUpstreamSignalResolver?: boolean; @@ -230,7 +232,7 @@ function startRelayed(args: { const signalChannel = args.signalChannel ?? createInMemorySignalChannel({ newId: () => newId("sig") }); - const spawnChild = + const baseSpawnChild = args.spawnChild ?? (args.holdUntilChildAwaits !== undefined ? holdSpawnUntilChildAwaits( @@ -248,6 +250,11 @@ function startRelayed(args: { }, repoStore, )); + const { wrapChild } = args; + const spawnChild: SpawnChildWorkflow = + wrapChild === undefined + ? baseSpawnChild + : async (input) => wrapChild(await baseSpawnChild(input)); const env: WorkflowRuntimeEnv = { repoStore, scheduler: createInMemoryScheduler({ repoStore, clock }), @@ -1823,6 +1830,91 @@ describe("a parked childWorkflow survives a process restart", () => { ); }); + test("hands a decision to a resuming child only once it parks again", async () => { + const parent = defineWorkflow({ + id: "resume-rearm-parent", + trigger: { type: "manual" }, + steps: { + spawn: childWorkflow({ + definition: defineWorkflow({ + id: "resume-rearm-child", + trigger: { type: "manual" }, + steps: { hold: step({ agent }) }, + }), + }), + }, + }); + const runId = "run-resume-rearm"; + const first = startRelayed({ + definition: parent, + runId, + invokeStep: async (req) => { + if (req.resume !== undefined) return { output: null }; + return { + suspend: { + correlationId: "corr-rearm", + kind: "approval", + approvalSnapshot: snapshot, + }, + }; + }, + }); + void first.run.complete.catch(() => undefined); + await waitForEvent( + first.repoStore, + runId, + (event) => + event.kind === "SignalAwaited" && + event.stepId === "spawn" && + event.parkKind === "approval", + ); + const parentLog = await copyLog(first.repoStore, runId); + const childId = spawnedChildId(parentLog, "spawn"); + const childLog = await copyLog(first.repoStore, childId); + + const repoStore = createInMemoryRepoStore(); + await seedLog(repoStore, childId, childLog); + const askedForRearm = Promise.withResolvers(); + const childRearmed = Promise.withResolvers(); + let handedDown = 0; + let resumed: unknown; + const { run } = startRelayed({ + definition: parent, + runId, + invokeStep: async (req) => { + resumed = req.resume?.decision; + return { output: null }; + }, + wrapChild: (handle) => ({ + ...handle, + resume: (correlationId, decision) => { + handedDown += 1; + return handle.resume(correlationId, decision); + }, + whenSignalAwaiter: async (name, onWake) => { + askedForRearm.resolve(undefined); + await childRearmed.promise; + await handle.whenSignalAwaiter?.(name, onWake); + }, + }), + repoStore, + resumeFromEvents: parentLog, + }); + await run.signal( + signalName("corr-rearm"), + { outcome: "approved" }, + "sig-rearm", + ); + await askedForRearm.promise; + expect(handedDown).toBe(0); + + childRearmed.resolve(undefined); + const result = await run.complete; + expect(result.terminalStatus).toBe("completed"); + expect(handedDown).toBe(1); + expect(resumed).toEqual({ outcome: "approved" }); + }); + test("reestablishes an input park and delivers the next turn", async () => { const parent = defineWorkflow({ id: "resume-input-parent", diff --git a/packages/workflow/src/runtime/run.ts b/packages/workflow/src/runtime/run.ts index 644d06704..13f8345ef 100644 --- a/packages/workflow/src/runtime/run.ts +++ b/packages/workflow/src/runtime/run.ts @@ -4176,7 +4176,14 @@ function containerRelays( `step ${stepId} awaiting-signal ${awaiting.name} is not a reserved control-plane channel`, ); } - return [{ kind: "control-reestablish", parkKind: kind, correlationId }]; + return [ + { + kind: "control-reestablish", + parkKind: kind, + correlationId, + startsOver: true, + }, + ]; } refuseHandedOnInput(stepId, log, sinceSeq, undefined); return []; @@ -6611,6 +6618,7 @@ async function awaitChildPark( abort: AbortSignal, sectionSteps: ReadonlySet, pull: ChildRelayPull = "next", + handDown: "at-once" | "after-rearm" = "at-once", ): Promise { const watch = handle.waitForParkRelease; if (watch === undefined) { @@ -6706,6 +6714,24 @@ async function awaitChildPark( if (again.kind === "released") { throw new ChildRelayRetiredError(stepId, parkKind, name); } + // A body still resuming acts on a park it read before this decision lands, + // so the decision waits until the body waits on it again. + const whenAwaiter = handle.whenSignalAwaiter; + if (handDown === "after-rearm" && whenAwaiter !== undefined) { + const rearm = await awaitChildRearm( + handle, + stepId, + name, + whenAwaiter, + abort, + ); + if (rearm.kind === "terminal") { + return { kind: "terminal", terminalStatus: rearm.terminalStatus }; + } + if (rearm.kind === "released") { + throw new ChildRelayRetiredError(stepId, parkKind, name); + } + } await handle.resume(correlationId, outcome.payload); return finishRelayPull(handle, pull); } @@ -6808,6 +6834,8 @@ type ChildRelay = kind: "control-reestablish"; parkKind: "approval" | "input"; correlationId: string; + // The body's log is gone, so it starts over: no re-park to wait for. + startsOver?: true; } | { kind: "control-deliver"; @@ -7505,6 +7533,7 @@ async function drivePlannedChildRelay( abort, sectionSteps, "hold", + relay.startsOver === true ? "at-once" : "after-rearm", ); case "control-deliver": { const name = signalName(relay.correlationId); @@ -7548,6 +7577,7 @@ async function drivePlannedChildRelay( abort, sectionSteps, "hold", + "after-rearm", ); } case "approval-fresh": { @@ -7568,6 +7598,7 @@ async function drivePlannedChildRelay( abort, sectionSteps, "hold", + "after-rearm", ); } default: { From d35318452af12daa471aa4e385d1b2d8b50be953 Mon Sep 17 00:00:00 2001 From: FarinM Date: Thu, 29 Oct 2026 01:18:57 +0100 Subject: [PATCH 15/20] Hand a drain deadline's cancellation to the workflow supervisor The drain timeout signed and committed its CancelRequested itself, with the substrate and signing bindings threaded in for it alone. It now hands the cancellation to a commit the supervisor injects, so the supervisor decides how a cancellation it owns is committed. What gets committed is unchanged. --- docs/IMPLEMENTATION.md | 13 ++- .../src/supervisor/drain-timeout.test.ts | 101 +++++++++--------- .../src/supervisor/drain-timeout.ts | 57 +++------- .../src/supervisor/supervisor.test.ts | 7 +- .../src/supervisor/supervisor.ts | 15 ++- 5 files changed, 87 insertions(+), 106 deletions(-) diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index f5f23d205..622dc410a 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -862,13 +862,12 @@ wall-clock time only while the supervisor believes at least one step transitions to `"wait"` behavior and resumes when cancel-mode work reappears. Time spent paused does not consume the deadline. -On `drainTimeout` expiry the accumulator invokes the supervisor's -injected `signAsPrincipal("supervisor", ...)` callback to mint a -signature over a canonical `CancelRequested{origin: -"supervisor-drain"}` event and commits it through the supervisor's -substrate handle. The runtime body observes the commit through its -existing cancel-cascade path; no special drain-aware cancel branch -is needed in `run.ts`. +On `drainTimeout` expiry the accumulator hands a +`CancelRequested{origin: "supervisor-drain"}` event to the +supervisor's injected `commitCancel`, which signs it and commits it +as the supervisor's own cancellation. The runtime body observes the +commit through its existing cancel-cascade path; no special +drain-aware cancel branch is needed in `run.ts`. The canonical observable sequence at the in-process layer (asserted in this package's tests via `subscribeKind` against a fake diff --git a/packages/workflow-host/src/supervisor/drain-timeout.test.ts b/packages/workflow-host/src/supervisor/drain-timeout.test.ts index 3d707f3d5..b18e15bd5 100644 --- a/packages/workflow-host/src/supervisor/drain-timeout.test.ts +++ b/packages/workflow-host/src/supervisor/drain-timeout.test.ts @@ -3,7 +3,8 @@ // The accumulator ticks against wall-clock time while the supervisor // believes the workflow has at least one `"cancel"`-behavior step in // flight. On timeout it commits `CancelRequested{origin: -// "supervisor-drain"}` through the injected substrate. The runtime +// "supervisor-drain"}` through the supervisor's injected commit, which +// these tests run against a stub substrate. The runtime // body's existing cancellation cascade handles teardown once the // CancelRequested event lands on the workflow-run log. // @@ -13,8 +14,8 @@ // side. The attachment point is the `RepoStore. // writeTreePreservingPrefix` interceptor on a stub substrate -- the // fake captures every commit's principal/files/ref so the test -// asserts directly on the CancelRequested blob the accumulator -// writes, instead of round-tripping through a `subscribeKind` +// asserts directly on the CancelRequested blob the escalation +// commits, instead of round-tripping through a `subscribeKind` // observer. The same observable is being tested; the attachment // point is just one layer closer to the write the accumulator // performs. @@ -31,9 +32,11 @@ import type { TerminalRunEvent, } from "./types"; +import { commitCancelRequested } from "./cancel-signing"; import { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, + type DrainTimeoutOpts, } from "./drain-timeout"; import { waitUntil } from "@intx/types/testing"; @@ -165,6 +168,24 @@ const REPO_ID: RepoId = { kind: "workflow-run", id: "deployment-x" }; const REF = "refs/heads/main"; const DEPLOYMENT_ID = "deployment-x"; +// Commits the escalation as the supervisor does, through the stub substrate. +function commitThrough( + substrate: RepoStore, + signAsPrincipal: ReturnType["signAsPrincipal"], + runId: string, +): DrainTimeoutOpts["commitCancel"] { + return (cancel) => + commitCancelRequested({ + substrate, + repoId: REPO_ID, + ref: REF, + anchorRunId: DEPLOYMENT_ID, + runId, + ...cancel, + signAsPrincipal, + }); +} + describe("createDrainTimeoutAccumulator", () => { test("DEFAULT_DRAIN_TIMEOUT_MS is 60 seconds", () => { expect(DEFAULT_DRAIN_TIMEOUT_MS).toBe(60_000); @@ -181,12 +202,8 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-1", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough(substrate, signSpy.signAsPrincipal, "run-1"), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -231,12 +248,8 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-2", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough(substrate, signSpy.signAsPrincipal, "run-2"), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -272,12 +285,8 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-3", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough(substrate, signSpy.signAsPrincipal, "run-3"), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -303,12 +312,8 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-4", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough(substrate, signSpy.signAsPrincipal, "run-4"), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -345,12 +350,12 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "parent-run", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough( + substrate, + signSpy.signAsPrincipal, + "parent-run", + ), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -379,12 +384,12 @@ describe("createDrainTimeoutAccumulator", () => { const clock = createFakeClock(); const host = createFakeTimerHost(); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "parent-run", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough( + substrate, + signSpy.signAsPrincipal, + "parent-run", + ), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -425,12 +430,12 @@ describe("createDrainTimeoutAccumulator", () => { }; }; const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-terminal-1", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough( + substrate, + signSpy.signAsPrincipal, + "run-terminal-1", + ), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -501,12 +506,12 @@ describe("createDrainTimeoutAccumulator", () => { }, }); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-timeout-1", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough( + substrate, + signSpy.signAsPrincipal, + "run-timeout-1", + ), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, @@ -548,12 +553,12 @@ describe("createDrainTimeoutAccumulator", () => { }, }); const accumulator = createDrainTimeoutAccumulator({ - substrate, - repoId: REPO_ID, - ref: REF, - anchorRunId: DEPLOYMENT_ID, runId: "run-stop-terminal-1", - signAsPrincipal: signSpy.signAsPrincipal, + commitCancel: commitThrough( + substrate, + signSpy.signAsPrincipal, + "run-stop-terminal-1", + ), drainTimeoutMs: 1_000, now: clock.now, setTimer: host.setTimer, diff --git a/packages/workflow-host/src/supervisor/drain-timeout.ts b/packages/workflow-host/src/supervisor/drain-timeout.ts index 6a8dd1c6f..73340974f 100644 --- a/packages/workflow-host/src/supervisor/drain-timeout.ts +++ b/packages/workflow-host/src/supervisor/drain-timeout.ts @@ -12,11 +12,10 @@ // while every in-flight step is `"wait"`-behavior does NOT tick the // accumulator -- waiting for a human-in-the-loop pause should not // burn the drain budget. On `drainTimeout` expiry, the accumulator -// invokes the supervisor's injected `signAsPrincipal("supervisor", -// ...)` callback to obtain a signed `CancelRequested{origin: -// "supervisor-drain"}` and commits it via the injected substrate -// handle. The runtime body's existing cancellation cascade handles -// teardown from there. +// hands a `CancelRequested{origin: "supervisor-drain"}` to the +// supervisor's injected `commitCancel`, which signs and commits it as +// the supervisor's own cancellation. The runtime body's existing +// cancellation cascade handles teardown from there. // // The accumulator is started by the supervisor on receipt of the // host's drain command; it is stopped when (a) drainTimeout expires @@ -29,12 +28,7 @@ import { getLogger } from "@intx/log"; import type { CancelOrigin } from "@intx/workflow"; -import { commitCancelRequested } from "./cancel-signing"; -import type { PrincipalSigner, TerminalEventSource } from "./types"; -import type { - RepoId, - RepoStore as SubstrateRepoStore, -} from "@intx/hub-sessions/substrate"; +import type { TerminalEventSource } from "./types"; const logger = getLogger(["workflow-host", "supervisor", "drain-timeout"]); @@ -45,28 +39,19 @@ const logger = getLogger(["workflow-host", "supervisor", "drain-timeout"]); export const DEFAULT_DRAIN_TIMEOUT_MS = 60_000; export type DrainTimeoutOpts = { - /** - * Substrate handle through which the escalation commit lands. - * Reused from the supervisor's bindings -- the accumulator does - * not own a substrate of its own. - */ - substrate: SubstrateRepoStore; - /** Workflow-run repo for this deployment. */ - repoId: RepoId; - /** Workflow-run repo ref the supervisor commits events to. */ - ref: string; - /** Anchor run id baked into the supervisor's signing principal. */ - anchorRunId: string; /** Run id the drain is being escalated against. */ runId: string; - /** - * Host-supplied per-principal signing callback the supervisor uses - * to mint the CancelRequested signature. The drain origin is - * `supervisor-drain`. - */ - signAsPrincipal: PrincipalSigner; /** ISO-8601 reason for the CancelRequested escalation. */ reason?: string; + /** + * Commits the escalation's CancelRequested for `runId`. The supervisor + * commits it as one of its own cancellations. + */ + commitCancel: (cancel: { + origin: CancelOrigin; + reason: string; + at: string; + }) => Promise; /** * Operator-overridable per-deployment timeout. Defaults to * `DEFAULT_DRAIN_TIMEOUT_MS`. @@ -278,17 +263,9 @@ export function createDrainTimeoutAccumulator( // resources. finaliseTerminalWatcher(); const origin: CancelOrigin = "supervisor-drain"; - escalationPending = commitCancelRequested({ - substrate: opts.substrate, - repoId: opts.repoId, - ref: opts.ref, - anchorRunId: opts.anchorRunId, - runId: opts.runId, - origin, - reason, - at: new Date().toISOString(), - signAsPrincipal: opts.signAsPrincipal, - }) + const cancel = { origin, reason, at: new Date().toISOString() }; + escalationPending = opts + .commitCancel(cancel) .then(() => undefined) .catch((cause: unknown) => { const message = cause instanceof Error ? cause.message : String(cause); diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 04d4657b8..9a59220c0 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -3455,12 +3455,7 @@ describe("createWorkflowSupervisor", () => { for (const stub of stubs) { expect(stub.__startCount).toBe(1); expect(stub.__stopCount).toBe(0); - expect(stub.__opts.anchorRunId).toBe("run_deployment-x"); - expect(stub.__opts.repoId).toEqual({ - kind: "workflow-run", - id: "run_deployment-x", - }); - expect(stub.__opts.ref).toBe("refs/heads/main"); + expect(typeof stub.__opts.commitCancel).toBe("function"); expect(stub.__opts.drainTimeoutMs).toBe(7_500); expect(typeof stub.__opts.runId).toBe("string"); expect(stub.__opts.runId.length).toBeGreaterThan(0); diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 4056231e7..48001073a 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -4678,16 +4678,21 @@ export function createWorkflowSupervisor( if (drainAccumulators.has(runId)) continue; if (runInputChannels.has(runId)) continue; const accumulator = accumulatorFactory({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, runId, - signAsPrincipal: bindings.signAsPrincipal, drainTimeoutMs, now: drainNow, setTimer: drainSetTimer, clearTimer: drainClearTimer, + commitCancel: (cancel) => + commitCancelRequested({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + ref: bindings.workflowRunRef, + anchorRunId: bindings.anchorRunId, + runId, + ...cancel, + signAsPrincipal: bindings.signAsPrincipal, + }), ...(cohortSource !== null ? { terminalEventSource: cohortSource } : {}), }); drainAccumulators.set(runId, accumulator); From 6403c1f81cd8647faac0659284e9e8fb2bec0ca2 Mon Sep 17 00:00:00 2001 From: FarinM Date: Thu, 29 Oct 2026 01:18:57 +0100 Subject: [PATCH 16/20] Move a workflow supervisor's boot recovery into its own function It also drops "on spawn" from the recovery's log messages, since the function need not know what called it. --- .../src/supervisor/supervisor.ts | 156 ++++++++---------- 1 file changed, 69 insertions(+), 87 deletions(-) diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 48001073a..0b32edc45 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -3145,99 +3145,13 @@ export function createWorkflowSupervisor( // teardown: it kills the handle and releases the mail subscription // and address registration installed below. try { - // Replay any orphaned `processing/` entries back to `inbox/` - // BEFORE the dispatch loop's first dequeue. A crash mid-dispatch - // in a prior supervisor incarnation can leave an entry in - // `processing/` with no owner; the FIFO contract requires the - // entry move back to `inbox/` so the next dispatch picks it up - // in its original arrival position. The replay runs off the - // spawn critical path (the substrate write may roundtrip through - // the pack-pushing wrap and a slow hub), but `runDispatchLoop` - // takes the promise as an argument and awaits it before its - // first `dequeueToProcessing` so a fresh inbound mail that lands - // during the replay window cannot ship ahead of the orphan once - // the replay completes. - // One scan of `runs/` feeds both spawn-time recovery consumers: the - // orphan replay (which gates dispatch) and the compaction sweep (which - // does not). Sharing the walk keeps recovery off a second O(total-runs) - // scan. - const scanDone = scanRunsForBoot( - bindings.repoStore, - supervisorPrincipal, - bindings.workflowRunRepoId, - bindings.workflowRunRef, - ); - const replayDone = scanDone - .then(({ ownedMessageIds }) => - inboxPrimitives.replayProcessingToInbox( - bindings.repoStore, - inboxWritePrincipal, - bindings.workflowRunRepoId, - bindings.deploymentMailAddress, - { ownedMessageIds }, - ), - ) - .then(({ retainedMessageIds }) => { - keepOwnedMessages(retainedMessageIds, { childRunning: true }); - wakeDispatch(); - }) - .catch((cause) => { - // Documented best-effort: a failed replay leaves orphaned - // `processing/` entries parked and the dispatch loop will - // then ship newly-enqueued mail ahead of them, violating - // the FIFO contract described in the comment above. - // Tightening this to a fatal `onChildCrash` was attempted - // but caused spurious crashes in the integration suite - // where the first spawn legitimately has no - // `processing/` directory to replay; resolving that - // requires either a no-op-on-missing variant of - // `replayProcessingToInbox` or a dispatch-loop periodic - // sweep that picks up parked orphans. Left as logged - // best-effort until that lands. - const message = - cause instanceof Error ? cause.message : String(cause); - logger.warn`boot recovery scan or processing replay failed on spawn: ${message}`; - }); + const { replayed: replayDone, sweepDone } = recoverAtBoot(); // Hold the replay promise on the active-state record so // `shutdownInternal` awaits its settlement before tearing the // bindings down. A shutdown that lands while the replay is in // flight would otherwise leave the substrate write pending past // the supervisor's exit. state.replayDone = replayDone; - bootReplay = replayDone; - - // Re-seal runs a crash left terminal-but-per-event when their - // fire-and-forget fold never ran. Unlike the replay above, this must - // NOT gate dispatch: reclaiming leaked per-event files is housekeeping - // and cannot be allowed to delay the first dequeue. Best-effort, held - // on the active-state record so shutdown awaits its settlement (see the - // `sweepDone` field docstring for the teardown-latency tradeoff). - const sweepDone = scanDone - .then(({ pendingSealRunIds }) => - recoverInterruptedCompactions({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, - pendingSealRunIds, - }), - ) - .then(({ sealed, failed }) => { - if (sealed > 0) { - logger.info`recovery sweep sealed ${String(sealed)} interrupted run(s)`; - } - if (failed.length > 0) { - const detail = failed - .map((f) => `${f.runId} (${f.message})`) - .join("; "); - logger.warn`recovery sweep left ${String(failed.length)} run(s) unsealed: ${detail}`; - } - }) - .catch((cause) => { - const message = - cause instanceof Error ? cause.message : String(cause); - logger.warn`boot recovery scan or compaction sweep failed on spawn: ${message}`; - }); state.sweepDone = sweepDone; bindings.mailBus.registerAddress(bindings.deploymentMailAddress); @@ -3411,6 +3325,74 @@ export function createWorkflowSupervisor( } } + // Move `processing/` entries a crashed incarnation left without an owner back + // to `inbox/` in arrival order. The write can wait on a slow Hub, so it runs + // in the background, and dispatch awaits it before its first dequeue so new + // mail cannot overtake them. The compaction sweep shares its scan of `runs/`. + function recoverAtBoot(): { + replayed: Promise; + sweepDone: Promise; + } { + const scanDone = scanRunsForBoot( + bindings.repoStore, + supervisorPrincipal, + bindings.workflowRunRepoId, + bindings.workflowRunRef, + ); + const replayed = scanDone + .then(({ ownedMessageIds }) => + inboxPrimitives.replayProcessingToInbox( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + { ownedMessageIds }, + ), + ) + .then(({ retainedMessageIds }) => { + keepOwnedMessages(retainedMessageIds, { childRunning: true }); + wakeDispatch(); + }) + .catch((cause: unknown) => { + // Best-effort: entries a failed replay leaves in `processing/` lose + // their place to newer mail. Making this fatal crashed first spawns + // with no `processing/` directory; that needs a replay that allows + // one, or a periodic sweep for orphans. + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`boot recovery scan or processing replay failed: ${message}`; + }); + bootReplay = replayed; + + // Re-seal terminal runs whose fold a crash interrupted. Housekeeping, so + // unlike the replay it does not gate dispatch; shutdown still awaits it. + const sweepDone = scanDone + .then(({ pendingSealRunIds }) => + recoverInterruptedCompactions({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + ref: bindings.workflowRunRef, + anchorRunId: bindings.anchorRunId, + pendingSealRunIds, + }), + ) + .then(({ sealed, failed }) => { + if (sealed > 0) { + logger.info`recovery sweep sealed ${String(sealed)} interrupted run(s)`; + } + if (failed.length > 0) { + const detail = failed + .map((f) => `${f.runId} (${f.message})`) + .join("; "); + logger.warn`recovery sweep left ${String(failed.length)} run(s) unsealed: ${detail}`; + } + }) + .catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`boot recovery scan or compaction sweep failed: ${message}`; + }); + return { replayed, sweepDone }; + } + // Arm the recycle policy for the child that just started at `spawnedAt`, // replacing the previous child's, so its bounds measure the running child. // The policy is a no-op when all bounds are `undefined`; bounds resolution From 0e1b5ca8353ec997bb60c945d649b4bb59ddd6c1 Mon Sep 17 00:00:00 2001 From: FarinM Date: Thu, 29 Oct 2026 01:18:57 +0100 Subject: [PATCH 17/20] Let a workflow supervisor put its child in standby while all runs wait A deployment waiting on an approval, a signal or its next trigger kept its workflow child running for the whole wait. The supervisor now stops the child once it reports quiet twice, and a re-registration delivers waiting mail, since mail that wakes the child from standby would otherwise wait out the 300 s backstop. A supervisor in standby respawns its child for mail, a signal, a cancellation, its earliest pending timer or a wake request. A host that records each standby through recordStandby can bring the deployment back with restoreInStandby after a restart, with no child running. A new input.taken frame answers a trigger or signal that asks for it once the child has committed the input or declined it, so an input the child took is not dropped with the stopped child's writes. A dispatch now consumes a signal's mail only once that answer says the child committed it, since another step's park or the run's end says nothing about this mail. A declined mail is rejected if the committed log shows its run ended. On a live run it fails the dispatch and stays in processing/ until a respawn replays it, since forcing a recycle to retry it could loop on a persistent fault. --- docs/AUTH.md | 1 + docs/IMPLEMENTATION.md | 31 +- docs/unified-execution-host-design.md | 5 +- packages/hub-sessions/src/substrate.ts | 1 + .../src/workflow-run-kind.test.ts | 47 + .../hub-sessions/src/workflow-run-kind.ts | 23 + packages/workflow-host/README.md | 97 +- .../src/child/run-child-source-ref.test.ts | 1 + .../workflow-host/src/child/run-child.test.ts | 317 +- packages/workflow-host/src/child/run-child.ts | 69 +- .../src/child/substrate-factory.ts | 1 + packages/workflow-host/src/index.ts | 6 + .../workflow-host/src/ipc/control-payloads.ts | 38 +- .../workflow-host/src/seams/scheduler.test.ts | 60 +- packages/workflow-host/src/seams/scheduler.ts | 3 + .../src/seams/signal-channel.test.ts | 54 + .../workflow-host/src/seams/signal-channel.ts | 17 +- .../src/supervisor/crash-respawn.test.ts | 3 + .../workflow-host/src/supervisor/index.ts | 6 + .../src/supervisor/lifecycle-races.test.ts | 3 + .../supervisor/outbound-signed-send.test.ts | 3 + .../src/supervisor/park-notify.test.ts | 3 + .../re-emit-parked-correlations.test.ts | 3 + .../src/supervisor/recorded-standby.ts | 21 + .../src/supervisor/recycle.test.ts | 3983 ++++++++++++++++- .../workflow-host/src/supervisor/recycle.ts | 56 +- .../src/supervisor/spawn-replay-fifo.test.ts | 14 +- .../supervisor/stale-cohort-routing.test.ts | 3 + .../src/supervisor/substrate-write.test.ts | 3 + .../src/supervisor/supervisor.test.ts | 30 + .../src/supervisor/supervisor.ts | 1804 ++++++-- .../workflow-host/src/supervisor/types.ts | 61 +- .../workflow-host/src/testing/mail-bus.ts | 10 + .../src/testing/stub-repo-store.ts | 5 + ...ingle-step-conversation-durability.test.ts | 1 + 35 files changed, 6437 insertions(+), 346 deletions(-) create mode 100644 packages/workflow-host/src/supervisor/recorded-standby.ts diff --git a/docs/AUTH.md b/docs/AUTH.md index 8a6ac7651..e0c268bed 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -250,6 +250,7 @@ A run needs three kinds of authority: its grants snapshot, its credential materi | Deploy | None until the first trigger; the `run.grants` barrier then brings it | The deploy frame. The Hub tokens arrive on the first `run.grants` barrier | | Sidecar restart | The run's grants file | The sealed mirror in the run record. On reconnect the Hub replays `run.grants` and reconciles the credentials | | Crash-respawn, and every recycle (operator, policy, self) | The run's grants file | The supervisor's mirror | +| Wake from standby | The run's grants file | The supervisor's mirror; after a sidecar restart, it starts from the sealed mirror in the run record | | Hub replacement deploy | The run's grants file, after the Hub restores the run's history | The deploy frame, which also carries the run's existing Hub tokens | A credential that the Hub revokes while the sidecar is down stays usable by the restored run until the reconnect reconcile revokes it. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 622dc410a..0513b5127 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -573,7 +573,7 @@ Implementation lives in `packages/workflow-host/src/supervisor/supervisor.ts`. The recycle sequence is library code in `packages/workflow-host/src/supervisor/recycle.ts`; see the Recycle -subsection below for the six-step contract and the three trigger +subsection below for the six-step contract and the five trigger origins. ### Workflow-Process Child @@ -655,11 +655,19 @@ ChannelSender` writes against; the event channel is the - `grants-updated` -> swap the active credentialsSnapshot. - `credentials-updated` -> merge into the material cell. - `drain` -> flip the drain controller's signal. + - `signal.deliver` -> commit the signal as `SignalReceived`. A frame + with a `requestId`, like a `trigger.fire` with one, is answered with + `input.taken` once the commit settles, saying whether it committed, + so a standby request waits for it. A log that already ended takes no + signal, and the answer says so. The dispatch that delivered a mail + consumes it only once that answer says committed; a declined mail is + rejected if the committed log shows its run ended, and otherwise fails + the dispatch and stays in `processing/` until a respawn replays it. - `quiet.request` -> make every buffered event durable and answer whether every run only waits on input from outside, with the earliest timer the scheduler has queued; with `wait`, answer once that holds. A run that has ended answers `ended`, and a check that - fails answers with its error. + fails answers with its error; either refuses the standby request. A check still waiting when the next one arrives is dropped unanswered, since the supervisor gave up on it. - `shutdown` -> stop accepting new triggers and exit the loop. @@ -1019,7 +1027,7 @@ to the channelId's lifetime; the rotation is what defends against frame replay across the recycle boundary that monotonic seq alone cannot defend against. -**Three trigger origins funnel through `triggerRecycle(reason)`:** +**Five trigger origins funnel through `triggerRecycle(reason)`:** - **Operator command.** The host receives a `recycle` request via its caller-facing API and routes it to the supervisor's @@ -1032,7 +1040,8 @@ cannot defend against. unless the operator sets a threshold). On a threshold trip the policy calls `triggerRecycle` with origin `policy` and a reason naming the tripped bound. Each child gets its own check, armed - when it starts, so `maxUptimeMs` measures the running child. + when it starts, so `maxUptimeMs` measures the running child; a + deployment in standby has no child and no check. - **Workflow-process self-initiated.** The child sends a `recycle.request` payload over control IPC when its own self- check decides it needs to be recycled (an internal consistency @@ -1042,9 +1051,17 @@ cannot defend against. IPC payload union extension is the only contract change recycle introduces; the envelope, signature, and channelId/seq rules are unchanged. - -All three origins land in the same code path. The reason string is -the only origin-specific data the path carries forward. +- **Crash respawn.** A child that exits unexpectedly is respawned + with origin `crash` once the respawn backoff has elapsed. +- **Wake.** A child in standby is respawned with origin `wake` for + mail, a signal, a cancellation, its earliest pending timer, or + `wake()`. + +All five origins land in the same code path. The origin tag and the +reason string are the only origin-specific data it carries forward. A +crash respawn and a wake drain nothing, since their child is already +gone, and the first wake of a deployment restored in standby has no +child to stop and skips the replay its boot already made. #### Placeholders And Future Work diff --git a/docs/unified-execution-host-design.md b/docs/unified-execution-host-design.md index c169928f6..6b056a0b4 100644 --- a/docs/unified-execution-host-design.md +++ b/docs/unified-execution-host-design.md @@ -356,8 +356,9 @@ workflows.** Where it lives, when it tears down, how recycle/drain interact: (or eagerly at spawn for a hot path; lazy is the default to avoid paying tool construction before any mail arrives). - **When it's torn down.** On `agent.undeploy` (deployment teardown), on child - `shutdown`, and on recycle. A long-lived agent is _not_ torn down between - messages — that is the entire point of warm-keep. + `shutdown`, on recycle, and when standby stops the child. A long-lived agent + is otherwise _not_ torn down between messages — that is the entire point of + warm-keep. - **Recycle interaction.** The supervisor's recycle policy (`packages/workflow-host/src/supervisor/recycle.ts`) kills and respawns the child on drain/SIGTERM/SIGKILL/respawn. On respawn the warm-agent cache is diff --git a/packages/hub-sessions/src/substrate.ts b/packages/hub-sessions/src/substrate.ts index 79dbc03b4..8c95de640 100644 --- a/packages/hub-sessions/src/substrate.ts +++ b/packages/hub-sessions/src/substrate.ts @@ -20,6 +20,7 @@ export { DEFAULT_CONSUMED_RETENTION_MS, dequeueToProcessing, enqueueInbox, + hasInboxEntries, markConsumed, NotInProcessingError, parseEventSeq, diff --git a/packages/hub-sessions/src/workflow-run-kind.test.ts b/packages/hub-sessions/src/workflow-run-kind.test.ts index 55b227733..5fabd51fb 100644 --- a/packages/hub-sessions/src/workflow-run-kind.test.ts +++ b/packages/hub-sessions/src/workflow-run-kind.test.ts @@ -12,6 +12,7 @@ import { classifyTerminalEvent, enqueueInbox, dequeueToProcessing, + hasInboxEntries, markConsumed, NotInProcessingError, parseEventSeq, @@ -3124,6 +3125,52 @@ describe("claim-check API — enqueueInbox", () => { }); }); +describe("claim-check API — hasInboxEntries", () => { + test("reports mail no dispatch has taken yet", async () => { + const { store, repoId, principal } = + await makeClaimCheckStore("cc-has-inbox-"); + expect(await hasInboxEntries(store, principal, repoId, ADDRESS)).toBe( + false, + ); + await enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-1", + receivedAt: 100, + }); + expect(await hasInboxEntries(store, principal, repoId, ADDRESS)).toBe(true); + await dequeueToProcessing(store, principal, repoId, ADDRESS); + expect(await hasInboxEntries(store, principal, repoId, ADDRESS)).toBe( + false, + ); + }); + + test("reads the committed inbox, not the checkout", async () => { + const { store, repoId, principal } = await makeClaimCheckStore( + "cc-has-inbox-committed-", + ); + await enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId: "msg-1", + receivedAt: 100, + }); + // A write that failed after deleting the checkout file leaves the entry in + // Git alone, and one that failed before committing leaves only the file. + const repoDir = store.getRepoDir(repoId); + await fs.promises.rm( + path.join(repoDir, inboxPathFor(ADDRESS_SEG, 100, "msg-1")), + ); + expect(await hasInboxEntries(store, principal, repoId, ADDRESS)).toBe(true); + await dequeueToProcessing(store, principal, repoId, ADDRESS); + await fs.promises.writeFile( + path.join(repoDir, inboxPathFor(ADDRESS_SEG, 200, "msg-2")), + "{}", + ); + expect(await hasInboxEntries(store, principal, repoId, ADDRESS)).toBe( + false, + ); + }); +}); + describe("claim-check API — markConsumed", () => { test("entries written with a since-removed envelope field keep moving", async () => { // The real handler refuses a consumed entry with no prior processing diff --git a/packages/hub-sessions/src/workflow-run-kind.ts b/packages/hub-sessions/src/workflow-run-kind.ts index 4079c8aa4..32c52539a 100644 --- a/packages/hub-sessions/src/workflow-run-kind.ts +++ b/packages/hub-sessions/src/workflow-run-kind.ts @@ -4077,6 +4077,29 @@ export async function readProcessingEntry( return null; } +/** + * Whether `address` holds inbox entries no dispatch has taken yet. Reads the + * committed tree, as dequeue does: a failed write can leave the checkout + * without an entry Git still holds. + */ +export async function hasInboxEntries( + store: RepoStore, + principal: Principal, + repoId: RepoId, + address: string, +): Promise { + const reads = await store.openCommittedReads( + principal, + repoId, + claimCheckCommitRef(), + ); + if (reads === null) return false; + const inbox = await reads.listDir( + `${addressPrefix(addressSegmentFor(address))}${WORKFLOW_RUN_INBOX_DIR}`, + ); + return inbox.some((entry) => entry.name.endsWith(".json")); +} + export type MarkConsumedArgs = { address: string; messageId: string; diff --git a/packages/workflow-host/README.md b/packages/workflow-host/README.md index 65a8a9822..b59e1c1c8 100644 --- a/packages/workflow-host/README.md +++ b/packages/workflow-host/README.md @@ -108,9 +108,10 @@ the deploy program read and write the filesystem. ## Supervisor `createWorkflowSupervisor(bindings)` returns a per-deployment -supervisor that owns one workflow-process child for the lifetime of -the deployment. The supervisor is host-agnostic library code; the -host supplies its own concrete bindings. +supervisor that owns the deployment's workflow-process child: one at a +time, replaced across recycles and respawns, and none in standby. The +supervisor is host-agnostic library code; the host supplies its own +concrete bindings. ### `WorkflowSupervisorBindings` @@ -225,7 +226,9 @@ On an unexpected exit the supervisor, with no external intervention: back into `inbox/` under their original `-` keys. Those keys sort ahead of any mail that arrived during the kill/respawn gap, so the stranded entry is re-dispatched first and - FIFO ordering holds across the respawn boundary. Mail a live run + FIFO ordering holds across the respawn boundary. Mail the child + declined is the exception: newer mail can overtake it while it waits + for the respawn. Mail a live run already took, its start mail or a turn it received, stays in `processing/`: the resumed run carries on with it, and the supervisor consumes it once the run reaches a new park or ends, retrying a failed @@ -251,6 +254,8 @@ the host. Every bound is operator-overridable via - **Stable-run reset.** Once a crash-respawned child stays up for `crashLoopStableResetMs` (60s), the crash counter and the backoff reset, so a flap followed by stability does not permanently latch. + A stop for standby is no crash: the wait goes on while the child is + stopped, and starts over for a child woken before it ends. `crash-looping` is an in-memory, per-process phase — no external reader observes it. The durable, externally-queryable signal is the run's @@ -262,6 +267,90 @@ sees the crash-loop as a failed run. The commit lands before the latch's teardown resolves and before the host hears of the self-termination, so a host that reads the stopped deployment's history finds it. +### Standby + +`enterStandby()` stops the child once every run only waits on input +from outside, and keeps the deployment: its mail subscription, +address registration, and history stay as they are. + +1. Once nothing the supervisor drives is in flight, it asks the child, + with `quiet.request`, to report when it is quiet. Dispatch and signals + go on meanwhile, so a busy deployment enters standby at its next quiet + moment. An input the child took counts as in flight until the child + answers it: `input.taken` answers each `trigger.fire` and + `signal.deliver` the supervisor sends, saying whether the child + committed the input. A dispatch waiting for its run to park or end + does not: its mail stays in `processing/` with the run, as it does + across any respawn (see Respawn policy). +2. The child is quiet when no run loop or step works: every step waits + on a signal, a timer, or a nested run, and no input it committed is + still on its way to the run that waits for it. A map counts as + working until it ends, since resume cannot continue one in flight, + and so does a step with a timeout, whose deadline lives only in the + child (a gate's timeout is a timer in the log, and does not count), + and a step waiting to retry, whose wait resume does not + continue. It makes every buffered event durable before it answers, + and reports the earliest timer its scheduler has queued. +3. The supervisor then holds dispatch and signals back and asks once + more. If the child is still quiet, the supervisor holds the child's + further writes and records the correlations the child is parked on. + Mail waiting in the inbox refuses the standby attempt, since it + would wake the child as soon as it stopped. So does mail the dispatch + took that the child has not committed (one behind a run that waits), + since nothing would wake the child for it, and mail a failed dispatch + left in `processing/`, such as one the child declined, which only a + respawn puts back. Otherwise the supervisor sends + `shutdown`, and kills the child if it does not exit. A write after + the quiet answer means the child is working again: before the stop + the standby attempt goes back to waiting for quiet, and after it the + child is woken for that work. + +Once the child is stopped, and before anything can wake it, the supervisor +calls the host's `recordStandby` with what a restart needs: the correlations +the child was parked on, its earliest timer, the tip of the run's history +ref, read before the child reported its parks, and the definition the first +child verified. A failure there runs the child again and refuses the standby +attempt. + +`enterStandby()` resolves with the earliest timer the child had queued, if +any. It rejects with `StandbyRefusedError` when the run ends, mail waits +in the inbox, for the child to commit it, or in `processing/`, the run's +history or the inbox cannot be read, the child does not report what it is +parked on, the standby cannot be recorded, or the child reports that its +quiet check failed. Its subclass `StandbyAbandonedError` is used when +`wake()`, `abandonStandby()`, a cancellation or a shutdown calls it off, +and `StandbyDeferredError` is used while the child is +starting or restarting, so the host can ask again once it runs. +`abandonStandby()` does nothing to a finished standby attempt, and runs the +child again when it comes after the stop began; `wake()` and a cancellation +that abandon one run the child themselves. + +A supervisor in standby respawns its child through the recycle path for mail, +a signal (delivered to the new child), a cancellation (committed first), +its earliest pending timer, or `wake()`, and then calls `onWake`, with +`requested` set when `wake()` started it. That respawn waits for a cancellation being committed, since the new child +applies it from its log. Mail that reaches the inbox after the last read +before the stop wakes the child as arriving mail does; a redelivery of mail a +run already took does not. A hub +re-establishment while in standby, or while the child stops for it, +re-registers the correlations the child reported, without asking it. + +`restoreInStandby(opts, standby)` restores a deployment its host recorded in +standby. With no child to report the definition, the recorded one lays the +steps out, once it hashes to the approved one as a child's report must. While +the run's history tip is still the recorded one, nothing has changed what the +runs wait on: the supervisor takes the deployment's mail, +re-sends the recorded correlations on each hub re-establishment, and arms the +recorded timer, waking at once for one already due, all without a child. The +boot replay of `processing/` runs as on a spawn, and mail it puts back or that +waits in the inbox wakes the child, as does a replay that fails. Mail a live +run took stays with the run. The first wake starts the child through the +recycle path, from `standby`, after the boot replay. A tip that moved, from a +child that ran again or a cancellation committed while it was stopped, a tip +that cannot be read, or a recorded definition that is not the approved one, +starts the child as `spawn` does, and the result says why, so the host can +report the deployment woken. + ### Host wiring A host that wants to instantiate a supervisor constructs the diff --git a/packages/workflow-host/src/child/run-child-source-ref.test.ts b/packages/workflow-host/src/child/run-child-source-ref.test.ts index d7f121413..81afadc2e 100644 --- a/packages/workflow-host/src/child/run-child-source-ref.test.ts +++ b/packages/workflow-host/src/child/run-child-source-ref.test.ts @@ -510,6 +510,7 @@ function buildBindings(baseDir: string): RunWorkflowChildBindings { spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, earliestQueuedTimer: () => undefined, + stopScheduler: async () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index f3b8c442c..b2364d88c 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -423,6 +423,7 @@ function buildBindings(opts: { scheduleIn: () => () => undefined, }, earliestQueuedTimer: () => undefined, + stopScheduler: async () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], @@ -528,7 +529,7 @@ describe("parseSpawnTimeEnv", () => { }); describe("runWorkflowChild", () => { - test("emits ready, processes a trigger.fire frame, and shuts down", async () => { + test("emits ready, processes a trigger.fire frame, answers it once the run started, and shuts down", async () => { const baseDir = await makeTempDir("child-trigger-"); const supervisorKeyPair = await generateKeyPair(); const childKeyPair = await generateKeyPair(); @@ -601,9 +602,26 @@ describe("runWorkflowChild", () => { runId: "run-1", messageId: "msg-1", receivedAt: 1, + requestId: "input-1", payload: textMail("hello from the inbox"), }, }); + const taken = await waitForUpstreamPayload( + childToSupervisor, + "input.taken", + ); + expect(taken.data).toEqual({ requestId: "input-1", committed: true }); + const events = await fs.readdir( + path.join( + baseDir, + "workflow-run", + "deployment-x", + "runs", + "run-1", + "events", + ), + ); + expect(events.length).toBeGreaterThan(0); await supervisorSender.send({ type: "shutdown", data: { reason: "test done" }, @@ -822,6 +840,7 @@ describe("runWorkflowChild", () => { const bindings: RunWorkflowChildBindings = { ...buildBindings({ baseDir, childKeyPair }), earliestQueuedTimer: () => new Date(wakeAt), + stopScheduler: async () => undefined, }; const supervisorSender = createControlChannelSender({ privateKeySeed: supervisorKeyPair.privateKey, @@ -877,6 +896,172 @@ describe("runWorkflowChild", () => { await runPromise; }); + test("stops the host scheduler once the control loop ends, so its timers do not keep the process open", async () => { + const baseDir = await makeTempDir("child-stop-scheduler-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + let schedulerStops = 0; + const bindings: RunWorkflowChildBindings = { + ...buildBindings({ baseDir, childKeyPair }), + stopScheduler: async () => { + schedulerStops += 1; + }, + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + expect(schedulerStops).toBe(0); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + expect(schedulerStops).toBeGreaterThan(0); + }); + + test("a shutdown stops the host scheduler before it cancels the writes, so a timer's commit it cancels is no failure", async () => { + const baseDir = await makeTempDir("child-shutdown-order-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + const order: string[] = []; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings: { + ...buildBindings({ baseDir, childKeyPair }), + stopScheduler: async () => { + order.push("stop scheduler"); + }, + }, + substrateWriteBridge: { + handleMergeRequest: () => undefined, + handleWriteResponse: () => undefined, + cancelAll: () => { + order.push("cancel writes"); + }, + }, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + expect(order[0]).toBe("stop scheduler"); + expect(order).toContain("cancel writes"); + }); + + test("a control channel that closes with no shutdown frame still stops the host scheduler", async () => { + const baseDir = await makeTempDir("child-stop-scheduler-eof-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-def-", emptyWorkflowDefinition()); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + let schedulerStops = 0; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: createMemoryFrameStream().writer, + bindings: { + ...buildBindings({ baseDir, childKeyPair }), + stopScheduler: async () => { + schedulerStops += 1; + }, + }, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + supervisorToChild.close(); + await runPromise; + expect(schedulerStops).toBeGreaterThan(0); + }); + test("answers a quiet check as not quiet while a step is in flight", async () => { const baseDir = await makeTempDir("child-quiet-step-"); const supervisorKeyPair = await generateKeyPair(); @@ -1193,6 +1378,124 @@ describe("runWorkflowChild", () => { await runPromise; }); + test("answers input.taken for a committed signal, a duplicate trigger and a signal for no run", async () => { + const baseDir = await makeTempDir("child-input-taken-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-input-taken-def-", { + id: "test-workflow", + triggers: [], + steps: { "step-1": agentStep("step-1") }, + stepOrder: ["step-1"], + }); + await seedProcessingEntry( + baseDir, + { kind: "workflow-run", id: "deployment-x" }, + { + address: "deployment-x@example.com", + messageId: "msg-1", + receivedAt: 1, + text: "hello from the inbox", + }, + ); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + const env = parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ); + // The step holds the run in flight until the test lets it finish. + const stepStarted = Promise.withResolvers(); + const finishStep = Promise.withResolvers(); + const bindings: RunWorkflowChildBindings = { + ...buildBindings({ baseDir, childKeyPair }), + invokeStep: async () => { + stepStarted.resolve(undefined); + await finishStep.promise; + return { output: null }; + }, + }; + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const runPromise = runWorkflowChild({ + env, + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + await waitForUpstreamPayload(childToSupervisor, "ready"); + const taken = async (requestId: string) => + ( + await waitForUpstreamPayload( + childToSupervisor, + "input.taken", + (payload) => payload.data.requestId === requestId, + ) + ).data.committed; + const trigger = { + type: "trigger.fire", + data: { + runId: "run-1", + messageId: "msg-1", + receivedAt: 1, + payload: textMail("hello from the inbox"), + }, + } as const; + const signal = (runId: string, requestId: string) => ({ + type: "signal.deliver" as const, + data: { + runId, + signalName: "gate", + signalId: `sig-${requestId}`, + payload: { approved: true }, + requestId, + }, + }); + + await supervisorSender.send({ + ...trigger, + data: { ...trigger.data, requestId: "first" }, + }); + expect(await taken("first")).toBe(true); + await stepStarted.promise; + await supervisorSender.send({ + ...trigger, + data: { ...trigger.data, requestId: "duplicate" }, + }); + expect(await taken("duplicate")).toBe(false); + await supervisorSender.send(signal("run-1", "signal")); + expect(await taken("signal")).toBe(true); + await supervisorSender.send(signal("run-ghost", "no-run")); + expect(await taken("no-run")).toBe(false); + + finishStep.resolve(undefined); + await waitForUpstreamPayload(childToSupervisor, "terminal.event"); + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await runPromise; + }); + test("routes a mailbox.notify to the watch registry as an exists event", async () => { const baseDir = await makeTempDir("child-mailbox-notify-"); const supervisorKeyPair = await generateKeyPair(); @@ -1285,7 +1588,7 @@ describe("runWorkflowChild", () => { ]); }); - test("drops a signal.deliver for a run this child is not driving", async () => { + test("drops a signal.deliver for a run this child is not driving, and still answers it", async () => { const baseDir = await makeTempDir("child-signal-drop-"); const supervisorKeyPair = await generateKeyPair(); const childKeyPair = await generateKeyPair(); @@ -1362,8 +1665,16 @@ describe("runWorkflowChild", () => { signalName: "go", signalId: "sig-1", payload: { x: 1 }, + requestId: "signal-1", }, }); + // A standby attempt waits for the reply, so a dropped delivery answers too, + // as not committed. + const delivered = await waitForUpstreamPayload( + childToSupervisor, + "input.taken", + ); + expect(delivered.data).toEqual({ requestId: "signal-1", committed: false }); await supervisorSender.send({ type: "shutdown", data: { reason: "test done" }, @@ -2770,6 +3081,7 @@ describe("warm-agent round-trip (Phase 4.4)", () => { spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, earliestQueuedTimer: () => undefined, + stopScheduler: async () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], @@ -2968,6 +3280,7 @@ describe("warm-agent round-trip (Phase 4.4)", () => { spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, earliestQueuedTimer: () => undefined, + stopScheduler: async () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index 5c7f9df7c..f4d3458d7 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -358,6 +358,8 @@ export interface RunWorkflowChildBindings { * must be woken for. */ earliestQueuedTimer: () => Date | undefined; + /** Cancels the host scheduler's timers, which would keep the process open. */ + stopScheduler: () => Promise; /** Grant evaluator wired against the host's grant-rule grammar. */ evaluateGrants: GrantEvaluator; /** @@ -1065,6 +1067,7 @@ export async function runWorkflowChild( }; const cleanupControlLoop = async (): Promise => { + stopHostScheduler(opts.bindings); cancellationBarrier.close("workflow-child control loop exited"); // Any exit path -- clean (iterator end), dirty (thrown error), // shutdown (already cancelled, repeat is a no-op on an empty map) @@ -1125,10 +1128,17 @@ export async function runWorkflowChild( }; } +function stopHostScheduler(bindings: RunWorkflowChildBindings): void { + void bindings.stopScheduler().catch((cause: unknown) => { + logger.warn`stopping the host scheduler failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); +} + // Whether every run only waits on input from outside, for a supervisor that // stops the child once it is. Buffered events are made durable and input on // its way to a run is delivered first, so the answer holds for the log the -// stopped child would leave. A check that fails is reported, not retried. +// stopped child would leave. A check that fails is reported, not retried: +// the supervisor refuses the standby attempt with it. async function answerQuiet( ctx: { bindings: RunWorkflowChildBindings; @@ -1253,6 +1263,12 @@ async function handleControlPayload( // (that would block the control loop). if (ctx.runsInFlight.has(payload.data.runId)) { ctx.triggeredRunIds.push(payload.data.runId); + void answerInputTaken( + ctx.upstreamSender, + payload.data.requestId, + payload.data.runId, + false, + ); return false; } // The supervisor resolved the inbound mail to the run's input (the @@ -1299,6 +1315,14 @@ async function handleControlPayload( triggerPayload, }); ctx.runsInFlight.set(payload.data.runId, handle); + void handle.started.then((committed) => + answerInputTaken( + ctx.upstreamSender, + payload.data.requestId, + payload.data.runId, + committed, + ), + ); // Fan the run's terminal status back to the supervisor over the // upstream control channel. The supervisor's dispatch loop and // any armed drainTimeout accumulator subscribe through the @@ -1380,8 +1404,16 @@ async function handleControlPayload( // re-discovered -- must not commit an orphan `SignalReceived` to a log no // awaiter is tailing. `runsInFlight` is the one-driver authority on which // runs this child drives. + const acknowledge = (committed: boolean) => + answerInputTaken( + ctx.upstreamSender, + payload.data.requestId, + payload.data.runId, + committed, + ); if (!ctx.runsInFlight.has(payload.data.runId)) { logger.warn`signal.deliver for run ${payload.data.runId} which is not in flight; dropping (signalName=${payload.data.signalName})`; + void acknowledge(false); return false; } // Land the signal as a `SignalReceived` commit on the run's @@ -1421,8 +1453,9 @@ async function handleControlPayload( clock: ctx.clock, }); void (async () => { + let committed = false; try { - await transientSignalChannel.deliver( + committed = await transientSignalChannel.land( payload.data.signalName, payload.data.payload, payload.data.signalId, @@ -1430,11 +1463,15 @@ async function handleControlPayload( ? [payload.data.correlationId] : []), ); + if (!committed) { + logger.warn`signal.deliver for run ${payload.data.runId} whose log already ended; refusing it (signalName=${payload.data.signalName})`; + } } catch (cause) { const reason = cause instanceof Error ? cause.message : String(cause); logger.warn`signal.deliver commit failed runId=${payload.data.runId} signalName=${payload.data.signalName}: ${reason}`; } finally { await transientSignalChannel.stop(); + await acknowledge(committed); endDelivery(); } })(); @@ -1461,6 +1498,9 @@ async function handleControlPayload( case "cancel.prepared": { throw new Error("workflow-child received an upstream cancellation reply"); } + case "input.taken": { + throw new Error("workflow-child received an upstream input reply"); + } case "drain": { // The supervisor's `drain` control mail flips the controller's // signal. The runtime body's four observation points read the @@ -1475,6 +1515,9 @@ async function handleControlPayload( } case "shutdown": { logger.info`workflow-child shutdown requested (${payload.data.reason})`; + // Before the writes are cancelled, so the scheduler takes a commit this + // cancels for the stop it is, not a failure. + stopHostScheduler(ctx.bindings); if (ctx.substrateWriteBridge !== undefined) { ctx.substrateWriteBridge.cancelAll("workflow-child shutdown requested"); } @@ -2015,6 +2058,28 @@ function buildRuntimeEnv(args: { return env; } +/** + * Tell the supervisor whether the input a `trigger.fire` or `signal.deliver` + * carried is committed, when the frame asked. Best-effort like the other + * upstream sends: a reply is lost only with the control channel, and until the + * child is replaced every standby attempt of this deployment waits for it, + * which `wake` abandons. + */ +async function answerInputTaken( + upstreamSender: ControlChannelSender, + requestId: string | undefined, + runId: string, + committed: boolean, +): Promise { + if (requestId === undefined) return; + await upstreamSender + .send({ type: "input.taken", data: { requestId, committed } }) + .catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`input.taken reply failed runId=${runId}: ${message}`; + }); +} + /** * Forward a control-plane suspension to the supervisor over the upstream * control channel. Fired from `env.onPark` each time a workflow agent step diff --git a/packages/workflow-host/src/child/substrate-factory.ts b/packages/workflow-host/src/child/substrate-factory.ts index c2cabe349..4546876ec 100644 --- a/packages/workflow-host/src/child/substrate-factory.ts +++ b/packages/workflow-host/src/child/substrate-factory.ts @@ -2883,6 +2883,7 @@ export function createSidecarSubstrateFactory( }, scheduler, earliestQueuedTimer: () => hostScheduler.earliestQueuedTimer(), + stopScheduler: () => hostScheduler.stop(), evaluateGrants: evaluateGrantsAdapter, ...parkHooks, // The same registry the warm agent's transport registers `watch` diff --git a/packages/workflow-host/src/index.ts b/packages/workflow-host/src/index.ts index 0a0a19920..5ddecb00d 100644 --- a/packages/workflow-host/src/index.ts +++ b/packages/workflow-host/src/index.ts @@ -42,6 +42,9 @@ export { } from "./adapters/spawn-child"; export { createWorkflowSupervisor, + StandbyAbandonedError, + StandbyDeferredError, + StandbyRefusedError, isErrnoNotFound, commitCancelRequested, createDrainTimeoutAccumulator, @@ -75,6 +78,9 @@ export { type RecyclePolicy, type RecyclePolicyBounds, type RecyclePolicyOpts, + type RecordedStandby, + type StandbyOutcome, + type StandbyRestore, type SignedPayload, type SpawnOpts, type SpawnResult, diff --git a/packages/workflow-host/src/ipc/control-payloads.ts b/packages/workflow-host/src/ipc/control-payloads.ts index c164d8c45..2af8d54ef 100644 --- a/packages/workflow-host/src/ipc/control-payloads.ts +++ b/packages/workflow-host/src/ipc/control-payloads.ts @@ -67,6 +67,18 @@ export const SourcesUpdatedData = type({ return true; }); +/** + * One correlation a child's run is parked on, as the child reports it. The + * snapshot is required for an approval park and absent for an input park. + */ +export const ParkedCorrelation = type({ + runId: "string > 0", + correlationId: "string > 0", + parkKind: ControlParkKind, + "snapshot?": BoundedApprovalSnapshot, +}); +export type ParkedCorrelation = typeof ParkedCorrelation.infer; + /** * Wire projection of an attachment on an outbound mail message. The * runtime `MessageAttachment.data` is raw bytes; the NDJSON control @@ -527,6 +539,9 @@ const controlPayloadFactories = defineControlPayloadFactories({ runId: "string", messageId: "string", receivedAt: "number", + // Asks for an `input.taken` reply once the run's `RunStarted` is + // committed. + "requestId?": "string", // The run's inbound-mail input, resolved by the supervisor (the sole // mail owner) before the frame: a decoded `Mail` (headers plus part // descriptors that reference the part bytes committed to the workflow-run @@ -547,6 +562,8 @@ const controlPayloadFactories = defineControlPayloadFactories({ // Addresses one onTrigger relay park. Omitted, the child commits // a name-FIFO delivery. "correlationId?": "string > 0", + // Asks for an `input.taken` reply once the child's commit settles. + "requestId?": "string", // The resume decision in FINAL form -- the child commits it as the // SignalReceived payload verbatim. Each sender owns any // provenance-specific preparation BEFORE this frame: the dispatch loop @@ -557,6 +574,16 @@ const controlPayloadFactories = defineControlPayloadFactories({ payload: "unknown", }, }), + "input.taken": () => + type({ + // Answers a `trigger.fire` or `signal.deliver` that carried a + // `requestId`, once the child has committed the input or declined it. A + // standby attempt waits for this, so an input the child took is not + // dropped with the stopped child's writes. Declined mail refuses the + // attempt, and a declined signal is dropped. + type: "'input.taken'", + data: { requestId: "string", committed: "boolean" }, + }), drain: () => type({ type: "'drain'", @@ -948,13 +975,7 @@ const controlPayloadFactories = defineControlPayloadFactories({ type: "'parked-correlations.response'", data: { requestId: "string > 0", - parked: type({ - runId: "string > 0", - correlationId: "string > 0", - parkKind: ControlParkKind, - // Snapshot is required for approval parks and absent for input parks. - "snapshot?": BoundedApprovalSnapshot, - }).array(), + parked: ParkedCorrelation.array(), }, }), "quiet.request": () => @@ -976,7 +997,8 @@ const controlPayloadFactories = defineControlPayloadFactories({ type({ // `ended` alone when the deployment's run has ended, so there is // nothing to stop it for. Otherwise `wakeAt` is the earliest timer the - // child has queued, and `error` says the child could not check. + // child has queued, and `error` says the child could not check, which + // refuses the standby attempt. type: "'quiet.response'", data: type({ requestId: "string > 0", diff --git a/packages/workflow-host/src/seams/scheduler.test.ts b/packages/workflow-host/src/seams/scheduler.test.ts index 8f9a9dfa9..8136972a4 100644 --- a/packages/workflow-host/src/seams/scheduler.test.ts +++ b/packages/workflow-host/src/seams/scheduler.test.ts @@ -14,7 +14,12 @@ import type { RepoStore, ValidatePushResult, } from "@intx/hub-sessions"; -import { commitBuffered, dropChain, reloadState } from "@intx/workflow/runtime"; +import { + commitBuffered, + dropChain, + reloadState, + whenDelivered, +} from "@intx/workflow/runtime"; import { createWorkflowRunRepoStore } from "../adapters/repo-store"; import { createWorkflowHostScheduler } from "./scheduler"; @@ -918,6 +923,59 @@ describe("workflow-host scheduler", () => { } }); + // The scheduler reports a failed commit only as an unhandled rejection, which + // fails the test that is running. + test("a commit that fails once the scheduler stopped is not reported, since the next child queues its timer again", async () => { + const dataDir = await makeTempDir("scheduler-stopped-"); + const store = createRepoStore({ + dataDir, + signingKey, + handlers: { + skill: permissiveHandler("workflow-runs-stopped"), + }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: "deployment-stopped" }; + const runId = "r-timer-stopped"; + const runtimeStore = runtimeStoreFor(store, repoId); + await runtimeStore.append(runId, { + kind: "TimerSet", + seq: 1, + at: "2026-01-01T00:00:00.000Z", + timerId: "t-commit", + fireAt: new Date(Date.now() + 60_000).toISOString(), + }); + const writing = Promise.withResolvers(); + const write = Promise.withResolvers(); + const timeouts = createManualTimeouts(); + const scheduler = createWorkflowHostScheduler({ + repoStore: { + ...store, + writeTreePreservingPrefix: () => { + writing.resolve(undefined); + return write.promise; + }, + }, + runtimeStore, + principal, + listActiveDeployments: () => [repoId], + ref: REF, + clock: () => new Date(), + scheduleTimeout: timeouts.scheduleTimeout, + }); + try { + await scheduler.start(); + timeouts.fireAll(); + await writing.promise; + await scheduler.stop(); + write.reject(new Error("write cancelled")); + await whenDelivered(); + } finally { + await scheduler.stop(); + dropChain(runId); + } + }); + test("an empty buffer appends TimerFired at the next durable sequence", async () => { const dataDir = await makeTempDir("scheduler-empty-"); const store = createRepoStore({ diff --git a/packages/workflow-host/src/seams/scheduler.ts b/packages/workflow-host/src/seams/scheduler.ts index d42d9e89c..749ad1ab4 100644 --- a/packages/workflow-host/src/seams/scheduler.ts +++ b/packages/workflow-host/src/seams/scheduler.ts @@ -267,6 +267,9 @@ export function createWorkflowHostScheduler( const delayMs = Math.max(0, fireAtMs - opts.clock().getTime()); const cancelTimeout = schedule(() => { void fireTimer(runId, timerId).catch((cause) => { + // A stop overtook this commit. The timer stays unfired in the log, so + // the next child's recovery queues it again. + if (stopped) return; // The scheduler's commit failed. Surface as unhandled so // operators see it; the runtime body's awaiter will hang // until restart triggers recovery. diff --git a/packages/workflow-host/src/seams/signal-channel.test.ts b/packages/workflow-host/src/seams/signal-channel.test.ts index 45900c455..92c34e5f4 100644 --- a/packages/workflow-host/src/seams/signal-channel.test.ts +++ b/packages/workflow-host/src/seams/signal-channel.test.ts @@ -612,6 +612,60 @@ describe("workflow-host signal channel", () => { } }); + test("land says a log that ended took no signal, and a repeated one is held", async () => { + const dataDir = await makeTempDir("sigchan-land-"); + const store = createRepoStore({ + dataDir, + signingKey, + handlers: { skill: permissiveHandler("workflow-runs-land") }, + authorize: allowAll, + }); + const repoId: RepoId = { kind: "skill", id: "deployment-land" }; + const runId = "r-land"; + const box = makeStateBox(runId); + const runtimeStore = runtimeStoreFor(store, repoId); + const at = "2026-01-01T00:00:00.000Z"; + const channel = createWorkflowHostSignalChannel({ + repoStore: store, + runtimeStore, + principal, + repoId, + ref: REF, + runId, + readState: () => box.state, + newId: makeIdGen("sig"), + clock: () => new Date(at), + }); + try { + expect(await channel.land("approve", { ok: true }, "sig-early")).toBe( + true, + ); + expect(await channel.land("approve", { ok: true }, "sig-early")).toBe( + true, + ); + await runtimeStore.append(runId, { + kind: "RunFailed", + seq: 1, + at, + error: { message: "already failed" }, + }); + expect(await channel.land("approve", { ok: true }, "sig-late")).toBe( + false, + ); + expect(await channel.land("approve", { ok: true }, "sig-early")).toBe( + true, + ); + const events = await runtimeStore.read(runId); + expect(events.map((event) => event.kind)).toEqual([ + "SignalReceived", + "RunFailed", + ]); + } finally { + await channel.stop(); + dropChain(runId); + } + }); + test("peek reads the name queue and the matching correlation separately", async () => { const dataDir = await makeTempDir("sigchan-peek-"); const store = createRepoStore({ diff --git a/packages/workflow-host/src/seams/signal-channel.ts b/packages/workflow-host/src/seams/signal-channel.ts index 6ffded61a..648e21652 100644 --- a/packages/workflow-host/src/seams/signal-channel.ts +++ b/packages/workflow-host/src/seams/signal-channel.ts @@ -145,6 +145,12 @@ type NameSubscription = { }; export type SignalChannelHandle = SignalChannel & { + /** + * `deliver`, answering whether the run holds the signal. A log that already + * ended takes none, so nothing is written and the answer is `false`; a + * repeated `signalId` is already held. + */ + land(...args: Parameters): Promise; /** * Tear down every slot subscription and reject every pending * awaiter. Idempotent. After `stop()` the channel holds no @@ -303,14 +309,18 @@ export function createWorkflowHostSignalChannel( }); } - return { - async deliver(name, payload, signalId, correlationId) { + const handle: SignalChannelHandle = { + async deliver(...args) { + await handle.land(...args); + }, + async land(name, payload, signalId, correlationId) { if (stopped) { throw new Error("signal channel: deliver after stop"); } const id = signalId ?? opts.newId(); const at = opts.clock().toISOString(); const prefix = `runs/${opts.runId}/events/`; + let landed = true; await withRunCommitBarrier( { repoStore: opts.runtimeStore }, opts.runId, @@ -357,6 +367,7 @@ export function createWorkflowHostSignalChannel( // past RunFailed/RunCompleted/RunCancelled would make the // next resume disagree with the terminal result already // reported. + landed = duplicate || !terminal; if (duplicate || terminal) return out; const nextSeq = maxSeq + 1; // The workflow-run kind handler's `EventEnvelope` @@ -380,6 +391,7 @@ export function createWorkflowHostSignalChannel( }, ), ); + return landed; }, async awaitNext(name, onWake, signal, correlationId) { if (stopped) { @@ -456,6 +468,7 @@ export function createWorkflowHostSignalChannel( } }, }; + return handle; } function isTerminalOnDiskEvent(parsed: unknown): boolean { diff --git a/packages/workflow-host/src/supervisor/crash-respawn.test.ts b/packages/workflow-host/src/supervisor/crash-respawn.test.ts index 52dba384f..081268c8e 100644 --- a/packages/workflow-host/src/supervisor/crash-respawn.test.ts +++ b/packages/workflow-host/src/supervisor/crash-respawn.test.ts @@ -327,6 +327,9 @@ function createMemoryInboxPrimitives(): InboxPrimitives & { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address) { replayCalls += 1; replayChanges.notify(); diff --git a/packages/workflow-host/src/supervisor/index.ts b/packages/workflow-host/src/supervisor/index.ts index 790105dda..1ce7b3106 100644 --- a/packages/workflow-host/src/supervisor/index.ts +++ b/packages/workflow-host/src/supervisor/index.ts @@ -1,11 +1,16 @@ export { createWorkflowSupervisor, + StandbyAbandonedError, + StandbyDeferredError, + StandbyRefusedError, type CancelCommitInfo, type CancelRequestOpts, type DeliverCredentialsOpts, type DeliverSignalOpts, type DeliverSourcesOpts, type DrainOpts, + type StandbyOutcome, + type StandbyRestore, type RecycleOpts, type SpawnOpts, type SpawnResult, @@ -58,6 +63,7 @@ export type { InboxPrimitives, MailBusBindings, PrincipalSigner, + RecordedStandby, SignedPayload, SubprocessHandle, SubprocessSpawner, diff --git a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts index 59eed1e58..3c7e3c11b 100644 --- a/packages/workflow-host/src/supervisor/lifecycle-races.test.ts +++ b/packages/workflow-host/src/supervisor/lifecycle-races.test.ts @@ -144,6 +144,9 @@ function createMemoryInboxPrimitives(): InboxPrimitives { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address) { const s = getOrCreate(address); const replayedKeys: string[] = []; diff --git a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts index d0cde9d09..c1938ccb9 100644 --- a/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts +++ b/packages/workflow-host/src/supervisor/outbound-signed-send.test.ts @@ -73,6 +73,9 @@ function createNoopInboxPrimitives(): InboxPrimitives { async replayProcessingToInbox() { return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, + async hasInboxEntries() { + return false; + }, }; } diff --git a/packages/workflow-host/src/supervisor/park-notify.test.ts b/packages/workflow-host/src/supervisor/park-notify.test.ts index bc7444d9c..1b2d5498d 100644 --- a/packages/workflow-host/src/supervisor/park-notify.test.ts +++ b/packages/workflow-host/src/supervisor/park-notify.test.ts @@ -55,6 +55,9 @@ function createNoopInboxPrimitives(): InboxPrimitives { async replayProcessingToInbox() { return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, + async hasInboxEntries() { + return false; + }, }; } diff --git a/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts b/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts index 04e7e644b..b5218a0ad 100644 --- a/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts +++ b/packages/workflow-host/src/supervisor/re-emit-parked-correlations.test.ts @@ -85,6 +85,9 @@ function createNoopInboxPrimitives(): InboxPrimitives { async replayProcessingToInbox() { return { commitSha: "noop", replayedKeys: [], retainedMessageIds: [] }; }, + async hasInboxEntries() { + return false; + }, }; } diff --git a/packages/workflow-host/src/supervisor/recorded-standby.ts b/packages/workflow-host/src/supervisor/recorded-standby.ts new file mode 100644 index 000000000..22f7dbb7c --- /dev/null +++ b/packages/workflow-host/src/supervisor/recorded-standby.ts @@ -0,0 +1,21 @@ +import { type } from "arktype"; + +import { WorkflowProjectionDefinition } from "@intx/types/sidecar"; + +import { ParkedCorrelation } from "../ipc/control-payloads"; + +/** + * A child in standby as its host records it. A restart restores the standby + * from this, without the child, while the run's history is still at `tip`: + * `parked` is what the stopped child's runs were parked on, `wakeAt` the + * earliest timer it had queued, `tip` the run's history tip, read before the + * child reported its parks, and `definition` the one the first child verified, + * which lays the steps out again with no child to report it. + */ +export const RecordedStandby = type({ + parked: ParkedCorrelation.array(), + "wakeAt?": "string.date.iso", + tip: "string | null", + definition: WorkflowProjectionDefinition, +}); +export type RecordedStandby = typeof RecordedStandby.infer; diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index 5b4f3e867..ab93c86f7 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -28,6 +28,7 @@ import { generateKeyPair } from "@intx/crypto"; import { base64Encode, hexEncode } from "@intx/types"; import type { CredentialDelivery } from "@intx/types/credential-delivery"; import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; +import { signalName } from "@intx/types/signals"; import type { RepoStore } from "@intx/hub-sessions"; import { NotInProcessingError } from "@intx/hub-sessions"; import { @@ -39,6 +40,7 @@ import { createChangeNotifier, waitForTriggerFireRunIds, waitForUpstreamPayload, + waitForUpstreamPayloads, createStubRepoStore, sendDefinition, TEST_DEFINITION, @@ -47,14 +49,22 @@ import { import { createWorkflowSupervisor, + StandbyAbandonedError, + StandbyDeferredError, + StandbyRefusedError, type InboxPrimitives, type MailBusBindings, + type RecordedStandby, type SignedPayload, type SubprocessHandle, type SubprocessSpawner, type WorkflowSupervisor, type WorkflowSupervisorBindings, } from "./index"; +import { + DEFAULT_KILL_TIMEOUT_MS, + defaultClearTimer, +} from "./child-termination"; import { createRecyclePolicy, DEFAULT_POLICY_INTERVAL_MS, @@ -69,6 +79,7 @@ import { SignedEnvelope, } from "../ipc/index"; import { seedStepGrants, seededOnRunStart } from "./step-grants.test-helpers"; +import type { ParkedCorrelation } from "../ipc/control-payloads"; function parseSignalDeliverRunIds(lines: readonly string[]): string[] { const ids: string[] = []; @@ -144,6 +155,11 @@ type SpawnTracker = { function createSpawnTracker(opts: { sigtermExits?: boolean; killThrows?: boolean; + // Fails the supervisor's control write of a line, as a closing pipe does. + controlWriteFails?: (line: string) => boolean; + // Holds the supervisor's control write of a line until the promise it + // returns settles. + holdControlWrite?: (line: string) => Promise | undefined; }): SpawnTracker { const children: FakeChild[] = []; const spawnEnvs: Record[] = []; @@ -172,9 +188,22 @@ function createSpawnTracker(opts: { }; children.push(child); changes.notify(); + const { controlWriteFails, holdControlWrite } = opts; const handle: SubprocessHandle = { pid: child.pid, - controlWriter: supervisorToChild.writer, + controlWriter: + controlWriteFails === undefined && holdControlWrite === undefined + ? supervisorToChild.writer + : { + write: (line) => { + if (controlWriteFails?.(line) === true) + throw new Error("the child's control pipe is closed"); + const held = holdControlWrite?.(line); + return held === undefined + ? supervisorToChild.writer.write(line) + : held.then(() => supervisorToChild.writer.write(line)); + }, + }, controlReader: childToSupervisor.reader, eventReader: eventChildToSupervisor.reader, kill: (signal) => { @@ -371,6 +400,9 @@ function createMemoryInboxPrimitives(): InboxPrimitives & { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address, opts) { const s = getOrCreate(address); const replayedKeys: string[] = []; @@ -2919,6 +2951,19 @@ describe("supervisor recycle: the new child's authority", () => { const RUN = "run_deployment-x"; const ADDRESS = "run_deployment-x@example.com"; +// The tip of the run's history that the standby tests' store reports. +const HISTORY_TIP = "tip-at-stop"; +const GATE_PARK: RecordedStandby["parked"][number] = { + runId: RUN, + correlationId: "corr-gate", + parkKind: "approval", + snapshot: { + name: "charge_card", + description: "Charge the customer's card", + inputSchema: { type: "object" }, + arguments: { amount: 100 }, + }, +}; // The run's log in the stub store's directory, which the stub serves as its // committed history. @@ -2946,27 +2991,27 @@ function runStarted(): Record { }; } +// Move a mail to processing/, as a dispatch that took it would. +async function takeMail( + inbox: ReturnType, + baseDir: string, + messageId: string, + receivedAt: number, +): Promise { + const store = createStubRepoStore(baseDir); + const repoId = { kind: "workflow-run", id: RUN } as const; + const principal = { kind: "supervisor", anchorRunId: RUN } as const; + await inbox.enqueueInbox(store, principal, repoId, { + address: ADDRESS, + messageId, + receivedAt, + }); + await inbox.dequeueToProcessing(store, principal, repoId, ADDRESS); +} + // A respawned child resumes the run from its log, so mail the run already // took stays in processing/ with it instead of being delivered again. describe("supervisor respawn: mail a resumed run took", () => { - // Move a mail to processing/, as a dispatch that took it would. - async function takeMail( - inbox: ReturnType, - baseDir: string, - messageId: string, - receivedAt: number, - ): Promise { - const store = createStubRepoStore(baseDir); - const repoId = { kind: "workflow-run", id: RUN } as const; - const principal = { kind: "supervisor", anchorRunId: RUN } as const; - await inbox.enqueueInbox(store, principal, repoId, { - address: ADDRESS, - messageId, - receivedAt, - }); - await inbox.dequeueToProcessing(store, principal, repoId, ADDRESS); - } - async function setupOwned(opts: { inbox: ReturnType; inboxPrimitives?: Partial; @@ -3664,3 +3709,3903 @@ describe("supervisor respawn: mail a resumed run took", () => { }); }); }); + +// Answer the `count`th correlation query on the child's channel. Every spawn +// sends one of its own, so a standby attempt's query is the second. +async function answerParkedQuery( + child: FakeChild, + childSender: ReturnType, + count: number, + parked: ParkedCorrelation[] = [], +): Promise { + const queries = await waitForUpstreamPayloads( + child.supervisorToChild, + "parked-correlations.request", + count, + ); + const query = queries[count - 1]; + if (query === undefined) throw new Error("correlation query missing"); + await childSender.send({ + type: "parked-correlations.response", + data: { requestId: query.data.requestId, parked }, + }); +} + +// Answer the `count`th quiet query on the child's channel, and return it. +async function answerQuiet( + child: FakeChild, + childSender: ReturnType, + count: number, + answer: { ended: true } | { quiet: boolean; wakeAt?: string; error?: string }, +): Promise<{ wait: boolean }> { + const queries = await waitForUpstreamPayloads( + child.supervisorToChild, + "quiet.request", + count, + ); + const query = queries[count - 1]; + if (query === undefined) throw new Error("quiet query missing"); + await childSender.send({ + type: "quiet.response", + data: { requestId: query.data.requestId, ...answer }, + }); + return query.data; +} + +// The fake child reports quiet for the wait and the final check, answers the +// standby attempt's correlation query, and exits on its `shutdown`, as the real +// child's control loop does. `firstQuery` numbers the wait's query. +async function actAsChildEnteringStandby( + child: FakeChild, + childSender: ReturnType, + opts: { firstQuery?: number; wakeAt?: string } = {}, +): Promise { + const first = opts.firstQuery ?? 1; + const answer = { + quiet: true, + ...(opts.wakeAt !== undefined ? { wakeAt: opts.wakeAt } : {}), + }; + expect((await answerQuiet(child, childSender, first, answer)).wait).toBe( + true, + ); + expect((await answerQuiet(child, childSender, first + 1, answer)).wait).toBe( + false, + ); + await answerParkedQuery(child, childSender, 2); + await waitForUpstreamPayload(child.supervisorToChild, "shutdown"); + child.eventChildToSupervisor.close(); + child.childToSupervisor.close(); + child.resolveExit?.(0); +} + +function sentFrameTypes(child: FakeChild): string[] { + return parseControlFrames(child.supervisorToChild.flushed()).map( + (frame) => frame.type, + ); +} + +// The fake child commits the trigger the supervisor fired at it. +async function takeTrigger( + child: FakeChild, + childSender: ReturnType, +): Promise { + const fire = await waitForUpstreamPayload( + child.supervisorToChild, + "trigger.fire", + ); + const requestId = fire.data.requestId; + if (requestId === undefined) throw new Error("the trigger has no requestId"); + await childSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); +} + +describe("supervisor standby and wake", () => { + async function setup( + opts: { + onWake?: WorkflowSupervisorBindings["onWake"]; + recordStandby?: (standby: RecordedStandby) => Promise; + inboxPrimitives?: InboxPrimitives; + beforeSpawn?: (baseDir: string) => Promise; + repoStore?: (baseDir: string) => RepoStore; + onSuspensionRegister?: WorkflowSupervisorBindings["onSuspensionRegister"]; + onStandbyWait?: WorkflowSupervisorBindings["onStandbyWait"]; + onShutdownWait?: WorkflowSupervisorBindings["onShutdownWait"]; + onDispatchWait?: WorkflowSupervisorBindings["onDispatchWait"]; + parkedQueryWatchdogMs?: number; + timers?: Pick< + WorkflowSupervisorBindings, + "setTimer" | "clearTimer" | "now" + >; + recyclePolicy?: Partial; + drainTimeoutAccumulatorFactory?: WorkflowSupervisorBindings["drainTimeoutAccumulatorFactory"]; + respawnBackoffInitialMs?: number; + controlWriteFails?: (line: string) => boolean; + holdControlWrite?: (line: string) => Promise | undefined; + } = {}, + ) { + const baseDir = await makeTempDir("standby-"); + await opts.beforeSpawn?.(baseDir); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({ + ...(opts.controlWriteFails !== undefined + ? { controlWriteFails: opts.controlWriteFails } + : {}), + ...(opts.holdControlWrite !== undefined + ? { holdControlWrite: opts.holdControlWrite } + : {}), + }); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + overrides: { + repoStore: + opts.repoStore?.(baseDir) ?? + createStubRepoStore(baseDir, { + writeTree: true, + refTip: HISTORY_TIP, + }), + ...(opts.onSuspensionRegister !== undefined + ? { onSuspensionRegister: opts.onSuspensionRegister } + : {}), + ...(opts.onWake !== undefined ? { onWake: opts.onWake } : {}), + ...(opts.recordStandby !== undefined + ? { recordStandby: opts.recordStandby } + : {}), + ...(opts.onStandbyWait !== undefined + ? { onStandbyWait: opts.onStandbyWait } + : {}), + ...(opts.onShutdownWait !== undefined + ? { onShutdownWait: opts.onShutdownWait } + : {}), + ...(opts.onDispatchWait !== undefined + ? { onDispatchWait: opts.onDispatchWait } + : {}), + ...(opts.parkedQueryWatchdogMs !== undefined + ? { parkedQueryWatchdogMs: opts.parkedQueryWatchdogMs } + : {}), + ...(opts.inboxPrimitives !== undefined + ? { inboxPrimitives: opts.inboxPrimitives } + : {}), + ...(opts.drainTimeoutAccumulatorFactory !== undefined + ? { + drainTimeoutAccumulatorFactory: + opts.drainTimeoutAccumulatorFactory, + } + : {}), + ...(opts.respawnBackoffInitialMs !== undefined + ? { respawnBackoffInitialMs: opts.respawnBackoffInitialMs } + : {}), + ...opts.timers, + ...opts.recyclePolicy, + }, + }); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + return { + supervisor, + firstSender, + first, + tracker, + mailBus, + ipcKeypair, + }; + } + + test("a child in standby has no recycle policy, and the woken child's uptime counts from its start", async () => { + const woke = Promise.withResolvers(); + const policy = manualRecyclePolicy(60_000); + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + onWake: ({ reason }) => woke.resolve(reason), + recyclePolicy: policy.bindings, + }); + expect(policy.armed()).toBe(1); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + expect(policy.armed()).toBe(0); + + policy.advance(120_000); + const armed = policy.nextArm(); + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("mail"); + await armed; + + const ticked = policy.nextArm(); + policy.tick(); + await ticked; + expect(tracker.totalSpawns).toBe(2); + + await supervisor.shutdown(); + }); + + test("stops the child once it reports quiet, and mail respawns it", async () => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ onWake: ({ reason }) => woke.resolve(reason) }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + expect(tracker.totalSpawns).toBe(1); + + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("mail"); + + await supervisor.shutdown(); + }); + + test("the child's exit for standby does not count as a crash", async () => { + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ recyclePolicy: { crashLoopMaxCount: 1 } }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + + await supervisor.shutdown(); + }); + + test("a bad frame from the child going into standby does not stop the supervisor", async () => { + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup(); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + first.childToSupervisor.inject("not-json{{{"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await inStandby).toEqual({}); + + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + + await supervisor.shutdown(); + }); + + // The inbox as the standby attempt reads it right before the stop, empty, and + // as `after` once the child stopped. + function emptyUntilStopped( + after: InboxPrimitives["hasInboxEntries"], + ): InboxPrimitives["hasInboxEntries"] { + let reads = 0; + return async (...args) => { + reads += 1; + return reads === 1 ? false : after(...args); + }; + } + + // Mail enqueued between the last read before the stop and the stop is in the + // inbox but was never dispatched. + const waitingInboxes: [string, InboxPrimitives["hasInboxEntries"]][] = [ + ["holds mail", async () => true], + [ + "cannot be read", + async () => { + throw new Error("the inbox is unreadable"); + }, + ], + ]; + test.each(waitingInboxes)( + "a child whose inbox %s once it stops is respawned", + async (_case, hasInboxEntries) => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = + await setup({ + onWake: ({ reason }) => woke.resolve(reason), + inboxPrimitives: { + ...createMemoryInboxPrimitives(), + hasInboxEntries: emptyUntilStopped(hasInboxEntries), + }, + }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("mail"); + + await supervisor.shutdown(); + }, + ); + + test("mail that lands while the last read before the stop runs wakes the stopped child", async () => { + const woke = Promise.withResolvers(); + const memory = createMemoryInboxPrimitives(); + let deliverMail = (): Promise => Promise.resolve(); + let reads = 0; + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + onWake: ({ reason }) => woke.resolve(reason), + inboxPrimitives: { + ...memory, + // The read began before the mail landed, and the mail's arrival + // finds the child still running, so only the read after the stop + // can wake it. + async hasInboxEntries(...args) { + reads += 1; + if (reads > 1) return memory.hasInboxEntries(...args); + await deliverMail(); + return false; + }, + }, + }); + deliverMail = () => + mailBus.deliverSettled(ADDRESS, new TextEncoder().encode("late mail")); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("mail"); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused while mail waits in the inbox, and the child keeps running", async () => { + const { supervisor, firstSender, first, tracker } = await setup({ + inboxPrimitives: { + ...createMemoryInboxPrimitives(), + // Mail behind a run that waits on an approval. + async hasInboxEntries() { + return true; + }, + }, + }); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "mail is waiting in the deployment's inbox", + ); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused when the inbox cannot be read before the stop, and the child keeps running", async () => { + const { supervisor, firstSender, first, tracker } = await setup({ + inboxPrimitives: { + ...createMemoryInboxPrimitives(), + async hasInboxEntries() { + throw new Error("the inbox is unreadable"); + }, + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "the deployment's inbox could not be read: the inbox is unreadable", + ); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused when the run's history cannot be read, and nothing is recorded", async () => { + const records: RecordedStandby[] = []; + const { supervisor, firstSender, first, tracker } = await setup({ + recordStandby: async (standby) => { + records.push(standby); + }, + repoStore: (baseDir) => { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "resolveRef") + return Reflect.get(target, prop, receiver); + return async () => { + throw new Error("the history ref is unreadable"); + }; + }, + }); + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await expect(inStandby).rejects.toThrow( + "the run's history could not be read: the history ref is unreadable", + ); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + expect(records).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("mail a failed dispatch left in processing/ refuses a standby attempt", async () => { + const writeFailed = Promise.withResolvers(); + let failPartWrite = true; + const inbox = createMemoryInboxPrimitives(); + const { supervisor, firstSender, first, mailBus } = await setup({ + inboxPrimitives: inbox, + repoStore: (baseDir) => + new Proxy(createStubRepoStore(baseDir, { writeTree: true }), { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") + return Reflect.get(target, prop, receiver); + return async ( + ...args: Parameters + ) => { + // The write of the mail's parts, before any input reaches the + // child. + if (failPartWrite && args[3].preservePrefix.includes("/parts/")) { + failPartWrite = false; + writeFailed.resolve(undefined); + throw new Error("the mail's parts could not be written"); + } + return target.writeTreePreservingPrefix(...args); + }; + }, + }), + }); + mailBus.deliver( + ADDRESS, + new TextEncoder().encode( + `From: a@example.com\r\nTo: ${ADDRESS}\r\nContent-Type: text/plain\r\n\r\ninput`, + ), + ); + await writeFailed.promise; + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "mail a failed dispatch or replay left waits in processing/ for the child to restart", + ); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + expect(inbox.messageIds(ADDRESS).processing).toHaveLength(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused after a failed boot replay, which can leave mail behind", async () => { + const inbox = createMemoryInboxPrimitives(); + const { supervisor, firstSender, first } = await setup({ + inboxPrimitives: { + ...inbox, + replayProcessingToInbox: async () => { + throw new Error("the boot replay failed"); + }, + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "mail a failed dispatch or replay left waits in processing/ for the child to restart", + ); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused after a failed respawn replay, and the next replay clears it", async () => { + const inbox = createMemoryInboxPrimitives(); + let replays = 0; + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + respawnBackoffInitialMs: 1, + inboxPrimitives: { + ...inbox, + replayProcessingToInbox: async (...args) => { + replays += 1; + if (replays === 2) throw new Error("the respawn replay failed"); + return inbox.replayProcessingToInbox(...args); + }, + }, + }); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + await takeTrigger(first, firstSender); + + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(inbox.messageIds(ADDRESS).processing).toHaveLength(1); + + const refused = supervisor.enterStandby(); + await answerQuiet(second, secondSender, 1, { quiet: true }); + await answerQuiet(second, secondSender, 2, { quiet: true }); + await answerParkedQuery(second, secondSender, 2); + await expect(refused).rejects.toThrow( + "mail a failed dispatch or replay left waits in processing/ for the child to restart", + ); + expect(sentFrameTypes(second)).not.toContain("shutdown"); + + second.eventChildToSupervisor.close(); + second.childToSupervisor.close(); + second.resolveExit?.(1); + await tracker.awaitChildren(3); + const third = tracker.children[2]; + if (third === undefined) throw new Error("third child missing"); + const thirdSender = await driveReady(third, ipcKeypair); + await answerParkedQuery(third, thirdSender, 1); + await takeTrigger(third, thirdSender); + expect(inbox.messageIds(ADDRESS).inbox).toEqual([]); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(third, thirdSender); + expect(await inStandby).toEqual({}); + + await supervisor.shutdown(); + }); + + test("a child that crashes during the final check puts the standby attempt off, and the respawned child takes mail", async () => { + const waits: string[] = []; + const waited = createChangeNotifier(); + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + onStandbyWait: (waiting) => { + waits.push(waiting); + waited.notify(); + }, + }); + + const inStandby = supervisor.enterStandby(); + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + // A signal the child has not taken yet holds the final check back. + await supervisor.deliverSignal({ + runId: RUN, + signalName: "gate", + signalId: "sig-1", + payload: { approved: true }, + }); + const before = waits.length; + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waited.until(() => waits.slice(before).includes("standby")); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + + await expect(inStandby).rejects.toBeInstanceOf(StandbyDeferredError); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + await takeTrigger(second, secondSender); + + await supervisor.shutdown(); + }); + + test("a policy recycle during the final check puts the standby attempt off, and the new child takes mail", async () => { + const policy = manualRecyclePolicy(60_000); + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ recyclePolicy: policy.bindings }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + policy.advance(61_000); + policy.tick(); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await expect(inStandby).rejects.toBeInstanceOf(StandbyDeferredError); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + await takeTrigger(second, secondSender); + + await supervisor.shutdown(); + }); + + test("an operator recycle while the standby stop waits on its correlation query puts the attempt off", async () => { + const { supervisor, firstSender, first, tracker, ipcKeypair } = + await setup(); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + const recycled = supervisor.recycle({ reason: "operator" }); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await recycled; + await expect(inStandby).rejects.toBeInstanceOf(StandbyDeferredError); + + await supervisor.shutdown(); + }); + + test("a standby attempt waits while the child works, and mail still reaches it", async () => { + const { supervisor, firstSender, first, mailBus } = await setup(); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: false })).wait, + ).toBe(true); + // Not quiet yet: the child is asked again, and dispatch goes on. + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + await takeTrigger(first, firstSender); + expect(first.killSignals).toEqual([]); + + await actAsChildEnteringStandby(first, firstSender, { firstQuery: 2 }); + expect(await inStandby).toEqual({}); + + await supervisor.shutdown(); + }); + + test("a child working again at the final check lets held mail through and waits for quiet once more", async () => { + const held = Promise.withResolvers(); + const { supervisor, firstSender, first, mailBus } = await setup({ + onStandbyWait: (waiting) => { + if (waiting === "input") held.resolve(undefined); + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + // Mail during the final check is held back. + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + await held.promise; + expect(sentFrameTypes(first)).not.toContain("trigger.fire"); + + expect( + (await answerQuiet(first, firstSender, 2, { quiet: false })).wait, + ).toBe(false); + await takeTrigger(first, firstSender); + expect(first.killSignals).toEqual([]); + + await actAsChildEnteringStandby(first, firstSender, { firstQuery: 3 }); + expect(await inStandby).toEqual({}); + + await supervisor.shutdown(); + }); + + test("a write that lands after the child reports quiet sends the standby attempt back to waiting", async () => { + // The first write the child asks for waits until the test lets it land. + const firstWrite = Promise.withResolvers(); + const registered = Promise.withResolvers(); + let writes = 0; + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + repoStore: (baseDir) => { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + writes += 1; + if (writes === 1) await firstWrite.promise; + return target.writeTreePreservingPrefix(...args); + }; + }, + }); + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + // A write still landing when the child answers holds the final check. + await sendChildWrite(firstSender, "landing"); + await answerQuiet(first, firstSender, 2, { quiet: true }); + // A write after the answer is held by the fence and found by the check, + // which waits for the landing write. The supervisor takes frames in order, + // so once it registers the park sent next, it has taken that write. + await sendChildWrite(firstSender, "after-quiet"); + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-marker", parkKind: "approval" }, + }); + await registered.promise; + firstWrite.resolve(undefined); + + expect( + (await answerQuiet(first, firstSender, 3, { quiet: true })).wait, + ).toBe(true); + expect(first.killSignals).toEqual([]); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + await supervisor.wake(); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + await supervisor.shutdown(); + }); + + test("writes a crashed child left behind the fence never land", async () => { + // The first write the child asks for waits until the test lets it land. + const firstWrite = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const messages: string[] = []; + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + repoStore: (baseDir) => { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + if (messages.length === 0) await firstWrite.promise; + messages.push(args[3].message); + return target.writeTreePreservingPrefix(...args); + }; + }, + }); + }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + await sendChildWrite(firstSender, "landing"); + await answerQuiet(first, firstSender, 2, { quiet: true }); + // Held by the fence; the park sent next shows the supervisor took it. + await sendChildWrite(firstSender, "after-quiet"); + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-marker", parkKind: "approval" }, + }); + await registered.promise; + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + // The supervisor watched this exit first, so it has seen it once this + // await returns, before the landing write lets the standby attempt go on. + await first.exited; + firstWrite.resolve(undefined); + + await expect(inStandby).rejects.toBeInstanceOf(StandbyDeferredError); + expect(messages).toEqual(["child write landing"]); + await supervisor.shutdown(); + }); + + test("a child that writes after the stop began is woken for that work once it stops", async () => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { onWake: ({ reason }) => woke.resolve(reason) }, + ); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + await sendChildWrite(firstSender, "after-stop"); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await inStandby).toEqual({}); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("work it began after it went quiet"); + + await supervisor.shutdown(); + }); + + test("a standby attempt waits for a mail the child has not taken yet", async () => { + const waits = Promise.withResolvers(); + const { supervisor, firstSender, first, mailBus } = await setup({ + onStandbyWait: (waiting) => { + if (waiting === "standby") waits.resolve(undefined); + }, + }); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + const fire = await waitForUpstreamPayload( + first.supervisorToChild, + "trigger.fire", + ); + const requestId = fire.data.requestId; + if (requestId === undefined) + throw new Error("the trigger has no requestId"); + + const inStandby = supervisor.enterStandby(); + await waits.promise; + expect(sentFrameTypes(first)).not.toContain("quiet.request"); + await firstSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + await supervisor.shutdown(); + }); + + test("a run in standby keeps the mail it took", async () => { + const inbox = createMemoryInboxPrimitives(); + const { supervisor, firstSender, first, mailBus } = await setup({ + inboxPrimitives: inbox, + }); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + const fire = await waitForUpstreamPayload( + first.supervisorToChild, + "trigger.fire", + ); + const requestId = fire.data.requestId; + if (requestId === undefined) + throw new Error("the trigger has no requestId"); + await firstSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); + + // The dispatch for a mail waits until its run parks or ends; it does not + // hold up a standby attempt, since the run keeps that mail across a + // respawn. + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: [fire.data.messageId], + consumed: [], + }); + + await supervisor.shutdown(); + }); + + test("a redelivery of mail a run in standby took does not wake the child", async () => { + const inbox = createMemoryInboxPrimitives(); + const wakes: boolean[] = []; + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + inboxPrimitives: { + ...inbox, + // As the durable inbox does, mail a run took is not enqueued again. + enqueueInbox: async (store, principal, repoId, args) => + inbox.messageIds(args.address).processing.includes(args.messageId) + ? { outcome: "already-present", reason: "processing" } + : inbox.enqueueInbox(store, principal, repoId, args), + }, + onWake: ({ requested }) => wakes.push(requested), + }); + const mail = new TextEncoder().encode("hello"); + mailBus.deliver(ADDRESS, mail); + const fire = await waitForUpstreamPayload( + first.supervisorToChild, + "trigger.fire", + ); + const requestId = fire.data.requestId; + if (requestId === undefined) + throw new Error("the trigger has no requestId"); + await firstSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + await mailBus.deliverSettled(ADDRESS, mail); + // A wake the redelivery started would be the one this joins. + const woken = supervisor.wake(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + expect(wakes).toEqual([true]); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused while a mail the child declined waits for it", async () => { + const { supervisor, firstSender, first, tracker, mailBus } = await setup(); + mailBus.deliver(ADDRESS, new TextEncoder().encode("hello")); + const fire = await waitForUpstreamPayload( + first.supervisorToChild, + "trigger.fire", + ); + const requestId = fire.data.requestId; + if (requestId === undefined) + throw new Error("the trigger has no requestId"); + await firstSender.send({ + type: "input.taken", + data: { requestId, committed: false }, + }); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "mail the dispatch took is still waiting for the child", + ); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + // The dispatch's holds on mail for a live run, in order. + function dispatchWaits() { + const held: PromiseWithResolvers[] = []; + const at = (n: number) => { + while (held.length < n) held.push(Promise.withResolvers()); + const entry = held[n - 1]; + if (entry === undefined) throw new Error("no dispatch hold"); + return entry; + }; + let count = 0; + return { + onDispatchWait: () => { + count += 1; + at(count).resolve(undefined); + }, + nth: (n: number) => at(n).promise, + }; + } + + // A run that is live on the first child. + async function setupLiveRun( + opts: { + inboxPrimitives?: InboxPrimitives; + } = {}, + ) { + const waits = dispatchWaits(); + const harness = await setup({ + ...opts, + beforeSpawn: (baseDir) => writeWorkingRunLog(baseDir, [runStarted()]), + onDispatchWait: waits.onDispatchWait, + }); + return { ...harness, waits }; + } + + // The child's answer to the `count`th `signal.deliver` it was sent. + async function confirmDelivery( + child: FakeChild, + childSender: ReturnType, + count: number, + committed: boolean, + ): Promise { + const deliveries = await waitForUpstreamPayloads( + child.supervisorToChild, + "signal.deliver", + count, + ); + const requestId = deliveries[count - 1]?.data.requestId; + if (requestId === undefined) { + throw new Error("the delivery asked for no confirmation"); + } + await childSender.send({ + type: "input.taken", + data: { requestId, committed }, + }); + } + + test("mail that wakes the child reaches a run the new child resumed parked on input", async () => { + const { + supervisor, + firstSender, + first, + tracker, + mailBus, + ipcKeypair, + waits, + } = await setupLiveRun(); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + mailBus.deliver(ADDRESS, new TextEncoder().encode("next turn")); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + // The dispatch holds the mail before the new child says what its resumed + // run is parked on, and a park the run already holds sends no notice. + await waits.nth(1); + await answerParkedQuery(second, secondSender, 1, [ + { runId: RUN, correlationId: "corr-next-turn", parkKind: "input" }, + ]); + const delivered = await waitForUpstreamPayload( + second.supervisorToChild, + "signal.deliver", + ); + expect(delivered.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-next-turn"), + }); + + await supervisor.shutdown(); + }); + + test("a later re-registration delivers mail waiting for the run's input", async () => { + const { supervisor, firstSender, first, mailBus, waits } = + await setupLiveRun(); + // The child's first answer shows no park, so the mail waits. + await answerParkedQuery(first, firstSender, 1); + mailBus.deliver(ADDRESS, new TextEncoder().encode("next turn")); + await waits.nth(1); + + const reEmitted = supervisor.reEmitParkedCorrelations(); + await answerParkedQuery(first, firstSender, 2, [ + { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + ]); + await reEmitted; + const delivered = await waitForUpstreamPayload( + first.supervisorToChild, + "signal.deliver", + ); + expect(delivered.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-next"), + }); + + await supervisor.shutdown(); + }); + + // History the child answers from can lag an input it took, so a + // re-registration can still show it, before or after the child confirms it. + const takenOrders: ["confirmation" | "re-registration"][] = [ + ["confirmation"], + ["re-registration"], + ]; + test.each(takenOrders)( + "an input the child took is not registered again when its %s comes first", + async (comesFirst) => { + const { supervisor, firstSender, first, mailBus, waits } = + await setupLiveRun(); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-taken", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("turn one")); + const reRegister = async () => { + const reEmitted = supervisor.reEmitParkedCorrelations(); + await answerParkedQuery(first, firstSender, 2, [ + { runId: RUN, correlationId: "corr-taken", parkKind: "input" }, + ]); + await reEmitted; + }; + if (comesFirst === "confirmation") { + await confirmDelivery(first, firstSender, 1, true); + await reRegister(); + } else { + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + await reRegister(); + await confirmDelivery(first, firstSender, 1, true); + } + + // The run then waits on an approval, which shows no input channel, so + // the next mail waits for the run's next input. + mailBus.deliver(ADDRESS, new TextEncoder().encode("turn two")); + await firstSender.send({ type: "park.notify", data: GATE_PARK }); + await waits.nth(1); + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-next", parkKind: "input" }, + }); + const deliveries = await waitForUpstreamPayloads( + first.supervisorToChild, + "signal.deliver", + 2, + ); + expect(deliveries[1]?.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-next"), + }); + + await supervisor.shutdown(); + }, + ); + + test("an input the child declined keeps its mail and is registered again for the next mail", async () => { + const inbox = createMemoryInboxPrimitives(); + const { supervisor, firstSender, first, mailBus, waits } = + await setupLiveRun({ inboxPrimitives: inbox }); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-declined", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("turn one")); + await confirmDelivery(first, firstSender, 1, false); + // The decline fails turn one's dispatch and leaves its mail for a respawn. + // Another step of the run parks on an approval, and the next mail waits + // for the run's input. + mailBus.deliver(ADDRESS, new TextEncoder().encode("turn two")); + await firstSender.send({ type: "park.notify", data: GATE_PARK }); + await waits.nth(1); + expect(inbox.messageIds(ADDRESS).consumed).toEqual([]); + expect(inbox.messageIds(ADDRESS).processing).toHaveLength(2); + + const reEmitted = supervisor.reEmitParkedCorrelations(); + await answerParkedQuery(first, firstSender, 2, [ + { runId: RUN, correlationId: "corr-declined", parkKind: "input" }, + ]); + await reEmitted; + const deliveries = await waitForUpstreamPayloads( + first.supervisorToChild, + "signal.deliver", + 2, + ); + expect(deliveries[1]?.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-declined"), + }); + + await supervisor.shutdown(); + }); + + test("an input whose child crashed before committing it goes to the next child", async () => { + const { + supervisor, + firstSender, + first, + tracker, + mailBus, + ipcKeypair, + waits, + } = await setupLiveRun(); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-lost", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("turn one")); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + + // The replay puts the mail back, and the new child's run still waits on + // the input the first child never committed. + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await waits.nth(1); + await answerParkedQuery(second, secondSender, 1, [ + { runId: RUN, correlationId: "corr-lost", parkKind: "input" }, + ]); + const delivered = await waitForUpstreamPayload( + second.supervisorToChild, + "signal.deliver", + ); + expect(delivered.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-lost"), + }); + + await supervisor.shutdown(); + }); + + test("mail the child refused because its run ended is rejected", async () => { + const inbox = createMemoryInboxPrimitives(); + const consumed: unknown[] = []; + const { supervisor, firstSender, first, mailBus } = await setupLiveRun({ + inboxPrimitives: { + ...inbox, + async markConsumed(...args) { + consumed.push(args[3]); + return inbox.markConsumed(...args); + }, + }, + }); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-ended", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("late turn")); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + await firstSender.send({ + type: "terminal.event", + data: { runId: RUN, kind: "RunCancelled", seq: 9, at: "test" }, + }); + await confirmDelivery(first, firstSender, 1, false); + await waitUntil(() => inbox.messageIds(ADDRESS).consumed.length === 1); + expect(consumed[0]).toMatchObject({ + rejection: { code: "workflow_run_terminal" }, + }); + + await supervisor.shutdown(); + }); + + test("mail declined before the terminal frame of a run that ended is rejected", async () => { + const inbox = createMemoryInboxPrimitives(); + const consumed: unknown[] = []; + let logDir = ""; + const { supervisor, firstSender, first, mailBus } = await setup({ + beforeSpawn: async (baseDir) => { + logDir = baseDir; + await writeWorkingRunLog(baseDir, [runStarted()]); + }, + inboxPrimitives: { + ...inbox, + async markConsumed(...args) { + consumed.push(args[3]); + return inbox.markConsumed(...args); + }, + }, + }); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-ended", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("late turn")); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + // The run's end is committed, and its terminal frame has not gone out. + await writeWorkingRunLog(logDir, [ + runStarted(), + { type: "RunCancelled", reason: "test" }, + ]); + await confirmDelivery(first, firstSender, 1, false); + await waitUntil(() => inbox.messageIds(ADDRESS).consumed.length === 1); + expect(consumed[0]).toMatchObject({ + rejection: { code: "workflow_run_terminal" }, + }); + + await supervisor.shutdown(); + }); + + test("a stop whose shutdown frame cannot be written still kills the child", async () => { + const blocked = Promise.withResolvers(); + const { supervisor, firstSender, first } = await setup({ + timers: { + setTimer: (cb, ms) => + setTimeout(cb, ms === DEFAULT_KILL_TIMEOUT_MS ? 1 : ms), + clearTimer: defaultClearTimer, + }, + holdControlWrite: (line) => + parseControlFrames([line])[0]?.type === "shutdown" + ? blocked.promise + : undefined, + }); + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await inStandby; + expect(first.killSignals).toEqual(["SIGTERM"]); + + blocked.resolve(undefined); + await supervisor.shutdown(); + }); + + test("abandoning a standby attempt ends a quiet check whose send is blocked", async () => { + const blocked = Promise.withResolvers(); + const sending = Promise.withResolvers(); + const { supervisor } = await setup({ + holdControlWrite: (line) => { + if (parseControlFrames([line])[0]?.type !== "quiet.request") + return undefined; + sending.resolve(undefined); + return blocked.promise; + }, + }); + const inStandby = supervisor.enterStandby(); + await sending.promise; + supervisor.abandonStandby("the Hub that asked for it disconnected"); + await expect(inStandby).rejects.toThrow( + "the Hub that asked for it disconnected", + ); + + blocked.resolve(undefined); + await supervisor.shutdown(); + }); + + test("a parked query whose send is blocked still gives up at its watchdog", async () => { + const blocked = Promise.withResolvers(); + let hold = false; + const { supervisor } = await setup({ + parkedQueryWatchdogMs: 10, + holdControlWrite: (line) => + hold && + parseControlFrames([line])[0]?.type === "parked-correlations.request" + ? blocked.promise + : undefined, + }); + hold = true; + await supervisor.reEmitParkedCorrelations(); + + blocked.resolve(undefined); + await supervisor.shutdown(); + }); + + test("a standby attempt is refused while a dispatch holds mail for the run's next input", async () => { + const inbox = createMemoryInboxPrimitives(); + const dequeued = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, mailBus } = await setup({ + // The run is live, but the supervisor knows no input channel for it. + beforeSpawn: (baseDir) => writeWorkingRunLog(baseDir, [runStarted()]), + inboxPrimitives: { + ...inbox, + async dequeueToProcessing(...args) { + const entry = await inbox.dequeueToProcessing(...args); + if (entry !== null) dequeued.resolve(undefined); + return entry; + }, + }, + }); + mailBus.deliver(ADDRESS, new TextEncoder().encode("follow-up")); + await dequeued.promise; + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow( + "mail the dispatch took is still waiting for the child", + ); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + // The child was not handed the mail yet. + expect( + sentFrameTypes(first).filter( + (frameType) => + frameType === "trigger.fire" || frameType === "signal.deliver", + ), + ).toEqual([]); + + await supervisor.shutdown(); + }); + + // A write that never lands, and the final-check deadlines for the test to fire. + async function setupStuckWrite() { + const stuckWrite = Promise.withResolvers(); + const watchdogs: (() => void)[] = []; + const waits: string[] = []; + const waited = createChangeNotifier(); + const registered = Promise.withResolvers(); + let writes = 0; + const running = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + onStandbyWait: (waiting) => { + waits.push(waiting); + waited.notify(); + }, + timers: { + setTimer: (callback: () => void, ms: number): unknown => { + if (ms === 30_000) { + watchdogs.push(callback); + return null; + } + const handle = setTimeout(callback, ms); + return { clear: () => clearTimeout(handle) }; + }, + clearTimer: (handle: unknown): void => { + if ( + typeof handle === "object" && + handle !== null && + "clear" in handle && + typeof handle.clear === "function" + ) + handle.clear(); + }, + }, + parkedQueryWatchdogMs: 45_000, + repoStore: (baseDir) => { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + writes += 1; + if (writes === 1) await stuckWrite.promise; + return target.writeTreePreservingPrefix(...args); + }; + }, + }); + }, + }); + return { ...running, stuckWrite, watchdogs, waits, waited, registered }; + } + + const heldSignal = { + runId: RUN, + signalName: "gate", + signalId: "sig-held", + payload: { approved: true }, + }; + + test("a write stuck before the final check lets a held signal through once the deadline passes", async () => { + const { + supervisor, + firstSender, + first, + stuckWrite, + watchdogs, + waits, + waited, + } = await setupStuckWrite(); + + const inStandby = supervisor.enterStandby(); + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + await sendChildWrite(firstSender, "stuck"); + const before = waits.length; + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waited.until(() => waits.slice(before).includes("standby")); + const delivered = supervisor.deliverSignal(heldSignal); + await waited.until(() => waits.includes("input")); + expect(sentFrameTypes(first)).not.toContain("signal.deliver"); + + watchdogs.at(-1)?.(); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + stuckWrite.resolve(undefined); + await delivered; + await supervisor.wake(); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + await supervisor.shutdown(); + }); + + test("a write stuck under the final check's answer lets a held signal through once the deadline passes", async () => { + const { + supervisor, + firstSender, + first, + stuckWrite, + watchdogs, + waits, + waited, + registered, + } = await setupStuckWrite(); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + await sendChildWrite(firstSender, "stuck"); + await answerQuiet(first, firstSender, 2, { quiet: true }); + // The supervisor takes frames in order, so once it registers the park sent + // next, it has the answer and waits on the write. + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-marker", parkKind: "approval" }, + }); + await registered.promise; + const delivered = supervisor.deliverSignal(heldSignal); + await waited.until(() => waits.includes("input")); + expect(sentFrameTypes(first)).not.toContain("signal.deliver"); + + watchdogs.at(-1)?.(); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + stuckWrite.resolve(undefined); + await delivered; + await supervisor.wake(); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + await supervisor.shutdown(); + }); + + test("a final check the child does not answer in time counts it as working", async () => { + // Quiet-check watchdogs are the test's to fire; every other timer runs. + const watchdogs: (() => void)[] = []; + const timers = { + setTimer: (callback: () => void, ms: number): unknown => { + if (ms === 30_000) { + watchdogs.push(callback); + return null; + } + const handle = setTimeout(callback, ms); + return { clear: () => clearTimeout(handle) }; + }, + clearTimer: (handle: unknown): void => { + if ( + typeof handle === "object" && + handle !== null && + "clear" in handle && + typeof handle.clear === "function" + ) + handle.clear(); + }, + }; + const { supervisor, firstSender, first } = await setup({ + timers, + parkedQueryWatchdogMs: 45_000, + }); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + const queries = await waitForUpstreamPayloads( + first.supervisorToChild, + "quiet.request", + 2, + ); + expect(queries[1]?.data.wait).toBe(false); + const watchdog = watchdogs.at(-1); + if (watchdog === undefined) throw new Error("no quiet-check watchdog"); + watchdog(); + // Back to waiting until the child says it is quiet. + const again = await waitForUpstreamPayloads( + first.supervisorToChild, + "quiet.request", + 3, + ); + expect(again[2]?.data.wait).toBe(true); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + const woken = supervisor.wake(); + await expect(inStandby).rejects.toBeInstanceOf(StandbyAbandonedError); + await woken; + await supervisor.shutdown(); + }); + + test("reports the earliest queued timer, and wakes for it", async () => { + const hour = 60 * 60 * 1000; + const now = Date.parse("2026-10-02T00:00:00.000Z"); + const fireAt = new Date(now + hour).toISOString(); + // The wake timer is the test's to fire; every other timer runs as usual. + let wakeTimer: (() => void) | undefined; + const timers = { + now: () => now, + setTimer: (callback: () => void, ms: number): unknown => { + if (ms === hour) { + wakeTimer = callback; + return null; + } + const handle = setTimeout(callback, ms); + return { clear: () => clearTimeout(handle) }; + }, + clearTimer: (handle: unknown): void => { + if ( + typeof handle === "object" && + handle !== null && + "clear" in handle && + typeof handle.clear === "function" + ) + handle.clear(); + }, + }; + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { onWake: ({ reason }) => woke.resolve(reason), timers }, + ); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender, { wakeAt: fireAt }); + expect(await inStandby).toEqual({ wakeAt: fireAt }); + expect(tracker.totalSpawns).toBe(1); + + if (wakeTimer === undefined) throw new Error("no wake timer was armed"); + wakeTimer(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + expect(await woke.promise).toBe("timer"); + + await supervisor.shutdown(); + }); + + test("a child that runs again clears the timer that would have woken it", async () => { + const hour = 3_600_000; + const now = Date.parse("2026-10-02T00:00:00.000Z"); + const wakeHandle = { wake: true }; + const cleared: unknown[] = []; + const timers = { + now: () => now, + setTimer: (callback: () => void, ms: number): unknown => { + if (ms === hour) return wakeHandle; + const handle = setTimeout(callback, ms); + return { clear: () => clearTimeout(handle) }; + }, + clearTimer: (handle: unknown): void => { + cleared.push(handle); + if ( + typeof handle === "object" && + handle !== null && + "clear" in handle && + typeof handle.clear === "function" + ) + handle.clear(); + }, + }; + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { timers }, + ); + + const inStandby = supervisor.enterStandby(); + const wakeAt = new Date(now + hour).toISOString(); + await actAsChildEnteringStandby(first, firstSender, { wakeAt }); + await inStandby; + expect(cleared).not.toContain(wakeHandle); + + const woken = supervisor.wake(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + expect(cleared).toContain(wakeHandle); + + await supervisor.shutdown(); + }); + + test("a wake further off than a timer can wait is armed again when that timer fires, and wakes the child once due", async () => { + // `setTimeout` fires at once past this many ms. + const maxTimerMs = 2_147_483_647; + let now = Date.now(); + const fireAt = new Date(now + 40 * 24 * 60 * 60 * 1000).toISOString(); + const wakeTimers: { callback: () => void; ms: number }[] = []; + const timers = { + now: () => now, + setTimer: (callback: () => void, ms: number): unknown => { + if (ms > 24 * 60 * 60 * 1000) { + wakeTimers.push({ callback, ms }); + return null; + } + const handle = setTimeout(callback, ms); + return { clear: () => clearTimeout(handle) }; + }, + clearTimer: (handle: unknown): void => { + if ( + typeof handle === "object" && + handle !== null && + "clear" in handle && + typeof handle.clear === "function" + ) + handle.clear(); + }, + }; + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { timers, onWake: ({ reason }) => woke.resolve(reason) }, + ); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender, { wakeAt: fireAt }); + expect(await inStandby).toEqual({ wakeAt: fireAt }); + expect(wakeTimers.map((timer) => timer.ms)).toEqual([maxTimerMs]); + + now += maxTimerMs; + wakeTimers[0]?.callback(); + const remainingMs = Date.parse(fireAt) - now; + expect(wakeTimers.map((timer) => timer.ms)).toEqual([ + maxTimerMs, + remainingMs, + ]); + expect(tracker.totalSpawns).toBe(1); + + now += remainingMs; + wakeTimers[1]?.callback(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + expect(await woke.promise).toBe("timer"); + + await supervisor.shutdown(); + }); + + test("a signal respawns a child in standby and is delivered to the new one", async () => { + const { supervisor, firstSender, first, tracker, ipcKeypair } = + await setup(); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await inStandby; + + const delivered = supervisor.deliverSignal({ + runId: RUN, + signalName: "gate", + signalId: "sig-1", + payload: { approved: true }, + }); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await delivered; + const signal = await waitForUpstreamPayload( + second.supervisorToChild, + "signal.deliver", + ); + expect(signal.data.signalId).toBe("sig-1"); + expect(parseSignalDeliverRunIds(first.supervisorToChild.flushed())).toEqual( + [], + ); + + await supervisor.shutdown(); + }); + + test("a signal that arrives while another signal's wake respawns the child reaches the new one", async () => { + const { supervisor, firstSender, first, tracker, ipcKeypair } = + await setup(); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await inStandby; + + const signal = (signalId: string) => + supervisor.deliverSignal({ + runId: RUN, + signalName: "gate", + signalId, + payload: { approved: true }, + }); + const waking = signal("sig-1"); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const during = signal("sig-2"); + await driveReady(second, ipcKeypair); + await Promise.all([waking, during]); + const signals = await waitForUpstreamPayloads( + second.supervisorToChild, + "signal.deliver", + 2, + ); + expect(signals.map((frame) => frame.data.signalId).sort()).toEqual([ + "sig-1", + "sig-2", + ]); + + await supervisor.shutdown(); + }); + + test("a cancellation asked for while a wake respawns the child goes to the new one", async () => { + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { repoStore: (baseDir) => mergingRepoStore(baseDir) }, + ); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await inStandby; + + const woken = supervisor.wake(); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const cancelled = supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + const prepare = await waitForUpstreamPayload( + second.supervisorToChild, + "cancel.prepare", + ); + await secondSender.send({ + type: "cancel.prepared", + data: { requestId: prepare.data.requestId }, + }); + await cancelled; + expect(sentFrameTypes(first)).not.toContain("cancel.prepare"); + + await supervisor.shutdown(); + }); + + test("a signal during the final check waits for it and reaches the respawned child", async () => { + const { supervisor, firstSender, first, tracker, ipcKeypair } = + await setup(); + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + + const delivered = supervisor.deliverSignal({ + runId: RUN, + signalName: "gate", + signalId: "sig-1", + payload: { approved: true }, + }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await inStandby).toEqual({}); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + await driveReady(second, ipcKeypair); + await delivered; + expect(parseSignalDeliverRunIds(first.supervisorToChild.flushed())).toEqual( + [], + ); + const signal = await waitForUpstreamPayload( + second.supervisorToChild, + "signal.deliver", + ); + expect(signal.data.signalId).toBe("sig-1"); + + await supervisor.shutdown(); + }); + + test("a hub reconnect while the child stops re-registers the correlations it reported", async () => { + const registered: string[] = []; + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: (registration) => { + registered.push(registration.correlationId); + }, + }); + + const entering = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + const queries = await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + const query = queries[1]; + if (query === undefined) throw new Error("correlation query missing"); + await firstSender.send({ + type: "parked-correlations.response", + data: { + requestId: query.data.requestId, + parked: [ + { + runId: RUN, + correlationId: "corr-gate", + parkKind: "approval", + snapshot: { + name: "charge_card", + description: "Charge the customer's card", + inputSchema: { type: "object" }, + arguments: { amount: 100 }, + }, + }, + ], + }, + }); + // The child has been told to stop and has not exited yet. + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + await supervisor.reEmitParkedCorrelations(); + expect(registered).toEqual(["corr-gate"]); + + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await entering).toEqual({}); + await supervisor.shutdown(); + }); + + test("a standby is recorded once the child stopped, and only once", async () => { + let childExited = false; + const records: boolean[] = []; + const { supervisor, firstSender, first } = await setup({ + recordStandby: async () => { + records.push(childExited); + }, + }); + void first.exited.then(() => { + childExited = true; + }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + expect(await supervisor.enterStandby()).toEqual({}); + expect(records).toEqual([true]); + + await supervisor.shutdown(); + }); + + // A run sleeping on a timer sends no park, so a drain arms a timer for it. + // The supervisor takes frames in order, so once it registers the park sent + // next, the run is one it drains. + async function drainSleepingRun( + supervisor: WorkflowSupervisor, + firstSender: ReturnType, + registered: Promise, + ): Promise { + await firstSender.send({ type: "resumed.runs", data: { runIds: [RUN] } }); + await firstSender.send({ + type: "park.notify", + data: { + runId: "run-marker", + correlationId: "corr-marker", + parkKind: "approval", + }, + }); + await registered; + await supervisor.drain({ deadlineMs: 60_000 }); + } + + // A drain timer whose deadline the test passes: it commits its cancellation + // through the commit the supervisor gave it, as the real one does. + function passableDrainDeadline() { + let commitCancel: + | ((cancel: { + origin: "supervisor-drain"; + reason: string; + at: string; + }) => Promise) + | undefined; + const factory: WorkflowSupervisorBindings["drainTimeoutAccumulatorFactory"] = + (opts) => { + commitCancel = opts.commitCancel; + return { + start: () => undefined, + pause: () => undefined, + resume: () => undefined, + stop: () => undefined, + accumulatedMs: () => 0, + escalated: false, + disposed: () => Promise.resolve(), + }; + }; + return { + factory, + pass(): Promise { + if (commitCancel === undefined) throw new Error("no drain was armed"); + return commitCancel({ + origin: "supervisor-drain", + reason: "drainTimeout expired", + at: "2026-10-01T00:00:00.000Z", + }); + }, + }; + } + + test("a standby stop stops the drain timers its child's runs armed", async () => { + const registered = Promise.withResolvers(); + const stopped: string[] = []; + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + drainTimeoutAccumulatorFactory: ({ runId }) => ({ + start: () => undefined, + pause: () => undefined, + resume: () => undefined, + stop: () => { + stopped.push(runId); + }, + accumulatedMs: () => 0, + escalated: false, + disposed: () => Promise.resolve(), + }), + }); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + expect(stopped).toEqual([RUN]); + + await supervisor.shutdown(); + }); + + test("a drain sent while a standby stop takes the child arms no drain timer", async () => { + const registered = Promise.withResolvers(); + const started: string[] = []; + const stoppedOne = Promise.withResolvers(); + const drainHeld = Promise.withResolvers(); + const releaseDrain = Promise.withResolvers(); + const inbox = createMemoryInboxPrimitives(); + let drains = 0; + let entering = false; + let lateDrain: Promise | undefined; + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + holdControlWrite: (line) => { + if (parseControlFrames([line])[0]?.type !== "drain") return undefined; + drains += 1; + if (drains !== 2) return undefined; + drainHeld.resolve(undefined); + return releaseDrain.promise; + }, + // The standby attempt reads the inbox last, while the child still runs. + inboxPrimitives: { + ...inbox, + async hasInboxEntries(...args) { + if (entering) lateDrain ??= supervisor.drain({ deadlineMs: 60_000 }); + return inbox.hasInboxEntries(...args); + }, + }, + drainTimeoutAccumulatorFactory: ({ runId }) => ({ + start: () => { + started.push(runId); + }, + pause: () => undefined, + resume: () => undefined, + stop: () => { + stoppedOne.resolve(undefined); + }, + accumulatedMs: () => 0, + escalated: false, + disposed: () => Promise.resolve(), + }), + }); + await drainSleepingRun(supervisor, firstSender, registered.promise); + expect(started).toEqual([RUN]); + + entering = true; + const inStandby = supervisor.enterStandby(); + const stopping = actAsChildEnteringStandby(first, firstSender); + await drainHeld.promise; + await stoppedOne.promise; + releaseDrain.resolve(undefined); + await lateDrain; + await stopping; + expect(await inStandby).toEqual({}); + expect(started).toEqual([RUN]); + + await supervisor.shutdown(); + }); + + test("a standby attempt waits for a drain deadline's cancellation being committed", async () => { + const registered = Promise.withResolvers(); + const deadline = passableDrainDeadline(); + const committing = Promise.withResolvers(); + const release = Promise.withResolvers(); + const waiting = Promise.withResolvers(); + const { supervisor, firstSender, first } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + drainTimeoutAccumulatorFactory: deadline.factory, + onStandbyWait: (on) => { + if (on === "standby") waiting.resolve(undefined); + }, + repoStore: (baseDir) => + mergingRepoStore(baseDir, async () => { + committing.resolve(undefined); + await release.promise; + }), + }); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const cancelling = deadline.pass(); + await committing.promise; + const entering = supervisor.enterStandby(); + void entering.catch(() => undefined); + const asked = waitForUpstreamPayload( + first.supervisorToChild, + "quiet.request", + ); + expect( + await Promise.race([ + waiting.promise.then(() => "waited"), + asked.then(() => "asked"), + ]), + ).toBe("waited"); + + release.resolve(undefined); + await cancelling; + await asked; + + await supervisor.shutdown(); + }); + + test("a drain deadline that passes during the final check calls the standby attempt off, and the child keeps running", async () => { + const registered = Promise.withResolvers(); + const deadline = passableDrainDeadline(); + let writes = 0; + const { supervisor, firstSender, first, tracker } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + drainTimeoutAccumulatorFactory: deadline.factory, + repoStore: (baseDir) => + mergingRepoStore(baseDir, async () => { + writes += 1; + }), + }); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + await waitForUpstreamPayloads(first.supervisorToChild, "quiet.request", 2); + const committed = deadline.pass(); + await expect(inStandby).rejects.toThrow("the drain deadline passed"); + await committed; + expect(writes).toBe(1); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a drain deadline that passes once the standby stop is under way wakes the child for its cancellation", async () => { + const registered = Promise.withResolvers(); + const deadline = passableDrainDeadline(); + const reasons: string[] = []; + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { + onWake: ({ reason }) => { + reasons.push(reason); + woke.resolve(undefined); + }, + onSuspensionRegister: () => registered.resolve(undefined), + drainTimeoutAccumulatorFactory: deadline.factory, + repoStore: (baseDir) => mergingRepoStore(baseDir), + }, + ); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + // Past the point a stop can be called off. + const committed = deadline.pass(); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + await expect(inStandby).rejects.toThrow("the drain deadline passed"); + await committed; + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woke.promise; + expect(reasons).toEqual(["cancel"]); + + await supervisor.shutdown(); + }); + + test("a shutdown waits for a drain deadline's cancellation the standby stop left in flight", async () => { + const registered = Promise.withResolvers(); + const deadline = passableDrainDeadline(); + const shutdownWaits = Promise.withResolvers(); + let landed = false; + const { supervisor, firstSender, first, tracker } = await setup({ + onSuspensionRegister: () => registered.resolve(undefined), + onShutdownWait: (waiting) => { + if (waiting === "cancellation") shutdownWaits.resolve(undefined); + }, + drainTimeoutAccumulatorFactory: deadline.factory, + repoStore: (baseDir) => + mergingRepoStore(baseDir, async () => { + await shutdownWaits.promise; + landed = true; + }), + }); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + const committed = deadline.pass(); + + // The shutdown kills the child the stop waits on. + await supervisor.shutdown(); + expect(landed).toBe(true); + await expect(inStandby).rejects.toThrow(); + await committed; + expect(tracker.totalSpawns).toBe(1); + }); + + test("mail that lands while the standby is recorded wakes the child once the record lands", async () => { + const memory = createMemoryInboxPrimitives(); + const recording = Promise.withResolvers(); + const release = Promise.withResolvers(); + const wakeWaits = Promise.withResolvers(); + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + onWake: ({ reason }) => { + woke.resolve(reason); + }, + recordStandby: async () => { + recording.resolve(undefined); + await release.promise; + }, + onStandbyWait: (waiting) => { + if (waiting === "attempt") wakeWaits.resolve(undefined); + }, + inboxPrimitives: memory, + }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await recording.promise; + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + await wakeWaits.promise; + expect(tracker.totalSpawns).toBe(1); + release.resolve(undefined); + expect(await inStandby).toEqual({}); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("mail"); + + await supervisor.shutdown(); + }); + + test("records the parks, the timer and the history tip it read before the child reported its parks", async () => { + const records: RecordedStandby[] = []; + const parkQueriesAtTipRead: number[] = []; + // The correlation queries sent to the child so far, once it exists. + let parkQueries = (): number => -1; + const { supervisor, firstSender, first } = await setup({ + recordStandby: async (standby) => { + records.push(standby); + }, + repoStore: (baseDir) => { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "resolveRef") + return Reflect.get(target, prop, receiver); + return async () => { + parkQueriesAtTipRead.push(parkQueries()); + return HISTORY_TIP; + }; + }, + }); + }, + }); + parkQueries = () => + sentFrameTypes(first).filter( + (type) => type === "parked-correlations.request", + ).length; + const wakeAt = new Date(Date.now() + 3_600_000).toISOString(); + + const entering = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true, wakeAt }); + await answerQuiet(first, firstSender, 2, { quiet: true, wakeAt }); + const queries = await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + const query = queries[1]; + if (query === undefined) throw new Error("correlation query missing"); + await firstSender.send({ + type: "parked-correlations.response", + data: { requestId: query.data.requestId, parked: [GATE_PARK] }, + }); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await entering).toEqual({ wakeAt }); + + // Only the spawn's own correlation query had gone out. + expect(parkQueriesAtTipRead).toEqual([1]); + expect(records).toEqual([ + { + parked: [GATE_PARK], + wakeAt, + tip: HISTORY_TIP, + definition: TEST_DEFINITION, + }, + ]); + await supervisor.shutdown(); + }); + + test("a shutdown while the standby is recorded says the shutdown called it off", async () => { + const recording = Promise.withResolvers(); + const release = Promise.withResolvers(); + const { supervisor, firstSender, first } = await setup({ + recordStandby: async () => { + recording.resolve(undefined); + await release.promise; + }, + }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await recording.promise; + const shutdown = supervisor.shutdown(); + release.resolve(undefined); + const cause = await inStandby.then( + () => undefined, + (error: unknown) => error, + ); + expect(cause).toBeInstanceOf(StandbyAbandonedError); + expect(cause).toHaveProperty( + "message", + "the workflow supervisor shut down while entering standby", + ); + await shutdown; + }); + + test("a shutdown resolves once the standby it called off is recorded", async () => { + const recording = Promise.withResolvers(); + const release = Promise.withResolvers(); + const shutdownWaits = Promise.withResolvers(); + let recorded = false; + const { supervisor, firstSender, first } = await setup({ + recordStandby: async () => { + recording.resolve(undefined); + await release.promise; + recorded = true; + }, + onShutdownWait: (waiting) => { + if (waiting === "attempt") shutdownWaits.resolve(undefined); + }, + }); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await recording.promise; + const shutdown = supervisor.shutdown(); + const recordedWhenShutDown = shutdown.then(() => recorded); + await shutdownWaits.promise; + release.resolve(undefined); + expect(await recordedWhenShutDown).toBe(true); + await expect(inStandby).rejects.toBeInstanceOf(StandbyAbandonedError); + }); + + test("a standby attempt its host cannot record runs the child again", async () => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { + onWake: ({ reason }) => woke.resolve(reason), + recordStandby: () => Promise.reject(new Error("disk full")), + }, + ); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await expect(inStandby).rejects.toThrow( + "the standby could not be recorded: disk full", + ); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("a standby that was not recorded"); + + await supervisor.shutdown(); + }); + + test("a child that does not report its parked correlations is not in standby", async () => { + const { supervisor, firstSender, first } = await setup({ + parkedQueryWatchdogMs: 1, + }); + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await expect(inStandby).rejects.toThrow( + "the workflow child did not report the correlations it is parked on", + ); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + await supervisor.shutdown(); + }); + + test("a standby attempt abandoned while its child stops runs the child again", async () => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { onWake: ({ reason }) => woke.resolve(reason) }, + ); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + // Past the point a stop can be called off. + supervisor.abandonStandby("the Hub that asked for it disconnected"); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + await expect(inStandby).rejects.toThrow( + "the Hub that asked for it disconnected", + ); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + expect(await woke.promise).toBe("an abandoned standby attempt"); + + await supervisor.shutdown(); + }); + + // A store whose writes run their merge, as a cancellation's commit needs; + // `beforeWrite` runs ahead of each one. + function mergingRepoStore( + baseDir: string, + beforeWrite: () => Promise = async () => undefined, + ): RepoStore { + const base = createStubRepoStore(baseDir, { writeTree: true }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + await beforeWrite(); + await args[3].merge(new Map()); + return target.writeTreePreservingPrefix(...args); + }; + }, + }); + } + + test("a cancellation that abandons a standby attempt while its child stops runs the child itself", async () => { + const reasons: string[] = []; + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { + onWake: ({ reason }) => { + reasons.push(reason); + woke.resolve(undefined); + }, + repoStore: (baseDir) => mergingRepoStore(baseDir), + }, + ); + + const inStandby = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + // Past the point a stop can be called off. + const cancelled = supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + await expect(inStandby).rejects.toThrow("a cancellation was requested"); + await cancelled; + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woke.promise; + expect(reasons).toEqual(["cancel"]); + + await supervisor.shutdown(); + }); + + test("a standby attempt waiting on a cancellation commit goes on once it lands", async () => { + const committing = Promise.withResolvers(); + const release = Promise.withResolvers(); + const waiting = Promise.withResolvers(); + let hold = true; + const { supervisor, firstSender, first } = await setup({ + onStandbyWait: (on) => { + if (on === "standby") waiting.resolve(undefined); + }, + repoStore: (baseDir) => + mergingRepoStore(baseDir, async () => { + if (!hold) return; + hold = false; + committing.resolve(undefined); + await release.promise; + }), + }); + + const cancelled = supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }); + // The running child prepares the cancellation before it is committed. + const prepare = await waitForUpstreamPayload( + first.supervisorToChild, + "cancel.prepare", + ); + await firstSender.send({ + type: "cancel.prepared", + data: { requestId: prepare.data.requestId }, + }); + await committing.promise; + const entering = supervisor.enterStandby(); + void entering.catch(() => undefined); + await waiting.promise; + expect(sentFrameTypes(first)).not.toContain("quiet.request"); + + release.resolve(undefined); + await cancelled; + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + + await supervisor.shutdown(); + }); + + test("a standby attempt waiting on a cancellation the child has not prepared goes on once it lands", async () => { + const waiting = Promise.withResolvers(); + const { supervisor, firstSender, first } = await setup({ + onStandbyWait: (on) => { + if (on === "standby") waiting.resolve(undefined); + }, + repoStore: (baseDir) => mergingRepoStore(baseDir), + }); + + const cancelled = supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }); + const prepare = await waitForUpstreamPayload( + first.supervisorToChild, + "cancel.prepare", + ); + const entering = supervisor.enterStandby(); + void entering.catch(() => undefined); + await waiting.promise; + expect(sentFrameTypes(first)).not.toContain("quiet.request"); + + await firstSender.send({ + type: "cancel.prepared", + data: { requestId: prepare.data.requestId }, + }); + await cancelled; + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + + await supervisor.shutdown(); + }); + + test("a wake waits for a drain deadline's cancellation that waits on the standby attempt", async () => { + const baseDir = await makeTempDir("standby-drain-wake-"); + const tracker = createSpawnTracker({}); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const registered = Promise.withResolvers(); + const wakeWaitsOnCancel = Promise.withResolvers(); + const signing = Promise.withResolvers(); + const allowSign = Promise.withResolvers(); + const deadline = passableDrainDeadline(); + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker, + mailBus, + ipcKeypair, + overrides: { + repoStore: mergingRepoStore(baseDir), + onSuspensionRegister: () => registered.resolve(undefined), + onStandbyWait: (waiting) => { + if (waiting === "cancellation") wakeWaitsOnCancel.resolve(undefined); + }, + // Holds the drain deadline's cancellation before it commits. + signAsPrincipal: async () => { + signing.resolve(undefined); + await allowSign.promise; + return { sig: new Uint8Array(64), principalKind: "supervisor" }; + }, + drainTimeoutAccumulatorFactory: deadline.factory, + }, + }); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + await drainSleepingRun(supervisor, firstSender, registered.promise); + + const entering = supervisor.enterStandby().then( + () => null, + (cause: unknown) => cause, + ); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await answerParkedQuery(first, firstSender, 2); + await waitForUpstreamPayload(first.supervisorToChild, "shutdown"); + // The wake waits for the stop first, so the drain deadline's cancellation + // waits behind it. + const woken = supervisor.wake(); + const cancelling = deadline.pass(); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(0); + expect(await entering).toBeInstanceOf(StandbyAbandonedError); + await signing.promise; + expect( + await Promise.race([ + wakeWaitsOnCancel.promise.then(() => "waited"), + tracker.awaitChildren(2).then(() => "spawned"), + ]), + ).toBe("waited"); + + allowSign.resolve(undefined); + await cancelling; + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + + await supervisor.shutdown(); + }); + + test("a child in standby runs again only once a cancellation being committed lands", async () => { + const committing = Promise.withResolvers(); + const release = Promise.withResolvers(); + const waiting = Promise.withResolvers(); + let hold = false; + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { + onStandbyWait: (on) => { + if (on === "cancellation") waiting.resolve(undefined); + }, + repoStore: (baseDir) => + mergingRepoStore(baseDir, async () => { + if (!hold) return; + hold = false; + committing.resolve(undefined); + await release.promise; + }), + }, + ); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + await inStandby; + hold = true; + const cancelled = supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }); + await committing.promise; + const woken = supervisor.wake(); + await waiting.promise; + expect(tracker.totalSpawns).toBe(1); + + release.resolve(undefined); + await cancelled; + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + expect(tracker.totalSpawns).toBe(2); + + await supervisor.shutdown(); + }); + + test("a standby attempt asked for while an abandoned stop finishes is put off, not answered as done", async () => { + const inboxRead = Promise.withResolvers(); + const releaseRead = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { + inboxPrimitives: { + ...createMemoryInboxPrimitives(), + hasInboxEntries: emptyUntilStopped(async () => { + inboxRead.resolve(undefined); + await releaseRead.promise; + return false; + }), + }, + }, + ); + + const abandoned = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + // The child is stopped and recorded; the stop still reads the inbox. + await inboxRead.promise; + supervisor.abandonStandby("the Hub that asked for it disconnected"); + await expect(supervisor.enterStandby()).rejects.toBeInstanceOf( + StandbyDeferredError, + ); + releaseRead.resolve(undefined); + await expect(abandoned).rejects.toThrow( + "the Hub that asked for it disconnected", + ); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + + await supervisor.shutdown(); + }); + + test("a standby attempt asked for while an abandoned one settles is put off, not refused", async () => { + const { supervisor, firstSender, first, tracker } = await setup(); + + const abandoned = supervisor.enterStandby(); + expect( + (await answerQuiet(first, firstSender, 1, { quiet: true })).wait, + ).toBe(true); + expect( + (await answerQuiet(first, firstSender, 2, { quiet: true })).wait, + ).toBe(false); + await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + supervisor.abandonStandby("the Hub that asked for it disconnected"); + await expect(supervisor.enterStandby()).rejects.toBeInstanceOf( + StandbyDeferredError, + ); + + await answerParkedQuery(first, firstSender, 2); + await expect(abandoned).rejects.toThrow( + "the Hub that asked for it disconnected", + ); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt asked for while the child wakes is put off, not refused", async () => { + const woke = Promise.withResolvers(); + const { supervisor, firstSender, first, tracker, ipcKeypair } = await setup( + { onWake: ({ reason }) => woke.resolve(reason) }, + ); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + + const woken = supervisor.wake(); + await expect(supervisor.enterStandby()).rejects.toBeInstanceOf( + StandbyDeferredError, + ); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + expect(await woke.promise).toBe("wake"); + + await supervisor.shutdown(); + }); + + // A child that crashed once, so the backoff is raised, with the replacement + // running. Only that crash's backoff runs out, and the stable-run reset is the + // test's to fire. + async function setupRaisedBackoff() { + const backoffMs = 1_234; + const stableResetMs = 65_432; + const backoffs: number[] = []; + const resets: (() => void)[] = []; + const armed = createChangeNotifier(); + const running = await setup({ + respawnBackoffInitialMs: backoffMs, + recyclePolicy: { crashLoopStableResetMs: stableResetMs }, + timers: { + setTimer: (callback, ms) => { + if (ms === stableResetMs) { + resets.push(callback); + armed.notify(); + return callback; + } + if (ms === backoffMs || ms === backoffMs * 2) { + backoffs.push(ms); + armed.notify(); + return backoffs.length === 1 ? setTimeout(callback, 0) : callback; + } + return setTimeout(callback, ms); + }, + }, + }); + const { first, tracker, ipcKeypair } = running; + const crash = (child: FakeChild) => { + child.eventChildToSupervisor.close(); + child.childToSupervisor.close(); + child.resolveExit?.(1); + }; + crash(first); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await armed.until(() => resets.length === 1); + const wakeThirdChild = async () => { + const woken = running.supervisor.wake(); + await tracker.awaitChildren(3); + const third = tracker.children[2]; + if (third === undefined) throw new Error("third child missing"); + const thirdSender = await driveReady(third, ipcKeypair); + await answerParkedQuery(third, thirdSender, 1); + await woken; + return third; + }; + const backoffAfterCrash = async (child: FakeChild) => { + crash(child); + await armed.until(() => backoffs.length === 2); + return backoffs[1]; + }; + return { + ...running, + second, + secondSender, + backoffMs, + resets, + wakeThirdChild, + backoffAfterCrash, + }; + } + + test("a raised respawn backoff resets when its time passes with the child in standby", async () => { + const { + supervisor, + second, + secondSender, + backoffMs, + resets, + wakeThirdChild, + backoffAfterCrash, + } = await setupRaisedBackoff(); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(second, secondSender); + await inStandby; + resets.at(-1)?.(); + + const third = await wakeThirdChild(); + expect(await backoffAfterCrash(third)).toBe(backoffMs); + + await supervisor.shutdown(); + }); + + test("a raised respawn backoff resets once a child woken from standby lasts", async () => { + const { + supervisor, + second, + secondSender, + backoffMs, + resets, + wakeThirdChild, + backoffAfterCrash, + } = await setupRaisedBackoff(); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(second, secondSender); + await inStandby; + + const third = await wakeThirdChild(); + expect(resets).toHaveLength(2); + resets.at(-1)?.(); + expect(await backoffAfterCrash(third)).toBe(backoffMs); + + await supervisor.shutdown(); + }); + + test("a child woken from standby before its reset comes does not reset the backoff if it crashes first", async () => { + const { + supervisor, + second, + secondSender, + backoffMs, + wakeThirdChild, + backoffAfterCrash, + } = await setupRaisedBackoff(); + + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(second, secondSender); + await inStandby; + + const third = await wakeThirdChild(); + expect(await backoffAfterCrash(third)).toBe(backoffMs * 2); + + await supervisor.shutdown(); + }); + + test("a standby request while a crashed child waits out its respawn backoff is put off", async () => { + // A backoff no other timer here shares, which is never let fire. + const backoffMs = 54_321; + const backoffArmed = Promise.withResolvers(); + const { supervisor, first } = await setup({ + respawnBackoffInitialMs: backoffMs, + timers: { + setTimer: (callback, ms) => { + if (ms !== backoffMs) return setTimeout(callback, ms); + backoffArmed.resolve(undefined); + return callback; + }, + }, + }); + + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + await backoffArmed.promise; + await expect(supervisor.enterStandby()).rejects.toBeInstanceOf( + StandbyDeferredError, + ); + expect(sentFrameTypes(first)).not.toContain("quiet.request"); + + await supervisor.shutdown(); + }); + + test("a quiet check the child can no longer be sent puts the standby attempt off", async () => { + let pipeClosed = false; + const { supervisor } = await setup({ + controlWriteFails: () => pipeClosed, + }); + + pipeClosed = true; + await expect(supervisor.enterStandby()).rejects.toBeInstanceOf( + StandbyDeferredError, + ); + + pipeClosed = false; + await supervisor.shutdown(); + }); + + test("a correlation query the child can no longer be sent puts the standby attempt off", async () => { + let queryFails = false; + const { supervisor, firstSender, first } = await setup({ + controlWriteFails: (line) => + queryFails && line.includes('"parked-correlations.request"'), + }); + + queryFails = true; + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await expect(inStandby).rejects.toBeInstanceOf(StandbyDeferredError); + expect(sentFrameTypes(first)).not.toContain("shutdown"); + + queryFails = false; + await supervisor.shutdown(); + }); + + test("a standby attempt is refused once the run has ended", async () => { + const { supervisor, firstSender, first, tracker } = await setup(); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { ended: true }); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt is refused when the child cannot check that it is quiet", async () => { + const { supervisor, firstSender, first, tracker } = await setup(); + + const inStandby = supervisor.enterStandby(); + await answerQuiet(first, firstSender, 1, { + quiet: false, + error: "store read failed", + }); + await expect(inStandby).rejects.toThrow(/store read failed/); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + // Refused at the first failed check, not asked again. + expect( + sentFrameTypes(first).filter((type) => type === "quiet.request"), + ).toHaveLength(1); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("wake abandons a standby attempt the child has not reported quiet for", async () => { + const { supervisor, first, tracker } = await setup(); + + const inStandby = supervisor.enterStandby(); + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + await supervisor.wake(); + await expect(inStandby).rejects.toBeInstanceOf(StandbyRefusedError); + expect(first.killSignals).toEqual([]); + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("a standby attempt waits out the new dispatch loop when the one a recycle retired finishes late", async () => { + const memory = createMemoryInboxPrimitives(); + const consumeHeld = Promise.withResolvers(); + const releaseConsume = Promise.withResolvers(); + let holdConsume = true; + let holdDequeue = false; + const dequeueHeld = Promise.withResolvers(); + const releaseDequeue = Promise.withResolvers(); + const standbyWaits = Promise.withResolvers(); + let firstMessageId: string | null = null; + const { supervisor, firstSender, first, tracker, mailBus, ipcKeypair } = + await setup({ + onStandbyWait: (waiting) => { + if (waiting === "standby") standbyWaits.resolve(undefined); + }, + inboxPrimitives: { + ...memory, + async markConsumed(...args) { + if (holdConsume) { + holdConsume = false; + consumeHeld.resolve(undefined); + await releaseConsume.promise; + } + return memory.markConsumed(...args); + }, + async replayProcessingToInbox( + store, + principal, + repoId, + address, + opts, + ) { + // Keeps the mail the run took, as the production scan does. + const owned = new Set(opts?.ownedMessageIds ?? []); + if (firstMessageId !== null) owned.add(firstMessageId); + return memory.replayProcessingToInbox( + store, + principal, + repoId, + address, + { ...opts, ownedMessageIds: owned }, + ); + }, + async dequeueToProcessing(...args) { + const taken = await memory.dequeueToProcessing(...args); + if (holdDequeue && taken !== null) { + holdDequeue = false; + dequeueHeld.resolve(undefined); + await releaseDequeue.promise; + } + return taken; + }, + }, + }); + + // The first loop's consume is held after its run parks. + mailBus.deliver(ADDRESS, new TextEncoder().encode("first")); + const fired = await waitForUpstreamPayload( + first.supervisorToChild, + "trigger.fire", + ); + firstMessageId = fired.data.messageId; + await takeTrigger(first, firstSender); + await firstSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-1", parkKind: "input" }, + }); + await consumeHeld.promise; + + // A recycle retires that loop, and the new loop holds the replayed mail + // right after moving it out of the inbox. + await mailBus.deliverSettled(ADDRESS, new TextEncoder().encode("second")); + holdDequeue = true; + const recycled = supervisor.recycle({ reason: "operator" }); + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await dequeueHeld.promise; + releaseConsume.resolve(undefined); + await recycled; + + const inStandby = supervisor.enterStandby(); + await standbyWaits.promise; + releaseDequeue.resolve(undefined); + await takeTrigger(second, secondSender); + await secondSender.send({ + type: "park.notify", + data: { runId: RUN, correlationId: "corr-2", parkKind: "input" }, + }); + await actAsChildEnteringStandby(second, secondSender); + await inStandby; + await supervisor.shutdown(); + expect(memory.messageIds(ADDRESS).processing).toEqual([]); + }); + + test("a refusal answers the writes a final check held back", async () => { + const { supervisor, firstSender, first, tracker } = await setup({ + inboxPrimitives: (() => { + const inbox = createMemoryInboxPrimitives(); + return { ...inbox, hasInboxEntries: async () => true }; + })(), + }); + const inStandby = supervisor.enterStandby(); + void inStandby.catch(() => undefined); + await answerQuiet(first, firstSender, 1, { quiet: true }); + await answerQuiet(first, firstSender, 2, { quiet: true }); + await waitForUpstreamPayloads( + first.supervisorToChild, + "parked-correlations.request", + 2, + ); + await sendChildWrite(firstSender, "held-by-fence"); + await answerParkedQuery(first, firstSender, 2); + await expect(inStandby).rejects.toThrow("mail is waiting"); + expect(tracker.totalSpawns).toBe(1); + const responses = await waitForUpstreamPayloads( + first.supervisorToChild, + "substrate.write.response", + 1, + ); + expect(responses[0]?.data.requestId).toBe("held-by-fence"); + await supervisor.shutdown(); + }); + + test("a signal mid delivery holds a standby attempt's final check back", async () => { + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + const waits = Promise.withResolvers(); + let holding = false; + const { supervisor, firstSender, first } = await setup({ + onStandbyWait: (on) => { + if (holding && on === "standby") waits.resolve(undefined); + }, + recyclePolicy: { + onRunStart: async () => { + if (holding) { + reached.resolve(undefined); + await release.promise; + } + return null; + }, + }, + }); + const inStandby = supervisor.enterStandby(); + void inStandby.catch(() => undefined); + await waitForUpstreamPayload(first.supervisorToChild, "quiet.request"); + holding = true; + const delivering = supervisor.deliverSignal({ + runId: RUN, + signalName: "go", + signalId: "sig-held", + payload: null, + }); + await reached.promise; + await answerQuiet(first, firstSender, 1, { quiet: true }); + expect( + await Promise.race([ + waits.promise.then(() => "waited"), + waitForUpstreamPayloads( + first.supervisorToChild, + "quiet.request", + 2, + ).then(() => "asked"), + ]), + ).toBe("waited"); + release.resolve(undefined); + await delivering; + await supervisor.shutdown(); + }); + + test("a boot replay still running holds a standby attempt back", async () => { + const inbox = createMemoryInboxPrimitives(); + const release = Promise.withResolvers(); + const waits = Promise.withResolvers(); + const { supervisor, first } = await setup({ + inboxPrimitives: { + ...inbox, + replayProcessingToInbox: async (...args) => { + await release.promise; + return inbox.replayProcessingToInbox(...args); + }, + }, + onStandbyWait: (on) => { + if (on === "standby") waits.resolve(undefined); + }, + }); + const inStandby = supervisor.enterStandby(); + void inStandby.catch(() => undefined); + const seen = await Promise.race([ + waits.promise.then(() => "waited"), + waitForUpstreamPayload(first.supervisorToChild, "quiet.request").then( + () => "asked", + ), + ]); + expect(seen).toBe("waited"); + release.resolve(undefined); + await supervisor.shutdown(); + }); + + test("mail waking a live standby whose respawn fails stops the supervisor and says so", async () => { + const stopped = Promise.withResolvers(); + const baseDir = await makeTempDir("standby-wake-fails-"); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + let spawns = 0; + const { supervisor, firstSender } = await spawnSupervisor({ + baseDir, + tracker: { + ...tracker, + spawner: (args) => { + spawns += 1; + if (spawns > 1) + throw new Error("the workflow-child binary is missing"); + return tracker.spawner(args); + }, + }, + mailBus, + ipcKeypair, + overrides: { + repoStore: createStubRepoStore(baseDir, { + writeTree: true, + refTip: HISTORY_TIP, + }), + onSelfTerminate: (info) => stopped.resolve(info.reason), + }, + }); + const first = tracker.children[0]; + if (first === undefined) throw new Error("first child missing"); + const inStandby = supervisor.enterStandby(); + await actAsChildEnteringStandby(first, firstSender); + expect(await inStandby).toEqual({}); + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + expect(await stopped.promise).toBe( + "recycle failed: the workflow-child binary is missing", + ); + }); +}); + +describe("supervisor restored in standby", () => { + const SPAWN_OPTS = { + definitionHash: TEST_DEFINITION_HASH, + onInferenceEvent: () => undefined, + }; + + // A store over a run whose history tip is `tip`, and whose writes merge, so + // a cancellation can be committed. `undefined` leaves the tip unreadable. + // `beforeTipRead` runs ahead of each read of the tip. + function historyStore( + baseDir: string, + tip: string | null | undefined, + beforeTipRead?: () => Promise, + ) { + const base = createStubRepoStore(baseDir, { + writeTree: true, + ...(tip !== undefined ? { refTip: tip } : {}), + }); + return new Proxy(base, { + get(target, prop, receiver) { + if (prop === "resolveRef" && beforeTipRead !== undefined) { + return async (...args: Parameters) => { + await beforeTipRead(); + return target.resolveRef(...args); + }; + } + if (prop !== "writeTreePreservingPrefix") { + return Reflect.get(target, prop, receiver); + } + return async ( + ...args: Parameters + ) => { + await args[3].merge(new Map()); + return target.writeTreePreservingPrefix(...args); + }; + }, + }); + } + + // Restore a supervisor from `standby` over a run whose history tip is + // `tip`, the recorded one unless a test moves it. + async function restore( + standby: RecordedStandby, + opts: { + tip?: string | null | undefined; + onWake?: WorkflowSupervisorBindings["onWake"]; + onSelfTerminate?: WorkflowSupervisorBindings["onSelfTerminate"]; + onSuspensionRegister?: WorkflowSupervisorBindings["onSuspensionRegister"]; + inbox?: ReturnType; + inboxPrimitives?: Partial; + beforeRestore?: ( + baseDir: string, + inbox: ReturnType, + ) => Promise; + spawner?: (tracker: SpawnTracker) => SubprocessSpawner; + recyclePolicy?: Partial; + beforeTipRead?: () => Promise; + } = {}, + ) { + const baseDir = await makeTempDir("standby-restore-"); + await seedStepGrants(baseDir, "step-1", [ + { resource: "thing", action: "read" }, + ]); + const inbox = opts.inbox ?? createMemoryInboxPrimitives(); + await opts.beforeRestore?.(baseDir, inbox); + const ipcKeypair = await generateKeyPair(); + const mailBus = createMockMailBus(); + const tracker = createSpawnTracker({}); + const bindings = await buildBindings({ + baseDir, + spawner: opts.spawner?.(tracker) ?? tracker.spawner, + mailBus, + ipcKeypair, + inboxPrimitives: { ...inbox, ...opts.inboxPrimitives }, + }); + const supervisor = createWorkflowSupervisor({ + ...bindings, + repoStore: historyStore( + baseDir, + "tip" in opts ? opts.tip : standby.tip, + opts.beforeTipRead, + ), + ...(opts.onWake !== undefined ? { onWake: opts.onWake } : {}), + ...(opts.onSelfTerminate !== undefined + ? { onSelfTerminate: opts.onSelfTerminate } + : {}), + ...(opts.onSuspensionRegister !== undefined + ? { onSuspensionRegister: opts.onSuspensionRegister } + : {}), + ...opts.recyclePolicy, + }); + const restoring = supervisor.restoreInStandby(SPAWN_OPTS, standby); + return { supervisor, restoring, tracker, mailBus, ipcKeypair, inbox }; + } + + // Drive the `count`th child through its handshake and the correlation query + // its install sends. + async function driveStartedChild( + tracker: SpawnTracker, + ipcKeypair: { privateKey: Uint8Array; publicKey: Uint8Array }, + count: number, + ) { + await tracker.awaitChildren(count); + const child = tracker.children[count - 1]; + if (child === undefined) throw new Error(`child ${String(count)} missing`); + const sender = await driveReady(child, ipcKeypair); + await answerParkedQuery(child, sender, 1); + return { child, sender }; + } + + const recorded: RecordedStandby = { + parked: [GATE_PARK], + tip: HISTORY_TIP, + definition: TEST_DEFINITION, + }; + + test("a shutdown while the restore reads the history tip waits for it, and the restore stops", async () => { + const reading = Promise.withResolvers(); + const release = Promise.withResolvers(); + const { supervisor, restoring, tracker, mailBus } = await restore( + recorded, + { + beforeTipRead: async () => { + reading.resolve(undefined); + await release.promise; + }, + }, + ); + await reading.promise; + + const shutdown = supervisor.shutdown(); + release.resolve(undefined); + await expect(restoring).rejects.toThrow("shut down before the restore"); + await shutdown; + expect(mailBus.registered()).toEqual([]); + expect(tracker.totalSpawns).toBe(0); + }); + + test("a wake whose child a shutdown leaves uninstalled reports no wake", async () => { + const killRequested = Promise.withResolvers(); + let finishChild: (() => void) | undefined; + const wakes: string[] = []; + const { supervisor, restoring, tracker, ipcKeypair } = await restore( + recorded, + { + onWake: ({ reason }) => wakes.push(reason), + // The shutdown's kill is held, so the child's ready arrives after it. + spawner: (tracker) => (args) => { + const handle = tracker.spawner(args); + finishChild = () => handle.kill("SIGKILL"); + return { ...handle, kill: () => killRequested.resolve(undefined) }; + }, + }, + ); + await restoring; + + const woken = supervisor.wake(); + await tracker.awaitChildren(1); + const stopped = supervisor.shutdown(); + await killRequested.promise; + const child = tracker.children[0]; + if (child === undefined) throw new Error("child missing"); + await driveReady(child, ipcKeypair); + await expect(woken).rejects.toThrow("shut down before the woken child ran"); + finishChild?.(); + await stopped; + expect(wakes).toEqual([]); + }); + + test("a restored standby arms no recycle policy before its first child, and measures that child from its start", async () => { + const policy = manualRecyclePolicy(60_000); + const { supervisor, restoring, tracker, ipcKeypair } = await restore( + recorded, + { recyclePolicy: policy.bindings }, + ); + expect(await restoring).toEqual({ inStandby: true }); + expect(policy.armed()).toBe(0); + + policy.advance(120_000); + const armed = policy.nextArm(); + const woken = supervisor.wake(); + await driveStartedChild(tracker, ipcKeypair, 1); + await woken; + await armed; + + const ticked = policy.nextArm(); + policy.tick(); + await ticked; + expect(tracker.totalSpawns).toBe(1); + + await supervisor.shutdown(); + }); + + test("the first child of a restored standby whose boot replay went through does not replay again", async () => { + const inbox = createMemoryInboxPrimitives(); + let replays = 0; + const { supervisor, restoring, tracker, ipcKeypair } = await restore( + recorded, + { + inbox, + inboxPrimitives: { + async replayProcessingToInbox(...args) { + replays += 1; + return inbox.replayProcessingToInbox(...args); + }, + }, + }, + ); + expect(await restoring).toEqual({ inStandby: true }); + + const woken = supervisor.wake(); + await driveStartedChild(tracker, ipcKeypair, 1); + await woken; + expect(replays).toBe(1); + + await supervisor.shutdown(); + }); + + test("a wake that reaches its replay after a shutdown began does not replay", async () => { + const inbox = createMemoryInboxPrimitives(); + const bootReplay = Promise.withResolvers(); + let replays = 0; + const { supervisor, restoring } = await restore(recorded, { + inbox, + inboxPrimitives: { + async replayProcessingToInbox(...args) { + replays += 1; + if (replays > 1) return inbox.replayProcessingToInbox(...args); + await bootReplay.promise; + throw new Error("the boot replay failed"); + }, + }, + }); + expect(await restoring).toEqual({ inStandby: true }); + + const woken = supervisor.wake().catch(() => undefined); + const stopped = supervisor.shutdown(); + bootReplay.resolve(undefined); + await stopped; + await woken; + expect(replays).toBe(1); + }); + + test("a standby restored over unchanged history starts no child and re-sends what it was parked on", async () => { + const registered: string[] = []; + const { supervisor, restoring, tracker, mailBus } = await restore( + recorded, + { + onSuspensionRegister: (registration) => { + registered.push(registration.correlationId); + }, + }, + ); + expect(await restoring).toEqual({ inStandby: true }); + expect(mailBus.registered()).toEqual([ADDRESS]); + + await supervisor.reEmitParkedCorrelations(); + expect(registered).toEqual(["corr-gate"]); + expect(await supervisor.enterStandby()).toEqual({}); + + await supervisor.shutdown(); + expect(mailBus.registered()).toEqual([]); + expect(tracker.totalSpawns).toBe(0); + }); + + const wakes: [ + string, + Partial, + (setup: Awaited>) => Promise, + ][] = [ + [ + "mail", + {}, + async ({ mailBus }) => { + mailBus.deliver(ADDRESS, new TextEncoder().encode("wake up")); + }, + ], + [ + "timer", + { wakeAt: new Date(Date.now() - 1_000).toISOString() }, + async () => undefined, + ], + ["wake", {}, ({ supervisor }) => supervisor.wake()], + [ + "signal", + {}, + ({ supervisor }) => + supervisor.deliverSignal({ + runId: RUN, + signalName: "gate", + signalId: "sig-1", + payload: { approved: true }, + }), + ], + [ + "cancel", + {}, + ({ supervisor }) => + supervisor.requestCancel({ + runId: RUN, + origin: "hub-admin", + reason: "Stop", + at: "2026-10-01T00:00:00.000Z", + }), + ], + ]; + test.each(wakes)( + "%s starts the first child of a restored standby", + async (reason, standby, trigger) => { + const woke = Promise.withResolvers<{ + reason: string; + requested: boolean; + }>(); + const restored = await restore( + { ...recorded, ...standby }, + { onWake: (info) => woke.resolve(info) }, + ); + expect(await restored.restoring).toEqual({ inStandby: true }); + const triggered = trigger(restored); + const { child } = await driveStartedChild( + restored.tracker, + restored.ipcKeypair, + 1, + ); + await triggered; + // Only a wake asked for through `wake()` is requested. + expect(await woke.promise).toEqual({ + reason, + requested: reason === "wake", + }); + expect(sentFrameTypes(child)[0]).toBe("authority.set"); + if (reason === "signal") { + expect( + parseSignalDeliverRunIds(child.supervisorToChild.flushed()), + ).toEqual([RUN]); + } + + await restored.supervisor.shutdown(); + }, + ); + + const startedAwake: [string, string | null | undefined, string][] = [ + [ + "written since it stopped", + "tip-after-a-cancellation", + "its restart, which found history written after it stopped", + ], + [ + "that cannot be read", + undefined, + "its restart, which could not read its history: stub RepoStore: resolveRef not implemented for this test", + ], + ]; + test.each(startedAwake)( + "a standby restored over history %s starts the child, and says why", + async (_case, tip, reason) => { + const { supervisor, restoring, tracker, ipcKeypair } = await restore( + recorded, + { tip }, + ); + await tracker.awaitChildren(1); + const child = tracker.children[0]; + if (child === undefined) throw new Error("child missing"); + const sender = await driveReady(child, ipcKeypair); + expect(await restoring).toEqual({ inStandby: false, reason }); + + // A running child is asked whether it is quiet, not answered for. + const entering = supervisor.enterStandby(); + await answerQuiet(child, sender, 1, { ended: true }); + await expect(entering).rejects.toThrow("the workflow run has ended"); + + await supervisor.shutdown(); + }, + ); + + test("a standby restored with a definition that is not the approved one starts the child, and says why", async () => { + const { supervisor, restoring, tracker, ipcKeypair } = await restore({ + ...recorded, + definition: { ...TEST_DEFINITION, id: "another-workflow" }, + }); + await tracker.awaitChildren(1); + const child = tracker.children[0]; + if (child === undefined) throw new Error("child missing"); + await driveReady(child, ipcKeypair); + expect(await restoring).toEqual({ + inStandby: false, + reason: + "its restart, which found a recorded definition that is not the approved one", + }); + + await supervisor.shutdown(); + }); + + test("a restored standby whose boot replay puts mail back in the inbox wakes for it", async () => { + const woke = Promise.withResolvers(); + const { restoring, tracker, ipcKeypair, supervisor } = await restore( + recorded, + { + onWake: (info) => woke.resolve(info.reason), + beforeRestore: (baseDir, memory) => + takeMail(memory, baseDir, "mail-orphan", 1), + }, + ); + expect(await restoring).toEqual({ inStandby: true }); + await driveStartedChild(tracker, ipcKeypair, 1); + expect(await woke.promise).toBe("mail"); + + await supervisor.shutdown(); + }); + + test("a restored standby keeps mail a live run took, and stays stopped", async () => { + const replayed = Promise.withResolvers(); + const { restoring, tracker, supervisor, inbox } = await restore(recorded, { + beforeRestore: async (baseDir, memory) => { + await writeWorkingRunLog(baseDir, [ + { ...runStarted(), consumedMessageId: "mail-owned" }, + ]); + await takeMail(memory, baseDir, "mail-owned", 1); + }, + inboxPrimitives: { + async hasInboxEntries(...args) { + replayed.resolve(undefined); + return createMemoryInboxPrimitives().hasInboxEntries(...args); + }, + }, + }); + expect(await restoring).toEqual({ inStandby: true }); + await replayed.promise; + expect(inbox.messageIds(ADDRESS)).toEqual({ + inbox: [], + processing: ["mail-owned"], + consumed: [], + }); + + await supervisor.shutdown(); + expect(tracker.totalSpawns).toBe(0); + }); + + test("a restored standby whose boot replay fails wakes in case it left mail behind", async () => { + const woke = Promise.withResolvers(); + const { restoring, tracker, ipcKeypair, supervisor } = await restore( + recorded, + { + onWake: (info) => woke.resolve(info.reason), + inboxPrimitives: { + async replayProcessingToInbox() { + throw new Error("the claim-check tree is unreadable"); + }, + }, + }, + ); + expect(await restoring).toEqual({ inStandby: true }); + await driveStartedChild(tracker, ipcKeypair, 1); + expect(await woke.promise).toBe( + "its restart, which could not recover its mail", + ); + + await supervisor.shutdown(); + }); + + test("a credential update while restored reaches the first child's authority.set", async () => { + const { restoring, tracker, ipcKeypair, supervisor } = + await restore(recorded); + expect(await restoring).toEqual({ inStandby: true }); + const delivery = { + bindings: [], + materials: [ + { + credentialId: "cred-1", + providerKey: "provider", + origin: "tenant", + secret: "s3cret", + }, + ], + }; + await supervisor.deliverCredentials({ delivery }); + + const woken = supervisor.wake(); + const { child } = await driveStartedChild(tracker, ipcKeypair, 1); + await woken; + const authority = await waitForUpstreamPayload( + child.supervisorToChild, + "authority.set", + ); + expect(authority.data.delivery).toEqual(delivery); + + await supervisor.shutdown(); + }); + + test("calls that need a child do nothing to a restored standby, or are refused", async () => { + const { restoring, tracker, supervisor } = await restore(recorded); + expect(await restoring).toEqual({ inStandby: true }); + + await supervisor.drain({ deadlineMs: 1_000 }); + await supervisor.deliverSources({ + sources: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-x", + model: "claude-test", + }, + ], + defaultSource: "primary", + }); + expect(await supervisor.deliverGrants(RUN)).toBe("skipped"); + await expect( + supervisor.recycle({ reason: "policy", origin: "operator" }), + ).rejects.toThrow("recycle called in phase standby"); + supervisor.abandonStandby("the Hub went away"); + expect(tracker.totalSpawns).toBe(0); + + await supervisor.shutdown(); + }); + + test("a shutdown while a restored standby's first child starts stops that child", async () => { + const { restoring, tracker, supervisor } = await restore(recorded); + expect(await restoring).toEqual({ inStandby: true }); + const woken = supervisor.wake(); + await tracker.awaitChildren(1); + const child = tracker.children[0]; + if (child === undefined) throw new Error("child missing"); + + await supervisor.shutdown(); + await expect(woken).rejects.toThrow(); + expect(child.killSignals.length).toBeGreaterThan(0); + }); + + test("a channel crash while a restored standby's first child starts stops the supervisor and says so", async () => { + const stopped = Promise.withResolvers(); + const { restoring, tracker, supervisor } = await restore(recorded, { + onSelfTerminate: (info) => stopped.resolve(info.reason), + }); + expect(await restoring).toEqual({ inStandby: true }); + const woken = supervisor.wake(); + await tracker.awaitChildren(1); + const child = tracker.children[0]; + if (child === undefined) throw new Error("child missing"); + child.childToSupervisor.inject("not-json{{{"); + + await expect(woken).rejects.toThrow(); + expect(await stopped.promise).toContain( + "control channel received non-JSON line", + ); + }); + + test("a restored standby whose first child cannot start stops the supervisor and says so", async () => { + const stopped = Promise.withResolvers(); + const { restoring, supervisor } = await restore(recorded, { + onSelfTerminate: (info) => stopped.resolve(info.reason), + spawner: () => () => { + throw new Error("the workflow-child binary is missing"); + }, + }); + expect(await restoring).toEqual({ inStandby: true }); + + await expect(supervisor.wake()).rejects.toThrow( + "the workflow-child binary is missing", + ); + expect(await stopped.promise).toBe( + "recycle failed: the workflow-child binary is missing", + ); + }); + + test("a standby attempt while a restored standby's first child starts is put off", async () => { + const { restoring, supervisor, tracker, ipcKeypair } = + await restore(recorded); + expect(await restoring).toEqual({ inStandby: true }); + const woken = supervisor.wake(); + await tracker.awaitChildren(1); + const attempt = await supervisor.enterStandby().then( + (outcome) => ({ answered: outcome }), + (cause: unknown) => ({ failed: cause }), + ); + expect( + "failed" in attempt && attempt.failed instanceof StandbyDeferredError, + ).toBe(true); + const second = tracker.children[0]; + if (second === undefined) throw new Error("child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await answerParkedQuery(second, secondSender, 1); + await woken; + await supervisor.shutdown(); + }); +}); diff --git a/packages/workflow-host/src/supervisor/recycle.ts b/packages/workflow-host/src/supervisor/recycle.ts index 4a03b9eff..f6a06a6c9 100644 --- a/packages/workflow-host/src/supervisor/recycle.ts +++ b/packages/workflow-host/src/supervisor/recycle.ts @@ -18,11 +18,11 @@ // The `drain` step and the `SubprocessHandle` handed in as `current` are // caller-parameterized. For the operator/policy/self recycle origins the // child is live: `drain` sends the real drain control mail and `kill` -// terminates a running process. For the `crash` origin the child has -// already exited unexpectedly, so the caller supplies a no-op `drain` -// (there is nothing to drain) and the `kill` in step 2 lands on an -// already-dead handle as a cheap no-op. Steps 3-6 are identical for every -// origin. +// terminates a running process. For the `crash` and `wake` origins the +// child has already exited, unexpectedly or because a standby attempt stopped +// it, so the caller supplies a no-op `drain` (there is nothing to drain) and +// the `kill` in step 2 lands on an already-dead handle as a cheap no-op. Steps +// 3-6 are identical for every origin. // // 1. `drain` -- send the existing drain control mail. Wait for // in-flight runs to drain per each step's `drainBehavior`. @@ -68,7 +68,7 @@ // instead: the resumed run carries on with it, and the supervisor // consumes it once the run next parks or ends. // -// Three trigger origins funnel through `triggerRecycle(reason, ctx)`: +// Five origins funnel through `triggerRecycle(reason, ctx)`: // // - Operator command -- the host receives a `recycle` request via // its caller-facing API and routes it to the supervisor's @@ -81,9 +81,15 @@ // `recycle.request` payload over control IPC. The supervisor's // upstream control-channel reader recognises the variant and // funnels it here. +// - Crash respawn -- the child exited unexpectedly, and the +// supervisor starts the next one. +// - Wake -- input or a timer wakes a child the supervisor stopped +// for standby. // -// All three origins land in the same code path. The reason string is -// the only origin-specific data the path carries forward. +// All five origins land in the same code path. The origin tag and the +// reason string are the only origin-specific data it carries forward. The +// first wake of a deployment restored in standby has no child to stop and +// skips the replay its boot already made. import { getLogger } from "@intx/log"; @@ -133,10 +139,11 @@ export const DEFAULT_POLICY_INTERVAL_MS = 60_000; * operator scanning logs can distinguish operator-initiated, * policy-initiated, self-initiated, and crash-respawn origins at a * glance. The `crash` origin drives the same respawn sequence after an - * unexpected child exit; see the six-step header note about steps 1-2 - * degrading to no-ops for it. + * unexpected child exit, and `wake` drives it for a child the supervisor + * stopped while every run waited; see the six-step header note about steps + * 1-2 degrading to no-ops for both. */ -export type RecycleOrigin = "operator" | "policy" | "self" | "crash"; +export type RecycleOrigin = "operator" | "policy" | "self" | "crash" | "wake"; export interface RecycleAttempt { /** Origin the recycle was initiated from. */ @@ -145,8 +152,8 @@ export interface RecycleAttempt { readonly reason: string; /** ChannelId the recycled child was minted with. */ readonly newChannelId: string; - /** ChannelId the previous child was running under. */ - readonly previousChannelId: string; + /** ChannelId the previous child ran under, or null when none ran before. */ + readonly previousChannelId: string | null; } /** @@ -179,8 +186,11 @@ export interface RecycleContext { readonly definitionHash: string; /** Forward target for InferenceEvents the new child publishes. */ readonly onInferenceEvent: (event: EventPayload) => void; - /** Live child wiring on entry; replaced before return. */ - readonly current: ChildWiring; + /** + * Live child wiring on entry, replaced before return; null when a restored + * standby starts its first child. + */ + readonly current: ChildWiring | null; /** Supervisor-side drain primitive; sends the existing drain mail. */ readonly drain: (deadlineMs: number) => Promise; /** @@ -285,9 +295,9 @@ export async function triggerRecycle( ): Promise { const drainDeadlineMs = ctx.drainDeadlineMs ?? 60_000; const killTimeoutMs = ctx.killTimeoutMs ?? DEFAULT_KILL_TIMEOUT_MS; - const previousChannelId = ctx.current.channelId; + const previousChannelId = ctx.current?.channelId ?? null; - logger.info`recycle ${opts.origin} requested: ${opts.reason} (previousChannelId=${previousChannelId})`; + logger.info`recycle ${opts.origin} requested: ${opts.reason} (previousChannelId=${previousChannelId ?? "none"})`; // Step 1: drain. The supervisor's drain primitive is the same one // the standalone-drain path uses; the recycle path does not @@ -310,11 +320,13 @@ export async function triggerRecycle( // `killTimeoutMs`, the handle's hard kill lands. The injected // spawner's `SubprocessHandle.kill()` is the surface the recycle // path touches; the spawner owns the Node primitives. - await killChildHandle(ctx.current.handle, killTimeoutMs, { - logger, - ...(ctx.setTimer !== undefined ? { setTimer: ctx.setTimer } : {}), - ...(ctx.clearTimer !== undefined ? { clearTimer: ctx.clearTimer } : {}), - }); + if (ctx.current !== null) { + await killChildHandle(ctx.current.handle, killTimeoutMs, { + logger, + ...(ctx.setTimer !== undefined ? { setTimer: ctx.setTimer } : {}), + ...(ctx.clearTimer !== undefined ? { clearTimer: ctx.clearTimer } : {}), + }); + } // Replay the in-flight `processing/` entries no live run took back to // `inbox/`. Without this replay, a child whose run was mid-dispatch diff --git a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts index b47dbe62d..4354a7841 100644 --- a/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts +++ b/packages/workflow-host/src/supervisor/spawn-replay-fifo.test.ts @@ -196,6 +196,9 @@ function createGatedInboxPrimitives(): { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address) { await gate; const s = getOrCreate(address); @@ -561,8 +564,15 @@ describe("supervisor spawn-time replay FIFO contract", () => { readPayloadsOfType(harness.supervisorToChild.flushed(), "trigger.fire"), ).toHaveLength(1); - // Complete the resumed run so the fresh mail's durable-consume wait - // releases, then verify both entries left the queue in FIFO order. + // Commit the signal and complete the resumed run so the fresh mail's + // durable-consume wait releases, then verify both entries left the queue + // in FIFO order. + const requestId = freshSignal.data.requestId; + if (requestId === undefined) throw new Error("no answer was asked for"); + await harness.childSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); await harness.childSender.send({ type: "terminal.event", data: { diff --git a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts index 57fd39df7..c7d4f4bb9 100644 --- a/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts +++ b/packages/workflow-host/src/supervisor/stale-cohort-routing.test.ts @@ -183,6 +183,9 @@ function createMemoryInbox(): InboxPrimitives & { } throw new Error("processing not found"); }, + async hasInboxEntries(_s, _p, _r, address) { + return get(address).inbox.size > 0; + }, async replayProcessingToInbox(_s, _p, _r, address) { const s = get(address); for (const [key, value] of s.processing) { diff --git a/packages/workflow-host/src/supervisor/substrate-write.test.ts b/packages/workflow-host/src/supervisor/substrate-write.test.ts index c0b434a64..159f3888c 100644 --- a/packages/workflow-host/src/supervisor/substrate-write.test.ts +++ b/packages/workflow-host/src/supervisor/substrate-write.test.ts @@ -301,6 +301,9 @@ function createMemoryInboxPrimitives(opts: { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address) { const s = getOrCreate(address); const replayedKeys: string[] = []; diff --git a/packages/workflow-host/src/supervisor/supervisor.test.ts b/packages/workflow-host/src/supervisor/supervisor.test.ts index 9a59220c0..98156a7e6 100644 --- a/packages/workflow-host/src/supervisor/supervisor.test.ts +++ b/packages/workflow-host/src/supervisor/supervisor.test.ts @@ -95,6 +95,29 @@ function parseControlFrameTypes(lines: readonly string[]): string[] { * Parse every `signal.deliver` frame in the supervisor-to-child stream, * returning the `{ signalName, signalId }` for each. */ +// The child's answer that it committed the `count`th `signal.deliver` it got. +async function confirmDelivery( + wired: { + supervisorToChild: Parameters[0]; + childSender: ReturnType; + }, + count: number, +): Promise { + const deliveries = await waitForUpstreamPayloads( + wired.supervisorToChild, + "signal.deliver", + count, + ); + const requestId = deliveries[count - 1]?.data.requestId; + if (requestId === undefined) { + throw new Error("the delivery asked for no confirmation"); + } + await wired.childSender.send({ + type: "input.taken", + data: { requestId, committed: true }, + }); +} + function parseSignalDelivers( lines: readonly string[], ): { signalName: string; signalId: string; payload: unknown }[] { @@ -952,6 +975,9 @@ function createMemoryInboxPrimitives(): MemoryInboxPrimitives { prunedMessageIds: [], }; }, + async hasInboxEntries(_store, _principal, _repoId, address) { + return getOrCreate(address).inbox.size > 0; + }, async replayProcessingToInbox(_store, _principal, _repoId, address) { const state = getOrCreate(address); const replayedKeys: string[] = []; @@ -4657,6 +4683,7 @@ describe("createWorkflowSupervisor", () => { // resumed run: markConsumed for a signal now holds until the child has // durably taken it up (re-parks or terminates), mirroring trigger.fire. await waitForUpstreamPayload(wired.supervisorToChild, "signal.deliver"); + await confirmDelivery(wired, 1); // The signal is sent but the run has not taken it up, so msg-2's // markConsumed is held until it re-parks or terminates. Only msg-1 is // consumed. @@ -4763,6 +4790,7 @@ describe("createWorkflowSupervisor", () => { const firstSignal = signals[0]; if (firstSignal === undefined) throw new Error("unreachable"); expect(firstSignal.signalName).toBe(signalName("corr-input-1")); + await confirmDelivery(wired, 1); // msg-1 is consumed off its park, but msg-2's signal is not yet taken up // by the run, so its markConsumed is still held. @@ -5262,6 +5290,7 @@ describe("createWorkflowSupervisor", () => { wired.mailBus.deliver(address, new TextEncoder().encode("msg-3")); await waitSignals(1); + await confirmDelivery(wired, 1); await wired.childSender.send({ type: "park.notify", data: { @@ -5273,6 +5302,7 @@ describe("createWorkflowSupervisor", () => { await waitConsumed(2); await waitSignals(2); + await confirmDelivery(wired, 2); await wired.childSender.send({ type: "terminal.event", data: { diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index 0b32edc45..e030caa23 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -54,6 +54,7 @@ import { generateKeyPair } from "@intx/crypto"; import { enqueueInbox as defaultEnqueueInbox, dequeueToProcessing as defaultDequeueToProcessing, + hasInboxEntries as defaultHasInboxEntries, markConsumed as defaultMarkConsumed, scanRunsForBoot, readCommittedWorkflowRunLifecycle, @@ -71,6 +72,8 @@ import { deriveWorkflowRunId } from "@intx/types/workflow-run-id"; import { signalName } from "@intx/types/signals"; import { RepoId } from "@intx/types/repo"; import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; +import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; import { inboundMessageToRaw } from "@intx/mail-memory"; import { executeSearch, @@ -152,6 +155,7 @@ import type { DispatchStructuralCounters, DispatchSubstrateLeg, InboxPrimitives, + RecordedStandby, SubprocessHandle, TerminalEventSource, TerminalRunEvent, @@ -234,6 +238,14 @@ export const DEFAULT_RESPAWN_BACKOFF_MAX_MS = 30_000; */ export const DEFAULT_PARKED_QUERY_WATCHDOG_MS = 30_000; +// The wake reason of a respawn `wake()` asked for. +const REQUESTED_WAKE = "wake"; + +// A standby attempt's final check holds dispatch and signals back, and a signal +// holds its address's frames behind it, a forced stop among them. A check that +// does not finish within this bound counts the child as working and they go on. +const QUIET_CHECK_WATCHDOG_MS = 30_000; + /** * Backstop for `waitForRunTerminalOrPark`. A dispatch waits here for the child * to park or terminate the run before releasing `markConsumed`; a lost park @@ -264,6 +276,16 @@ export interface WorkflowSupervisor { * been pushed. */ spawn(opts: SpawnOpts): Promise; + /** + * Restore a deployment its host recorded in standby. While the run's history + * is unchanged it takes mail and re-sends its parks without a child, until an + * input wakes it; otherwise the child starts as on `spawn`, and the result + * says why. + */ + restoreInStandby( + opts: SpawnOpts, + standby: RecordedStandby, + ): Promise; /** * Sign and commit a CancelRequested event under the named origin. * Used by the host directly for `supervisor-operator` and `hub- @@ -310,7 +332,9 @@ export interface WorkflowSupervisor { * * Throws when the supervisor is not in a phase where it can address * the child (idle / stopping / stopped); the caller is responsible - * for serializing delivery against `spawn` completion. + * for serializing delivery against `spawn` completion. A standby attempt past + * its final check settles first, and a child in standby is respawned and the + * signal delivered to the new one. */ deliverSignal(opts: DeliverSignalOpts): Promise; /** @@ -318,7 +342,8 @@ export interface WorkflowSupervisor { * agent. Mirrors `deliverSignal`: the supervisor is the single producer * of `sources-updated` control frames, and delivery is phase-guarded to * starting/running so a frame is never written into a recycling child's - * closing pipe. Throws otherwise. + * closing pipe. While in standby it returns at once, since the respawn reads + * the host's persisted sources; any other phase throws. */ deliverSources(opts: DeliverSourcesOpts): Promise; /** @@ -346,7 +371,8 @@ export interface WorkflowSupervisor { * Re-register every correlation the child is currently parked on by * querying it for its parked correlations and re-emitting each through * `onSuspensionRegister`. Recovers a `park.notify` register the hub may have - * missed while it was down at suspend time. + * missed while it was down at suspend time. While in standby it re-sends the + * correlations the stopped child was parked on. * * Best-effort and safe to call at any time, including concurrently. Unlike * `deliverSignal`/`deliverSources`, which throw when the child is not @@ -364,6 +390,64 @@ export interface WorkflowSupervisor { * the run record. `null` when the deployment has never had any. */ credentialDelivery(): CredentialDelivery | null; + /** + * Stop the child once every run only waits on input from outside, keeping + * the deployment's mail subscription and address registration. The child + * reports when it is quiet; dispatch and signals go on meanwhile. Then the + * supervisor stops handing it input, asks again, and holds the child's + * writes until the child is stopped. A held write means the child is + * working again: before the stop begins the standby attempt waits for quiet + * once more, and after it the child is woken for that work. Resolves when + * the child is stopped. Rejects with `StandbyAbandonedError` when `wake`, + * `abandonStandby`, a cancellation or a shutdown calls it off, and with + * `StandbyRefusedError` when the child restarts, the run ends, or mail + * waits in the inbox. + * + * A supervisor in standby respawns its child for mail, a signal, a + * cancellation, its earliest pending timer, or `wake`. + */ + enterStandby(): Promise; + /** Abandon a standby attempt in progress, or respawn a child in standby. */ + wake(): Promise; + /** + * Abandon a standby attempt in progress for `reason`. One whose stop has + * begun runs the child again; a child an earlier standby attempt stopped + * stays in standby. + */ + abandonStandby(reason: string): void; +} + +/** A standby attempt that stopped the child. */ +export type StandbyOutcome = { + /** The earliest timer the child had queued: the supervisor wakes for it. */ + wakeAt?: string; +}; + +/** How `restoreInStandby` restored a deployment. */ +export type StandbyRestore = + | { inStandby: true } + | { + inStandby: false; + /** Why the child started, worded to follow "woken by". */ + reason: string; + }; + +/** A standby attempt that was refused or abandoned. */ +export class StandbyRefusedError extends Error { + override name = "StandbyRefusedError"; +} + +/** + * A standby attempt refused because the child is starting or restarting; it can + * be asked for again once the child runs. + */ +export class StandbyDeferredError extends StandbyRefusedError { + override name = "StandbyDeferredError"; +} + +/** A standby attempt called off by a wake, a cancellation, a shutdown or its caller. */ +export class StandbyAbandonedError extends StandbyRefusedError { + override name = "StandbyAbandonedError"; } export type SpawnOpts = { @@ -841,6 +925,18 @@ export function createWorkflowSupervisor( * fire `setTimeout` after the supervisor has been disposed). */ const drainAccumulators = new Map(); + // Each stop is guarded, so one that throws does not leave the rest armed. + function stopDrainAccumulators(during: string): void { + for (const accumulator of drainAccumulators.values()) { + try { + accumulator.stop(); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`drain accumulator stop threw during ${during}: ${message}`; + } + } + drainAccumulators.clear(); + } const accumulatorFactory = bindings.drainTimeoutAccumulatorFactory ?? createDrainTimeoutAccumulator; const drainNow = bindings.now ?? Date.now; @@ -1012,6 +1108,7 @@ export function createWorkflowSupervisor( dequeueToProcessing: defaultDequeueToProcessing, markConsumed: defaultMarkConsumed, replayProcessingToInbox: defaultReplayProcessingToInbox, + hasInboxEntries: defaultHasInboxEntries, }; const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { kind: "supervisor", @@ -1083,11 +1180,14 @@ export function createWorkflowSupervisor( let spawnContext: SpawnContext | null = null; /** * The step ids the host derived from the first child's verified - * definition. Set once, while `spawn` is still starting that child, and - * kept for the deployment's life: every later child reports a definition - * with the same approved hash. + * definition. Set once, while `spawn` is still starting that child or + * `restoreInStandby` restores its standby, and kept for the deployment's + * life: every later child reports a definition with the same approved hash. */ let stepIds: readonly string[] | null = null; + // Recorded with a standby, so a restart can lay the steps out without a + // child. + let verifiedDefinition: WorkflowProjectionDefinition | null = null; function requireStepIds(): readonly string[] { if (stepIds === null) { throw new Error( @@ -1102,6 +1202,47 @@ export function createWorkflowSupervisor( // caller owns the contention read because the two paths diverge on // contention (recycle throws, crash-respawn declines silently). let respawnInProgress = false; + // A standby attempt in progress. Dispatch and signals go on until the child + // reports it is quiet; `frozen` then holds them back for the final check, + // and `fence` queues the child's repository writes until the child is + // stopped, so the log the child reported quiet on is the log it leaves. + type StandbyAttempt = { + readonly generation: number; + readonly abandon: AbortController; + readonly outcome: Promise; + readonly settled: Promise; + progress: PromiseWithResolvers; + frozen: PromiseWithResolvers | null; + fence: SubstrateWriteRequest[] | null; + // Set when the host abandons the standby attempt, which leaves nobody else + // to run a child it already stopped. + wakeIfStopped: boolean; + // Set when the child exits on its own while the standby attempt runs. + childExited: boolean; + // What the child reported it is parked on, before the stop. A hub + // re-establishment while the child stops re-registers these. + correlations: ParkedCorrelations | null; + }; + type SubstrateWriteRequest = Extract< + ControlPayload, + { type: "substrate.write.request" } + >["data"]; + let standbyAttempt: StandbyAttempt | null = null; + // While in standby: the correlations the stopped child was parked on, re-sent + // on a hub re-establishment, and the timer for its earliest pending timer. + let inStandby: { + correlations: ParkedCorrelations; + outcome: StandbyOutcome; + wakeTimer: unknown; + } | null = null; + let waking: Promise | null = null; + // Repository writes the child has asked for, so a standby attempt can tell + // that the child wrote after it reported quiet. + let childWrites = 0; + let deliveringSignals = 0; + // The cohort whose dispatch loop is delivering mail. A retired loop that + // finishes late must not clear the mark of the loop that replaced it. + let dispatching: AbortController | null = null; // Monotonic child-cohort generation, bumped atomically with each // transition to `running` (initial spawn and every respawn's // `installNewChild`). The exit-watcher captures the generation it was @@ -1115,6 +1256,10 @@ export function createWorkflowSupervisor( // and drained by `maybeHandleChildExit` once `runRespawn` clears the // latch; a pending exit for a superseded generation is dropped as stale. let pendingChildExit: { generation: number; reason: string } | null = null; + // The generation whose child last exited. While it is still current, the + // child is gone: its respawn waits out the backoff, or it stopped for + // standby. + let exitedGeneration: number | null = null; // Monotonic timestamps (ms, from `crashNow`) of recent unexpected exits, // pruned to `crashLoopWindowMs`. The crash-loop guard latches when the // count within the window reaches `crashLoopMaxCount`. @@ -1181,23 +1326,24 @@ export function createWorkflowSupervisor( // respawn/crash-loop path a clean death takes -- a violation that emits a // garbage frame just before dying must not divert to `stopped` while a // clean SIGKILL respawns. In any other phase the owning lifecycle path - // (spawn handshake, recycle reap, shutdown) owns teardown. + // (spawn handshake, recycle reap, standby, shutdown) owns teardown. function onChildCrash(reason: string): void { + if (state.phase === "standby" && !respawning()) { + logger.info`workflow-process channel ended after the child was put in standby: ${reason}`; + return; + } if (state.phase === "running") { logger.error`workflow-process channel crash on live cohort; forcing child down to respawn: ${reason}`; state.handle.kill(); return; } logger.error`workflow-process channel crash: ${reason}`; - // Only a live, registered supervisor driven down by a channel crash is a - // self-termination the host must reclaim, and that is `recycling`: - // `running` took the kill branch above (its exit reaches the crash-loop - // latch, which carries its own flag), `starting` is the pre-registration - // initial spawn handshake whose failure the deploy unwind owns, and - // `stopping` is a teardown already in flight (a host `shutdown()`, or a - // self-termination already firing). An allowlist, not a denylist, so a - // future phase defaults to no self-terminate rather than a spurious one. - const selfTerminated = state.phase === "recycling"; + // Only a crash during a respawn is a self-termination the host must + // reclaim: `running` took the kill branch above, and its exit reaches the + // crash-loop latch; the deploy unwind owns a failed `starting`, and + // `stopping` is already tearing down. An allowlist, so a new phase defaults + // to no self-termination. + const selfTerminated = respawning(); void shutdownInternal({ reason, selfTerminated }); } @@ -1219,16 +1365,19 @@ export function createWorkflowSupervisor( } // Arm (or re-arm) the stable-run reset for the cohort that just reached - // `running`. If the child is still that same live cohort when the timer - // fires, the run has been stable for `crashLoopStableResetMs` and the - // crash counter is cleared so a flap-then-stabilize sequence does not + // `running`. If that same cohort is still live, or stopped for standby, when + // the timer fires, the run has been stable for `crashLoopStableResetMs` and + // the crash counter is cleared so a flap-then-stabilize sequence does not // latch. A crash before the timer fires re-arms it for the next cohort; // teardown clears it. function armStableRunResetTimer(generation: number): void { clearStableRunResetTimer(); stableRunResetTimer = readySetTimer(() => { stableRunResetTimer = null; - if (generation === childGeneration && state.phase === "running") { + if ( + generation === childGeneration && + (state.phase === "running" || state.phase === "standby") + ) { // The cohort ran stably: reset both the crash counter and the // exponential backoff so a flap followed by stability starts over. crashTimestamps.length = 0; @@ -1293,6 +1442,11 @@ export function createWorkflowSupervisor( // already installed) are dropped. function onChildExited(generation: number, reason: string): void { if (generation !== childGeneration) return; + exitedGeneration = generation; + if (standbyAttempt?.generation === generation) { + standbyAttempt.childExited = true; + noteStandbyProgress(); + } pendingChildExit = { generation, reason }; maybeHandleChildExit(); } @@ -1648,8 +1802,9 @@ export function createWorkflowSupervisor( `inbound mail not accepted: supervisor phase is "${state.phase}"`, ); } + let waiting: boolean; try { - await enqueueInboundMail(rawMessage); + waiting = await enqueueInboundMail(rawMessage); } catch (cause) { // Both branches WITHHOLD (rethrow); the split only sets log severity so // a stale refusal surfaces as its own loud signal rather than blending @@ -1663,9 +1818,17 @@ export function createWorkflowSupervisor( } throw cause; } + // The mail is durable in the inbox; the respawned child dispatches it. A + // redelivery of mail a run already took leaves nothing to wake for. + if (waiting && state.phase === "standby") { + void wakeFromStandby("mail").catch((cause: unknown) => { + logger.error`waking a child in standby for mail failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } } - async function enqueueInboundMail(rawMessage: Uint8Array): Promise { + // Resolves whether the mail waits in the inbox for dispatch. + async function enqueueInboundMail(rawMessage: Uint8Array): Promise { const messageId = await deriveMessageId(rawMessage); const receivedAt = Date.now(); // Inline the raw mail bytes on the claim-check envelope so the @@ -1705,12 +1868,13 @@ export function createWorkflowSupervisor( // the durable delivery, so this never withholds the ack. await commitInboundToMailbox(messageId, rawMessage); wakeDispatch(); - } else { - // A redelivery of a message already durably present: the ack still - // fires (it is on disk), but no new run is dispatched. Surface it so an - // at-least-once redelivery being made effectively-once is observable. - logger.info`inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`; + return true; } + // A redelivery of a message already durably present: the ack still fires + // (it is on disk), but no new run is dispatched. Surface it so an + // at-least-once redelivery being made effectively-once is observable. + logger.info`inbound mail ${messageId} already durably present (${outcome.reason}); acknowledging without re-dispatch`; + return outcome.reason === "duplicate" || outcome.reason === "already_inbox"; } /** @@ -1728,6 +1892,23 @@ export function createWorkflowSupervisor( cohortBroadcaster: TerminalBroadcaster, ): Promise { for await (const payload of iter) { + if (payload.type === "input.taken") { + const input = inputCommits.get(payload.data.requestId); + inputCommits.delete(payload.data.requestId); + // A declined delivery leaves the run waiting on that input. + if (input !== undefined && input !== null && !payload.data.committed) + releaseTakenInput(input); + if (payload.data.requestId === dispatchRequestId) { + // A declined input leaves its mail untaken, which refuses a standby + // attempt as mail still in the inbox does. + if (payload.data.committed) untakenMessageId = null; + dispatchRequestId = null; + dispatchAnswer?.(payload.data.committed); + dispatchAnswer = null; + } + noteStandbyProgress(); + continue; + } if (payload.type === "cancel.prepared") { const pending = pendingCancellations.get(payload.data.requestId); if (pending?.broadcaster === cohortBroadcaster) @@ -1751,6 +1932,11 @@ export function createWorkflowSupervisor( return; } if (payload.type === "substrate.write.request") { + childWrites += 1; + if (standbyAttempt !== null && standbyAttempt.fence !== null) { + standbyAttempt.fence.push(payload.data); + continue; + } // Run the write off the iterator's loop so the iterator can // continue draining other upstream frames (notably the // substrate.merge.response that resolves the merge round-trip @@ -1839,6 +2025,7 @@ export function createWorkflowSupervisor( // runs have no dispatch-loop entry, so their cleanup happens here. cohortRunIds.delete(payload.data.runId); runInputChannels.delete(payload.data.runId); + takenInputs.delete(payload.data.runId); continue; } if (payload.type === "park.notify") { @@ -1902,6 +2089,14 @@ export function createWorkflowSupervisor( consumeOwnedMessages(payload.data.runId); continue; } + if (payload.type === "quiet.response") { + // A response for a query the standby attempt already gave up on is + // dropped. + const settle = pendingQuietQueries.get(payload.data.requestId); + pendingQuietQueries.delete(payload.data.requestId); + settle?.({ ...payload.data, writes: childWrites }); + continue; + } if (payload.type === "parked-correlations.response") { // The child answered a `reEmitParkedCorrelations` query. Resolve the // awaiting driver; a response with no pending entry (the query already @@ -1964,15 +2159,44 @@ export function createWorkflowSupervisor( >(); let cancellationSeq = 0; const cancellationCommits = new Set>(); + // Drain deadlines' cancellations, which outlive the accumulator a standby + // stop or a recycle drops. + const drainCancellations = new Set>(); + // Inputs sent to the child that it has not committed yet, by request, with + // the run input a mail delivery went to. Its writes are dropped when a + // standby attempt stops it, so a standby attempt waits for these. + const inputCommits = new Map(); + // The input of each run that mail was last delivered to. History the child + // read before it committed that input still shows the run waiting on it, so + // a re-registration must not register it again. + const takenInputs = new Map(); + // A spawn's replay of `processing/` still running: mail it moves to the + // inbox after a standby attempt stopped the child would reach no child. + let processingReplayPending = false; + let inputSeq = 0; + // The mail the dispatch took out of the inbox, until the child commits it. A + // standby attempt is refused while there is one: the stop reads only the + // inbox, so nothing would wake the child for it. + let untakenMessageId: string | null = null; + // A failed dispatch or replay can leave mail in processing/ until a + // respawn's replay puts it back. A standby stop would end the recycles that + // respawn the child, so a standby attempt is refused until then. + let strandedMail = false; + let dispatchRequestId: string | null = null; + // Takes the child's answer for the signal the dispatch delivered. + let dispatchAnswer: ((committed: boolean) => void) | null = null; + // The cohort whose dispatch is waiting on its run or on a standby attempt, + // neither of which a standby attempt has to wait out. + let dispatchWaiting: AbortController | null = null; // Mail a resumed run took before its child was replaced. The dispatch that // took it is gone, so it is consumed when the run next parks or ends. const ownedMessages = new Set(); // Retries of a failed consume, which a run that has ended would not // trigger again. const ownedConsumeRetries = new Set(); - // The boot replay. It read history from before any respawn, so a respawn - // waits for it. - let bootReplay: Promise | null = null; + // The boot replay, and whether it went through. It read history from + // before any respawn, so a respawn waits for it. + let bootReplay: Promise | null = null; // Repository writes started off the control pump. Each promise settles // without rejecting; its handler logs any failure. const detachedWrites = new Set>(); @@ -1986,6 +2210,7 @@ export function createWorkflowSupervisor( detachedWrites.add(write); void write.then(() => { detachedWrites.delete(write); + noteStandbyProgress(); }); } @@ -2014,6 +2239,11 @@ export function createWorkflowSupervisor( for (const pending of pendingCancellations.values()) pending.resolve(`cohort aborted: ${reason}`); pendingCancellations.clear(); + // A replaced child commits nothing more, and the next one reports a run + // input it did not commit as one the run still waits on. + for (const input of inputCommits.values()) + if (input !== null) releaseTakenInput(input); + inputCommits.clear(); for (const [requestId, entry] of pendingMerges) { pendingMerges.delete(requestId); entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` }); @@ -2022,10 +2252,15 @@ export function createWorkflowSupervisor( pendingParkedQueries.delete(requestId); entry.settle(null); } + for (const [requestId, settle] of pendingQuietQueries) { + pendingQuietQueries.delete(requestId); + settle(null); + } for (const [runId, resolve] of parkNotifyWaiters.entries()) { parkNotifyWaiters.delete(runId); resolve(); } + noteStandbyProgress(); } function resolveMergeResponse( @@ -2121,6 +2356,18 @@ export function createWorkflowSupervisor( }; const pendingParkedQueries = new Map(); let parkedQuerySeq = 0; + // A child's quiet answer, with how many writes it had asked for when the + // answer arrived: a later write means it is working again. + type QuietAnswer = Extract< + ControlPayload, + { type: "quiet.response" } + >["data"] & { writes: number }; + // Pending `quiet.request`s, settled with `null` when the cohort goes away. + const pendingQuietQueries = new Map< + string, + (answer: QuietAnswer | null) => void + >(); + let quietQuerySeq = 0; function resolveParkedResponse( data: Extract< @@ -2138,6 +2385,14 @@ export function createWorkflowSupervisor( } async function reEmitParkedCorrelations(): Promise { + // A child in standby, or one being stopped for it, cannot be asked; + // re-send what it was parked on. + if (state.phase === "standby") { + const parked = inStandby?.correlations ?? standbyAttempt?.correlations; + if (parked !== null && parked !== undefined) + registerParkedCorrelations(parked); + return; + } // No-op (NOT throw) when the child is not addressable. `deliverSignal` // throws on non-running/starting (including recycling) because it points a // write at the dying child's closing pipe and wants the caller to retry; @@ -2148,7 +2403,17 @@ export function createWorkflowSupervisor( logger.info`reEmitParkedCorrelations: child not addressable (phase=${state.phase}); skipping`; return; } - const controlSender = state.controlSender; + const outcome = await queryParkedCorrelations(state.controlSender); + // A failed query is re-driven by the next re-establishment. + if (typeof outcome !== "string") registerParkedCorrelations(outcome); + } + + // Ask the child which correlations its runs are parked on: `timeout` when it + // does not answer in time, `unreachable` when the send fails or the cohort + // is torn down before it answers. + async function queryParkedCorrelations( + controlSender: ControlChannelSender, + ): Promise { const requestId = `pc-${String((parkedQuerySeq += 1))}`; const responded = new Promise((resolve) => { pendingParkedQueries.set(requestId, { settle: resolve }); @@ -2169,11 +2434,16 @@ export function createWorkflowSupervisor( }); let outcome: ParkedCorrelations | null | "timeout"; try { - await controlSender.send({ - type: "parked-correlations.request", - data: { requestId }, - }); - outcome = await Promise.race([responded, watchdog]); + // A wedged child can block the send too, so the watchdog bounds it. + outcome = await Promise.race([ + controlSender + .send({ + type: "parked-correlations.request", + data: { requestId }, + }) + .then(() => responded), + watchdog, + ]); } catch (cause) { // The downstream send failed (a closing pipe). Best-effort: drop the // pending entry, log, and return; the next re-establishment re-drives. @@ -2181,29 +2451,40 @@ export function createWorkflowSupervisor( if (timeoutHandle !== null) clearTimeout(timeoutHandle); const message = cause instanceof Error ? cause.message : String(cause); logger.warn`reEmitParkedCorrelations query failed: ${message}; re-registration retries on the next re-establishment`; - return; + return "unreachable"; } if (timeoutHandle !== null) clearTimeout(timeoutHandle); - // `timeout` (watchdog fired) or `null` (cohort torn down before the child - // answered): nothing to re-emit; the next re-establishment re-drives. - if (outcome === "timeout" || outcome === null) return; - for (const parked of outcome) { - if (parked.parkKind === "input") { + return outcome ?? "unreachable"; + } + + function registerParkedCorrelations(outcome: ParkedCorrelations): void { + for (const entry of outcome) { + if (entry.parkKind === "input") { + // A supervisor in standby has no cohort to route input to; the + // respawned child's own re-emit registers the channel. + if (state.phase === "standby") continue; + if (takenInputs.get(entry.runId) === entry.correlationId) continue; // Register cohort membership BEFORE the input channel so the run is // never a channel-without-cohort entry: the dispatch loop's routing // hygiene drops exactly such entries, and a live resumed run must not // be mistaken for a dead one and have its channel deleted. - cohortRunIds.add(parked.runId); - runInputChannels.set(parked.runId, { - correlationId: parked.correlationId, + cohortRunIds.add(entry.runId); + runInputChannels.set(entry.runId, { + correlationId: entry.correlationId, parkKind: "input", }); + inputChannelWaiters.get(entry.runId)?.(); } else { - registerSuspension({ ...parked, parkKind: "approval" }); + registerSuspension({ ...entry, parkKind: "approval" }); } } } + // Mail waiting for a live run to show its input channel, by run. A + // re-registration can show it: a run parked before its child started sends no + // park notice. + const inputChannelWaiters = new Map void>(); + function resolveParkNotifyWaiter(runId: string): void { const resolve = parkNotifyWaiters.get(runId); if (resolve === undefined) return; @@ -2650,17 +2931,14 @@ export function createWorkflowSupervisor( ): Promise { const cohort = activeCohort(); if (cohort === null) { - // The request arrived after `activeControlSender()` was - // cleared (the supervisor is in `recycling` mid-swap, or - // `draining`/`stopping`/`stopped`). The child's read end of - // the IPC pipe is being torn down alongside this transition, - // so the child's pending waiter will surface a pipe-close - // error on its own read rather than wedge. Dropping the - // frame here is the only available action -- there is no - // sender to write the response on, and routing the response - // to whatever next-cohort sender exists would deliver it to - // the wrong child. Logged loudly so persistent occurrences - // surface in operator logs. + // The request arrived with no live cohort: the supervisor is + // stopping, stopped, crash-looping or in standby, so the child + // that sent it is gone or going, and its pending waiter fails + // when its pipe closes rather than wedge. Dropping the frame + // here is the only available action -- there is no sender to + // write the response on, and routing the response to a later + // cohort's sender would deliver it to the wrong child. Logged + // loudly so persistent occurrences surface in operator logs. logger.warn`substrate.write.request received outside running phase; requestId=${data.requestId} dropped (child waiter will fail on pipe close)`; return; } @@ -3080,6 +3358,7 @@ export function createWorkflowSupervisor( if (definitionRace.kind === "failed") throw definitionRace.err; childPid = definitionRace.report.childPid; stepIds = await bindings.layoutSteps(definitionRace.report.projection); + verifiedDefinition = definitionRace.report.projection; authority = await readAuthority(); } catch (cause) { await killUnpublishedChild(); @@ -3145,7 +3424,8 @@ export function createWorkflowSupervisor( // teardown: it kills the handle and releases the mail subscription // and address registration installed below. try { - const { replayed: replayDone, sweepDone } = recoverAtBoot(); + const { replayed, sweepDone } = recoverAtBoot({ childRunning: true }); + const replayDone = replayed.then(() => undefined); // Hold the replay promise on the active-state record so // `shutdownInternal` awaits its settlement before tearing the // bindings down. A shutdown that lands while the replay is in @@ -3325,12 +3605,146 @@ export function createWorkflowSupervisor( } } + async function restoreInStandby( + opts: SpawnOpts, + standby: RecordedStandby, + ): Promise { + if (state.phase !== "idle") { + throw new Error( + `supervisor: restoreInStandby called in phase ${state.phase}; expected idle`, + ); + } + // Until the restore publishes `state`, the supervisor still reads as + // idle, so a shutdown cancels it before it publishes, as it does a spawn. + const pending: PendingSpawn = { + cancelled: false, + cancel: Promise.withResolvers(), + done: Promise.withResolvers(), + }; + pendingSpawn = pending; + let startedBy: string | null; + try { + startedBy = await whyRestoreStartsChild(opts, standby); + // No child reports the definition, so the recorded one lays the steps + // out. + if (startedBy === null) { + stepIds = await bindings.layoutSteps(standby.definition); + verifiedDefinition = standby.definition; + } + } finally { + pendingSpawn = null; + pending.done.resolve(undefined); + } + if (pending.cancelled) { + throw new Error( + "workflow-host supervisor: shut down before the restore in standby finished", + ); + } + if (startedBy !== null) { + await spawn(opts); + return { inStandby: false, reason: startedBy }; + } + const now = bindings.recyclePolicyNow ?? defaultNow; + spawnContext = { + definitionHash: opts.definitionHash, + onInferenceEvent: opts.onInferenceEvent, + spawnedAt: now(), + }; + const { replayed, sweepDone } = recoverAtBoot({ childRunning: false }); + const restored: RestoredState = { + mailUnsubscribe: null, + replayDone: replayed.then(() => undefined), + sweepDone, + }; + state = { phase: "standby", restored: true, ...restored }; + inStandby = { + correlations: standby.parked, + outcome: standby.wakeAt === undefined ? {} : { wakeAt: standby.wakeAt }, + wakeTimer: + standby.wakeAt === undefined + ? null + : armStandbyWakeTimer(standby.wakeAt), + }; + try { + bindings.mailBus.registerAddress(bindings.deploymentMailAddress); + state.mailUnsubscribe = bindings.mailBus.subscribeMailForAddress( + bindings.deploymentMailAddress, + onMailMessage, + ); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + await shutdownInternal({ + reason: `restoring in standby failed: ${message}`, + }).catch((shutdownCause: unknown) => { + logger.error`shutdown after a failed restore in standby also threw: ${shutdownCause instanceof Error ? shutdownCause.message : String(shutdownCause)}`; + }); + throw cause; + } + // Mail the replay put back or that already waited wakes the child, and so + // does a replay that failed, which can leave mail behind. + void replayed + .then(async (ok) => { + // A shutdown during the replay leaves nothing to wake. + if (state.phase !== "standby") return; + const mailWaiting = + !ok || + (await inboxHoldsMail().catch((cause: unknown) => { + if (state.phase !== "standby") return false; + logger.error`reading the inbox of a deployment restored in standby failed; waking it in case mail waits there: ${cause instanceof Error ? cause.message : String(cause)}`; + return true; + })); + if (!mailWaiting) return; + await wakeFromStandby( + ok ? "mail" : "its restart, which could not recover its mail", + ); + }) + .catch((cause: unknown) => { + logger.error`waking a deployment restored in standby for its mail failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + return { inStandby: true }; + } + + // Why a restored standby must start its child, or null while its record + // holds. History written since the recorded tip, by a child that ran again + // or a cancellation, can make the recorded parks stale; the child reads its + // own. The recorded definition gets the check a child's report gets. + async function whyRestoreStartsChild( + opts: SpawnOpts, + standby: RecordedStandby, + ): Promise { + try { + if ((await readHistoryTip()) !== standby.tip) { + return "its restart, which found history written after it stopped"; + } + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + return `its restart, which could not read its history: ${message}`; + } + if ( + (await computeWireDefinitionHash(standby.definition)) !== + opts.definitionHash + ) { + return "its restart, which found a recorded definition that is not the approved one"; + } + return null; + } + + // The run's history ref, where its steps record what they are parked on and + // the timers they wait for. + function readHistoryTip(): Promise { + return bindings.repoStore.resolveRef( + supervisorPrincipal, + bindings.workflowRunRepoId, + bindings.workflowRunRef, + ); + } + // Move `processing/` entries a crashed incarnation left without an owner back // to `inbox/` in arrival order. The write can wait on a slow Hub, so it runs // in the background, and dispatch awaits it before its first dequeue so new // mail cannot overtake them. The compaction sweep shares its scan of `runs/`. - function recoverAtBoot(): { - replayed: Promise; + function recoverAtBoot(opts: { childRunning: boolean }): { + replayed: Promise; sweepDone: Promise; } { const scanDone = scanRunsForBoot( @@ -3339,6 +3753,7 @@ export function createWorkflowSupervisor( bindings.workflowRunRepoId, bindings.workflowRunRef, ); + processingReplayPending = true; const replayed = scanDone .then(({ ownedMessageIds }) => inboxPrimitives.replayProcessingToInbox( @@ -3350,16 +3765,24 @@ export function createWorkflowSupervisor( ), ) .then(({ retainedMessageIds }) => { - keepOwnedMessages(retainedMessageIds, { childRunning: true }); + keepOwnedMessages(retainedMessageIds, opts); wakeDispatch(); + return true; }) .catch((cause: unknown) => { // Best-effort: entries a failed replay leaves in `processing/` lose // their place to newer mail. Making this fatal crashed first spawns // with no `processing/` directory; that needs a replay that allows - // one, or a periodic sweep for orphans. + // one, or a periodic sweep for orphans. A stopped child would leave + // them with nothing to bring them back, so standby waits for a respawn. const message = cause instanceof Error ? cause.message : String(cause); logger.warn`boot recovery scan or processing replay failed: ${message}`; + strandedMail = true; + return false; + }) + .finally(() => { + processingReplayPending = false; + noteStandbyProgress(); }); bootReplay = replayed; @@ -3515,15 +3938,23 @@ export function createWorkflowSupervisor( runId: string, payload: Mail, ): Promise { - await sender.send({ - type: "trigger.fire", - data: { - runId, - messageId, - receivedAt, - payload, - }, - }); + const requestId = nextInputRequest(null); + dispatchRequestId = requestId; + try { + await sender.send({ + type: "trigger.fire", + data: { + runId, + messageId, + receivedAt, + requestId, + payload, + }, + }); + } catch (cause) { + inputCommits.delete(requestId); + throw cause; + } cohortRunIds.add(runId); return runId; } @@ -3691,6 +4122,7 @@ export function createWorkflowSupervisor( const envelope = dequeued.envelope; const runId = deriveWorkflowRunId(bindings.deploymentMailAddress); const messageId = envelope.messageId; + untakenMessageId = messageId; let rejection: | { code: string; @@ -3740,14 +4172,7 @@ export function createWorkflowSupervisor( if (!cohortRunIds.has(runId)) { const lifecycle = terminalRunIds.has(runId) ? "terminal" - : await readCommittedWorkflowRunLifecycle( - await bindings.repoStore.openCommittedReads( - supervisorPrincipal, - bindings.workflowRunRepoId, - bindings.workflowRunRef, - ), - runId, - ); + : await readRunLifecycle(runId); if (lifecycle === "terminal") { rejectTerminalRun(); } else if (lifecycle === "live") { @@ -3775,6 +4200,7 @@ export function createWorkflowSupervisor( // cleaned up by pumpUpstreamControl, but synthetic ones are not). cohortRunIds.delete(runId); runInputChannels.delete(runId); + takenInputs.delete(runId); } else { // Dispose the pre-created iterator; the normal path creates fresh // iterators inside the dispatch-decision loop when waiting. @@ -3829,18 +4255,35 @@ export function createWorkflowSupervisor( // a terminal the resumed run reaches right after applying the signal // is not missed; the park watcher is armed inside // waitForRunTerminalOrPark. + if (!(await deliveryAllowed(cohortAbort))) break; const iter = broadcaster.source(runId)[Symbol.asyncIterator](); let waitEntered = false; try { - await sender.send({ - type: "signal.deliver", - data: { - runId, - signalName: signalName(inputChannel.correlationId), - signalId: envelope.messageId, - payload: prepared.mail, - }, - }); + const input = { + runId, + correlationId: inputChannel.correlationId, + }; + const requestId = nextInputRequest(input); + dispatchRequestId = requestId; + const answer = Promise.withResolvers(); + dispatchAnswer = answer.resolve; + takenInputs.set(runId, input.correlationId); + await sender + .send({ + type: "signal.deliver", + data: { + runId, + signalName: signalName(inputChannel.correlationId), + signalId: envelope.messageId, + payload: prepared.mail, + requestId, + }, + }) + .catch((cause: unknown) => { + inputCommits.delete(requestId); + releaseTakenInput(input); + throw cause; + }); // Invalidate the cached input channel: its correlation is now // consumed by this delivery, so the NEXT mail must not reuse it. // The resumed run re-parks on a FRESH correlation (a new @@ -3853,27 +4296,52 @@ export function createWorkflowSupervisor( // entry \Seen/$Processed. Fire-and-forget off the dispatch path. markMailboxProcessed(envelope.messageId); // Durable-consume contract, mirroring the trigger.fire path: hold - // markConsumed until the child has durably taken up the signal -- - // the resumed run re-parks or reaches a terminal event. That gate - // is downstream of durability DESPITE the child's fire-and-forget - // SignalReceived writer: the runtime reaches re-park/terminal only - // by resuming from the COMMITTED SignalReceived, which its per-run - // subscribeKind substrate subscription surfaces only after the - // commit lands -- so the substrate subscription IS the ack, and a - // failed deliver commit is observed by nothing, never re-parks, and - // never releases markConsumed (the mail stays reclaimable). A crash + // markConsumed until the child has committed the signal and the + // run parks or reaches a terminal event. Another step's park ends + // this wait too, and a run that ends may never have taken the + // signal, so only the child's answer says who holds the mail. A + // signal it declined fails the dispatch, which leaves the mail + // reclaimable, or is rejected if the run ended. A crash // before the child commits the signal leaves the claim-check entry // in processing/ for replayProcessingToInbox to re-deliver; after // it, the resumed run owns the mail and the supervisor consumes it // once the run next parks or ends. On cohort abort the wait returns - // and the post-loop guard skips markConsumed. + // and the post-loop guard skips markConsumed. A standby attempt + // does not wait out this wait, for the same reason: once the + // child has committed the signal, the run owns the mail. waitEntered = true; - await waitForRunTerminalOrPark( - iter, - cohortAbort.signal, + const declined = new AbortController(); + void answer.promise.then((committed) => { + if (!committed) declined.abort(); + }); + const outcome = await whileDispatchWaits( + cohortAbort, + waitForRunTerminalOrPark( + iter, + AbortSignal.any([cohortAbort.signal, declined.signal]), + runId, + sinceGen, + ), + ); + const committed = await waitForSignalAnswer( + cohortAbort, + answer.promise, runId, - sinceGen, ); + if (committed === false) { + // The child's terminal frame can follow its answer, so the + // committed log says whether the run ended. + if ( + outcome !== "terminal" && + !terminalRunIds.has(runId) && + (await readRunLifecycle(runId)) !== "terminal" + ) { + throw new Error( + `the workflow child declined mail ${messageId} for live run ${runId}; it stays in processing/ until a respawn replays it`, + ); + } + rejectTerminalRun(); + } } finally { // waitForRunTerminalOrPark finalizes the iterator it consumes; the // only leak is when `sender.send` throws before the wait is @@ -3908,6 +4376,7 @@ export function createWorkflowSupervisor( // before-fire contract), so a terminal that lands while // forwardDispatchedEntry is in flight would be lost and the wait // would hang to the backstop. + if (!(await deliveryAllowed(cohortAbort))) break; const iter = broadcaster.source(runId)[Symbol.asyncIterator](); let waitEntered = false; try { @@ -3931,13 +4400,17 @@ export function createWorkflowSupervisor( // committed, so a crash before RunStarted leaves the entry in // processing/ for replayProcessingToInbox to re-deliver. On cohort // abort the wait returns and the post-loop guard skips - // markConsumed. + // markConsumed. As on the signal path, a standby attempt does not + // wait this out once the child has committed `RunStarted`. waitEntered = true; - await waitForRunTerminalOrPark( - iter, - cohortAbort.signal, - runId, - sinceGen, + await whileDispatchWaits( + cohortAbort, + waitForRunTerminalOrPark( + iter, + cohortAbort.signal, + runId, + sinceGen, + ), ); } finally { // waitForRunTerminalOrPark finalizes the iterator it consumes; the @@ -3951,14 +4424,33 @@ export function createWorkflowSupervisor( } break; } + // The mail waits here undelivered, and a standby attempt is refused + // until the child takes it. A re-registration can show the input + // channel instead of a park notice. const iter = broadcaster.source(runId)[Symbol.asyncIterator](); - const outcome = await waitForRunTerminalOrPark( - iter, - cohortAbort.signal, - runId, - sinceGen, - ); - if (outcome === "aborted") break; + const channelShown = new AbortController(); + inputChannelWaiters.set(runId, () => channelShown.abort()); + bindings.onDispatchWait?.(runId); + let outcome: Awaited>; + try { + outcome = await whileDispatchWaits( + cohortAbort, + waitForRunTerminalOrPark( + iter, + AbortSignal.any([cohortAbort.signal, channelShown.signal]), + runId, + sinceGen, + ), + ); + } finally { + inputChannelWaiters.delete(runId); + } + if (outcome === "aborted") { + // A channel shown first re-evaluates the run, now with its input. + if (channelShown.signal.aborted && !cohortAbort.signal.aborted) + continue; + break; + } if (outcome === "terminal") { // This message was waiting for an already-live run to expose its // next input correlation. The run terminated first, so the mail @@ -4144,6 +4636,48 @@ export function createWorkflowSupervisor( } } + async function readRunLifecycle(runId: string) { + return readCommittedWorkflowRunLifecycle( + await bindings.repoStore.openCommittedReads( + supervisorPrincipal, + bindings.workflowRunRepoId, + bindings.workflowRunRef, + ), + runId, + ); + } + + // The child's answer for a signal a dispatch delivered, or `undefined` once + // the cohort aborts. A child that never answers fails the dispatch, as the + // park wait's backstop does, so the mail stays reclaimable. + async function waitForSignalAnswer( + cohortAbort: AbortController, + answered: Promise, + runId: string, + ): Promise { + let timeoutHandle: ReturnType | undefined; + const backstop = new Promise<"timeout">((resolve) => { + timeoutHandle = setTimeout( + () => resolve("timeout"), + TERMINAL_OR_PARK_BACKSTOP_MS, + ); + }); + try { + const answer = await whileDispatchWaits( + cohortAbort, + raceAbort(cohortAbort.signal, Promise.race([answered, backstop])), + ); + if (answer === "timeout") { + throw new Error( + `the workflow child did not answer for a signal to run ${runId} within ${TERMINAL_OR_PARK_BACKSTOP_MS}ms`, + ); + } + return answer; + } finally { + clearTimeout(timeoutHandle); + } + } + /** * The dispatch loop body. Runs until the cohort aborts; each * iteration drains one inbox entry through the FIFO claim-check @@ -4177,36 +4711,104 @@ export function createWorkflowSupervisor( // dispatchOne would await the fresh, unresolved promise and strand that // mail until some later wake. const wake = dispatchWake.promise; + const frozen = standbyAttempt === null ? null : standbyAttempt.frozen; + if (frozen !== null) { + // A standby attempt's final check takes no new mail: the child takes it + // once the check fails, or is respawned for it once stopped. + bindings.onStandbyWait?.("input"); + await raceAbort(cohortAbort.signal, frozen.promise); + continue; + } let dispatched: boolean; + dispatching = cohortAbort; try { dispatched = await dispatchOne(sender, cohortAbort, broadcaster); } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); logger.error`dispatch loop iteration failed: ${message}`; + if (!cohortAbort.signal.aborted && untakenMessageId !== null) { + strandedMail = true; + } // A failure deep inside the substrate is one the operator // must see. The loop continues -- a transient failure should // not wedge the deployment -- but the loop pauses on the // wake so we do not busy-spin against a persistent fault. dispatched = false; + } finally { + if (dispatching === cohortAbort) dispatching = null; + // A stopped child leaves its untaken mail for the wake. + if (!cohortAbort.signal.aborted) { + untakenMessageId = null; + dispatchRequestId = null; + dispatchAnswer = null; + } + noteStandbyProgress(); } if (dispatched) continue; if (cohortAbort.signal.aborted) return; - const abortPromise = new Promise((resolve) => { - if (cohortAbort.signal.aborted) { - resolve(); - return; - } - cohortAbort.signal.addEventListener("abort", () => resolve(), { - once: true, - }); - }); - await Promise.race([wake, abortPromise]); + await raceAbort(cohortAbort.signal, wake); } } + async function commitCancellation( + opts: CancelRequestOpts, + ): Promise { + const committing = commitCancelRequested({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + ref: bindings.workflowRunRef, + anchorRunId: bindings.anchorRunId, + runId: opts.runId, + origin: opts.origin, + reason: opts.reason, + at: opts.at, + signAsPrincipal: bindings.signAsPrincipal, + }); + cancellationCommits.add(committing); + try { + const result = await committing; + return { commitSha: result.commitSha, seq: result.seq }; + } finally { + cancellationCommits.delete(committing); + noteStandbyProgress(); + } + } + + // A drain deadline's cancellation. Like a requested one it calls off a + // standby attempt, as the child that attempt would stop is the one that + // applies it, and a child that stopped anyway is woken to apply it. + function commitDrainCancellation(opts: CancelRequestOpts): Promise { + const committing = (async () => { + const attempt = standbyAttempt; + if (attempt !== null) { + abandonStandby(attempt, "the drain deadline passed"); + await attempt.settled; + } + await commitCancellation(opts); + if (state.phase === "standby") { + void wakeFromStandby("cancel").catch((cause: unknown) => { + logger.error`waking a child in standby for a drain cancellation failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } + })(); + drainCancellations.add(committing); + void committing.then( + () => drainCancellations.delete(committing), + () => drainCancellations.delete(committing), + ); + return committing; + } + async function requestCancel( opts: CancelRequestOpts, ): Promise { + const attempt = standbyAttempt; + if (attempt !== null) { + abandonStandby(attempt, "a cancellation was requested"); + await attempt.settled; + } + // A wake in progress has no child to ask yet; the woken one is. + if (waking !== null) await waking; if ( state.phase === "stopping" || state.phase === "stopped" || @@ -4214,33 +4816,22 @@ export function createWorkflowSupervisor( ) { throw new Error("Cannot cancel a stopped workflow supervisor"); } - const commitCancellation = async () => { - const committing = commitCancelRequested({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, - runId: opts.runId, - origin: opts.origin, - reason: opts.reason, - at: opts.at, - signAsPrincipal: bindings.signAsPrincipal, - }); - cancellationCommits.add(committing); - try { - const result = await committing; - return { commitSha: result.commitSha, seq: result.seq }; - } finally { - cancellationCommits.delete(committing); - } - }; const cohort = state; + if (cohort.phase === "standby") { + // A child in standby cannot settle the run; the respawned one applies the + // committed cancellation when it resumes. + const committed = await commitCancellation(opts); + void wakeFromStandby("cancel").catch((cause: unknown) => { + logger.error`waking a child in standby for a cancellation failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + return committed; + } if ( cohort.phase !== "starting" && cohort.phase !== "running" && cohort.phase !== "recycling" ) - return commitCancellation(); + return commitCancellation(opts); const sender = cohort.controlSender; const requestId = `cancel-${String((cancellationSeq += 1))}`; @@ -4259,7 +4850,7 @@ export function createWorkflowSupervisor( cohort.terminalCohortAbort.signal.throwIfAborted(); if (activeControlSender() !== sender) throw new Error("Cancellation's workflow child was replaced"); - const result = await commitCancellation(); + const result = await commitCancellation(opts); await sender .send({ type: "cancel.committed", data: { requestId } }) .catch((cause: unknown) => { @@ -4279,6 +4870,7 @@ export function createWorkflowSupervisor( throw cause; } finally { pendingCancellations.delete(requestId); + noteStandbyProgress(); } } @@ -4331,6 +4923,29 @@ export function createWorkflowSupervisor( async function performShutdown(opts: ShutdownOptions): Promise { const prior = state; state = { phase: "stopping" }; + // The child's cohort, if any, and what the deployment holds either way: a + // restored standby holds the mail subscription and boot recovery alone. + const cohort = + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" || + (prior.phase === "standby" && !prior.restored) + ? prior + : null; + const held = cohort ?? (prior.phase === "standby" ? prior : null); + // Waited for below, so a stop it already made is recorded before + // shutdown resolves. + const abandonedAttempt = standbyAttempt; + if (abandonedAttempt !== null) { + abandonStandby( + abandonedAttempt, + "the workflow supervisor is shutting down", + ); + } + if (inStandby !== null) { + if (inStandby.wakeTimer !== null) readyClearTimer(inStandby.wakeTimer); + inStandby = null; + } for (const retry of ownedConsumeRetries) readyClearTimer(retry); ownedConsumeRetries.clear(); // shutdownInternal is designed to be TOTAL: when a child is up it must @@ -4350,12 +4965,7 @@ export function createWorkflowSupervisor( // invariant survives if that ever changes.) const accumulatorsToDispose = [...drainAccumulators.values()]; const childrenToStop = new Set(uninstalledChildren); - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) - childrenToStop.add(prior.handle); + if (cohort !== null) childrenToStop.add(cohort.handle); const childTerminations: Promise[] = []; let killRequested = false; function killChildren(): void { @@ -4381,28 +4991,14 @@ export function createWorkflowSupervisor( // Stop every armed drainTimeout accumulator before tearing the child // down. An accumulator left running would otherwise fire its // `setTimeout` callback (or its terminal-event watcher's settle hook) - // against a shutdown-mid-flight supervisor. Guard each `stop` so one - // throwing accumulator does not leave the rest armed. - for (const accumulator of accumulatorsToDispose) { - try { - accumulator.stop(); - } catch (cause) { - const message = - cause instanceof Error ? cause.message : String(cause); - logger.warn`drain accumulator stop threw during shutdown: ${message}`; - } - } - drainAccumulators.clear(); + // against a shutdown-mid-flight supervisor. + stopDrainAccumulators("shutdown"); cohortRunIds.clear(); runInputChannels.clear(); parkNotifyWaiters.clear(); parkGenerations.clear(); - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - prior.terminalCohortAbort.abort(); + if (cohort !== null) { + cohort.terminalCohortAbort.abort(); // Reject every pending merge round-trip and park-notify waiter // so handler closures awaiting them (including fire-and-forget // `handleSubstrateWriteRequest` instances) cannot outlive the @@ -4414,7 +5010,7 @@ export function createWorkflowSupervisor( // with `done: true` -- the dispatch loop's `waitForRunTerminal` // and any drainTimeout watcher unblock through the same shutdown // path the cohort abort drives. - prior.terminalBroadcaster.dispose(); + cohort.terminalBroadcaster.dispose(); // Wake the dispatch loop so its `dispatchWake` await settles // and the loop notices the cohort abort. Without the wake, the // loop's `Promise.race` would sit on the wake promise until @@ -4437,45 +5033,43 @@ export function createWorkflowSupervisor( }), ), ); + while (drainCancellations.size > 0) { + bindings.onShutdownWait?.("cancellation"); + await Promise.allSettled([...drainCancellations]); + } // Includes a loop a recycle has retired from `state`, so its last // consumption write lands before shutdown resolves. Each loop's own // logger surfaces its failure. await Promise.allSettled([...dispatchLoops]); - if ( - (prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling") && - prior.replayDone !== null - ) { + if (held !== null && held.replayDone !== null) { // Await the spawn-time replayProcessingToInbox before tearing // the bindings down. The replay's substrate write // (`processing/` -> `inbox/` rename via a tree commit) must // settle before the supervisor's exit; without the await the // substrate I/O outlives the supervisor and a subsequent boot // can observe a partially-applied replay. - await prior.replayDone.catch(() => { + await held.replayDone.catch(() => { /* swallowed: the replay's own catch already surfaces the failure to the supervisor's warn channel; the shutdown path only waits for the substrate write to settle. */ }); } - if ( - (prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling") && - prior.sweepDone !== null - ) { + if (held !== null && held.sweepDone !== null) { // Await the spawn-time compaction sweep before teardown so an // in-flight fold's substrate commit does not outlive the supervisor // and interleave with the next incarnation's boot. Teardown latency // is bounded by the recovery backlog (see the `sweepDone` field // docstring); a normal boot has zero or one pending fold. - await prior.sweepDone.catch(() => { + await held.sweepDone.catch(() => { /* swallowed: the sweep's own catch already surfaces failures to the supervisor's warn channel; the shutdown path only waits for the in-flight fold's substrate commit to settle. */ }); } + if (abandonedAttempt !== null) { + bindings.onShutdownWait?.("attempt"); + await abandonedAttempt.settled; + } if (respawnReplays.size > 0) { bindings.onShutdownWait?.("replay"); await Promise.allSettled([...respawnReplays]); @@ -4518,14 +5112,10 @@ export function createWorkflowSupervisor( cancelRespawnBackoffWaits(); pendingChildExit = null; spawnContext = null; - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - if (prior.mailUnsubscribe !== null) { + if (held !== null) { + if (held.mailUnsubscribe !== null) { try { - prior.mailUnsubscribe(); + held.mailUnsubscribe(); } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); @@ -4554,12 +5144,8 @@ export function createWorkflowSupervisor( }), ), ); - if ( - prior.phase === "starting" || - prior.phase === "running" || - prior.phase === "recycling" - ) { - await prior.eventPump.catch(() => { + if (cohort !== null) { + await cohort.eventPump.catch(() => { /* swallowed for the same reason as above. */ }); } @@ -4636,10 +5222,15 @@ export function createWorkflowSupervisor( // tick. The supervisor never blocks on the child's acknowledgement // -- the accumulator below is the deadline-keeper, not the round // trip. - await state.controlSender.send({ + const sender = state.controlSender; + await sender.send({ type: "drain", data: { deadlineMs: opts.deadlineMs }, }); + // A standby stop or a respawn during the send already dropped the drain + // deadlines, and the child they were for is gone. + const cohort = activeCohort(); + if (cohort === null || cohort.controlSender !== sender) return; // Arm one accumulator per in-flight run. Each accumulator's // `escalate` path commits a signed `CancelRequested{origin: // "supervisor-drain"}` against the workflow-run repo via the @@ -4653,8 +5244,8 @@ export function createWorkflowSupervisor( // new mail. The cohort abort on shutdown/recycle will eventually // tear the run down. const cohortSource = perCohortTerminalSource( - state.terminalCohortAbort, - state.terminalBroadcaster, + cohort.terminalCohortAbort, + cohort.terminalBroadcaster, ); for (const runId of cohortRunIds) { if (drainAccumulators.has(runId)) continue; @@ -4665,16 +5256,7 @@ export function createWorkflowSupervisor( now: drainNow, setTimer: drainSetTimer, clearTimer: drainClearTimer, - commitCancel: (cancel) => - commitCancelRequested({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, - runId, - ...cancel, - signAsPrincipal: bindings.signAsPrincipal, - }), + commitCancel: (cancel) => commitDrainCancellation({ runId, ...cancel }), ...(cohortSource !== null ? { terminalEventSource: cohortSource } : {}), }); drainAccumulators.set(runId, accumulator); @@ -4718,20 +5300,602 @@ export function createWorkflowSupervisor( }); } + // `setTimeout` fires at once past this many ms, so a far wake is re-armed. + const MAX_TIMER_MS = 2_147_483_647; + + function noteStandbyProgress(): void { + const attempt = standbyAttempt; + if (attempt === null) return; + const reached = attempt.progress; + attempt.progress = Promise.withResolvers(); + reached.resolve(undefined); + } + + function abandonStandby(attempt: StandbyAttempt, reason: string): void { + if (!attempt.abandon.signal.aborted) { + attempt.abandon.abort(new StandbyAbandonedError(reason)); + } + } + + function enterStandby(): Promise { + if ( + state.phase === "idle" || + state.phase === "stopping" || + state.phase === "stopped" || + state.phase === "crash-looping" + ) { + return Promise.reject( + new StandbyRefusedError(`the workflow child is ${state.phase}`), + ); + } + // A child being woken is past the standby attempt it had. + if (waking !== null || respawnInProgress) { + return Promise.reject( + new StandbyDeferredError("the workflow child is restarting"), + ); + } + // An attempt still settling answers for itself, even once the child it + // stopped is recorded. + if (standbyAttempt !== null) { + // An abandoned attempt refuses with its own reason, which is not this + // request's; asked again once it settles, a fresh attempt starts. + return standbyAttempt.abandon.signal.aborted + ? Promise.reject( + new StandbyDeferredError( + "an abandoned standby attempt is still settling", + ), + ) + : standbyAttempt.outcome; + } + if (state.phase === "standby" && inStandby !== null) { + return Promise.resolve(inStandby.outcome); + } + // A child that exited is replaced once its respawn backoff passes. + if (state.phase !== "running" || exitedGeneration === childGeneration) { + return Promise.reject( + new StandbyDeferredError("the workflow child is restarting"), + ); + } + const done = Promise.withResolvers(); + const attempt: StandbyAttempt = { + generation: childGeneration, + abandon: new AbortController(), + outcome: done.promise, + settled: done.promise.then( + () => undefined, + () => undefined, + ), + progress: Promise.withResolvers(), + frozen: null, + fence: null, + wakeIfStopped: false, + childExited: false, + correlations: null, + }; + standbyAttempt = attempt; + void (async () => { + let outcome: StandbyOutcome | undefined; + let failure: unknown; + try { + outcome = await runStandbyAttempt(attempt); + } catch (cause) { + failure = cause; + } + // Cleared before the outcome settles, so the dispatch loop that wakes + // on it sees no standby attempt. + if (standbyAttempt === attempt) standbyAttempt = null; + wakeDispatch(); + if (outcome !== undefined) done.resolve(outcome); + else done.reject(failure); + })(); + return done.promise; + } + + function abandonCurrentStandby(reason: string): void { + if (standbyAttempt === null) return; + standbyAttempt.wakeIfStopped = true; + abandonStandby(standbyAttempt, reason); + } + + async function wake(): Promise { + const attempt = standbyAttempt; + if (attempt !== null) { + abandonStandby(attempt, "the deployment was woken"); + await attempt.settled; + } + // A standby attempt past its final check stops the child anyway. + await wakeFromStandby(REQUESTED_WAKE); + } + + async function runStandbyAttempt( + attempt: StandbyAttempt, + ): Promise { + const runId = deriveWorkflowRunId(bindings.deploymentMailAddress); + try { + for (;;) { + await untilStandbyQuiet(attempt); + // Wait for the child to report it is quiet while it still takes input. + const idle = await askChildQuiet(attempt, runId); + if (!idle.quiet) continue; + freezeStandby(attempt); + const deadline = waitDeadline(readySetTimer, QUIET_CHECK_WATCHDOG_MS); + try { + if (!(await untilStandbyQuiet(attempt, deadline.promise))) continue; + const final = await askChildQuiet(attempt, runId, deadline.promise); + if (!final.quiet) continue; + attempt.fence = []; + if ( + !(await untilDetachedWritesLand(deadline.promise)) || + childWrites !== final.writes || + !isStandbyQuiet() + ) { + releaseStandbyFence(attempt); + continue; + } + return await stopChildForStandby( + attempt, + final.wakeAt === undefined ? {} : { wakeAt: final.wakeAt }, + ); + } finally { + readyClearTimer(deadline.handle); + thawStandby(attempt); + } + } + } catch (cause) { + releaseStandbyFence(attempt); + throw cause; + } + } + + // Hold dispatch and signals back for a standby attempt's final check. + function freezeStandby(attempt: StandbyAttempt): void { + if (attempt.frozen === null) { + attempt.frozen = Promise.withResolvers(); + } + } + + function thawStandby(attempt: StandbyAttempt): void { + const frozen = attempt.frozen; + attempt.frozen = null; + frozen?.resolve(undefined); + } + + // Whether the supervisor went quiet, which `deadline` can cut short. + async function untilStandbyQuiet( + attempt: StandbyAttempt, + deadline?: Promise, + ): Promise { + const expired = deadline?.then(() => false as const); + for (;;) { + const progress = attempt.progress.promise; + currentStandbyCohort(attempt); + if (isStandbyQuiet()) return true; + bindings.onStandbyWait?.("standby"); + const going = await Promise.race([ + untilStandbyProgress(attempt, progress).then(() => true as const), + ...(expired === undefined ? [] : [expired]), + ]); + if (!going) return false; + } + } + + async function untilDetachedWritesLand( + deadline: Promise, + ): Promise { + const expired = deadline.then(() => false as const); + while (detachedWrites.size > 0) { + const landed = await Promise.race([ + Promise.all([...detachedWrites]).then(() => true as const), + expired, + ]); + if (!landed) return false; + } + return true; + } + + // Ask the child whether every run only waits on input from outside. Without + // a `deadline` the child answers once it is; with one, a child that does not + // answer in time counts as working. The child answers only the latest check, + // so this is the one sender and one standby attempt asks at a time. + async function askChildQuiet( + attempt: StandbyAttempt, + runId: string, + deadline?: Promise, + ): Promise<{ quiet: boolean; wakeAt?: string; writes: number }> { + const cohort = currentStandbyCohort(attempt); + const requestId = `quiet-${String((quietQuerySeq += 1))}`; + const answered = new Promise((resolve) => { + pendingQuietQueries.set(requestId, resolve); + }); + const wait = deadline === undefined; + try { + // A send fails only once the child is going away. A wedged child can + // block it instead, which the abandon and the deadline must still end. + const sent = cohort.controlSender + .send({ type: "quiet.request", data: { requestId, runId, wait } }) + .catch((cause: unknown) => { + throw new StandbyDeferredError( + `the workflow child could not be asked whether it is quiet: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + }); + const answer = await raceAbort( + attempt.abandon.signal, + Promise.race([ + answered, + sent.then(() => answered), + ...(deadline === undefined + ? [] + : [deadline.then(() => "late" as const)]), + ]), + ); + currentStandbyCohort(attempt); + if (answer === undefined || answer === "late") { + return { quiet: false, writes: childWrites }; + } + if (answer === null) { + throw new StandbyDeferredError( + "the workflow child restarted while entering standby", + ); + } + if ("ended" in answer) { + throw new StandbyRefusedError("the workflow run has ended"); + } + if (answer.error !== undefined) { + throw new StandbyRefusedError( + `the workflow child could not check that it is quiet: ${answer.error}`, + ); + } + return answer; + } finally { + pendingQuietQueries.delete(requestId); + } + } + + // The cohort a standby attempt is for, or a refusal when it has been + // abandoned or the child it was for is gone. + function currentStandbyCohort( + attempt: StandbyAttempt, + ): Extract { + attempt.abandon.signal.throwIfAborted(); + if ( + state.phase !== "running" || + childGeneration !== attempt.generation || + attempt.childExited + ) { + throw new StandbyDeferredError( + "the workflow child restarted while entering standby", + ); + } + return state; + } + + // Nothing the supervisor drives for the child is in flight. + function isStandbyQuiet(): boolean { + return ( + (dispatching === null || dispatchWaiting === dispatching) && + detachedWrites.size === 0 && + pendingCancellations.size === 0 && + cancellationCommits.size === 0 && + deliveringSignals === 0 && + inputCommits.size === 0 && + !processingReplayPending + ); + } + + async function untilStandbyProgress( + attempt: StandbyAttempt, + progress: Promise, + ): Promise { + await raceAbort(attempt.abandon.signal, progress); + attempt.abandon.signal.throwIfAborted(); + } + + // Apply the writes the fence held, in the order the child sent them. A child + // that exited never has them land, as a completed stop drops them. + function releaseStandbyFence(attempt: StandbyAttempt): void { + const held = attempt.fence; + attempt.fence = null; + if (held === null || attempt.childExited) return; + for (const data of held) { + ownDetachedWrite( + handleSubstrateWriteRequest(data).catch((cause) => { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.error`substrate.write.request handler crashed: ${message}`; + }), + ); + } + } + + function inboxHoldsMail(): Promise { + return inboxPrimitives.hasInboxEntries( + bindings.repoStore, + inboxWritePrincipal, + bindings.workflowRunRepoId, + bindings.deploymentMailAddress, + ); + } + + async function stopChildForStandby( + attempt: StandbyAttempt, + outcome: StandbyOutcome, + ): Promise { + const cohort = currentStandbyCohort(attempt); + const record = bindings.recordStandby; + // Read before the child reports its parks, so history written after the + // read ends a standby restored from the record. + let tip: string | null = null; + if (record !== undefined) { + try { + tip = await readHistoryTip(); + } catch (cause) { + throw new StandbyRefusedError( + `the run's history could not be read: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + } + currentStandbyCohort(attempt); + } + const correlations = await queryParkedCorrelations(cohort.controlSender); + currentStandbyCohort(attempt); + if (correlations === "unreachable") { + throw new StandbyDeferredError( + "the workflow child could not be asked for the correlations it is parked on", + ); + } + // A stopped child's parks are registered again from this record alone, so + // a child that does not report them keeps running. + if (correlations === "timeout") { + throw new StandbyRefusedError( + "the workflow child did not report the correlations it is parked on", + ); + } + attempt.correlations = correlations; + // Mail the dispatch cannot hand over yet, such as mail behind a run that + // waits on an approval or a signal. Mail still in the inbox would wake the + // child as soon as it stopped, and nothing would wake it for mail the + // dispatch took that the child has not committed. + if (untakenMessageId !== null) { + throw new StandbyRefusedError( + "mail the dispatch took is still waiting for the child", + ); + } + if (strandedMail) { + throw new StandbyRefusedError( + "mail a failed dispatch or replay left waits in processing/ for the child to restart", + ); + } + const mailQueued = await inboxHoldsMail().catch((cause: unknown) => { + throw new StandbyRefusedError( + `the deployment's inbox could not be read: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + }); + if (mailQueued) { + throw new StandbyRefusedError( + "mail is waiting in the deployment's inbox", + ); + } + const live = currentStandbyCohort(attempt); + // From here the child's exit is planned, not a crash. + state = { ...live, phase: "standby", restored: false }; + stopRecyclePolicy(); + // A drain deadline does not carry across standby, as it does not across + // a recycle. One that already passed wakes the child for its cancel. + stopDrainAccumulators("standby"); + const exited = live.handle.exited.then( + () => true, + () => true, + ); + // Not awaited: a wedged child can block the send, and the kill must not + // wait for it. + void live.controlSender + .send({ type: "shutdown", data: { reason: "standby" } }) + .catch((cause: unknown) => { + logger.warn`sending shutdown to the child in standby failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + const deadline = waitDeadline(readySetTimer, DEFAULT_KILL_TIMEOUT_MS); + const stopped = await Promise.race([ + exited, + deadline.promise.then(() => false), + ]); + readyClearTimer(deadline.handle); + if (!stopped) { + await killChildHandle(live.handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }).catch((cause: unknown) => { + logger.warn`killing the child in standby failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } + await live.eventPump.catch(() => { + /* the child is gone; its event channel ending is expected */ + }); + live.terminalCohortAbort.abort(); + rejectCohortAwaiters("standby"); + if (pendingChildExit?.generation === attempt.generation) { + pendingChildExit = null; + } + // The stopped child's held writes never land. A child that wrote after it + // reported quiet was working again, so it is woken for that work. + const wroteAfterQuiet = attempt.fence !== null && attempt.fence.length > 0; + attempt.fence = null; + // Inputs wait for the attempt to settle, so the host records the child as + // it stopped. + if (record !== undefined) { + try { + if (verifiedDefinition === null) { + throw new Error("no child has reported a verified definition"); + } + await record({ + parked: correlations, + ...(outcome.wakeAt !== undefined ? { wakeAt: outcome.wakeAt } : {}), + tip, + definition: verifiedDefinition, + }); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + if (state.phase === "standby") { + inStandby = { correlations, outcome, wakeTimer: null }; + void wakeFromStandby("a standby that was not recorded").catch( + (wakeCause: unknown) => { + logger.error`waking a child whose standby was not recorded failed: ${wakeCause instanceof Error ? wakeCause.message : String(wakeCause)}`; + }, + ); + } + throw new StandbyRefusedError( + `the standby could not be recorded: ${message}`, + { cause }, + ); + } + } + // Only a shutdown moves the state on before the attempt settles. + if (state.phase !== "standby") { + throw new StandbyAbandonedError( + "the workflow supervisor shut down while entering standby", + ); + } + inStandby = { + correlations, + outcome, + wakeTimer: + outcome.wakeAt === undefined + ? null + : armStandbyWakeTimer(outcome.wakeAt), + }; + // Mail enqueued after the check before the stop waits in the inbox; mail + // enqueued after this read finds the child in standby and wakes it. + const mailWaiting = await inboxHoldsMail().catch((cause: unknown) => { + logger.error`reading the inbox of the child in standby failed; waking it in case mail waits there: ${cause instanceof Error ? cause.message : String(cause)}`; + return true; + }); + if (mailWaiting) { + void wakeFromStandby("mail").catch((cause: unknown) => { + logger.error`waking a child in standby for mail waiting in its inbox failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } else if (wroteAfterQuiet) { + void wakeFromStandby("work it began after it went quiet").catch( + (cause: unknown) => { + logger.error`waking a child in standby for its interrupted work failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }, + ); + } + // Abandoned too late to keep the child running. A cancellation or a wake + // runs the child again itself, and a shutdown keeps it stopped. + if (attempt.abandon.signal.aborted) { + if (attempt.wakeIfStopped) { + void wakeFromStandby("an abandoned standby attempt").catch( + (cause: unknown) => { + logger.error`waking a child whose standby was abandoned failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }, + ); + } + attempt.abandon.signal.throwIfAborted(); + } + return outcome; + } + + function armStandbyWakeTimer(wakeAt: string): unknown { + const delay = Math.max(0, Date.parse(wakeAt) - drainNow()); + return readySetTimer( + () => { + if (state.phase !== "standby" || inStandby === null) return; + if (delay > MAX_TIMER_MS) { + inStandby.wakeTimer = armStandbyWakeTimer(wakeAt); + return; + } + void wakeFromStandby("timer").catch((cause: unknown) => { + logger.error`waking a child in standby for its timer failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + }, + Math.min(delay, MAX_TIMER_MS), + ); + } + + // Respawn a child in standby through the recycle path: its drain and kill + // steps have nothing left to do, and the respawn and cohort install are the + // ones a recycle runs. + async function wakeFromStandby(reason: string): Promise { + // A standby attempt still stopping the child finishes first. + const attempt = standbyAttempt; + if (attempt !== null) { + bindings.onStandbyWait?.("attempt"); + await attempt.settled; + } + // The respawned child applies a cancellation from its log, so one being + // committed lands first, as does a drain deadline's that waits on the + // standby attempt it called off. + while (cancellationCommits.size > 0 || drainCancellations.size > 0) { + bindings.onStandbyWait?.("cancellation"); + await Promise.allSettled([...cancellationCommits, ...drainCancellations]); + } + if (waking !== null) return waking; + if (state.phase !== "standby") return; + const prior = state; + const priorContext = spawnContext; + if (priorContext === null) { + throw new Error("a workflow supervisor in standby has no spawn context"); + } + if (inStandby !== null && inStandby.wakeTimer !== null) { + readyClearTimer(inStandby.wakeTimer); + } + const woken = runRespawn({ + origin: "wake", + reason, + prior, + priorContext, + drain: async () => undefined, + }).then(() => { + // A shutdown that began during the respawn left the child uninstalled. + if (state.phase === "stopping" || state.phase === "stopped") { + throw new Error( + "the workflow supervisor shut down before the woken child ran", + ); + } + inStandby = null; + // A stop and wake are planned, so the woken child can still earn back a + // backoff that a crash raised. + if (respawnBackoffMs !== respawnBackoffInitialMs) { + armStableRunResetTimer(childGeneration); + } + try { + bindings.onWake?.({ reason, requested: reason === REQUESTED_WAKE }); + } catch (cause) { + logger.error`onWake sink threw: ${cause instanceof Error ? cause.message : String(cause)}`; + } + }); + waking = woken.finally(() => { + waking = null; + }); + return waking; + } + + // A respawn is starting the next child. A restored standby's wake does so + // from `standby`. + function respawning(): boolean { + return ( + state.phase === "recycling" || + (state.phase === "standby" && respawnInProgress) + ); + } + /** * Shared kill/replay/respawn/install driver. The caller has already - * verified the supervisor is `running` with a live spawn context and - * snapshotted both as `prior`/`priorContext`; this function transitions - * to `recycling`, runs the six-step `triggerRecycle` sequence with the - * caller-supplied `drain` step, and swaps in the new cohort via the - * inline `installNewChild` callback. The operator/policy/self recycle - * path is the sole caller today; the crash-respawn path will call it with - * a no-op drain (its child is already dead). + * snapshotted the supervisor's active state and spawn context as + * `prior`/`priorContext`; this function transitions to `recycling`, runs the + * six-step `triggerRecycle` sequence with the caller-supplied `drain` step, + * and swaps in the new cohort via the inline `installNewChild` callback. A + * recycle, a crash respawn, and the wake of a child in standby call it; the + * last two pass a no-op drain, since their child is already dead. A restored + * standby has no cohort: it stays in `standby` until its first child is + * installed, and replays only mail a failed boot replay left behind. */ async function runRespawn(args: { origin: RecycleOrigin; reason: string; - prior: ActiveState; + prior: Extract; priorContext: SpawnContext; drain: (deadlineMs: number) => Promise; }): Promise { @@ -4740,6 +5904,7 @@ export function createWorkflowSupervisor( // turn; two respawns can never interleave. respawnInProgress = true; const { origin, reason, prior, priorContext, drain } = args; + const cohort = prior.phase === "standby" && prior.restored ? null : prior; // The cohort abort no longer fires up-front. triggerRecycle drives // the drain step against a LIVE cohort first, then invokes // `abortPriorCohort` (the callback below) between the drain and the @@ -4747,7 +5912,7 @@ export function createWorkflowSupervisor( // up-front would starve drain accumulators of live terminal events; // aborting after the kill would race the dispatch loop's next // iteration against the controlSender that's about to disappear. - const priorDispatchLoop = prior.dispatchLoop; + const priorDispatchLoop = cohort === null ? null : cohort.dispatchLoop; // Transition to `recycling`. Inbound mail continues to flow through // `enqueueInbox` unchanged; the prior dispatch loop is still alive // for the drain window and keeps forwarding to the dying child. @@ -4755,20 +5920,22 @@ export function createWorkflowSupervisor( // loop notices the abort and exits before the kill lands. The new // dispatch loop picks up the inbox once `installNewChild` swaps // the wiring. - state = { - phase: "recycling", - handle: prior.handle, - controlSender: prior.controlSender, - channelId: prior.channelId, - eventPump: prior.eventPump, - onInferenceEvent: prior.onInferenceEvent, - mailUnsubscribe: prior.mailUnsubscribe, - terminalCohortAbort: prior.terminalCohortAbort, - terminalBroadcaster: prior.terminalBroadcaster, - dispatchLoop: null, - replayDone: null, - sweepDone: prior.sweepDone, - }; + if (cohort !== null) { + state = { + phase: "recycling", + handle: cohort.handle, + controlSender: cohort.controlSender, + channelId: cohort.channelId, + eventPump: cohort.eventPump, + onInferenceEvent: cohort.onInferenceEvent, + mailUnsubscribe: cohort.mailUnsubscribe, + terminalCohortAbort: cohort.terminalCohortAbort, + terminalBroadcaster: cohort.terminalBroadcaster, + dispatchLoop: null, + replayDone: null, + sweepDone: cohort.sweepDone, + }; + } let attempt: RecycleAttempt; try { attempt = await triggerRecycle( @@ -4776,7 +5943,7 @@ export function createWorkflowSupervisor( bindings: { ...bindings, subprocessSpawner: (spawnArgs) => { - if (state.phase !== "recycling") { + if (!respawning()) { throw new Error( `Cannot spawn a replacement in supervisor phase ${state.phase}`, ); @@ -4791,19 +5958,24 @@ export function createWorkflowSupervisor( }, definitionHash: priorContext.definitionHash, onInferenceEvent: priorContext.onInferenceEvent, - current: { - handle: prior.handle, - controlSender: prior.controlSender, - channelId: prior.channelId, - eventPump: prior.eventPump, - }, + current: + cohort === null + ? null + : { + handle: cohort.handle, + controlSender: cohort.controlSender, + channelId: cohort.channelId, + eventPump: cohort.eventPump, + }, drain, replayProcessingToInbox: async () => { // A boot replay still running would put back mail the replacement - // takes. + // takes. No child has run since a restored standby's boot replay, + // so only mail a failed one left behind needs replaying. if (bootReplay !== null) { bindings.onRespawnWait?.("replay"); - await bootReplay; + const booted = await bootReplay; + if (booted && prior.phase === "standby" && prior.restored) return; } // Mail a live run already took stays in processing/, as on spawn: // the resumed run carries on with it. Redelivered, its start mail @@ -4818,7 +5990,7 @@ export function createWorkflowSupervisor( } // Shutdown waits for a replay that began before it; one after it // would write a repository its host may already have removed. - if (state.phase !== "recycling") { + if (!respawning()) { throw new Error("the workflow supervisor is shutting down"); } const replay = (async () => { @@ -4837,11 +6009,15 @@ export function createWorkflowSupervisor( bindings.deploymentMailAddress, { ownedMessageIds }, ); + strandedMail = false; keepOwnedMessages(retainedMessageIds, { childRunning: false }); })(); respawnReplays.add(replay); try { await replay; + } catch (cause) { + strandedMail = true; + throw cause; } finally { respawnReplays.delete(replay); } @@ -4850,7 +6026,7 @@ export function createWorkflowSupervisor( // Fired by triggerRecycle between the drain and the kill. // The prior dispatch loop notices the abort on its next // wake and exits before the kill drops the child. - prior.terminalCohortAbort.abort(); + cohort?.terminalCohortAbort.abort(); wakeDispatch(); }, // Kept inline rather than extracted: this cohort-swap closes over @@ -4875,7 +6051,7 @@ export function createWorkflowSupervisor( // Shutdown already owns the uninstalled child's kill and exit. // A late ready frame must only drain its IPC resources, never // install it as a new running cohort. - if (state.phase !== "recycling") { + if (!respawning()) { void wiring.eventPump.catch((cause: unknown) => { const message = cause instanceof Error ? cause.message : String(cause); @@ -4896,10 +6072,7 @@ export function createWorkflowSupervisor( // child); the resumed child re-discovers any survivors // and the next `drain()` mints fresh accumulators // against the new cohort. - for (const accumulator of drainAccumulators.values()) { - accumulator.stop(); - } - drainAccumulators.clear(); + stopDrainAccumulators("recycle"); cohortRunIds.clear(); runInputChannels.clear(); parkNotifyWaiters.clear(); @@ -4910,11 +6083,14 @@ export function createWorkflowSupervisor( // child will re-issue substrate writes through fresh // handlers under the new cohort's channel. rejectCohortAwaiters("recycle"); + untakenMessageId = null; + dispatchRequestId = null; + dispatchAnswer = null; // Dispose the prior cohort's broadcaster so any minted // iterator still held by the aborted dispatch loop or a // stopped accumulator settles with `done: true`. The next // cohort gets a fresh broadcaster wired below. - prior.terminalBroadcaster.dispose(); + cohort?.terminalBroadcaster.dispose(); // Mint a fresh cohort abort and start a new dispatch // loop against the new child's controlSender. const newCohortAbort = new AbortController(); @@ -5071,6 +6247,29 @@ export function createWorkflowSupervisor( } async function deliverSignal(opts: DeliverSignalOpts): Promise { + // A signal delivered during a standby attempt's final check would land + // after the child reported quiet, so it waits for the check to end either + // way. + const frozen = standbyAttempt === null ? null : standbyAttempt.frozen; + if (frozen !== null) { + bindings.onStandbyWait?.("input"); + await frozen.promise; + } + // A child in standby, or one a wake is starting, takes the signal once it + // runs. + if (state.phase === "standby" || waking !== null) { + await wakeFromStandby("signal"); + } + deliveringSignals += 1; + try { + await deliverSignalToChild(opts); + } finally { + deliveringSignals -= 1; + noteStandbyProgress(); + } + } + + async function deliverSignalToChild(opts: DeliverSignalOpts): Promise { // The supervisor is the single producer of `signal.deliver` control // IPC frames. Routing every signal delivery through the same child // makes the workflow-process the single writer of `runs//events/` @@ -5112,18 +6311,65 @@ export function createWorkflowSupervisor( `supervisor: deliverSignal raced a recycle in phase ${phaseAfterRefresh}; expected starting/running`, ); } - await state.controlSender.send({ - type: "signal.deliver", - data: { - runId: opts.runId, - signalName: opts.signalName, - signalId: opts.signalId, - payload: opts.payload, - ...(opts.correlationId !== undefined - ? { correlationId: opts.correlationId } - : {}), - }, - }); + const requestId = nextInputRequest(null); + try { + await state.controlSender.send({ + type: "signal.deliver", + data: { + runId: opts.runId, + signalName: opts.signalName, + signalId: opts.signalId, + payload: opts.payload, + ...(opts.correlationId !== undefined + ? { correlationId: opts.correlationId } + : {}), + requestId, + }, + }); + } catch (cause) { + inputCommits.delete(requestId); + throw cause; + } + } + + function nextInputRequest(input: RunInput | null): string { + const requestId = `input-${String((inputSeq += 1))}`; + inputCommits.set(requestId, input); + return requestId; + } + + function releaseTakenInput(input: RunInput): void { + if (takenInputs.get(input.runId) === input.correlationId) + takenInputs.delete(input.runId); + } + + // A dispatch sends no input during a standby attempt's final check. The + // attempt refuses while the dispatch holds mail the child has not taken, so + // the input goes out once the check ends. + async function deliveryAllowed(cohortAbort: AbortController) { + for (;;) { + const frozen = standbyAttempt === null ? null : standbyAttempt.frozen; + if (frozen === null) return !cohortAbort.signal.aborted; + bindings.onStandbyWait?.("input"); + await whileDispatchWaits( + cohortAbort, + raceAbort(cohortAbort.signal, frozen.promise), + ); + if (cohortAbort.signal.aborted) return false; + } + } + + async function whileDispatchWaits( + cohortAbort: AbortController, + wait: Promise, + ): Promise { + dispatchWaiting = cohortAbort; + noteStandbyProgress(); + try { + return await wait; + } finally { + if (dispatchWaiting === cohortAbort) dispatchWaiting = null; + } } // Mail a replay left with the run that took it. A run that parked or ended @@ -5201,7 +6447,11 @@ export function createWorkflowSupervisor( // `deliverSignal`: `state.controlSender` still points at the dying // child, so a frame would either buffer behind the SIGTERM or write // into a closed pipe and be lost. Rejecting surfaces the race so the - // caller can retry once the recycle completes. + // caller can retry once the recycle completes. A child in standby has none + // to update: its respawn reads the host's persisted sources. A wake + // starting the first child of a restored standby is rejected as a + // recycle is. + if (state.phase === "standby" && !respawning()) return; if (state.phase !== "running" && state.phase !== "starting") { throw new Error( `supervisor: deliverSources called in phase ${state.phase}; expected starting/running`, @@ -5363,6 +6613,7 @@ export function createWorkflowSupervisor( return { spawn, + restoreInStandby, requestCancel, shutdown, drain, @@ -5373,9 +6624,15 @@ export function createWorkflowSupervisor( deliverGrants, reEmitParkedCorrelations, credentialDelivery: () => currentCredentialDelivery, + enterStandby, + wake, + abandonStandby: abandonCurrentStandby, }; } +// A run's input a mail was delivered on. +type RunInput = { runId: string; correlationId: string }; + type SupervisorState = | { phase: "idle" } | { phase: "stopping" } @@ -5388,7 +6645,22 @@ type SupervisorState = | { phase: "crash-looping" } | ({ phase: "starting" } & ActiveState) | ({ phase: "running" } & ActiveState) - | ({ phase: "recycling" } & ActiveState); + | ({ phase: "recycling" } & ActiveState) + // The child was stopped while every run waited. The cohort record is kept, + // with its exited handle, so a wake respawns into the same mail + // subscription and address registration. + | ({ phase: "standby"; restored: false } & ActiveState) + // A standby restored after a restart, before any child has run: a wake + // starts the first one into the mail subscription the restore took. + | ({ phase: "standby"; restored: true } & RestoredState); + +// What a standby restored after a restart holds without a child. Its boot +// replay gates the first child's dispatch, as a spawn's does. +type RestoredState = { + mailUnsubscribe: (() => void) | null; + replayDone: Promise; + sweepDone: Promise; +}; type ActiveState = { handle: SubprocessHandle; diff --git a/packages/workflow-host/src/supervisor/types.ts b/packages/workflow-host/src/supervisor/types.ts index 33d944abf..c3b1fc828 100644 --- a/packages/workflow-host/src/supervisor/types.ts +++ b/packages/workflow-host/src/supervisor/types.ts @@ -29,6 +29,9 @@ import type { import type { RunCancelled, RunCompleted, RunFailed } from "@intx/workflow"; import type { FrameReader, NdjsonReader, NdjsonWriter } from "../ipc/index"; +import type { RecordedStandby } from "./recorded-standby"; + +export type { RecordedStandby }; /** * Terminal workflow-run event the supervisor's drain accumulators @@ -200,7 +203,7 @@ export type SubprocessSpawner = (args: { * Inbox claim-check primitives the supervisor's mail-arrival path * and dispatch loop reach into. Production wires this against the * concrete `enqueueInbox` / `dequeueToProcessing` / `markConsumed` - * / `replayProcessingToInbox` functions exported from + * / `replayProcessingToInbox` / `hasInboxEntries` functions exported from * `@intx/hub-sessions`; tests inject a deterministic in-memory * stub so the supervisor's dispatch loop is observable without a * real git substrate. @@ -235,6 +238,12 @@ export interface InboxPrimitives { address: string, opts?: ReplayProcessingToInboxOpts, ): Promise; + hasInboxEntries( + store: SubstrateRepoStore, + principal: Principal, + repoId: RepoId, + address: string, + ): Promise; } /** @@ -319,19 +328,51 @@ export interface WorkflowSupervisorBindings { phase: "stopped" | "crash-looping"; reason: string; }) => void; + /** + * Fired once a supervisor in standby has a running child again, whatever woke + * it: mail, a signal, a cancellation, its earliest pending timer, or + * `wake`. `requested` is true only when `wake()` started the respawn. + */ + onWake?: (info: { reason: string; requested: boolean }) => void; + /** + * Called once a standby attempt has stopped the child and before anything can + * wake it, so a wake's record of the child running lands after it. A + * rejection runs the child again and refuses the standby attempt. What it + * records is what `restoreInStandby` takes after a restart. + */ + recordStandby?: (standby: RecordedStandby) => Promise; + /** + * Test seam, unset in production. Called each time a standby attempt makes + * something wait: `input` when mail or a signal waits out the final quiet + * check, `standby` when the standby attempt waits for work the supervisor + * still drives for the child, `attempt` when a wake waits for a standby + * attempt to finish stopping the child, `cancellation` when a wake waits + * for a cancellation being committed. + */ + onStandbyWait?: ( + waiting: "input" | "standby" | "attempt" | "cancellation", + ) => void; /** * Test seam, unset in production. Called each time a shutdown waits for - * work it owns before it resolves: `replay` for a respawn's replay of - * `processing/`, `writes` for repository writes started off the control - * pump. + * work it owns before it resolves: `cancellation` for a drain deadline's + * cancellation, `attempt` for a standby attempt it called off, `replay` for + * a respawn's replay of `processing/`, `writes` for repository writes + * started off the control pump. */ - onShutdownWait?: (waiting: "replay" | "writes") => void; + onShutdownWait?: ( + waiting: "cancellation" | "attempt" | "replay" | "writes", + ) => void; /** * Test seam, unset in production. Called each time a respawn, before it * scans `runs/`, waits for the boot replay (`replay`) or for repository * writes started off the control pump (`writes`). */ onRespawnWait?: (waiting: "replay" | "writes") => void; + /** + * Test seam, unset in production. Called when the dispatch holds mail for a + * live run until the run shows its input, parks or ends. + */ + onDispatchWait?: (runId: string) => void; /** * Per-run grants source. The dispatch loop consults it before it forwards a * `trigger.fire`, and every new child (spawn, crash-respawn, recycle) gets @@ -445,15 +486,17 @@ export interface WorkflowSupervisorBindings { */ drainTimeoutAccumulatorFactory?: import("./drain-timeout").DrainTimeoutAccumulatorFactory; /** - * Clock the supervisor threads into the drainTimeout accumulator. - * Production wires `() => Date.now()`; tests inject a deterministic - * fake clock. Defaults to `Date.now` when omitted. + * Clock the supervisor threads into the drainTimeout accumulator, and + * that times the standby wake timer against the earliest timer the + * child reported. Production wires `() => Date.now()`; tests inject a + * deterministic fake clock. Defaults to `Date.now` when omitted. */ now?: () => number; /** * General scheduling primitive the supervisor threads into its timed * waits: the drainTimeout accumulator, the spawn ready-handshake - * timeout, and that timeout's SIGTERM->SIGKILL kill escalation. + * timeout and its SIGTERM->SIGKILL kill escalation, and standby's wake + * timer, quiet-check watchdog and stop deadline. * Production wires `(cb, ms) => setTimeout(cb, ms)`; tests inject a * deterministic timer host. Defaults to `setTimeout` when omitted. */ diff --git a/packages/workflow-host/src/testing/mail-bus.ts b/packages/workflow-host/src/testing/mail-bus.ts index 0323fb8b0..3cf993b36 100644 --- a/packages/workflow-host/src/testing/mail-bus.ts +++ b/packages/workflow-host/src/testing/mail-bus.ts @@ -21,6 +21,11 @@ export type MockMailBus = MailBusBindings & { registrationHistory(): readonly string[]; /** Deliver a raw message to every handler subscribed for `address`. */ deliver(address: string, message: Uint8Array): void; + /** + * As `deliver`, resolving once every handler is done with the message, for + * a test that needs the arrival finished before it goes on. + */ + deliverSettled(address: string, message: Uint8Array): Promise; /** * Resolve once `address` is registered, whether it already is or is * registered later. @@ -90,6 +95,11 @@ export function createMockMailBus(): MockMailBus { if (set === undefined) return; for (const handler of set) void handler(message).catch(() => undefined); }, + async deliverSettled(address: string, message: Uint8Array) { + const set = subscribers.get(address); + if (set === undefined) return; + await Promise.allSettled([...set].map((handler) => handler(message))); + }, async awaitRegistered(address: string) { for (;;) { // Re-read on every pass, so an address registered before this call diff --git a/packages/workflow-host/src/testing/stub-repo-store.ts b/packages/workflow-host/src/testing/stub-repo-store.ts index a59e5354e..e9974945a 100644 --- a/packages/workflow-host/src/testing/stub-repo-store.ts +++ b/packages/workflow-host/src/testing/stub-repo-store.ts @@ -27,6 +27,8 @@ export type StubRepoStoreOpts = { * fails on the call rather than accepting it. */ writeTree?: boolean; + /** Answer `resolveRef` with this tip for every ref; unset, the call fails. */ + refTip?: string | null; }; export function createStubRepoStore( @@ -48,6 +50,9 @@ export function createStubRepoStore( }), } : {}), + ...(opts.refTip !== undefined + ? { resolveRef: async () => opts.refTip ?? null } + : {}), }; // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; every method the opts did not enable surfaces a precise failure via the proxy return new Proxy(stub as RepoStore, { diff --git a/tests/workflow-deploy/single-step-conversation-durability.test.ts b/tests/workflow-deploy/single-step-conversation-durability.test.ts index b598a55d2..9f4f2c559 100644 --- a/tests/workflow-deploy/single-step-conversation-durability.test.ts +++ b/tests/workflow-deploy/single-step-conversation-durability.test.ts @@ -570,6 +570,7 @@ describe("single-step conversation durability across respawn (Phase 4.5)", () => spawnChild: async () => createTerminalChildHandle("completed"), scheduler: { scheduleIn: () => () => undefined }, earliestQueuedTimer: () => undefined, + stopScheduler: async () => undefined, evaluateGrants: async () => ({ effect: "allow" as const, matchingGrants: [], From bcddd0fd511408574ce87125a91d6636b652b789 Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 2 Nov 2026 14:13:14 +0100 Subject: [PATCH 18/20] Hold a signal for a resumed park until the run waits again A signal committed while a resumed run plans moves a step the run has just read as parked, which fails an agent step or loses a section's input. A resume guard cannot tell this from a crash after the decision was taken, when re-invoking the step could run a tool twice, so the child holds the signal instead. --- docs/IMPLEMENTATION.md | 5 +- .../src/child/resume-delivery-hold.test.ts | 350 ++++++++++++++++++ .../src/child/resume-delivery-hold.ts | 99 +++++ .../workflow-host/src/child/run-child.test.ts | 334 ++++++++++++++++- packages/workflow-host/src/child/run-child.ts | 155 +++++--- .../src/supervisor/recycle.test.ts | 74 ++++ .../src/supervisor/supervisor.ts | 3 +- .../src/runtime/loop-iteration-handle.test.ts | 23 ++ .../src/runtime/loop-iteration-handle.ts | 9 +- .../runtime/nested-loop-resume-race.test.ts | 297 +++++++++++++++ 10 files changed, 1285 insertions(+), 64 deletions(-) create mode 100644 packages/workflow-host/src/child/resume-delivery-hold.test.ts create mode 100644 packages/workflow-host/src/child/resume-delivery-hold.ts create mode 100644 packages/workflow/src/runtime/nested-loop-resume-race.test.ts diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 0513b5127..ccc4b13e6 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -662,7 +662,10 @@ ChannelSender` writes against; the event channel is the signal, and the answer says so. The dispatch that delivered a mail consumes it only once that answer says committed; a declined mail is rejected if the committed log shows its run ended, and otherwise fails - the dispatch and stays in `processing/` until a respawn replays it. + the dispatch and stays in `processing/` until a respawn replays it. A signal for a park a + resumed run holds waits until the run parks on it again, the run ends, or the + process goes quiet: landing during the resume would move a step the + run has just read as parked. - `quiet.request` -> make every buffered event durable and answer whether every run only waits on input from outside, with the earliest timer the scheduler has queued; with `wait`, answer once diff --git a/packages/workflow-host/src/child/resume-delivery-hold.test.ts b/packages/workflow-host/src/child/resume-delivery-hold.test.ts new file mode 100644 index 000000000..7e12261cd --- /dev/null +++ b/packages/workflow-host/src/child/resume-delivery-hold.test.ts @@ -0,0 +1,350 @@ +// A signal for a park a run resumed with lands only once the run waits on that +// park again. The sweep lands the signal after each read a resume makes, which +// is where a signal landing unheld fails the step or loses the input. + +import { describe, expect, test } from "bun:test"; + +import { createDefaultDirectorRegistry, defineAgent } from "@intx/agent"; +import { signalName } from "@intx/types"; +import { + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, + createNoopDrainController, + createTerminalChildHandle, + defineWorkflow, + resumeFromLog, + runtimeRun, + step, + type WorkflowDefinition, + type WorkflowEvent, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { beginDelivery, sectionBodyRunId } from "@intx/workflow/runtime"; + +import { + createResumeDeliveryHold, + parkedSignalNames, +} from "./resume-delivery-hold"; + +const at = new Date().toISOString(); + +const tick = (): Promise => + new Promise((resolve) => setTimeout(resolve, 0)); + +async function settled( + waiting: Promise | undefined, +): Promise { + await tick(); + return Promise.race([waiting, Promise.resolve("pending")]); +} + +describe("resume delivery hold", () => { + test("a name no resumed park holds can land at once", () => { + const hold = createResumeDeliveryHold(); + hold.hold("run", ["held"]); + expect(hold.until("run", "other")).toBeUndefined(); + expect(hold.until("other-run", "held")).toBeUndefined(); + hold.close(); + }); + + test("a signal waits until the run parks on it again, and later ones land at once", async () => { + const endWork = beginDelivery(); + const hold = createResumeDeliveryHold(); + hold.hold("run", ["a", "b"]); + const a = hold.until("run", "a"); + const b = hold.until("run", "b"); + expect(await settled(a)).toBe("pending"); + + hold.reparked("run", "a"); + expect(await settled(a)).toBe(true); + expect(await settled(b)).toBe("pending"); + expect(hold.until("run", "a")).toBeUndefined(); + + hold.reparked("run", "b"); + expect(await settled(b)).toBe(true); + hold.close(); + endWork(); + }); + + test("the run ending refuses a held signal", async () => { + const endWork = beginDelivery(); + const hold = createResumeDeliveryHold(); + hold.hold("run", ["a"]); + const a = hold.until("run", "a"); + hold.runEnded("run"); + expect(await settled(a)).toBe(false); + hold.close(); + endWork(); + }); + + test("the process going quiet lets a held signal go on", async () => { + const endWork = beginDelivery(); + const hold = createResumeDeliveryHold(); + hold.hold("run", ["a"]); + const a = hold.until("run", "a"); + expect(await settled(a)).toBe("pending"); + endWork(); + expect(await settled(a)).toBe(true); + hold.close(); + }); + + test("closing refuses a held signal", async () => { + const endWork = beginDelivery(); + const hold = createResumeDeliveryHold(); + hold.hold("run", ["a"]); + const a = hold.until("run", "a"); + hold.close(); + expect(await settled(a)).toBe(false); + endWork(); + }); +}); + +const agent = defineAgent({ + id: "a", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, +}); + +const started = (runId: string): WorkflowEvent => ({ + kind: "RunStarted", + seq: 1, + at, + runId, + definitionHash: "x", + trigger: { type: "manual", payload: { text: "event-0" } }, +}); +const stepStarted = (stepId: string): WorkflowEvent => ({ + kind: "StepStarted", + seq: 2, + at, + stepId, + attempt: 1, + input: { ref: "inline:null" }, +}); + +type Scenario = { + label: string; + definition: WorkflowDefinition; + name: string; + seed: (runId: string) => WorkflowEvent[]; + sectionBody?: boolean; +}; + +const scenarios: Scenario[] = [ + { + label: "an agent step's approval", + definition: defineWorkflow({ + id: "hold-approval", + trigger: { type: "manual" }, + steps: { a: step({ agent }) }, + }), + name: signalName("corr-approval"), + seed: (runId) => [ + started(runId), + stepStarted("a"), + { + kind: "SignalAwaited", + seq: 3, + at, + stepId: "a", + signalName: signalName("corr-approval"), + parkKind: "approval", + }, + ], + }, + { + label: "an agent step's input", + definition: defineWorkflow({ + id: "hold-input", + trigger: { type: "manual" }, + steps: { a: step({ agent, triggers: 2 }) }, + }), + name: signalName("corr-input"), + seed: (runId) => [ + started(runId), + stepStarted("a"), + { + kind: "SignalAwaited", + seq: 3, + at, + stepId: "a", + signalName: signalName("corr-input"), + parkKind: "input", + }, + ], + }, + { + label: "an onTrigger section's next event", + definition: defineWorkflow({ + id: "hold-section", + steps: { + section: { + kind: "onTrigger", + id: "", + on: { type: "mail", to: "run_sec@t.example" }, + body: { ref: "body-ref" }, + drainBehavior: "wait", + }, + }, + }), + name: signalName("corr-rearm"), + sectionBody: true, + seed: (runId) => [ + started(runId), + stepStarted("section"), + { + kind: "ChildSpawned", + seq: 3, + at, + stepId: "section", + childRunId: sectionBodyRunId(runId, "section", 0), + childDefinitionRef: "body-ref", + }, + { + kind: "ChildCompleted", + seq: 4, + at, + childRunId: sectionBodyRunId(runId, "section", 0), + terminalStatus: "completed", + }, + { + kind: "SignalAwaited", + seq: 5, + at, + stepId: "section", + signalName: signalName("corr-rearm"), + parkKind: "input", + }, + ], + }, +]; + +// Asks to deliver the signal after read `requestAfterRead` of the run's log, the +// way the child does: through the hold, then a commit, then the channel. +async function trial( + scenario: Scenario, + requestAfterRead: number, +): Promise<{ pastTheWindow: boolean }> { + const runId = `run-${scenario.definition.id}-${String(requestAfterRead)}`; + const clock = (): Date => new Date(); + const repoStore = createInMemoryRepoStore(); + const channel = createInMemorySignalChannel(); + const seed = scenario.seed(runId); + const hold = createResumeDeliveryHold(); + hold.hold(runId, parkedSignalNames(resumeFromLog(runId, seed))); + + const decisions: unknown[] = []; + const spawned = Promise.withResolvers(); + let spawnCount = 0; + let requested: Promise | undefined; + const requestDelivery = (): Promise => + (requested ??= (async () => { + const canLand = await (hold.until(runId, scenario.name) ?? true); + expect(canLand).toBe(true); + const tip = (await repoStore.read(runId)).at(-1); + if (tip === undefined) throw new Error("run log empty"); + await repoStore.append(runId, { + kind: "SignalReceived", + seq: tip.seq + 1, + at, + signalName: scenario.name, + signalId: "sig-hold", + payload: { text: "decision" }, + }); + await channel.deliver(scenario.name, { text: "decision" }, "sig-hold"); + })()); + + let reads = 0; + let pastTheWindow = false; + const env: WorkflowRuntimeEnv = { + repoStore: { + ...repoStore, + read: async (id) => { + const log = [...(await repoStore.read(id))]; + if (id === runId && log.length > 0) { + reads += 1; + if (reads === requestAfterRead) void requestDelivery(); + } + return log; + }, + }, + scheduler: createInMemoryScheduler({ repoStore, clock }), + signalChannel: channel, + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: async (req) => { + if (req.resume !== undefined) decisions.push(req.resume.decision); + return { output: null }; + }, + spawnChild: async () => createTerminalChildHandle("completed"), + ...(scenario.sectionBody === true + ? { + spawnSuspendableChild: async () => { + spawnCount += 1; + spawned.resolve(undefined); + return { + next: () => new Promise(() => undefined), + resume: async () => undefined, + deliverSignal: async () => undefined, + }; + }, + } + : {}), + clock, + newId: (prefix) => `${prefix}-${Math.random().toString(36).slice(2, 8)}`, + drain: createNoopDrainController(scenario.definition), + hasUpstreamSignalResolver: true, + onSignalAwaiter: (park) => { + hold.reparked(park.runId, park.name); + if (requested === undefined) { + pastTheWindow = true; + void requestDelivery(); + } + }, + }; + + const run = runtimeRun(scenario.definition, env, { + runId, + resumeFromEvents: seed, + }); + if (scenario.sectionBody === true) { + await spawned.promise; + expect(spawnCount).toBe(1); + } else { + const result = await run.complete; + expect(result.terminalStatus).toBe("completed"); + expect(decisions).toEqual([{ text: "decision" }]); + } + await requested; + const awaits = (await repoStore.read(runId)).filter( + (event) => + event.kind === "SignalAwaited" && event.signalName === scenario.name, + ); + expect(awaits).toHaveLength(1); + if (scenario.sectionBody === true) { + await run.cancel("supervisor-operator", "test done").catch(() => undefined); + } + hold.close(); + return { pastTheWindow }; +} + +describe("a signal asked for after each read of a resume", () => { + for (const scenario of scenarios) { + test(scenario.label, async () => { + for (let read = 1; read <= 40; read += 1) { + const { pastTheWindow } = await trial(scenario, read); + if (pastTheWindow) return; + } + throw new Error("the run never parked on the signal again"); + }); + } +}); diff --git a/packages/workflow-host/src/child/resume-delivery-hold.ts b/packages/workflow-host/src/child/resume-delivery-hold.ts new file mode 100644 index 000000000..aa1480514 --- /dev/null +++ b/packages/workflow-host/src/child/resume-delivery-hold.ts @@ -0,0 +1,99 @@ +// When a signal for a resumed run may be committed. +// +// A resumed run plans from reads of its log. A signal committed between those +// reads moves a step the run has just read as parked, so the run acts on a park +// that is gone: an agent step in that state looks crashed and fails, since +// re-invoking one could run a tool twice, and a section parks again and loses +// the input. The log cannot tell this apart from a crash, so the child commits +// a signal for a park the run resumed with only once the run waits on it again. +// +// A run that does not park again must not keep the signal forever. A hold also +// ends with the run, which refuses the signal, and once the process goes quiet, +// when nothing is left that could park again. + +import { whenQuiet } from "@intx/workflow/runtime"; +import type { RunState } from "@intx/workflow"; + +export type ResumeDeliveryHold = { + /** Hold signals for `names` until `runId` parks on them again. */ + hold(runId: string, names: readonly string[]): void; + /** `runId` waits on `name` again, so a signal for it can land. */ + reparked(runId: string, name: string): void; + runEnded(runId: string): void; + /** + * `undefined` when a signal for `name` can land now. Otherwise resolves once + * it can, or with `false` when the run ended or the child is closing and it + * must not. + */ + until(runId: string, name: string): Promise | undefined; + close(): void; +}; + +/** The channels a run's steps are parked on. */ +export function parkedSignalNames(state: RunState): string[] { + const names: string[] = []; + for (const step of state.steps.values()) { + if (step.phase === "awaiting-signal" && step.awaitingSignal !== undefined) { + names.push(step.awaitingSignal.name); + } + } + return names; +} + +export function createResumeDeliveryHold(): ResumeDeliveryHold { + type Waiter = (canLand: boolean) => void; + const held = new Map>(); + const quietWatch = new AbortController(); + let watchingQuiet = false; + + const settle = (waiters: readonly Waiter[], canLand: boolean): void => { + for (const waiter of waiters) waiter(canLand); + }; + const settleRun = (runId: string, canLand: boolean): void => { + const names = held.get(runId); + if (names === undefined) return; + held.delete(runId); + for (const waiters of names.values()) settle(waiters, canLand); + }; + const settleAll = (canLand: boolean): void => { + for (const runId of [...held.keys()]) settleRun(runId, canLand); + }; + + return { + hold(runId, names) { + if (names.length === 0) return; + held.set(runId, new Map(names.map((name) => [name, []]))); + }, + reparked(runId, name) { + const names = held.get(runId); + const waiters = names?.get(name); + if (names === undefined || waiters === undefined) return; + names.delete(name); + if (names.size === 0) held.delete(runId); + settle(waiters, true); + }, + runEnded(runId) { + settleRun(runId, false); + }, + until(runId, name) { + const waiters = held.get(runId)?.get(name); + if (waiters === undefined) return undefined; + if (!watchingQuiet) { + watchingQuiet = true; + void whenQuiet(quietWatch.signal).then( + () => { + settleAll(true); + }, + () => undefined, + ); + } + return new Promise((resolve) => { + waiters.push(resolve); + }); + }, + close() { + quietWatch.abort(); + settleAll(false); + }, + }; +} diff --git a/packages/workflow-host/src/child/run-child.test.ts b/packages/workflow-host/src/child/run-child.test.ts index b2364d88c..bffe293ef 100644 --- a/packages/workflow-host/src/child/run-child.test.ts +++ b/packages/workflow-host/src/child/run-child.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { generateKeyPair } from "@intx/crypto"; -import { base64Encode, hexEncode } from "@intx/types"; +import { base64Encode, hexEncode, signalName } from "@intx/types"; import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; import { computeLiveDefinitionHash, @@ -32,6 +32,7 @@ import { type MessageHeaders, } from "@intx/mime"; +import { createWorkflowRunRepoStore } from "../adapters/repo-store"; import { createWorkflowStepInvoker, type StepEnvBase, @@ -2716,6 +2717,337 @@ describe("runWorkflowChild", () => { }); }); +describe("a signal for a park a resumed run holds", () => { + const RUN_ID = "run-parked"; + const NAME = signalName("corr-parked"); + const AT = "2026-01-01T00:00:00.000Z"; + + // A child resuming a run parked on an approval, held at the run's first read + // of its log, so a signal arrives while the resume is under way. + async function resumeParkedRun() { + const baseDir = await makeTempDir("child-resume-hold-"); + const supervisorKeyPair = await generateKeyPair(); + const childKeyPair = await generateKeyPair(); + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const repoId: RepoId = { kind: "workflow-run", id: "deployment-x" }; + const principalShape = { + kind: "workflow-process", + anchorRunId: "deployment-x", + }; + const principal: Principal = principalShape; + const substrate = createRepoStore({ + dataDir: baseDir, + signingKey: await generateKeyPair(), + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: () => ({ allowed: true }), + }); + await substrate.writeTree({ kind: "hub" }, repoId, "refs/heads/main", { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + const seeded = createWorkflowRunRepoStore({ + substrate, + repoId, + principal, + ref: "refs/heads/main", + }); + await seeded.appendBatch(RUN_ID, [ + { + kind: "RunStarted", + seq: 1, + at: AT, + runId: RUN_ID, + definitionHash: "definition-hash-abc", + trigger: { type: "manual", payload: null }, + }, + { + kind: "StepStarted", + seq: 2, + at: AT, + stepId: "step-1", + attempt: 1, + input: { ref: "inline:null" }, + }, + { + kind: "SignalAwaited", + seq: 3, + at: AT, + stepId: "step-1", + signalName: NAME, + parkKind: "approval", + }, + ]); + const { + packageDir: closurePackageDir, + approvedHash: definitionHash, + definition, + } = await materializeClosure("child-resume-hold-def-", { + id: "test-workflow", + triggers: [], + steps: { "step-1": agentStep("step-1") }, + stepOrder: ["step-1"], + }); + + const readHeld = Promise.withResolvers(); + const releaseRead = Promise.withResolvers(); + // Self-discovery reads the log once before the run starts. + let reads = 0; + const writeHold = { + on: false, + reached: Promise.withResolvers(), + release: Promise.withResolvers(), + }; + const heldAtFirstRead = new Proxy(substrate, { + get(target, prop, receiver) { + if (prop === "writeTreePreservingPrefix") { + return async ( + ...args: Parameters + ) => { + if (writeHold.on) { + writeHold.reached.resolve(undefined); + await writeHold.release.promise; + } + return target.writeTreePreservingPrefix(...args); + }; + } + if (prop !== "openCommittedReads") { + return Reflect.get(target, prop, receiver); + } + return async (...args: Parameters) => { + reads += 1; + if (reads === 2) { + readHeld.resolve(undefined); + await releaseRead.promise; + } + return target.openCommittedReads(...args); + }; + }, + }); + + const decisions: unknown[] = []; + const bindings: RunWorkflowChildBindings = { + ...buildBindings({ baseDir, childKeyPair }), + substrate: heldAtFirstRead, + principal, + invokeStep: async (req) => { + if (req.resume !== undefined) decisions.push(req.resume.decision); + return { output: null }; + }, + }; + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventStream = createMemoryFrameStream(); + const supervisorSender = createControlChannelSender({ + privateKeySeed: supervisorKeyPair.privateKey, + channelId, + writer: supervisorToChild.writer, + }); + await sendAuthoritySet(supervisorSender, { snapshot: AUTHORITY_SNAPSHOT }); + const childDone = runWorkflowChild({ + env: parseSpawnTimeEnv( + makeSpawnEnv({ + channelId, + hmacKeyHex: hexEncode(hmacKey), + hostPubKeyHex: hexEncode(supervisorKeyPair.publicKey), + definitionHash, + closurePackageDir, + }), + ), + definition, + controlReader: supervisorToChild.reader, + controlWriter: childToSupervisor.writer, + eventWriter: eventStream.writer, + bindings, + }); + await readHeld.promise; + + // Sends the signal, then waits until the child has taken that frame and + // has no committed input on its way: a signal not held has landed by then. + const deliverAndSettle = async (): Promise => { + await supervisorSender.send({ + type: "signal.deliver", + data: { + runId: RUN_ID, + signalName: NAME, + signalId: "sig-1", + payload: { outcome: "approved" }, + requestId: "signal-1", + }, + }); + await supervisorSender.send({ + type: "quiet.request", + data: { requestId: "settled", runId: RUN_ID, wait: false }, + }); + await waitForUpstreamPayload(childToSupervisor, "quiet.response"); + }; + const shutDown = async (): Promise => { + await supervisorSender.send({ + type: "shutdown", + data: { reason: "test done" }, + }); + supervisorToChild.close(); + await childDone; + }; + return { + childToSupervisor, + supervisorSender, + decisions, + deliverAndSettle, + releaseRead: () => { + releaseRead.resolve(undefined); + }, + seeded, + shutDown, + writeHold, + }; + } + + test("a signal whose commit is in flight keeps the child from answering quiet", async () => { + const child = await resumeParkedRun(); + child.releaseRead(); + await child.supervisorSender.send({ + type: "quiet.request", + data: { requestId: "parked", runId: RUN_ID, wait: true }, + }); + const parked = await waitForUpstreamPayload( + child.childToSupervisor, + "quiet.response", + ); + expect(parked.data).toMatchObject({ requestId: "parked", quiet: true }); + + child.writeHold.on = true; + await child.supervisorSender.send({ + type: "signal.deliver", + data: { + runId: RUN_ID, + signalName: NAME, + signalId: "sig-1", + payload: { outcome: "approved" }, + requestId: "signal-1", + }, + }); + await child.writeHold.reached.promise; + await child.supervisorSender.send({ + type: "quiet.request", + data: { requestId: "during-commit", runId: RUN_ID, wait: false }, + }); + // Frames are handled in order, so the answer for a run this child does not + // drive shows the check has started. + await child.supervisorSender.send({ + type: "signal.deliver", + data: { + runId: "run-absent", + signalName: NAME, + signalId: "sig-absent", + payload: null, + requestId: "after-check", + }, + }); + await waitForUpstreamPayload(child.childToSupervisor, "input.taken"); + + child.writeHold.on = false; + child.writeHold.release.resolve(undefined); + await waitForUpstreamPayload(child.childToSupervisor, "terminal.event"); + await waitForUpstreamPayloads(child.childToSupervisor, "quiet.response", 2); + // A child that counts the delivery answers only once the commit settled. + const frames = child.childToSupervisor.flushed(); + const committedAt = frames.findIndex((line) => line.includes('"signal-1"')); + const answeredAt = frames.findIndex((line) => + line.includes('"during-commit"'), + ); + expect(committedAt).toBeGreaterThanOrEqual(0); + expect(answeredAt).toBeGreaterThan(committedAt); + await child.shutDown(); + }); + + test("lands once the run waits on the park again", async () => { + const child = await resumeParkedRun(); + await child.deliverAndSettle(); + expect((await child.seeded.read(RUN_ID)).map((e) => e.kind)).toEqual([ + "RunStarted", + "StepStarted", + "SignalAwaited", + ]); + + child.releaseRead(); + const taken = await waitForUpstreamPayload( + child.childToSupervisor, + "input.taken", + ); + expect(taken.data).toEqual({ requestId: "signal-1", committed: true }); + const terminal = await waitForUpstreamPayload( + child.childToSupervisor, + "terminal.event", + ); + expect(terminal.data.runId).toBe(RUN_ID); + await child.shutDown(); + + expect(child.decisions).toEqual([{ outcome: "approved" }]); + const log = await child.seeded.read(RUN_ID); + expect(log.filter((e) => e.kind === "SignalAwaited")).toHaveLength(1); + expect(log.at(-1)?.kind).toBe("RunCompleted"); + }); + + test("a signal for a run whose log already ended is answered as not committed", async () => { + const child = await resumeParkedRun(); + child.releaseRead(); + await child.supervisorSender.send({ + type: "quiet.request", + data: { requestId: "parked", runId: RUN_ID, wait: true }, + }); + await waitForUpstreamPayload(child.childToSupervisor, "quiet.response"); + + // The end is committed, and the child still counts the run as in flight + // until its driver finishes. + await child.seeded.append(RUN_ID, { + kind: "RunFailed", + seq: 4, + at: AT, + error: { message: "ended" }, + }); + await child.supervisorSender.send({ + type: "signal.deliver", + data: { + runId: RUN_ID, + signalName: NAME, + signalId: "sig-1", + payload: { outcome: "approved" }, + requestId: "signal-1", + }, + }); + const taken = await waitForUpstreamPayload( + child.childToSupervisor, + "input.taken", + ); + expect(taken.data).toEqual({ requestId: "signal-1", committed: false }); + await child.shutDown(); + expect( + (await child.seeded.read(RUN_ID)).filter( + (e) => e.kind === "SignalReceived", + ), + ).toEqual([]); + }); + + test("is answered as not committed when the child shuts down first", async () => { + const child = await resumeParkedRun(); + await child.deliverAndSettle(); + const shutdown = child.shutDown(); + const taken = await waitForUpstreamPayload( + child.childToSupervisor, + "input.taken", + ); + expect(taken.data).toEqual({ requestId: "signal-1", committed: false }); + child.releaseRead(); + await shutdown; + expect( + (await child.seeded.read(RUN_ID)).filter( + (e) => e.kind === "SignalReceived", + ), + ).toEqual([]); + }); +}); + describe("runWorkflowChildFromProcessEnv", () => { test("missing spawn-time env throws via parseSpawnTimeEnv", async () => { const { runWorkflowChildFromProcessEnv } = await import("./index"); diff --git a/packages/workflow-host/src/child/run-child.ts b/packages/workflow-host/src/child/run-child.ts index f4d3458d7..8d887f12e 100644 --- a/packages/workflow-host/src/child/run-child.ts +++ b/packages/workflow-host/src/child/run-child.ts @@ -118,6 +118,11 @@ import type { InferenceSource, MailPartReader } from "@intx/types/runtime"; import type { CredentialDelivery } from "@intx/types/credential-delivery"; import { createWorkflowRunRepoStore } from "../adapters/repo-store"; +import { + createResumeDeliveryHold, + parkedSignalNames, + type ResumeDeliveryHold, +} from "./resume-delivery-hold"; import { createCancellationBarrier } from "./cancellation-barrier"; import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate"; import type { @@ -910,6 +915,7 @@ export async function runWorkflowChild( // TransitionError into its fire-and-forget continuation. Each site // removes its entry when the run reaches terminal. const runsInFlight = new Map(); + const resumeDeliveryHold = createResumeDeliveryHold(); const cancellationBarrier = createCancellationBarrier( runtimeRepoStore, upstreamSender, @@ -943,7 +949,11 @@ export async function runWorkflowChild( }); }, upstreamSender, + onSignalAwaiter: (park) => { + resumeDeliveryHold.reparked(park.runId, park.name); + }, }); + resumeDeliveryHold.hold(run.runId, parkedSignalNames(run.resumedState)); const handle = runtimeRun(definition, env, { runId: run.runId, resumeFromEvents: run.seedEvents, @@ -962,10 +972,12 @@ export async function runWorkflowChild( runId: run.runId, }); runsInFlight.delete(run.runId); + resumeDeliveryHold.runEnded(run.runId); return emitTerminalEvent(upstreamSender, result); }) .catch((cause) => { runsInFlight.delete(run.runId); + resumeDeliveryHold.runEnded(run.runId); logger.error`resumed run ${run.runId} failed: ${String(cause)}`; }); resumedRunIds.push(run.runId); @@ -1036,6 +1048,7 @@ export async function runWorkflowChild( drainController, triggeredRunIds, runsInFlight, + resumeDeliveryHold, cancellationBarrier, quietCheck, warmCache, @@ -1068,6 +1081,7 @@ export async function runWorkflowChild( const cleanupControlLoop = async (): Promise => { stopHostScheduler(opts.bindings); + resumeDeliveryHold.close(); cancellationBarrier.close("workflow-child control loop exited"); // Any exit path -- clean (iterator end), dirty (thrown error), // shutdown (already cancelled, repeat is a no-op on an empty map) @@ -1231,6 +1245,7 @@ async function handleControlPayload( drainController: DrainController; triggeredRunIds: string[]; runsInFlight: Map; + resumeDeliveryHold: ResumeDeliveryHold; cancellationBarrier: ReturnType; quietCheck: { current: AbortController | null }; warmCache: WarmAgentCache | undefined; @@ -1416,65 +1431,87 @@ async function handleControlPayload( void acknowledge(false); return false; } - // Land the signal as a `SignalReceived` commit on the run's - // event log. The signal-channel substrate's `subscribeKind` - // peer (the per-run signal channel installed at run start) is - // what resolves any pending `awaitNext` awaiter -- the - // control-loop's job is just to commit. Constructing an - // ad-hoc signal channel scoped to this runId keeps the - // control-loop free of per-run signal-channel bookkeeping - // while still routing through the canonical writer path. - // - // The deliver path writes through `writeTreePreservingPrefix`, - // which the sidecar's substrate factory wraps with a pack-push - // hook. The hook emits a `pack.push.request` on the upstream - // control channel and awaits the supervisor's matching - // `pack.push.response` on the same downstream stream this - // iterator pulls from. Awaiting the deliver inline blocks the - // iterator from pulling the response that resolves the deliver - // -- a deadlock observed end-to-end with the workflow-run - // pack-pushing wrapper. Fire the deliver off the loop so the - // iterator continues pumping `pack.push.response` (and any other - // downstream payload) while the deliver settles in the - // background. A commit failure surfaces via the logger; the - // runtime body's `signalChannel.awaitNext` peer either resolves - // (deliver landed) or remains pending until a subsequent - // delivery. - const endDelivery = beginDelivery(); - const transientSignalChannel = createWorkflowHostSignalChannel({ - repoStore: ctx.bindings.substrate, - runtimeStore: ctx.runtimeRepoStore, - principal: ctx.bindings.principal, - repoId: ctx.bindings.workflowRunRepoId, - ref: ctx.bindings.workflowRunRef, - runId: payload.data.runId, - readState: () => emptyState(payload.data.runId), - newId: () => ctx.newId("sig"), - clock: ctx.clock, - }); - void (async () => { - let committed = false; - try { - committed = await transientSignalChannel.land( - payload.data.signalName, - payload.data.payload, - payload.data.signalId, - ...(payload.data.correlationId !== undefined - ? [payload.data.correlationId] - : []), - ); - if (!committed) { - logger.warn`signal.deliver for run ${payload.data.runId} whose log already ended; refusing it (signalName=${payload.data.signalName})`; + const deliver = (): void => { + // Land the signal as a `SignalReceived` commit on the run's + // event log. The signal-channel substrate's `subscribeKind` + // peer (the per-run signal channel installed at run start) is + // what resolves any pending `awaitNext` awaiter -- the + // control-loop's job is just to commit. Constructing an + // ad-hoc signal channel scoped to this runId keeps the + // control-loop free of per-run signal-channel bookkeeping + // while still routing through the canonical writer path. + // + // The deliver path writes through `writeTreePreservingPrefix`, + // which the sidecar's substrate factory wraps with a pack-push + // hook. The hook emits a `pack.push.request` on the upstream + // control channel and awaits the supervisor's matching + // `pack.push.response` on the same downstream stream this + // iterator pulls from. Awaiting the deliver inline blocks the + // iterator from pulling the response that resolves the deliver + // -- a deadlock observed end-to-end with the workflow-run + // pack-pushing wrapper. Fire the deliver off the loop so the + // iterator continues pumping `pack.push.response` (and any other + // downstream payload) while the deliver settles in the + // background. A commit failure surfaces via the logger; the + // runtime body's `signalChannel.awaitNext` peer either resolves + // (deliver landed) or remains pending until a subsequent + // delivery. + const endDelivery = beginDelivery(); + const transientSignalChannel = createWorkflowHostSignalChannel({ + repoStore: ctx.bindings.substrate, + runtimeStore: ctx.runtimeRepoStore, + principal: ctx.bindings.principal, + repoId: ctx.bindings.workflowRunRepoId, + ref: ctx.bindings.workflowRunRef, + runId: payload.data.runId, + readState: () => emptyState(payload.data.runId), + newId: () => ctx.newId("sig"), + clock: ctx.clock, + }); + void (async () => { + let committed = false; + try { + committed = await transientSignalChannel.land( + payload.data.signalName, + payload.data.payload, + payload.data.signalId, + ...(payload.data.correlationId !== undefined + ? [payload.data.correlationId] + : []), + ); + if (!committed) { + logger.warn`signal.deliver for run ${payload.data.runId} whose log already ended; refusing it (signalName=${payload.data.signalName})`; + } + } catch (cause) { + const reason = + cause instanceof Error ? cause.message : String(cause); + logger.warn`signal.deliver commit failed runId=${payload.data.runId} signalName=${payload.data.signalName}: ${reason}`; + } finally { + await transientSignalChannel.stop(); + await acknowledge(committed); + endDelivery(); } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`signal.deliver commit failed runId=${payload.data.runId} signalName=${payload.data.signalName}: ${reason}`; - } finally { - await transientSignalChannel.stop(); - await acknowledge(committed); - endDelivery(); + })(); + }; + // A held signal is not counted as a delivery: it would keep the process + // from going quiet, which is what ends a hold no re-park ends. The + // supervisor still waits for the answer before it stops the child. + const held = ctx.resumeDeliveryHold.until( + payload.data.runId, + payload.data.signalName, + ); + if (held === undefined) { + deliver(); + return false; + } + void held.then((canLand) => { + if (canLand && ctx.runsInFlight.has(payload.data.runId)) { + deliver(); + } else { + logger.warn`signal.deliver held for run ${payload.data.runId} signalName=${payload.data.signalName} can no longer land; refusing it`; + void acknowledge(false); } - })(); + }); return false; } case "cancel.prepare": { @@ -1849,6 +1886,7 @@ function buildRuntimeEnv(args: { mailboxCallBridge?: ChildMailboxCallBridge; onEvent: (event: EventPayload) => void; upstreamSender: ControlChannelSender; + onSignalAwaiter?: (park: { runId: string; name: string }) => void; }): WorkflowRuntimeEnv { const signalChannel = createWorkflowHostSignalChannel({ repoStore: args.bindings.substrate, @@ -1955,6 +1993,9 @@ function buildRuntimeEnv(args: { onPark: (park) => { void emitParkNotify(args.upstreamSender, park); }, + ...(args.onSignalAwaiter !== undefined + ? { onSignalAwaiter: args.onSignalAwaiter } + : {}), // Let the resume classifier recover a step that crashed across the park // boundary. Absent (tests, the recursive child-workflow adapter) leaves a // crashed invocation a terminal failure. diff --git a/packages/workflow-host/src/supervisor/recycle.test.ts b/packages/workflow-host/src/supervisor/recycle.test.ts index ab93c86f7..9b8a76470 100644 --- a/packages/workflow-host/src/supervisor/recycle.test.ts +++ b/packages/workflow-host/src/supervisor/recycle.test.ts @@ -4721,6 +4721,7 @@ describe("supervisor standby and wake", () => { async function setupLiveRun( opts: { inboxPrimitives?: InboxPrimitives; + onSuspensionRegister?: WorkflowSupervisorBindings["onSuspensionRegister"]; } = {}, ) { const waits = dispatchWaits(); @@ -4947,6 +4948,79 @@ describe("supervisor standby and wake", () => { await supervisor.shutdown(); }); + test("mail the child holds past another step's park is consumed once it commits", async () => { + const inbox = createMemoryInboxPrimitives(); + const parked = Promise.withResolvers(); + const { supervisor, firstSender, first, mailBus } = await setupLiveRun({ + inboxPrimitives: inbox, + onSuspensionRegister: () => parked.resolve(undefined), + }); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-held", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("held turn")); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + await firstSender.send({ type: "park.notify", data: GATE_PARK }); + await parked.promise; + // The forbidden event is a consume before the answer, and overshooting + // this window only gives it more room to land. + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(inbox.messageIds(ADDRESS).consumed).toEqual([]); + + await confirmDelivery(first, firstSender, 1, true); + await waitUntil(() => inbox.messageIds(ADDRESS).consumed.length === 1); + + await supervisor.shutdown(); + }); + + test("mail the child holds past another step's park goes to the next child when it crashes", async () => { + const inbox = createMemoryInboxPrimitives(); + const parked = Promise.withResolvers(); + const { + supervisor, + firstSender, + first, + tracker, + mailBus, + ipcKeypair, + waits, + } = await setupLiveRun({ + inboxPrimitives: inbox, + onSuspensionRegister: () => parked.resolve(undefined), + }); + await answerParkedQuery(first, firstSender, 1, [ + { runId: RUN, correlationId: "corr-held", parkKind: "input" }, + ]); + mailBus.deliver(ADDRESS, new TextEncoder().encode("held turn")); + await waitForUpstreamPayload(first.supervisorToChild, "signal.deliver"); + await firstSender.send({ type: "park.notify", data: GATE_PARK }); + await parked.promise; + await new Promise((resolve) => setTimeout(resolve, 25)); + first.eventChildToSupervisor.close(); + first.childToSupervisor.close(); + first.resolveExit?.(1); + + await tracker.awaitChildren(2); + const second = tracker.children[1]; + if (second === undefined) throw new Error("second child missing"); + const secondSender = await driveReady(second, ipcKeypair); + await waits.nth(1); + await answerParkedQuery(second, secondSender, 1, [ + { runId: RUN, correlationId: "corr-held", parkKind: "input" }, + ]); + const delivered = await waitForUpstreamPayload( + second.supervisorToChild, + "signal.deliver", + ); + expect(delivered.data).toMatchObject({ + runId: RUN, + signalName: signalName("corr-held"), + }); + expect(inbox.messageIds(ADDRESS).consumed).toEqual([]); + + await supervisor.shutdown(); + }); + test("mail the child refused because its run ended is rejected", async () => { const inbox = createMemoryInboxPrimitives(); const consumed: unknown[] = []; diff --git a/packages/workflow-host/src/supervisor/supervisor.ts b/packages/workflow-host/src/supervisor/supervisor.ts index e030caa23..979cc3d99 100644 --- a/packages/workflow-host/src/supervisor/supervisor.ts +++ b/packages/workflow-host/src/supervisor/supervisor.ts @@ -4648,7 +4648,8 @@ export function createWorkflowSupervisor( } // The child's answer for a signal a dispatch delivered, or `undefined` once - // the cohort aborts. A child that never answers fails the dispatch, as the + // the cohort aborts. The child can hold a signal for a resumed park well past + // another step's park. A child that never answers fails the dispatch, as the // park wait's backstop does, so the mail stays reclaimable. async function waitForSignalAnswer( cohortAbort: AbortController, diff --git a/packages/workflow/src/runtime/loop-iteration-handle.test.ts b/packages/workflow/src/runtime/loop-iteration-handle.test.ts index a638f679c..ff6c2e518 100644 --- a/packages/workflow/src/runtime/loop-iteration-handle.test.ts +++ b/packages/workflow/src/runtime/loop-iteration-handle.test.ts @@ -77,6 +77,29 @@ function buildEnv( } describe("createLoopIterationHandle", () => { + test("a body does not take over the sinks of the run that spawns it", () => { + const blobs = createInMemoryBlobSubstrate(); + const resolver = (): ActionHandler => async (input) => input; + const handle = createLoopIterationHandle( + { + ...buildEnv(blobs, resolver), + onPark: noWake, + onSignalPark: noWake, + onSignalAwaiter: noWake, + }, + { + definition: body, + childRunId: "child-sinks", + input: 7, + depth: 0, + maxChildSpawnDepth: 32, + signal: new AbortController().signal, + signalChannel: createInMemorySignalChannel(), + }, + ); + expect(handle.next).toBeDefined(); + }); + test("resumes from a terminal child log without re-running the body", async () => { let runs = 0; const resolver = (): ActionHandler => async (input) => { diff --git a/packages/workflow/src/runtime/loop-iteration-handle.ts b/packages/workflow/src/runtime/loop-iteration-handle.ts index 1c722517b..e2711210d 100644 --- a/packages/workflow/src/runtime/loop-iteration-handle.ts +++ b/packages/workflow/src/runtime/loop-iteration-handle.ts @@ -63,10 +63,11 @@ export function createLoopIterationHandle( cleanup?: () => void | Promise; }, ): SuspendableChildHandle { - // `onPark`/`onSignalPark` are destructured out (and left unread) so the - // inherited env carries neither: the shared handle owns those sinks, and the - // body's parks must feed its queue, not the parent's supervisor-facing onPark. - const { onPark, onSignalPark, ...inherited } = baseEnv; + // `onPark`/`onSignalPark`/`onSignalAwaiter` are destructured out (and left + // unread) so the inherited env carries none: the shared handle owns those + // sinks, and the body's parks must feed its queue, not the parent's + // supervisor-facing onPark. + const { onPark, onSignalPark, onSignalAwaiter, ...inherited } = baseEnv; const childEnv: WorkflowRuntimeEnv = { ...inherited, signalChannel: args.signalChannel, diff --git a/packages/workflow/src/runtime/nested-loop-resume-race.test.ts b/packages/workflow/src/runtime/nested-loop-resume-race.test.ts new file mode 100644 index 000000000..0d600c834 --- /dev/null +++ b/packages/workflow/src/runtime/nested-loop-resume-race.test.ts @@ -0,0 +1,297 @@ +// A signal handed down by an outer loop while a resuming nested loop plans its +// relays. The nested loop reads its state and its log; the hand-down lands in +// the log, so it must not land where the state sees it and the plan does not. +import { expect, test } from "bun:test"; + +import { createDefaultDirectorRegistry } from "@intx/agent"; +import { + action, + awaitSignal, + createInMemoryBlobSubstrate, + createInMemoryRepoStore, + createInMemoryScheduler, + createInMemorySignalChannel, + createLoopIterationHandle, + createNoopDrainController, + defineWorkflow, + enumerateInlineLoopBodies, + loop, + loopBodyRunId, + runtimeRun, + type LoopFn, + type RepoStore, + type SignalChannel, + type WorkflowDefinition, + type WorkflowEvent, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { waitForNthEvent } from "@intx/workflow/testing"; + +import { withRunCommitBarrier } from "./commit-chain"; + +const loopFns = (ref: string): LoopFn => { + if (ref === "cont") return () => false; + if (ref === "next") return () => 0; + throw new Error(ref); +}; + +const leaf = defineWorkflow({ + id: "leaf", + trigger: { type: "manual" }, + steps: { hold: awaitSignal({ name: "go" }) }, +}); +const middle = defineWorkflow({ + id: "middle", + trigger: { type: "manual" }, + steps: { + inner: loop({ + body: leaf, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 2, + onExhausted: "innerEscalate", + }), + innerEscalate: action({ handler: "escalate", after: ["inner"] }), + }, +}); +const top: WorkflowDefinition = defineWorkflow({ + id: "top", + trigger: { type: "manual" }, + steps: { + outer: loop({ + body: middle, + while: "cont", + carry: "next", + input: { literal: 0 }, + maxIterations: 2, + onExhausted: "escalate", + }), + escalate: action({ handler: "escalate", after: ["outer"] }), + }, +}); + +// Like the host channel: a delivery lands in the run's own log, and an awaiter +// sees only what lands after it subscribed. +function logWritingChannel(repoStore: RepoStore, runId: string): SignalChannel { + const awaiters: { + name: string; + wake: () => void; + resolve: (delivery: { payload: unknown; signalId: string }) => void; + }[] = []; + return { + async deliver(name, payload, signalId) { + const id = signalId ?? `signal-${runId}`; + let landed = false; + await withRunCommitBarrier({ repoStore }, runId, async () => { + const log = await repoStore.read(runId); + if (log.some((e) => e.kind === "SignalReceived" && e.signalId === id)) { + return; + } + await repoStore.append(runId, { + kind: "SignalReceived", + seq: (log.at(-1)?.seq ?? 0) + 1, + at: new Date().toISOString(), + signalName: name, + signalId: id, + payload, + }); + landed = true; + }); + if (landed) { + const index = awaiters.findIndex((a) => a.name === name); + if (index >= 0) { + const [awaiter] = awaiters.splice(index, 1); + awaiter?.wake(); + awaiter?.resolve({ payload, signalId: id }); + } + } + }, + awaitNext(name, onWake, signal) { + if (signal?.aborted === true) { + onWake(); + return Promise.reject(new Error("aborted")); + } + return new Promise((resolve, reject) => { + const entry = { name, wake: onWake, resolve }; + awaiters.push(entry); + signal?.addEventListener( + "abort", + () => { + const index = awaiters.indexOf(entry); + if (index >= 0) awaiters.splice(index, 1); + onWake(); + reject(new Error("aborted")); + }, + { once: true }, + ); + }); + }, + }; +} + +function buildEnv( + repoStore: RepoStore, + channelFor: (runId: string) => SignalChannel, + rootRunId: string, +): WorkflowRuntimeEnv { + const clock = (): Date => new Date(); + const env: WorkflowRuntimeEnv = { + repoStore, + scheduler: createInMemoryScheduler({ repoStore, clock }), + signalChannel: channelFor(rootRunId), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: () => { + throw new Error("no agents here"); + }, + invokeAction: async () => ({ output: 1 }), + spawnChild: () => { + throw new Error("no child workflows here"); + }, + clock, + newId: (prefix) => `${prefix}-${Math.random().toString(36).slice(2, 8)}`, + drain: createNoopDrainController(top), + hasUpstreamSignalResolver: true, + loopFns, + }; + const bodies = new Map(); + const collect = (definition: WorkflowDefinition): void => { + for (const body of enumerateInlineLoopBodies(definition)) { + bodies.set(body.ref, body.definition); + collect(body.definition); + } + }; + collect(top); + env.spawnLoopIteration = async (args) => { + const definition = bodies.get(args.definitionRef); + if (definition === undefined) throw new Error(args.definitionRef); + return createLoopIterationHandle(env, { + definition, + childRunId: args.childRunId, + input: args.input, + depth: args.depth, + maxChildSpawnDepth: args.maxChildSpawnDepth, + ...(args.resumeFromEvents !== undefined + ? { resumeFromEvents: args.resumeFromEvents } + : {}), + signal: args.signal, + signalChannel: channelFor(args.childRunId), + }); + }; + return env; +} + +function channelsFor( + make: (runId: string) => SignalChannel, +): (runId: string) => SignalChannel { + const channels = new Map(); + return (runId) => { + let channel = channels.get(runId); + if (channel === undefined) { + channel = make(runId); + channels.set(runId, channel); + } + return channel; + }; +} + +const RUN_ID = "nested-race"; +const MIDDLE_RUN_ID = loopBodyRunId(RUN_ID, "outer", 0); +const LEAF_RUN_ID = loopBodyRunId(MIDDLE_RUN_ID, "inner", 0); + +async function parkedLogs(): Promise> { + const store = createInMemoryRepoStore(); + const channelFor = channelsFor(() => createInMemorySignalChannel()); + const abort = new AbortController(); + const run = runtimeRun(top, buildEnv(store, channelFor, RUN_ID), { + runId: RUN_ID, + localAbort: abort.signal, + }); + await waitForNthEvent( + store, + RUN_ID, + (e) => e.kind === "SignalAwaited" && e.parkKind === "signal-relay", + 1, + ); + const logs = new Map(); + for (const id of [RUN_ID, MIDDLE_RUN_ID, LEAF_RUN_ID]) { + logs.set(id, [...(await store.read(id))]); + } + abort.abort(); + await run.complete; + return logs; +} + +// `runLoop`'s own read of the log, not one by a helper it calls. +function isRunLoopRead(): boolean { + const frame = (new Error().stack ?? "") + .split("\n") + .find((line) => line.includes("runtime/run.ts")); + return frame?.includes("runLoop") === true; +} + +test.each([ + { landsAfter: "the log read", snapshotFirst: true }, + { landsAfter: "the state read", snapshotFirst: false }, +])( + "a signal handed down to a resuming nested loop right after $landsAfter still reaches its body", + async ({ snapshotFirst }) => { + const logs = await parkedLogs(); + const inner = createInMemoryRepoStore(); + for (const id of [MIDDLE_RUN_ID, LEAF_RUN_ID]) { + for (const event of logs.get(id) ?? []) await inner.append(id, event); + } + + const handedDown = Promise.withResolvers(); + const loopRead = Promise.withResolvers(); + let held = false; + const store: RepoStore = { + ...inner, + read: async (id) => { + const holds = id === MIDDLE_RUN_ID && !held && isRunLoopRead(); + if (!holds) return inner.read(id); + held = true; + loopRead.resolve(undefined); + if (snapshotFirst) { + const snapshot = [...(await inner.read(id))]; + await handedDown.promise; + return snapshot; + } + await handedDown.promise; + return inner.read(id); + }, + }; + const channelFor = channelsFor((id) => { + const channel = logWritingChannel(store, id); + if (id !== MIDDLE_RUN_ID) return channel; + return { + ...channel, + deliver: async (...args) => { + await channel.deliver(...args); + handedDown.resolve(undefined); + }, + }; + }); + + const env = buildEnv(store, channelFor, RUN_ID); + const awaiter = Promise.withResolvers(); + env.onSignalAwaiter = () => { + awaiter.resolve(undefined); + }; + const run = runtimeRun(top, env, { + runId: RUN_ID, + resumeFromEvents: logs.get(RUN_ID) ?? [], + }); + await loopRead.promise; + await awaiter.promise; + await channelFor(RUN_ID).deliver("go", { ok: true }, "signal-1"); + + expect((await run.complete).terminalStatus).toBe("completed"); + }, +); From b06fdefa04381f1947e148e444bb3f881420ef18 Mon Sep 17 00:00:00 2001 From: FarinM Date: Wed, 28 Oct 2026 15:28:58 +0100 Subject: [PATCH 19/20] Put a workflow deployment in standby and wake it over the sidecar link The supervisor can put a waiting deployment's child in standby, but nothing outside the sidecar could ask it to. The Hub can now send standby and wake commands over the link, and confirms a standby request only once it holds the history the stopped child left. The deployment stays routed, so an input wakes it; the sidecar reports that wake, holding the report through a link outage, and every hello says whether the child is in standby. The run record keeps what the child was parked on, so a sidecar restart keeps the deployment in standby without the child while its history is unchanged. The sidecar bounds the length of a control answer's error, and the Hub rejects a longer one. An answer that fails validation rejects the pending control it names, which could otherwise wait forever. --- Makefile | 2 +- apps/sidecar/src/index.ts | 2 + docs/HARNESS_DESIGN.md | 19 +- docs/IMPLEMENTATION.md | 15 +- .../hub-agent/src/sidecar-orchestrator.ts | 16 + packages/hub-agent/src/ws/hub-link.test.ts | 215 ++++++- packages/hub-agent/src/ws/hub-link.ts | 94 ++- .../hub-sessions/src/session-service.test.ts | 6 + .../hub-sessions/src/ws/pending-tracker.ts | 40 +- .../hub-sessions/src/ws/sidecar-events.ts | 14 + .../src/ws/sidecar-handler.control.test.ts | 354 ++++++++++++ .../hub-sessions/src/ws/sidecar-handler.ts | 270 ++++++++- .../ws/sidecar-handler.work-admission.test.ts | 62 ++ packages/types/src/sidecar.test.ts | 59 ++ packages/types/src/sidecar.ts | 71 ++- .../src/deploy/workflow-host-wiring.test.ts | 539 +++++++++++++++++- .../src/deploy/workflow-host-wiring.ts | 229 +++++++- .../src/deploy/workflow-run-record.test.ts | 46 ++ .../src/deploy/workflow-run-record.ts | 30 + tests/workflow-deploy/drain-roundtrip.test.ts | 150 +++++ .../standby-approval-wake.test.ts | 262 +++++++++ .../standby-cancel-wake.test.ts | 217 +++++++ .../standby-childworkflow-signal.test.ts | 279 +++++++++ tests/workflow-deploy/standby-helpers.ts | 31 + .../standby-mail-wakes-input-park.test.ts | 245 ++++++++ .../standby-on-trigger-between-events.test.ts | 274 +++++++++ .../standby-restart-approval.test.ts | 337 +++++++++++ tests/workflow-deploy/standby-restart.test.ts | 282 +++++++++ .../standby-signal-wake.test.ts | 221 +++++++ .../standby-two-child-parks.test.ts | 242 ++++++++ 30 files changed, 4532 insertions(+), 91 deletions(-) create mode 100644 tests/workflow-deploy/standby-approval-wake.test.ts create mode 100644 tests/workflow-deploy/standby-cancel-wake.test.ts create mode 100644 tests/workflow-deploy/standby-childworkflow-signal.test.ts create mode 100644 tests/workflow-deploy/standby-helpers.ts create mode 100644 tests/workflow-deploy/standby-mail-wakes-input-park.test.ts create mode 100644 tests/workflow-deploy/standby-on-trigger-between-events.test.ts create mode 100644 tests/workflow-deploy/standby-restart-approval.test.ts create mode 100644 tests/workflow-deploy/standby-restart.test.ts create mode 100644 tests/workflow-deploy/standby-signal-wake.test.ts create mode 100644 tests/workflow-deploy/standby-two-child-parks.test.ts diff --git a/Makefile b/Makefile index edc9e0e21..e7db10512 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-unit: FORCE # (port 0, mkdtemp dirs, unique schemas) and none rely on fresh module state, # so sharing the registry across files is safe. test-workflow: FORCE - $(BUN) test --timeout 120000 --parallel=$(WF_PARALLEL) --no-isolate tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/hub-link-reconnect.test.ts tests/workflow-deploy/reconnect-reemits-parked-correlation.test.ts tests/workflow-deploy/interrupted-pack-reconnect-recovery.test.ts tests/workflow-deploy/deploy-window-reconnect-recovery.test.ts tests/workflow-deploy/midrun-signal-survives-reconnect.test.ts tests/workflow-deploy/multistep-perstep-reroute-reconnect.test.ts tests/workflow-deploy/crash-restart-reconnect-resumes.test.ts tests/workflow-deploy/crash-respawn-fifo.test.ts tests/workflow-deploy/crash-loop-latch.test.ts tests/workflow-deploy/on-trigger-signal-delivery.test.ts tests/workflow-deploy/on-trigger-between-events-restart.test.ts tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts tests/workflow-deploy/sleep-restart-keeps-authority.test.ts tests/workflow-deploy/sleep-restart-hub-access.test.ts tests/workflow-deploy/on-trigger-agent-body.test.ts tests/workflow-deploy/on-trigger-action-body.test.ts tests/workflow-deploy/on-trigger-loop-body.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/map-fan-out-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-credential-tool.test.ts tests/workflow-deploy/credential-bound-route-deploy.test.ts tests/workflow-deploy/action-step-credential.test.ts tests/workflow-deploy/action-hub-access.test.ts tests/workflow-deploy/tool-hub-access.test.ts tests/workflow-deploy/invoker-hub-access.test.ts tests/workflow-deploy/body-hub-access.test.ts tests/workflow-deploy/revoked-creator-grant-running-run.test.ts tests/workflow-deploy/expired-run-grant-running-run.test.ts tests/workflow-deploy/tarball-route-deploy.test.ts tests/workflow-deploy/folded-tools-failover-real-agent.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-pinned-ask-tool.test.ts tests/workflow-deploy/single-step-per-run-grants.test.ts tests/workflow-deploy/approval-tracer-roundtrip.test.ts tests/workflow-deploy/scope-always-roundtrip.test.ts tests/workflow-deploy/childworkflow-scope-always-roundtrip.test.ts tests/workflow-deploy/mail-trigger-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-trigger-run-completes-real-route.test.ts tests/workflow-deploy/mail-send-denied-real-route.test.ts tests/workflow-deploy/inbound-mail-enforcement-real-route.test.ts tests/workflow-deploy/run-mail-send-real-route.test.ts tests/workflow-deploy/mail-federation-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-federation-run-completes.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/multistep-conversation-failover.test.ts tests/workflow-deploy/per-level-pipeline-real-agents.test.ts tests/workflow-deploy/dispatch-orchestrator-real-agents.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/single-step-mail-loop.test.ts tests/workflow-deploy/single-step-mail-in-reply-to.test.ts tests/workflow-deploy/single-step-mail-wait-wake.test.ts tests/workflow-deploy/single-step-mail-two-turn-thread.test.ts tests/workflow-deploy/single-step-mail-interactive-turns.test.ts tests/workflow-deploy/single-step-mail-flag-expunge.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/conversation-state-connector-seed.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/fixtures/child-workflow.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/child-workflow-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-inherited-grants-roundtrip.test.ts tests/workflow-deploy/on-trigger-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-roundtrip.test.ts tests/workflow-deploy/loop-nested-roundtrip.test.ts tests/workflow-deploy/loop-action-roundtrip.test.ts tests/workflow-deploy/all-action-deploy.test.ts tests/workflow-deploy/sidecar-runs-no-workflow-code.test.ts tests/workflow-deploy/onfailure-action-roundtrip.test.ts tests/workflow-deploy/loop-await-signal-restart-resumes.test.ts tests/workflow-deploy/loop-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-work-then-signal-restart-resumes.test.ts tests/workflow-deploy/loop-await-signal-drain-roundtrip.test.ts tests/workflow-deploy/loop-childworkflow-inflight-grandchild-drain.test.ts tests/workflow-deploy/childworkflow-restart-resumes.test.ts tests/workflow-deploy/body-action-restart-effect-once.test.ts tests/workflow-deploy/body-action-crash-mid-effect.test.ts tests/workflow-deploy/childworkflow-drain-wait-resumes.test.ts tests/workflow-deploy/childworkflow-unanswerable-gate.test.ts tests/workflow-deploy/on-trigger-drain-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/workflow-deploy/run-event-batching.test.ts tests/workflow-deploy/walking-skeleton.e2e.test.ts tests/workflow-deploy/asset-source.e2e.test.ts tests/workflow-deploy/asset-tool-restarts.e2e.test.ts tests/workflow-deploy/source-workflow.e2e.test.ts tests/workflow-deploy/source-monorepo.e2e.test.ts tests/workflow-deploy/source-monorepo-many.e2e.test.ts tests/workflow-deploy/source-tamper-reject.e2e.test.ts tests/workflow-deploy/source-catalog.e2e.test.ts tests/workflow-deploy/source-mixed-closure.e2e.test.ts tests/workflow-deploy/source-closure-director.e2e.test.ts tests/workflow-deploy/source-credential.e2e.test.ts tests/workflow-deploy/source-inline-tool.e2e.test.ts tests/workflow-deploy/source-posix-lsp.e2e.test.ts tests/workflow-deploy/loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-credential-tool.test.ts tests/workflow-deploy/loop-map-tool-invoke-roundtrip.test.ts tests/workflow-deploy/loop-nested-tool-invoke-roundtrip.test.ts + $(BUN) test --timeout 120000 --parallel=$(WF_PARALLEL) --no-isolate tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/hub-link-reconnect.test.ts tests/workflow-deploy/reconnect-reemits-parked-correlation.test.ts tests/workflow-deploy/interrupted-pack-reconnect-recovery.test.ts tests/workflow-deploy/deploy-window-reconnect-recovery.test.ts tests/workflow-deploy/midrun-signal-survives-reconnect.test.ts tests/workflow-deploy/multistep-perstep-reroute-reconnect.test.ts tests/workflow-deploy/crash-restart-reconnect-resumes.test.ts tests/workflow-deploy/standby-restart.test.ts tests/workflow-deploy/standby-restart-approval.test.ts tests/workflow-deploy/standby-approval-wake.test.ts tests/workflow-deploy/standby-mail-wakes-input-park.test.ts tests/workflow-deploy/standby-on-trigger-between-events.test.ts tests/workflow-deploy/standby-childworkflow-signal.test.ts tests/workflow-deploy/standby-signal-wake.test.ts tests/workflow-deploy/standby-cancel-wake.test.ts tests/workflow-deploy/standby-two-child-parks.test.ts tests/workflow-deploy/crash-respawn-fifo.test.ts tests/workflow-deploy/crash-loop-latch.test.ts tests/workflow-deploy/on-trigger-signal-delivery.test.ts tests/workflow-deploy/on-trigger-between-events-restart.test.ts tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts tests/workflow-deploy/sleep-restart-keeps-authority.test.ts tests/workflow-deploy/sleep-restart-hub-access.test.ts tests/workflow-deploy/on-trigger-agent-body.test.ts tests/workflow-deploy/on-trigger-action-body.test.ts tests/workflow-deploy/on-trigger-loop-body.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/map-fan-out-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-credential-tool.test.ts tests/workflow-deploy/credential-bound-route-deploy.test.ts tests/workflow-deploy/action-step-credential.test.ts tests/workflow-deploy/action-hub-access.test.ts tests/workflow-deploy/tool-hub-access.test.ts tests/workflow-deploy/invoker-hub-access.test.ts tests/workflow-deploy/body-hub-access.test.ts tests/workflow-deploy/revoked-creator-grant-running-run.test.ts tests/workflow-deploy/expired-run-grant-running-run.test.ts tests/workflow-deploy/tarball-route-deploy.test.ts tests/workflow-deploy/folded-tools-failover-real-agent.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-pinned-ask-tool.test.ts tests/workflow-deploy/single-step-per-run-grants.test.ts tests/workflow-deploy/approval-tracer-roundtrip.test.ts tests/workflow-deploy/scope-always-roundtrip.test.ts tests/workflow-deploy/childworkflow-scope-always-roundtrip.test.ts tests/workflow-deploy/mail-trigger-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-trigger-run-completes-real-route.test.ts tests/workflow-deploy/inbound-mail-enforcement-real-route.test.ts tests/workflow-deploy/run-mail-send-real-route.test.ts tests/workflow-deploy/mail-send-denied-real-route.test.ts tests/workflow-deploy/mail-federation-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-federation-run-completes.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/multistep-conversation-failover.test.ts tests/workflow-deploy/per-level-pipeline-real-agents.test.ts tests/workflow-deploy/dispatch-orchestrator-real-agents.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/single-step-mail-loop.test.ts tests/workflow-deploy/single-step-mail-in-reply-to.test.ts tests/workflow-deploy/single-step-mail-wait-wake.test.ts tests/workflow-deploy/single-step-mail-two-turn-thread.test.ts tests/workflow-deploy/single-step-mail-interactive-turns.test.ts tests/workflow-deploy/single-step-mail-flag-expunge.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/conversation-state-connector-seed.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/fixtures/child-workflow.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/child-workflow-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-inherited-grants-roundtrip.test.ts tests/workflow-deploy/on-trigger-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-roundtrip.test.ts tests/workflow-deploy/loop-nested-roundtrip.test.ts tests/workflow-deploy/loop-action-roundtrip.test.ts tests/workflow-deploy/all-action-deploy.test.ts tests/workflow-deploy/sidecar-runs-no-workflow-code.test.ts tests/workflow-deploy/onfailure-action-roundtrip.test.ts tests/workflow-deploy/loop-await-signal-restart-resumes.test.ts tests/workflow-deploy/loop-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-work-then-signal-restart-resumes.test.ts tests/workflow-deploy/loop-await-signal-drain-roundtrip.test.ts tests/workflow-deploy/loop-childworkflow-inflight-grandchild-drain.test.ts tests/workflow-deploy/childworkflow-restart-resumes.test.ts tests/workflow-deploy/body-action-restart-effect-once.test.ts tests/workflow-deploy/body-action-crash-mid-effect.test.ts tests/workflow-deploy/childworkflow-drain-wait-resumes.test.ts tests/workflow-deploy/childworkflow-unanswerable-gate.test.ts tests/workflow-deploy/on-trigger-drain-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/workflow-deploy/run-event-batching.test.ts tests/workflow-deploy/walking-skeleton.e2e.test.ts tests/workflow-deploy/asset-source.e2e.test.ts tests/workflow-deploy/asset-tool-restarts.e2e.test.ts tests/workflow-deploy/source-workflow.e2e.test.ts tests/workflow-deploy/source-monorepo.e2e.test.ts tests/workflow-deploy/source-monorepo-many.e2e.test.ts tests/workflow-deploy/source-tamper-reject.e2e.test.ts tests/workflow-deploy/source-catalog.e2e.test.ts tests/workflow-deploy/source-mixed-closure.e2e.test.ts tests/workflow-deploy/source-closure-director.e2e.test.ts tests/workflow-deploy/source-credential.e2e.test.ts tests/workflow-deploy/source-inline-tool.e2e.test.ts tests/workflow-deploy/source-posix-lsp.e2e.test.ts tests/workflow-deploy/loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-credential-tool.test.ts tests/workflow-deploy/loop-map-tool-invoke-roundtrip.test.ts tests/workflow-deploy/loop-nested-tool-invoke-roundtrip.test.ts # The core integration pass: inference, hub-api, and db suites that need # real Postgres and the extended timeout. Files run in parallel diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 60b5a77dd..4bbeba70e 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -607,6 +607,7 @@ const orchestrator = createSidecarOrchestrator({ publishWorkflowInferenceEvent, publishWorkflowSuspension, publishDeploymentStopped, + publishDeploymentWoken, }) => { const router = createSidecarDeployRouter({ ...capacity, @@ -628,6 +629,7 @@ const orchestrator = createSidecarOrchestrator({ reportDeploymentRefTips: (incarnation) => wrappedRepoStore.reportWorkflowRunRefTips(incarnation), publishDeploymentStopped, + publishDeploymentWoken, // The run record lives in the repository and goes last, so a crash // partway through leaves it for the next boot to report. removeRunRepository: (runId) => diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index c645d8ddd..4cfcbe93f 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -141,7 +141,7 @@ The per-agent key directory is keyed by the sanitized run address; the workflow Nothing creates or reads `agents/` under `SIDECAR_DATA_DIR` or `HUB_DATA_DIR`. Earlier versions kept agent-state repositories there; one left behind is safe to delete. -The `deployment.json` record stores only what a restart cannot otherwise recover: the deployment's `agentAddress` and the allocation `generation` it was deployed for, the `definitionId` naming its workflow definition, the `sourceRef` pin a restore re-materializes the closure from, the sealed credential cell, each step's ordered inference-**sources** failover chain (`sources`), and the optional inference `sessionId`. A `version` field guards the schema so a stale record can be rejected rather than parsed blindly. A deployment that no longer runs keeps its record until the Hub undeploys it, marked `stopped`, with the error that ended it unless the Hub stopped it, or `tearing-down`, so a restart reports it instead of spawning it. Its optional `retention` marker records whether keeping the copy was accepted (`kept`) or refused (`refused`); only durably kept copies leave the active capacity pool. The record deliberately does **not** duplicate the workflow definition (re-materialized from its `sourceRef` closure at restore). The record is written owner-only (mode 0600). +The `deployment.json` record stores only what a restart cannot otherwise recover: the deployment's `agentAddress` and the allocation `generation` it was deployed for, the `definitionId` naming its workflow definition, the `sourceRef` pin a restore re-materializes the closure from, the sealed credential cell, each step's ordered inference-**sources** failover chain (`sources`), and the optional inference `sessionId`. A `version` field guards the schema so a stale record can be rejected rather than parsed blindly. A deployment that no longer runs keeps its record until the Hub undeploys it, marked `stopped`, with the error that ended it unless the Hub stopped it, or `tearing-down`, so a restart reports it instead of spawning it. Its optional `retention` marker records whether keeping the copy was accepted (`kept`) or refused (`refused`); only durably kept copies leave the active capacity pool. A deployment whose child is in standby also carries `standby`: the correlations the child was parked on, its earliest timer, the tip of the run's history, and the projection its first child verified, so a restart that finds that tip unchanged keeps it in standby without starting the child, laying its steps out from that projection once it hashes to the approved one. Outside `standby`, the record deliberately does **not** duplicate the workflow definition (re-materialized from its `sourceRef` closure at restore). The record is written owner-only (mode 0600). A live source rotation for a single-step deployment overwrites this record's `sources` before it takes effect. Persistence is what makes a rotation durable: a rotation whose write fails is not durable, and the deployment falls back to the last durably-recorded source list on the next recycle or restart. @@ -177,21 +177,22 @@ The Hub mints a sidecar identity for each probe or allocation generation and han Possession of the raw token is sufficient to authenticate as that sidecar, for every probe and allocation it currently hosts, including cleanup bindings. Provisioners and workers must never log it. A provisioner that must persist the token for restartable capacity must use access-controlled secret storage and keep it with any worker state that can be restarted, deleting it only when the worker and its restorable state are permanently removed. Connections crossing a non-loopback or otherwise untrusted transport must use `wss://`; plaintext `ws://` is only appropriate for local loopback development. -Every connection opens with a `hello` frame reporting each incarnation the worker holds and what it is doing with it: `deploying`, `live`, `stopped`, or `tearing-down`. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop, because its workflow child ended itself or the worker could not restore it, carries the `error` that ended it, and the Hub fails its deployment. The worker also reports such a stop in a `deployment.stopped` frame when it happens and after each `welcome` until retention is decided, with the tips of the deployment's history branches, and the Hub fails the deployment once it holds that history, or after a bounded wait. The worker restores its deployments from storage before it first connects, so that `hello` already lists them. The Hub accepts the frame when the token resolves to current work or a cleanup obligation (a release, a replacement, or a failed cleanup); a cleanup-only connection receives no routes. It routes a reported incarnation only when it is live, its generation is the current generation of an allocation the sidecar hosts, that generation has finished initializing, and its run has not ended. It keeps current stopped copies and initialized terminal copies unrouted. After welcome it stops terminal live copies, requesting retention only while their policy allows it. Durable kept copies use the retained pool. Cleanup bindings leave removal to allocation reconciliation; the Hub sends `agent.undeploy` for other unwanted incarnations instead of closing the socket, so one stale deployment cannot disconnect the sidecar's other work, and then answers `welcome` with the incarnations it routes. +Every connection opens with a `hello` frame reporting each incarnation the worker holds and what it is doing with it: `deploying`, `live`, `stopped`, or `tearing-down`. A `live` one whose workflow child is in standby also carries `inStandby`: the worker stopped the child while every run waited and respawns it for its next input, and it reports such a wake in a `deployment.woken` frame. A deployment in standby the worker restores stays in standby without a child while its history is where the stopped child left it, and is reported so. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop, because its workflow child ended itself or the worker could not restore it, carries the `error` that ended it, and the Hub fails its deployment. The worker also reports such a stop in a `deployment.stopped` frame when it happens and after each `welcome` until retention is decided, with the tips of the deployment's history branches, and the Hub fails the deployment once it holds that history, or after a bounded wait. The worker restores its deployments from storage before it first connects, so that `hello` already lists them. The Hub accepts the frame when the token resolves to current work or a cleanup obligation (a release, a replacement, or a failed cleanup); a cleanup-only connection receives no routes. It routes a reported incarnation only when it is live, its generation is the current generation of an allocation the sidecar hosts, that generation has finished initializing, and its run has not ended. It keeps current stopped copies and initialized terminal copies unrouted. After welcome it stops terminal live copies, requesting retention only while their policy allows it. Durable kept copies use the retained pool. Cleanup bindings leave removal to allocation reconciliation; the Hub sends `agent.undeploy` for other unwanted incarnations instead of closing the socket, so one stale deployment cannot disconnect the sidecar's other work, and then answers `welcome` with the incarnations it routes. The sidecar defaults to 128 active reservations and 256 kept records, configured at startup with `SIDECAR_MAX_ACTIVE_DEPLOYMENTS` and `SIDECAR_MAX_RETAINED_DEPLOYMENTS`. It advertises those limits and reports both pools in full; the Hub uses the advertised active limit for placement. The combined protocol ceiling is 2,048 records, and the complete `hello` must fit the 64 MiB WebSocket frame-size limit. Hello errors use the same 4,096-character limit as individual stop reports. Restoration exceeding either configured limit fails startup; it does not omit records from `hello`. -The worker sends nothing the Hub must receive before `welcome`. It queues the reports it owes (outbound mail and signal correlation registrations), drops best-effort events, and holds back workflow-run pack pushes. On `welcome` it sends the queue and re-drives each routed incarnation's pending pack and parked correlations, so nothing it re-sends can overtake route restoration. The Hub answers every `hello`: with `welcome`, or by closing the socket when the frame is invalid or registration fails before the welcome. A worker that gets no `welcome` within 30 seconds reconnects. A worker whose token is valid but that hosts neither current work nor cleanup obligations gets a `welcome` with nothing routed after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, until its provisioner places new work on it or stops it. Trigger and signal durability lives in `workflow_run_dispatch`; the Hub does not maintain an unscoped, in-memory queue for arbitrary disconnected sidecars. +The worker sends nothing the Hub must receive before `welcome`. It queues the reports it owes (outbound mail and signal correlation registrations), keeps the latest wake report of each deployment, drops best-effort events, and holds back workflow-run pack pushes. On `welcome` it sends the wake reports and then the queue, and re-drives each routed incarnation's pending pack and parked correlations, so nothing it re-sends can overtake route restoration. The Hub answers every `hello`: with `welcome`, or by closing the socket when the frame is invalid or registration fails before the welcome. A worker that gets no `welcome` within 30 seconds reconnects. A worker whose token is valid but that hosts neither current work nor cleanup obligations gets a `welcome` with nothing routed after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, until its provisioner places new work on it or stops it. Trigger and signal durability lives in `workflow_run_dispatch`; the Hub does not maintain an unscoped, in-memory queue for arbitrary disconnected sidecars. -| Direction | Frame | Fields | Description | -| ------------- | -------------------- | -------------------------------------------------------------------------- | --------------------------------------------------------------- | -| Sidecar → Hub | `hello` | `sidecarId`, `token`, `capacity`, `incarnations`, `cachedSenderAddresses?` | Authenticate and report every incarnation the sidecar holds | -| Hub → Sidecar | `welcome` | `routed` | The incarnations the Hub routes on this connection | -| Sidecar → Hub | `deployment.stopped` | `agentAddress`, `generation`, `error`, `refTips?` | Report a deployment that stopped though the Hub did not stop it | +| Direction | Frame | Fields | Description | +| ------------- | -------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| Sidecar → Hub | `hello` | `sidecarId`, `token`, `capacity`, `incarnations`, `cachedSenderAddresses?` | Authenticate and report every incarnation the sidecar holds | +| Hub → Sidecar | `welcome` | `routed` | The incarnations the Hub routes on this connection | +| Sidecar → Hub | `deployment.stopped` | `agentAddress`, `generation`, `error`, `refTips?` | Report a deployment that stopped though the Hub did not stop it | +| Sidecar → Hub | `deployment.woken` | `agentAddress`, `generation`, `reason` | Report what woke a deployment in standby when the Hub did not ask for it | ## Self-Restoration -At boot, the sidecar validates its stored deployment records and restores runnable copies through the normal supervised spawn path, eight at a time, before connecting. Retained, stopped and tearing-down copies stay unspawned. Failed restores are recorded as stopped with their error until the Hub undeploys them. Storage survival depends on the provisioner's advertised isolation and recovery guarantees. +At boot, the sidecar validates its stored deployment records and restores runnable copies through the normal supervised spawn path, eight at a time, before connecting. A copy recorded in standby whose run history has not changed since, and whose recorded projection hashes to the approved one, is restored in standby instead: no child is spawned, the correlations its runs wait on are registered again, and input or its earliest timer wakes it. Any other copy recorded in standby starts its child and reports itself woken. Retained, stopped and tearing-down copies stay unspawned. Failed restores are recorded as stopped with their error until the Hub undeploys them. Storage survival depends on the provisioner's advertised isolation and recovery guarantees. If all `N` respawns use their full ready timeout, their waits take roughly `ceil(N / 8) × readyTimeout`: 8 minutes for the default 128 active deployments, or 64 minutes for 1,024, at the default 30 seconds. Allow additional startup time when setting [`maxDisconnected`](workflow-lifecycle-policy.md), especially when raising `SIDECAR_MAX_ACTIVE_DEPLOYMENTS`; capacity changes do not extend the deadline. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index ccc4b13e6..cb263006a 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -263,7 +263,7 @@ The sidecar WebSocket protocol includes frames for agent deployment and reconnec Agent deployment stages through the workflow-run substrate rather than a separate provision-then-start handshake. The hub sends `agent.deploy`; the sidecar's deploy router spawns the supervised workflow-process child, then acks with `agent.deploy.ack` carrying the public key. The workflow-process child starts inference itself once spawned -- there is no separate `session.start` step. -A deployment on a sidecar is an incarnation: its address plus the allocation generation it was deployed for. `agent.deploy` and `agent.undeploy` carry a `requestId` and the `generation` they act on, and every reply echoes both, so a reply settles only the request it answers and a late reply to a request the Hub already gave up on matches nothing. The sidecar handles the frames for one address one at a time, in the order it received them. Two kinds of frame do not hold the address's turn: the Hub's answers to its own pack transfers and correlation registrations, which only settle the request they answer, and a `workflow.control`, which starts in order but frees the turn while a cooperative cancellation waits for the child. Frames for different addresses, probe requests, and heartbeat pongs do not wait on each other, so one deployment's slow deploy or undeploy delays neither another deployment nor the link's liveness. It answers each request only on the connection that carried it; a request whose connection closed before its turn is dropped, and the Hub has already failed it. An undeploy is the exception: it still runs and only its answer is dropped, because the Hub sends it for a generation it will not route again and may close the socket right behind it. +A deployment on a sidecar is an incarnation: its address plus the allocation generation it was deployed for. `agent.deploy` and `agent.undeploy` carry a `requestId` and the `generation` they act on, and every reply echoes both, so a reply settles only the request it answers and a late reply to a request the Hub already gave up on matches nothing. The sidecar handles the frames for one address one at a time, in the order it received them. Two kinds of frame do not hold the address's turn: the Hub's answers to its own pack transfers and correlation registrations, which only settle the request they answer, and a `workflow.control`, which starts in order but frees the turn while a cooperative cancellation or a standby request waits for the child. Frames for different addresses, probe requests, and heartbeat pongs do not wait on each other, so one deployment's slow deploy or undeploy delays neither another deployment nor the link's liveness. It answers each request only on the connection that carried it; a request whose connection closed before its turn is dropped, and the Hub has already failed it. An undeploy is the exception: it still runs and only its answer is dropped, because the Hub sends it for a generation it will not route again and may close the socket right behind it. A sidecar holds at most one incarnation of an address, and an address is deployed once: a deploy of an address the sidecar holds is refused with `agent.deploy.error`, and a Hub pack for it with `conflict`. @@ -271,17 +271,18 @@ Undeploy is an acknowledged operation. The sidecar shuts the named generation's **Reconnection:** -| Direction | Frame | Purpose | -| ------------- | -------------------- | --------------------------------------------------------------- | -| Sidecar → Hub | `hello` | Authenticate and report every incarnation the sidecar holds | -| Hub → Sidecar | `welcome` | Name the incarnations the Hub routes after reconciling `hello` | -| Sidecar → Hub | `deployment.stopped` | Report a deployment that stopped though the Hub did not stop it | +| Direction | Frame | Purpose | +| ------------- | -------------------- | ------------------------------------------------------------------------ | +| Sidecar → Hub | `hello` | Authenticate and report every incarnation the sidecar holds | +| Hub → Sidecar | `welcome` | Name the incarnations the Hub routes after reconciling `hello` | +| Sidecar → Hub | `deployment.stopped` | Report a deployment that stopped though the Hub did not stop it | +| Sidecar → Hub | `deployment.woken` | Report what woke a deployment in standby when the Hub did not ask for it | **Grant management:** The live `grants.update` wire path has been retired. The hub sends the run's committed grants in a `run.grants` frame ahead of each signal delivery and each trigger, and to each deployment that it routes again when a sidecar reconnects, so a standing approval decision reaches a running deployment. The sidecar keeps them in the run's grants file, and the supervisor passes them to the child over its IPC credentials snapshot at spawn and recycle. When a change to a delegator's grants re-derives a run's rows, the hub sends the run its grants on their own after the change commits (see AUTH.md, _Grant Revocation_). -Every connection opens with a `hello` that reports each incarnation the sidecar holds and whether it is `deploying`, `live`, `stopped`, or `tearing-down`. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop carries the error that ended it: its workflow child ended itself, or the sidecar could not restore it. The Hub then fails the deployment's live runs with that error and keeps its capacity for the retention its policy sets for failed runs, so the stopped copy can be inspected. The sidecar also marks either state in the run record, so a restart reports the incarnation instead of spawning it, and reports such a stop in a `deployment.stopped` frame once that write settles, and again after each `welcome` until retention is durably decided, with the tips of the deployment's history branches. It does not wait for its pushes of that history: a push that fails is retried on every reconnect, so that wait would have no bound. The Hub fails the deployment once it holds those tips, or a minute after it first heard of the stop, and the failure then names the history it never received. A reconnect does not restart that minute, and a sidecar that disconnects instead and stays away past its disconnect limit fails the deployment with the reason it reported. The Hub holds a reported stop only in memory until it fails the deployment: a Hub restart starts that minute again from the sidecar's next report, and if the sidecar does not come back, the deployment fails without that reason. The Hub accepts the frame when the bearer token resolves to a sidecar that is not retired or still hosts a probe, allocation generation, or cleanup binding. It restores a reported incarnation's route when the incarnation is live, its generation is the current, initialized generation of a routable allocation the sidecar hosts, and its run has not ended, keeps a stopped one of the current generation of an allocation the sidecar hosts unrouted, initialized or not, and an initialized live one whose run is terminal until lifecycle retention stops it, leaves incarnations covered by cleanup bindings to the reconciler, and sends `agent.undeploy` naming the generation of every other reported incarnation rather than rejecting the connection, so one stale deployment cannot disconnect the others. It then answers `welcome` listing the incarnations it routes, as soon as routing is settled. A hello is never left unanswered: the Hub closes the socket when the hello is invalid or its registration fails before the welcome, and the sidecar reconnects when no `welcome` arrives within 30 seconds (`welcomeTimeoutMs`). A failure after the welcome is logged, since registering again would meet it again. A sidecar whose token is valid but that hosts neither current work nor cleanup obligations is welcomed with nothing routed, after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, for its provisioner to reuse. Until `welcome` the sidecar sends nothing the Hub must receive: it queues outbound mail and signal correlation registrations, drops best-effort events, and holds back the workflow-run pack pushes of every incarnation it holds; a push no connection could carry stays due instead of failing the deployment's next write. The queue holds 1024 frames for all the sidecar's deployments together: an outage that outlasts it drops queued correlation registrations first, which the next `welcome` re-sends, then the audit copies the Hub records sent mail from, and then fails a deployment's send rather than drop mail it has not yet routed. On `welcome` it sends the queue, re-drives its routed incarnations' pending packs and parked correlations, and reports stopped deployments whose retention is undecided. +Every connection opens with a `hello` that reports each incarnation the sidecar holds and whether it is `deploying`, `live`, `stopped`, or `tearing-down`. A `live` incarnation whose workflow child is in standby also carries `inStandby`. A `workflow.control` with `action: standby` stops the child once every run waits and answers with the history tips read after it stopped and the earliest pending timer, and is refused while mail waits for the child, in the deployment's inbox or taken but not committed; the Hub confirms the standby request once it holds those tips, and the incarnation keeps its route. The sidecar abandons the standby request if the connection that asked for it closes before the standby attempt settles, and runs the child again if the attempt already stopped it. It records the standby in the run record before anything can wake the child: the correlations the child was parked on, its earliest timer, the tip of the run's history read before the child reported them, and the definition the child verified. A restart that finds that tip unchanged, and that definition still hashing to the approved one, restores the standby without the child and reports it in standby; one that finds history written since, by a child that ran again or a cancellation committed while it was stopped, or cannot read the tip, starts the child, clears the record and sends `deployment.woken`. Every answer, a repeated one too, reads the tips again, since the supervisor can write history while the child is stopped, and pushes what the Hub still lacks of it; a child that ran again before the answer refuses the standby request instead. Mail, a signal, a cancellation, or the timer wakes a child in standby; the sidecar clears the mark and sends `deployment.woken`, held until the next `welcome` if the link is down. `action: wake` abandons a standby request in progress or wakes the child. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop carries the error that ended it: its workflow child ended itself, or the sidecar could not restore it. The Hub then fails the deployment's live runs with that error and keeps its capacity for the retention its policy sets for failed runs, so the stopped copy can be inspected. The sidecar also marks either state in the run record, so a restart reports the incarnation instead of spawning it, and reports such a stop in a `deployment.stopped` frame once that write settles, and again after each `welcome` until retention is durably decided, with the tips of the deployment's history branches. It does not wait for its pushes of that history: a push that fails is retried on every reconnect, so that wait would have no bound. The Hub fails the deployment once it holds those tips, or a minute after it first heard of the stop, and the failure then names the history it never received. A reconnect does not restart that minute, and a sidecar that disconnects instead and stays away past its disconnect limit fails the deployment with the reason it reported. The Hub holds a reported stop only in memory until it fails the deployment: a Hub restart starts that minute again from the sidecar's next report, and if the sidecar does not come back, the deployment fails without that reason. The Hub accepts the frame when the bearer token resolves to a sidecar that is not retired or still hosts a probe, allocation generation, or cleanup binding. It restores a reported incarnation's route when the incarnation is live, its generation is the current, initialized generation of a routable allocation the sidecar hosts, and its run has not ended, keeps a stopped one of the current generation of an allocation the sidecar hosts unrouted, initialized or not, and an initialized live one whose run is terminal until lifecycle retention stops it, leaves incarnations covered by cleanup bindings to the reconciler, and sends `agent.undeploy` naming the generation of every other reported incarnation rather than rejecting the connection, so one stale deployment cannot disconnect the others. It then answers `welcome` listing the incarnations it routes, as soon as routing is settled. A hello is never left unanswered: the Hub closes the socket when the hello is invalid or its registration fails before the welcome, and the sidecar reconnects when no `welcome` arrives within 30 seconds (`welcomeTimeoutMs`). A failure after the welcome is logged, since registering again would meet it again. A sidecar whose token is valid but that hosts neither current work nor cleanup obligations is welcomed with nothing routed, after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, for its provisioner to reuse. Until `welcome` the sidecar sends nothing the Hub must receive: it queues outbound mail and signal correlation registrations, keeps the latest wake report of each deployment, drops best-effort events, and holds back the workflow-run pack pushes of every incarnation it holds; a push no connection could carry stays due instead of failing the deployment's next write. The queue holds 1024 frames for all the sidecar's deployments together: an outage that outlasts it drops queued correlation registrations first, which the next `welcome` re-sends, then the audit copies the Hub records sent mail from, and then fails a deployment's send rather than drop mail it has not yet routed. On `welcome` it sends the wake reports and then the queue, re-drives its routed incarnations' pending packs and parked correlations, and reports stopped deployments whose retention is undecided. ### Debug and Telemetry Streams diff --git a/packages/hub-agent/src/sidecar-orchestrator.ts b/packages/hub-agent/src/sidecar-orchestrator.ts index 57fff739c..cf1b1cd83 100644 --- a/packages/hub-agent/src/sidecar-orchestrator.ts +++ b/packages/hub-agent/src/sidecar-orchestrator.ts @@ -16,6 +16,7 @@ import type { HubTransport } from "@intx/mail-memory"; import type { SignalKind } from "@intx/types"; import type { DeploymentStoppedFrame, + DeploymentWokenFrame, HostedIncarnation, SidecarCapacity, } from "@intx/types/sidecar"; @@ -103,6 +104,11 @@ export type CreateDeployRouter = (deps: { publishDeploymentStopped: ( report: Omit, ) => void; + /** + * Sink for reports that an input woke a deployment in standby. Wired to the + * hub-link's `sendDeploymentWoken`, like `publishDeploymentStopped`. + */ + publishDeploymentWoken: (report: Omit) => void; }) => DeployRouter; export type SidecarOrchestratorConfig = { @@ -331,6 +337,12 @@ export function createSidecarOrchestrator( /* replaced after HubLink construction */ }; + let dispatchDeploymentWoken: ( + report: Omit, + ) => void = () => { + /* replaced after HubLink construction */ + }; + const sessions = createSessionManager({ repoStore }); const deployRouter = createDeployRouter({ @@ -369,6 +381,9 @@ export function createSidecarOrchestrator( publishDeploymentStopped: (report) => { dispatchDeploymentStopped(report); }, + publishDeploymentWoken: (report) => { + dispatchDeploymentWoken(report); + }, }); const hubLink = createHubLink({ @@ -411,6 +426,7 @@ export function createSidecarOrchestrator( dispatchEvent = hubLink.sendEvent; dispatchSuspension = hubLink.sendSignalCorrelationRegister; dispatchDeploymentStopped = hubLink.sendDeploymentStopped; + dispatchDeploymentWoken = hubLink.sendDeploymentWoken; function start(): void { hubLink.connect(); diff --git a/packages/hub-agent/src/ws/hub-link.test.ts b/packages/hub-agent/src/ws/hub-link.test.ts index add18e140..4f1ec1a52 100644 --- a/packages/hub-agent/src/ws/hub-link.test.ts +++ b/packages/hub-agent/src/ws/hub-link.test.ts @@ -544,7 +544,58 @@ describe("sidecar↔hub integration", () => { }, ); - test("a cancellation finishing after reconnect does not reply on the new connection", async () => { + test("fits a failed control's error to the bound on sidecar text", async () => { + const hello = Promise.withResolvers(); + const acknowledged = Promise.withResolvers(); + const hub = startWelcomingHub((raw) => { + const frame = SidecarFrame.assert(JSON.parse(raw)); + if (frame.type === "hello") hello.resolve(undefined); + if ( + frame.type === "workflow.control.ack" && + frame.requestId === "long-error" + ) + acknowledged.resolve(frame); + }); + const bindings = withTestDeployBindings(); + const client = createHubLink({ + capacity: DEFAULT_SIDECAR_CAPACITY, + hubURL: `ws://localhost:${hub.server.port}/ws`, + sidecarId: "long-error", + token: "test-token", + transport: createInMemoryTransport(), + sessions: createMockSessionManager(), + ...bindings, + deployRouter: { + ...bindings.deployRouter, + control: async () => { + throw new Error("x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH * 2)); + }, + }, + }); + try { + client.connect(); + await hello.promise; + hub.send({ + type: "workflow.control", + requestId: "long-error", + action: "standby", + runId: "run_long", + agentAddress: "run_long@example.test", + generation: 1, + reason: "Asked for standby", + }); + expect(await acknowledged.promise).toMatchObject({ + type: "workflow.control.ack", + requestId: "long-error", + error: expect.stringMatching(/^x+\.\.\.$/), + }); + } finally { + client.close(); + await hub.server.stop(true); + } + }); + + test("a cancellation finishing after reconnect does not reply on the new connection, and learns its connection closed", async () => { const sockets: TestSocket[] = []; const stopped = Promise.withResolvers(); class TestSocket extends EventTarget { @@ -586,6 +637,7 @@ describe("sidecar↔hub integration", () => { const release = Promise.withResolvers(); const finished = Promise.withResolvers(); const reconnect = Promise.withResolvers<() => void>(); + let cancelSignal: AbortSignal | undefined; const bindings = withTestDeployBindings(); const client = createHubLink({ capacity: DEFAULT_SIDECAR_CAPACITY, @@ -601,8 +653,9 @@ describe("sidecar↔hub integration", () => { }, deployRouter: { ...bindings.deployRouter, - async control(frame) { + async control(frame, { signal }) { if (frame.action === "stop") return { refTips: STOPPED_REF_TIPS }; + cancelSignal = signal; started.resolve(undefined); await release.promise; finished.resolve(undefined); @@ -624,7 +677,9 @@ describe("sidecar↔hub integration", () => { }; first.receive({ ...command, action: "cancel", requestId: "cancel" }); await started.promise; + expect(cancelSignal?.aborted).toBe(false); first.close(); + expect(cancelSignal?.aborted).toBe(true); (await reconnect.promise)(); const second = sockets[1]; if (second === undefined) throw new Error("Missing second socket"); @@ -1876,8 +1931,8 @@ describe("initial handshake on connect", () => { client.connect(); try { // An event is best-effort and goes nowhere before the welcome, and so - // does a stop report, which the sidecar sends again after it; the - // register is owed and waits for it. + // does a stop report, which the sidecar sends again after it; a woken + // report and the register are owed and wait for it. client.sendEvent("live@integration.interchange", 3, "sess-queued", { type: "reactor.start", seq: 0, @@ -1890,6 +1945,12 @@ describe("initial handshake on connect", () => { refTips: { "refs/heads/main": "c".repeat(40) }, }; client.sendDeploymentStopped(stopped); + const woken = { + agentAddress: "live@integration.interchange", + generation: 3, + reason: "Woken by mail", + }; + client.sendDeploymentWoken(woken); client.sendSignalCorrelationRegister({ correlationId: "corr-queued", runId: "run-1", @@ -1915,7 +1976,15 @@ describe("initial handshake on connect", () => { sockets[0]!.send(JSON.stringify({ type: "welcome", routed: [] })); await waitUntil(() => types().includes("signal.correlation.register")); - expect(types()).toEqual(["hello", "signal.correlation.register"]); + expect(types()).toEqual([ + "hello", + "deployment.woken", + "signal.correlation.register", + ]); + expect(JSON.parse(frames[1]!)).toEqual({ + type: "deployment.woken", + ...woken, + }); client.sendDeploymentStopped(stopped); await waitUntil(() => types().includes("deployment.stopped")); @@ -2024,6 +2093,142 @@ describe("initial handshake on connect", () => { } }); + test("wake reports owed during an outage leave the queue to mail and go out once per incarnation", async () => { + const frames: string[] = []; + const sockets: { send(data: string): void; close(): void }[] = []; + const app = new Hono(); + app.get( + "/ws", + upgradeWebSocket((_c) => ({ + onOpen(_evt, ws) { + sockets.push(ws); + }, + onMessage(evt, _ws) { + if (typeof evt.data === "string") frames.push(evt.data); + }, + })), + ); + const server = Bun.serve({ fetch: app.fetch, websocket, port: 0 }); + const sender = "run_outage@integration.interchange"; + const sleepers = Array.from( + { length: 1100 }, + (_, index) => `run_sleeper${String(index)}@integration.interchange`, + ); + const transport = createInMemoryTransport(); + transport.register(sender, createEd25519Crypto(await generateKeyPair())); + let reconnect = Promise.withResolvers<() => void>(); + const client = createHubLink({ + capacity: DEFAULT_SIDECAR_CAPACITY, + hubURL: `ws://localhost:${server.port}/ws`, + sidecarId: "sc-wakes", + token: "test-token", + transport, + sessions: createMockSessionManager(), + ...withTestDeployBindings(), + getIncarnations: () => + [sender, ...sleepers].map((address) => ({ + address, + generation: 1, + state: "live" as const, + })), + scheduleReconnect(callback) { + reconnect.resolve(callback); + return () => undefined; + }, + }); + const parsed = () => + frames.map((raw) => { + const frame: { + type: string; + agentAddress?: string; + reason?: string; + delivered?: boolean; + } = JSON.parse(raw); + return frame; + }); + const hellos = () => + parsed().filter((frame) => frame.type === "hello").length; + // Each connection's frames, from its hello on. + const sinceHello = (count: number) => { + const all = parsed(); + let seen = 0; + const start = all.findIndex( + (frame) => frame.type === "hello" && (seen += 1) === count, + ); + return start < 0 ? [] : all.slice(start); + }; + const welcome = async (index: number) => { + await waitUntil(() => hellos() === index + 1); + sockets[index]!.send(JSON.stringify({ type: "welcome", routed: [] })); + }; + const dropAndReconnect = async () => { + sockets[sockets.length - 1]!.close(); + const callback = await reconnect.promise; + reconnect = Promise.withResolvers<() => void>(); + return callback; + }; + const mail = (content: string) => + transport.getTransportFor(sender).send({ + to: "remote@example.test", + type: "conversation.message", + content, + }); + // A mail goes out to be routed, then again marked delivered for audit. + const mailed = (count: number) => + parsed().filter( + (frame) => frame.type === "mail.outbound" && frame.delivered !== true, + ).length === count; + + client.connect(); + try { + await welcome(0); + const resume = await dropAndReconnect(); + for (const address of sleepers) { + client.sendDeploymentWoken({ + agentAddress: address, + generation: 1, + reason: "Woken by timer", + }); + } + // A later wake of the same incarnation replaces the one held for it, + // and one the sidecar no longer holds is not reported. + client.sendDeploymentWoken({ + agentAddress: sleepers[0]!, + generation: 1, + reason: "Woken by mail", + }); + client.sendDeploymentWoken({ + agentAddress: "run_gone@integration.interchange", + generation: 1, + reason: "Woken by mail", + }); + await mail("mail during the outage"); + + resume(); + await welcome(1); + await waitUntil(() => mailed(1)); + const second = sinceHello(2); + const woken = second.filter((frame) => frame.type === "deployment.woken"); + expect(woken.map((frame) => frame.agentAddress)).toEqual(sleepers); + expect(woken[0]?.reason).toBe("Woken by mail"); + expect(second.findIndex((frame) => frame.type === "mail.outbound")).toBe( + 1 + sleepers.length, + ); + + // A report the Hub already has is not sent again on the next connection. + (await dropAndReconnect())(); + await welcome(2); + await mail("mail after the next reconnect"); + await waitUntil(() => mailed(2)); + expect( + sinceHello(3).some((frame) => frame.type === "deployment.woken"), + ).toBe(false); + } finally { + client.close(); + await server.stop(true); + } + }); + test("an outage that fills the queue drops audit copies before it refuses mail to route", async () => { const frames: string[] = []; const sockets: { send(data: string): void }[] = []; diff --git a/packages/hub-agent/src/ws/hub-link.ts b/packages/hub-agent/src/ws/hub-link.ts index 0586a28af..7727d9796 100644 --- a/packages/hub-agent/src/ws/hub-link.ts +++ b/packages/hub-agent/src/ws/hub-link.ts @@ -21,6 +21,7 @@ import { type SessionErrorFrame, type AgentUndeployFrame, type DeploymentStoppedFrame, + type DeploymentWokenFrame, type DeploymentRetention, type WorkflowControlFrame, type WorkflowRunRefTips, @@ -349,15 +350,27 @@ export interface DeployRouter { * routers can omit the implementation; the link then deletes the directory. */ undeploy?: (frame: AgentUndeployFrame) => Promise; - /** Cancel, stop, or request durable retention of a workflow's local inspection state. */ - control?: (frame: WorkflowControlFrame) => Promise; + /** + * Cancel, stop, put in standby, or wake a workflow deployment, or request durable + * retention of its local inspection state. `signal` aborts once the + * connection that carried the frame closes. + */ + control?: ( + frame: WorkflowControlFrame, + opts: { signal: AbortSignal }, + ) => Promise; } /** What a handled workflow control reports back to the Hub. */ export type WorkflowControlOutcome = { retention?: DeploymentRetention; - /** Final history for a live stop, which the Hub must confirm receiving. */ + /** + * Final history for a live stop or a standby request, which the Hub must + * confirm receiving. + */ refTips?: WorkflowRunRefTips; + /** The earliest pending timer of the child in standby. */ + wakeAt?: string; }; /** @@ -801,6 +814,12 @@ export type HubLink = { * again after each `welcome`. */ sendDeploymentStopped: (report: Omit) => void; + /** + * Report that an input woke a deployment in standby. Held while the link + * is not welcomed, the latest per incarnation, and sent on the next + * `welcome`. + */ + sendDeploymentWoken: (report: Omit) => void; }; const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]); @@ -980,6 +999,25 @@ export function createHubLink(config: HubLinkConfig): HubLink { ); } + // Wake reports owed while the link is not welcomed, one per incarnation: + // the next `hello` can predate the wake. Kept out of the queue, so they never + // crowd out mail, and bounded by the deployments the sidecar holds. + const pendingWakes = new Map>(); + + function sendPendingWakes(socket: WebSocket): void { + const held = new Set( + getIncarnations().map( + (incarnation) => + `${incarnation.address}\n${String(incarnation.generation)}`, + ), + ); + for (const [key, woken] of pendingWakes) { + if (held.has(key)) + socket.send(JSON.stringify({ type: "deployment.woken", ...woken })); + } + pendingWakes.clear(); + } + function welcomedSocket(): WebSocket | null { return ws !== null && welcomed && ws.readyState === WebSocket.OPEN ? ws @@ -1047,6 +1085,19 @@ export function createHubLink(config: HubLinkConfig): HubLink { // unrelated request. type Reply = (frame: SidecarFrame) => void; + // Aborts once the connection closes, for work that only matters while the + // Hub that asked for it can hear the answer. + const connectionsClosed = new WeakMap(); + function closedSignalOf(connection: WebSocket): AbortSignal { + let closed = connectionsClosed.get(connection); + if (closed === undefined) { + closed = new AbortController(); + if (connection.readyState === WebSocket.CLOSED) closed.abort(); + connectionsClosed.set(connection, closed); + } + return closed.signal; + } + function replyOn(connection: WebSocket): Reply { return (frame) => { if (!sendOnConnection(connection, frame)) { @@ -1878,16 +1929,22 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleWorkflowControl( frame: WorkflowControlFrame, reply: Reply, + signal: AbortSignal, ): Promise { let error: string | undefined; let refTips: WorkflowRunRefTips | undefined; let retention: DeploymentRetention | undefined; + let wakeAt: string | undefined; try { if (deployRouter.control === undefined) throw new Error("Workflow control is not supported by this sidecar"); - ({ refTips, retention } = await deployRouter.control(frame)); + ({ refTips, retention, wakeAt } = await deployRouter.control(frame, { + signal, + })); } catch (cause) { - error = cause instanceof Error ? cause.message : String(cause); + error = fitDeploymentError( + cause instanceof Error ? cause.message : String(cause), + ); } reply({ type: "workflow.control.ack", @@ -1895,6 +1952,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { ...(error !== undefined ? { error } : {}), ...(refTips !== undefined ? { refTips } : {}), ...(retention !== undefined ? { retention } : {}), + ...(wakeAt !== undefined ? { wakeAt } : {}), }); } @@ -1918,6 +1976,8 @@ export function createHubLink(config: HubLinkConfig): HubLink { if (ws !== connection || welcomed) return; welcomed = true; stopWelcomeTimer(); + const socket = welcomedSocket(); + if (socket !== null) sendPendingWakes(socket); flush(); if (onWorkflowAddressesRoutable === undefined) return; const held = new Map( @@ -2030,10 +2090,14 @@ export function createHubLink(config: HubLinkConfig): HubLink { } case "workflow.control": // Start in the address's order, but leave its lane free for a forced - // stop and the deployment's undeploy while cooperative cancellation - // waits for the child. + // stop and the deployment's undeploy while cooperative cancellation or + // a standby request waits for the child. onLanes(frame.type, [frame.agentAddress], async () => { - void handleWorkflowControl(frame, reply).catch((cause: unknown) => { + void handleWorkflowControl( + frame, + reply, + closedSignalOf(connection), + ).catch((cause: unknown) => { logger.warn`Workflow control reply failed: ${cause instanceof Error ? cause.message : String(cause)}`; }); }); @@ -2207,6 +2271,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { }); connection.addEventListener("close", () => { + connectionsClosed.get(connection)?.abort(); // A late close from a superseded attempt must not null or reschedule the // active socket. Normal reconnects also pass this fence: the next socket // is not created until this handler schedules it. @@ -2314,6 +2379,18 @@ export function createHubLink(config: HubLinkConfig): HubLink { ); }; + const sendDeploymentWoken: HubLink["sendDeploymentWoken"] = (woken) => { + const socket = welcomedSocket(); + if (socket !== null) { + socket.send(JSON.stringify({ type: "deployment.woken", ...woken })); + return; + } + pendingWakes.set( + `${woken.agentAddress}\n${String(woken.generation)}`, + woken, + ); + }; + return { connect, close, @@ -2321,5 +2398,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { sendSignalCorrelationRegister, pushWorkflowRunPack, sendDeploymentStopped, + sendDeploymentWoken, }; } diff --git a/packages/hub-sessions/src/session-service.test.ts b/packages/hub-sessions/src/session-service.test.ts index c7b047632..32884a160 100644 --- a/packages/hub-sessions/src/session-service.test.ts +++ b/packages/hub-sessions/src/session-service.test.ts @@ -197,6 +197,12 @@ function createMockAllocationRouter( async sendWorkflowControl() { throw new Error("mock workflow control is not used by session service"); }, + async putAllocationInStandby() { + throw new Error("mock standby is not used by session service"); + }, + async wakeAllocation() { + throw new Error("mock wake is not used by session service"); + }, retireAllocation() { throw new Error( "mock allocation retirement is not used by session service", diff --git a/packages/hub-sessions/src/ws/pending-tracker.ts b/packages/hub-sessions/src/ws/pending-tracker.ts index 2c14f1057..d87793706 100644 --- a/packages/hub-sessions/src/ws/pending-tracker.ts +++ b/packages/hub-sessions/src/ws/pending-tracker.ts @@ -104,7 +104,8 @@ export class PendingTracker< * transports, tests) settles it. When `timeoutMs` elapses the entry is * dropped and `reject` is invoked with `timeoutMessage` — the same * rejection path an error reply frame uses, so per-site cleanup (routing - * rollback, address bookkeeping) runs exactly once either way. + * rollback, address bookkeeping) runs exactly once either way. A null + * `timeoutMs` arms none: the entry waits for its reply or its connection. * * `meta` is the opaque per-entry payload settle-time ownership checks read * off `get`; pass `undefined` when the round-trip carries none. @@ -113,7 +114,7 @@ export class PendingTracker< key: Key, ws: WsHandle, options: { - timeoutMs: number; + timeoutMs: number | null; timeoutMessage: string; resolve(value: Value): void; reject(error: string | Failure): void; @@ -124,22 +125,25 @@ export class PendingTracker< // below can recognize it. Allocated before `schedule`, because the // handler closes over it and the entry itself does not exist yet. const token = Symbol("pending-entry"); - const cancelTimeout = this.schedule(() => { - // Only the entry this call registered may be timed out. The global - // timer never fires a cancelled timeout, but `schedule` is injectable - // now and a caller's implementation is not owed that guarantee -- so - // the handler checks rather than trusting cancellation to be perfect. - // The check compares `token` rather than asking whether the key is - // occupied: keys are reused (`pendingDeploys` is keyed by agent - // address), so a stale fire can find the key held by a live successor. - // Treating that as its own entry would drop the successor without - // settling it and run this already-settled entry's per-site cleanup a - // second time. - const current = this.entries.get(key); - if (current === undefined || current.token !== token) return; - this.entries.delete(key); - options.reject(options.timeoutMessage); - }, options.timeoutMs); + const cancelTimeout = + options.timeoutMs === null + ? () => undefined + : this.schedule(() => { + // Only the entry this call registered may be timed out. The global + // timer never fires a cancelled timeout, but `schedule` is injectable + // now and a caller's implementation is not owed that guarantee -- so + // the handler checks rather than trusting cancellation to be perfect. + // The check compares `token` rather than asking whether the key is + // occupied: keys are reused (`pendingDeploys` is keyed by agent + // address), so a stale fire can find the key held by a live successor. + // Treating that as its own entry would drop the successor without + // settling it and run this already-settled entry's per-site cleanup a + // second time. + const current = this.entries.get(key); + if (current === undefined || current.token !== token) return; + this.entries.delete(key); + options.reject(options.timeoutMessage); + }, options.timeoutMs); this.entries.set(key, { key, ws, diff --git a/packages/hub-sessions/src/ws/sidecar-events.ts b/packages/hub-sessions/src/ws/sidecar-events.ts index ee76bcb4c..4e5110887 100644 --- a/packages/hub-sessions/src/ws/sidecar-events.ts +++ b/packages/hub-sessions/src/ws/sidecar-events.ts @@ -108,6 +108,19 @@ export type SidecarEventMap = { generation: number; }; + /** + * Notification of whether the current generation's workflow child is + * in standby, as the sidecar reports it: on every `hello` for each live + * incarnation it routes, and when an input wakes a child in standby. + */ + "deployment.standby": { + allocationId: string; + generation: number; + inStandby: boolean; + /** Why a child in standby runs again, when an input woke it. */ + reason?: string; + }; + /** Notification. Emitted for mail the Hub gives up on: recipients no * connection routes, un-acked mail it stops redelivering that no * dispatch row stands behind, and mail held for a sender key that never @@ -171,6 +184,7 @@ export function createSidecarEmitter(): SidecarEventEmitter { "sidecar.disconnect": new Set(), "sidecar.allocated.connected": new Set(), "deployment.stopped": new Set(), + "deployment.standby": new Set(), "mail.outbound.undelivered": new Set(), "mail.persisted": new Set(), "mail.inbound.acknowledged": new Set(), diff --git a/packages/hub-sessions/src/ws/sidecar-handler.control.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.control.test.ts index 12936260f..6cf5eaf43 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.control.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.control.test.ts @@ -26,7 +26,9 @@ import { connectAllocated, createAllocatedRouter, createManualRetries, + createMockWs, deployReply, + helloFrame, parsedFrames, TEST_CONFIG, TEST_WORKFLOW, @@ -701,6 +703,358 @@ describe("SidecarRouter allocation control protocols", () => { expect(finalityReads).toBe(0); }); + test("confirms a standby request once the Hub holds its history and keeps the route", async () => { + let hubRefTips: WorkflowRunRefTips = { + ...TEST_REF_TIPS, + "refs/heads/main": "b".repeat(40), + }; + const router = createAllocatedRouter({ + lookups: { readWorkflowRunRefTips: async () => hubRefTips }, + }); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const acknowledgeStandby = async () => { + const pending = router.putAllocationInStandby( + TEST_IDENTITY, + CONTROL_TIMEOUT_MS, + ); + await tick(); + const frame = WorkflowControlFrame.assert( + framesOfType(ws, "workflow.control").at(-1), + ); + expect(frame.action).toBe("standby"); + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + refTips: TEST_REF_TIPS, + wakeAt: "2026-10-02T00:00:00.000Z", + }), + ); + return pending; + }; + + await expect(acknowledgeStandby()).rejects.toThrow( + WorkflowControlHistoryPendingError, + ); + hubRefTips = TEST_REF_TIPS; + expect(await acknowledgeStandby()).toEqual({ + wakeAt: "2026-10-02T00:00:00.000Z", + }); + expect(router.getRoutableAddresses()).toContain( + TEST_IDENTITY.workflowRunAddress, + ); + }); + + test("an invalid answer to a standby request fails it at once", async () => { + const router = createAllocatedRouter(); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const pending = router.putAllocationInStandby( + TEST_IDENTITY, + CONTROL_TIMEOUT_MS, + ); + await tick(); + const frame = WorkflowControlFrame.assert( + framesOfType(ws, "workflow.control").at(-1), + ); + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + wakeAt: "2026-W05", + }), + ); + await expect(pending).rejects.toThrow("invalid answer from the sidecar"); + }); + + test("waits for a standby request's answer however long every run takes to wait", async () => { + const armed: number[] = []; + const router = createAllocatedRouter({ + scheduleTimeout: (_handler, ms) => { + armed.push(ms); + return () => undefined; + }, + lookups: { readWorkflowRunRefTips: async () => TEST_REF_TIPS }, + }); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const pending = router.putAllocationInStandby(TEST_IDENTITY, 7_000); + await tick(); + const frame = WorkflowControlFrame.assert( + framesOfType(ws, "workflow.control").at(-1), + ); + // The request is registered before its frame goes out. + expect(armed).not.toContain(7_000); + + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + refTips: TEST_REF_TIPS, + }), + ); + expect(await pending).toEqual({}); + }); + + test("sends a wake asked after a standby request behind it, however long the standby request's connection check takes", async () => { + let stall = false; + const checking = Promise.withResolvers(); + const release = Promise.withResolvers(); + const router = createAllocatedRouter({ + validateSidecarIdentity: async (_identity, use) => { + if (use === "routing" && stall) { + stall = false; + checking.resolve(undefined); + await release.promise; + } + return true; + }, + }); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const before = framesOfType(ws, "workflow.control").length; + stall = true; + const inStandby = router.putAllocationInStandby(TEST_IDENTITY, 7_000); + await checking.promise; + const woken = router.wakeAllocation(TEST_IDENTITY, 7_000); + await tick(); + expect(framesOfType(ws, "workflow.control")).toHaveLength(before); + + release.resolve(undefined); + await ws.awaitSent( + () => framesOfType(ws, "workflow.control").length === before + 2, + ); + expect( + framesOfType(ws, "workflow.control") + .slice(before) + .map((frame) => WorkflowControlFrame.assert(frame).action), + ).toEqual(["standby", "wake"]); + const settled = Promise.allSettled([inStandby, woken]); + router.handleClose(ws); + expect((await settled).map((outcome) => outcome.status)).toEqual([ + "rejected", + "rejected", + ]); + }); + + test("a standby request whose connection closes right after its lookup settles", async () => { + // The close lands a varying number of microtasks after the lookup, so one + // of them falls between the lookup and the send. + for (let hops = 0; hops < 12; hops += 1) { + let closeAfterLookup: (() => void) | undefined; + const router = createAllocatedRouter({ + validateSidecarIdentity: async (_identity, use) => { + const close = closeAfterLookup; + if (use === "routing" && close !== undefined) { + closeAfterLookup = undefined; + void (async () => { + for (let hop = 0; hop < hops; hop += 1) await null; + close(); + })(); + } + return true; + }, + }); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + closeAfterLookup = () => router.handleClose(ws); + const outcome = router.putAllocationInStandby(TEST_IDENTITY, 7_000).then( + () => "answered", + () => "settled", + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(await Promise.race([outcome, Promise.resolve("pending")])).toBe( + "settled", + ); + } + }); + + test("takes up a standby request's answer after the frames the sidecar sent before it", async () => { + const releasePack = Promise.withResolvers(); + const router = createAllocatedRouter({ + lookups: { + readWorkflowRunRefTips: async () => TEST_REF_TIPS, + async receiveWorkflowRunPack() { + await releasePack.promise; + return { accepted: true }; + }, + }, + }); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const seen: string[] = []; + router.events.on("deployment.standby", ({ reason }) => { + seen.push(reason ?? "hello"); + }); + const pending = router.putAllocationInStandby(TEST_IDENTITY, 7_000, () => { + seen.push("answer"); + }); + await tick(); + const frame = WorkflowControlFrame.assert( + framesOfType(ws, "workflow.control").at(-1), + ); + pushPack(router, ws, "transfer-slow"); + router.handleMessage( + ws, + JSON.stringify({ + type: "deployment.woken", + agentAddress: TEST_IDENTITY.workflowRunAddress, + generation: TEST_IDENTITY.generation, + reason: "Woken by mail", + }), + ); + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + refTips: TEST_REF_TIPS, + }), + ); + await tick(); + expect(seen).toEqual([]); + + releasePack.resolve(undefined); + expect(await pending).toEqual({}); + expect(seen).toEqual(["Woken by mail", "answer"]); + }); + + test("reports a standby request the worker refused", async () => { + const router = createAllocatedRouter(); + const ws = await connectAllocated(router, [ + TEST_IDENTITY.workflowRunAddress, + ]); + const pending = router.putAllocationInStandby( + TEST_IDENTITY, + CONTROL_TIMEOUT_MS, + ); + await tick(); + const frame = WorkflowControlFrame.assert( + framesOfType(ws, "workflow.control").at(-1), + ); + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + error: + "The deployment was not put in standby: the workflow run has ended", + }), + ); + await expect(pending).rejects.toThrow(WorkflowControlRejectedError); + }); + + test("reports whether a routed worker's child is in standby, on hello and on a wake", async () => { + const router = createAllocatedRouter(); + const reports: { inStandby: boolean; reason?: string }[] = []; + router.events.on("deployment.standby", ({ inStandby, reason }) => { + reports.push({ inStandby, ...(reason !== undefined ? { reason } : {}) }); + }); + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + helloFrame(TEST_IDENTITY.sidecarId, [ + { + address: TEST_IDENTITY.workflowRunAddress, + generation: TEST_IDENTITY.generation, + state: "live", + inStandby: true, + }, + ]), + ); + await tick(); + router.handleMessage( + ws, + JSON.stringify({ + type: "deployment.woken", + agentAddress: TEST_IDENTITY.workflowRunAddress, + generation: TEST_IDENTITY.generation, + reason: "Woken by mail", + }), + ); + await tick(); + expect(reports).toEqual([ + { inStandby: true }, + { inStandby: false, reason: "Woken by mail" }, + ]); + }); + + test("takes up a wake report only from the generation it routes", async () => { + const router = createAllocatedRouter(); + const reasons: string[] = []; + const current = Promise.withResolvers(); + router.events.on("deployment.standby", ({ reason }) => { + if (reason === undefined) return; + reasons.push(reason); + if (reason === "Woken by mail") current.resolve(undefined); + }); + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + helloFrame(TEST_IDENTITY.sidecarId, [ + { + address: TEST_IDENTITY.workflowRunAddress, + generation: TEST_IDENTITY.generation, + state: "live", + inStandby: true, + }, + ]), + ); + await ws.awaitSent((sent) => sent.some((raw) => raw.includes('"welcome"'))); + const woken = (generation: number, reason: string) => + JSON.stringify({ + type: "deployment.woken", + agentAddress: TEST_IDENTITY.workflowRunAddress, + generation, + reason, + }); + router.handleMessage( + ws, + woken(TEST_IDENTITY.generation + 1, "Woken by another generation"), + ); + router.handleMessage(ws, woken(TEST_IDENTITY.generation, "Woken by mail")); + // Reports are taken up in the order they arrive. + await current.promise; + expect(reasons).toEqual(["Woken by mail"]); + }); + + test("a hello reports no standby for a worker the Hub does not route", async () => { + const router = createAllocatedRouter(); + const reports: unknown[] = []; + router.events.on("deployment.standby", (report) => { + reports.push(report); + }); + router.retireAllocation(TEST_TARGET); + const ws = createMockWs(); + router.handleOpen(ws); + router.handleMessage( + ws, + helloFrame(TEST_IDENTITY.sidecarId, [ + { + address: TEST_IDENTITY.workflowRunAddress, + generation: TEST_IDENTITY.generation, + state: "live", + inStandby: true, + }, + ]), + ); + // The reports go out right after the welcome. + await ws.awaitSent((sent) => sent.some((raw) => raw.includes('"welcome"'))); + expect(reports).toEqual([]); + }); + test("a stopped worker that reconnects without its address still delivers its history", async () => { let hubRefTips: WorkflowRunRefTips = { ...TEST_REF_TIPS, diff --git a/packages/hub-sessions/src/ws/sidecar-handler.ts b/packages/hub-sessions/src/ws/sidecar-handler.ts index e3138b8ab..de7a19bb3 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.ts @@ -34,6 +34,7 @@ import { type AgentUndeployAckFrame, type AgentUndeployErrorFrame, type DeploymentStoppedFrame, + type DeploymentWokenFrame, type DeploymentRetention, type HostedIncarnation, type SidecarCapacity, @@ -494,6 +495,12 @@ type WorkflowControlCommand = Omit< "type" | "requestId" | "generation" | "action" > & { action: "cancel" | "stop" }; +// What an acknowledged workflow control carries back to its caller. +type WorkflowControlResult = { + readonly retention?: DeploymentRetention; + readonly wakeAt?: string; +}; + export type SidecarAllocationRouter = { /** Opaque identity of the currently attached cleanup connection. */ getCleanupConnection(target: AllocatedSidecarTarget): object | undefined; @@ -515,6 +522,31 @@ export type SidecarAllocationRouter = { command: WorkflowControlCommand, timeoutMs: number, ): Promise; + /** + * Put the deployment's workflow child in standby. Resolves with its earliest + * pending timer once the sidecar has stopped the child and the Hub holds the + * history the sidecar reported for it. Rejects with the sidecar's reason when + * the standby request was abandoned, and with + * `WorkflowControlHistoryPendingError` while that history is still arriving; + * a repeated standby request answers again. The sidecar answers once every + * run waits, however long that takes, or the connection closes; `timeoutMs` + * bounds only the Hub's handling of that answer. `onAnswered` runs when the + * Hub takes up the answer, in order with the other frames on its connection. + */ + putAllocationInStandby( + target: AllocatedSidecarTarget, + timeoutMs: number, + onAnswered?: () => void, + ): Promise<{ readonly wakeAt?: string }>; + /** + * Abandon a standby request in progress, or run a workflow child + * in standby again. `onAnswered` runs as for `putAllocationInStandby`. + */ + wakeAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + onAnswered?: () => void, + ): Promise; /** Concurrent callers share the existing request and its deadline. */ retainAllocation( target: AllocatedSidecarTarget, @@ -770,6 +802,11 @@ const INCARNATION_VALIDATION_CONCURRENCY = 8; // A longer `setTimeout` fires at once, and a disconnect limit can be longer. const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; +const MalformedControlAck = type({ + type: "'workflow.control.ack'", + requestId: "string > 0", +}); + const MalformedPackDone = type({ type: "'repo.pack.done'", transferId: "string > 0", @@ -1105,6 +1142,8 @@ export function createSidecarRouter( type PackTransferMeta = { agentAddress: string; repoId: RepoId }; const pendingPacks = new PendingTracker(); + // allocation:generation → the last standby or wake waiting to go out. + const standbySends = new Map>(); // agentAddress → the undeploy a caller awaits for it, answered by the // agent.undeploy.ack or agent.undeploy.error carrying its request id. const pendingUndeploys = new PendingTracker< @@ -1114,12 +1153,13 @@ export function createSidecarRouter( >(scheduleTimeout); const pendingWorkflowControls = new PendingTracker< string, - DeploymentRetention | undefined, + WorkflowControlResult, AllocatedSidecarTarget & Pick & { terminal: boolean; fail(error: Error): void; received(): void; + answered(): void; } >(scheduleTimeout); @@ -1719,6 +1759,18 @@ export function createSidecarRouter( handleClose(ws); ws.close(); } + // The sidecar answered, and a request with no time limit would otherwise + // wait for an answer that is not coming. + const ack = MalformedControlAck(raw); + if ( + !(ack instanceof type.errors) && + pendingWorkflowControls.get(ack.requestId)?.ws === ws + ) { + pendingWorkflowControls.reject( + ack.requestId, + `invalid answer from the sidecar: ${validated.summary}`, + ); + } const done = MalformedPackDone(raw); if (!(done instanceof type.errors)) { // Behind the frames the sidecar sent before it, such as the @@ -1823,6 +1875,7 @@ export function createSidecarRouter( case "signal.correlation.register": case "workflow.control.ack": case "deployment.stopped": + case "deployment.woken": case "repo.pack.push": case "repo.pack.done": return false; @@ -2030,6 +2083,9 @@ export function createSidecarRouter( case "deployment.stopped": handleDeploymentStopped(ws, frame); return; + case "deployment.woken": + handleDeploymentWoken(ws, frame); + return; default: return assertNever(frame); } @@ -2851,6 +2907,19 @@ export function createSidecarRouter( conn.send({ type: "welcome", routed }); welcomedSockets.add(ws); notifySidecarInventory(sidecarId); + // Only incarnations this connection routes report standby, not one being + // cleaned up or of a stale generation. + const routedAddresses = new Set(routed.map(({ address }) => address)); + for (const incarnation of incarnations) { + const binding = reclaimed.get(incarnation.address); + if (binding === undefined || !routedAddresses.has(incarnation.address)) + continue; + events.emit("deployment.standby", { + allocationId: binding.allocationId, + generation: binding.generation, + inStandby: incarnation.inStandby === true, + }); + } // Live retained copies were validated in readHostedWork's bounded batches. // Stop them without repeating those queries or awaiting acknowledgements // inside the hello handler that must finish before processing the replies. @@ -5350,6 +5419,7 @@ export function createSidecarRouter( ): Promise { const entry = pendingWorkflowControls.get(frame.requestId); if (entry === undefined || entry.ws !== ws) return; + entry.meta.answered(); const fail = (error: Error) => { pendingWorkflowControls.delete(frame.requestId); entry.meta.fail(error); @@ -5396,10 +5466,16 @@ export function createSidecarRouter( connectionRetention.set(ws, retained); removeRoute(ws, entry.meta.agentAddress); } - if (entry.meta.action === "stop" || entry.meta.action === "retain") { - // A live stop waits for the worker's tips. Terminal controls follow the - // admission cutoff instead: later history cannot change an ended run. - // Both wait for earlier accepted writes before fencing further packs. + if ( + entry.meta.action === "stop" || + entry.meta.action === "retain" || + entry.meta.action === "standby" + ) { + // A live stop or a standby request waits for the worker's tips. + // Terminal controls follow the admission cutoff instead: later history + // cannot change an ended run. A stop and a retention wait for earlier + // accepted writes before fencing further packs; a worker in standby + // keeps its routes. let unreceived: string | null = null; let unreceivedCause: unknown; if ( @@ -5458,13 +5534,22 @@ export function createSidecarRouter( ); return; } - stoppedAllocations.set(entry.meta.allocationId, entry.meta.generation); - removeRoute(ws, entry.meta.agentAddress); + if (entry.meta.action !== "standby") { + stoppedAllocations.set( + entry.meta.allocationId, + entry.meta.generation, + ); + removeRoute(ws, entry.meta.agentAddress); + } } - pendingWorkflowControls.resolve( - frame.requestId, - entry.meta.action === "retain" ? frame.retention : undefined, - ); + pendingWorkflowControls.resolve(frame.requestId, { + ...(entry.meta.action === "retain" && frame.retention !== undefined + ? { retention: frame.retention } + : {}), + ...(entry.meta.action === "standby" && frame.wakeAt !== undefined + ? { wakeAt: frame.wakeAt } + : {}), + }); } } @@ -5532,6 +5617,25 @@ export function createSidecarRouter( }); } + // A child in standby that an input woke. Only the current generation's report + // counts. + function handleDeploymentWoken( + ws: WsHandle, + frame: DeploymentWokenFrame, + ): void { + const conn = connections.get(ws); + if (conn === undefined) return; + const binding = deploymentBinding(conn, frame.agentAddress); + if (binding?.generation !== frame.generation || !isFencedAsCurrent(binding)) + return; + events.emit("deployment.standby", { + allocationId: binding.allocationId, + generation: binding.generation, + inStandby: false, + reason: frame.reason, + }); + } + // Describes the first ref whose reported tip the Hub does not hold, or // returns null when the Hub holds the worker's whole history. async function findUnreceivedWorkflowHistory( @@ -5563,6 +5667,108 @@ export function createSidecarRouter( await requestWorkflowControl(target, command, timeoutMs); } + async function putAllocationInStandby( + target: AllocatedSidecarTarget, + timeoutMs: number, + onAnswered?: () => void, + ): Promise<{ readonly wakeAt?: string }> { + const result = await sendStandbyControl(target, "standby", timeoutMs, { + untilAnswered: true, + ...(onAnswered !== undefined ? { onAnswered } : {}), + }); + return result.wakeAt === undefined ? {} : { wakeAt: result.wakeAt }; + } + + async function wakeAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + onAnswered?: () => void, + ): Promise { + await sendStandbyControl( + target, + "wake", + timeoutMs, + onAnswered !== undefined ? { onAnswered } : {}, + ); + } + + // A deployment's standby and wake commands go out in the order they were + // asked for, so a wake reaches the sidecar after the standby request it + // abandons. Each waits only for the frame before it to go out, not for its + // answer. + async function sendStandbyControl( + target: AllocatedSidecarTarget, + action: "standby" | "wake", + timeoutMs: number, + opts: { untilAnswered?: boolean; onAnswered?: () => void }, + ): Promise { + const key = `${target.allocationId}:${String(target.generation)}`; + const previous = standbySends.get(key) ?? Promise.resolve(); + const sent = Promise.withResolvers(); + const tail = previous.then(() => sent.promise); + standbySends.set(key, tail); + let answer: Promise | undefined; + try { + await previous; + const connection = await routingConnection(target); + // Sent under the anchor lock, so a wake never starts a child whose run + // a cancellation or expiry already ended. + await withAllocationWorkAdmission( + connection.ws, + connection.binding, + () => { + // The connection can have closed since the lookup, and its close + // sweep never finds a request registered after it. + const held = allocatedConnections.get(target.allocationId); + if ( + held?.ws !== connection.ws || + held.identity.generation !== target.generation || + connections.get(connection.ws) !== connection.conn || + connection.conn.bindings.get(target.allocationId) !== + connection.binding || + allocationFences.get(target.allocationId)?.generation !== + target.generation + ) { + throw new WorkflowControlUnreachableError( + `Allocated sidecar connection changed for allocation ${target.allocationId}`, + ); + } + answer = sendWorkflowControlOnConnection( + connection, + target, + { + runId: connection.binding.anchorRunId, + agentAddress: connection.binding.workflowRunAddress, + action, + reason: + action === "standby" + ? "Put the deployment's workflow child in standby" + : "Wake the deployment's workflow child", + }, + timeoutMs, + { terminal: false, ...opts }, + ); + // Own an early rejection while the admission transaction ends. + void answer.catch(() => undefined); + return true; + }, + ).catch((cause: unknown) => { + // The lock only orders the send against a cancellation or expiry, so + // once the frame is out its answer stands. + if (answer === undefined) throw cause; + logger.warn`Workflow ${action} for allocation ${target.allocationId} was sent, but its admission did not end cleanly: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + if (answer === undefined) + throw new Error(`Workflow ${action} was not sent`); + } finally { + sent.resolve(undefined); + void tail.then(() => { + if (standbySends.get(key) === tail) standbySends.delete(key); + }); + } + return answer; + } + async function retainAllocation( target: AllocatedSidecarTarget, timeoutMs: number, @@ -5597,7 +5803,7 @@ export function createSidecarRouter( }, timeoutMs, { terminal: true }, - ).then((retention) => { + ).then(({ retention }) => { if (retention === undefined) throw new WorkflowControlRejectedError("Retention was not confirmed"); return retention; @@ -5612,14 +5818,13 @@ export function createSidecarRouter( } } - async function requestWorkflowControl( + // The connection a control goes out on; unreachable while the sidecar has + // none for the allocation. + async function routingConnection( target: AllocatedSidecarTarget, - command: WorkflowControlCommand, - timeoutMs: number, - ): Promise { - let connection: Awaited>; + ): Promise>> { try { - connection = await getAllocatedConnection(target, "routing"); + return await getAllocatedConnection(target, "routing"); } catch (cause) { if (cause instanceof SidecarIdentityValidationError) throw cause; if (cause instanceof SidecarNotConnectedError) @@ -5629,8 +5834,15 @@ export function createSidecarRouter( cause, ); } + } + + async function requestWorkflowControl( + target: AllocatedSidecarTarget, + command: Omit, + timeoutMs: number, + ): Promise { return sendWorkflowControlOnConnection( - connection, + await routingConnection(target), target, command, timeoutMs, @@ -5643,8 +5855,12 @@ export function createSidecarRouter( target: AllocatedSidecarTarget, command: Omit, timeoutMs: number, - { terminal }: { terminal: boolean }, - ): Promise { + { + terminal, + untilAnswered = false, + onAnswered, + }: { terminal: boolean; untilAnswered?: boolean; onAnswered?: () => void }, + ): Promise { const { ws, conn, binding } = connection; if ( command.agentAddress !== binding.workflowRunAddress || @@ -5657,8 +5873,9 @@ export function createSidecarRouter( const requestId = nextRequestId(); const timeoutMessage = `Workflow control ${requestId} timed out`; const unconfirmedMessage = `Workflow control ${requestId} was acknowledged but not processed in time`; - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { let cancelProcessing: (() => void) | undefined; + let answered = false; const settle = () => { cancelProcessing?.(); }; @@ -5666,7 +5883,7 @@ export function createSidecarRouter( requestId, ws, { - timeoutMs, + timeoutMs: untilAnswered ? null : timeoutMs, timeoutMessage, resolve: (retention) => { settle(); @@ -5708,6 +5925,11 @@ export function createSidecarRouter( pendingWorkflowControls.reject(requestId, unconfirmedMessage); }, timeoutMs); }, + answered: () => { + if (answered) return; + answered = true; + onAnswered?.(); + }, }, ); try { @@ -6082,6 +6304,8 @@ export function createSidecarRouter( undeployAllocation, getCleanupConnection, sendWorkflowControl, + putAllocationInStandby, + wakeAllocation, retainAllocation, getAllocationRetention, getRetentionCandidates, diff --git a/packages/hub-sessions/src/ws/sidecar-handler.work-admission.test.ts b/packages/hub-sessions/src/ws/sidecar-handler.work-admission.test.ts index b09501366..1ddf0f435 100644 --- a/packages/hub-sessions/src/ws/sidecar-handler.work-admission.test.ts +++ b/packages/hub-sessions/src/ws/sidecar-handler.work-admission.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "bun:test"; import { canExecuteWorkflowRun, WorkflowRunNotExecutableError } from "@intx/db"; +import { WorkflowControlFrame } from "@intx/types/sidecar"; import { connectAllocated, @@ -10,6 +11,7 @@ import { liveIncarnations, parsedFrames, TEST_IDENTITY, + TEST_REF_TIPS, TEST_TARGET, tick, } from "./sidecar-handler.test-helpers"; @@ -235,6 +237,66 @@ test.each(["mail", "signal", "direct mail", "direct signal"] as const)( }, ); +test.each(["standby", "wake"] as const)( + "guards a %s command", + async (action) => { + const router = createAllocatedRouter({ + withExecutableWorkflowRun: async () => { + throw new WorkflowRunNotExecutableError( + TEST_IDENTITY.anchorRunId, + "stopping", + ); + }, + }); + const ws = await connectAllocated(router, [address]); + const before = [...ws.sent]; + const sending = + action === "standby" + ? router.putAllocationInStandby(TEST_TARGET, 7_000) + : router.wakeAllocation(TEST_TARGET, 7_000); + await expect(sending).rejects.toBeInstanceOf(WorkflowRunNotExecutableError); + expect(ws.sent).toEqual(before); + router.handleClose(ws); + }, +); + +test("a standby command whose admission fails after it went out still takes its answer", async () => { + const router = createAllocatedRouter({ + withExecutableWorkflowRun: async (_target, send) => { + send(); + throw new Error("The admission commit failed"); + }, + }); + const ws = await connectAllocated(router, [address]); + let answered = false; + const pending = router.putAllocationInStandby(TEST_TARGET, 7_000, () => { + answered = true; + }); + await tick(); + const frame = WorkflowControlFrame.assert( + parsedFrames(ws) + .filter( + (sent) => + typeof sent === "object" && + sent !== null && + "type" in sent && + sent.type === "workflow.control", + ) + .at(-1), + ); + router.handleMessage( + ws, + JSON.stringify({ + type: "workflow.control.ack", + requestId: frame.requestId, + refTips: TEST_REF_TIPS, + }), + ); + expect(await pending).toEqual({}); + expect(answered).toBe(true); + router.handleClose(ws); +}); + test("reports untracked relay mail when admission fails transiently", async () => { const undelivered = Promise.withResolvers<{ rawMessage: string; diff --git a/packages/types/src/sidecar.test.ts b/packages/types/src/sidecar.test.ts index 246532e06..2ec1de8f2 100644 --- a/packages/types/src/sidecar.test.ts +++ b/packages/types/src/sidecar.test.ts @@ -1039,6 +1039,65 @@ describe("DeploymentStoppedFrame", () => { }); }); +describe("deployments in standby", () => { + test("a hello marks only a live incarnation in standby", () => { + const hello = (state: string) => ({ + type: "hello", + sidecarId: "sc-1", + token: "token", + capacity: { maxActiveDeployments: 1, maxRetainedDeployments: 0 }, + incarnations: [ + { address: "wf@example.test", generation: 1, state, inStandby: true }, + ], + }); + expect(SidecarFrame(hello("live")) instanceof type.errors).toBe(false); + expect(SidecarFrame(hello("stopped")) instanceof type.errors).toBe(true); + }); + + test("a control answer's error fits the bound on sidecar text", () => { + const ack = (error: string) => ({ + type: "workflow.control.ack", + requestId: "control-1", + error, + }); + expect( + SidecarFrame(ack("x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH))) instanceof + type.errors, + ).toBe(false); + expect( + SidecarFrame(ack("x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH + 1))) instanceof + type.errors, + ).toBe(true); + }); + + test("a standby request's answer names its earliest timer as an ISO time", () => { + const ack = (wakeAt: string) => ({ + type: "workflow.control.ack", + requestId: "control-1", + wakeAt, + }); + expect( + SidecarFrame(ack("2026-10-02T00:00:00.000Z")) instanceof type.errors, + ).toBe(false); + expect(SidecarFrame(ack("tomorrow")) instanceof type.errors).toBe(true); + // ISO 8601, but not a time the Hub's `Date` can read. + expect(SidecarFrame(ack("2026-W05")) instanceof type.errors).toBe(true); + }); + + test("the SidecarFrame union admits a wake report with why", () => { + const frame = { + type: "deployment.woken" as const, + agentAddress: "wf@example.test", + generation: 1, + reason: "Woken by mail", + }; + expect(SidecarFrame(frame)).toEqual(frame); + expect(SidecarFrame({ ...frame, reason: "" }) instanceof type.errors).toBe( + true, + ); + }); +}); + describe("SenderKeyEvictFrame", () => { const frame = { type: "sender.key.evict", diff --git a/packages/types/src/sidecar.ts b/packages/types/src/sidecar.ts index 53019f556..295bbde7d 100644 --- a/packages/types/src/sidecar.ts +++ b/packages/types/src/sidecar.ts @@ -195,18 +195,28 @@ export const HostedIncarnation = type({ address: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), generation: Generation, state: IncarnationState, + // A live incarnation whose workflow child is in standby: stopped while every + // run waits, and respawned for its next input. + "inStandby?": "true", "retention?": DeploymentRetention, // Why a stopped incarnation stopped when the Hub did not stop it: its // workflow child ended itself, or the sidecar could not restore it. The Hub // fails such a deployment. "error?": type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), -}).narrow( - (incarnation, ctx) => - incarnation.retention === undefined || - incarnation.state === "stopped" || - incarnation.state === "tearing-down" || - ctx.mustBe("stopped before deciding retention"), -); +}) + .narrow( + (incarnation, ctx) => + incarnation.retention === undefined || + incarnation.state === "stopped" || + incarnation.state === "tearing-down" || + ctx.mustBe("stopped before deciding retention"), + ) + .narrow( + (incarnation, ctx) => + incarnation.inStandby === undefined || + incarnation.state === "live" || + ctx.mustBe("live to be in standby"), + ); export type HostedIncarnation = typeof HostedIncarnation.infer; // --------------------------------------------------------------------------- @@ -837,9 +847,13 @@ export const AgentUndeployFrame = type({ export type AgentUndeployFrame = typeof AgentUndeployFrame.infer; /** - * Cancel a workflow, stop its process, or transfer a terminal copy into the - * retained pool. Retention reports kept/refused only after its decision is - * durable. The sidecar refuses a command for another generation. + * Cancel a workflow, stop its process, transfer a terminal copy into the + * retained pool, or put its workflow child in standby or wake it. Retention + * reports kept/refused only after its decision is durable. `standby` answers + * once the child is stopped and the standby mark is durable, or with the reason + * it was abandoned; `wake` answers once a standby request in progress is + * abandoned or the child is running again. The sidecar refuses a command for + * another generation. */ export const WorkflowControlFrame = type({ type: "'workflow.control'", @@ -847,7 +861,7 @@ export const WorkflowControlFrame = type({ agentAddress: "string", generation: Generation, runId: "string", - action: "'cancel' | 'stop' | 'retain'", + action: "'cancel' | 'stop' | 'retain' | 'standby' | 'wake'", reason: "string", }); export type WorkflowControlFrame = typeof WorkflowControlFrame.infer; @@ -869,17 +883,28 @@ export const WorkflowRunRefTips = type({ ); export type WorkflowRunRefTips = typeof WorkflowRunRefTips.infer; +// `string.date.iso` also admits forms such as week dates that `Date` cannot read. +const WakeAtTime = type("string.date.iso").narrow( + (value, ctx) => + !Number.isNaN(Date.parse(value)) || ctx.mustBe("a time a Date can read"), +); + export const WorkflowControlAckFrame = type({ type: "'workflow.control.ack'", requestId: "string", - "error?": "string", + "error?": type("string").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), "retention?": DeploymentRetention, /** - * A stopped worker's ref tips, read after its child exited. The Hub - * confirms a live stop only once it holds the same tips. Retain follows - * the existing terminal-history cutoff instead of importing later commits. + * The ref tips of a stopped worker or one in standby, read after its child + * exited. The Hub confirms a live stop or a standby request only once it + * holds the same tips. Retain follows the existing terminal-history cutoff + * instead of importing later commits. */ "refTips?": WorkflowRunRefTips, + /** + * The earliest pending timer of the child in standby, which wakes it. + */ + "wakeAt?": WakeAtTime, }); export type WorkflowControlAckFrame = typeof WorkflowControlAckFrame.infer; @@ -900,6 +925,21 @@ export const DeploymentStoppedFrame = type({ }); export type DeploymentStoppedFrame = typeof DeploymentStoppedFrame.infer; +/** + * Reports a deployment in standby whose workflow child is running again though + * the Hub did not wake it. `reason` says what did, such as mail, a signal, a + * cancellation, its earliest pending timer, or a restart. The sidecar queues + * it while no connection is welcomed, so a report sent around a reconnect is + * delivered after `welcome`. + */ +export const DeploymentWokenFrame = type({ + type: "'deployment.woken'", + agentAddress: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), + generation: Generation, + reason: type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), +}); +export type DeploymentWokenFrame = typeof DeploymentWokenFrame.infer; + /** * The Hub's answer to `hello`, sent once it has reconciled the reported * incarnations. `routed` lists the incarnations it routes on this connection; @@ -1184,6 +1224,7 @@ export const SidecarFrame = type.or( WorkflowProbeErrorFrame, WorkflowControlAckFrame, DeploymentStoppedFrame, + DeploymentWokenFrame, ); export type SidecarFrame = typeof SidecarFrame.infer; diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts index afdb43b07..0a26fb3de 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring.test.ts @@ -42,6 +42,7 @@ import { WORKFLOW_CONTROL_INITIALIZING_ERROR, WorkflowProjectionDefinition, type AgentDeployFrame, + type WorkflowRunRefTips, } from "@intx/types/sidecar"; import { waitUntil } from "@intx/types/testing"; import { @@ -51,6 +52,7 @@ import { createChangeNotifier, sendDefinition, waitForUpstreamPayload, + waitForUpstreamPayloads, } from "@intx/workflow-host/testing"; import { assembleRunCredentialsSnapshot, @@ -75,6 +77,7 @@ import { import { scanWorkflowRunRecords, markWorkflowRunRecord, + markWorkflowRunRecordInStandby, writeWorkflowRunRecord, updateWorkflowRunRecordCredentials, updateWorkflowRunRecordSources, @@ -428,8 +431,26 @@ function downstreamPayloadType(line: string): string | undefined { * Build a stub RepoStore whose `getRepoDir` resolves under the supplied * tempBase. */ -function createSpawnTestRepoStore(tempBase: string): RepoStore { +// `historyTip`, when given, is the tip every ref resolves to, over an empty +// claim-check tree, so a restored standby's boot recovery can read the run. +function createSpawnTestRepoStore( + tempBase: string, + historyTip?: string | null, +): RepoStore { const stub: Partial = { + ...(historyTip !== undefined + ? { + resolveRef: async () => historyTip, + openCommittedReads: async () => null, + async writeTreeDelta(_p, _id, _ref, args) { + await args.computeDelta(null, { + readBlobByOid: async () => new Uint8Array(), + listDirOids: async () => [], + }); + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + } + : {}), getRepoDir(repoId: RepoId): string { return path.join(tempBase, repoId.kind, repoId.id); }, @@ -777,6 +798,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { publishDeploymentStopped?: Parameters< typeof createSidecarDeployRouter >[0]["publishDeploymentStopped"]; + publishDeploymentWoken?: Parameters< + typeof createSidecarDeployRouter + >[0]["publishDeploymentWoken"]; assertSourceBuildable?: Parameters< typeof createSidecarDeployRouter >[0]["assertSourceBuildable"]; @@ -826,11 +850,13 @@ describe("createSidecarDeployRouter multi-step branch", () => { typeof createSidecarDeployRouter >[0]["applyFrozenWorkflowClosure"]; /** Runs before the repo store writes a tree, such as a step's grants. */ + /** The tip of every deployment's run history, for a standby restore. */ + historyTip?: string | null; }) { const transport = opts.transport ?? createInMemoryTransport(); const keyPair = await generateKeyPair(); const tempBase = await createTempBaseDir("sidecar-multistep-"); - const repoStore = createSpawnTestRepoStore(tempBase); + const repoStore = createSpawnTestRepoStore(tempBase, opts.historyTip); // The deploy router's source-ref branch materializes the pinned closure // under `${SIDECAR_DATA_DIR}/workflow-definition-closures//` before // invoking the spawner. The test fixture defaults the data dir to a @@ -887,6 +913,9 @@ describe("createSidecarDeployRouter multi-step branch", () => { }), reportDeploymentRefTips: opts.reportDeploymentRefTips ?? (async () => ({})), + ...(opts.publishDeploymentWoken !== undefined + ? { publishDeploymentWoken: opts.publishDeploymentWoken } + : {}), ...(opts.publishDeploymentStopped !== undefined ? { publishDeploymentStopped: opts.publishDeploymentStopped } : {}), @@ -1308,6 +1337,32 @@ describe("createSidecarDeployRouter multi-step branch", () => { expect(reported).toEqual([agentAddress]); }); + test("refuses to put a deployment in standby or wake it with no workflow child here", async () => { + const { router } = await buildMultistepFixture({ + spawner: () => { + throw new Error("control must not spawn"); + }, + }); + if (router.control === undefined) + throw new Error("router.control is undefined"); + const command = { + type: "workflow.control", + requestId: "standby-missing", + agentAddress: "run_standbymissing@example.com", + generation: 1, + runId: "run_standbymissing", + action: "standby", + reason: "Put the deployment's workflow child in standby", + } as const; + + await expect(router.control(command)).rejects.toThrow( + "The deployment has no workflow child to put in standby", + ); + await expect( + router.control({ ...command, requestId: "wake", action: "wake" }), + ).rejects.toThrow("The deployment has no workflow child to wake"); + }); + test("registers a multistepMailRouter handler against the deployment address once spawn succeeds", async () => { // Drives the spawn handshake the same way the first multi-step // test does, but injects a `multistepMailRouter` and asserts the @@ -2348,6 +2403,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { supervisorToChild: ReturnType; childToSupervisor: ReturnType; eventChildToSupervisor: ReturnType; + resolveExit?: (code: number) => void; childSender?: ReturnType; }; const spawnsChanges = createChangeNotifier(); @@ -2374,6 +2430,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { supervisorToChild, childToSupervisor, eventChildToSupervisor, + ...(resolveExit !== undefined ? { resolveExit } : {}), }); spawnsChanges.notify(); const handle: SubprocessHandle = { @@ -2466,6 +2523,7 @@ describe("createSidecarDeployRouter multi-step branch", () => { if (payload instanceof type.errors) return []; return [payload]; }), + controlFor: (index: number) => spawns[index]?.supervisorToChild, async recycleRequestFor(index: number): Promise { const sender = spawns[index]?.childSender; if (sender === undefined) { @@ -2482,6 +2540,105 @@ describe("createSidecarDeployRouter multi-step branch", () => { failNextSpawn(): void { failNext = true; }, + // Exit the child as a crash does, without being asked to. + crash(index: number): void { + const spawn = spawns[index]; + if (spawn === undefined) + throw new Error(`spawn ${String(index)} missing`); + spawn.eventChildToSupervisor.close(); + spawn.childToSupervisor.close(); + spawn.resolveExit?.(1); + }, + // Answer the `count`th correlation query a child asks, with none. + async answerParkedQuery(index: number, count: number): Promise { + const spawn = spawns[index]; + const sender = spawn?.childSender; + if (spawn === undefined || sender === undefined) { + throw new Error( + `spawn ${String(index)} has no sender; drive its ready first`, + ); + } + const query = ( + await waitForUpstreamPayloads( + spawn.supervisorToChild, + "parked-correlations.request", + count, + ) + )[count - 1]; + if (query === undefined) throw new Error("correlation query missing"); + await sender.send({ + type: "parked-correlations.response", + data: { requestId: query.data.requestId, parked: [] }, + }); + }, + // Answer the `count`th quiet query a child asks. + async answerQuiet( + index: number, + count: number, + answer: { quiet: boolean; ended?: true }, + ): Promise { + const spawn = spawns[index]; + const sender = spawn?.childSender; + if (spawn === undefined || sender === undefined) { + throw new Error( + `spawn ${String(index)} has no sender; drive its ready first`, + ); + } + const query = ( + await waitForUpstreamPayloads( + spawn.supervisorToChild, + "quiet.request", + count, + ) + )[count - 1]; + if (query === undefined) throw new Error("quiet query missing"); + await sender.send({ + type: "quiet.response", + data: { requestId: query.data.requestId, ...answer }, + }); + }, + // Answer a standby attempt as the real child does: quiet for the wait and + // the final check, no parked correlations, then exit on `shutdown`. + async putChildInStandby(index: number): Promise { + const spawn = spawns[index]; + const sender = spawn?.childSender; + if (spawn === undefined || sender === undefined) { + throw new Error( + `spawn ${String(index)} has no sender; drive its ready first`, + ); + } + for (const count of [1, 2]) { + const query = ( + await waitForUpstreamPayloads( + spawn.supervisorToChild, + "quiet.request", + count, + ) + )[count - 1]; + if (query === undefined) throw new Error("quiet query missing"); + await sender.send({ + type: "quiet.response", + data: { requestId: query.data.requestId, quiet: true }, + }); + } + // The first correlation query is the one the child's start asks. + const parked = ( + await waitForUpstreamPayloads( + spawn.supervisorToChild, + "parked-correlations.request", + 2, + ) + )[1]; + if (parked === undefined) throw new Error("correlation query missing"); + await sender.send({ + type: "parked-correlations.response", + data: { requestId: parked.data.requestId, parked: [] }, + }); + await waitForUpstreamPayload(spawn.supervisorToChild, "shutdown"); + spawn.eventChildToSupervisor.close(); + spawn.childToSupervisor.close(); + spawn.resolveExit?.(0); + }, }; } @@ -2602,6 +2759,384 @@ describe("createSidecarDeployRouter multi-step branch", () => { expect(isRegistered(freshTransport, head)).toBe(true); }); + const askedQuiet = async ( + spawner: ReturnType, + ) => { + const control = spawner.controlFor(0); + if (control === undefined) throw new Error("no spawned child"); + return (await waitForUpstreamPayload(control, "quiet.request")).data.wait; + }; + + const HISTORY_TIP = "tip-at-stop"; + + // Record a standby at `HISTORY_TIP`, then leave its restore on a fresh router + // whose run history is at `historyTip` for the test to drive. + async function restoreRecordedStandby( + dataDir: string, + head: string, + pidBase: number, + opts: { + historyTip: string; + publishDeploymentWoken?: (report: { reason: string }) => void; + }, + ) { + const frame = singleStepFrame(head, `wf-${head.split("@")[0] ?? head}`); + const first = makeReadyDrivingSpawner(pidBase); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(frame); + await first.driveReadyFor(0); + await deployPromise; + const deployedInStandby = routerA + .incarnations() + .find((incarnation) => incarnation.address === head)?.inStandby; + const definition = projectionByHash.get( + frame.workflow?.approvedWireHash ?? "", + ); + if (definition === undefined) { + throw new Error("the frame's definition is not registered"); + } + await markWorkflowRunRecordInStandby( + dataDir, + deriveWorkflowRunRepoId(head), + { parked: [], tip: HISTORY_TIP, definition }, + ); + + const second = makeReadyDrivingSpawner(pidBase + 100); + const { router } = await buildMultistepFixture({ + spawner: second.spawner, + transport: createInMemoryTransport(), + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + historyTip: opts.historyTip, + ...(opts.publishDeploymentWoken !== undefined + ? { publishDeploymentWoken: opts.publishDeploymentWoken } + : {}), + }); + const restoring = router.restoreWorkflowRuns(); + return { + frame, + router, + spawned: second, + deployedInStandby, + restoring, + definition, + }; + } + + const standbyMark = async (dataDir: string, head: string) => + (await waitForRecord(dataDir, deriveWorkflowRunRepoId(head), () => true))?.[ + "standby" + ]; + + test("a deploy starts its child awake, and a record in standby over unchanged history restores it without a child", async () => { + const dataDir = await createTempBaseDir("sidecar-standby-restore-"); + const head = "run_standbydrestore@example.com"; + const reports: string[] = []; + const { router, spawned, deployedInStandby, restoring, definition } = + await restoreRecordedStandby(dataDir, head, 9150, { + historyTip: HISTORY_TIP, + publishDeploymentWoken: (report) => reports.push(report.reason), + }); + await restoring; + + expect(deployedInStandby).toBeUndefined(); + expect(spawned.spawnCount()).toBe(0); + expect( + router.incarnations().find((incarnation) => incarnation.address === head) + ?.inStandby, + ).toBe(true); + expect(await standbyMark(dataDir, head)).toEqual({ + parked: [], + tip: HISTORY_TIP, + definition, + }); + expect(reports).toEqual([]); + }); + + test("a record in standby over history written since restores the child awake, clears the mark and reports it woken", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-moved-"); + const head = "run_restoremoved@example.com"; + const reported = Promise.withResolvers(); + const { router, spawned, restoring } = await restoreRecordedStandby( + dataDir, + head, + 9180, + { + historyTip: "tip-after-a-cancellation", + publishDeploymentWoken: (report) => reported.resolve(report.reason), + }, + ); + await spawned.driveReadyFor(0); + await restoring; + + expect(await reported.promise).toBe( + "Woken by its restart, which found history written after it stopped", + ); + expect(await standbyMark(dataDir, head)).toBeUndefined(); + expect( + router.incarnations().find((incarnation) => incarnation.address === head) + ?.inStandby, + ).toBeUndefined(); + }); + + test("a wake of a restored standby is answered once the cleared mark is on disk, and sends no wake report", async () => { + const dataDir = await createTempBaseDir("sidecar-wake-mark-"); + const head = "run_wakemark@example.com"; + const reports: string[] = []; + const { frame, router, spawned, restoring } = await restoreRecordedStandby( + dataDir, + head, + 9170, + { + historyTip: HISTORY_TIP, + publishDeploymentWoken: (report) => reports.push(report.reason), + }, + ); + await restoring; + expect(await standbyMark(dataDir, head)).toBeDefined(); + if (router.control === undefined) + throw new Error("router.control is undefined"); + + const woken = router.control({ + type: "workflow.control", + requestId: "wake-mark", + agentAddress: head, + generation: frame.generation, + runId: deriveWorkflowRunId(head), + action: "wake", + reason: "Wake the deployment's workflow child", + }); + await spawned.driveReadyFor(0); + expect(await woken).toEqual({}); + expect(spawned.spawnCount()).toBe(1); + expect(await standbyMark(dataDir, head)).toBeUndefined(); + // The Hub that asked for the wake records it from the answer. + expect(reports).toEqual([]); + }); + + // Deploy `head` awake for standby and wake controls, with the history + // tips each standby answer reads. + async function deployForStandby( + head: string, + pidBase: number, + reportDeploymentRefTips: () => Promise, + ) { + const frame = singleStepFrame(head, `wf-${head.split("@")[0] ?? head}`); + const spawned = makeReadyDrivingSpawner(pidBase); + const { router, substrateEnv } = await buildMultistepFixture({ + spawner: spawned.spawner, + reportDeploymentRefTips, + historyTip: HISTORY_TIP, + }); + const deployPromise = router.deploy(frame); + await spawned.driveReadyFor(0); + await deployPromise; + const control = router.control; + if (control === undefined) throw new Error("router.control is undefined"); + return { + router, + frame, + spawned, + dataDir: substrateEnv["SIDECAR_DATA_DIR"], + control: (action: "standby" | "wake", requestId: string) => + control({ + type: "workflow.control", + requestId, + agentAddress: head, + generation: frame.generation, + runId: deriveWorkflowRunId(head), + action, + reason: `${action === "standby" ? "Put the deployment's workflow child in standby" : "Wake the deployment's workflow child"}`, + }), + }; + } + + test("a deployment stopped in standby says so in no hello", async () => { + const head = "run_standbystopped@example.com"; + const { router, frame, spawned, control } = await deployForStandby( + head, + 9990, + async () => ({ "refs/heads/main": "1".repeat(40) }), + ); + const first = control("standby", "standby-first"); + await spawned.putChildInStandby(0); + await first; + await router.control({ + type: "workflow.control", + requestId: "stop-1", + agentAddress: head, + generation: frame.generation, + runId: deriveWorkflowRunId(head), + action: "stop", + reason: "Stop", + }); + const incarnation = router + .incarnations() + .find((candidate) => candidate.address === head); + expect(incarnation?.state).toBe("stopped"); + expect(incarnation?.inStandby).toBeUndefined(); + }); + + test("a repeated standby attempt answers with the history the supervisor wrote since the child stopped", async () => { + let reads = 0; + const { spawned, control } = await deployForStandby( + "run_standbyrepeat@example.com", + 9230, + async () => { + reads += 1; + return { "refs/heads/main": String(reads % 10).repeat(40) }; + }, + ); + + const first = control("standby", "standby-first"); + await spawned.putChildInStandby(0); + expect(await first).toEqual({ + refTips: { "refs/heads/main": "1".repeat(40) }, + }); + expect(await control("standby", "standby-again")).toEqual({ + refTips: { "refs/heads/main": "2".repeat(40) }, + }); + expect(spawned.spawnCount()).toBe(1); + }); + + test("a standby attempt whose child ran again before it was answered is refused", async () => { + const readStarted = Promise.withResolvers(); + const read = Promise.withResolvers(); + const { spawned, control } = await deployForStandby( + "run_standbywoken@example.com", + 9250, + () => { + readStarted.resolve(undefined); + return read.promise; + }, + ); + + const inStandby = control("standby", "standby-woken"); + await spawned.putChildInStandby(0); + await readStarted.promise; + const woken = control("wake", "wake-during-read"); + await spawned.driveReadyFor(1); + await spawned.answerParkedQuery(1, 1); + expect(await woken).toEqual({}); + read.resolve({ "refs/heads/main": "a".repeat(40) }); + await expect(inStandby).rejects.toThrow( + "The deployment was not put in standby: its child ran again before the standby attempt was answered", + ); + }); + + test("a standby attempt whose mark cannot be written is refused, and the child runs again", async () => { + const head = "run_standbyunmarked@example.com"; + const { spawned, control, dataDir } = await deployForStandby( + head, + 9310, + async () => ({ "refs/heads/main": "a".repeat(40) }), + ); + if (dataDir === undefined) throw new Error("the fixture has no data dir"); + await fs.rm( + path.join( + dataDir, + "workflow-runs", + deriveWorkflowRunRepoId(head), + "deployment.json", + ), + ); + + const inStandby = control("standby", "standby-unmarked"); + await spawned.putChildInStandby(0); + await expect(inStandby).rejects.toThrow( + "the standby could not be recorded", + ); + await spawned.driveReadyFor(1); + await spawned.answerParkedQuery(1, 1); + expect(spawned.spawnCount()).toBe(2); + }); + + test("a standby attempt a child crash puts off is answered as initializing, so the Hub asks again", async () => { + const { spawned, control } = await deployForStandby( + "run_standbycrash@example.com", + 9290, + async () => ({ "refs/heads/main": "a".repeat(40) }), + ); + + const inStandby = control("standby", "standby-crash"); + await spawned.answerQuiet(0, 1, { quiet: true }); + const toChild = spawned.controlFor(0); + if (toChild === undefined) throw new Error("spawn 0 missing"); + await waitForUpstreamPayloads(toChild, "quiet.request", 2); + spawned.crash(0); + await expect(inStandby).rejects.toThrow( + WORKFLOW_CONTROL_INITIALIZING_ERROR, + ); + await spawned.driveReadyFor(1); + await spawned.answerParkedQuery(1, 1); + }); + + test("a standby attempt whose history tips cannot be read is put off with the child kept stopped", async () => { + let reads = 0; + const { spawned, control } = await deployForStandby( + "run_standbyunread@example.com", + 9270, + async () => { + reads += 1; + if (reads === 1) throw new Error("disk unreadable"); + return { "refs/heads/main": "a".repeat(40) }; + }, + ); + + const inStandby = control("standby", "standby-unread"); + await spawned.putChildInStandby(0); + await expect(inStandby).rejects.toThrow( + WORKFLOW_CONTROL_INITIALIZING_ERROR, + ); + expect(await control("standby", "standby-read")).toEqual({ + refTips: { "refs/heads/main": "a".repeat(40) }, + }); + expect(spawned.spawnCount()).toBe(1); + }); + + test.each(["while asking", "before asking"] as const)( + "a standby attempt is abandoned once the connection that asked for it closes %s", + async (when) => { + const closedFirst = when === "before asking"; + const head = `run_standbyclosed${closedFirst ? "first" : ""}@example.com`; + const frame = singleStepFrame(head, "wf-standby-closed"); + const spawned = makeReadyDrivingSpawner(closedFirst ? 9195 : 9190); + const { router } = await buildMultistepFixture({ + spawner: spawned.spawner, + }); + const deployPromise = router.deploy(frame); + await spawned.driveReadyFor(0); + await deployPromise; + if (router.control === undefined) + throw new Error("router.control is undefined"); + + const connection = new AbortController(); + if (closedFirst) connection.abort(); + const inStandby = router.control( + { + type: "workflow.control", + requestId: "standby-closed", + agentAddress: head, + generation: frame.generation, + runId: deriveWorkflowRunId(head), + action: "standby", + reason: "Put the deployment's workflow child in standby", + }, + { signal: connection.signal }, + ); + if (!closedFirst) { + expect(await askedQuiet(spawned)).toBe(true); + connection.abort(); + } + await expect(inStandby).rejects.toThrow( + "The deployment was not put in standby: the Hub that asked for it disconnected", + ); + expect(spawned.spawnCount()).toBe(1); + }, + ); + test("restore holds a record whose closure will not materialize as stopped and restores the rest", async () => { const dataDir = await createTempBaseDir("sidecar-restore-softfail-data-"); const goodHead = "run_good@example.com"; diff --git a/packages/workflow-host/src/deploy/workflow-host-wiring.ts b/packages/workflow-host/src/deploy/workflow-host-wiring.ts index ca4d8255c..cfbf80471 100644 --- a/packages/workflow-host/src/deploy/workflow-host-wiring.ts +++ b/packages/workflow-host/src/deploy/workflow-host-wiring.ts @@ -44,6 +44,7 @@ import { type AgentUndeployFrame, type DeploymentStoppedFrame, type DeploymentRetention, + type DeploymentWokenFrame, type HostedIncarnation, type IncarnationState, type SourceRefPin, @@ -67,11 +68,15 @@ import { } from "../supervisor/credentials"; import { createWorkflowSupervisor, + StandbyDeferredError, + StandbyRefusedError, + type StandbyOutcome, type SpawnOpts, type WorkflowSupervisor, } from "../supervisor/supervisor"; import type { DispatchTimingMark, + RecordedStandby, SubprocessSpawner, SuspensionRegistration, } from "../supervisor/types"; @@ -86,6 +91,7 @@ import type { import { deleteWorkflowRunRecord, markWorkflowRunRecord, + markWorkflowRunRecordInStandby, scanWorkflowRunRecords, updateWorkflowRunRecordCredentials, updateWorkflowRunRecordSources, @@ -459,6 +465,10 @@ export type CreateSidecarWorkflowSupervisorOpts = { * poisoned. */ isRunPoisoned?: (runId: string) => boolean; + /** Called when the child of a supervisor in standby runs again. */ + onWake?: (info: { reason: string; requested: boolean }) => void; + /** Records the standby before anything can wake the stopped child. */ + recordStandby?: (standby: RecordedStandby) => Promise; }; export type SidecarWorkflowSupervisor = { @@ -645,6 +655,7 @@ type DeployRouterResult = { type WorkflowControlOutcome = { refTips?: WorkflowRunRefTips; retention?: DeploymentRetention; + wakeAt?: string; }; type ResolvedInboundMailPolicy = Record; @@ -659,7 +670,14 @@ type ResolvedInboundMailPolicy = Record; export interface SidecarDeployRouter { deploy(frame: AgentDeployFrame): Promise; undeploy(frame: AgentUndeployFrame): Promise; - control(frame: WorkflowControlFrame): Promise; + /** + * `signal` aborts once the connection that carried the frame closes; a + * standby attempt it asked for is abandoned then. + */ + control( + frame: WorkflowControlFrame, + opts?: { signal?: AbortSignal }, + ): Promise; /** * Re-establish every persisted workflow deployment on this sidecar's local * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step @@ -1079,6 +1097,11 @@ export function createSidecarDeployRouter< publishDeploymentStopped?: ( report: Omit, ) => void; + /** + * Send the Hub a report that an input woke a deployment in standby. Tests + * that do not exercise the report omit it. + */ + publishDeploymentWoken?: (report: Omit) => void; }): SidecarDeployRouter { // Validate the signing seed at construction so a malformed key fails // sidecar boot rather than the first deploy. The seed signs every @@ -1153,6 +1176,8 @@ export function createSidecarDeployRouter< wired?: SidecarWorkflowSupervisor; // Why a stopped incarnation stopped when the Hub did not stop it. error?: string; + // The workflow child is in standby: stopped while every run waits. + inStandby?: true; }; const deployments = new Map(); const retaining = new Set(); @@ -1336,6 +1361,39 @@ export function createSidecarDeployRouter< deps.multistepCredentialsRouter?.unregister(agentAddress); } + // A child in standby runs again. Clear the mark so a restart does not + // put it in standby, and tell the Hub when it did not ask for the wake. + function noteDeploymentWoken(args: { + runId: string; + agentAddress: string; + hosted: HostedDeployment; + reason: string; + // The Hub asked for this wake, so it needs no report. + requested: boolean; + }): void { + if (deployments.get(args.agentAddress) !== args.hosted) return; + delete args.hosted.inStandby; + const report = (): void => { + if (args.requested) return; + if (deployments.get(args.agentAddress) !== args.hosted) return; + deps.publishDeploymentWoken?.({ + agentAddress: args.agentAddress, + generation: args.hosted.generation, + reason: fitDeploymentError(`Woken by ${args.reason}`), + }); + }; + if (stepStateDataDir === undefined) { + report(); + return; + } + void markWorkflowRunRecordInStandby(stepStateDataDir, args.runId, null) + .catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`Could not clear the standby mark of ${args.agentAddress}; a restart could restore it in standby: ${message}`; + }) + .finally(report); + } + // Hold a deployment whose supervisor drove ITSELF to a terminal phase // (crash-loop latch, channel crash, recycle failure) as stopped, with why, // until the Hub undeploys it: the hello reports it, the Hub fails the @@ -1507,6 +1565,8 @@ export function createSidecarDeployRouter< * must not live that long. */ recordDataDir?: string; + /** The standby the deployment's record holds; unset on a deploy. */ + standby?: RecordedStandby; } /** @@ -1818,6 +1878,30 @@ export function createSidecarDeployRouter< error: `Its workflow child ended itself (${phase}): ${reason}`, }); }, + onWake: ({ reason, requested }) => { + noteDeploymentWoken({ + runId, + agentAddress: spec.agentAddress, + hosted, + reason, + requested, + }); + }, + recordStandby: async (standby) => { + hosted.inStandby = true; + if ( + stepStateDataDir !== undefined && + !(await markWorkflowRunRecordInStandby( + stepStateDataDir, + runId, + standby, + )) + ) { + throw new Error( + `the run record of ${spec.agentAddress} is missing or invalid, so a restart could not restore the standby`, + ); + } + }, substrateEnv, // Recomputed on every spawn AND recycle respawn. The rotation // handler below revises `currentSources` in place, so a respawn @@ -1902,7 +1986,19 @@ export function createSidecarDeployRouter< // rejection into a structured failure frame. The supervisor is // registered against the deployment address only after spawn succeeds, // so a spawn-time rejection leaves the registry untouched. - await wired.supervisor.spawn(spawnOpts); + // A restored standby whose child had to start is reported woken once + // the deployment is set up. + let restoredAwake: string | null = null; + if (spec.standby === undefined) { + await wired.supervisor.spawn(spawnOpts); + } else { + const restored = await wired.supervisor.restoreInStandby( + spawnOpts, + spec.standby, + ); + if (restored.inStandby) hosted.inStandby = true; + else restoredAwake = restored.reason; + } wiredForUnwind = wired; hosted.wired = wired; supervisorRegistered = true; @@ -2117,6 +2213,16 @@ export function createSidecarDeployRouter< ); routersRegistered = true; + if (restoredAwake !== null) { + logger.info`${spec.agentAddress} was in standby, and its restart started the child: ${restoredAwake}`; + noteDeploymentWoken({ + runId, + agentAddress: spec.agentAddress, + hosted, + reason: restoredAwake, + requested: false, + }); + } succeeded = true; return { publicKey: deploymentPublicKey }; } finally { @@ -2333,8 +2439,97 @@ export function createSidecarDeployRouter< } } + // A Hub that disconnects before the attempt it asked for settles calls it + // off, since it would not hear the answer, and a child the attempt already + // stopped is woken again. One that disconnects later leaves the child + // stopped and asks again. The supervisor records the standby before anything + // can wake the child. + async function putDeploymentInStandby( + frame: WorkflowControlFrame, + hosted: HostedDeployment, + wired: SidecarWorkflowSupervisor, + signal: AbortSignal | undefined, + ): Promise { + const abandon = (): void => { + wired.supervisor.abandonStandby("the Hub that asked for it disconnected"); + }; + signal?.addEventListener("abort", abandon, { once: true }); + let outcome: StandbyOutcome; + try { + const enteringStandby = wired.supervisor.enterStandby(); + // Only an attempt that exists can be called off. + if (signal?.aborted === true) abandon(); + outcome = await enteringStandby; + } catch (cause) { + // The Hub asks again for a standby attempt the child's restart put off. + if (cause instanceof StandbyDeferredError) { + throw new Error(WORKFLOW_CONTROL_INITIALIZING_ERROR, { cause }); + } + if (cause instanceof StandbyRefusedError) { + throw new Error( + `The deployment was not put in standby: ${cause.message}`, + { + cause, + }, + ); + } + throw cause; + } finally { + signal?.removeEventListener("abort", abandon); + } + // Read for every answer, a repeated one too: the supervisor can write + // history while the child is stopped, and reading ships what the Hub + // still lacks. + let refTips: WorkflowRunRefTips; + try { + refTips = await deps.reportDeploymentRefTips({ + agentAddress: frame.agentAddress, + generation: frame.generation, + }); + } catch (cause) { + // The child stays stopped, and the Hub asks again. Nothing else here + // acts on it: this request may be stale already. + logger.error`Could not read the history tips of ${frame.agentAddress}, which is in standby; the Hub asks again: ${cause instanceof Error ? cause.message : String(cause)}`; + throw new Error(WORKFLOW_CONTROL_INITIALIZING_ERROR, { cause }); + } + if ( + deployments.get(frame.agentAddress) !== hosted || + hosted.wired !== wired + ) + throw new Error("The deployment changed while it was entering standby"); + // A wake reports itself only after a record write, so this answer reaches + // the Hub before it. A child that already ran again ended its standby + // before the Hub heard of it. + if (hosted.inStandby !== true) + throw new Error( + "The deployment was not put in standby: its child ran again before the standby attempt was answered", + ); + return { + refTips, + ...(outcome.wakeAt !== undefined ? { wakeAt: outcome.wakeAt } : {}), + }; + } + + // The cleared mark is durable before the Hub hears the child runs, so a + // restart does not restore it in standby. + async function wakeDeployment( + frame: WorkflowControlFrame, + wired: SidecarWorkflowSupervisor, + ): Promise { + await wired.supervisor.wake(); + if (stepStateDataDir !== undefined) { + await markWorkflowRunRecordInStandby( + stepStateDataDir, + deps.deriveWorkflowRunRepoId(frame.agentAddress), + null, + ); + } + return {}; + } + async function control( frame: WorkflowControlFrame, + opts: { signal?: AbortSignal } = {}, ): Promise { if (deriveWorkflowRunId(frame.agentAddress) !== frame.runId) { throw new Error( @@ -2352,10 +2547,28 @@ export function createSidecarDeployRouter< const stopping = workflowStopTasks.get(frame.agentAddress); if (hosted?.state === "tearing-down") throw new Error("The deployment is being removed"); + if (frame.action === "standby" || frame.action === "wake") { + if (stopping !== undefined) throw new Error("The deployment is stopping"); + if (hosted?.state === "deploying") { + throw new Error(WORKFLOW_CONTROL_INITIALIZING_ERROR); + } + const wired = hosted?.wired; + if ( + hosted === undefined || + wired === undefined || + hosted.state !== "live" + ) + throw new Error( + `The deployment has no workflow child to ${frame.action === "standby" ? "put in standby" : "wake"}`, + ); + return frame.action === "standby" + ? putDeploymentInStandby(frame, hosted, wired, opts.signal) + : wakeDeployment(frame, wired); + } if (stopping !== undefined) { await stopping; if (frame.action === "retain" && hosted?.retention === undefined) - return control(frame); + return control(frame, opts); return reportControlOutcome(frame); } if (hosted?.state === "deploying") { @@ -2738,6 +2951,9 @@ export function createSidecarDeployRouter< // without this a rotation would silently drop it and wedge the // NEXT restart. sourceRef: record.sourceRef, + ...(record.standby !== undefined + ? { standby: record.standby } + : {}), }; await spawnWorkflowRun(spec, hosted); @@ -2808,6 +3024,9 @@ export function createSidecarDeployRouter< address, generation: hosted.generation, state: hosted.state, + ...(hosted.inStandby === true && hosted.state === "live" + ? { inStandby: true as const } + : {}), ...(hosted.retention !== undefined ? { retention: hosted.retention } : {}), @@ -2930,6 +3149,10 @@ export function createSidecarWorkflowSupervisor( ...(opts.onSelfTerminate !== undefined ? { onSelfTerminate: opts.onSelfTerminate } : {}), + ...(opts.onWake !== undefined ? { onWake: opts.onWake } : {}), + ...(opts.recordStandby !== undefined + ? { recordStandby: opts.recordStandby } + : {}), ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } : {}), diff --git a/packages/workflow-host/src/deploy/workflow-run-record.test.ts b/packages/workflow-host/src/deploy/workflow-run-record.test.ts index b02f4d40f..dc042ac72 100644 --- a/packages/workflow-host/src/deploy/workflow-run-record.test.ts +++ b/packages/workflow-host/src/deploy/workflow-run-record.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { type } from "arktype"; import { createEnvKeyCredentialCipher } from "@intx/crypto"; import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import { TEST_DEFINITION } from "@intx/workflow-host/testing"; import * as atomicWrite from "./atomic-write"; import { @@ -13,6 +14,7 @@ import { writeWorkflowRunRecord, deleteWorkflowRunRecord, markWorkflowRunRecord, + markWorkflowRunRecordInStandby, updateWorkflowRunRecordCredentials, updateWorkflowRunRecordSources, withWorkflowRunRecord, @@ -847,6 +849,50 @@ describe("workflow run record store", () => { } }); + test("a standby mark keeps the rest of the record and clears without trace", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "standby-1"; + const standby = { + parked: [ + { + runId: anchorRunId, + correlationId: "corr-gate", + parkKind: "approval" as const, + snapshot: { + name: "charge_card", + description: "Charge the customer's card", + inputSchema: { type: "object" }, + arguments: { amount: 100 }, + }, + }, + { + runId: anchorRunId, + correlationId: "corr-input", + parkKind: "input" as const, + }, + ], + wakeAt: "2026-11-01T00:00:00.000Z", + tip: "0123456789abcdef0123456789abcdef01234567", + definition: TEST_DEFINITION, + }; + expect( + await markWorkflowRunRecordInStandby(dataDir, anchorRunId, standby), + ).toBe(false); + + await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); + expect( + await markWorkflowRunRecordInStandby(dataDir, anchorRunId, standby), + ).toBe(true); + expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([ + { runId: anchorRunId, record: { ...SINGLE_STEP, standby } }, + ]); + + await markWorkflowRunRecordInStandby(dataDir, anchorRunId, null); + expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([ + { runId: anchorRunId, record: SINGLE_STEP }, + ]); + }); + test("marking leaves a record the boot scan would skip as it is", async () => { const dataDir = await makeDataDir(); const { generation: _dropped, ...withoutGeneration } = SINGLE_STEP; diff --git a/packages/workflow-host/src/deploy/workflow-run-record.ts b/packages/workflow-host/src/deploy/workflow-run-record.ts index 9fe78087c..0511604d4 100644 --- a/packages/workflow-host/src/deploy/workflow-run-record.ts +++ b/packages/workflow-host/src/deploy/workflow-run-record.ts @@ -38,6 +38,7 @@ import { SourceRefPin, } from "@intx/types/sidecar"; +import { RecordedStandby } from "../supervisor/recorded-standby"; import { removeFileAtomicDurable, writeFileAtomicDurable, @@ -105,6 +106,9 @@ const workflowRunRecordBase = { "state?": "'stopped' | 'tearing-down'", "retention?": DeploymentRetention, "error?": "string > 0", + // Set while the child is in standby: what a restart needs to restore the + // standby without the child while the run's history is still at `tip`. + "standby?": RecordedStandby, } as const; /** @@ -398,6 +402,32 @@ export async function markWorkflowRunRecord( }); } +/** + * Record a live deployment's child as in standby, or clear it with `null`. + * Returns false when the boot scan would not accept the run's record. + */ +export async function markWorkflowRunRecordInStandby( + dataDir: string, + runId: string, + standby: RecordedStandby | null, +): Promise { + return withWorkflowRunRecord(dataDir, runId, async (path) => { + const checked = await readWorkflowRunRecord(path, runId); + if (checked === null) return false; + const { standby: _replaced, ...record } = checked; + await writeFileAtomicDurable( + path, + JSON.stringify( + standby === null ? record : { ...record, standby }, + null, + 2, + ), + { mode: 0o600, requireDirectorySync: true }, + ); + return true; + }); +} + /** * Remove a run record. Called on a soft-failed deploy so a never-completed * run is not restored on the next boot; undeploy removes the record together diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index e1ab60c50..8e94e7031 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -54,7 +54,9 @@ import { } from "@intx/test-harness/seed"; import { WorkflowControlHistoryPendingError, + WorkflowControlRejectedError, type RepoId, + type SidecarEventMap, } from "@intx/hub-sessions"; import { @@ -64,6 +66,7 @@ import { fireMailTrigger, initiateDrain, listRunIds, + readClaimCheckDir, readWorkflowRunEvents, startDeployFlowEnv, waitFor, @@ -77,6 +80,8 @@ const DEPLOYMENT_ID = "run_drain-roundtrip-1"; const WAIT_DEPLOYMENT_ID = "run_drain-roundtrip-wait-1"; const CONTROL_DEPLOYMENT_ID = "run_control-roundtrip"; const STOP_DEPLOYMENT_ID = "run_stop-roundtrip"; +const STANDBY_DEPLOYMENT_ID = "run_standby-roundtrip"; +const REFUSED_STANDBY_DEPLOYMENT_ID = "run_standby-refused-roundtrip"; // Longer than the test's budget, so the worker's acknowledgement, not the // clock, settles the control request. const CONTROL_TIMEOUT_MS = 120_000; @@ -99,6 +104,8 @@ const CALLER_PRINCIPAL_ID = "prn_drain_roundtrip"; const DEFINITION_ASSET_IDS: Record = { [CONTROL_DEPLOYMENT_ID]: "ast_control_roundtrip", [STOP_DEPLOYMENT_ID]: "ast_stop_roundtrip", + [STANDBY_DEPLOYMENT_ID]: "ast_standby_roundtrip", + [REFUSED_STANDBY_DEPLOYMENT_ID]: "ast_standby_refused_roundtrip", [DEPLOYMENT_ID]: "ast_drain_roundtrip_cancel_wf", [WAIT_DEPLOYMENT_ID]: "ast_drain_roundtrip_wait_wf", }; @@ -580,6 +587,132 @@ test.skipIf(!harnessDbEnvAvailable())( 30_000, ); +// A wait on an author signal sends the supervisor no park notice, so the +// trigger's dispatch never sees this run park; the standby attempt goes through +// anyway once the child has taken the trigger. It runs before the stop test, +// whose stop fences the allocation identity every deployment here shares. +test.skipIf(!harnessDbEnvAvailable())( + "Hub puts a real child in standby and wakes it, and mail wakes one in standby", + async () => { + const { address, handle } = await deployParkedOnSignal({ + anchorRunId: STANDBY_DEPLOYMENT_ID, + definitionAssetId: "ast_standby_roundtrip", + workflowId: "standby-workflow", + }); + const target = env.hub.prepareAllocationIdentity( + STANDBY_DEPLOYMENT_ID, + address, + ); + const inStandbyMark = () => readInStandbyMark(handle); + // The standby attempt can overtake the child's last pushes; a repeated + // standby attempt answers again once that history has landed. + const enterStandby = () => + waitFor( + async () => { + try { + await env.hub.router.putAllocationInStandby( + target, + CONTROL_TIMEOUT_MS, + ); + return true; + } catch (error) { + if (error instanceof WorkflowControlHistoryPendingError) + return false; + throw error; + } + }, + { diagnostics: env.sidecarDiagnostics }, + ); + + await enterStandby(); + expect(await inStandbyMark()).toBeDefined(); + // A deployment in standby keeps its routes, so mail can still wake it. + expect(env.hub.router.getRoutableAddresses()).toContain(address); + + // The sidecar answers a wake once the cleared mark is on disk. + await env.hub.router.wakeAllocation(target, CONTROL_TIMEOUT_MS); + expect(await inStandbyMark()).toBeUndefined(); + + await enterStandby(); + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + await fireMailTrigger(env, address, { + messageId: `<${STANDBY_DEPLOYMENT_ID}-wake@integration.interchange>`, + }); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by mail", + }); + expect(await inStandbyMark()).toBeUndefined(); + } finally { + unsubscribe(); + } + }, +); + +// Mail waiting behind the run would wake the child as soon as it stopped, so +// the sidecar refuses the standby and the child keeps running. +test.skipIf(!harnessDbEnvAvailable())( + "Hub's standby for a real child is refused while mail waits for it", + async () => { + const anchorRunId = REFUSED_STANDBY_DEPLOYMENT_ID; + const { address, handle } = await deployParkedOnSignal({ + anchorRunId, + definitionAssetId: "ast_standby_refused_roundtrip", + workflowId: "standby-refused-workflow", + }); + const target = env.hub.prepareAllocationIdentity(anchorRunId, address); + const { messageId } = await fireMailTrigger(env, address, { + messageId: `<${anchorRunId}-waiting@integration.interchange>`, + }); + // The sidecar writes its inbox and pushes it to the Hub, so mail the Hub + // holds there is mail the sidecar has. + await waitFor( + async () => { + for (const subdir of ["inbox", "processing"] as const) { + const entries = await readClaimCheckDir( + env, + handle.workflowRunRepoId, + address, + subdir, + ); + if ( + entries.some((entry) => + new TextDecoder().decode(entry.bytes).includes(messageId), + ) + ) + return true; + } + return false; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + + const refusal = await env.hub.router + .putAllocationInStandby(target, CONTROL_TIMEOUT_MS) + .then( + () => undefined, + (error: unknown) => error, + ); + expect(refusal).toBeInstanceOf(WorkflowControlRejectedError); + // Mail still in the inbox or already taken by the dispatch, depending on + // how far the dispatch got. + expect(refusal instanceof Error ? refusal.message : "").toContain( + "The deployment was not put in standby: mail ", + ); + expect(await readInStandbyMark(handle)).toBeUndefined(); + expect(env.hub.router.getRoutableAddresses()).toContain(address); + }, +); + test.skipIf(!harnessDbEnvAvailable())( "Hub confirms a workflow stop once it holds the stopped worker's history", async () => { @@ -620,6 +753,23 @@ test.skipIf(!harnessDbEnvAvailable())( 30_000, ); +// The mark the sidecar keeps in a deployment's run record while its child is +// in standby. +async function readInStandbyMark(handle: { + workflowRunRepoId: RepoId; +}): Promise { + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const record: unknown = JSON.parse(await readFile(recordFile, "utf8")); + return typeof record === "object" && record !== null && "standby" in record + ? record.standby + : undefined; +} + /** * Read every workflow-run event under any `runs//events/` * subtree on the deployment's workflow-run repo. Used to discover the diff --git a/tests/workflow-deploy/standby-approval-wake.test.ts b/tests/workflow-deploy/standby-approval-wake.test.ts new file mode 100644 index 000000000..9b5f7c962 --- /dev/null +++ b/tests/workflow-deploy/standby-approval-wake.test.ts @@ -0,0 +1,262 @@ +// A deployment in standby on an approval wakes for the approval's decision. +// +// The run parks on an approval, and standby stops its child. The decision is the +// signal the approve route sends after claiming the approval. It starts a new +// child, which resumes the run and completes it. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type SidecarEventMap } from "@intx/hub-sessions"; +import { signalName } from "@intx/types"; +import type { WireGrantRule } from "@intx/types/grant-wire"; +import type { + ApprovalSnapshot, + HarnessConfig, + InferenceSource, +} from "@intx/types/runtime"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; +import { + createApprovalSet, + deriveRunAddress, + type ApprovalSet, +} from "@intx/workflow-deploy"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { MAIL_TOOL_NAME } from "./fixtures/mail-tool"; +import { singleStepMailToolEntry } from "./fixtures/single-step-mail-tool"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-approval-wake-1"; +const TENANT_ID = "tnt_standby_approval_wake"; +const CALLER_PRINCIPAL_ID = "prn_standby_approval_wake"; +const DEFINITION_ASSET_ID = "ast_standby_approval_wake_wf"; +const STEP_ID = "step1"; +const CALL_INPUT = { + to: "standby-approval-wake", + body: "standby-approval-wake-tool-ran.txt", +}; + +const ASK_GRANT: WireGrantRule = { + id: "grant-tool-ask", + resource: `tool:${MAIL_TOOL_NAME}`, + action: "invoke", + effect: "ask", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, +}; + +type Registration = { + correlationId: string; + runId: string; + approvalSnapshot: ApprovalSnapshot; +}; + +let env: DeployFlowEnv; +let h: TestDb; +// Resolved by the run's approval registration. +const nextRegistration = Promise.withResolvers(); + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-approval-wake-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + + env = await startDeployFlowEnv({ + // The model calls the ask-granted tool until its result is in history, + // so the run parks on the approval and completes once it is approved. + inferenceApprovalToolCall: { + toolName: MAIL_TOOL_NAME, + input: CALL_INPUT, + resultPrefix: "done: ", + }, + registerSignalCorrelation: async ({ + correlationId, + runId, + approvalSnapshot, + }) => { + nextRegistration.resolve({ correlationId, runId, approvalSnapshot }); + }, + }); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby on an approval", + () => { + test("the approval's decision starts a new child, which completes the run", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const approvals: ApprovalSet = createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + `tool:${MAIL_TOOL_NAME}`, + ]); + const handle = await deployWorkflowSourceForTest(env, { + entryModule: singleStepMailToolEntry({ + variant: "ask", + stepId: STEP_ID, + systemPrompt: "You are the single-step agent under approval control.", + address, + agentId: "agent-standby-approval-wake", + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals, + config, + sources: { [STEP_ID]: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + grants: [ASK_GRANT], + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + // The deployment's one run is the only one that registers an approval. + const parked = await nextRegistration.promise; + expect(parked.runId).toBe(runId); + const correlationId = parked.correlationId; + await waitFor( + async () => + (await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId)).some( + (e) => + e.type === "SignalAwaited" && + e.body["signalName"] === signalName(correlationId), + ), + { diagnostics: env.sidecarDiagnostics }, + ); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + + // ---- the approval's decision starts the child, which resumes the run ---- + const woken = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + woken.resolve(report); + }, + ); + try { + await injectSignal( + env, + DEPLOYMENT_ID, + runId, + signalName(correlationId), + { outcome: "approved" }, + ); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + expect(await woken.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + const count = (match: (e: (typeof events)[number]) => boolean) => + events.filter(match).length; + expect(count((e) => e.type === "RunStarted")).toBe(1); + expect( + count( + (e) => + e.type === "SignalReceived" && + e.body["signalName"] === signalName(correlationId), + ), + ).toBe(1); + expect(count((e) => e.type === "RunCompleted")).toBe(1); + }, 180_000); + }, +); diff --git a/tests/workflow-deploy/standby-cancel-wake.test.ts b/tests/workflow-deploy/standby-cancel-wake.test.ts new file mode 100644 index 000000000..73de8de5e --- /dev/null +++ b/tests/workflow-deploy/standby-cancel-wake.test.ts @@ -0,0 +1,217 @@ +// A deployment whose top-level run waits on an author signal is put in standby, +// and a cancellation wakes it on the same sidecar. +// +// The deployment is put in standby, which stops the child process. The Hub's +// cancel then commits the request and wakes the child, and the new child applies +// it from the log and ends the run as cancelled. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type RepoId, type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { CONTROL_TIMEOUT_MS, putInStandby } from "./standby-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-cancel-wake"; +const TENANT_ID = "tnt_standby_cancel_wake"; +const CALLER_PRINCIPAL_ID = "prn_standby_cancel_wake"; +const DEFINITION_ASSET_ID = "ast_standby_cancel_wake_wf"; +const SIGNAL_NAME = "go"; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-cancel-wake-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function entry(address: string): string { + return ` +import { awaitSignal, defineWorkflow } from "@intx/workflow/definition"; + +export const workflow = defineWorkflow({ + id: ${JSON.stringify(`wf_${DEPLOYMENT_ID}`)}, + trigger: { type: "mail", to: ${JSON.stringify(address)} }, + steps: { + hold: awaitSignal({ name: ${JSON.stringify(SIGNAL_NAME)} }), + }, +}); +`; +} + +async function findRunId( + workflowRunRepoId: RepoId, +): Promise { + for (const id of await listRunIds(env, workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if ( + events.some( + (e) => + e.type === "SignalAwaited" && + e.body["stepId"] === "hold" && + e.body["signalName"] === SIGNAL_NAME, + ) + ) + return id; + } + return undefined; +} + +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby on a top-level awaitSignal", + () => { + test("is woken by a cancellation, and the run ends as cancelled", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: entry(address), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let found: string | undefined; + await waitFor( + async () => { + found = await findRunId(handle.workflowRunRepoId); + return found !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (found === undefined) throw new Error("no run on the gate"); + const runId = found; + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + await env.hub.router.sendWorkflowControl( + target, + { + action: "cancel", + agentAddress: address, + runId: DEPLOYMENT_ID, + reason: "No longer needed", + }, + CONTROL_TIMEOUT_MS, + ); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCancelled"); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by cancel", + }); + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + expect(events.filter((e) => e.type === "RunStarted")).toHaveLength(1); + expect(events.filter((e) => e.type === "CancelRequested")).toHaveLength( + 1, + ); + expect(events.filter((e) => e.type === "SignalReceived")).toEqual([]); + } finally { + unsubscribe(); + } + }); + }, +); diff --git a/tests/workflow-deploy/standby-childworkflow-signal.test.ts b/tests/workflow-deploy/standby-childworkflow-signal.test.ts new file mode 100644 index 000000000..12fd1c54a --- /dev/null +++ b/tests/workflow-deploy/standby-childworkflow-signal.test.ts @@ -0,0 +1,279 @@ +// A deployment whose run has a childWorkflow parked inside it is put in +// standby, and the signal the child workflow waits on wakes it. +// +// The child workflow waits on an author signal, which its parent relays onto +// the deployment run. The deployment is put in standby, which stops the child +// process. The signal then wakes it: the new child resumes the parent and the +// child workflow it spawned, the child workflow takes the signal and +// completes, and so does the parent. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type RepoId, type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { createApprovalSet, deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { childWorkflowUnanswerableGateEntry } from "./fixtures/childworkflow-unanswerable-gate"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-childworkflow-signal"; +const TENANT_ID = "tnt_standby_childworkflow"; +const CALLER_PRINCIPAL_ID = "prn_standby_childworkflow"; +const DEFINITION_ASSET_ID = "ast_standby_childworkflow_wf"; +const SIGNAL_NAME = "go"; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-childworkflow-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +// The parent is the run whose relay of the child workflow's gate sits on the +// spawn step. +async function findParentRunId( + workflowRunRepoId: RepoId, +): Promise { + for (const id of await listRunIds(env, workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if ( + events.some( + (e) => + e.type === "SignalAwaited" && + e.body["stepId"] === "spawn" && + e.body["signalName"] === SIGNAL_NAME, + ) + ) + return id; + } + return undefined; +} + +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby with a childWorkflow parked in its run", + () => { + test("is woken by the child workflow's signal, and the child workflow and its parent complete", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: childWorkflowUnanswerableGateEntry({ + address, + workflowId: `wf_${DEPLOYMENT_ID}`, + childWorkflowId: `wf_child_${DEPLOYMENT_ID}`, + signalName: SIGNAL_NAME, + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]), + config, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let found: string | undefined; + await waitFor( + async () => { + found = await findParentRunId(handle.workflowRunRepoId); + return found !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (found === undefined) throw new Error("no parent run on the spawn"); + const parentRunId = found; + const spawned = ( + await readWorkflowRunEvents(env, DEPLOYMENT_ID, parentRunId) + ).flatMap((e) => + e.type === "ChildSpawned" ? [e.body["childRunId"]] : [], + ); + expect(spawned).toHaveLength(1); + const childRunId = spawned[0]; + if (typeof childRunId !== "string") + throw new Error("the spawn named no child run"); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const standbyMark = async (): Promise => { + const record: unknown = JSON.parse( + await fs.promises.readFile(recordFile, "utf8"), + ); + return typeof record === "object" && + record !== null && + "standby" in record + ? record.standby + : undefined; + }; + // The child workflow's gate has no deadline, so only the signal wakes it. + expect(await standbyMark()).toEqual({ + parked: [], + tip: expect.any(String), + definition: expect.objectContaining({ id: `wf_${DEPLOYMENT_ID}` }), + }); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + const injected = await injectSignal( + env, + DEPLOYMENT_ID, + parentRunId, + SIGNAL_NAME, + { resumed: true }, + ); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + parentRunId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + expect(await standbyMark()).toBeUndefined(); + + const parent = await readWorkflowRunEvents( + env, + DEPLOYMENT_ID, + parentRunId, + ); + const onParent = (type: string): number => + parent.filter((e) => e.type === type).length; + expect(onParent("RunStarted")).toBe(1); + expect(onParent("ChildSpawned")).toBe(1); + expect(onParent("RunCompleted")).toBe(1); + expect( + parent.filter( + (e) => + e.type === "SignalReceived" && + e.body["signalName"] === SIGNAL_NAME, + ), + ).toMatchObject([ + { body: { signalId: injected.signalId, payload: { resumed: true } } }, + ]); + expect( + parent.some( + (e) => + e.type === "ChildCompleted" && + e.body["childRunId"] === childRunId, + ), + ).toBe(true); + } finally { + unsubscribe(); + } + + // The new child resumed the child workflow it found parked, rather than + // spawning it again. + const child = await readWorkflowRunEvents(env, DEPLOYMENT_ID, childRunId); + const onChild = (type: string): number => + child.filter((e) => e.type === type).length; + expect(onChild("RunStarted")).toBe(1); + expect(onChild("RunCompleted")).toBe(1); + }); + }, +); diff --git a/tests/workflow-deploy/standby-helpers.ts b/tests/workflow-deploy/standby-helpers.ts new file mode 100644 index 000000000..bf89e9017 --- /dev/null +++ b/tests/workflow-deploy/standby-helpers.ts @@ -0,0 +1,31 @@ +// Helpers the standby tests share: put a deployment in standby over the link. + +import { WorkflowControlHistoryPendingError } from "@intx/hub-sessions"; + +import { waitFor, type DeployFlowEnv } from "../hub-agent/lib/deploy-flow-env"; + +// Longer than any test's budget, so the worker's acknowledgement, not the +// clock, settles a control request. +export const CONTROL_TIMEOUT_MS = 300_000; + +type AllocationTarget = { allocationId: string; generation: number }; + +// The standby attempt can overtake the child's last pushes; a repeated one +// answers again once that history has landed. +export async function putInStandby( + env: DeployFlowEnv, + target: AllocationTarget, +): Promise { + await waitFor( + async () => { + try { + await env.hub.router.putAllocationInStandby(target, CONTROL_TIMEOUT_MS); + return true; + } catch (error) { + if (error instanceof WorkflowControlHistoryPendingError) return false; + throw error; + } + }, + { diagnostics: env.sidecarDiagnostics }, + ); +} diff --git a/tests/workflow-deploy/standby-mail-wakes-input-park.test.ts b/tests/workflow-deploy/standby-mail-wakes-input-park.test.ts new file mode 100644 index 000000000..ecaaa4c00 --- /dev/null +++ b/tests/workflow-deploy/standby-mail-wakes-input-park.test.ts @@ -0,0 +1,245 @@ +// Mail wakes a deployment in standby whose run waits on input, and the new +// child answers it as the run's next turn. +// +// An unbounded single-step mail agent replies to mail 1 and parks on input. +// The deployment is put in standby, which stops the child. Mail 2 then wakes +// it: the new child resumes the run parked on input, and the supervisor hands +// it mail 2 on that input once the child reports the park, though the park +// was taken before the child started and sends no notice of its own. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { parseHeaderSection } from "@intx/mime"; +import { tenant as tenantTable } from "@intx/db/schema"; +import { type SidecarEventMap } from "@intx/hub-sessions"; +import type { WireGrantRule } from "@intx/types/grant-wire"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; +import { deriveRunAddress } from "@intx/workflow-deploy"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + type DeployFlowEnv, + type WorkflowRunEvent, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { singleStepMailInboxEntry } from "./fixtures/single-step-mail-inbox-agent"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_5ba1d0ba11ce5a11b0ba5eba11cafe0f"; +const TENANT_ID = "tnt_standby_mail_input"; +const CALLER_PRINCIPAL_ID = "prn_standby_mail_input"; +const DEFINITION_ASSET_ID = "ast_standby_mail_input_wf"; +const STEP_ID = "step1"; + +const MAIL1_MESSAGE_ID = ""; +const MAIL2_MESSAGE_ID = ""; +const MAIL_FROM = "user-standby-input@integration.interchange"; + +const ALL_TOOL_GRANTS: WireGrantRule[] = [ + "mail_send", + "mail_reply", + "mail_search", + "mail_read", + "mail_wait", +].map((name) => ({ + id: `grant-tool-${name}`, + resource: `tool:${name}`, + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, +})); + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-mail-input-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + // Each turn replies with plain text and calls no tool, so the unbounded + // step parks on input after every turn. + env = await startDeployFlowEnv({ + inferenceScriptedTurns: [{ text: "Acknowledged." }], + }); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function inputParks(events: WorkflowRunEvent[]): number { + return events.filter( + (e) => e.type === "SignalAwaited" && e.body["parkKind"] === "input", + ).length; +} + +describe.skipIf(!harnessDbEnvAvailable())( + "mail that wakes a deployment in standby", + () => { + test("reaches the run parked on input as its next turn", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: singleStepMailInboxEntry({ + stepId: STEP_ID, + systemPrompt: "You are the single-step mail-inbox agent.", + address, + agentId: "agent-standby-mail-input", + triggers: "unbounded", + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + sources: { [STEP_ID]: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + const replyTo = (messageId: string) => + env.hub.outboundMail + .map((m) => ({ m, headers: parseHeaderSection(m.raw).headers })) + .find( + ({ m, headers }) => + m.senderAddress === address && + headers.get("in-reply-to") === messageId, + )?.headers; + + await fireMailTrigger(env, address, { + messageId: MAIL1_MESSAGE_ID, + from: MAIL_FROM, + content: "First turn.", + grants: ALL_TOOL_GRANTS, + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + await waitFor(() => replyTo(MAIL1_MESSAGE_ID) !== undefined, { + diagnostics: env.sidecarDiagnostics, + }); + const reply1MessageId = replyTo(MAIL1_MESSAGE_ID)?.get("message-id"); + if (reply1MessageId === undefined) { + throw new Error("the first reply carried no Message-Id"); + } + await waitFor( + async () => + inputParks(await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId)) >= + 1, + { diagnostics: env.sidecarDiagnostics }, + ); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + // The user answers the first reply, continuing the conversation. + await fireMailTrigger(env, address, { + messageId: MAIL2_MESSAGE_ID, + from: MAIL_FROM, + content: "Second turn.", + inReplyTo: reply1MessageId, + references: [MAIL1_MESSAGE_ID, reply1MessageId], + grants: ALL_TOOL_GRANTS, + }); + await waitFor(() => replyTo(MAIL2_MESSAGE_ID) !== undefined, { + diagnostics: env.sidecarDiagnostics, + }); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by mail", + }); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + // Mail 2 was the run's next turn on its input, not a run of its own. + expect(events.filter((e) => e.type === "RunStarted")).toHaveLength(1); + expect( + events.filter( + (e) => + e.type === "SignalReceived" && + e.body["signalId"] === MAIL2_MESSAGE_ID, + ), + ).toHaveLength(1); + }, 180_000); + }, +); diff --git a/tests/workflow-deploy/standby-on-trigger-between-events.test.ts b/tests/workflow-deploy/standby-on-trigger-between-events.test.ts new file mode 100644 index 000000000..64a00240d --- /dev/null +++ b/tests/workflow-deploy/standby-on-trigger-between-events.test.ts @@ -0,0 +1,274 @@ +// An onTrigger section idle between events is put in standby, and the next +// event wakes it and is serviced by the new child. +// +// Mail 1 starts the container run, whose section runs event 0's body (a short +// sleep) and re-arms on its input park. The deployment is put in standby, which +// stops the child. Mail 2 then wakes it: the new child resumes the container +// on its input park, and the section runs event 1's body to completion. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child driving `runOnTrigger`, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type RepoId, type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { sectionBodyRunId } from "@intx/workflow"; +import { createApprovalSet, deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + type DeployFlowEnv, + type WorkflowRunEvent, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { onTriggerBodyEntry } from "./fixtures/on-trigger-body"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-on-trigger-between-events"; +const TENANT_ID = "tnt_standby_on_trigger"; +const CALLER_PRINCIPAL_ID = "prn_standby_on_trigger"; +const DEFINITION_ASSET_ID = "ast_standby_on_trigger_wf"; +const SECTION_ID = "section"; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-on-trigger-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +// The container is the one run whose id is not a minted section body's. +async function findContainerRunId( + workflowRunRepoId: RepoId, +): Promise { + const ids = await listRunIds(env, workflowRunRepoId); + return ids.find((id) => !id.startsWith(`${SECTION_ID}__`)); +} + +function childCompletedAt( + events: readonly WorkflowRunEvent[], + childRunId: string, +): number { + return events.findIndex( + (e) => e.type === "ChildCompleted" && e.body["childRunId"] === childRunId, + ); +} + +describe.skipIf(!harnessDbEnvAvailable())( + "an onTrigger section in standby between events", + () => { + test("is woken by the next event, whose body the new child runs to completion", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + // A body that completes on its own, so the section re-arms on its + // input after each event. + entryModule: onTriggerBodyEntry({ + address, + sectionId: SECTION_ID, + body: { variant: "sleep", stepId: "wait", duration: 10 }, + workflowId: `wf_${DEPLOYMENT_ID}`, + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]), + config, + sources: { [SECTION_ID]: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + content: "First event.", + }); + // Event 0's body completed and the section re-armed on its input after. + let found: string | undefined; + await waitFor( + async () => { + found = await findContainerRunId(handle.workflowRunRepoId); + if (found === undefined) return false; + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, found); + const completed = childCompletedAt( + events, + sectionBodyRunId(found, SECTION_ID, 0), + ); + return ( + completed >= 0 && + events + .slice(completed) + .some( + (e) => + e.type === "SignalAwaited" && e.body["parkKind"] === "input", + ) + ); + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (found === undefined) throw new Error("no container run"); + const containerRunId = found; + const event0 = sectionBodyRunId(containerRunId, SECTION_ID, 0); + const event1 = sectionBodyRunId(containerRunId, SECTION_ID, 1); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const standbyMark = async (): Promise => { + const record: unknown = JSON.parse( + await fs.promises.readFile(recordFile, "utf8"), + ); + return typeof record === "object" && + record !== null && + "standby" in record + ? record.standby + : undefined; + }; + // The container waits on its input alone: event 0's sleep has fired, so + // no timer wakes the section on its own. + expect(await standbyMark()).toEqual({ + parked: [ + { + correlationId: expect.any(String), + parkKind: "input", + runId: containerRunId, + }, + ], + tip: expect.any(String), + definition: expect.objectContaining({ id: `wf_${DEPLOYMENT_ID}` }), + }); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + await fireMailTrigger(env, address, { + messageId: "", + content: "Second event.", + }); + await waitFor( + async () => + childCompletedAt( + await readWorkflowRunEvents(env, DEPLOYMENT_ID, containerRunId), + event1, + ) >= 0, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by mail", + }); + expect(await standbyMark()).toBeUndefined(); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents( + env, + DEPLOYMENT_ID, + containerRunId, + ); + const types = events.map((e) => e.type); + // Mail 2 was the section's next event, not a run of its own. + expect(types.filter((t) => t === "RunStarted")).toHaveLength(1); + expect( + events.flatMap((e) => + e.type === "ChildSpawned" ? [e.body["childRunId"]] : [], + ), + ).toEqual([event0, event1]); + expect(types).not.toContain("RunCompleted"); + expect(types).not.toContain("RunFailed"); + expect(types).not.toContain("RunCancelled"); + }); + }, +); diff --git a/tests/workflow-deploy/standby-restart-approval.test.ts b/tests/workflow-deploy/standby-restart-approval.test.ts new file mode 100644 index 000000000..92d3d6b19 --- /dev/null +++ b/tests/workflow-deploy/standby-restart-approval.test.ts @@ -0,0 +1,337 @@ +// A deployment in standby on an approval re-sends that approval after a +// sidecar restart without starting its child, and the decision then starts the +// child, which completes the run. The Hub's correlation lookup records each +// registration instead of writing approval rows, and the decision is the signal +// the approve route sends after claiming the approval. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type SidecarEventMap } from "@intx/hub-sessions"; +import { signalName } from "@intx/types"; +import type { WireGrantRule } from "@intx/types/grant-wire"; +import type { + ApprovalSnapshot, + HarnessConfig, + InferenceSource, +} from "@intx/types/runtime"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; +import { + createApprovalSet, + deriveRunAddress, + type ApprovalSet, +} from "@intx/workflow-deploy"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + startSidecarSubprocess, + terminateSidecarSubprocess, + waitFor, + waitForFirstRunId, + waitForReconnect, + waitForWorkflowRunComplete, + type DeployFlowEnv, + type SidecarHandle, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { MAIL_TOOL_NAME } from "./fixtures/mail-tool"; +import { singleStepMailToolEntry } from "./fixtures/single-step-mail-tool"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-restart-approval-1"; +const TENANT_ID = "tnt_standby_restart_approval"; +const CALLER_PRINCIPAL_ID = "prn_standby_restart_approval"; +const DEFINITION_ASSET_ID = "ast_standby_restart_approval_wf"; +const STEP_ID = "step1"; +const CALL_INPUT = { to: "standby-restart", body: "standby-tool-ran.txt" }; + +const ASK_GRANT: WireGrantRule = { + id: "grant-tool-ask", + resource: `tool:${MAIL_TOOL_NAME}`, + action: "invoke", + effect: "ask", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, +}; + +type Registration = { + correlationId: string; + runId: string; + approvalSnapshot: ApprovalSnapshot; +}; + +let env: DeployFlowEnv; +let h: TestDb; +let restartedSidecar: SidecarHandle | undefined; +const restartTempDirs: string[] = []; +// Resolved by the next approval registration; the test re-arms it before the +// registration it awaits. +let nextRegistration = Promise.withResolvers(); + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-restart-approval-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + + env = await startDeployFlowEnv({ + // The model calls the ask-granted tool until its result is in history, + // so the run parks on the approval and completes once it is approved. + inferenceApprovalToolCall: { + toolName: MAIL_TOOL_NAME, + input: CALL_INPUT, + resultPrefix: "done: ", + }, + registerSignalCorrelation: async ({ + correlationId, + runId, + approvalSnapshot, + }) => { + nextRegistration.resolve({ correlationId, runId, approvalSnapshot }); + }, + }); +}); + +afterAll(async () => { + if (restartedSidecar !== undefined) { + await terminateSidecarSubprocess(restartedSidecar); + } + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); + for (const dir of restartTempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }); + } +}); + +describe.skipIf(!harnessDbEnvAvailable())( + "sidecar restart of a deployment in standby on an approval", + () => { + test("the restore re-registers the approval without a child, and its decision resumes the run", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const approvals: ApprovalSet = createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + `tool:${MAIL_TOOL_NAME}`, + ]); + const handle = await deployWorkflowSourceForTest(env, { + entryModule: singleStepMailToolEntry({ + variant: "ask", + stepId: STEP_ID, + systemPrompt: "You are the single-step agent under approval control.", + address, + agentId: "agent-standby-restart-approval", + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals, + config, + sources: { [STEP_ID]: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + grants: [ASK_GRANT], + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + // The deployment's one run is the only one that registers an approval. + const parked = await nextRegistration.promise; + expect(parked.runId).toBe(runId); + const correlationId = parked.correlationId; + await waitFor( + async () => + (await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId)).some( + (e) => + e.type === "SignalAwaited" && + e.body["signalName"] === signalName(correlationId), + ), + { diagnostics: env.sidecarDiagnostics }, + ); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const standbyMark = async (): Promise => { + const record: unknown = JSON.parse( + await fs.promises.readFile(recordFile, "utf8"), + ); + return typeof record === "object" && + record !== null && + "standby" in record + ? record.standby + : undefined; + }; + expect(await standbyMark()).toEqual({ + parked: [ + expect.objectContaining({ + runId, + correlationId, + parkKind: "approval", + }), + ], + tip: expect.any(String), + definition: expect.objectContaining({ id: "wf_single_step_mail_tool" }), + }); + + // ---- kill the sidecar, and start a fresh one on its data dir ---- + const dataDir = env.sidecar.dataDir; + await terminateSidecarSubprocess(env.sidecar); + await waitFor( + () => !env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + nextRegistration = Promise.withResolvers(); + const reports: SidecarEventMap["deployment.standby"][] = []; + // The restore reports first, so the second report is the wake's. + const wakeReported = Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reports.push(report); + if (reports.length === 2) wakeReported.resolve(undefined); + }, + ); + try { + const hubPort = env.hub.server.port; + if (hubPort === undefined) + throw new Error("hub.server.port is undefined after the kill"); + restartedSidecar = await startSidecarSubprocess({ + hubPort, + registerTempDir: (dir) => { + restartTempDirs.push(dir); + }, + extraEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + env.registerSidecar(restartedSidecar); + await waitForReconnect(env, address); + + // ---- the restored standby re-sends the approval it waits on ---- + const reRegistered = await nextRegistration.promise; + const restarted = { ...env, sidecar: restartedSidecar }; + expect(listWorkflowHostChildren(restarted)).toEqual([]); + expect(reRegistered).toEqual(parked); + expect(reports).toEqual([{ ...target, inStandby: true }]); + expect(await standbyMark()).toBeDefined(); + + // ---- the approval's decision starts the child, which resumes the run ---- + await injectSignal( + env, + DEPLOYMENT_ID, + runId, + signalName(correlationId), + { + outcome: "approved", + }, + ); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + await wakeReported.promise; + expect(reports[1]).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + expect(await standbyMark()).toBeUndefined(); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + const count = (match: (e: (typeof events)[number]) => boolean) => + events.filter(match).length; + expect(count((e) => e.type === "RunStarted")).toBe(1); + expect( + count( + (e) => + e.type === "SignalReceived" && + e.body["signalName"] === signalName(correlationId), + ), + ).toBe(1); + expect(count((e) => e.type === "RunCompleted")).toBe(1); + }, 180_000); + }, +); diff --git a/tests/workflow-deploy/standby-restart.test.ts b/tests/workflow-deploy/standby-restart.test.ts new file mode 100644 index 000000000..3a3cc53f8 --- /dev/null +++ b/tests/workflow-deploy/standby-restart.test.ts @@ -0,0 +1,282 @@ +// A deployment in standby survives a sidecar restart without its child: a run +// parked on a signal is put in standby, the sidecar is killed and restarted on +// its data dir, and the restore reports the standby with no child running. The +// signal then starts the child, which completes the run with each effect once. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { + createApprovalSet, + deriveRunAddress, + type ApprovalSet, +} from "@intx/workflow-deploy"; +import { tenant as tenantTable } from "@intx/db/schema"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; +import { type SidecarEventMap } from "@intx/hub-sessions"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + startSidecarSubprocess, + terminateSidecarSubprocess, + waitFor, + waitForReconnect, + waitForWorkflowRunComplete, + type DeployFlowEnv, + type SidecarHandle, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; +import { signalGateEntry } from "./fixtures/signal-gate"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-restart-1"; +const TENANT_ID = "tnt_standby_restart"; +const CALLER_PRINCIPAL_ID = "prn_standby_restart"; +const DEFINITION_ASSET_ID = "ast_standby_restart_wf"; + +let env: DeployFlowEnv; +let h: TestDb; +let restartedSidecar: SidecarHandle | undefined; +const restartTempDirs: string[] = []; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-restart-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (restartedSidecar !== undefined) { + await terminateSidecarSubprocess(restartedSidecar); + } + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); + for (const dir of restartTempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }); + } +}); + +describe.skipIf(!harnessDbEnvAvailable())( + "sidecar restart of a deployment in standby", + () => { + test("the restore keeps the deployment in standby without a child, and the awaited signal resumes the run", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const approvals: ApprovalSet = createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]); + const handle = await deployWorkflowSourceForTest(env, { + entryModule: signalGateEntry({ + address, + signalName: "go", + systemPrompt1: "You are the first step agent.", + systemPrompt2: "You are the second step agent.", + agentId1: "agent-step1", + agentId2: "agent-step2", + workflowId: `wf_${DEPLOYMENT_ID}`, + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals, + config, + sources: { + step1: [inferenceSource], + gate: [inferenceSource], + step2: [inferenceSource], + }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let runId: string | undefined; + await waitFor( + async () => { + for (const id of await listRunIds(env, handle.workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if ( + events.some( + (e) => + e.type === "SignalAwaited" && e.body["signalName"] === "go", + ) + ) + runId = id; + } + return runId !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (runId === undefined) throw new Error("no run reached the gate"); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const standbyMark = async (): Promise => { + const record: unknown = JSON.parse( + await fs.promises.readFile(recordFile, "utf8"), + ); + return typeof record === "object" && + record !== null && + "standby" in record + ? record.standby + : undefined; + }; + expect(await standbyMark()).toEqual({ + parked: [], + tip: expect.any(String), + definition: expect.objectContaining({ id: `wf_${DEPLOYMENT_ID}` }), + }); + + // ---- kill the sidecar, and start a fresh one on its data dir ---- + const dataDir = env.sidecar.dataDir; + await terminateSidecarSubprocess(env.sidecar); + await waitFor( + () => !env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + const reports: SidecarEventMap["deployment.standby"][] = []; + // The restore reports first, so the second report is the wake's. + const wakeReported = Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reports.push(report); + if (reports.length === 2) wakeReported.resolve(undefined); + }, + ); + try { + const hubPort = env.hub.server.port; + if (hubPort === undefined) + throw new Error("hub.server.port is undefined after the kill"); + restartedSidecar = await startSidecarSubprocess({ + hubPort, + registerTempDir: (dir) => { + restartTempDirs.push(dir); + }, + extraEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + env.registerSidecar(restartedSidecar); + await waitForReconnect(env, address); + + const restarted = { ...env, sidecar: restartedSidecar }; + expect(listWorkflowHostChildren(restarted)).toEqual([]); + expect(reports).toEqual([{ ...target, inStandby: true }]); + expect(await standbyMark()).toBeDefined(); + + // ---- the awaited signal starts the child, which resumes the run ---- + await injectSignal(env, DEPLOYMENT_ID, runId, "go", { resumed: true }); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + await wakeReported.promise; + expect(reports[1]).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + expect(await standbyMark()).toBeUndefined(); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + const count = (match: (e: (typeof events)[number]) => boolean) => + events.filter(match).length; + expect(count((e) => e.type === "RunStarted")).toBe(1); + expect( + count( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "step1", + ), + ).toBe(1); + expect( + count( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "step2", + ), + ).toBe(1); + expect(count((e) => e.type === "RunCompleted")).toBe(1); + }, 180_000); + }, +); diff --git a/tests/workflow-deploy/standby-signal-wake.test.ts b/tests/workflow-deploy/standby-signal-wake.test.ts new file mode 100644 index 000000000..3e0705759 --- /dev/null +++ b/tests/workflow-deploy/standby-signal-wake.test.ts @@ -0,0 +1,221 @@ +// A deployment whose top-level run waits on an author signal is put in standby, +// and that signal wakes it on the same sidecar. +// +// The run's own `awaitSignal` is the park, with no child workflow to relay it. +// The deployment is put in standby, which stops the child process. The signal +// then wakes it, the new child resumes the run, the gate takes the signal and +// the run completes. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type RepoId, type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-signal-wake"; +const TENANT_ID = "tnt_standby_signal_wake"; +const CALLER_PRINCIPAL_ID = "prn_standby_signal_wake"; +const DEFINITION_ASSET_ID = "ast_standby_signal_wake_wf"; +const SIGNAL_NAME = "go"; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-signal-wake-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function entry(address: string): string { + return ` +import { awaitSignal, defineWorkflow } from "@intx/workflow/definition"; + +export const workflow = defineWorkflow({ + id: ${JSON.stringify(`wf_${DEPLOYMENT_ID}`)}, + trigger: { type: "mail", to: ${JSON.stringify(address)} }, + steps: { + hold: awaitSignal({ name: ${JSON.stringify(SIGNAL_NAME)} }), + }, +}); +`; +} + +async function findRunId( + workflowRunRepoId: RepoId, +): Promise { + for (const id of await listRunIds(env, workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if ( + events.some( + (e) => + e.type === "SignalAwaited" && + e.body["stepId"] === "hold" && + e.body["signalName"] === SIGNAL_NAME, + ) + ) + return id; + } + return undefined; +} + +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby on a top-level awaitSignal", + () => { + test("is woken by that signal, and the run completes", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: entry(address), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let found: string | undefined; + await waitFor( + async () => { + found = await findRunId(handle.workflowRunRepoId); + return found !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (found === undefined) throw new Error("no run on the gate"); + const runId = found; + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + const injected = await injectSignal( + env, + DEPLOYMENT_ID, + runId, + SIGNAL_NAME, + { resumed: true }, + ); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + expect(events.filter((e) => e.type === "RunStarted")).toHaveLength(1); + expect( + events.filter( + (e) => + e.type === "SignalReceived" && + e.body["signalName"] === SIGNAL_NAME, + ), + ).toMatchObject([ + { body: { signalId: injected.signalId, payload: { resumed: true } } }, + ]); + } finally { + unsubscribe(); + } + }); + }, +); diff --git a/tests/workflow-deploy/standby-two-child-parks.test.ts b/tests/workflow-deploy/standby-two-child-parks.test.ts new file mode 100644 index 000000000..da52c546d --- /dev/null +++ b/tests/workflow-deploy/standby-two-child-parks.test.ts @@ -0,0 +1,242 @@ +// A deployment whose child workflow holds two open parks is put in standby, and +// each park is answered in turn. +// +// The child workflow waits on two author signals at once. Its parent relays one, +// and the other stays queued in the child, since the spawn step awaits one signal +// at a time. Standby stops the child process with the second park relayed by +// nobody. The signal for the first wakes it; the new child resumes the child +// workflow and its parent relays the second, and that signal finishes the run. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type RepoId, type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForWorkflowRunComplete, + type DeployFlowEnv, + type WorkflowRunEvent, +} from "../hub-agent/lib/deploy-flow-env"; +import { putInStandby } from "./standby-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-two-child-parks"; +const TENANT_ID = "tnt_standby_two_child_parks"; +const CALLER_PRINCIPAL_ID = "prn_standby_two_child_parks"; +const DEFINITION_ASSET_ID = "ast_standby_two_child_parks_wf"; +const SIGNAL_NAMES = ["first", "second"]; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-two-child-parks-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function entry(address: string): string { + return ` +import { awaitSignal, childWorkflow, defineWorkflow } from "@intx/workflow/definition"; + +const child = defineWorkflow({ + id: ${JSON.stringify(`wf_child_${DEPLOYMENT_ID}`)}, + trigger: { type: "manual" }, + steps: { + first: awaitSignal({ name: ${JSON.stringify(SIGNAL_NAMES[0])} }), + second: awaitSignal({ name: ${JSON.stringify(SIGNAL_NAMES[1])} }), + }, +}); + +export const workflow = defineWorkflow({ + id: ${JSON.stringify(`wf_${DEPLOYMENT_ID}`)}, + trigger: { type: "mail", to: ${JSON.stringify(address)} }, + steps: { + spawn: childWorkflow({ definition: child }), + }, +}); +`; +} + +// The signals the parent has relayed so far, in the order it relayed them. +function relayed(events: WorkflowRunEvent[]): unknown[] { + return events.flatMap((e) => + e.type === "SignalAwaited" && e.body["stepId"] === "spawn" + ? [e.body["signalName"]] + : [], + ); +} + +async function findParentRunId( + workflowRunRepoId: RepoId, +): Promise { + for (const id of await listRunIds(env, workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if (relayed(events).length > 0) return id; + } + return undefined; +} + +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby with a child workflow holding two open parks", + () => { + test("takes each park's signal in turn, and the run completes", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: entry(address), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let found: string | undefined; + await waitFor( + async () => { + found = await findParentRunId(handle.workflowRunRepoId); + return found !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (found === undefined) throw new Error("no parent run on the spawn"); + const parentRunId = found; + const relays = async () => + relayed(await readWorkflowRunEvents(env, DEPLOYMENT_ID, parentRunId)); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + await putInStandby(env, target); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const reported = + Promise.withResolvers(); + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + reported.resolve(report); + }, + ); + try { + // Only one park is on the parent while the child is stopped. + const before = await relays(); + expect(before).toHaveLength(1); + const firstName = before[0]; + if (typeof firstName !== "string") + throw new Error("the parent relayed no signal name"); + const secondName = SIGNAL_NAMES.find((name) => name !== firstName); + if (secondName === undefined) throw new Error("no second signal"); + + await injectSignal(env, DEPLOYMENT_ID, parentRunId, firstName, { + answered: firstName, + }); + expect(await reported.promise).toEqual({ + ...target, + inStandby: false, + reason: "Woken by signal", + }); + await waitFor(async () => (await relays()).includes(secondName), { + diagnostics: env.sidecarDiagnostics, + }); + await injectSignal(env, DEPLOYMENT_ID, parentRunId, secondName, { + answered: secondName, + }); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + parentRunId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + + const parent = await readWorkflowRunEvents( + env, + DEPLOYMENT_ID, + parentRunId, + ); + expect(parent.filter((e) => e.type === "RunStarted")).toHaveLength(1); + expect(parent.filter((e) => e.type === "ChildSpawned")).toHaveLength(1); + } finally { + unsubscribe(); + } + }); + }, +); From 65a681a18c722486bc22b14615abde94eb3a072f Mon Sep 17 00:00:00 2001 From: FarinM Date: Mon, 26 Oct 2026 14:58:05 +0100 Subject: [PATCH 20/20] Let users put a deployment in standby and wake it through the Hub API A user could not ask for a deployment to be put in standby or woken. The API records the request and the lifecycle sweep carries it out against the sidecar, so the call does not wait for every run to go quiet. The Hub now also acts on what the sidecar reports about standby: a child in standby that nobody asked for is recorded waking and woken, and a hello finding the child running ends a recorded standby. Moving a deployment to replacement capacity resets its standby state, since the replacement starts awake: a standby still wanted is asked for again there, and a wake has nothing left to do. --- Makefile | 2 +- apps/hub/src/server.ts | 4 + docs/API.md | 30 +- docs/HARNESS_DESIGN.md | 2 +- docs/IMPLEMENTATION.md | 2 +- docs/SIDECAR_PLACEMENT.md | 6 +- docs/workflow-lifecycle-policy.md | 79 +- .../migrations/0125_workflow_run_standby.sql | 4 + .../db/migrations/meta/0125_snapshot.json | 5019 +++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/src/parse-row.test.ts | 18 + packages/db/src/parse-row.ts | 6 + packages/db/src/schema/workflow-run.ts | 16 + packages/db/src/sidecar-allocation-store.ts | 32 + packages/hub-api/src/routes/runs.test.ts | 66 + packages/hub-api/src/routes/runs.ts | 112 +- .../src/workflow-lifecycle-service.ts | 705 ++- packages/types/src/workflow-lifecycle.ts | 13 + .../db/sidecar-lifecycle-integration.test.ts | 3 +- tests/db/sidecar-test-helpers.ts | 8 + tests/db/workflow-execution-admission.test.ts | 5 +- tests/db/workflow-lifecycle-service.test.ts | 1317 ++++- tests/db/workflow-run-terminal-flip.test.ts | 5 + tests/workflow-deploy/drain-roundtrip.test.ts | 211 +- tests/workflow-deploy/standby-helpers.ts | 105 +- .../standby-restart-hub-record.test.ts | 318 ++ .../standby-sleep-wake.test.ts | 205 + .../standby-timed-gate-wake.test.ts | 208 + .../standby-timer-wake-at.test.ts | 200 + 29 files changed, 8658 insertions(+), 50 deletions(-) create mode 100644 packages/db/migrations/0125_workflow_run_standby.sql create mode 100644 packages/db/migrations/meta/0125_snapshot.json create mode 100644 tests/workflow-deploy/standby-restart-hub-record.test.ts create mode 100644 tests/workflow-deploy/standby-sleep-wake.test.ts create mode 100644 tests/workflow-deploy/standby-timed-gate-wake.test.ts create mode 100644 tests/workflow-deploy/standby-timer-wake-at.test.ts diff --git a/Makefile b/Makefile index e7db10512..1ff53cb77 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ test-unit: FORCE # (port 0, mkdtemp dirs, unique schemas) and none rely on fresh module state, # so sharing the registry across files is safe. test-workflow: FORCE - $(BUN) test --timeout 120000 --parallel=$(WF_PARALLEL) --no-isolate tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/hub-link-reconnect.test.ts tests/workflow-deploy/reconnect-reemits-parked-correlation.test.ts tests/workflow-deploy/interrupted-pack-reconnect-recovery.test.ts tests/workflow-deploy/deploy-window-reconnect-recovery.test.ts tests/workflow-deploy/midrun-signal-survives-reconnect.test.ts tests/workflow-deploy/multistep-perstep-reroute-reconnect.test.ts tests/workflow-deploy/crash-restart-reconnect-resumes.test.ts tests/workflow-deploy/standby-restart.test.ts tests/workflow-deploy/standby-restart-approval.test.ts tests/workflow-deploy/standby-approval-wake.test.ts tests/workflow-deploy/standby-mail-wakes-input-park.test.ts tests/workflow-deploy/standby-on-trigger-between-events.test.ts tests/workflow-deploy/standby-childworkflow-signal.test.ts tests/workflow-deploy/standby-signal-wake.test.ts tests/workflow-deploy/standby-cancel-wake.test.ts tests/workflow-deploy/standby-two-child-parks.test.ts tests/workflow-deploy/crash-respawn-fifo.test.ts tests/workflow-deploy/crash-loop-latch.test.ts tests/workflow-deploy/on-trigger-signal-delivery.test.ts tests/workflow-deploy/on-trigger-between-events-restart.test.ts tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts tests/workflow-deploy/sleep-restart-keeps-authority.test.ts tests/workflow-deploy/sleep-restart-hub-access.test.ts tests/workflow-deploy/on-trigger-agent-body.test.ts tests/workflow-deploy/on-trigger-action-body.test.ts tests/workflow-deploy/on-trigger-loop-body.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/map-fan-out-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-credential-tool.test.ts tests/workflow-deploy/credential-bound-route-deploy.test.ts tests/workflow-deploy/action-step-credential.test.ts tests/workflow-deploy/action-hub-access.test.ts tests/workflow-deploy/tool-hub-access.test.ts tests/workflow-deploy/invoker-hub-access.test.ts tests/workflow-deploy/body-hub-access.test.ts tests/workflow-deploy/revoked-creator-grant-running-run.test.ts tests/workflow-deploy/expired-run-grant-running-run.test.ts tests/workflow-deploy/tarball-route-deploy.test.ts tests/workflow-deploy/folded-tools-failover-real-agent.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-pinned-ask-tool.test.ts tests/workflow-deploy/single-step-per-run-grants.test.ts tests/workflow-deploy/approval-tracer-roundtrip.test.ts tests/workflow-deploy/scope-always-roundtrip.test.ts tests/workflow-deploy/childworkflow-scope-always-roundtrip.test.ts tests/workflow-deploy/mail-trigger-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-trigger-run-completes-real-route.test.ts tests/workflow-deploy/inbound-mail-enforcement-real-route.test.ts tests/workflow-deploy/run-mail-send-real-route.test.ts tests/workflow-deploy/mail-send-denied-real-route.test.ts tests/workflow-deploy/mail-federation-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-federation-run-completes.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/multistep-conversation-failover.test.ts tests/workflow-deploy/per-level-pipeline-real-agents.test.ts tests/workflow-deploy/dispatch-orchestrator-real-agents.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/single-step-mail-loop.test.ts tests/workflow-deploy/single-step-mail-in-reply-to.test.ts tests/workflow-deploy/single-step-mail-wait-wake.test.ts tests/workflow-deploy/single-step-mail-two-turn-thread.test.ts tests/workflow-deploy/single-step-mail-interactive-turns.test.ts tests/workflow-deploy/single-step-mail-flag-expunge.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/conversation-state-connector-seed.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/fixtures/child-workflow.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/child-workflow-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-inherited-grants-roundtrip.test.ts tests/workflow-deploy/on-trigger-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-roundtrip.test.ts tests/workflow-deploy/loop-nested-roundtrip.test.ts tests/workflow-deploy/loop-action-roundtrip.test.ts tests/workflow-deploy/all-action-deploy.test.ts tests/workflow-deploy/sidecar-runs-no-workflow-code.test.ts tests/workflow-deploy/onfailure-action-roundtrip.test.ts tests/workflow-deploy/loop-await-signal-restart-resumes.test.ts tests/workflow-deploy/loop-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-work-then-signal-restart-resumes.test.ts tests/workflow-deploy/loop-await-signal-drain-roundtrip.test.ts tests/workflow-deploy/loop-childworkflow-inflight-grandchild-drain.test.ts tests/workflow-deploy/childworkflow-restart-resumes.test.ts tests/workflow-deploy/body-action-restart-effect-once.test.ts tests/workflow-deploy/body-action-crash-mid-effect.test.ts tests/workflow-deploy/childworkflow-drain-wait-resumes.test.ts tests/workflow-deploy/childworkflow-unanswerable-gate.test.ts tests/workflow-deploy/on-trigger-drain-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/workflow-deploy/run-event-batching.test.ts tests/workflow-deploy/walking-skeleton.e2e.test.ts tests/workflow-deploy/asset-source.e2e.test.ts tests/workflow-deploy/asset-tool-restarts.e2e.test.ts tests/workflow-deploy/source-workflow.e2e.test.ts tests/workflow-deploy/source-monorepo.e2e.test.ts tests/workflow-deploy/source-monorepo-many.e2e.test.ts tests/workflow-deploy/source-tamper-reject.e2e.test.ts tests/workflow-deploy/source-catalog.e2e.test.ts tests/workflow-deploy/source-mixed-closure.e2e.test.ts tests/workflow-deploy/source-closure-director.e2e.test.ts tests/workflow-deploy/source-credential.e2e.test.ts tests/workflow-deploy/source-inline-tool.e2e.test.ts tests/workflow-deploy/source-posix-lsp.e2e.test.ts tests/workflow-deploy/loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-credential-tool.test.ts tests/workflow-deploy/loop-map-tool-invoke-roundtrip.test.ts tests/workflow-deploy/loop-nested-tool-invoke-roundtrip.test.ts + $(BUN) test --timeout 120000 --parallel=$(WF_PARALLEL) --no-isolate tests/workflow-deploy/multistep-signal.test.ts tests/workflow-deploy/hub-link-reconnect.test.ts tests/workflow-deploy/reconnect-reemits-parked-correlation.test.ts tests/workflow-deploy/interrupted-pack-reconnect-recovery.test.ts tests/workflow-deploy/deploy-window-reconnect-recovery.test.ts tests/workflow-deploy/midrun-signal-survives-reconnect.test.ts tests/workflow-deploy/multistep-perstep-reroute-reconnect.test.ts tests/workflow-deploy/crash-restart-reconnect-resumes.test.ts tests/workflow-deploy/standby-restart.test.ts tests/workflow-deploy/standby-restart-approval.test.ts tests/workflow-deploy/standby-approval-wake.test.ts tests/workflow-deploy/standby-restart-hub-record.test.ts tests/workflow-deploy/standby-mail-wakes-input-park.test.ts tests/workflow-deploy/standby-timed-gate-wake.test.ts tests/workflow-deploy/standby-sleep-wake.test.ts tests/workflow-deploy/standby-timer-wake-at.test.ts tests/workflow-deploy/standby-on-trigger-between-events.test.ts tests/workflow-deploy/standby-childworkflow-signal.test.ts tests/workflow-deploy/standby-signal-wake.test.ts tests/workflow-deploy/standby-cancel-wake.test.ts tests/workflow-deploy/standby-two-child-parks.test.ts tests/workflow-deploy/crash-respawn-fifo.test.ts tests/workflow-deploy/crash-loop-latch.test.ts tests/workflow-deploy/on-trigger-signal-delivery.test.ts tests/workflow-deploy/on-trigger-between-events-restart.test.ts tests/workflow-deploy/on-trigger-body-sleep-restart-resumes.test.ts tests/workflow-deploy/sleep-restart-keeps-authority.test.ts tests/workflow-deploy/sleep-restart-hub-access.test.ts tests/workflow-deploy/on-trigger-agent-body.test.ts tests/workflow-deploy/on-trigger-action-body.test.ts tests/workflow-deploy/on-trigger-loop-body.test.ts tests/workflow-deploy/single-step-real-agent.test.ts tests/workflow-deploy/map-fan-out-real-agent.test.ts tests/workflow-deploy/multistep-signed-send.test.ts tests/workflow-deploy/single-step-message-input.test.ts tests/workflow-deploy/single-step-posix-tool.test.ts tests/workflow-deploy/single-step-credential-tool.test.ts tests/workflow-deploy/credential-bound-route-deploy.test.ts tests/workflow-deploy/action-step-credential.test.ts tests/workflow-deploy/action-hub-access.test.ts tests/workflow-deploy/tool-hub-access.test.ts tests/workflow-deploy/invoker-hub-access.test.ts tests/workflow-deploy/body-hub-access.test.ts tests/workflow-deploy/revoked-creator-grant-running-run.test.ts tests/workflow-deploy/expired-run-grant-running-run.test.ts tests/workflow-deploy/tarball-route-deploy.test.ts tests/workflow-deploy/folded-tools-failover-real-agent.test.ts tests/workflow-deploy/single-step-grants-bridge.test.ts tests/workflow-deploy/single-step-pinned-ask-tool.test.ts tests/workflow-deploy/single-step-per-run-grants.test.ts tests/workflow-deploy/approval-tracer-roundtrip.test.ts tests/workflow-deploy/scope-always-roundtrip.test.ts tests/workflow-deploy/childworkflow-scope-always-roundtrip.test.ts tests/workflow-deploy/mail-trigger-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-trigger-run-completes-real-route.test.ts tests/workflow-deploy/inbound-mail-enforcement-real-route.test.ts tests/workflow-deploy/run-mail-send-real-route.test.ts tests/workflow-deploy/mail-send-denied-real-route.test.ts tests/workflow-deploy/mail-federation-derived-grants-roundtrip.test.ts tests/workflow-deploy/mail-federation-run-completes.test.ts tests/workflow-deploy/single-step-event-threading.test.ts tests/workflow-deploy/single-step-conversation-durability.test.ts tests/workflow-deploy/multistep-conversation-failover.test.ts tests/workflow-deploy/per-level-pipeline-real-agents.test.ts tests/workflow-deploy/dispatch-orchestrator-real-agents.test.ts tests/workflow-deploy/single-step-full-lifecycle.test.ts tests/workflow-deploy/single-step-mail-loop.test.ts tests/workflow-deploy/single-step-mail-in-reply-to.test.ts tests/workflow-deploy/single-step-mail-wait-wake.test.ts tests/workflow-deploy/single-step-mail-two-turn-thread.test.ts tests/workflow-deploy/single-step-mail-interactive-turns.test.ts tests/workflow-deploy/single-step-mail-flag-expunge.test.ts tests/workflow-deploy/cross-process-custom-adapter.test.ts tests/workflow-deploy/conversation-state-wal.test.ts tests/workflow-deploy/conversation-state-connector-seed.test.ts tests/workflow-deploy/drain-roundtrip.test.ts tests/workflow-deploy/fixtures/child-workflow.test.ts tests/workflow-deploy/child-workflow-roundtrip.test.ts tests/workflow-deploy/child-workflow-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-inherited-grants-roundtrip.test.ts tests/workflow-deploy/on-trigger-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-roundtrip.test.ts tests/workflow-deploy/loop-nested-roundtrip.test.ts tests/workflow-deploy/loop-action-roundtrip.test.ts tests/workflow-deploy/all-action-deploy.test.ts tests/workflow-deploy/sidecar-runs-no-workflow-code.test.ts tests/workflow-deploy/onfailure-action-roundtrip.test.ts tests/workflow-deploy/loop-await-signal-restart-resumes.test.ts tests/workflow-deploy/loop-childworkflow-roundtrip.test.ts tests/workflow-deploy/loop-work-then-signal-restart-resumes.test.ts tests/workflow-deploy/loop-await-signal-drain-roundtrip.test.ts tests/workflow-deploy/loop-childworkflow-inflight-grandchild-drain.test.ts tests/workflow-deploy/childworkflow-restart-resumes.test.ts tests/workflow-deploy/body-action-restart-effect-once.test.ts tests/workflow-deploy/body-action-crash-mid-effect.test.ts tests/workflow-deploy/childworkflow-drain-wait-resumes.test.ts tests/workflow-deploy/childworkflow-unanswerable-gate.test.ts tests/workflow-deploy/on-trigger-drain-roundtrip.test.ts tests/workflow-deploy/unresolvable-director.test.ts tests/workflow-deploy/fifo-mail.test.ts tests/workflow-deploy/mail-edge-cases.test.ts tests/workflow-deploy/run-event-batching.test.ts tests/workflow-deploy/walking-skeleton.e2e.test.ts tests/workflow-deploy/asset-source.e2e.test.ts tests/workflow-deploy/asset-tool-restarts.e2e.test.ts tests/workflow-deploy/source-workflow.e2e.test.ts tests/workflow-deploy/source-monorepo.e2e.test.ts tests/workflow-deploy/source-monorepo-many.e2e.test.ts tests/workflow-deploy/source-tamper-reject.e2e.test.ts tests/workflow-deploy/source-catalog.e2e.test.ts tests/workflow-deploy/source-mixed-closure.e2e.test.ts tests/workflow-deploy/source-closure-director.e2e.test.ts tests/workflow-deploy/source-credential.e2e.test.ts tests/workflow-deploy/source-inline-tool.e2e.test.ts tests/workflow-deploy/source-posix-lsp.e2e.test.ts tests/workflow-deploy/loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/child-workflow-loop-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-tool-invoke-roundtrip.test.ts tests/workflow-deploy/on-trigger-credential-tool.test.ts tests/workflow-deploy/loop-map-tool-invoke-roundtrip.test.ts tests/workflow-deploy/loop-nested-tool-invoke-roundtrip.test.ts # The core integration pass: inference, hub-api, and db suites that need # real Postgres and the extended timeout. Files run in parallel diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index b000c330c..3f724d537 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -478,6 +478,7 @@ export async function createHubServer({ historyReceives: workflowHistoryReceives, sendControl: (target, command, timeoutMs) => sidecarRouter.sendWorkflowControl(target, command, timeoutMs), + standbyRouter: sidecarRouter, }); const dispatchProjection = createWorkflowDispatchProjection({ db, @@ -555,6 +556,9 @@ export async function createHubServer({ sidecarRouter.events.on("deployment.stopped", (allocated) => sidecarAllocationReconciler.handleConnected(allocated), ); + sidecarRouter.events.on("deployment.standby", (report) => + workflowLifecycleService.noteDeploymentInStandby(report), + ); sidecarRouter.events.on( "mail.inbound.acknowledged", ({ messageId, allocated }) => { diff --git a/docs/API.md b/docs/API.md index 6924ab93b..0f8417b81 100644 --- a/docs/API.md +++ b/docs/API.md @@ -46,6 +46,8 @@ | GET | /api/tenants/:tenantId/workflows/runs/:runId/lifecycle | Get workflow lifecycle status | | POST | /api/tenants/:tenantId/workflows/runs/:runId/capacity/release | Release workflow capacity | | POST | /api/tenants/:tenantId/workflows/runs/:runId/capacity/recover | Release capacity whose cleanup stopped | +| POST | /api/tenants/:tenantId/workflows/runs/:runId/standby | Put a deployment in standby | +| POST | /api/tenants/:tenantId/workflows/runs/:runId/wake | Wake a deployment | | GET | /api/tenants/:tenantId/workflows/runs/:runId/events | Read a run's event log | | POST | /api/tenants/:tenantId/workflows/runs/:runId/mail | Send mail to a run | | GET | /api/tenants/:tenantId/workflows/runs/:runId/mail | List mail for a run | @@ -537,7 +539,7 @@ Returns the offerings associated with the run's workflow definition. These repre ### GET /api/tenants/:tenantId/workflows/runs/:runId/lifecycle Get workflow lifecycle status -Returns the deployment's saved policy, deadlines, and allocation cleanup status. Policy edits apply to new deployments. +Returns the deployment's saved policy, deadlines, standby state, and allocation cleanup status. Policy edits apply to new deployments. 200: WorkflowLifecycleResponse -- Lifecycle status 404: ErrorResponse -- Run not found @@ -566,6 +568,28 @@ Body: RecoverWorkflowCapacityRequest 409: ErrorResponse -- Cleanup has not stopped, or the allocation generation changed 503: ErrorResponse -- Workflow lifecycle service unavailable +### POST /api/tenants/:tenantId/workflows/runs/:runId/standby +Put a deployment in standby + +Durably requests that a live top-level run's workflow child stop once every run of the deployment only waits on input from outside. Work in progress goes on until then, and the deployment keeps its capacity. Mail, a signal, an approval decision, or a due timer runs the child again, as does `POST .../wake`. Poll the Location header for progress; a standby request the sidecar refuses returns to `awake` with its reason. + +202: (no content) -- Standby requested +204: (no content) -- The deployment is already in standby +404: ErrorResponse -- Run not found +409: ErrorResponse -- Run is terminal, being cancelled, or its allocation can no longer be dispatched to a sidecar +503: ErrorResponse -- Workflow lifecycle service unavailable + +### POST /api/tenants/:tenantId/workflows/runs/:runId/wake +Wake a deployment + +Durably requests that the workflow child of a deployment in standby run again, or that a standby request in progress be abandoned. Poll the Location header for progress. + +202: (no content) -- Wake requested +204: (no content) -- The deployment is not in standby +404: ErrorResponse -- Run not found +409: ErrorResponse -- Run is terminal, being cancelled, or its allocation can no longer be dispatched to a sidecar +503: ErrorResponse -- Workflow lifecycle service unavailable + ### GET /api/tenants/:tenantId/workflows/runs/:runId/events Read a run's event log @@ -1740,9 +1764,11 @@ Source: packages/types/src/workflows.ts **status**: Deployment lifecycle status. `recovering` means the Hub is replacing deployment capacity. `failed` is a terminal allocation failure with no infrastructure. `destroy_failed` means automatic cleanup stopped. Read `GET /api/tenants/:tenantId/workflows/runs/:runId/lifecycle` for the allocation's failure code and message. ### WorkflowLifecycleResponse -`{ allocation: { failureCode: string | null, failureMessage: string | null, generation: number, id: string, status: "allocated" | "destroy_failed" | "failed" | "pending" | "provisioning" | "released" | "releasing" | "replacing" } | null, cancellationDeadline: string | null, cancellationReason: string | null, cancellationRequestedAt: string | null, capacityReleaseAt: string | null, expiresAt: string | null, policy: { capacityRetention?: { cancelled?: string , completed?: string , failed?: string , + (undeclared): reject }, maxDisconnected?: string , maxLifetime?: string , + (undeclared): reject }, runId: string, status: "cancelled" | "completed" | "deployed" | "failed" | "running" }` +`{ allocation: { failureCode: string | null, failureMessage: string | null, generation: number, id: string, status: "allocated" | "destroy_failed" | "failed" | "pending" | "provisioning" | "released" | "releasing" | "replacing" } | null, cancellationDeadline: string | null, cancellationReason: string | null, cancellationRequestedAt: string | null, capacityReleaseAt: string | null, expiresAt: string | null, policy: { capacityRetention?: { cancelled?: string , completed?: string , failed?: string , + (undeclared): reject }, maxDisconnected?: string , maxLifetime?: string , + (undeclared): reject }, runId: string, standby: { changedAt: string | null, reason: string | null, state: "awake" | "entering" | "standby" | "waking", wakeAt: string | null } | null, status: "cancelled" | "completed" | "deployed" | "failed" | "running" }` Source: packages/types/src/workflow-lifecycle.ts +**standby**: Null once the run is terminal. `state` is `awake` while the workflow child runs and no standby is asked for, `entering` until the sidecar stops the child, `standby` once the Hub holds the history it left, and `waking` until the child runs again. `wakeAt` is the earliest timer a child in standby wakes for by itself. `reason` says why the state last changed other than as requested: the sidecar refused a standby or wake request, something other than a wake request woke the child in standby, such as mail or a restart that found new history, the sidecar reported the child running, it reported the child in standby though no standby was asked for, a cancellation ended the standby, or the deployment moved to replacement capacity. While a request is pending it says why the Hub could not carry it out yet. + ### WorkflowRollbackRequest `{ version: string }` Source: packages/types/src/workflows.ts diff --git a/docs/HARNESS_DESIGN.md b/docs/HARNESS_DESIGN.md index 4cfcbe93f..08f1506bb 100644 --- a/docs/HARNESS_DESIGN.md +++ b/docs/HARNESS_DESIGN.md @@ -177,7 +177,7 @@ The Hub mints a sidecar identity for each probe or allocation generation and han Possession of the raw token is sufficient to authenticate as that sidecar, for every probe and allocation it currently hosts, including cleanup bindings. Provisioners and workers must never log it. A provisioner that must persist the token for restartable capacity must use access-controlled secret storage and keep it with any worker state that can be restarted, deleting it only when the worker and its restorable state are permanently removed. Connections crossing a non-loopback or otherwise untrusted transport must use `wss://`; plaintext `ws://` is only appropriate for local loopback development. -Every connection opens with a `hello` frame reporting each incarnation the worker holds and what it is doing with it: `deploying`, `live`, `stopped`, or `tearing-down`. A `live` one whose workflow child is in standby also carries `inStandby`: the worker stopped the child while every run waited and respawns it for its next input, and it reports such a wake in a `deployment.woken` frame. A deployment in standby the worker restores stays in standby without a child while its history is where the stopped child left it, and is reported so. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop, because its workflow child ended itself or the worker could not restore it, carries the `error` that ended it, and the Hub fails its deployment. The worker also reports such a stop in a `deployment.stopped` frame when it happens and after each `welcome` until retention is decided, with the tips of the deployment's history branches, and the Hub fails the deployment once it holds that history, or after a bounded wait. The worker restores its deployments from storage before it first connects, so that `hello` already lists them. The Hub accepts the frame when the token resolves to current work or a cleanup obligation (a release, a replacement, or a failed cleanup); a cleanup-only connection receives no routes. It routes a reported incarnation only when it is live, its generation is the current generation of an allocation the sidecar hosts, that generation has finished initializing, and its run has not ended. It keeps current stopped copies and initialized terminal copies unrouted. After welcome it stops terminal live copies, requesting retention only while their policy allows it. Durable kept copies use the retained pool. Cleanup bindings leave removal to allocation reconciliation; the Hub sends `agent.undeploy` for other unwanted incarnations instead of closing the socket, so one stale deployment cannot disconnect the sidecar's other work, and then answers `welcome` with the incarnations it routes. +Every connection opens with a `hello` frame reporting each incarnation the worker holds and what it is doing with it: `deploying`, `live`, `stopped`, or `tearing-down`. A `live` one whose workflow child is in standby also carries `inStandby`: the worker stopped the child while every run waited and respawns it for its next input, and it reports such a wake in a `deployment.woken` frame. A deployment in standby the worker restores stays in standby without a child while its history is where the stopped child left it, and is reported so; the Hub wakes it when no standby is asked for. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop, because its workflow child ended itself or the worker could not restore it, carries the `error` that ended it, and the Hub fails its deployment. The worker also reports such a stop in a `deployment.stopped` frame when it happens and after each `welcome` until retention is decided, with the tips of the deployment's history branches, and the Hub fails the deployment once it holds that history, or after a bounded wait. The worker restores its deployments from storage before it first connects, so that `hello` already lists them. The Hub accepts the frame when the token resolves to current work or a cleanup obligation (a release, a replacement, or a failed cleanup); a cleanup-only connection receives no routes. It routes a reported incarnation only when it is live, its generation is the current generation of an allocation the sidecar hosts, that generation has finished initializing, and its run has not ended. It keeps current stopped copies and initialized terminal copies unrouted. After welcome it stops terminal live copies, requesting retention only while their policy allows it. Durable kept copies use the retained pool. Cleanup bindings leave removal to allocation reconciliation; the Hub sends `agent.undeploy` for other unwanted incarnations instead of closing the socket, so one stale deployment cannot disconnect the sidecar's other work, and then answers `welcome` with the incarnations it routes. The sidecar defaults to 128 active reservations and 256 kept records, configured at startup with `SIDECAR_MAX_ACTIVE_DEPLOYMENTS` and `SIDECAR_MAX_RETAINED_DEPLOYMENTS`. It advertises those limits and reports both pools in full; the Hub uses the advertised active limit for placement. The combined protocol ceiling is 2,048 records, and the complete `hello` must fit the 64 MiB WebSocket frame-size limit. Hello errors use the same 4,096-character limit as individual stop reports. Restoration exceeding either configured limit fails startup; it does not omit records from `hello`. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index cb263006a..10652e4e0 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -282,7 +282,7 @@ Undeploy is an acknowledged operation. The sidecar shuts the named generation's The live `grants.update` wire path has been retired. The hub sends the run's committed grants in a `run.grants` frame ahead of each signal delivery and each trigger, and to each deployment that it routes again when a sidecar reconnects, so a standing approval decision reaches a running deployment. The sidecar keeps them in the run's grants file, and the supervisor passes them to the child over its IPC credentials snapshot at spawn and recycle. When a change to a delegator's grants re-derives a run's rows, the hub sends the run its grants on their own after the change commits (see AUTH.md, _Grant Revocation_). -Every connection opens with a `hello` that reports each incarnation the sidecar holds and whether it is `deploying`, `live`, `stopped`, or `tearing-down`. A `live` incarnation whose workflow child is in standby also carries `inStandby`. A `workflow.control` with `action: standby` stops the child once every run waits and answers with the history tips read after it stopped and the earliest pending timer, and is refused while mail waits for the child, in the deployment's inbox or taken but not committed; the Hub confirms the standby request once it holds those tips, and the incarnation keeps its route. The sidecar abandons the standby request if the connection that asked for it closes before the standby attempt settles, and runs the child again if the attempt already stopped it. It records the standby in the run record before anything can wake the child: the correlations the child was parked on, its earliest timer, the tip of the run's history read before the child reported them, and the definition the child verified. A restart that finds that tip unchanged, and that definition still hashing to the approved one, restores the standby without the child and reports it in standby; one that finds history written since, by a child that ran again or a cancellation committed while it was stopped, or cannot read the tip, starts the child, clears the record and sends `deployment.woken`. Every answer, a repeated one too, reads the tips again, since the supervisor can write history while the child is stopped, and pushes what the Hub still lacks of it; a child that ran again before the answer refuses the standby request instead. Mail, a signal, a cancellation, or the timer wakes a child in standby; the sidecar clears the mark and sends `deployment.woken`, held until the next `welcome` if the link is down. `action: wake` abandons a standby request in progress or wakes the child. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop carries the error that ended it: its workflow child ended itself, or the sidecar could not restore it. The Hub then fails the deployment's live runs with that error and keeps its capacity for the retention its policy sets for failed runs, so the stopped copy can be inspected. The sidecar also marks either state in the run record, so a restart reports the incarnation instead of spawning it, and reports such a stop in a `deployment.stopped` frame once that write settles, and again after each `welcome` until retention is durably decided, with the tips of the deployment's history branches. It does not wait for its pushes of that history: a push that fails is retried on every reconnect, so that wait would have no bound. The Hub fails the deployment once it holds those tips, or a minute after it first heard of the stop, and the failure then names the history it never received. A reconnect does not restart that minute, and a sidecar that disconnects instead and stays away past its disconnect limit fails the deployment with the reason it reported. The Hub holds a reported stop only in memory until it fails the deployment: a Hub restart starts that minute again from the sidecar's next report, and if the sidecar does not come back, the deployment fails without that reason. The Hub accepts the frame when the bearer token resolves to a sidecar that is not retired or still hosts a probe, allocation generation, or cleanup binding. It restores a reported incarnation's route when the incarnation is live, its generation is the current, initialized generation of a routable allocation the sidecar hosts, and its run has not ended, keeps a stopped one of the current generation of an allocation the sidecar hosts unrouted, initialized or not, and an initialized live one whose run is terminal until lifecycle retention stops it, leaves incarnations covered by cleanup bindings to the reconciler, and sends `agent.undeploy` naming the generation of every other reported incarnation rather than rejecting the connection, so one stale deployment cannot disconnect the others. It then answers `welcome` listing the incarnations it routes, as soon as routing is settled. A hello is never left unanswered: the Hub closes the socket when the hello is invalid or its registration fails before the welcome, and the sidecar reconnects when no `welcome` arrives within 30 seconds (`welcomeTimeoutMs`). A failure after the welcome is logged, since registering again would meet it again. A sidecar whose token is valid but that hosts neither current work nor cleanup obligations is welcomed with nothing routed, after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, for its provisioner to reuse. Until `welcome` the sidecar sends nothing the Hub must receive: it queues outbound mail and signal correlation registrations, keeps the latest wake report of each deployment, drops best-effort events, and holds back the workflow-run pack pushes of every incarnation it holds; a push no connection could carry stays due instead of failing the deployment's next write. The queue holds 1024 frames for all the sidecar's deployments together: an outage that outlasts it drops queued correlation registrations first, which the next `welcome` re-sends, then the audit copies the Hub records sent mail from, and then fails a deployment's send rather than drop mail it has not yet routed. On `welcome` it sends the wake reports and then the queue, re-drives its routed incarnations' pending packs and parked correlations, and reports stopped deployments whose retention is undecided. +Every connection opens with a `hello` that reports each incarnation the sidecar holds and whether it is `deploying`, `live`, `stopped`, or `tearing-down`. A `live` incarnation whose workflow child is in standby also carries `inStandby`. A `workflow.control` with `action: standby` stops the child once every run waits and answers with the history tips read after it stopped and the earliest pending timer, and is refused while mail waits for the child, in the deployment's inbox or taken but not committed; the Hub confirms the standby request once it holds those tips, and the incarnation keeps its route. The sidecar abandons the standby request if the connection that asked for it closes before the standby attempt settles, and runs the child again if the attempt already stopped it. It records the standby in the run record before anything can wake the child: the correlations the child was parked on, its earliest timer, the tip of the run's history read before the child reported them, and the definition the child verified. A restart that finds that tip unchanged, and that definition still hashing to the approved one, restores the standby without the child and reports it in standby; one that finds history written since, by a child that ran again or a cancellation committed while it was stopped, or cannot read the tip, starts the child, clears the record and sends `deployment.woken`. Every answer, a repeated one too, reads the tips again, since the supervisor can write history while the child is stopped, and pushes what the Hub still lacks of it; a child that ran again before the answer refuses the standby request instead. Mail, a signal, a cancellation, or the timer wakes a child in standby; the sidecar clears the mark and sends `deployment.woken`, held until the next `welcome` if the link is down. The Hub takes up that report and the answers to its standby and wake commands in the order they arrive on the connection, so the report never ends a standby request the sidecar answered after it. `action: wake` abandons a standby request in progress or wakes the child. A `stopped` incarnation no longer runs and keeps its local state for inspection until the Hub undeploys it. One the Hub did not stop carries the error that ended it: its workflow child ended itself, or the sidecar could not restore it. The Hub then fails the deployment's live runs with that error and keeps its capacity for the retention its policy sets for failed runs, so the stopped copy can be inspected. The sidecar also marks either state in the run record, so a restart reports the incarnation instead of spawning it, and reports such a stop in a `deployment.stopped` frame once that write settles, and again after each `welcome` until retention is durably decided, with the tips of the deployment's history branches. It does not wait for its pushes of that history: a push that fails is retried on every reconnect, so that wait would have no bound. The Hub fails the deployment once it holds those tips, or a minute after it first heard of the stop, and the failure then names the history it never received. A reconnect does not restart that minute, and a sidecar that disconnects instead and stays away past its disconnect limit fails the deployment with the reason it reported. The Hub holds a reported stop only in memory until it fails the deployment: a Hub restart starts that minute again from the sidecar's next report, and if the sidecar does not come back, the deployment fails without that reason. The Hub accepts the frame when the bearer token resolves to a sidecar that is not retired or still hosts a probe, allocation generation, or cleanup binding. It restores a reported incarnation's route when the incarnation is live, its generation is the current, initialized generation of a routable allocation the sidecar hosts, and its run has not ended, keeps a stopped one of the current generation of an allocation the sidecar hosts unrouted, initialized or not, and an initialized live one whose run is terminal until lifecycle retention stops it, leaves incarnations covered by cleanup bindings to the reconciler, and sends `agent.undeploy` naming the generation of every other reported incarnation rather than rejecting the connection, so one stale deployment cannot disconnect the others. It then answers `welcome` listing the incarnations it routes, as soon as routing is settled. A hello is never left unanswered: the Hub closes the socket when the hello is invalid or its registration fails before the welcome, and the sidecar reconnects when no `welcome` arrives within 30 seconds (`welcomeTimeoutMs`). A failure after the welcome is logged, since registering again would meet it again. A sidecar whose token is valid but that hosts neither current work nor cleanup obligations is welcomed with nothing routed, after the Hub sends `agent.undeploy` for everything it reports, and stays connected, idle, for its provisioner to reuse. Until `welcome` the sidecar sends nothing the Hub must receive: it queues outbound mail and signal correlation registrations, keeps the latest wake report of each deployment, drops best-effort events, and holds back the workflow-run pack pushes of every incarnation it holds; a push no connection could carry stays due instead of failing the deployment's next write. The queue holds 1024 frames for all the sidecar's deployments together: an outage that outlasts it drops queued correlation registrations first, which the next `welcome` re-sends, then the audit copies the Hub records sent mail from, and then fails a deployment's send rather than drop mail it has not yet routed. On `welcome` it sends the wake reports and then the queue, re-drives its routed incarnations' pending packs and parked correlations, and reports stopped deployments whose retention is undecided. ### Debug and Telemetry Streams diff --git a/docs/SIDECAR_PLACEMENT.md b/docs/SIDECAR_PLACEMENT.md index 34a64d3cf..db38dcfaa 100644 --- a/docs/SIDECAR_PLACEMENT.md +++ b/docs/SIDECAR_PLACEMENT.md @@ -139,8 +139,10 @@ elsewhere. A provisioner that knows the work will not come back on that sidecar passes `endReconnectWindow`. The Hub records that on the sidecar and ends the reconnect window of its disconnected work at once, handing it to the usual connection loss path, which today replaces a parked run and fails an -active one. Work that disconnects later, and work a Hub restart would -otherwise give a fresh window, gets no window either. Work still connected to +active one. Standby alone does not make a deployment replaceable: +one in standby whose run sleeps on a timer, or that has no run yet, fails too. +Work that disconnects later, and work a Hub restart would otherwise give a +fresh window, gets no window either. Work still connected to the sidecar contradicts the report; the Hub logs it and leaves that work running until it disconnects. `endReconnectWindow` covers allocations only, since a probe needs no help: one still waiting for its sidecar fails when its diff --git a/docs/workflow-lifecycle-policy.md b/docs/workflow-lifecycle-policy.md index ba9c3c2fa..b6200a22d 100644 --- a/docs/workflow-lifecycle-policy.md +++ b/docs/workflow-lifecycle-policy.md @@ -102,8 +102,8 @@ The platform default is not a ceiling: a tenant or installed workflow may set a longer duration. To effectively disable an action, set its field to `36500d`. `maxLifetime` measures wall-clock time from creation of the deployment's anchor -run. Provisioning and waiting count; restarts and any future hibernation do -not reset the clock. This also covers deployments that +run. Provisioning, waiting, and standby count; restarts do not reset the +clock. This also covers deployments that never receive their first trigger. If still live at expiry, the Hub stops the deployment as `cancelled`, allowing a 30-second cancellation grace period before stopping the process. Forced stop preempts a pending cooperative cancellation. The sidecar acknowledges @@ -251,7 +251,9 @@ A missed sidecar connect deadline is a different decision from that capacity-los record. When the committed run log is one the workflow runtime would continue, and it contains a park the runtime would re-attach, the Hub replaces the worker and restores the run onto it. Otherwise it releases the allocation and logs the -anchor, the allocation, and the reason. Sleep with no signal park, a park that +anchor, the allocation, and the reason. A deployment in standby is judged the +same way, so one whose run sleeps on a timer, or that has no run yet, is +released. Sleep with no signal park, a park that was not pushed to the committed ref, and an approval that was never flushed to the log stay on the release path: the first has no re-attachable park, and the other two are not visible on the committed ref. A replacement that dies while @@ -267,10 +269,11 @@ cleanup binding without workflow routes. Lifecycle reconciliation uses the shared reconciliation scheduler with eight independent slots per Hub. Each slot selects one run at a time, and active runs are excluded from further selection until their work finishes. Selection -considers only deployments with work due: a live run that is cancelling or -past its expiry, terminal copies awaiting retention, retained capacity whose -release time has passed or is not yet recorded, a capacity loss waiting on -reconciliation, and accepted history left unprojected past its grace. A +considers only deployments with work due: a live run that is cancelling, past +its expiry, or waiting on a standby or wake request, terminal copies awaiting +retention, retained capacity whose release time has passed or is not yet +recorded, a capacity loss waiting on reconciliation, and accepted history left +unprojected past its grace. A deployment that ended without a saved policy is released only on request. Selection walks run IDs up to a captured upper bound, then waits one second before another scan. Work that becomes due during a scan can wait for the next @@ -356,6 +359,63 @@ recorded yet, DELETE therefore returns `202`. The Hub records the run's own outcome when it recovers that history, and it never records `cancelled` over an accepted terminal event. +`POST /api/tenants/:tenantId/workflows/runs/:runId/standby` puts a +live deployment in standby: its sidecar stops the workflow child once every run +only waits on input from outside, a signal, an approval, a timer, or mail, and +the deployment keeps its capacity. Work in progress goes on until then, +including input that just arrived, and a map or a step with a timeout counts as +work until it ends, since a fresh child could not continue the map or keep the +step's deadline. So does a step waiting to retry, since a fresh child would not +continue that wait. A gate's timeout does not count, since it is a timer in the +run's log. The sidecar refuses the standby request when the run has ended. The Hub records `entering`, and the lifecycle sweep sends +the request while the Hub holds the connection of the sidecar the deployment +runs on: the sidecar answers once every run waits, however long that takes, and +abandons the standby request if that connection closes before the attempt +settles, running the child again if it already stopped it; either way the +sweep asks again on the next one. A sidecar restarting the child +puts the request off, and the sweep asks again. The sidecar marks the child in standby before anything +can wake it, and answers with the history tips it reads then, refusing +when the child ran again first. The Hub records `standby` only once it holds +that history, together with the earliest timer the child wakes for. Until it +does, the sweep keeps asking, however long that takes: history that never +arrives means the Hub or the sidecar is broken, which giving up on the request +cannot fix. Mail, a signal, an approval decision, a due timer, or +cancellation runs the child again, and the sidecar reports why. Standby keeps +the deployment on its sidecar, so the sidecar wakes the child itself and the Hub +only checks that it holds the history. Waking through the Hub and checking the +history there matter only for resuming a deployment elsewhere, which remains +separate work. `POST .../wake` +records `waking`, and the sweep sends the wake until the sidecar answers; it +runs the child again or abandons a standby request in progress. An answer +applies only to the request it carried out, and the Hub takes up the sidecar's +reports and answers in the order the sidecar sent them, so a report never ends a +later standby. A child in standby that its sidecar reports though no standby is +asked for is recorded `waking` and woken the same way, and a `hello` finding +the child running ends a recorded standby but leaves a pending request to +its command. The lifecycle status's `standby` field shows the state, when +it last changed, the earliest timer a child in standby wakes for by itself +(`wakeAt`), and the reason for a refused standby or wake request, for an +input that woke the child, or for a cancellation or replacement capacity that +ended the standby. A command that gets no answer goes out again after a +wait that doubles up to 30 seconds, and the reason says why, unless all it waits +for is history still on its way. The sidecar refuses a standby request while mail +waits for the child: mail in the deployment's inbox would wake the child at +once, and nothing would wake it for mail taken but not yet committed. A +child in standby stays so across a sidecar restart: the sidecar restores it +without starting the child while the run's history is where the stopped child +left it and the recorded definition still hashes to the approved one, and +reports it in standby. History written since starts the child, +which the sidecar reports as a wake. Replacement capacity starts awake: a deployment in standby or entering it is put in standby again +there once it is deployed and every run waits, and a pending wake has nothing +left to do. +Cancellation and expiry end standby. Both routes need `manage` on the +run and return `202` while the change is pending, `204` when the deployment is +already in the requested state, and `409` for a terminal or cancelling run, +or one whose allocation no sidecar can carry the request out on. Like DELETE, +a request takes only the anchor run's row lock and reads no history, so it +does not wait for a pack receive; one for a run whose accepted history has +ended returns `202`, and the sweep records the run's own outcome instead. + Beginning release removes workflow routes. The allocation reaches `released` only after the provider's hold is released and deployment cleanup is confirmed. Confirmed removal frees the active sidecar slot even if the @@ -377,5 +437,6 @@ from connected sidecars independently of provider availability and releases the provider hold through `SidecarProvisioner.destroy()`. Either an undeploy acknowledgement or confirmed provider destruction proves removal. These duration rules do not need a separate -workflow-aware reaping plugin. Hibernating and resuming a live run remains -separate work, especially when it depends on files held only on the sidecar. +workflow-aware reaping plugin. Releasing the capacity of a deployment in +standby and resuming it elsewhere remains separate work, especially when it +depends on files held only on the sidecar. diff --git a/packages/db/migrations/0125_workflow_run_standby.sql b/packages/db/migrations/0125_workflow_run_standby.sql new file mode 100644 index 000000000..f7e8e7863 --- /dev/null +++ b/packages/db/migrations/0125_workflow_run_standby.sql @@ -0,0 +1,4 @@ +ALTER TABLE "workflow_run" ADD COLUMN "standby_state" text;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "standby_state_at" timestamp;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "standby_wake_at" timestamp;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "standby_reason" text; \ No newline at end of file diff --git a/packages/db/migrations/meta/0125_snapshot.json b/packages/db/migrations/meta/0125_snapshot.json new file mode 100644 index 000000000..57e8b0b86 --- /dev/null +++ b/packages/db/migrations/meta/0125_snapshot.json @@ -0,0 +1,5019 @@ +{ + "id": "a07e016c-cfb2-4f20-b75b-15de4835fd71", + "prevId": "23dabe89-c44b-494c-b2a4-b60d4ba3221d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.federation_trust": { + "name": "federation_trust", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_tenant_id": { + "name": "target_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "federation_trust_tenant_id_tenant_id_fk": { + "name": "federation_trust_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "federation_trust_target_tenant_id_tenant_id_fk": { + "name": "federation_trust_target_tenant_id_tenant_id_fk", + "tableFrom": "federation_trust", + "tableTo": "tenant", + "columnsFrom": ["target_tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "federation_trust_tenant_id_target_tenant_id_unique": { + "name": "federation_trust_tenant_id_target_tenant_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "target_tenant_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tenant": { + "name": "tenant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tenant_domain_lower_idx": { + "name": "tenant_domain_lower_idx", + "columns": [ + { + "expression": "lower(\"domain\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tenant_parent_id_tenant_id_fk": { + "name": "tenant_parent_id_tenant_id_fk", + "tableFrom": "tenant", + "tableTo": "tenant", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tenant_slug_unique": { + "name": "tenant_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": { + "tenant_slug_dns_label_check": { + "name": "tenant_slug_dns_label_check", + "value": "\"tenant\".\"slug\" ~ '^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$' and length(\"tenant\".\"slug\") <= 63" + } + }, + "isRLSEnabled": false + }, + "public.principal": { + "name": "principal", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_tenant_id_tenant_id_fk": { + "name": "principal_tenant_id_tenant_id_fk", + "tableFrom": "principal", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_tenant_id_kind_ref_id_unique": { + "name": "principal_tenant_id_kind_ref_id_unique", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "ref_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_key": { + "name": "principal_key", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "principal_key_one_active": { + "name": "principal_key_one_active", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"principal_key\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "principal_key_principal_id_principal_id_fk": { + "name": "principal_key_principal_id_principal_id_fk", + "tableFrom": "principal_key", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "principal_key_public_key_unique": { + "name": "principal_key_public_key_unique", + "nullsNotDistinct": false, + "columns": ["public_key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_role": { + "name": "agent_role", + "schema": "", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_role_agent_id_workflow_definition_id_fk": { + "name": "agent_role_agent_id_workflow_definition_id_fk", + "tableFrom": "agent_role", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_role_role_id_role_id_fk": { + "name": "agent_role_role_id_role_id_fk", + "tableFrom": "agent_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_role_agent_id_role_id_pk": { + "name": "agent_role_agent_id_role_id_pk", + "columns": ["agent_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.principal_role": { + "name": "principal_role", + "schema": "", + "columns": { + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "principal_role_principal_id_principal_id_fk": { + "name": "principal_role_principal_id_principal_id_fk", + "tableFrom": "principal_role", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "principal_role_role_id_role_id_fk": { + "name": "principal_role_role_id_role_id_fk", + "tableFrom": "principal_role", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "principal_role_principal_id_role_id_pk": { + "name": "principal_role_principal_id_role_id_pk", + "columns": ["principal_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.role": { + "name": "role", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_system": { + "name": "is_system", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "role_tenant_id_tenant_id_fk": { + "name": "role_tenant_id_tenant_id_fk", + "tableFrom": "role", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.grant": { + "name": "grant", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "effect": { + "name": "effect", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "grant_invoker_allow_principal_idx": { + "name": "grant_invoker_allow_principal_idx", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"grant\".\"origin\" = 'invoker' and \"grant\".\"effect\" = 'allow'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "grant_tenant_id_tenant_id_fk": { + "name": "grant_tenant_id_tenant_id_fk", + "tableFrom": "grant", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_role_id_role_id_fk": { + "name": "grant_role_id_role_id_fk", + "tableFrom": "grant", + "tableTo": "role", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "grant_principal_id_principal_id_fk": { + "name": "grant_principal_id_principal_id_fk", + "tableFrom": "grant", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "grant_target_exactly_one": { + "name": "grant_target_exactly_one", + "value": "num_nonnulls(\"grant\".\"principal_id\", \"grant\".\"role_id\") = 1" + } + }, + "isRLSEnabled": false + }, + "public.approval": { + "name": "approval", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_address": { + "name": "agent_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_definition": { + "name": "tool_definition", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "tool_arguments": { + "name": "tool_arguments", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "timeout_at": { + "name": "timeout_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "approval_tenant_status_idx": { + "name": "approval_tenant_status_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_anchor_run_idx": { + "name": "approval_anchor_run_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_run_idx": { + "name": "approval_run_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approval_tenant_id_tenant_id_fk": { + "name": "approval_tenant_id_tenant_id_fk", + "tableFrom": "approval", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_anchor_run_id_workflow_run_id_fk": { + "name": "approval_anchor_run_id_workflow_run_id_fk", + "tableFrom": "approval", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_run_id_workflow_run_id_fk": { + "name": "approval_run_id_workflow_run_id_fk", + "tableFrom": "approval", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "approval_correlation_id_unique": { + "name": "approval_correlation_id_unique", + "nullsNotDistinct": false, + "columns": ["correlation_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.signal_correlation": { + "name": "signal_correlation", + "schema": "", + "columns": { + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_address": { + "name": "agent_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_name": { + "name": "signal_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_id": { + "name": "signal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "signal_correlation_tenant_id_tenant_id_fk": { + "name": "signal_correlation_tenant_id_tenant_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "signal_correlation_anchor_run_id_workflow_run_id_fk": { + "name": "signal_correlation_anchor_run_id_workflow_run_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "signal_correlation_run_id_workflow_run_id_fk": { + "name": "signal_correlation_run_id_workflow_run_id_fk", + "tableFrom": "signal_correlation", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider": { + "name": "provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "api_base_url": { + "name": "api_base_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_info_url": { + "name": "user_info_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_tenant_id_tenant_id_fk": { + "name": "provider_tenant_id_tenant_id_fk", + "tableFrom": "provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_tenant_name": { + "name": "provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "default_scopes": { + "name": "default_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "oauth_client_tenant_id_tenant_id_fk": { + "name": "oauth_client_tenant_id_tenant_id_fk", + "tableFrom": "oauth_client", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_provider_id_provider_id_fk": { + "name": "oauth_client_provider_id_provider_id_fk", + "tableFrom": "oauth_client", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_tenant_provider": { + "name": "oauth_client_tenant_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.credential": { + "name": "credential", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oauth_client_id": { + "name": "oauth_client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_version": { + "name": "secret_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "refresh_secret": { + "name": "refresh_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "credential_tenant_id_tenant_id_fk": { + "name": "credential_tenant_id_tenant_id_fk", + "tableFrom": "credential", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_principal_id_principal_id_fk": { + "name": "credential_principal_id_principal_id_fk", + "tableFrom": "credential", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "credential_provider_id_provider_id_fk": { + "name": "credential_provider_id_provider_id_fk", + "tableFrom": "credential", + "tableTo": "provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "credential_oauth_client_id_oauth_client_id_fk": { + "name": "credential_oauth_client_id_oauth_client_id_fk", + "tableFrom": "credential", + "tableTo": "oauth_client", + "columnsFrom": ["oauth_client_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "credential_tenant_name": { + "name": "credential_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset": { + "name": "asset", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_tenant_id_tenant_id_fk": { + "name": "asset_tenant_id_tenant_id_fk", + "tableFrom": "asset", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "asset_creator_principal_id_principal_id_fk": { + "name": "asset_creator_principal_id_principal_id_fk", + "tableFrom": "asset", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "asset_tenant_kind_name": { + "name": "asset_tenant_kind_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "kind", "name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transaction": { + "name": "transaction", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_id": { + "name": "sender_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "transaction_wallet_id_wallet_id_fk": { + "name": "transaction_wallet_id_wallet_id_fk", + "tableFrom": "transaction", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "transaction_run_id_workflow_run_id_fk": { + "name": "transaction_run_id_workflow_run_id_fk", + "tableFrom": "transaction", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.wallet": { + "name": "wallet", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backend_type": { + "name": "backend_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "wallet_tenant_id_tenant_id_fk": { + "name": "wallet_tenant_id_tenant_id_fk", + "tableFrom": "wallet", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.offering": { + "name": "offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pricing": { + "name": "pricing", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "offering_agent_id_workflow_definition_id_fk": { + "name": "offering_agent_id_workflow_definition_id_fk", + "tableFrom": "offering", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "offering_tenant_id_tenant_id_fk": { + "name": "offering_tenant_id_tenant_id_fk", + "tableFrom": "offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model": { + "name": "model", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_name": { + "name": "canonical_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_tenant_id_tenant_id_fk": { + "name": "model_tenant_id_tenant_id_fk", + "tableFrom": "model", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_tenant_canonical_name": { + "name": "model_tenant_canonical_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "canonical_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_offering": { + "name": "model_offering", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "deployment_tags": { + "name": "deployment_tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "capabilities": { + "name": "capabilities", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "quirks": { + "name": "quirks", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_offering_tenant_id_tenant_id_fk": { + "name": "model_offering_tenant_id_tenant_id_fk", + "tableFrom": "model_offering", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_model_id_model_id_fk": { + "name": "model_offering_model_id_model_id_fk", + "tableFrom": "model_offering", + "tableTo": "model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_offering_provider_id_model_provider_id_fk": { + "name": "model_offering_provider_id_model_provider_id_fk", + "tableFrom": "model_offering", + "tableTo": "model_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_offering_tenant_model_provider": { + "name": "model_offering_tenant_model_provider", + "nullsNotDistinct": false, + "columns": ["tenant_id", "model_id", "provider_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_pricing": { + "name": "model_pricing", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "offering_id": { + "name": "offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_price": { + "name": "input_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output_token_price": { + "name": "output_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_read_token_price": { + "name": "cache_read_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_write_token_price": { + "name": "cache_write_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "thinking_token_price": { + "name": "thinking_token_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_request_fee": { + "name": "per_request_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_image_fee": { + "name": "per_image_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "per_audio_fee": { + "name": "per_audio_fee", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effective_from": { + "name": "effective_from", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_pricing_tenant_id_tenant_id_fk": { + "name": "model_pricing_tenant_id_tenant_id_fk", + "tableFrom": "model_pricing", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_pricing_offering_id_model_offering_id_fk": { + "name": "model_pricing_offering_id_model_offering_id_fk", + "tableFrom": "model_pricing", + "tableTo": "model_offering", + "columnsFrom": ["offering_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_pricing_offering_currency_effective_from": { + "name": "model_pricing_offering_currency_effective_from", + "nullsNotDistinct": false, + "columns": ["offering_id", "currency", "effective_from"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_provider": { + "name": "model_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin": { + "name": "plugin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wallet_id": { + "name": "wallet_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "model_provider_tenant_id_tenant_id_fk": { + "name": "model_provider_tenant_id_tenant_id_fk", + "tableFrom": "model_provider", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "model_provider_credential_id_credential_id_fk": { + "name": "model_provider_credential_id_credential_id_fk", + "tableFrom": "model_provider", + "tableTo": "credential", + "columnsFrom": ["credential_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "model_provider_wallet_id_wallet_id_fk": { + "name": "model_provider_wallet_id_wallet_id_fk", + "tableFrom": "model_provider", + "tableTo": "wallet", + "columnsFrom": ["wallet_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "model_provider_tenant_name": { + "name": "model_provider_tenant_name", + "nullsNotDistinct": false, + "columns": ["tenant_id", "name"] + } + }, + "policies": {}, + "checkConstraints": { + "model_provider_auth_xor": { + "name": "model_provider_auth_xor", + "value": "(\"model_provider\".\"credential_id\" is not null) <> (\"model_provider\".\"wallet_id\" is not null)" + } + }, + "isRLSEnabled": false + }, + "public.sidecar": { + "name": "sidecar", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reconnect_window_ended_at": { + "name": "reconnect_window_ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'online'" + }, + "last_heartbeat": { + "name": "last_heartbeat", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sidecar_token_hash_sha256_unique": { + "name": "sidecar_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sidecar_allocation": { + "name": "sidecar_allocation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "ensure_accepted_generation": { + "name": "ensure_accepted_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "external_ref": { + "name": "external_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "reconciliation_lease_id": { + "name": "reconciliation_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reconciliation_lease_expires_at": { + "name": "reconciliation_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "initialization_lease_id": { + "name": "initialization_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ensure_attempts": { + "name": "ensure_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "destroy_attempts": { + "name": "destroy_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "deployment_cleanup_confirmed": { + "name": "deployment_cleanup_confirmed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "provider_hold_released": { + "name": "provider_hold_released", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "connect_deadline": { + "name": "connect_deadline", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "max_disconnected_ms": { + "name": "max_disconnected_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "first_deploy_failed_at": { + "name": "first_deploy_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sidecar_allocation_anchor_run_idx": { + "name": "sidecar_allocation_anchor_run_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_sidecar_idx": { + "name": "sidecar_allocation_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_reconciliation_idx": { + "name": "sidecar_allocation_reconciliation_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing') and \"sidecar_allocation\".\"next_attempt_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sidecar_allocation_dispatchable_anchor_idx": { + "name": "sidecar_allocation_dispatchable_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sidecar_allocation_anchor_run_id_workflow_run_id_fk": { + "name": "sidecar_allocation_anchor_run_id_workflow_run_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "sidecar_allocation_tenant_id_tenant_id_fk": { + "name": "sidecar_allocation_tenant_id_tenant_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "sidecar_allocation_sidecar_id_sidecar_id_fk": { + "name": "sidecar_allocation_sidecar_id_sidecar_id_fk", + "tableFrom": "sidecar_allocation", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "sidecar_allocation_status_check": { + "name": "sidecar_allocation_status_check", + "value": "\"sidecar_allocation\".\"status\" in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing', 'destroy_failed', 'released', 'failed')" + }, + "sidecar_allocation_generation_check": { + "name": "sidecar_allocation_generation_check", + "value": "\"sidecar_allocation\".\"generation\" >= 0" + }, + "sidecar_allocation_max_disconnected_check": { + "name": "sidecar_allocation_max_disconnected_check", + "value": "\"sidecar_allocation\".\"max_disconnected_ms\" > 0" + }, + "sidecar_allocation_accepted_generation_check": { + "name": "sidecar_allocation_accepted_generation_check", + "value": "\"sidecar_allocation\".\"ensure_accepted_generation\" is null or \"sidecar_allocation\".\"ensure_accepted_generation\" <= \"sidecar_allocation\".\"generation\"" + }, + "sidecar_allocation_initialization_check": { + "name": "sidecar_allocation_initialization_check", + "value": "\"sidecar_allocation\".\"initialization_lease_id\" is null or (\"sidecar_allocation\".\"status\" = 'allocated' and \"sidecar_allocation\".\"ensure_accepted_generation\" is not null and \"sidecar_allocation\".\"ensure_accepted_generation\" = \"sidecar_allocation\".\"generation\")" + } + }, + "isRLSEnabled": false + }, + "public.agent_session": { + "name": "agent_session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_session_tenant_id_tenant_id_fk": { + "name": "agent_session_tenant_id_tenant_id_fk", + "tableFrom": "agent_session", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_agent_id_workflow_definition_id_fk": { + "name": "agent_session_agent_id_workflow_definition_id_fk", + "tableFrom": "agent_session", + "tableTo": "workflow_definition", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "agent_session_principal_id_principal_id_fk": { + "name": "agent_session_principal_id_principal_id_fk", + "tableFrom": "agent_session", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_asset": { + "name": "session_asset", + "schema": "", + "columns": { + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mount_path": { + "name": "mount_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_pack_sha": { + "name": "asset_pack_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_commit_sha": { + "name": "source_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "materialized_at": { + "name": "materialized_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_asset_pack_sha_idx": { + "name": "session_asset_pack_sha_idx", + "columns": [ + { + "expression": "asset_pack_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "session_asset_instance_id_mount_path_pk": { + "name": "session_asset_instance_id_mount_path_pk", + "columns": ["instance_id", "mount_path"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inference_turn": { + "name": "inference_turn", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "inference_turn_instance_id_started_at_idx": { + "name": "inference_turn_instance_id_started_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inference_turn_session_id_agent_session_id_fk": { + "name": "inference_turn_session_id_agent_session_id_fk", + "tableFrom": "inference_turn", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "inference_turn_tenant_id_tenant_id_fk": { + "name": "inference_turn_tenant_id_tenant_id_fk", + "tableFrom": "inference_turn", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_mail": { + "name": "session_mail", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_mail_instance_id_created_at_idx": { + "name": "session_mail_instance_id_created_at_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_mail_session_id_created_at_idx": { + "name": "session_mail_session_id_created_at_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_mail_session_id_agent_session_id_fk": { + "name": "session_mail_session_id_agent_session_id_fk", + "tableFrom": "session_mail", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_mail_tenant_id_tenant_id_fk": { + "name": "session_mail_tenant_id_tenant_id_fk", + "tableFrom": "session_mail", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.turn_part": { + "name": "turn_part", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "turn_part_turn_id_inference_turn_id_fk": { + "name": "turn_part_turn_id_inference_turn_id_fk", + "tableFrom": "turn_part", + "tableTo": "inference_turn", + "columnsFrom": ["turn_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_part_session_id_agent_session_id_fk": { + "name": "turn_part_session_id_agent_session_id_fk", + "tableFrom": "turn_part", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.git_token": { + "name": "git_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ref_pattern": { + "name": "ref_pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actions": { + "name": "actions", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "git_token_user_id_name_active_idx": { + "name": "git_token_user_id_name_active_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"git_token\".\"revoked_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "git_token_tenant_id_tenant_id_fk": { + "name": "git_token_tenant_id_tenant_id_fk", + "tableFrom": "git_token", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "git_token_user_id_user_id_fk": { + "name": "git_token_user_id_user_id_fk", + "tableFrom": "git_token", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "git_token_principal_id_principal_id_fk": { + "name": "git_token_principal_id_principal_id_fk", + "tableFrom": "git_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "git_token_token_hash_sha256_unique": { + "name": "git_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_token": { + "name": "run_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "consumer": { + "name": "consumer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash_sha256": { + "name": "token_hash_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "sealed_token": { + "name": "sealed_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "run_token_principal_consumer_idx": { + "name": "run_token_principal_consumer_idx", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "consumer", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_token_principal_id_principal_id_fk": { + "name": "run_token_principal_id_principal_id_fk", + "tableFrom": "run_token", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "run_token_token_hash_sha256_unique": { + "name": "run_token_token_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": ["token_hash_sha256"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_definition": { + "name": "workflow_definition", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creator_principal_id": { + "name": "creator_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "wire_hash": { + "name": "wire_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_requirements": { + "name": "grant_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "model_requirements": { + "name": "model_requirements", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_bindings": { + "name": "credential_bindings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lifecycle_policy": { + "name": "lifecycle_policy", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_version": { + "name": "current_version", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'1'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'deployed'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_tenant_idx": { + "name": "workflow_definition_tenant_idx", + "columns": [ + { + "expression": "tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_definition_asset_wire_hash_idx": { + "name": "workflow_definition_asset_wire_hash_idx", + "columns": [ + { + "expression": "asset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "wire_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_tenant_id_tenant_id_fk": { + "name": "workflow_definition_tenant_id_tenant_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_definition_creator_principal_id_principal_id_fk": { + "name": "workflow_definition_creator_principal_id_principal_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "principal", + "columnsFrom": ["creator_principal_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workflow_definition_asset_id_asset_id_fk": { + "name": "workflow_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_definition", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_definition_version": { + "name": "workflow_definition_version", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "definition_id": { + "name": "definition_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "approved_wire_hash": { + "name": "approved_wire_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_snapshot": { + "name": "grant_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_version_definition_idx": { + "name": "workflow_definition_version_definition_idx", + "columns": [ + { + "expression": "definition_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_version_definition_id_workflow_definition_id_fk": { + "name": "workflow_definition_version_definition_id_workflow_definition_id_fk", + "tableFrom": "workflow_definition_version", + "tableTo": "workflow_definition", + "columnsFrom": ["definition_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workflow_definition_version_definition_version": { + "name": "workflow_definition_version_definition_version", + "nullsNotDistinct": false, + "columns": ["definition_id", "version"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run": { + "name": "workflow_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "definition_id": { + "name": "definition_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kernel_id": { + "name": "kernel_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_preferences": { + "name": "model_preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "credential_refs": { + "name": "credential_refs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lifecycle_policy": { + "name": "lifecycle_policy", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancellation_requested_at": { + "name": "cancellation_requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "operator_cancellation_requested_at": { + "name": "operator_cancellation_requested_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "cancellation_reason": { + "name": "cancellation_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cancellation_deadline": { + "name": "cancellation_deadline", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "standby_state": { + "name": "standby_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "standby_state_at": { + "name": "standby_state_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "standby_wake_at": { + "name": "standby_wake_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "standby_reason": { + "name": "standby_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capacity_release_at": { + "name": "capacity_release_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "infrastructure_failed_at": { + "name": "infrastructure_failed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "legacy_mail_accept": { + "name": "legacy_mail_accept", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_definition_idx": { + "name": "workflow_run_definition_idx", + "columns": [ + { + "expression": "definition_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_address_idx": { + "name": "workflow_run_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"workflow_run\".\"address\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_anchor_idx": { + "name": "workflow_run_anchor_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"id\" = \"workflow_run\".\"anchor_run_id\"", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_live_anchor_idx": { + "name": "workflow_run_live_anchor_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"id\" = \"workflow_run\".\"anchor_run_id\" and \"workflow_run\".\"status\" in ('deployed', 'running')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_infrastructure_failed_idx": { + "name": "workflow_run_infrastructure_failed_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"infrastructure_failed_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_sidecar_idx": { + "name": "workflow_run_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run\".\"sidecar_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_definition_id_workflow_definition_id_fk": { + "name": "workflow_run_definition_id_workflow_definition_id_fk", + "tableFrom": "workflow_run", + "tableTo": "workflow_definition", + "columnsFrom": ["definition_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_tenant_id_tenant_id_fk": { + "name": "workflow_run_tenant_id_tenant_id_fk", + "tableFrom": "workflow_run", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_principal_id_principal_id_fk": { + "name": "workflow_run_principal_id_principal_id_fk", + "tableFrom": "workflow_run", + "tableTo": "principal", + "columnsFrom": ["principal_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "workflow_run_sidecar_id_sidecar_id_fk": { + "name": "workflow_run_sidecar_id_sidecar_id_fk", + "tableFrom": "workflow_run", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_run_legacy_mail_accept_check": { + "name": "workflow_run_legacy_mail_accept_check", + "value": "\"workflow_run\".\"legacy_mail_accept\" is null or \"workflow_run\".\"legacy_mail_accept\" in ('invoker-correspondent', 'correspondent')" + } + }, + "isRLSEnabled": false + }, + "public.workflow_run_dispatch": { + "name": "workflow_run_dispatch", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mail'" + }, + "sender_address": { + "name": "sender_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "raw_message": { + "name": "raw_message", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "step_grants": { + "name": "step_grants", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "acknowledged_generation": { + "name": "acknowledged_generation", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "delivery_lease_id": { + "name": "delivery_lease_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "delivery_lease_expires_at": { + "name": "delivery_lease_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "acknowledged_at": { + "name": "acknowledged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_dispatch_anchor_message_idx": { + "name": "workflow_run_dispatch_anchor_message_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_delivery_idx": { + "name": "workflow_run_dispatch_delivery_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run_dispatch\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_anchor_status_idx": { + "name": "workflow_run_dispatch_anchor_status_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_dispatch_unresolved_anchor_idx": { + "name": "workflow_run_dispatch_unresolved_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_run_dispatch\".\"status\" in ('pending', 'acknowledged') or (\"workflow_run_dispatch\".\"status\" = 'abandoned' and \"workflow_run_dispatch\".\"next_attempt_at\" is not null)", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_dispatch_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_dispatch_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_dispatch", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_run_dispatch_status_check": { + "name": "workflow_run_dispatch_status_check", + "value": "\"workflow_run_dispatch\".\"status\" in ('pending', 'acknowledged', 'abandoned', 'settled', 'failed')" + }, + "workflow_run_dispatch_kind_check": { + "name": "workflow_run_dispatch_kind_check", + "value": "\"workflow_run_dispatch\".\"kind\" in ('mail', 'signal')" + }, + "workflow_run_dispatch_attempt_count_check": { + "name": "workflow_run_dispatch_attempt_count_check", + "value": "\"workflow_run_dispatch\".\"attempt_count\" >= 0" + }, + "workflow_run_dispatch_acknowledged_generation_check": { + "name": "workflow_run_dispatch_acknowledged_generation_check", + "value": "\"workflow_run_dispatch\".\"acknowledged_generation\" is null or \"workflow_run_dispatch\".\"acknowledged_generation\" >= 0" + }, + "workflow_run_dispatch_acknowledged_state_check": { + "name": "workflow_run_dispatch_acknowledged_state_check", + "value": "\"workflow_run_dispatch\".\"status\" <> 'acknowledged' or \"workflow_run_dispatch\".\"acknowledged_generation\" is not null" + }, + "workflow_run_dispatch_pending_schedule_check": { + "name": "workflow_run_dispatch_pending_schedule_check", + "value": "\"workflow_run_dispatch\".\"status\" <> 'pending' or \"workflow_run_dispatch\".\"next_attempt_at\" is not null" + }, + "workflow_run_dispatch_mail_sender_check": { + "name": "workflow_run_dispatch_mail_sender_check", + "value": "\"workflow_run_dispatch\".\"kind\" <> 'mail' or \"workflow_run_dispatch\".\"status\" in ('abandoned', 'settled', 'failed') or \"workflow_run_dispatch\".\"sender_address\" is not null" + } + }, + "isRLSEnabled": false + }, + "public.workflow_pending_projection": { + "name": "workflow_pending_projection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "history_required": { + "name": "history_required", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": { + "workflow_pending_projection_anchor_idx": { + "name": "workflow_pending_projection_anchor_idx", + "columns": [ + { + "expression": "anchor_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_pending_projection_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_pending_projection_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_pending_projection", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run_launch_spec": { + "name": "workflow_run_launch_spec", + "schema": "", + "columns": { + "anchor_run_id": { + "name": "anchor_run_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_domain": { + "name": "deployment_domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_authority_principal_id": { + "name": "source_authority_principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frozen_approval_bundle": { + "name": "frozen_approval_bundle", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "definition_artifact_id": { + "name": "definition_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_snapshot": { + "name": "grant_snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "source_offering_ids": { + "name": "source_offering_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "default_source_offering_id": { + "name": "default_source_offering_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_run_launch_spec_anchor_run_id_workflow_run_id_fk": { + "name": "workflow_run_launch_spec_anchor_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "workflow_run", + "columnsFrom": ["anchor_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "workflow_run_launch_spec_source_authority_principal_id_principal_id_fk": { + "name": "workflow_run_launch_spec_source_authority_principal_id_principal_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "principal", + "columnsFrom": ["source_authority_principal_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_run_launch_spec_definition_artifact_id_workflow_definition_artifact_id_fk": { + "name": "workflow_run_launch_spec_definition_artifact_id_workflow_definition_artifact_id_fk", + "tableFrom": "workflow_run_launch_spec", + "tableTo": "workflow_definition_artifact", + "columnsFrom": ["definition_artifact_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_run_execution": { + "name": "workflow_run_execution", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_run_execution_run_id_id_idx": { + "name": "workflow_run_execution_run_id_id_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_run_execution_status_idx": { + "name": "workflow_run_execution_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_run_execution_run_id_workflow_run_id_fk": { + "name": "workflow_run_execution_run_id_workflow_run_id_fk", + "tableFrom": "workflow_run_execution", + "tableTo": "workflow_run", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workflow_probe": { + "name": "workflow_probe", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "definition_asset_id": { + "name": "definition_asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entry": { + "name": "entry", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pin": { + "name": "pin", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "provisioner_id": { + "name": "provisioner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provisioner_api_version": { + "name": "provisioner_api_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provisioner_binding_fingerprint": { + "name": "provisioner_binding_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sidecar_id": { + "name": "sidecar_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "adapter_manifest_digest": { + "name": "adapter_manifest_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "external_ref": { + "name": "external_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "failure_code": { + "name": "failure_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_message": { + "name": "failure_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_probe_active_idx": { + "name": "workflow_probe_active_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"workflow_probe\".\"status\" in ('pending', 'provisioning', 'probing', 'releasing')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_probe_sidecar_idx": { + "name": "workflow_probe_sidecar_idx", + "columns": [ + { + "expression": "sidecar_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_probe_tenant_id_tenant_id_fk": { + "name": "workflow_probe_tenant_id_tenant_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_probe_definition_asset_id_asset_id_fk": { + "name": "workflow_probe_definition_asset_id_asset_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "asset", + "columnsFrom": ["definition_asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_probe_sidecar_id_sidecar_id_fk": { + "name": "workflow_probe_sidecar_id_sidecar_id_fk", + "tableFrom": "workflow_probe", + "tableTo": "sidecar", + "columnsFrom": ["sidecar_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_probe_status_check": { + "name": "workflow_probe_status_check", + "value": "\"workflow_probe\".\"status\" in ('pending', 'provisioning', 'probing', 'releasing', 'succeeded', 'failed')" + }, + "workflow_probe_succeeded_result_check": { + "name": "workflow_probe_succeeded_result_check", + "value": "\"workflow_probe\".\"status\" <> 'succeeded' or \"workflow_probe\".\"result\" is not null" + }, + "workflow_probe_generation_check": { + "name": "workflow_probe_generation_check", + "value": "\"workflow_probe\".\"generation\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.workflow_definition_artifact": { + "name": "workflow_definition_artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision_id": { + "name": "revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workflow_definition_artifact_revision_idx": { + "name": "workflow_definition_artifact_revision_idx", + "columns": [ + { + "expression": "revision_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "workflow_definition_artifact_revision_id_workflow_definition_revision_id_fk": { + "name": "workflow_definition_artifact_revision_id_workflow_definition_revision_id_fk", + "tableFrom": "workflow_definition_artifact", + "tableTo": "workflow_definition_revision", + "columnsFrom": ["revision_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "workflow_definition_artifact_state_check": { + "name": "workflow_definition_artifact_state_check", + "value": "(\"workflow_definition_artifact\".\"result\" is null and \"workflow_definition_artifact\".\"lease_owner\" is not null and \"workflow_definition_artifact\".\"lease_expires_at\" is not null) or (\"workflow_definition_artifact\".\"result\" is not null and \"workflow_definition_artifact\".\"lease_owner\" is null and \"workflow_definition_artifact\".\"lease_expires_at\" is null)" + } + }, + "isRLSEnabled": false + }, + "public.workflow_definition_revision": { + "name": "workflow_definition_revision", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "tenant_id": { + "name": "tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_id": { + "name": "asset_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "asset_commit_sha": { + "name": "asset_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entry": { + "name": "entry", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "closure": { + "name": "closure", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workflow_definition_revision_tenant_id_tenant_id_fk": { + "name": "workflow_definition_revision_tenant_id_tenant_id_fk", + "tableFrom": "workflow_definition_revision", + "tableTo": "tenant", + "columnsFrom": ["tenant_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "workflow_definition_revision_asset_id_asset_id_fk": { + "name": "workflow_definition_revision_asset_id_asset_id_fk", + "tableFrom": "workflow_definition_revision", + "tableTo": "asset", + "columnsFrom": ["asset_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index ecb0e61e5..891c0cb4d 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -869,6 +869,13 @@ "when": 1793960501284, "tag": "0124_mail_send_backfill", "breakpoints": true + }, + { + "idx": 125, + "version": "7", + "when": 1793960501285, + "tag": "0125_workflow_run_standby", + "breakpoints": true } ] } diff --git a/packages/db/src/parse-row.test.ts b/packages/db/src/parse-row.test.ts index d29609ea1..dc2362f5d 100644 --- a/packages/db/src/parse-row.test.ts +++ b/packages/db/src/parse-row.test.ts @@ -317,6 +317,10 @@ function makeWorkflowRunRow( operatorCancellationRequestedAt: null, cancellationReason: null, cancellationDeadline: null, + standbyState: null, + standbyStateAt: null, + standbyWakeAt: null, + standbyReason: null, capacityReleaseAt: null, infrastructureFailedAt: null, failureCode: null, @@ -354,6 +358,20 @@ describe("parseWorkflowRunRow", () => { ), ).toThrow(); }); + + test("rejects a standby state outside the known states", () => { + expect(parseWorkflowRunRow(makeWorkflowRunRow()).standbyState).toBeNull(); + expect( + parseWorkflowRunRow(makeWorkflowRunRow({ standbyState: "standby" })) + .standbyState, + ).toBe("standby"); + // The column is plain text, so a row can hold any string. + expect(() => + parseWorkflowRunRow( + Object.assign(makeWorkflowRunRow(), { standbyState: "stopped" }), + ), + ).toThrow(); + }); }); describe("parseWorkflowDefinitionRow", () => { diff --git a/packages/db/src/parse-row.ts b/packages/db/src/parse-row.ts index 3f3f6eabb..6cd6badcc 100644 --- a/packages/db/src/parse-row.ts +++ b/packages/db/src/parse-row.ts @@ -16,6 +16,7 @@ import { TenantConfig, WorkflowLifecyclePolicy, workflowDefinitionStatuses, + workflowStandbyStates, workflowDefinitionVersionStatuses, } from "@intx/types"; import { WireGrantRule } from "@intx/types/grant-wire"; @@ -76,6 +77,7 @@ const workflowRunStatuses = [ "cancelled", ] as const; const WorkflowRunStatusValidator = type.enumerated(...workflowRunStatuses); +const WorkflowStandbyStateValidator = type.enumerated(...workflowStandbyStates); const WorkflowRunDispatchStatusValidator = type.enumerated( ...workflowRunDispatchStatuses, ); @@ -223,6 +225,10 @@ export function parseWorkflowRunRow(row: typeof workflowRun.$inferSelect) { return { ...row, status: WorkflowRunStatusValidator.assert(row.status), + standbyState: + row.standbyState === null + ? null + : WorkflowStandbyStateValidator.assert(row.standbyState), lifecyclePolicy: row.lifecyclePolicy == null ? null diff --git a/packages/db/src/schema/workflow-run.ts b/packages/db/src/schema/workflow-run.ts index 77ab09696..f68b4105f 100644 --- a/packages/db/src/schema/workflow-run.ts +++ b/packages/db/src/schema/workflow-run.ts @@ -10,6 +10,8 @@ import { uniqueIndex, } from "drizzle-orm/pg-core"; +import type { WorkflowStandbyState } from "@intx/types"; + import { principal } from "./principals"; import { sidecar } from "./sidecar"; import { tenant } from "./tenants"; @@ -146,6 +148,20 @@ export const workflowRun = pgTable( ), cancellationReason: text("cancellation_reason"), cancellationDeadline: timestamp("cancellation_deadline"), + // Where a deployment anchor's workflow child stands on the way into or out + // of standby: `entering` until its sidecar stops the child, `standby` once + // the Hub holds the history the child left, `waking` until the child runs + // again. Null while the child runs and no standby is asked for. + standbyState: text("standby_state").$type(), + standbyStateAt: timestamp("standby_state_at"), + // The earliest pending timer a child in standby is woken for. + standbyWakeAt: timestamp("standby_wake_at"), + // Why the standby state last changed other than as requested: the + // sidecar refused a standby or wake request, something other than a wake + // request woke the child in standby, the sidecar reported the child + // running or in standby though none was asked for, the deployment is + // being cancelled, or it moved to replacement capacity. + standbyReason: text("standby_reason"), capacityReleaseAt: timestamp("capacity_release_at"), // When unrecoverable capacity loss happened while accepted history could // still settle some runs. Set on the anchor instead of failing its live diff --git a/packages/db/src/sidecar-allocation-store.ts b/packages/db/src/sidecar-allocation-store.ts index ae1220fa1..d6bd4ed61 100644 --- a/packages/db/src/sidecar-allocation-store.ts +++ b/packages/db/src/sidecar-allocation-store.ts @@ -1542,6 +1542,38 @@ function buildSidecarAllocationStore( .set({ credentialRefs: null }) .where(eq(workflowRun.id, updated.anchorRunId)); await retireRunPrincipalKey(tx, updated.tenantId, updated.anchorRunId); + // The replacement copy starts awake: a standby request still wanted is + // carried out there, and a wake has nothing left to do. A JS date, + // since an answer is matched against this value read back, which + // drops microseconds. + const standbyChangedAt = args.now ?? new Date(); + await tx + .update(workflowRun) + .set({ + standbyState: "entering", + standbyStateAt: standbyChangedAt, + standbyWakeAt: null, + standbyReason: + "The deployment moved to replacement capacity, which starts awake", + }) + .where( + and( + eq(workflowRun.id, updated.anchorRunId), + eq(workflowRun.standbyState, "standby"), + ), + ); + await tx + .update(workflowRun) + .set({ + standbyState: null, + standbyStateAt: standbyChangedAt, + }) + .where( + and( + eq(workflowRun.id, updated.anchorRunId), + eq(workflowRun.standbyState, "waking"), + ), + ); return parseSidecarAllocationRow(updated); }); diff --git a/packages/hub-api/src/routes/runs.test.ts b/packages/hub-api/src/routes/runs.test.ts index 194bb07a3..eda2b6fc6 100644 --- a/packages/hub-api/src/routes/runs.test.ts +++ b/packages/hub-api/src/routes/runs.test.ts @@ -1432,11 +1432,15 @@ function lifecycleService( cancellationDeadline: null, cancellationReason: null, capacityReleaseAt: null, + standby: null, allocation: null, }), releaseCapacity: async () => "pending", releaseStoppedCleanup: async () => "released", requestCancellation: async () => "pending", + requestStandby: async () => "pending", + requestWake: async () => "pending", + noteDeploymentInStandby: async () => undefined, reconcileNext: async () => false, ...overrides, }; @@ -1482,6 +1486,68 @@ describe("workflow lifecycle endpoints", () => { expect(calls).toEqual([[TENANT_ID, RUN_ID]]); }); + test.each(["standby", "wake"] as const)( + "reading a run does not grant %s", + async (wanted) => { + let called = false; + const request = async () => { + called = true; + return "pending" as const; + }; + const app = createTestApp({ + grants: [makeGrant({ action: "read" })], + lifecycleService: lifecycleService({ + requestStandby: request, + requestWake: request, + }), + }); + expect( + (await app.request(`${runURL()}/${wanted}`, { method: "POST" })).status, + ).toBe(403); + expect(called).toBe(false); + }, + ); + + test.each([ + ["standby", "pending", 202], + ["standby", "settled", 204], + ["standby", "terminal", 409], + ["standby", "cancelling", 409], + ["standby", "unallocated", 409], + ["standby", "not_found", 404], + ["wake", "pending", 202], + ["wake", "settled", 204], + ["wake", "terminal", 409], + ["wake", "cancelling", 409], + ["wake", "unallocated", 409], + ["wake", "not_found", 404], + ] as const)("%s reports %s as %d", async (wanted, result, status) => { + const calls: string[] = []; + const app = createTestApp({ + grants: [ + makeGrant({ resource: `workflow-run:${RUN_ID}`, action: "manage" }), + ], + lifecycleService: lifecycleService({ + requestStandby: async (tenantId, runId) => { + calls.push(`standby ${tenantId} ${runId}`); + return result; + }, + requestWake: async (tenantId, runId) => { + calls.push(`wake ${tenantId} ${runId}`); + return result; + }, + }), + }); + const response = await app.request(`${runURL()}/${wanted}`, { + method: "POST", + }); + expect(response.status).toBe(status); + expect(response.headers.get("Location")).toBe( + status === 202 ? `${runURL()}/lifecycle` : null, + ); + expect(calls).toEqual([`${wanted} ${TENANT_ID} ${RUN_ID}`]); + }); + test("exhausted cleanup reports the required intervention", async () => { const app = createTestApp({ grants: [makeGrant({ action: "manage" })], diff --git a/packages/hub-api/src/routes/runs.ts b/packages/hub-api/src/routes/runs.ts index c3d463c01..08a35644c 100644 --- a/packages/hub-api/src/routes/runs.ts +++ b/packages/hub-api/src/routes/runs.ts @@ -94,6 +94,36 @@ function workflowRunOperationUnsupported( ); } +function standbyResponse( + c: Context, + tenantId: string, + runId: string, + result: Awaited>, +): Response { + switch (result) { + case "not_found": + return errorResponse(c, "not_found", "Run not found"); + case "terminal": + return errorResponse(c, "conflict", "Run is already terminal"); + case "cancelling": + return errorResponse(c, "conflict", "Run is being cancelled"); + case "unallocated": + return errorResponse( + c, + "conflict", + "The deployment's allocation can no longer be dispatched to a sidecar", + ); + case "settled": + return c.body(null, 204); + case "pending": + c.header( + "Location", + `/api/tenants/${tenantId}/workflows/runs/${runId}/lifecycle`, + ); + return c.body(null, 202); + } +} + type RunStatusFilter = | "deployed" | "running" @@ -678,7 +708,7 @@ export function createRunRoutes({ tags: ["Runs"], summary: "Get workflow lifecycle status", description: - "Returns the deployment's saved policy, deadlines, and allocation cleanup status. Policy edits apply to new deployments.", + "Returns the deployment's saved policy, deadlines, standby state, and allocation cleanup status. Policy edits apply to new deployments.", responses: { 200: jsonResponse("Lifecycle status", WorkflowLifecycleResponse), 404: jsonResponse("Run not found", ErrorResponse), @@ -882,6 +912,86 @@ export function createRunRoutes({ }, ); + app.post( + "/:runId/standby", + requireGrant(idResource("workflow-run", "runId"), "manage"), + describeRoute({ + tags: ["Runs"], + summary: "Put a deployment in standby", + description: + "Durably requests that a live top-level run's workflow child stop once every run of the deployment only waits on input from outside. Work in progress goes on until then, and the deployment keeps its capacity. Mail, a signal, an approval decision, or a due timer runs the child again, as does `POST .../wake`. Poll the Location header for progress; a standby request the sidecar refuses returns to `awake` with its reason.", + responses: { + 202: { description: "Standby requested" }, + 204: { description: "The deployment is already in standby" }, + 404: jsonResponse("Run not found", ErrorResponse), + 409: jsonResponse( + "Run is terminal, being cancelled, or its allocation can no longer be dispatched to a sidecar", + ErrorResponse, + ), + 503: jsonResponse( + "Workflow lifecycle service unavailable", + ErrorResponse, + ), + }, + }), + async (c) => { + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const tenantId = c.get("tenant").id; + const runId = c.req.param("runId"); + return standbyResponse( + c, + tenantId, + runId, + await lifecycleService.requestStandby(tenantId, runId), + ); + }, + ); + + app.post( + "/:runId/wake", + requireGrant(idResource("workflow-run", "runId"), "manage"), + describeRoute({ + tags: ["Runs"], + summary: "Wake a deployment", + description: + "Durably requests that the workflow child of a deployment in standby run again, or that a standby request in progress be abandoned. Poll the Location header for progress.", + responses: { + 202: { description: "Wake requested" }, + 204: { description: "The deployment is not in standby" }, + 404: jsonResponse("Run not found", ErrorResponse), + 409: jsonResponse( + "Run is terminal, being cancelled, or its allocation can no longer be dispatched to a sidecar", + ErrorResponse, + ), + 503: jsonResponse( + "Workflow lifecycle service unavailable", + ErrorResponse, + ), + }, + }), + async (c) => { + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const tenantId = c.get("tenant").id; + const runId = c.req.param("runId"); + return standbyResponse( + c, + tenantId, + runId, + await lifecycleService.requestWake(tenantId, runId), + ); + }, + ); + app.get( "/:runId/events", requireGrant(idResource("workflow-run", "runId"), "read"), diff --git a/packages/hub-sessions/src/workflow-lifecycle-service.ts b/packages/hub-sessions/src/workflow-lifecycle-service.ts index cb3952318..d2d9bbc28 100644 --- a/packages/hub-sessions/src/workflow-lifecycle-service.ts +++ b/packages/hub-sessions/src/workflow-lifecycle-service.ts @@ -39,14 +39,22 @@ import { workflowRun, } from "@intx/db/schema"; import { getLogger } from "@intx/log"; -import { isSidecarAllocationDispatchable } from "@intx/types"; +import { + isSidecarAllocationDispatchable, + type WorkflowStandbyState, +} from "@intx/types"; import type { WorkflowHistoryReceiveTracker } from "./workflow-history-receives"; +import type { SidecarEventMap } from "./ws/sidecar-events"; import type { WorkflowRunReader } from "./workflow-run-reader"; import { WorkflowControlNotConnectedError, + WorkflowControlHistoryPendingError, + WorkflowControlInitializingError, WorkflowControlRejectedError, WorkflowControlTimeoutError, + WorkflowControlUnconfirmedError, + WorkflowControlUnreachableError, type AllocatedSidecarTarget, type SidecarAllocationRouter, } from "./ws/sidecar-handler"; @@ -73,11 +81,29 @@ const RECOVERY_BACKOFF_MIN_MS = 5_000; const RECOVERY_BACKOFF_MAX_MS = 5 * 60_000; const CANCEL_CONTROL_TIMEOUT_MS = 5_000; const STOP_CONTROL_TIMEOUT_MS = 10_000; +// A standby request is answered once every run waits, however long that takes, +// so this bounds a wake and the Hub's handling of an answer. +const STANDBY_CONTROL_TIMEOUT_MS = 60_000; +// A standby command the sidecar did not settle goes out again after this long, +// doubling up to the maximum. A retry whose request ended some other way is +// forgotten once it is long past due. +const STANDBY_RETRY_MIN_MS = 1_000; +const STANDBY_RETRY_MAX_MS = 30_000; +const STANDBY_RETRY_FORGET_MS = 60 * 60_000; +// A standby report the Hub could not record is retried for this long, holding +// back its allocation's later reports and answers, before the sidecar's next +// hello is left to correct it. +const REPORT_RETRY_MIN_MS = 5_000; +const REPORT_RETRY_MAX_MS = 60_000; +const REPORT_RETRY_GIVE_UP_MS = 5 * 60_000; const MAX_CONCURRENT_RETAINS_PER_SIDECAR = 8; const STOP_FAILED_RELEASE = { failureCode: "workflow_stop_failed", failureMessage: "Workflow stop could not be confirmed; reclaiming capacity", }; +const REPORTED_RUNNING_REASON = + "The sidecar reported the workflow child running"; +const CANCELLED_STANDBY_REASON = "The deployment is being cancelled"; type Run = ReturnType; type Allocation = typeof sidecarAllocation.$inferSelect; @@ -99,6 +125,7 @@ type ReleaseStoppedCleanupResult = // while the sweep is backing off; `stop` and `sweep` respect the backoff, and // only `sweep` leaves young rows to the receive that wrote them. type RecoveryTrigger = "request" | "stop" | "sweep"; +type StandbyAction = "standby" | "wake"; export type WorkflowLifecycleServiceDeps = { db: DB["db"]; @@ -122,6 +149,12 @@ export type WorkflowLifecycleServiceDeps = { */ operatorCancelReconnectGraceMs?: number; sendControl?: SidecarAllocationRouter["sendWorkflowControl"]; + standbyRouter: Pick< + SidecarAllocationRouter, + "putAllocationInStandby" | "wakeAllocation" | "holdsAllocatedBinding" + >; + /** Called after each standby or wake command ends, answered or not. */ + onStandbyCommandSettled?: (runId: string) => void; }; export function createWorkflowLifecycleService({ @@ -134,6 +167,8 @@ export function createWorkflowLifecycleService({ unreachableStopGraceMs = 60_000, operatorCancelReconnectGraceMs = 15_000, sendControl, + standbyRouter, + onStandbyCommandSettled, }: WorkflowLifecycleServiceDeps) { if (cancelGraceMs < 0 || !Number.isFinite(cancelGraceMs)) throw new Error("Cancellation grace must be finite and non-negative"); @@ -166,6 +201,59 @@ export function createWorkflowLifecycleService({ } >(); const retentionsInFlight = new Set(); + // By run: when the sweep next repeats a request that got no answer, and why + // it got none. Kept off the run's row, so it cannot outlast the request. + const standbyRetries = new Map< + string, + { + action: StandbyAction; + failures: number; + nextAt: number; + note: string | null; + } + >(); + // The standby or wake request each run has outstanding, so a sweep does not + // repeat one the sidecar is still answering. + const standbyCommands = new Map(); + // The last turn taken for each allocation generation's reports and answers. + const observationTails = new Map>(); + // The last standby drive queued for each run. + const standbyDrives = new Map>(); + // Standby reports whose write failed, by allocation generation. Each keeps + // its turn until the sweep records it or gives up. + const failedReports = new Map< + string, + { + report: SidecarEventMap["deployment.standby"]; + firstFailedAt: number; + failures: number; + nextAt: number; + done: () => void; + } + >(); + + // Reports and answers from a generation's sidecar are recorded in the order + // the Hub received them, so an earlier one never overwrites a later one. A + // turn is taken when its report or answer arrives. + function takeObservationTurn(target: AllocatedSidecarTarget): { + ready: Promise; + done: () => void; + } { + const key = `${target.allocationId}:${String(target.generation)}`; + const previous = observationTails.get(key) ?? Promise.resolve(); + const turn = Promise.withResolvers(); + const tail = previous.then(() => turn.promise); + observationTails.set(key, tail); + return { + ready: previous, + done: () => { + turn.resolve(undefined); + void tail.then(() => { + if (observationTails.get(key) === tail) observationTails.delete(key); + }); + }, + }; + } const retentionsPerSidecar = new Map(); function hasRetentionRoom(sidecarId: string): boolean { @@ -426,6 +514,17 @@ export function createWorkflowLifecycleService({ cancellationDeadline: run.cancellationDeadline?.toISOString() ?? null, cancellationReason: run.cancellationReason ?? null, capacityReleaseAt: run.capacityReleaseAt?.toISOString() ?? null, + standby: isLiveWorkflowRunStatus(run.status) + ? { + state: run.standbyState ?? ("awake" as const), + changedAt: run.standbyStateAt?.toISOString() ?? null, + wakeAt: + run.standbyState === "standby" + ? (run.standbyWakeAt?.toISOString() ?? null) + : null, + reason: standbyRetryNote(run) ?? run.standbyReason, + } + : null, allocation: allocation === null ? null @@ -529,6 +628,20 @@ export function createWorkflowLifecycleService({ } const deadline = new Date(at.getTime() + cancelGraceMs); const operatorCancellationRequestedAt = origin === "operator" ? at : null; + // Cancelling wakes a child in standby and abandons a standby request in + // progress, so a standby request no longer stands, nor does the reason + // an earlier one was refused. + const wake = + run.standbyState === null + ? run.standbyReason === null + ? {} + : { standbyReason: null } + : { + standbyState: null, + standbyStateAt: at, + standbyWakeAt: null, + standbyReason: CANCELLED_STANDBY_REASON, + }; await tx .update(workflowRun) .set({ @@ -536,6 +649,7 @@ export function createWorkflowLifecycleService({ cancellationDeadline: deadline, cancellationReason: reason, operatorCancellationRequestedAt, + ...wake, }) .where(eq(workflowRun.id, run.id)); return { @@ -544,6 +658,7 @@ export function createWorkflowLifecycleService({ cancellationDeadline: deadline, cancellationReason: reason, operatorCancellationRequestedAt, + ...wake, }; } @@ -599,6 +714,496 @@ export function createWorkflowLifecycleService({ }); } + // The Hub records the wanted state, and the sweep asks the sidecar for it + // until the sidecar answers. + async function requestStandbyState( + tenantId: string, + runId: string, + wanted: "standby" | "awake", + ): Promise< + | "pending" + | "settled" + | "terminal" + | "cancelling" + | "unallocated" + | "not_found" + > { + const [settled, pending] = + wanted === "standby" + ? (["standby", "entering"] as const) + : ([null, "waking"] as const); + // Not `withRun`, as for a cancellation: a pack receive and the sweep hold + // the allocation row, and a request must not wait on either. The anchor + // lock orders it against every `withRun`. History is not recovered here + // either; the sweep recovers it before its own drive. + const result = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(workflowRun) + .where( + and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + eq(workflowRun.anchorRunId, runId), + ), + ) + .for("no key update"); + if (row === undefined) return "not_found" as const; + const run = parseWorkflowRunRow(row); + if (!isLiveWorkflowRunStatus(run.status)) return "terminal" as const; + if (run.cancellationRequestedAt !== null) return "cancelling" as const; + const [allocation] = await tx + .select({ status: sidecarAllocation.status }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.anchorRunId, runId), + eq(sidecarAllocation.tenantId, tenantId), + ), + ); + // No sidecar could ever carry the request out, nor holds a child in the + // state asked for. + if ( + allocation === undefined || + !isSidecarAllocationDispatchable(allocation.status) + ) + return "unallocated" as const; + if (run.standbyState === settled) return "settled" as const; + if (run.standbyState !== pending) { + // A JS date, since an answer is matched against this value read back, + // which drops microseconds. + await tx + .update(workflowRun) + .set({ + standbyState: pending, + standbyStateAt: now(), + standbyWakeAt: null, + standbyReason: null, + }) + .where(eq(workflowRun.id, runId)); + standbyRetries.delete(runId); + } + return "pending" as const; + }); + // A standby request in progress answers only once every run waits, so a + // wake that abandons it goes out now. + if ( + result === "pending" && + wanted === "awake" && + standbyCommands.get(runId)?.action === "standby" + ) { + void driveStandbyState(runId).catch((cause: unknown) => { + logger.warn`Workflow wake for ${runId} waits for the sweep: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } + return result; + } + + type StandbyCommand = { + target: AllocatedSidecarTarget; + runId: string; + action: StandbyAction; + // The request being carried out; an answer applies only while it stands. + stateAt: Date; + }; + + // The anchor run an allocation generation serves. + function anchorOf(target: AllocatedSidecarTarget) { + return db + .select({ id: sidecarAllocation.anchorRunId }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.id, target.allocationId), + eq(sidecarAllocation.generation, target.generation), + ), + ); + } + + // Records what a generation's sidecar did with that allocation row locked + // first, as pack ingestion does, so a replacement cannot commit in between; + // skipped once the generation moved on. + async function withReportingGeneration( + target: AllocatedSidecarTarget, + write: (tx: DBExecutor, runId: string) => Promise, + ): Promise { + await db.transaction(async (tx) => { + const [allocation] = await tx + .select({ runId: sidecarAllocation.anchorRunId }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.id, target.allocationId), + eq(sidecarAllocation.generation, target.generation), + ), + ) + .for("update"); + if (allocation !== undefined) await write(tx, allocation.runId); + }); + } + + // A run's drives decide one at a time, each from a read taken after the one + // before it sent its command, so a command chosen from an older read never + // reaches the sidecar after one chosen from a newer read. Answers are + // awaited outside the queue. + function driveStandbyState(runId: string): Promise { + const previous = standbyDrives.get(runId) ?? Promise.resolve(); + const drive = previous.then(() => sendPendingStandbyCommand(runId)); + const tail = drive.catch(() => undefined); + standbyDrives.set(runId, tail); + void tail.then(() => { + if (standbyDrives.get(runId) === tail) standbyDrives.delete(runId); + }); + return drive; + } + + // Sends the command that carries out a pending standby or wake request once + // the Hub can deliver it: the copy is deployed and its sidecar connected. A + // command waits for its answer or its connection; the sweep sends again + // after either. + async function sendPendingStandbyCommand(runId: string) { + const [row] = await db + .select({ + run: workflowRun, + allocationId: sidecarAllocation.id, + allocationStatus: sidecarAllocation.status, + generation: sidecarAllocation.generation, + sidecarId: sidecarAllocation.sidecarId, + }) + .from(workflowRun) + .innerJoin( + sidecarAllocation, + eq(sidecarAllocation.anchorRunId, workflowRun.id), + ) + .where( + and( + eq(workflowRun.id, runId), + eq(workflowRun.anchorRunId, runId), + runPrincipalHasActiveKey(workflowRun.tenantId, workflowRun.id), + ), + ); + if (row === undefined) return; + const run = parseWorkflowRunRow(row.run); + if ( + (run.standbyState !== "entering" && run.standbyState !== "waking") || + run.standbyStateAt === null || + !isLiveWorkflowRunStatus(run.status) || + run.cancellationRequestedAt !== null || + row.allocationStatus !== "allocated" || + row.sidecarId === null + ) + return; + const action: StandbyAction = + run.standbyState === "entering" ? "standby" : "wake"; + const target = { + allocationId: row.allocationId, + generation: row.generation, + }; + if (!standbyRouter.holdsAllocatedBinding(target)) return; + const inFlight = standbyCommands.get(runId); + if ( + inFlight?.action === action && + inFlight.target.generation === row.generation + ) + return; + const command = { target, runId, action, stateAt: run.standbyStateAt }; + standbyCommands.set(runId, command); + void sendStandbyCommand(command).then(() => { + if (standbyCommands.get(runId) === command) standbyCommands.delete(runId); + onStandbyCommandSettled?.(runId); + }); + } + + async function sendStandbyCommand(command: StandbyCommand): Promise { + let turn: ReturnType | undefined; + const onAnswered = (): void => { + turn ??= takeObservationTurn(command.target); + }; + try { + let change: Parameters[1]; + try { + if (command.action === "standby") { + const { wakeAt } = await standbyRouter.putAllocationInStandby( + command.target, + STANDBY_CONTROL_TIMEOUT_MS, + onAnswered, + ); + change = { + standbyState: "standby", + standbyWakeAt: wakeAt === undefined ? null : new Date(wakeAt), + // It went as requested, so no earlier reason stands. + standbyReason: null, + }; + } else { + await standbyRouter.wakeAllocation( + command.target, + STANDBY_CONTROL_TIMEOUT_MS, + onAnswered, + ); + change = { standbyState: null }; + } + } catch (error) { + await turn?.ready; + const message = error instanceof Error ? error.message : String(error); + // Only the sidecar's refusal ends a request; the sweep repeats one that + // got no answer. One whose history the Hub does not hold is asked again + // however long that takes: history that never arrives means the Hub or + // the sidecar is broken, which giving up on the request cannot fix. + if (!(error instanceof WorkflowControlRejectedError)) { + logger.info`Workflow ${command.action} for ${command.runId} deferred: ${message}`; + deferStandbyCommand(command, error, message); + return; + } + logger.warn`Workflow ${command.action} for ${command.runId} was refused: ${message}`; + clearStandbyRetry(command); + await settleStandbyCommand(command, { + standbyState: null, + standbyReason: `The sidecar refused to ${command.action === "standby" ? "put the deployment in standby" : "wake the deployment"}: ${message}`, + }).catch((cause: unknown) => { + logger.error`Could not record the refused ${command.action} for ${command.runId}: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + return; + } + await turn?.ready; + // The request still stands, so the sweep asks again and records the + // answer the sidecar gives then. + await settleStandbyCommand(command, change) + .then(() => { + clearStandbyRetry(command); + }) + .catch((cause: unknown) => { + logger.error`Could not record the answered ${command.action} for ${command.runId}; the sweep asks again: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } finally { + turn?.done(); + } + } + + // The sweep leaves the command alone until its retry is due. History still on + // its way is the one failure nobody has to look at, so any other says why. + function deferStandbyCommand( + command: StandbyCommand, + error: unknown, + message: string, + ): void { + if (standbyCommands.get(command.runId) !== command) return; + const retry = standbyRetries.get(command.runId); + const previous = retry?.action === command.action ? retry : undefined; + const failures = (previous?.failures ?? 0) + 1; + standbyRetries.set(command.runId, { + action: command.action, + failures, + nextAt: + now().getTime() + + Math.min( + STANDBY_RETRY_MAX_MS, + STANDBY_RETRY_MIN_MS * 2 ** (failures - 1), + ), + note: + error instanceof WorkflowControlHistoryPendingError + ? (previous?.note ?? null) + : `The Hub could not ${command.action === "standby" ? "put the deployment in standby" : "wake the deployment"} yet and keeps asking${saysWhyStandbyWaits(error) ? `: ${message}` : ""}`, + }); + } + + // The note is public to anyone who can read the run. Any other failure, such + // as a failed query, can hold internals, so only the log carries its text. + function saysWhyStandbyWaits(error: unknown): boolean { + return ( + error instanceof WorkflowControlUnreachableError || + error instanceof WorkflowControlInitializingError || + error instanceof WorkflowControlUnconfirmedError || + error instanceof WorkflowControlTimeoutError + ); + } + + // A command a newer one replaced has no say in the retry of what came after. + function clearStandbyRetry(command: StandbyCommand): void { + if (standbyCommands.get(command.runId) === command) { + standbyRetries.delete(command.runId); + } + } + + function standbyRetryWaits(runId: string): boolean { + const retry = standbyRetries.get(runId); + return retry !== undefined && now().getTime() < retry.nextAt; + } + + // Why the request a run has pending got no answer yet, if this Hub knows. + function standbyRetryNote(run: Run): string | null { + const pending = + run.standbyState === "entering" + ? "standby" + : run.standbyState === "waking" + ? "wake" + : null; + const retry = standbyRetries.get(run.id); + return retry !== undefined && retry.action === pending ? retry.note : null; + } + + // Records an answer while the request it carried out still stands. + async function settleStandbyCommand( + command: StandbyCommand, + change: { + standbyState: WorkflowStandbyState | null; + standbyWakeAt?: Date | null; + standbyReason?: string | null; + }, + ): Promise { + await withReportingGeneration(command.target, (tx, runId) => + tx + .update(workflowRun) + .set({ standbyWakeAt: null, ...change, standbyStateAt: now() }) + .where( + and( + eq(workflowRun.id, runId), + eq( + workflowRun.standbyState, + command.action === "standby" ? "entering" : "waking", + ), + eq(workflowRun.standbyStateAt, command.stateAt), + ), + ), + ); + } + + // The sidecar's word on whether the child is in standby: each hello reports + // it, and a child in standby that an input woke says why. It takes its turn + // on arrival, so it is recorded after every answer received before it. The + // sidecar sends it once, so a write that fails keeps the turn for the sweep + // to retry. + async function noteDeploymentInStandby( + report: SidecarEventMap["deployment.standby"], + ): Promise { + const turn = takeObservationTurn(report); + try { + await recordDeploymentInStandby(report, turn.ready); + } catch (cause) { + // Held first in line, so its retry never waits on another report. + await turn.ready; + const failedAt = now().getTime(); + failedReports.set(`${report.allocationId}:${String(report.generation)}`, { + report, + firstFailedAt: failedAt, + failures: 1, + nextAt: failedAt + REPORT_RETRY_MIN_MS, + done: turn.done, + }); + logger.warn`Could not record the standby report of allocation ${report.allocationId}; the sweep retries it: ${cause instanceof Error ? cause.message : String(cause)}`; + return; + } + turn.done(); + } + + // Retries one failed standby report that is due. Returns whether it tried. + async function retryFailedReport(): Promise { + const at = now().getTime(); + const due = [...failedReports].find(([, failed]) => failed.nextAt <= at); + if (due === undefined) return false; + const [key, failed] = due; + failedReports.delete(key); + try { + await recordDeploymentInStandby(failed.report, Promise.resolve()); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + const failures = failed.failures + 1; + if (at - failed.firstFailedAt >= REPORT_RETRY_GIVE_UP_MS) { + logger.error`Gave up recording the standby report of allocation ${failed.report.allocationId} after ${String(failures)} attempts; the Hub's standby record may be wrong until its sidecar's next hello: ${message}`; + failed.done(); + return true; + } + failedReports.set(key, { + ...failed, + failures, + nextAt: + at + + Math.min( + REPORT_RETRY_MAX_MS, + REPORT_RETRY_MIN_MS * 2 ** (failures - 1), + ), + }); + return true; + } + failed.done(); + return true; + } + + async function recordDeploymentInStandby( + report: SidecarEventMap["deployment.standby"], + earlier: Promise, + ): Promise { + const [anchor] = await anchorOf(report); + if (anchor === undefined) return; + await earlier; + if (report.inStandby) { + // A child in standby that no standby request stands for is woken; a + // pending standby request is recorded from its command's answer. + let unrequested = false; + await withReportingGeneration(report, async (tx, runId) => { + const woken = await tx + .update(workflowRun) + .set({ + standbyState: "waking", + standbyStateAt: now(), + standbyWakeAt: null, + standbyReason: + "The sidecar reported the workflow child in standby, and no standby was asked for", + }) + .where( + and( + eq(workflowRun.id, runId), + isNull(workflowRun.standbyState), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + isNull(workflowRun.cancellationRequestedAt), + ), + ) + .returning({ id: workflowRun.id }); + unrequested = woken.length > 0; + if (unrequested) standbyRetries.delete(runId); + }); + if (unrequested) + logger.warn`Waking the workflow child of ${anchor.id}: its sidecar reports it in standby, and no standby is asked for`; + await driveStandbyState(anchor.id); + return; + } + // A running child ends a recorded standby, and an input that woke it + // also carried out a pending wake. A pending standby request stands: the + // sidecar carries it out next. + await withReportingGeneration(report, async (tx, runId) => { + await tx + .update(workflowRun) + .set({ + standbyState: null, + standbyStateAt: now(), + standbyWakeAt: null, + standbyReason: report.reason ?? REPORTED_RUNNING_REASON, + }) + .where( + and( + eq(workflowRun.id, runId), + report.reason === undefined + ? eq(workflowRun.standbyState, "standby") + : inArray(workflowRun.standbyState, ["standby", "waking"]), + ), + ); + // After a reconnect the `hello` can end the standby request before the + // wake report held across it arrives; that report still says what woke + // the child. + if (report.reason !== undefined) { + await tx + .update(workflowRun) + .set({ standbyReason: report.reason }) + .where( + and( + eq(workflowRun.id, runId), + isNull(workflowRun.standbyState), + eq(workflowRun.standbyReason, REPORTED_RUNNING_REASON), + ), + ); + } + }); + } + async function markStopped(tx: DBExecutor, run: Run): Promise { // Forced termination may leave only a partial event log. The Hub records // the outcome after the worker confirms its stop or the provisioner confirms @@ -841,8 +1446,14 @@ export function createWorkflowLifecycleService({ if ( retained?.retention !== "refused" && !(await needsLockedPass(tenantId, runId)) - ) + ) { + // A run that needs the locked pass is being cancelled, has lost its + // capacity, or has ended, so no sidecar could carry out a standby + // request for it. A pending projection also brings a run here, which + // must not cut its retry's wait short. + if (!standbyRetryWaits(runId)) await driveStandbyState(runId); return; + } const command = await withRun( tenantId, runId, @@ -1031,35 +1642,62 @@ export function createWorkflowLifecycleService({ // or history under backoff, remains eligible. Pause at the end of each pass // so the scheduler's immediate refill cannot repeatedly scan it in a loop. const sweep = createAnchorRunSweep( - createLifecycleSweepQueries(db, now, () => { - const candidates = retentionRouter.getRetentionCandidates(); - const current = new Map( - candidates.map((target) => [target.allocationId, target.generation]), - ); - for (const [id, backoff] of retentionBackoff) - if (current.get(id) !== backoff.generation) retentionBackoff.delete(id); - return candidates.filter((target) => { - // A confirmed refusal needs release, not a retention request. Other - // lifecycle branches still select expiry, cancellation and history work - // even when this retention branch has no request capacity. - if (retentionRouter.getAllocationRetention(target) === "refused") - return true; - if ( - retentionsInFlight.has(target.allocationId) || - !hasRetentionRoom(target.sidecarId) - ) - return false; - const backoff = retentionBackoff.get(target.allocationId); - return backoff === undefined || backoff.nextAt <= now().getTime(); - }); - }), + createLifecycleSweepQueries( + db, + now, + () => { + const candidates = retentionRouter.getRetentionCandidates(); + const current = new Map( + candidates.map((target) => [target.allocationId, target.generation]), + ); + for (const [id, backoff] of retentionBackoff) + if (current.get(id) !== backoff.generation) + retentionBackoff.delete(id); + return candidates.filter((target) => { + // A confirmed refusal needs release, not a retention request. Other + // lifecycle branches still select expiry, cancellation and history work + // even when this retention branch has no request capacity. + if (retentionRouter.getAllocationRetention(target) === "refused") + return true; + if ( + retentionsInFlight.has(target.allocationId) || + !hasRetentionRoom(target.sidecarId) + ) + return false; + const backoff = retentionBackoff.get(target.allocationId); + return backoff === undefined || backoff.nextAt <= now().getTime(); + }); + }, + standbyCommandsInFlight, + ), { intervalMs: SWEEP_INTERVAL_MS, now, }, ); + // A command whose binding is gone, or that a newer request replaced, does + // not hold its run back. + function standbyCommandsInFlight() { + const enterStandby: string[] = []; + const wake: string[] = []; + for (const [runId, { action, target }] of standbyCommands) { + if (!standbyRouter.holdsAllocatedBinding(target)) continue; + (action === "standby" ? enterStandby : wake).push(runId); + } + const at = now().getTime(); + for (const [runId, retry] of standbyRetries) { + if (at < retry.nextAt) { + (retry.action === "standby" ? enterStandby : wake).push(runId); + } else if (at - retry.nextAt > STANDBY_RETRY_FORGET_MS) { + standbyRetries.delete(runId); + } + } + return { enterStandby, wake }; + } + async function reconcileNext(): Promise { + if (await retryFailedReport()) return true; const run = await sweep.select(); if (run === null) return false; try { @@ -1077,6 +1715,11 @@ export function createWorkflowLifecycleService({ releaseCapacity, releaseStoppedCleanup, requestCancellation, + requestStandby: (tenantId: string, runId: string) => + requestStandbyState(tenantId, runId, "standby"), + requestWake: (tenantId: string, runId: string) => + requestStandbyState(tenantId, runId, "awake"), + noteDeploymentInStandby, reconcileNext, }; } @@ -1087,8 +1730,9 @@ export type WorkflowLifecycleService = ReturnType< /** * Selects the deployments `reconcileRun` can act on: a live run that is - * cancelling or past its expiry, terminal copies needing retention or - * release, a deferred infrastructure failure, and + * cancelling, past its expiry, or waiting on a standby or wake request its + * sidecar is not already answering, terminal + * copies needing retention or release, a deferred infrastructure failure, and * accepted history left unprojected past its grace. A new action there needs a * branch here. Deployments are never deleted, so each branch starts from an * index over rows still active in its own sense and reaches an anchor only by @@ -1106,6 +1750,10 @@ export function createLifecycleSweepQueries( db: DB["db"], now: () => Date, getRetentionCandidates: SidecarAllocationRouter["getRetentionCandidates"], + getStandbyCommandsInFlight: () => { + enterStandby: string[]; + wake: string[]; + }, ): AnchorRunSweepQueries<{ id: string; tenantId: string }> { // Selection runs once per candidate of every pass, so it is built once. function prepareBranches(afterCursor: boolean) { @@ -1219,6 +1867,8 @@ export function createLifecycleSweepQueries( workflowRun.expiresAt, sql.param(sql.placeholder("now"), workflowRun.expiresAt), ), + sql`(${workflowRun.standbyState} = 'entering' and ${workflowRun.id} <> all(${sql.placeholder("standbyInFlight")}::text[]))`, + sql`(${workflowRun.standbyState} = 'waking' and ${workflowRun.id} <> all(${sql.placeholder("wakesInFlight")}::text[]))`, ), inPass(workflowRun.id), ), @@ -1318,11 +1968,14 @@ export function createLifecycleSweepQueries( ); } const at = now(); + const inFlight = getStandbyCommandsInFlight(); const values = { passEnd, activeRunIds, retentionIds, retentionGenerations, + standbyInFlight: inFlight.enterStandby, + wakesInFlight: inFlight.wake, now: at, pendingCutoff: new Date(at.getTime() - PENDING_PROJECTION_GRACE_MS), }; diff --git a/packages/types/src/workflow-lifecycle.ts b/packages/types/src/workflow-lifecycle.ts index 49319f70a..8c3988d78 100644 --- a/packages/types/src/workflow-lifecycle.ts +++ b/packages/types/src/workflow-lifecycle.ts @@ -78,6 +78,9 @@ export type ResolvedWorkflowLifecyclePolicy = { >; }; +export const workflowStandbyStates = ["entering", "standby", "waking"] as const; +export type WorkflowStandbyState = (typeof workflowStandbyStates)[number]; + export const WorkflowLifecycleResponse = type({ runId: "string", status: "'deployed' | 'running' | 'completed' | 'failed' | 'cancelled'", @@ -87,6 +90,16 @@ export const WorkflowLifecycleResponse = type({ cancellationDeadline: "string | null", cancellationReason: "string | null", capacityReleaseAt: "string | null", + standby: type({ + state: type.enumerated("awake", ...workflowStandbyStates), + changedAt: "string | null", + wakeAt: "string | null", + reason: "string | null", + }) + .or("null") + .describe( + "Null once the run is terminal. `state` is `awake` while the workflow child runs and no standby is asked for, `entering` until the sidecar stops the child, `standby` once the Hub holds the history it left, and `waking` until the child runs again. `wakeAt` is the earliest timer a child in standby wakes for by itself. `reason` says why the state last changed other than as requested: the sidecar refused a standby or wake request, something other than a wake request woke the child in standby, such as mail or a restart that found new history, the sidecar reported the child running, it reported the child in standby though no standby was asked for, a cancellation ended the standby, or the deployment moved to replacement capacity. While a request is pending it says why the Hub could not carry it out yet.", + ), allocation: type({ id: "string", generation: "number", diff --git a/tests/db/sidecar-lifecycle-integration.test.ts b/tests/db/sidecar-lifecycle-integration.test.ts index 35e10acb2..b411cd809 100644 --- a/tests/db/sidecar-lifecycle-integration.test.ts +++ b/tests/db/sidecar-lifecycle-integration.test.ts @@ -47,7 +47,7 @@ import { } from "@intx/types/sidecar"; import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; -import { createMockWs } from "./sidecar-test-helpers"; +import { createMockWs, NO_STANDBY_ROUTER } from "./sidecar-test-helpers"; const NO_MANIFEST_DIGEST = await adapterManifestDigest(undefined); const TENANT = "tenant-lifecycle-integration"; @@ -191,6 +191,7 @@ describe.skipIf(!harnessDbEnvAvailable())( []; const lifecycle = createWorkflowLifecycleService({ db: h.db, + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { ...router, retainAllocation: (...args) => { diff --git a/tests/db/sidecar-test-helpers.ts b/tests/db/sidecar-test-helpers.ts index 37bf52cda..a2367797f 100644 --- a/tests/db/sidecar-test-helpers.ts +++ b/tests/db/sidecar-test-helpers.ts @@ -29,3 +29,11 @@ export function createMockWs(): WsHandle & { }, }; } + +// For a lifecycle service whose test asks for no standby: it holds no binding, +// so the service never sends a command. +export const NO_STANDBY_ROUTER = { + holdsAllocatedBinding: () => false, + putAllocationInStandby: () => Promise.reject(new Error("no standby here")), + wakeAllocation: () => Promise.reject(new Error("no standby here")), +}; diff --git a/tests/db/workflow-execution-admission.test.ts b/tests/db/workflow-execution-admission.test.ts index 691514e7b..0f27b7181 100644 --- a/tests/db/workflow-execution-admission.test.ts +++ b/tests/db/workflow-execution-admission.test.ts @@ -37,7 +37,7 @@ import { DEFAULT_SIDECAR_CAPACITY, HubFrame, } from "@intx/types/sidecar"; -import { createMockWs } from "./sidecar-test-helpers"; +import { createMockWs, NO_STANDBY_ROUTER } from "./sidecar-test-helpers"; const MANIFEST_DIGEST = "1".repeat(64); @@ -191,6 +191,7 @@ describe.skipIf(!harnessDbEnvAvailable())( try { await Promise.race([prepared.promise, deployment]); const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, @@ -311,6 +312,7 @@ describe.skipIf(!harnessDbEnvAvailable())( try { await Promise.race([sent.promise, result]); const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, @@ -462,6 +464,7 @@ describe.skipIf(!harnessDbEnvAvailable())( try { await preparing.promise; const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, diff --git a/tests/db/workflow-lifecycle-service.test.ts b/tests/db/workflow-lifecycle-service.test.ts index 5394cfa5f..395578484 100644 --- a/tests/db/workflow-lifecycle-service.test.ts +++ b/tests/db/workflow-lifecycle-service.test.ts @@ -49,8 +49,10 @@ import { createWorkflowLifecycleService, SidecarIdentityValidationError, SidecarNotConnectedError, + WorkflowControlHistoryPendingError, WorkflowControlInitializingError, WorkflowControlNotConnectedError, + WorkflowControlRejectedError, WorkflowControlTimeoutError, WorkflowControlUnconfirmedError, WorkflowControlUnreachableError, @@ -58,6 +60,7 @@ import { type WorkflowLifecycleServiceDeps, type WorkflowRunReader, } from "@intx/hub-sessions"; +import { NO_STANDBY_ROUTER } from "./sidecar-test-helpers"; const MANIFEST_DIGEST = "1".repeat(64); @@ -78,7 +81,12 @@ describe.skipIf(!harnessDbEnvAvailable())( options: Partial< Pick< WorkflowLifecycleServiceDeps, - "db" | "sendControl" | "historyReceives" | "cancelGraceMs" + | "db" + | "sendControl" + | "historyReceives" + | "cancelGraceMs" + | "standbyRouter" + | "onStandbyCommandSettled" > > & { runReader?: Partial; @@ -95,6 +103,7 @@ describe.skipIf(!harnessDbEnvAvailable())( }; return createWorkflowLifecycleService({ ...options, + standbyRouter: options.standbyRouter ?? NO_STANDBY_ROUTER, db: options.db ?? h.db, retentionRouter: { fenceAllocation: () => undefined, @@ -3969,5 +3978,1311 @@ describe.skipIf(!harnessDbEnvAvailable())( expect(await lifecycle.reconcileNext()).toBe(true); expect(await runStates()).toEqual({ [runId]: "failed" }); }); + + describe("standby", () => { + type StandbyCall = { + action: "standby" | "wake"; + target: { allocationId: string; generation: number }; + answer: PromiseWithResolvers<{ wakeAt?: string }>; + answered: () => void; + }; + + // The copy is deployed, so commands wait only for the sidecar's binding. + beforeEach(async () => { + await recordRunKey(runId); + }); + + // A lifecycle on a fake sidecar whose commands the test answers, and + // whose recorded answers it can wait for. + function standbyService( + opts: Partial< + Pick< + WorkflowLifecycleServiceDeps, + "db" | "sendControl" | "historyReceives" + > + > = {}, + ) { + const calls: StandbyCall[] = []; + const callWaiters: (() => void)[] = []; + let recorded = Promise.withResolvers(); + const binding = { held: true }; + const call = ( + action: StandbyCall["action"], + target: StandbyCall["target"], + answered: (() => void) | undefined, + ) => { + const answer = Promise.withResolvers<{ wakeAt?: string }>(); + calls.push({ + action, + target, + answer, + answered: answered ?? (() => undefined), + }); + for (const wake of callWaiters.splice(0)) wake(); + return answer.promise; + }; + const lifecycle = service({ + ...opts, + standbyRouter: { + putAllocationInStandby: (target, _timeoutMs, answered) => + call("standby", target, answered), + wakeAllocation: async (target, _timeoutMs, answered) => { + await call("wake", target, answered); + }, + holdsAllocatedBinding: () => binding.held, + }, + onStandbyCommandSettled: () => { + recorded.resolve(undefined); + }, + }); + return { + lifecycle, + calls, + binding, + actions: () => calls.map((sent) => sent.action), + async untilCalls(count: number) { + while (calls.length < count) { + await new Promise((resolve) => callWaiters.push(resolve)); + } + }, + // Settle a command and wait until the Hub has recorded its answer. + async answer(settle: () => void) { + recorded = Promise.withResolvers(); + settle(); + await recorded.promise; + }, + }; + } + + async function standby() { + return (await service().getStatus(tenantId, runId))?.standby; + } + + async function standbyOn(lifecycle: ReturnType) { + return (await lifecycle.getStatus(tenantId, runId))?.standby; + } + + // The database, with the read that picks a standby command + // answering late when asked: the query runs at once, and its rows come + // back once released, as a slow response would. + function lateDriveReads() { + const late = { + next: false, + reads: 0, + read: Promise.withResolvers(), + release: Promise.withResolvers(), + }; + const answeringLate = (builder: object): object => + new Proxy(builder, { + get(target, prop) { + const value: unknown = Reflect.get(target, prop, target); + if (typeof value !== "function") return value; + if (prop !== "then") { + return (...args: unknown[]) => { + const built: unknown = Reflect.apply(value, target, args); + return typeof built === "object" && built !== null + ? answeringLate(built) + : built; + }; + } + return ( + onFulfilled?: (rows: unknown) => unknown, + onRejected?: (cause: unknown) => unknown, + ) => { + late.reads += 1; + const held = late.next; + late.next = false; + const rows = new Promise((resolve, reject) => { + Reflect.apply(value, target, [resolve, reject]); + }); + const answered = held + ? rows.then((read) => { + late.read.resolve(undefined); + return late.release.promise.then(() => read); + }) + : rows; + return answered.then(onFulfilled, onRejected); + }; + }, + }); + const db = new Proxy(h.db, { + get(target, prop) { + const value: unknown = Reflect.get(target, prop, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + const built: unknown = Reflect.apply(value, target, args); + const fields: unknown = args[0]; + return prop === "select" && + typeof fields === "object" && + fields !== null && + "run" in fields && + "allocationStatus" in fields && + typeof built === "object" && + built !== null + ? answeringLate(built) + : built; + }; + }, + }); + return { late, db }; + } + + // The sweep pauses for its interval after each pass. + async function nextPass(lifecycle: ReturnType) { + current = new Date(current.getTime() + 1_000); + return lifecycle.reconcileNext(); + } + + async function setStandby( + state: "entering" | "standby" | "waking" | null, + ) { + await h.db + .update(workflowRun) + .set({ standbyState: state, standbyStateAt: current }) + .where(eq(workflowRun.id, runId)); + } + + async function beginReplacement() { + await h.db + .update(sidecarAllocation) + .set({ + reconciliationLeaseId: "lease", + reconciliationLeaseExpiresAt: new Date(Date.now() + 60_000), + }) + .where(eq(sidecarAllocation.id, allocationId)); + return createSidecarAllocationStore(h.db).beginReplacement({ + allocationId, + expectedStatus: "allocated", + expectedGeneration: 1, + expectedLeaseId: "lease", + firstAttemptAt: current, + nextAttemptAt: current, + failureCode: "sidecar_lost", + failureMessage: "The sidecar was lost", + now: current, + }); + } + + test("a standby request is asked of the sidecar once and records the wake time it answered", async () => { + const { lifecycle, calls, actions, answer } = standbyService(); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe("pending"); + expect(await standby()).toEqual({ + state: "entering", + changedAt: current.toISOString(), + wakeAt: null, + reason: null, + }); + + expect(await lifecycle.reconcileNext()).toBe(true); + expect(await nextPass(lifecycle)).toBe(false); + expect(actions()).toEqual(["standby"]); + expect(calls[0]?.target).toEqual({ allocationId, generation: 1 }); + + const wakeAt = new Date(current.getTime() + 60_000).toISOString(); + await answer(() => calls[0]?.answer.resolve({ wakeAt })); + expect(await standby()).toEqual({ + state: "standby", + changedAt: current.toISOString(), + wakeAt, + reason: null, + }); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe("settled"); + expect(await nextPass(lifecycle)).toBe(false); + expect(calls).toHaveLength(1); + }); + + test("a standby request and a wake do not wait for a pack receive holding the allocation row", async () => { + const { lifecycle } = standbyService(); + const locked = Promise.withResolvers(); + const release = Promise.withResolvers(); + const receive = h.db.transaction(async (tx) => { + await tx + .select() + .from(sidecarAllocation) + .where(eq(sidecarAllocation.id, allocationId)) + .for("update"); + locked.resolve(undefined); + await release.promise; + }); + try { + await Promise.race([ + locked.promise, + receive.then(() => { + throw new Error("Receive ended before locking"); + }), + ]); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe( + "pending", + ); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + } finally { + release.resolve(undefined); + await receive; + } + }); + + test("a cancellation clears the reason a standby request was refused", async () => { + const lifecycle = service(); + await h.db + .update(workflowRun) + .set({ + standbyStateAt: current, + standbyReason: + "The sidecar refused to put the deployment in standby", + }) + .where(eq(workflowRun.id, runId)); + + await lifecycle.requestCancellation(tenantId, runId, "Stop"); + expect((await standby())?.reason).toBeNull(); + }); + + test("a deployment that holds no sidecar capacity refuses a standby request and a wake, awake or in standby", async () => { + const { lifecycle, actions } = standbyService(); + await h.db + .update(sidecarAllocation) + .set({ status: "released" }) + .where(eq(sidecarAllocation.id, allocationId)); + + expect(await lifecycle.requestStandby(tenantId, runId)).toBe( + "unallocated", + ); + expect(await lifecycle.requestWake(tenantId, runId)).toBe( + "unallocated", + ); + await setStandby("standby"); + expect(await lifecycle.requestWake(tenantId, runId)).toBe( + "unallocated", + ); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe( + "unallocated", + ); + expect(await standby()).toEqual({ + state: "standby", + changedAt: current.toISOString(), + wakeAt: null, + reason: null, + }); + expect(await nextPass(lifecycle)).toBe(false); + expect(actions()).toEqual([]); + }); + + test("the sweep leaves a standby request its sidecar is answering, but still takes the run's expiry", async () => { + const controls: string[] = []; + const { lifecycle, actions, binding } = standbyService({ + sendControl: async (_target, command) => { + controls.push(command.action); + }, + }); + await lifecycle.requestStandby(tenantId, runId); + expect(await lifecycle.reconcileNext()).toBe(true); + expect(actions()).toEqual(["standby"]); + expect(await nextPass(lifecycle)).toBe(false); + + // Without its binding the sidecar answers nothing more. + binding.held = false; + expect(await nextPass(lifecycle)).toBe(true); + binding.held = true; + expect(await nextPass(lifecycle)).toBe(false); + + await h.db + .update(workflowRun) + .set({ expiresAt: current }) + .where(eq(workflowRun.id, runId)); + expect(await nextPass(lifecycle)).toBe(true); + expect(controls).toEqual(["cancel"]); + expect(actions()).toEqual(["standby"]); + // The expiry abandons the standby request in progress. + expect(await standby()).toMatchObject({ + state: "awake", + changedAt: current.toISOString(), + }); + }); + + test("the sweep leaves a wake its sidecar is answering", async () => { + await setStandby("standby"); + const { lifecycle, calls, actions, answer, binding } = standbyService(); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + expect(await lifecycle.reconcileNext()).toBe(true); + expect(actions()).toEqual(["wake"]); + expect(await nextPass(lifecycle)).toBe(false); + binding.held = false; + expect(await nextPass(lifecycle)).toBe(true); + binding.held = true; + expect(await nextPass(lifecycle)).toBe(false); + expect(actions()).toEqual(["wake"]); + await answer(() => calls[0]?.answer.resolve({})); + expect((await standby())?.state).toBe("awake"); + }); + + test.each([ + ["cancellation", "standby"], + ["cancellation", "waking"], + ["expiry", "standby"], + ["expiry", "waking"], + ] as const)( + "a %s of a run whose child is %s leaves it awake and sends no standby command", + async (ending, from) => { + const { lifecycle, actions } = standbyService({ + sendControl: async () => undefined, + }); + await setStandby(from); + if (ending === "cancellation") { + expect( + await lifecycle.requestCancellation(tenantId, runId, "Stop"), + ).toBe("pending"); + } else { + await h.db + .update(workflowRun) + .set({ expiresAt: current }) + .where(eq(workflowRun.id, runId)); + await nextPass(lifecycle); + } + const [run] = await h.db + .select({ cancelledAt: workflowRun.cancellationRequestedAt }) + .from(workflowRun) + .where(eq(workflowRun.id, runId)); + expect(run?.cancelledAt).not.toBeNull(); + expect((await standby())?.state).toBe("awake"); + await nextPass(lifecycle); + expect(actions()).toEqual([]); + }, + ); + + test("a standby request answered after cancellation is not recorded", async () => { + const controls: string[] = []; + const { lifecycle, calls, answer } = standbyService({ + sendControl: async (_target, command) => { + controls.push(command.action); + }, + }); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + expect(calls).toHaveLength(1); + expect( + await lifecycle.requestCancellation(tenantId, runId, "Stop"), + ).toBe("pending"); + await answer(() => + calls[0]?.answer.resolve({ + wakeAt: new Date(current.getTime() + 60_000).toISOString(), + }), + ); + expect(await standby()).toMatchObject({ + state: "awake", + wakeAt: null, + }); + expect(await nextPass(lifecycle)).toBe(true); + expect(controls).toHaveLength(1); + expect(calls).toHaveLength(1); + }); + + test("an old generation's standby answer after replacement is not recorded", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + expect(calls[0]?.target).toEqual({ allocationId, generation: 1 }); + expect((await beginReplacement())?.status).toBe("replacing"); + await answer(() => calls[0]?.answer.resolve({})); + expect((await standby())?.state).toBe("entering"); + }); + + test("a wake runs the child again and a repeated one is settled", async () => { + await setStandby("standby"); + const { lifecycle, calls, actions, answer } = standbyService(); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + expect((await standby())?.state).toBe("waking"); + await lifecycle.reconcileNext(); + expect(actions()).toEqual(["wake"]); + await answer(() => calls[0]?.answer.resolve({})); + expect((await standby())?.state).toBe("awake"); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("settled"); + }); + + test("a command waits for its sidecar's binding and goes out again once its connection closed, and a refusal records why", async () => { + const { lifecycle, calls, actions, answer, binding } = standbyService(); + binding.held = false; + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + expect(calls).toEqual([]); + + binding.held = true; + await nextPass(lifecycle); + expect(actions()).toEqual(["standby"]); + + binding.held = false; + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + await nextPass(lifecycle); + expect(actions()).toEqual(["standby"]); + binding.held = true; + await nextPass(lifecycle); + expect(actions()).toEqual(["standby", "standby"]); + + // A command that got no answer is asked again once its wait is up. + await answer(() => + calls[1]?.answer.reject( + new WorkflowControlTimeoutError("Workflow control timed out"), + ), + ); + await nextPass(lifecycle); + await nextPass(lifecycle); + expect(actions()).toEqual(["standby", "standby", "standby"]); + + await answer(() => + calls[2]?.answer.reject( + new WorkflowControlRejectedError( + "The deployment was not put in standby: the workflow run has ended", + ), + ), + ); + expect(await standby()).toEqual({ + state: "awake", + changedAt: current.toISOString(), + wakeAt: null, + reason: + "The sidecar refused to put the deployment in standby: The deployment was not put in standby: the workflow run has ended", + }); + }); + + test.each(["wake", "standby"] as const)( + "a wake reaches a standby request still in progress, when the %s's answer is recorded first", + async (first) => { + const { lifecycle, calls, actions, answer, untilCalls } = + standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + await untilCalls(2); + expect(actions()).toEqual(["standby", "wake"]); + + // The child stopped before the wake reached it. + const [enterStandby, wake] = [calls[0], calls[1]]; + for (const sent of first === "wake" + ? [wake, enterStandby] + : [enterStandby, wake]) + await answer(() => sent?.answer.resolve({})); + expect((await standby())?.state).toBe("awake"); + expect(await nextPass(lifecycle)).toBe(false); + expect(calls).toHaveLength(2); + }, + ); + + test("a wake after a standby request whose history was still arriving reaches the stopped child", async () => { + const { lifecycle, calls, actions, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlHistoryPendingError( + "refs/heads/events is at another commit on the Hub", + ), + ), + ); + expect((await standby())?.state).toBe("entering"); + + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + await nextPass(lifecycle); + expect(actions()).toEqual(["standby", "wake"]); + await answer(() => calls[1]?.answer.resolve({})); + expect((await standby())?.state).toBe("awake"); + }); + + test("a command that gets no answer goes out again more slowly, and says why until it is answered", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "entering", + reason: expect.stringContaining("The connection closed"), + }); + expect(await lifecycle.reconcileNext()).toBe(false); + expect(calls).toHaveLength(1); + + await nextPass(lifecycle); + expect(calls).toHaveLength(2); + await answer(() => + calls[1]?.answer.reject( + new WorkflowControlTimeoutError("Nothing came back"), + ), + ); + // The wait doubled. + expect(await nextPass(lifecycle)).toBe(false); + expect(calls).toHaveLength(2); + await nextPass(lifecycle); + expect(calls).toHaveLength(3); + + await answer(() => calls[2]?.answer.resolve({})); + expect(await standby()).toMatchObject({ + state: "standby", + reason: null, + }); + }); + + test("a standby request whose history is on its way waits more slowly without a reason", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlHistoryPendingError("still landing"), + ), + ); + expect(await standby()).toMatchObject({ + state: "entering", + reason: null, + }); + expect(await lifecycle.reconcileNext()).toBe(false); + expect(calls).toHaveLength(1); + await nextPass(lifecycle); + expect(calls).toHaveLength(2); + }); + + test("a pending projection does not cut a standby retry's wait short", async () => { + const historyReceives = createWorkflowHistoryReceiveTracker(); + const pending = await leavePending(); + historyReceives.begin(pending); + const { lifecycle, calls, answer } = standbyService({ + historyReceives, + }); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlHistoryPendingError("still landing"), + ), + ); + await nextPass(lifecycle); + await answer(() => + calls[1]?.answer.reject( + new WorkflowControlHistoryPendingError("still landing"), + ), + ); + // The second wait is two seconds. Each pass takes the run for its + // projection, and only the one after that wait asks again. + expect(await nextPass(lifecycle)).toBe(true); + expect(calls).toHaveLength(2); + expect(await nextPass(lifecycle)).toBe(true); + expect(calls).toHaveLength(3); + historyReceives.end(pending); + }); + + test("a standby request or a wake for a run whose accepted history ended is pending, and the sweep records its outcome", async () => { + const { lifecycle, calls } = standbyService(); + complete("RunCompleted"); + await leavePending(); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe("pending"); + await setStandby("standby"); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + expect(await lifecycle.reconcileNext()).toBe(true); + expect(calls).toHaveLength(0); + expect((await lifecycle.getStatus(tenantId, runId))?.status).toBe( + "completed", + ); + }); + + test("a failure that is not the sidecar's keeps its text out of the reason", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new Error('Failed query: select "secret" from "workflow_run"'), + ), + ); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "entering", + reason: + "The Hub could not put the deployment in standby yet and keeps asking", + }); + }); + + test("a retry's note stays on the Hub that retries, and a restart starts without it", async () => { + const before = standbyService(); + await before.lifecycle.requestStandby(tenantId, runId); + await before.lifecycle.reconcileNext(); + await before.answer(() => + before.calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect((await standbyOn(before.lifecycle))?.reason).toContain( + "The connection closed", + ); + + const after = standbyService(); + expect(await standbyOn(after.lifecycle)).toMatchObject({ + state: "entering", + reason: null, + }); + await after.lifecycle.reconcileNext(); + await after.answer(() => after.calls[0]?.answer.resolve({})); + expect(await standbyOn(after.lifecycle)).toMatchObject({ + state: "standby", + reason: null, + }); + }); + + test("an answered wake says nothing of the retry before it", async () => { + const { lifecycle, calls, answer } = standbyService(); + await setStandby("standby"); + await lifecycle.requestWake(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect((await standbyOn(lifecycle))?.reason).toContain( + "The connection closed", + ); + + await nextPass(lifecycle); + await answer(() => calls[1]?.answer.resolve({})); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "awake", + reason: null, + }); + }); + + test("a retry's note does not replace why the wake was needed", async () => { + const { lifecycle, calls, answer } = standbyService(); + await setStandby("waking"); + await h.db + .update(workflowRun) + .set({ standbyReason: "No standby was asked for" }) + .where(eq(workflowRun.id, runId)); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect((await standbyOn(lifecycle))?.reason).toContain( + "The connection closed", + ); + + await nextPass(lifecycle); + await answer(() => calls[1]?.answer.resolve({})); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "awake", + reason: "No standby was asked for", + }); + }); + + test("a next request starts without the note an earlier one left", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect((await standbyOn(lifecycle))?.reason).toContain( + "The connection closed", + ); + + await lifecycle.requestWake(tenantId, runId); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "waking", + reason: null, + }); + }); + + test("a wake the sidecar refuses ends the request and says why", async () => { + const { lifecycle, calls, actions, answer } = standbyService(); + await setStandby("standby"); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + expect(await lifecycle.reconcileNext()).toBe(true); + expect(actions()).toEqual(["wake"]); + + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlRejectedError( + "The deployment has no workflow child to wake", + ), + ), + ); + expect(await standby()).toEqual({ + state: "awake", + changedAt: current.toISOString(), + wakeAt: null, + reason: + "The sidecar refused to wake the deployment: The deployment has no workflow child to wake", + }); + expect(await nextPass(lifecycle)).toBe(false); + expect(calls).toHaveLength(1); + }); + + test("an answer to an earlier request does not settle a later one", async () => { + const { lifecycle, calls, actions, answer, untilCalls } = + standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await lifecycle.requestWake(tenantId, runId); + await untilCalls(2); + await answer(() => calls[1]?.answer.resolve({})); + current = new Date(current.getTime() + 1_000); + await lifecycle.requestStandby(tenantId, runId); + + await answer(() => calls[0]?.answer.resolve({})); + expect((await standby())?.state).toBe("entering"); + await nextPass(lifecycle); + expect(actions()).toEqual(["standby", "wake", "standby"]); + await answer(() => calls[2]?.answer.resolve({})); + expect((await standby())?.state).toBe("standby"); + }); + + test("a failure of an earlier request neither delays nor explains a later one", async () => { + const { lifecycle, calls, actions, answer, untilCalls } = + standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await lifecycle.requestWake(tenantId, runId); + await untilCalls(2); + await answer(() => calls[1]?.answer.resolve({})); + current = new Date(current.getTime() + 1_000); + await lifecycle.requestStandby(tenantId, runId); + + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "entering", + reason: null, + }); + await lifecycle.reconcileNext(); + await untilCalls(3); + expect(actions()).toEqual(["standby", "wake", "standby"]); + }); + + test("an answer to an earlier request leaves a later one's retry alone", async () => { + const { lifecycle, calls, answer, untilCalls } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await lifecycle.requestWake(tenantId, runId); + await untilCalls(2); + await answer(() => calls[1]?.answer.resolve({})); + current = new Date(current.getTime() + 1_000); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + await untilCalls(3); + await answer(() => + calls[2]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + + await answer(() => calls[0]?.answer.resolve({})); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "entering", + reason: expect.stringContaining("The connection closed"), + }); + }); + + test("a wake asked while an earlier drive's read answers late goes out after that drive decided", async () => { + const { late, db } = lateDriveReads(); + const { lifecycle, calls, actions, answer, untilCalls } = + standbyService({ db }); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + expect(actions()).toEqual(["standby"]); + + // A hello's drive read `entering` before the wake was asked for. + late.next = true; + const hello = lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: true, + }); + await late.read.promise; + const reads = late.reads; + expect(await lifecycle.requestWake(tenantId, runId)).toBe("pending"); + expect(late.reads).toBe(reads); + + late.release.resolve(undefined); + await hello; + await untilCalls(2); + expect(actions()).toEqual(["standby", "wake"]); + await answer(() => calls[1]?.answer.resolve({})); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlRejectedError( + "The deployment was not put in standby: the deployment was woken", + ), + ), + ); + expect((await standby())?.state).toBe("awake"); + expect(await nextPass(lifecycle)).toBe(false); + expect(calls).toHaveLength(2); + }); + + test("a report the sidecar sent after a standby request's answer is applied after it", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + + calls[0]?.answered(); + const woken = lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: false, + reason: "Woken by mail", + }); + await answer(() => calls[0]?.answer.resolve({})); + await woken; + expect(await standby()).toMatchObject({ + state: "awake", + reason: "Woken by mail", + }); + }); + + test("a hello received before a standby request's answer is recorded before it", async () => { + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + + // A hello finding the child running arrives, then the answer. + const hello = lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: false, + }); + calls[0]?.answered(); + await answer(() => calls[0]?.answer.resolve({})); + await hello; + expect((await standby())?.state).toBe("standby"); + }); + + test("an answer the Hub cannot record is asked for again", async () => { + const { outage, db } = failingTransactions(); + outage.failing = false; + const { lifecycle, calls, answer } = standbyService({ db }); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + outage.failing = true; + await answer(() => calls[0]?.answer.resolve({})); + expect((await standby())?.state).toBe("entering"); + outage.failing = false; + await nextPass(lifecycle); + expect(calls).toHaveLength(2); + await answer(() => calls[1]?.answer.resolve({})); + expect((await standby())?.state).toBe("standby"); + }); + + test("a refusal the Hub cannot record is asked for again", async () => { + const { outage, db } = failingTransactions(); + outage.failing = false; + const { lifecycle, calls, answer } = standbyService({ db }); + await lifecycle.requestStandby(tenantId, runId); + await lifecycle.reconcileNext(); + outage.failing = true; + await answer(() => + calls[0]?.answer.reject(new WorkflowControlRejectedError("no")), + ); + expect((await standby())?.state).toBe("entering"); + outage.failing = false; + await nextPass(lifecycle); + expect(calls).toHaveLength(2); + }); + + // The sidecar of a deployment in standby misses its connect deadline, and + // the classifier reads the committed log: a sleeping run and a run with no + // log yet answer `not-parked`, as `parked-run.test.ts` covers. + async function loseSidecar(verdict: "parked" | "not-parked" | "unknown") { + await setStandby("standby"); + await h.db + .update(sidecarAllocation) + .set({ + ensureAcceptedGeneration: 1, + connectDeadline: current, + nextAttemptAt: current, + }) + .where(eq(sidecarAllocation.id, allocationId)); + const allocations = createSidecarAllocationStore(h.db, { + getSidecarInventory: () => ({ + capacity: DEFAULT_SIDECAR_CAPACITY, + retainedIncarnations: [], + }), + }); + await allocations.wakeReconciliation(allocationId, 1); + const reconciler = createSidecarAllocationReconciler({ + allocationStore: allocations, + plugins: createSidecarPluginRegistry({ + provisioners: [ + { + id: "test", + apiVersion: 2, + bindingFingerprint: "test:1", + capabilities: [], + ensure: async () => ({ kind: "accepted" }), + destroy: async () => ({ + kind: "destroyed", + cleanup: "confirmed", + }), + }, + ], + }), + router: { + getCleanupConnection: () => undefined, + waitForSidecarInventory: async () => undefined, + undeployAllocation: async () => undefined, + fenceAllocation: () => undefined, + retireAllocation: () => undefined, + isAllocatedSidecarReady: async () => false, + holdsAllocatedBinding: () => false, + waitForAllocatedSidecar: async () => { + throw new Error("The sidecar never connected"); + }, + reportedDeploymentFailure: () => undefined, + syncSidecar: async () => undefined, + }, + classifyParkedRun: async () => ({ verdict, reason: "test" }), + hubWebSocketUrl: "ws://hub.example/ws", + now: () => current, + }); + expect(await reconciler.reconcileNext()).toBe(true); + return allocations.findById(allocationId); + } + + test("a deployment in standby whose runs are parked moves to replacement capacity when its sidecar is lost", async () => { + expect(await loseSidecar("parked")).toMatchObject({ + status: "replacing", + generation: 2, + }); + expect(await standby()).toMatchObject({ + state: "entering", + reason: expect.stringContaining("replacement capacity"), + }); + expect((await service().getStatus(tenantId, runId))?.status).toBe( + "running", + ); + }); + + test.each(["not-parked", "unknown"] as const)( + "a deployment in standby fails when its sidecar is lost and the classifier answers %s", + async (verdict) => { + expect((await loseSidecar(verdict))?.status).toBe("releasing"); + const state = await service().getStatus(tenantId, runId); + expect(state).toMatchObject({ + status: "failed", + standby: null, + allocation: { + failureCode: "sidecar_connect_failed", + failureMessage: "The sidecar never connected", + }, + }); + }, + ); + + test("replacement capacity ends a wake and the note its retry left", async () => { + const { lifecycle, calls, answer } = standbyService(); + await setStandby("standby"); + await lifecycle.requestWake(tenantId, runId); + await lifecycle.reconcileNext(); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + expect((await standbyOn(lifecycle))?.reason).toContain( + "The connection closed", + ); + + await beginReplacement(); + expect(await standbyOn(lifecycle)).toMatchObject({ + state: "awake", + reason: null, + }); + }); + + test("replacement capacity starts awake: a standby request still wanted is asked of it once it is deployed", async () => { + await setStandby("standby"); + const replacing = await beginReplacement(); + expect(replacing?.status).toBe("replacing"); + expect(await standby()).toMatchObject({ + state: "entering", + reason: expect.stringContaining("replacement capacity"), + }); + + const { lifecycle, calls, answer } = standbyService(); + await lifecycle.reconcileNext(); + expect(calls).toEqual([]); + // Allocated, and not yet deployed on the replacement. + await h.db + .update(sidecarAllocation) + .set({ status: "allocated", generation: 3 }) + .where(eq(sidecarAllocation.id, allocationId)); + await nextPass(lifecycle); + expect(calls).toEqual([]); + await seedRunPrincipalKey(h.db, tenantId, runId, "cd".repeat(32)); + await nextPass(lifecycle); + expect(calls.map((sent) => sent.target)).toEqual([ + { allocationId, generation: 3 }, + ]); + await answer(() => calls[0]?.answer.resolve({})); + expect(await standby()).toMatchObject({ + state: "standby", + reason: null, + }); + }); + + test("replacement capacity starts awake: a pending wake has nothing left to do", async () => { + await setStandby("waking"); + expect((await beginReplacement())?.status).toBe("replacing"); + expect((await standby())?.state).toBe("awake"); + }); + + test("cancellation ends a standby request, and a stopping or ended run refuses one", async () => { + const lifecycle = service(); + await lifecycle.requestStandby(tenantId, runId); + expect( + await lifecycle.requestCancellation(tenantId, runId, "Stop"), + ).toBe("pending"); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "The deployment is being cancelled", + }); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe( + "cancelling", + ); + expect(await lifecycle.requestWake(tenantId, runId)).toBe("cancelling"); + expect(await lifecycle.requestStandby("tnt_other", runId)).toBe( + "not_found", + ); + + await h.db + .update(workflowRun) + .set({ status: "completed", standbyState: "standby" }) + .where(eq(workflowRun.id, runId)); + expect(await lifecycle.requestStandby(tenantId, runId)).toBe( + "terminal", + ); + expect(await standby()).toBeNull(); + }); + + test("a hello reporting a child in standby wakes one that no standby request stands for, and answers a pending one", async () => { + const { lifecycle, calls, actions, answer } = standbyService(); + const report = { allocationId, generation: 1, inStandby: true }; + + // The wake stands until the sidecar answers it. + await lifecycle.noteDeploymentInStandby(report); + expect(await standby()).toMatchObject({ + state: "waking", + reason: + "The sidecar reported the workflow child in standby, and no standby was asked for", + }); + expect(actions()).toEqual(["wake"]); + await answer(() => + calls[0]?.answer.reject( + new WorkflowControlUnreachableError("The connection closed"), + ), + ); + await nextPass(lifecycle); + expect(actions()).toEqual(["wake", "wake"]); + await answer(() => calls[1]?.answer.resolve({})); + expect((await standby())?.state).toBe("awake"); + + await setStandby("entering"); + await lifecycle.noteDeploymentInStandby(report); + expect(actions()).toEqual(["wake", "wake", "standby"]); + await answer(() => calls[2]?.answer.resolve({})); + expect((await standby())?.state).toBe("standby"); + + await lifecycle.noteDeploymentInStandby(report); + expect(actions()).toEqual(["wake", "wake", "standby"]); + + await setStandby("waking"); + await lifecycle.noteDeploymentInStandby(report); + expect(actions()).toEqual(["wake", "wake", "standby", "wake"]); + }); + + // The cancellation's stop ends the child, standby or not. + test("a hello reporting a child in standby does not wake a run being cancelled", async () => { + const { lifecycle, actions } = standbyService(); + expect( + await lifecycle.requestCancellation(tenantId, runId, "Stop"), + ).toBe("pending"); + + await lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: true, + }); + expect(await standby()).toMatchObject({ state: "awake", reason: null }); + await nextPass(lifecycle); + expect(actions()).toEqual([]); + }); + + // The database, with every transaction failing while `failing` is set. + function failingTransactions() { + const outage = { failing: true, attempts: 0 }; + const db = new Proxy(h.db, { + get(target, prop) { + const value: unknown = Reflect.get(target, prop, target); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + if (prop === "transaction") { + outage.attempts += 1; + if (outage.failing) + return Promise.reject(new Error("the database is down")); + } + return Reflect.apply(value, target, args); + }; + }, + }); + return { outage, db }; + } + + const wokenByMail = { + allocationId, + generation: 1, + inStandby: false, + reason: "Woken by mail", + }; + + test("a standby report the Hub could not record is recorded once the sweep retries it", async () => { + const { outage, db } = failingTransactions(); + const lifecycle = service({ db }); + await setStandby("standby"); + + await lifecycle.noteDeploymentInStandby(wokenByMail); + expect((await standby())?.state).toBe("standby"); + + outage.failing = false; + await nextPass(lifecycle); + expect((await standby())?.state).toBe("standby"); + current = new Date(current.getTime() + 4_000); + expect(await lifecycle.reconcileNext()).toBe(true); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "Woken by mail", + }); + }); + + test("a report that arrives behind one the Hub could not record is recorded after it", async () => { + const { outage, db } = failingTransactions(); + const { lifecycle, actions } = standbyService({ db }); + await setStandby("standby"); + + await lifecycle.noteDeploymentInStandby(wokenByMail); + outage.failing = false; + // The child stopped again without a standby asked for, so applied in + // order the Hub wakes it. + const later = lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: true, + }); + expect((await standby())?.state).toBe("standby"); + + current = new Date(current.getTime() + 5_000); + expect(await lifecycle.reconcileNext()).toBe(true); + await later; + expect((await standby())?.state).toBe("waking"); + expect(actions()).toEqual(["wake"]); + }); + + test("a report the Hub cannot record for five minutes is given up, and later reports are recorded", async () => { + const { outage, db } = failingTransactions(); + const lifecycle = service({ db }); + await setStandby("standby"); + + await lifecycle.noteDeploymentInStandby(wokenByMail); + for (let minute = 1; minute <= 6; minute++) { + current = new Date(current.getTime() + 60_000); + await lifecycle.reconcileNext(); + } + const attempts = outage.attempts; + current = new Date(current.getTime() + 60_000); + await lifecycle.reconcileNext(); + expect(outage.attempts).toBe(attempts); + + outage.failing = false; + await lifecycle.noteDeploymentInStandby({ + ...wokenByMail, + reason: "Woken by signal", + }); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "Woken by signal", + }); + }); + + test("a hello finding the child running ends a recorded standby and leaves a pending request standing", async () => { + const lifecycle = service(); + const running = { allocationId, generation: 1, inStandby: false }; + + for (const pending of ["entering", "waking"] as const) { + await setStandby(pending); + await lifecycle.noteDeploymentInStandby(running); + expect((await standby())?.state).toBe(pending); + } + + await setStandby("standby"); + await lifecycle.noteDeploymentInStandby({ + ...running, + generation: 2, + }); + expect((await standby())?.state).toBe("standby"); + await lifecycle.noteDeploymentInStandby(running); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "The sidecar reported the workflow child running", + }); + }); + + test("a wake report held across a reconnect still names what woke the child", async () => { + const lifecycle = service(); + await setStandby("standby"); + await lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: false, + }); + await lifecycle.noteDeploymentInStandby({ + allocationId, + generation: 1, + inStandby: false, + reason: "Woken by timer", + }); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "Woken by timer", + }); + }); + + test("an input's report ends a recorded standby or a pending wake, and leaves a pending standby request standing", async () => { + const lifecycle = service(); + const woken = { + allocationId, + generation: 1, + inStandby: false, + reason: "Woken by mail", + }; + + await setStandby("entering"); + await lifecycle.noteDeploymentInStandby(woken); + expect((await standby())?.state).toBe("entering"); + + for (const ended of ["standby", "waking"] as const) { + await setStandby(ended); + await lifecycle.noteDeploymentInStandby(woken); + expect(await standby()).toMatchObject({ + state: "awake", + reason: "Woken by mail", + }); + } + }); + }); }, ); diff --git a/tests/db/workflow-run-terminal-flip.test.ts b/tests/db/workflow-run-terminal-flip.test.ts index e90f68f81..8c23b744c 100644 --- a/tests/db/workflow-run-terminal-flip.test.ts +++ b/tests/db/workflow-run-terminal-flip.test.ts @@ -59,6 +59,7 @@ import { } from "@intx/test-harness/seed"; import { applyWorkflowRunPack } from "../hub-agent/lib/deploy-flow-env"; +import { NO_STANDBY_ROUTER } from "./sidecar-test-helpers"; import { buildWorkflowRunLine, type WorkflowRunPackRun, @@ -324,6 +325,7 @@ describe.skipIf(!harnessDbEnvAvailable())( let current = observedAt; const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, @@ -1037,6 +1039,7 @@ describe.skipIf(!harnessDbEnvAvailable())( }); await h.db.update(sidecarAllocation).set({ sidecarId: "stop-worker" }); const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, @@ -1348,6 +1351,7 @@ describe.skipIf(!harnessDbEnvAvailable())( await Promise.race([reading.promise, result]); await enqueue("later-mail"); const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, @@ -1783,6 +1787,7 @@ describe.skipIf(!harnessDbEnvAvailable())( 1, ); const lifecycle = createWorkflowLifecycleService({ + standbyRouter: NO_STANDBY_ROUTER, retentionRouter: { fenceAllocation: () => undefined, retireAllocation: () => undefined, diff --git a/tests/workflow-deploy/drain-roundtrip.test.ts b/tests/workflow-deploy/drain-roundtrip.test.ts index 8e94e7031..f08bbfef2 100644 --- a/tests/workflow-deploy/drain-roundtrip.test.ts +++ b/tests/workflow-deploy/drain-roundtrip.test.ts @@ -41,7 +41,14 @@ import { deriveRunAddress, type ApprovalSet, } from "@intx/workflow-deploy"; -import { tenant as tenantTable } from "@intx/db/schema"; +import { + sidecar as sidecarTable, + sidecarAllocation, + tenant as tenantTable, +} from "@intx/db/schema"; +import { createInMemoryGrantStore } from "@intx/authz"; +import { createApp, type GetSession } from "@intx/hub-api"; +import type { GrantRule } from "@intx/types/authz"; import { createTestDb, harnessDbEnvAvailable, @@ -53,9 +60,14 @@ import { tenantSlugFromId, } from "@intx/test-harness/seed"; import { + createWorkflowHistoryReceiveTracker, + createWorkflowLifecycleService, + createWorkflowRunReader, WorkflowControlHistoryPendingError, WorkflowControlRejectedError, + type EventCollectorRegistry, type RepoId, + type SessionService, type SidecarEventMap, } from "@intx/hub-sessions"; @@ -65,6 +77,7 @@ import { deployWorkflowSourceForTest, fireMailTrigger, initiateDrain, + injectSignal, listRunIds, readClaimCheckDir, readWorkflowRunEvents, @@ -81,6 +94,7 @@ const WAIT_DEPLOYMENT_ID = "run_drain-roundtrip-wait-1"; const CONTROL_DEPLOYMENT_ID = "run_control-roundtrip"; const STOP_DEPLOYMENT_ID = "run_stop-roundtrip"; const STANDBY_DEPLOYMENT_ID = "run_standby-roundtrip"; +const ROUTE_STANDBY_DEPLOYMENT_ID = "run_standby-route-roundtrip"; const REFUSED_STANDBY_DEPLOYMENT_ID = "run_standby-refused-roundtrip"; // Longer than the test's budget, so the worker's acknowledgement, not the // clock, settles the control request. @@ -105,6 +119,7 @@ const DEFINITION_ASSET_IDS: Record = { [CONTROL_DEPLOYMENT_ID]: "ast_control_roundtrip", [STOP_DEPLOYMENT_ID]: "ast_stop_roundtrip", [STANDBY_DEPLOYMENT_ID]: "ast_standby_roundtrip", + [ROUTE_STANDBY_DEPLOYMENT_ID]: "ast_standby_route_roundtrip", [REFUSED_STANDBY_DEPLOYMENT_ID]: "ast_standby_refused_roundtrip", [DEPLOYMENT_ID]: "ast_drain_roundtrip_cancel_wf", [WAIT_DEPLOYMENT_ID]: "ast_drain_roundtrip_wait_wf", @@ -658,6 +673,118 @@ test.skipIf(!harnessDbEnvAvailable())( }, ); +// The routes record the request and the lifecycle sweep carries it out against +// the real child, through the same service the Hub composes. +test.skipIf(!harnessDbEnvAvailable())( + "the standby and wake routes reach a real child through the lifecycle sweep", + async () => { + const anchorRunId = ROUTE_STANDBY_DEPLOYMENT_ID; + const { address, handle } = await deployParkedOnSignal({ + anchorRunId, + definitionAssetId: "ast_standby_route_roundtrip", + workflowId: "standby-route-workflow", + }); + const target = env.hub.prepareAllocationIdentity(anchorRunId, address); + await seedAllocation(anchorRunId, target); + let commandSettled = Promise.withResolvers(); + // The sweep pauses between passes on this clock, which the test moves. + let nowMs = Date.now(); + const lifecycle = createWorkflowLifecycleService({ + db: h.db, + retentionRouter: env.hub.router, + runReader: createWorkflowRunReader(env.hub.agentRepoStore.repoStore), + historyReceives: createWorkflowHistoryReceiveTracker(), + sendControl: (control, command, timeoutMs) => + env.hub.router.sendWorkflowControl(control, command, timeoutMs), + standbyRouter: env.hub.router, + onStandbyCommandSettled: (runId) => { + if (runId === anchorRunId) commandSettled.resolve(undefined); + }, + now: () => new Date(nowMs), + }); + await seedPrincipal(h.db, { + id: OPERATOR_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + refId: OPERATOR_USER_ID, + status: "active", + }); + const operatorGrants: GrantRule[] = ["manage", "read"].map((action) => ({ + id: `grant-operator-${action}`, + resource: `workflow-run:${anchorRunId}`, + action, + effect: "allow", + origin: "system", + conditions: null, + expiresAt: null, + roleId: null, + principalId: OPERATOR_PRINCIPAL_ID, + })); + const app = createApp({ + getSession: operatorSession(), + authHandler: () => new Response("", { status: 404 }), + db: h.db, + grantStore: createInMemoryGrantStore(operatorGrants), + sidecarRouter: env.hub.router, + sessionService: unusedSessionService(), + eventCollectors: unusedEventCollectors(), + assetService: null, + repoStore: null, + maxTarballBytes: 10_000_000, + workflowLifecycleService: lifecycle, + }); + const runURL = `/api/tenants/${TENANT_ID}/workflows/runs/${anchorRunId}`; + const standbyState = async (): Promise => { + const status = await app.request(`${runURL}/lifecycle`); + const body: unknown = await status.json(); + return typeof body === "object" && + body !== null && + "standby" in body && + typeof body.standby === "object" && + body.standby !== null && + "state" in body.standby + ? body.standby.state + : undefined; + }; + // The state once the sweep has carried out the pending request. This run + // is the only one the sweep has work for, so a pass that selects a run + // sends its command, and sends it again once that settles unanswered: the + // sidecar's first answer can wait on history the child is still pushing. + const settled = async (): Promise => { + for (;;) { + const state = await standbyState(); + if (state !== "entering" && state !== "waking") return state; + commandSettled = Promise.withResolvers(); + if (await lifecycle.reconcileNext()) await commandSettled.promise; + // The sweep selects nothing while it pauses between passes. + else nowMs += 1_000; + } + }; + + expect( + (await app.request(`${runURL}/standby`, { method: "POST" })).status, + ).toBe(202); + expect(await settled()).toBe("standby"); + expect(await readInStandbyMark(handle)).toBeDefined(); + + expect( + (await app.request(`${runURL}/wake`, { method: "POST" })).status, + ).toBe(202); + expect(await settled()).toBe("awake"); + expect(await readInStandbyMark(handle)).toBeUndefined(); + + // The woken child still holds the run's park, so the signal completes it. + await injectSignal(env, anchorRunId, anchorRunId, "never-arrives", null); + const terminal = await waitForWorkflowRunComplete( + env, + anchorRunId, + anchorRunId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + }, +); + // Mail waiting behind the run would wake the child as soon as it stopped, so // the sidecar refuses the standby and the child keeps running. test.skipIf(!harnessDbEnvAvailable())( @@ -753,6 +880,88 @@ test.skipIf(!harnessDbEnvAvailable())( 30_000, ); +const OPERATOR_USER_ID = "usr_drain_roundtrip_operator"; +const OPERATOR_PRINCIPAL_ID = "prn_drain_roundtrip_operator"; +const MANIFEST_DIGEST = "1".repeat(64); + +// The allocation the fixture's router binds this deployment to, as the Hub +// records it when it places the deployment. +async function seedAllocation( + anchorRunId: string, + target: { allocationId: string; generation: number }, +): Promise { + const provisioner = { + provisionerId: "test", + provisionerApiVersion: 2 as const, + provisionerBindingFingerprint: "test:1", + adapterManifestDigest: MANIFEST_DIGEST, + }; + await h.db + .insert(sidecarTable) + .values({ + id: SIDECAR_ID, + tokenHashSha256: new Uint8Array(32), + ...provisioner, + }) + .onConflictDoNothing(); + await h.db.insert(sidecarAllocation).values({ + id: target.allocationId, + tenantId: TENANT_ID, + anchorRunId, + ...provisioner, + maxDisconnectedMs: 900_000, + sidecarId: SIDECAR_ID, + status: "allocated", + generation: target.generation, + ensureAcceptedGeneration: target.generation, + }); +} + +function operatorSession(): GetSession { + const now = new Date("2026-01-01"); + return async () => ({ + user: { + id: OPERATOR_USER_ID, + email: "operator@example.com", + emailVerified: true, + name: "Operator", + createdAt: now, + updatedAt: now, + }, + session: { + id: "session_operator", + userId: OPERATOR_USER_ID, + token: "tok_operator", + expiresAt: new Date("2999-01-01"), + createdAt: now, + updatedAt: now, + }, + }); +} + +function unused(name: string): never { + throw new Error(`drain round-trip: ${name} is not used`); +} + +function unusedSessionService(): SessionService { + return { + endSession: () => unused("endSession"), + }; +} + +function unusedEventCollectors(): EventCollectorRegistry { + return { + create: () => unused("create"), + dispatch: () => unused("dispatch"), + abandon: () => unused("abandon"), + has: () => false, + getStatus: () => undefined, + getAccumulatedText: () => undefined, + getCurrentTurnId: () => undefined, + getLastTurnId: () => undefined, + }; +} + // The mark the sidecar keeps in a deployment's run record while its child is // in standby. async function readInStandbyMark(handle: { diff --git a/tests/workflow-deploy/standby-helpers.ts b/tests/workflow-deploy/standby-helpers.ts index bf89e9017..9ade972a2 100644 --- a/tests/workflow-deploy/standby-helpers.ts +++ b/tests/workflow-deploy/standby-helpers.ts @@ -1,13 +1,27 @@ -// Helpers the standby tests share: put a deployment in standby over the link. +// Helpers the standby tests share: put a deployment in standby over the link, +// or drive the lifecycle service's sweep to it as the Hub does. -import { WorkflowControlHistoryPendingError } from "@intx/hub-sessions"; +import { sidecar as sidecarTable, sidecarAllocation } from "@intx/db/schema"; +import { + WorkflowControlHistoryPendingError, + createWorkflowHistoryReceiveTracker, + createWorkflowLifecycleService, + createWorkflowRunReader, +} from "@intx/hub-sessions"; +import type { TestDb } from "@intx/test-harness/db-harness"; -import { waitFor, type DeployFlowEnv } from "../hub-agent/lib/deploy-flow-env"; +import { + SIDECAR_ID, + waitFor, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; // Longer than any test's budget, so the worker's acknowledgement, not the // clock, settles a control request. export const CONTROL_TIMEOUT_MS = 300_000; +const MANIFEST_DIGEST = "1".repeat(64); + type AllocationTarget = { allocationId: string; generation: number }; // The standby attempt can overtake the child's last pushes; a repeated one @@ -29,3 +43,88 @@ export async function putInStandby( { diagnostics: env.sidecarDiagnostics }, ); } + +async function seedAllocation( + db: TestDb["db"], + ids: { tenantId: string; deploymentId: string }, + target: AllocationTarget, +): Promise { + const provisioner = { + provisionerId: "test", + provisionerApiVersion: 2 as const, + provisionerBindingFingerprint: "test:1", + adapterManifestDigest: MANIFEST_DIGEST, + }; + await db.insert(sidecarTable).values({ + id: SIDECAR_ID, + tokenHashSha256: new Uint8Array(32), + ...provisioner, + }); + await db.insert(sidecarAllocation).values({ + id: target.allocationId, + tenantId: ids.tenantId, + anchorRunId: ids.deploymentId, + ...provisioner, + maxDisconnectedMs: 900_000, + sidecarId: SIDECAR_ID, + status: "allocated", + generation: target.generation, + ensureAcceptedGeneration: target.generation, + }); +} + +// A lifecycle service for one deployment, whose allocation this records, and +// the sweep that carries out its standby and wake requests. +export async function startStandbySweep( + env: DeployFlowEnv, + db: TestDb["db"], + opts: { + tenantId: string; + deploymentId: string; + target: AllocationTarget; + }, +) { + const { tenantId, deploymentId, target } = opts; + await seedAllocation(db, { tenantId, deploymentId }, target); + let commandSettled = Promise.withResolvers(); + // The sweep pauses between passes on this clock, which the sweep moves. + let nowMs = Date.now(); + const lifecycle = createWorkflowLifecycleService({ + db, + retentionRouter: env.hub.router, + runReader: createWorkflowRunReader(env.hub.agentRepoStore.repoStore), + historyReceives: createWorkflowHistoryReceiveTracker(), + standbyRouter: env.hub.router, + onStandbyCommandSettled: (runId) => { + if (runId === deploymentId) commandSettled.resolve(undefined); + }, + now: () => new Date(nowMs), + }); + const standby = async () => + (await lifecycle.getStatus(tenantId, deploymentId))?.standby; + // The state once the sweep has carried out the pending request. This run is + // the only one the sweep has work for, so a pass that selects a run sends its + // command, and sends it again once that settles unanswered: the sidecar's + // first answer can wait on history the child is still pushing. + const settled = async () => { + for (;;) { + const current = await standby(); + if (current?.state !== "entering" && current?.state !== "waking") + return current; + commandSettled = Promise.withResolvers(); + if (await lifecycle.reconcileNext()) await commandSettled.promise; + // The sweep selects nothing while it pauses between passes. + else nowMs += 1_000; + } + }; + const untilInStandby = async (): Promise => { + const current = await settled(); + if (current === null || current === undefined) { + throw new Error("the run ended before standby"); + } + if (current.state !== "standby") { + throw new Error(`standby was refused: ${String(current.reason)}`); + } + }; + return { lifecycle, standby, settled, untilInStandby }; +} diff --git a/tests/workflow-deploy/standby-restart-hub-record.test.ts b/tests/workflow-deploy/standby-restart-hub-record.test.ts new file mode 100644 index 000000000..7cdb13ff1 --- /dev/null +++ b/tests/workflow-deploy/standby-restart-hub-record.test.ts @@ -0,0 +1,318 @@ +// The Hub's record of a standby it asked for survives a sidecar restart: the +// lifecycle service puts a run parked on a signal in standby, the sidecar is +// killed and restarted on its data dir, and the restore's report leaves the +// Hub recording the standby with no child running. A wake through the +// lifecycle service then runs the child again, and the signal completes the +// run with each step once. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import { type SidecarEventMap } from "@intx/hub-sessions"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { + createApprovalSet, + deriveRunAddress, + type ApprovalSet, +} from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + injectSignal, + listRunIds, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + startSidecarSubprocess, + terminateSidecarSubprocess, + waitFor, + waitForReconnect, + waitForWorkflowRunComplete, + type DeployFlowEnv, + type SidecarHandle, +} from "../hub-agent/lib/deploy-flow-env"; +import { startStandbySweep } from "./standby-helpers"; +import { signalGateEntry } from "./fixtures/signal-gate"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-restart-hub-record-1"; +const TENANT_ID = "tnt_standby_restart_hub_record"; +const CALLER_PRINCIPAL_ID = "prn_standby_restart_hub_record"; +const DEFINITION_ASSET_ID = "ast_standby_restart_hub_record_wf"; + +let env: DeployFlowEnv; +let h: TestDb; +let restartedSidecar: SidecarHandle | undefined; +const restartTempDirs: string[] = []; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-restart-hub-record-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (restartedSidecar !== undefined) { + await terminateSidecarSubprocess(restartedSidecar); + } + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); + for (const dir of restartTempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }); + } +}); + +// The allocation the fixture's router binds this deployment to, as the Hub +// records it when it places the deployment, so the lifecycle service can reach +// the deployment's sidecar. +describe.skipIf(!harnessDbEnvAvailable())( + "sidecar restart of a deployment the Hub put in standby", + () => { + test("the Hub still records the standby after the restore, and a wake it asks for resumes the run", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const approvals: ApprovalSet = createApprovalSet([ + "inference.source:anthropic:mock-model", + "director:@intx/agent/default", + `mail.address:${address}`, + `mail.send:${DEPLOYMENT_DOMAIN}`, + ]); + const handle = await deployWorkflowSourceForTest(env, { + entryModule: signalGateEntry({ + address, + signalName: "go", + systemPrompt1: "You are the first step agent.", + systemPrompt2: "You are the second step agent.", + agentId1: "agent-step1", + agentId2: "agent-step2", + workflowId: `wf_${DEPLOYMENT_ID}`, + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals, + config, + sources: { + step1: [inferenceSource], + gate: [inferenceSource], + step2: [inferenceSource], + }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + let runId: string | undefined; + await waitFor( + async () => { + for (const id of await listRunIds(env, handle.workflowRunRepoId)) { + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, id); + if ( + events.some( + (e) => + e.type === "SignalAwaited" && e.body["signalName"] === "go", + ) + ) + runId = id; + } + return runId !== undefined; + }, + { diagnostics: env.sidecarDiagnostics }, + ); + if (runId === undefined) throw new Error("no run reached the gate"); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + const { lifecycle, standby, settled } = await startStandbySweep( + env, + h.db, + { tenantId: TENANT_ID, deploymentId: DEPLOYMENT_ID, target }, + ); + const recordFile = path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ); + const standbyMark = async (): Promise => { + const record: unknown = JSON.parse( + await fs.promises.readFile(recordFile, "utf8"), + ); + return typeof record === "object" && + record !== null && + "standby" in record + ? record.standby + : undefined; + }; + + // Each report and the Hub's handling of it, wired as the Hub composes it. + const observed: { + report: SidecarEventMap["deployment.standby"]; + noted: Promise; + }[] = []; + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + const noted = lifecycle.noteDeploymentInStandby(report); + observed.push({ report, noted }); + return noted; + }, + ); + try { + expect(await lifecycle.requestStandby(TENANT_ID, DEPLOYMENT_ID)).toBe( + "pending", + ); + const recorded = await settled(); + expect(recorded).toEqual({ + state: "standby", + changedAt: expect.any(String), + wakeAt: null, + reason: null, + }); + expect(listWorkflowHostChildren(env)).toEqual([]); + expect(await standbyMark()).toEqual({ + parked: [], + tip: expect.any(String), + definition: expect.objectContaining({ id: `wf_${DEPLOYMENT_ID}` }), + }); + + // ---- kill the sidecar, and start a fresh one on its data dir ---- + const dataDir = env.sidecar.dataDir; + await terminateSidecarSubprocess(env.sidecar); + await waitFor( + () => !env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + const hubPort = env.hub.server.port; + if (hubPort === undefined) + throw new Error("hub.server.port is undefined after the kill"); + restartedSidecar = await startSidecarSubprocess({ + hubPort, + registerTempDir: (dir) => { + restartTempDirs.push(dir); + }, + extraEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + env.registerSidecar(restartedSidecar); + await waitForReconnect(env, address); + + // The Hub emits the hello's reports as it restores the routes, so + // the restore's report is in by now. + expect(observed.map(({ report }) => report)).toEqual([ + { ...target, inStandby: true }, + ]); + await Promise.all(observed.map(({ noted }) => noted)); + expect(await standby()).toEqual(recorded); + const restarted = { ...env, sidecar: restartedSidecar }; + expect(listWorkflowHostChildren(restarted)).toEqual([]); + expect(await standbyMark()).toBeDefined(); + + // ---- the Hub wakes the child, and the signal resumes the run ---- + expect(await lifecycle.requestWake(TENANT_ID, DEPLOYMENT_ID)).toBe( + "pending", + ); + expect(await settled()).toEqual({ + state: "awake", + changedAt: expect.any(String), + wakeAt: null, + reason: null, + }); + expect(await standbyMark()).toBeUndefined(); + + await injectSignal(env, DEPLOYMENT_ID, runId, "go", { resumed: true }); + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + { diagnostics: env.sidecarDiagnostics }, + ); + expect(terminal.type).toBe("RunCompleted"); + // A wake the Hub asked for, and a signal to a running child, report + // nothing. + expect(observed).toHaveLength(1); + } finally { + unsubscribe(); + } + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + const count = (match: (e: (typeof events)[number]) => boolean) => + events.filter(match).length; + expect(count((e) => e.type === "RunStarted")).toBe(1); + expect( + count( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "step1", + ), + ).toBe(1); + expect( + count( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "step2", + ), + ).toBe(1); + expect(count((e) => e.type === "RunCompleted")).toBe(1); + }, 180_000); + }, +); diff --git a/tests/workflow-deploy/standby-sleep-wake.test.ts b/tests/workflow-deploy/standby-sleep-wake.test.ts new file mode 100644 index 000000000..66136cafe --- /dev/null +++ b/tests/workflow-deploy/standby-sleep-wake.test.ts @@ -0,0 +1,205 @@ +// A due timer wakes a deployment in standby, and the sleep it belongs to ends. +// +// The run sleeps before its agent step. The Hub's lifecycle service puts the +// deployment in standby, which stops the child, and the sidecar wakes the child +// when the sleep's timer is due. The new child fires the timer and the run goes +// on to its agent step. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { startStandbySweep } from "./standby-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-sleep-wake"; +const TENANT_ID = "tnt_standby_sleep_wake"; +const CALLER_PRINCIPAL_ID = "prn_standby_sleep_wake"; +const DEFINITION_ASSET_ID = "ast_standby_sleep_wake_wf"; +// Long enough for standby to land before the timer is due, also when the +// test's lane runs other files alongside it. +const SLEEP_MS = 20_000; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-sleep-wake-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function sleepEntry(address: string): string { + return ` +import { defineWorkflow, sleep, step } from "@intx/workflow/definition"; +import { defineAgent } from "@intx/agent"; + +const agent = defineAgent({ + id: "settle-agent", + systemPrompt: "settle agent", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, +}); + +export const workflow = defineWorkflow({ + id: "wf_standby_sleep_wake", + trigger: { type: "mail", to: ${JSON.stringify(address)} }, + steps: { + hold: sleep({ duration: ${String(SLEEP_MS)} }), + settle: step({ agent, after: ["hold"] }), + }, +}); +`; +} + +// The allocation the fixture's router binds this deployment to, as the Hub +// records it when it places the deployment, so the lifecycle service can reach +// the deployment's sidecar. +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby on a sleep", + () => { + test("runs past the sleep once its due timer wakes the child", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: sleepEntry(address), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + sources: { hold: [inferenceSource], settle: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + await fireMailTrigger(env, address, { + messageId: "", + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + await waitFor( + async () => + (await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId)).some( + (e) => e.type === "TimerSet", + ), + { diagnostics: env.sidecarDiagnostics }, + ); + + const { lifecycle, untilInStandby } = await startStandbySweep(env, h.db, { + tenantId: TENANT_ID, + deploymentId: DEPLOYMENT_ID, + target: env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ), + }); + const wakeReasons: (string | undefined)[] = []; + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + wakeReasons.push(report.reason); + }, + ); + expect(await lifecycle.requestStandby(TENANT_ID, DEPLOYMENT_ID)).toBe( + "pending", + ); + await untilInStandby(); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + ); + expect(terminal.type).toBe("RunCompleted"); + await waitFor(() => wakeReasons.includes("Woken by timer"), { + diagnostics: env.sidecarDiagnostics, + }); + unsubscribe(); + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + expect(events.filter((e) => e.type === "TimerSet")).toHaveLength(1); + expect(events.filter((e) => e.type === "TimerFired")).toHaveLength(1); + expect( + events.some( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "settle", + ), + ).toBe(true); + }, 120_000); + }, +); diff --git a/tests/workflow-deploy/standby-timed-gate-wake.test.ts b/tests/workflow-deploy/standby-timed-gate-wake.test.ts new file mode 100644 index 000000000..71abfae56 --- /dev/null +++ b/tests/workflow-deploy/standby-timed-gate-wake.test.ts @@ -0,0 +1,208 @@ +// A due timer wakes a deployment in standby, and the timed gate it belongs to +// times out. +// +// The run parks on a timed `awaitSignal` gate. The Hub's lifecycle service puts +// the deployment in standby, which stops the child, and the sidecar wakes the +// child when the gate's timer is due. The new child's scheduler fires the timer +// while the resumed run is still on its way back to the gate, so the gate has to +// find that `TimerFired` in the log; the run then goes on to the gate's +// `onTimeout` step. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listWorkflowHostChildren, + readWorkflowRunEvents, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + waitForWorkflowRunComplete, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { startStandbySweep } from "./standby-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-timed-gate-wake"; +const TENANT_ID = "tnt_standby_timed_gate_wake"; +const CALLER_PRINCIPAL_ID = "prn_standby_timed_gate_wake"; +const DEFINITION_ASSET_ID = "ast_standby_timed_gate_wake_wf"; +// Long enough for standby to land before the timer is due, also when the +// test's lane runs other files alongside it. +const GATE_MS = 20_000; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-timed-gate-wake-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +function timedGateEntry(address: string): string { + return ` +import { awaitSignal, defineWorkflow, step } from "@intx/workflow/definition"; +import { defineAgent } from "@intx/agent"; + +const agent = defineAgent({ + id: "settle-agent", + systemPrompt: "settle agent", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, +}); + +export const workflow = defineWorkflow({ + id: "wf_standby_timed_gate_wake", + trigger: { type: "mail", to: ${JSON.stringify(address)} }, + steps: { + hold: awaitSignal({ name: "go", timeout: ${String(GATE_MS)}, onTimeout: "settle" }), + settle: step({ agent, after: ["hold"] }), + }, +}); +`; +} + +// The allocation the fixture's router binds this deployment to, as the Hub +// records it when it places the deployment, so the lifecycle service can reach +// the deployment's sidecar. +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby on a timed gate", + () => { + test("runs past the gate's timeout once its due timer wakes the child", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: timedGateEntry(address), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + sources: { hold: [inferenceSource], settle: [inferenceSource] }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + await fireMailTrigger(env, address, { + messageId: "", + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + await waitFor( + async () => + (await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId)).some( + (e) => e.type === "TimerSet", + ), + { diagnostics: env.sidecarDiagnostics }, + ); + + const { lifecycle, untilInStandby } = await startStandbySweep(env, h.db, { + tenantId: TENANT_ID, + deploymentId: DEPLOYMENT_ID, + target: env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ), + }); + const wakeReasons: (string | undefined)[] = []; + const unsubscribe = env.hub.router.events.on( + "deployment.standby", + (report) => { + wakeReasons.push(report.reason); + }, + ); + expect(await lifecycle.requestStandby(TENANT_ID, DEPLOYMENT_ID)).toBe( + "pending", + ); + await untilInStandby(); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const terminal = await waitForWorkflowRunComplete( + env, + DEPLOYMENT_ID, + runId, + ); + expect(terminal.type).toBe("RunCompleted"); + await waitFor(() => wakeReasons.includes("Woken by timer"), { + diagnostics: env.sidecarDiagnostics, + }); + unsubscribe(); + + const events = await readWorkflowRunEvents(env, DEPLOYMENT_ID, runId); + expect(events.filter((e) => e.type === "TimerSet")).toHaveLength(1); + expect(events.filter((e) => e.type === "TimerFired")).toHaveLength(1); + expect( + events.some( + (e) => e.type === "StepCompleted" && e.body["stepId"] === "settle", + ), + ).toBe(true); + }, 120_000); + }, +); diff --git a/tests/workflow-deploy/standby-timer-wake-at.test.ts b/tests/workflow-deploy/standby-timer-wake-at.test.ts new file mode 100644 index 000000000..3fa3f7999 --- /dev/null +++ b/tests/workflow-deploy/standby-timer-wake-at.test.ts @@ -0,0 +1,200 @@ +// A deployment in standby records the time its earliest timer fires, on the +// sidecar and on the Hub. +// +// The run sleeps for an hour before its agent step, so its only queued timer +// is the sleep's. The Hub's lifecycle service puts the deployment in standby, +// which stops the child. The sidecar's standby record and the Hub's stored +// standby then both carry the sleep's `fireAt` as the wake time. +// +// The test does not wait for the timer to wake the child: the sidecar +// subprocess arms that wake on its real clock and offers no seam to drive it. +// +// Harness justification: SPAWN-REAL: real hub, real sidecar subprocess, real +// workflow-process child, mock inference. + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { tenant as tenantTable } from "@intx/db/schema"; +import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; +import { deriveRunAddress } from "@intx/workflow-deploy"; +import { + createTestDb, + harnessDbEnvAvailable, + type TestDb, +} from "@intx/test-harness/db-harness"; +import { + seedAsset, + seedPrincipal, + tenantSlugFromId, +} from "@intx/test-harness/seed"; + +import { + SESSION_ID, + SIDECAR_ID, + deployWorkflowSourceForTest, + fireMailTrigger, + listWorkflowHostChildren, + startDeployFlowEnv, + waitFor, + waitForFirstRunId, + type DeployFlowEnv, +} from "../hub-agent/lib/deploy-flow-env"; +import { startStandbySweep } from "./standby-helpers"; +import { singleStepMailToolEntry } from "./fixtures/single-step-mail-tool"; +import { timerFireAt, waitForTimerSet } from "./sleep-restart-helpers"; + +const DEPLOYMENT_DOMAIN = "integration.interchange"; +const DEPLOYMENT_ID = "run_standby-timer-wake-at"; +const TENANT_ID = "tnt_standby_timer_wake_at"; +const CALLER_PRINCIPAL_ID = "prn_standby_timer_wake_at"; +const DEFINITION_ASSET_ID = "ast_standby_timer_wake_at_wf"; +const NAP_STEP_ID = "nap"; +const AGENT_STEP_ID = "step1"; +// Far past the test's budget, so the timer cannot fire while it runs. +const NAP_MS = 60 * 60 * 1000; + +let env: DeployFlowEnv; +let h: TestDb; + +beforeAll(async () => { + if (!harnessDbEnvAvailable()) return; + h = await createTestDb(); + await h.db.insert(tenantTable).values({ + id: TENANT_ID, + name: TENANT_ID, + slug: tenantSlugFromId(TENANT_ID), + domain: DEPLOYMENT_DOMAIN, + parentId: null, + }); + await seedPrincipal(h.db, { + id: CALLER_PRINCIPAL_ID, + tenantId: TENANT_ID, + kind: "user", + }); + await seedAsset(h.db, { + id: DEFINITION_ASSET_ID, + tenantId: TENANT_ID, + kind: "workflow", + name: "standby-timer-wake-at-wf", + creatorPrincipalId: CALLER_PRINCIPAL_ID, + }); + env = await startDeployFlowEnv(); +}); + +afterAll(async () => { + if (env !== undefined) await env.teardown(); + if (h !== undefined) await h.close(); +}); + +// The allocation the fixture's router binds this deployment to, as the Hub +// records it when it places the deployment, so the lifecycle service can reach +// the deployment's sidecar. +describe.skipIf(!harnessDbEnvAvailable())( + "a deployment in standby with a queued timer", + () => { + test("records the timer's fire time as its wake time on the sidecar and the Hub", async () => { + const address = deriveRunAddress({ + runId: DEPLOYMENT_ID, + domain: DEPLOYMENT_DOMAIN, + }); + const inferenceSource: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: `http://localhost:${String(env.inference.server.port)}`, + credentialId: "sk-mock", + model: "mock-model", + }; + const config: HarnessConfig = { + sessionId: SESSION_ID, + agentAddress: address, + sources: [inferenceSource], + defaultSource: "anthropic:mock-model", + }; + const handle = await deployWorkflowSourceForTest(env, { + entryModule: singleStepMailToolEntry({ + variant: "fs", + stepId: AGENT_STEP_ID, + systemPrompt: "You are the standby timer agent.", + address, + workflowId: `wf_${DEPLOYMENT_ID}`, + napMs: NAP_MS, + }), + db: h.db, + tenantId: TENANT_ID, + definitionAssetId: DEFINITION_ASSET_ID, + anchorRunId: DEPLOYMENT_ID, + deploymentDomain: DEPLOYMENT_DOMAIN, + agentAddress: address, + approvals: "approve-probed", + config, + // The deploy frame needs a source chain for every step, the sleep + // included. + sources: { + [NAP_STEP_ID]: [inferenceSource], + [AGENT_STEP_ID]: [inferenceSource], + }, + }); + await waitFor( + () => env.hub.router.getRoutableAddresses().includes(address), + { diagnostics: env.sidecarDiagnostics }, + ); + + await fireMailTrigger(env, address, { + messageId: "", + }); + const runId = await waitForFirstRunId(env, handle.workflowRunRepoId, { + diagnostics: env.sidecarDiagnostics, + }); + await waitForTimerSet(env, DEPLOYMENT_ID, runId, NAP_STEP_ID); + const wakeAt = new Date( + await timerFireAt(env, DEPLOYMENT_ID, runId, NAP_STEP_ID), + ).toISOString(); + + const target = env.hub.prepareAllocationIdentity( + DEPLOYMENT_ID, + address, + SIDECAR_ID, + ); + const { lifecycle, standby, untilInStandby } = await startStandbySweep( + env, + h.db, + { + tenantId: TENANT_ID, + deploymentId: DEPLOYMENT_ID, + target, + }, + ); + expect(await lifecycle.requestStandby(TENANT_ID, DEPLOYMENT_ID)).toBe( + "pending", + ); + await untilInStandby(); + expect(listWorkflowHostChildren(env)).toEqual([]); + + const record: unknown = JSON.parse( + await fs.promises.readFile( + path.join( + env.sidecar.dataDir, + "workflow-runs", + handle.workflowRunRepoId.id, + "deployment.json", + ), + "utf8", + ), + ); + expect( + typeof record === "object" && record !== null && "standby" in record + ? record.standby + : undefined, + ).toEqual({ + parked: [], + wakeAt, + tip: expect.any(String), + definition: expect.objectContaining({ id: `wf_${DEPLOYMENT_ID}` }), + }); + expect(await standby()).toMatchObject({ state: "standby", wakeAt }); + }); + }, +);