diff --git a/VENDORED.md b/VENDORED.md index 58dc233d96..ccb78ab08a 100644 --- a/VENDORED.md +++ b/VENDORED.md @@ -23,30 +23,31 @@ never a convenience. ## Ledger Pinned to [faremeter/interchange](https://github.com/faremeter/interchange) -@ `9febf69` (origin/main, v0.4.0). +@ `0a4eb59` (origin/main, v0.5.0). | Vendored path | What it is | Owner | Kill date | | ------------------------------ | ------------------------------------------------------------------------ | ------ | ---------- | -| `apps/sidecar` | Copy of upstream's sidecar app; apps are never npm-published. Three local deltas: `workflow-substrate-factory.ts` registers `@corbits/credential-header`'s and `@corbits/credential-mcp`'s providers alongside `builtinCredentialProviders()`; `bin/workflow-child` and `bin/workflow-probe-child` run under plain `bun` instead of upstream's `intx-src` resolve condition; `createSidecarStepBuildEnv` resolves the step's director registry from the closure package (`interchange.directors`) instead of the built-ins only, so a workflow's own director runs and not just probes (kill condition: upstream's step env loads the closure registry) | maintainers | 2026-11-03 | -| `vendor/intx/agent` | `@intx/agent` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/db` | `@intx/db`, with local fork migrations (wire_projection, launch origin, tolerate rewrite, tenant slug check), a serving-time credential-refresh seam, and an openai-responses catalog id | maintainers | 2026-11-03 | -| `vendor/intx/harness` | `@intx/harness` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/hub-agent` | `@intx/hub-agent`, with local childRunId event threading and an OAuth-login-over-websocket delta | maintainers | 2026-11-03 | -| `vendor/intx/hub-api` | `@intx/hub-api`, with the local workflow-deploy-bearer route and a serving-time credential-refresh hook | maintainers | 2026-11-03 | -| `vendor/intx/hub-sessions` | `@intx/hub-sessions`, with local childRunId event threading, serving-time credential refresh, loopback OAuth login, and prepared-deploy step staging | maintainers | 2026-11-03 | -| `vendor/intx/inference` | `@intx/inference`, with one local Google file-upload compile fix | maintainers | 2026-11-03 | -| `vendor/intx/mail-memory` | `@intx/mail-memory` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/mailbox` | `@intx/mailbox`, never yet published to npm | maintainers | 2026-11-03 | -| `vendor/intx/mime` | `@intx/mime` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/types` | `@intx/types`, with local childRunId, openai-responses plugin id, and OAuth-login token shapes | maintainers | 2026-11-03 | -| `vendor/intx/hub-common` | `@intx/hub-common` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/workflow` | `@intx/workflow`, with one local step-timeout-budget delta | maintainers | 2026-11-03 | -| `vendor/intx/workflow-deploy` | `@intx/workflow-deploy` at a newer commit than npm has published | maintainers | 2026-11-03 | -| `vendor/intx/workflow-host` | `@intx/workflow-host`, with local credential-wiring/childRunId threading, a step-grants-collapse delta, and a temporary delta omitting an empty inbound `Subject` header instead of passing it through empty (kill condition: upstream tolerates an empty Subject, INTR-577) | maintainers | 2026-11-03 | +| `apps/sidecar` | Copy of upstream's sidecar app; apps are never npm-published. Local delta: `bin/workflow-child` and `bin/workflow-probe-child` run under plain `bun` instead of upstream's `intx-src` resolve condition (the substrate-factory and deploy-router wiring moved to `@intx/workflow-host` upstream) | maintainers | 2026-11-17 | +| `vendor/intx/agent` | `@intx/agent` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/authz` | `@intx/authz` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/db` | `@intx/db`, with local fork migrations (wire_projection, launch origin, tolerate rewrite, tenant slug check, plus upstream 0095–0110 renumbered to 0100–0115) and an openai-responses catalog id | maintainers | 2026-11-17 | +| `vendor/intx/harness` | `@intx/harness` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/hub-agent` | `@intx/hub-agent`, with local childRunId event threading | maintainers | 2026-11-17 | +| `vendor/intx/hub-api` | `@intx/hub-api`, with the local workflow-deploy-bearer route | maintainers | 2026-11-17 | +| `vendor/intx/hub-sessions` | `@intx/hub-sessions`, with local childRunId event threading and prepared-deploy step staging | maintainers | 2026-11-17 | +| `vendor/intx/inference` | `@intx/inference`, with one local Google file-upload compile fix | maintainers | 2026-11-17 | +| `vendor/intx/mail-memory` | `@intx/mail-memory` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/mailbox` | `@intx/mailbox`, never yet published to npm | maintainers | 2026-11-17 | +| `vendor/intx/mime` | `@intx/mime` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/types` | `@intx/types`, with local childRunId, openai-responses plugin id, and `formatSafetyRatingText` re-export on `./runtime` | maintainers | 2026-11-17 | +| `vendor/intx/hub-common` | `@intx/hub-common` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/workflow` | `@intx/workflow`, with one local step-timeout-budget delta | maintainers | 2026-11-17 | +| `vendor/intx/workflow-deploy` | `@intx/workflow-deploy` at a newer commit than npm has published | maintainers | 2026-11-17 | +| `vendor/intx/workflow-host` | `@intx/workflow-host`, with local childRunId event threading, `@corbits/credential-header` and `@corbits/credential-mcp` provider registration + closure director loading in `child/substrate-factory.ts`, and a temporary delta omitting an empty inbound `Subject` header instead of passing it through empty (kill condition: upstream tolerates an empty Subject, INTR-577) | maintainers | 2026-11-17 | Every row above is re-vendored at the same commit rather than mixed pins -(npm's `0.4.0` is this commit; rows with no local delta are un-vendoring -candidates); the root +(npm's `0.5.0` is `d89f87c`; `0a4eb59` is `origin/main` HEAD; rows with no +local delta are un-vendoring candidates); the root `package.json` `overrides` point each vendored name at `workspace:*` so their own `@intx/*` dependencies resolve onto the vendored copies too. Local deltas are restated in each package's own `VENDORED-FROM` file. diff --git a/apps/hub/package.json b/apps/hub/package.json index 351a78db4a..817e8c18de 100644 --- a/apps/hub/package.json +++ b/apps/hub/package.json @@ -25,13 +25,13 @@ "@corbits/oauth-core": "^0.3.0", "@corbits/webhooks": "github:corbitsdev/webhooks#3f4f83147fd826d3aec5bcac7ef4b0ae7f75c00d", "@corbits/xai-provider": "0.1.2", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", diff --git a/apps/hub/src/provisioners/process.ts b/apps/hub/src/provisioners/process.ts index 5b7328c8ad..e0fe214257 100644 --- a/apps/hub/src/provisioners/process.ts +++ b/apps/hub/src/provisioners/process.ts @@ -426,7 +426,7 @@ function isErrnoCode(error: unknown, code: string): boolean { return error instanceof Error && "code" in error && error.code === code; } -const PROVISIONER_API_VERSION = 1 as const; +const PROVISIONER_API_VERSION = 2 as const; export const PROCESS_PROVISIONER_ID = "process"; diff --git a/apps/hub/src/provisioners/sandbox-sidecar.ts b/apps/hub/src/provisioners/sandbox-sidecar.ts index 932a1a46d7..c14854ec01 100644 --- a/apps/hub/src/provisioners/sandbox-sidecar.ts +++ b/apps/hub/src/provisioners/sandbox-sidecar.ts @@ -358,7 +358,7 @@ function isErrorCode(error: unknown, code: string): boolean { export type CreateSidecarProvisionerOpts = { readonly id: string; - readonly apiVersion: 1; + readonly apiVersion: 2; readonly bindingFingerprint: string; /** * Empty matches any deployment stating no requirement — the behaviour @@ -491,7 +491,7 @@ export function createSidecarProvisioner(opts: CreateSidecarProvisionerOpts): Si return rejected(...classify(error, "destroy_unit_failed")); } } - return { kind: "destroyed" }; + return { kind: "destroyed", cleanup: "confirmed" }; }, }; } diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index 8f723c8deb..9ca0f1c675 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -4,8 +4,10 @@ import { createPrincipalKeyStore, createSidecarAllocationStore, createWorkflowRunDispatchStore, + createWorkflowRunLaunchSpecStore, resolveFrameSenderKey, resolveSenderKey, + withExecutableWorkflowRun, } from "@intx/db"; import { asset as assetTable, @@ -14,7 +16,16 @@ import { } from "@intx/db/schema"; import { and, eq } from "drizzle-orm"; import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import { hexDecode, hexEncode, type SidecarCapabilityRule } from "@intx/types"; +import { + type AdapterManifest, + hexDecode, + hexEncode, + LifecycleDuration, + lifecycleDurationMs, + parseAdapterManifestEnv, + type ResolvedWorkflowLifecyclePolicy, + type SidecarCapabilityRule, +} from "@intx/types"; import { createApp, createAuth, @@ -23,6 +34,7 @@ import { type TenantEnv, } from "@intx/hub-api"; import { + attachSidecarProvisioners, createAgentRepoStore, createAssetService, createEventCollectorRegistry, @@ -36,6 +48,13 @@ import { createWorkflowAllocationService, createWorkflowDispatchService, createReconciliationScheduler, + createWorkflowLifecycleService, + createWorkflowDispatchProjection, + DEFAULT_WORKFLOW_PROJECTION_CONCURRENCY, + createWorkflowRunReader, + createParkedRunClassifier, + createWorkflowHistoryReceiveTracker, + recoverSenderDeploy, DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, pushCredentialReconcile, pushSourceUpdates, @@ -160,6 +179,8 @@ export type CreateHubServerOpts = { readonly sidecarAllocationConcurrency?: number; /** Deadline for provider calls, allocation claims, lease validation, and connection waits. Defaults to 120 seconds. */ readonly sidecarOperationTimeoutMs?: number; + /** Custom inference adapters every provisioned sidecar must load, forwarded on each ensure request. */ + readonly sidecarAdapterManifest?: AdapterManifest; }; export async function createHubServer({ @@ -170,6 +191,7 @@ export async function createHubServer({ probeSidecarCapabilityRules = [], sidecarAllocationConcurrency = DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, sidecarOperationTimeoutMs, + sidecarAdapterManifest, }: CreateHubServerOpts = {}) { await setup(); @@ -235,6 +257,44 @@ export async function createHubServer({ const hubSigningKey = await generateKeyPair(); log.info("Generated hub deploy signing key"); + function readOptionalPositiveIntEnv(name: string): number | undefined { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === "") return undefined; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer; got ${JSON.stringify(raw)}`); + } + return value; + } + + function readLifecycleDurationEnv(name: string, fallback: string): string { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === "") return fallback; + try { + return LifecycleDuration.assert(raw.trim()); + } catch (cause) { + throw new Error( + `${name} must be a lifecycle duration such as 30m or 7d; got ${JSON.stringify(raw)}`, + { cause }, + ); + } + } + + const defaultLifecyclePolicy: ResolvedWorkflowLifecyclePolicy = { + maxLifetime: readLifecycleDurationEnv("WORKFLOW_DEFAULT_MAX_LIFETIME", "7d"), + maxDisconnected: readLifecycleDurationEnv("WORKFLOW_DEFAULT_MAX_DISCONNECTED", "15m"), + capacityRetention: { + completed: readLifecycleDurationEnv("WORKFLOW_DEFAULT_RETENTION_COMPLETED", "30m"), + failed: readLifecycleDurationEnv("WORKFLOW_DEFAULT_RETENTION_FAILED", "24h"), + cancelled: readLifecycleDurationEnv("WORKFLOW_DEFAULT_RETENTION_CANCELLED", "1h"), + }, + }; + if (lifecycleDurationMs(defaultLifecyclePolicy.maxLifetime) === 0) + throw new Error("WORKFLOW_DEFAULT_MAX_LIFETIME must be greater than zero"); + if (lifecycleDurationMs(defaultLifecyclePolicy.maxDisconnected) === 0) { + throw new Error("WORKFLOW_DEFAULT_MAX_DISCONNECTED must be greater than zero"); + } + const agentRepoStore = createAgentRepoStore({ dataDir: hubDataDir, signingKey: hubSigningKey, @@ -250,8 +310,13 @@ export async function createHubServer({ const grantStore = createGrantStore(db); + const workflowHistoryReceives = createWorkflowHistoryReceiveTracker(); const lookups: SidecarLookups = { - ...createHubSessionLookups({ db, agentRepoStore }), + ...createHubSessionLookups({ + db, + agentRepoStore, + historyReceives: workflowHistoryReceives, + }), materializeMailTriggeredRunGrants: createMailTriggeredRunGrantsMaterializer({ db, principalKeyStore, @@ -263,12 +328,17 @@ export async function createHubServer({ }; const sidecarCredentials = createSidecarCredentialResolver({ db }); + const probeTimeoutMs = readOptionalPositiveIntEnv("HUB_PROBE_TIMEOUT_MS"); const sidecarRouter = createSidecarRouter({ hubPublicKey: hexEncode(hubSigningKey.publicKey), authenticateSidecar: async ({ token }) => sidecarCredentials.resolve(token), validateSidecarIdentity: sidecarCredentials.isCurrent, + withExecutableWorkflowRun: (target, send, signal) => + withExecutableWorkflowRun(db, target, send, signal), + resolveSidecarBindings: sidecarCredentials.resolveBindings, lookups, + ...(probeTimeoutMs !== undefined ? { probeTimeoutMs } : {}), }); lookups.resyncCredentials = (agentAddress) => { @@ -337,6 +407,8 @@ export async function createHubServer({ ? { chooser: probeSidecarProvisionerChooser } : {}), }); + const adapterManifest = + sidecarAdapterManifest ?? parseAdapterManifestEnv(process.env["SIDECAR_ADAPTER_MANIFEST"]); const workflowAllocationService = createWorkflowAllocationService({ db, deploymentPlugins: sidecarPlugins, @@ -346,11 +418,43 @@ export async function createHubServer({ probeCapabilityRules: probeSidecarCapabilityRules, allocationRouter: sidecarRouter, hubWebSocketUrl: hubSidecarWebSocketUrl, + defaultLifecyclePolicy, + ...(adapterManifest !== undefined ? { adapterManifest } : {}), ...(sidecarOperationTimeoutMs !== undefined ? { operationTimeoutMs: sidecarOperationTimeoutMs } : {}), }); - const sidecarAllocationStore = createSidecarAllocationStore(db); + const sidecarAllocationStore = createSidecarAllocationStore(db, { + getSidecarInventory: sidecarRouter.getSidecarInventory, + }); + const workflowRunReader = createWorkflowRunReader(agentRepoStore.repoStore); + const launchSpecs = createWorkflowRunLaunchSpecStore(db); + const classifyParkedRun = createParkedRunClassifier({ + runReader: workflowRunReader, + addressForAnchor: async (anchorRunId) => { + const row = await db.query.workflowRun.findFirst({ + where: (run, { eq: equals }) => equals(run.id, anchorRunId), + columns: { address: true }, + }); + return row?.address ?? null; + }, + projectionForAnchor: async (anchorRunId) => { + const spec = await launchSpecs.get(anchorRunId); + return spec?.frozenApprovalBundle.projection ?? null; + }, + }); + const workflowLifecycleService = createWorkflowLifecycleService({ + db, + retentionRouter: sidecarRouter, + runReader: workflowRunReader, + historyReceives: workflowHistoryReceives, + sendControl: (target, command, timeoutMs) => + sidecarRouter.sendWorkflowControl(target, command, timeoutMs), + }); + const dispatchProjection = createWorkflowDispatchProjection({ + db, + repoStore: agentRepoStore.repoStore, + }); const workflowDispatchService = createWorkflowDispatchService({ dispatchStore: createWorkflowRunDispatchStore(db), allocationStore: sidecarAllocationStore, @@ -365,12 +469,17 @@ export async function createHubServer({ }); const sidecarAllocationReconciler = createSidecarAllocationReconciler({ allocationStore: sidecarAllocationStore, + maxConcurrentClaims: sidecarAllocationConcurrency, plugins: sidecarPlugins, router: sidecarRouter, hubWebSocketUrl: hubSidecarWebSocketUrl, + ...(adapterManifest !== undefined ? { adapterManifest } : {}), ...(sidecarOperationTimeoutMs !== undefined ? { operationTimeoutMs: sidecarOperationTimeoutMs } : {}), + onInitializationRecovery: (allocation, reconciliation) => + recoverSenderDeploy({ db, sidecarRouter, allocation, reconciliation }), + classifyParkedRun, onReady: async (allocation, reconciliation) => { await workflowAllocationService.deployReadyAllocation(allocation, reconciliation); reconciliation.signal.throwIfAborted(); @@ -380,13 +489,33 @@ export async function createHubServer({ await workflowAllocationService.initialize?.(); await sidecarAllocationReconciler.initialize(); - sidecarRouter.events.on("sidecar.disconnect", ({ allocated }) => { - if (allocated === undefined) return; - return sidecarAllocationReconciler.handleDisconnect(allocated); + attachSidecarProvisioners({ + provisioners: [...deploymentProvisioners, ...probeProvisioners], + retireProvisionerSidecar: sidecarAllocationStore.retireProvisionerSidecar, + isConnected: sidecarRouter.holdsAllocatedBinding, + }); + sidecarRouter.events.on("sidecar.disconnect", async ({ allocated }) => { + await Promise.all( + allocated.map((target) => + sidecarAllocationReconciler.handleDisconnect(target).catch((err: unknown) => { + log.error( + "Failed to handle the disconnect of allocation {allocationId} generation {generation}: {error}", + { + allocationId: target.allocationId, + generation: target.generation, + error: err instanceof Error ? err.message : String(err), + }, + ); + }), + ), + ); }); sidecarRouter.events.on("sidecar.allocated.connected", (allocated) => sidecarAllocationReconciler.handleConnected(allocated), ); + sidecarRouter.events.on("deployment.stopped", (allocated) => + sidecarAllocationReconciler.handleConnected(allocated), + ); sidecarRouter.events.on("mail.inbound.acknowledged", ({ messageId, allocated }) => { if (allocated === undefined) return; return workflowDispatchService.acknowledge({ ...allocated, messageId }); @@ -406,11 +535,23 @@ export async function createHubServer({ return false; }, }); + const lifecycleScheduler = createReconciliationScheduler({ + name: "Workflow lifecycle", + reconcileNext: () => workflowLifecycleService.reconcileNext(), + }); const dispatchScheduler = createReconciliationScheduler({ name: "Workflow dispatch", concurrency: 1, reconcileNext: async () => { - workflowDispatchService.wake(); + await workflowDispatchService.reconcileUntilIdle(); + return false; + }, + }); + const dispatchProjectionScheduler = createReconciliationScheduler({ + name: "Workflow dispatch projection", + concurrency: DEFAULT_WORKFLOW_PROJECTION_CONCURRENCY, + reconcileNext: async () => { + await dispatchProjection.reconcileNext(); return false; }, }); @@ -426,6 +567,8 @@ export async function createHubServer({ allocationScheduler.start(); probeCleanupScheduler.start(); + lifecycleScheduler.start(); + dispatchProjectionScheduler.start(); dispatchScheduler.start(); connectionRepairScheduler.start(); @@ -440,6 +583,7 @@ export async function createHubServer({ sessionService, workflowAllocationService, workflowDispatchService, + workflowLifecycleService, eventCollectors, credentialCipher, principalKeyStore, @@ -486,7 +630,11 @@ export async function createHubServer({ upstream: createHubPersistMailWithSessionEnsure( db, eventCollectorsRef, - createHubSessionLookups({ db, agentRepoStore }).persistMail, + createHubSessionLookups({ + db, + agentRepoStore, + historyReceives: workflowHistoryReceives, + }).persistMail, ), authorizeSender: createHubMailboxAuthorizeSender(db), bus: mailboxBus, @@ -622,8 +770,20 @@ export async function createHubServer({ // package boundary; this just narrows them back to `sidecarRouter`'s own // types on the way through, with no behavior change. const systemTriggerMailRouter: HookMailRouter = { - routeMail: (address, rawMessage, authenticatedSender, messageId) => - sidecarRouter.routeMail(address, rawMessage, authenticatedSender, messageId), + routeMail: (address, rawMessage, authenticatedSender, messageId) => { + if (!sidecarRouter.getRoutableAddresses().includes(address)) { + return false; + } + void sidecarRouter + .routeMail(address, rawMessage, authenticatedSender, messageId) + .catch((error: unknown) => { + reportError(error, { + operation: "hub.systemTriggerMailRouter.routeMail", + extra: { address, authenticatedSender }, + }); + }); + return true; + }, sendRunGrants: (address, runId, stepGrants, senderIdentities) => sidecarRouter.sendRunGrants( address, @@ -865,6 +1025,8 @@ export async function createHubServer({ async shutdown(): Promise { allocationScheduler.stop(); probeCleanupScheduler.stop(); + lifecycleScheduler.stop(); + dispatchProjectionScheduler.stop(); dispatchScheduler.stop(); connectionRepairScheduler.stop(); cronTicker?.stop(); diff --git a/apps/hub/src/workflow-run-authenticator.ts b/apps/hub/src/workflow-run-authenticator.ts index 4340fb3e80..7caf0a0cf1 100644 --- a/apps/hub/src/workflow-run-authenticator.ts +++ b/apps/hub/src/workflow-run-authenticator.ts @@ -14,14 +14,20 @@ export function createWorkflowRunAuthenticator(deps: { db: DB["db"] }) { return { async resolve(token: string, runAddress: string) { if (token === "" || runAddress === "") return null; - const identity = await sidecars.resolve(token); - if (identity?.kind !== "allocated") return null; + const sidecar = await sidecars.resolve(token); + if (sidecar === null || sidecar.retired === true) return null; + const bindings = await sidecars.resolveBindings(sidecar.sidecarId); const run = await deps.db.query.workflowRun.findFirst({ where: eq(workflowRun.address, runAddress), }); if (run === undefined || run.principalId === null) return null; - if (run.tenantId !== identity.tenantId) return null; - if (run.anchorRunId !== identity.anchorRunId) return null; + const matches = bindings.some( + (identity) => + identity.kind === "allocated" && + identity.tenantId === run.tenantId && + identity.anchorRunId === run.anchorRunId, + ); + if (!matches) return null; return { tenantId: run.tenantId, principalId: run.principalId, diff --git a/apps/sidecar/bin/workflow-child b/apps/sidecar/bin/workflow-child index 6e732a63f6..166caffcc3 100755 --- a/apps/sidecar/bin/workflow-child +++ b/apps/sidecar/bin/workflow-child @@ -1,11 +1,13 @@ #!/usr/bin/env bun -// Plain bun: @intx/* resolves through published packages and the -// vendored source exports, never a resolve condition. -import { runWorkflowChildFromProcessEnv } from "@intx/workflow-host"; +// The intx-src condition resolves @intx/* to TypeScript source. The +// sidecar spawns this child by path (`Bun.spawn([binaryPath])`) and +// `--conditions` is not inherited, so it is set on the shebang. The dev +// loop builds no dist; a bundled deployment does not run this file. +import { runWorkflowChildFromProcessEnv } from "@intx/workflow-host/child"; import { SIDECAR_SUBSTRATE_CONFIG_KEYS, createSubstrate, -} from "../src/workflow-substrate-factory"; +} from "../src/workflow-child-bindings"; await runWorkflowChildFromProcessEnv(createSubstrate, { substrateConfigKeys: SIDECAR_SUBSTRATE_CONFIG_KEYS, diff --git a/apps/sidecar/bin/workflow-probe-child b/apps/sidecar/bin/workflow-probe-child index 04c1d65eb3..b2a6aeae77 100755 --- a/apps/sidecar/bin/workflow-probe-child +++ b/apps/sidecar/bin/workflow-probe-child @@ -1,6 +1,8 @@ #!/usr/bin/env bun -// One-shot workflow-probe child, spawned by path with plain bun so -// @intx/* resolves through published packages and vendored source. +// One-shot workflow-probe child. The sidecar host spawns this by path +// (`Bun.spawn([binaryPath])`); `--conditions` is not inherited, so it is +// set on the shebang to resolve @intx/* to TypeScript source in the dev +// loop. A bundled deployment does not run this file. // // The child reads the materialized package dir and IPC anchors from its // fresh env, evaluates the workflow entry behind the airlock, and ships @@ -8,10 +10,11 @@ // as an `ok: false` frame (handled inside the runner), so a throw reaching // here is a pre-evaluation defect (bad env, unwritable stdout) that exits // non-zero -- the host then reaps and answers `workflow.probe.error`. -import { runWorkflowProbeChildFromProcessEnv } from "../src/workflow-probe-handler"; +import { walkCapabilities } from "@intx/workflow-deploy/capabilities"; +import { runWorkflowProbeChildFromProcessEnv } from "@intx/workflow-host/probe/child"; try { - await runWorkflowProbeChildFromProcessEnv(); + await runWorkflowProbeChildFromProcessEnv(walkCapabilities); process.exit(0); } catch (err) { process.stderr.write( diff --git a/apps/sidecar/package.json b/apps/sidecar/package.json index 6b0566d7f0..5771f2a13d 100644 --- a/apps/sidecar/package.json +++ b/apps/sidecar/package.json @@ -1,6 +1,6 @@ { "name": "@workbench/sidecar", - "version": "0.4.0", + "version": "0.5.0", "private": true, "description": "Host process running a fleet of hub-orchestrated agent runtimes in one Bun process", "license": "SEE LICENSE IN ../../LICENSE.md", @@ -11,27 +11,24 @@ "scripts": { "start": "bun src/index.ts", "dev": "bun --watch src/index.ts", - "typecheck": "tsc --noEmit", - "test": "bun test" + "typecheck": "tsc --noEmit" }, "dependencies": { - "@corbits/credential-header": "^0.1.0", - "@corbits/credential-mcp": "^0.1.0", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", - "@intx/tools-lsp": "0.4.0", - "@intx/tools-mail": "0.4.0", - "@intx/tools-posix": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", + "@intx/tools-lsp": "0.5.0", + "@intx/tools-mail": "0.5.0", + "@intx/tools-posix": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/apps/sidecar/src/atomic-write.test.ts b/apps/sidecar/src/atomic-write.test.ts deleted file mode 100644 index 34fcfc8d08..0000000000 --- a/apps/sidecar/src/atomic-write.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { removeFileAtomicDurable, writeFileAtomicDurable } from "./atomic-write"; - -async function makeDir(): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), "atomic-write-")); -} - -async function listNames(dir: string): Promise { - return (await fs.readdir(dir)).sort(); -} - -describe("writeFileAtomicDurable", () => { - test("round-trips contents to the target path", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - - await writeFileAtomicDurable(file, '{"a":1}', { mode: 0o600 }); - expect(await fs.readFile(file, "utf8")).toBe('{"a":1}'); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("applies mode on the created file", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - - await writeFileAtomicDurable(file, "x", { mode: 0o600 }); - const stat = await fs.stat(file); - expect(stat.mode & 0o777).toBe(0o600); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("re-applies mode when overwriting an existing file", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - - // Seed a pre-existing file with a laxer mode. A plain in-place - // overwrite keeps the original mode; the temp+rename path creates a - // fresh file every write, so the requested mode actually lands. - await fs.writeFile(file, "old", { mode: 0o644 }); - await writeFileAtomicDurable(file, "new", { mode: 0o600 }); - - expect(await fs.readFile(file, "utf8")).toBe("new"); - const stat = await fs.stat(file); - expect(stat.mode & 0o777).toBe(0o600); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("leaves no temp orphan after a successful write", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - - await writeFileAtomicDurable(file, "x", { mode: 0o600 }); - expect(await listNames(dir)).toEqual(["record.json"]); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("a write that fails before staging leaves the prior file intact", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - await fs.writeFile(file, "old", { mode: 0o600 }); - - // Deny writes to the directory so the temp-file creation fails - // outright. The prior complete record must survive untouched -- an - // interrupted rotation must never corrupt the sole restore source. - await fs.chmod(dir, 0o500); - try { - await expect(writeFileAtomicDurable(file, "new", { mode: 0o600 })).rejects.toThrow(); - expect(await fs.readFile(file, "utf8")).toBe("old"); - expect(await listNames(dir)).toEqual(["record.json"]); - } finally { - await fs.chmod(dir, 0o700); - await fs.rm(dir, { recursive: true, force: true }); - } - }); - - test("a write that fails at the rename unlinks its staged temp", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - - // Occupy the target path with a directory so the rename fails after - // the temp file has already been staged and fsynced. This exercises - // the error-path unlink: the staged temp must be removed rather than - // stranded as an orphan. - await fs.mkdir(file); - await expect(writeFileAtomicDurable(file, "new", { mode: 0o600 })).rejects.toThrow(); - expect(await listNames(dir)).toEqual(["record.json"]); - - await fs.rm(dir, { recursive: true, force: true }); - }); -}); - -describe("removeFileAtomicDurable", () => { - test("removes an existing file", async () => { - const dir = await makeDir(); - const file = path.join(dir, "record.json"); - await fs.writeFile(file, "x"); - - await removeFileAtomicDurable(file); - expect(await listNames(dir)).toEqual([]); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("is a no-op for an already-absent file", async () => { - const dir = await makeDir(); - const file = path.join(dir, "never-existed.json"); - - // Idempotent: removing an absent file completes without throwing, so a - // re-driven eviction of an already-evicted key is not a fault. - await removeFileAtomicDurable(file); - expect(await listNames(dir)).toEqual([]); - - await fs.rm(dir, { recursive: true, force: true }); - }); - - test("leaves sibling files untouched", async () => { - const dir = await makeDir(); - const target = path.join(dir, "target.json"); - const sibling = path.join(dir, "keep.json"); - await fs.writeFile(target, "gone"); - await fs.writeFile(sibling, "stay"); - - await removeFileAtomicDurable(target); - expect(await listNames(dir)).toEqual(["keep.json"]); - expect(await fs.readFile(sibling, "utf8")).toBe("stay"); - - await fs.rm(dir, { recursive: true, force: true }); - }); -}); diff --git a/apps/sidecar/src/atomic-write.ts b/apps/sidecar/src/atomic-write.ts deleted file mode 100644 index 4f897eafab..0000000000 --- a/apps/sidecar/src/atomic-write.ts +++ /dev/null @@ -1,79 +0,0 @@ -// For a sole restore source that must survive a process kill or power -// loss without ever exposing a torn record — stronger than the cache's -// rebuildable temp+rename or fsyncWriteFile's non-atomic in-place write. - -import { open, rename, unlink } from "node:fs/promises"; -import { dirname } from "node:path"; - -import { getLogger } from "@intx/log"; -import { hasCode, hexEncode } from "@intx/types"; - -const logger = getLogger(["interchange", "sidecar", "atomic-write"]); - -export interface AtomicWriteOptions { - /** Permission mode applied when the temp file is created. */ - mode: number; -} - -/** - * fsync before rename extends atomicity past process death to power loss - * (without it, ext4 delayed allocation can surface a zero-length file). - * Directory fsync failure (FAT/exFAT, some network mounts) only degrades - * durability, so it's logged, not thrown. - */ -export async function writeFileAtomicDurable( - path: string, - contents: string, - options: AtomicWriteOptions, -): Promise { - const tmp = `${path}.tmp.${String(process.pid)}.${hexEncode(crypto.getRandomValues(new Uint8Array(8)))}`; - try { - const handle = await open(tmp, "w", options.mode); - try { - await handle.writeFile(contents); - await handle.sync(); - } finally { - await handle.close(); - } - await rename(tmp, path); - } catch (cause) { - // Best-effort cleanup: must not mask the original cause on a second failure. - await unlink(tmp).catch(() => undefined); - throw cause; - } - - try { - const dirHandle = await open(dirname(path), "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is renamed and fsynced — ${err instanceof Error ? err.message : String(err)}`; - } -} - -/** - * Fsyncs the parent dir after unlink so the removal survives a power loss — - * without it a resurrected file would re-stale a cache whose entry this is - * the restore source for. Idempotent: ENOENT on unlink is success. - */ -export async function removeFileAtomicDurable(path: string): Promise { - try { - await unlink(path); - } catch (err) { - if (!(hasCode(err) && err.code === "ENOENT")) throw err; - } - - try { - const dirHandle = await open(dirname(path), "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is unlinked — ${err instanceof Error ? err.message : String(err)}`; - } -} diff --git a/apps/sidecar/src/child-grant-filter.test.ts b/apps/sidecar/src/child-grant-filter.test.ts deleted file mode 100644 index c573e06845..0000000000 --- a/apps/sidecar/src/child-grant-filter.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -// The child-grant cap narrows a parent's grant rules to what a spawned child -// body declares, capping the child at the parent as the ceiling. The filter's -// safety rests on ONE direction being safe: it may over-keep an inert rule, but -// it must never drop a restriction or a rule the child legitimately needs. -// These cases pin that direction. - -import { describe, test, expect } from "bun:test"; - -import { defineAgent } from "@intx/agent"; -import { createDefaultDirectorRegistry } from "@intx/agent"; -import { evaluateGrants } from "@intx/authz"; -import type { GrantRule } from "@intx/authz"; -import { defineWorkflow, step } from "@intx/workflow"; - -import { collectDeclaredResources, filterGrantsToDeclaredResources } from "./child-grant-filter"; - -// A grant rule in the shape the per-run grants file carries. `effect` and -// `action` vary per case; the rest is boilerplate the filter never inspects. -function rule(resource: string, effect: GrantRule["effect"], action = "invoke"): GrantRule { - return { - id: `grant-${effect}-${resource}-${action}`, - resource, - action, - effect, - origin: "creator", - conditions: null, - expiresAt: null, - roleId: null, - principalId: null, - }; -} - -function resources(grants: readonly unknown[]): string[] { - return grants.map((g) => { - if (typeof g === "object" && g !== null && "resource" in g && typeof g.resource === "string") { - return g.resource; - } - throw new Error(`test grant has no string resource: ${JSON.stringify(g)}`); - }); -} - -describe("filterGrantsToDeclaredResources", () => { - test("keeps an allow whose exact resource is declared", () => { - const declared = new Set(["tool:foo"]); - const filtered = filterGrantsToDeclaredResources([rule("tool:foo", "allow")], declared); - expect(resources(filtered)).toEqual(["tool:foo"]); - }); - - test("keeps a wildcard allow that covers a declared resource", () => { - // The single grant that authorizes the most must survive: string equality - // would drop `tool:*`, but `matchPattern("tool:*", "tool:foo")` is true. - const declared = new Set(["tool:foo"]); - const filtered = filterGrantsToDeclaredResources([rule("tool:*", "allow")], declared); - expect(resources(filtered)).toEqual(["tool:*"]); - }); - - test("drops an allow for a resource the child never declares", () => { - // The escalation this cap closes: a parent-only effect/credential/tool the - // child body did not declare is removed, while a declared one survives. - const declared = new Set(["tool:foo"]); - const filtered = filterGrantsToDeclaredResources( - [ - rule("effect:secretCap", "allow"), - rule("credential:secretId", "allow"), - rule("tool:parentOnly", "allow"), - rule("tool:foo", "allow"), - ], - declared, - ); - expect(resources(filtered)).toEqual(["tool:foo"]); - }); - - test("keeps every deny and ask rule unconditionally", () => { - // deny/ask only ever restrict, so they survive even when their resource is - // undeclared -- dropping one would weaken safety. - const declared = new Set(["tool:foo"]); - const filtered = filterGrantsToDeclaredResources( - [rule("tool:bar", "deny"), rule("effect:whatever", "ask"), rule("tool:bar", "allow")], - declared, - ); - expect(resources(filtered)).toEqual(["tool:bar", "effect:whatever"]); - }); - - test("keeps an ask floor so the approval gate survives the cap", async () => { - // A tool's static approval mark rides as an `ask` grant. evaluateGrants - // ranks ask above allow at equal specificity so a workflow cannot declare - // its way under the gate. The filter must keep the ask even alongside an - // allow for the same declared resource, or the child punches through. - const declared = new Set(["tool:foo"]); - const filtered = filterGrantsToDeclaredResources( - [rule("tool:foo", "ask"), rule("tool:foo", "allow")], - declared, - ); - const decision = await evaluateGrants( - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the filter returns unknown[]; the seeded rows are GrantRule by construction here - [...(filtered as readonly GrantRule[])], - "tool:foo", - "invoke", - ); - expect(decision.effect).toBe("ask"); - }); - - test("keeps entries that are not allow rules with a string resource", () => { - // Keeping-on-doubt is safe: a non-rule entry is inert at evaluation - // (evaluateGrants reads `.resource`), so it never widens authority, and the - // filter can never wrongly drop a restriction it failed to classify. - const declared = new Set(); - const notRules: unknown[] = [ - { effect: "allow" }, // no resource - { resource: "tool:foo" }, // no effect - "not-an-object", - null, - 42, - ]; - const filtered = filterGrantsToDeclaredResources(notRules, declared); - expect(filtered).toEqual(notRules); - }); - - test("drops a provable allow while keeping unclassifiable entries", () => { - const declared = new Set(["tool:foo"]); - const kept = { resource: "tool:foo" }; // no effect -> kept - const dropped = rule("tool:parentOnly", "allow"); // undeclared allow -> gone - const filtered = filterGrantsToDeclaredResources( - [kept, dropped, rule("tool:foo", "allow")], - declared, - ); - expect(filtered).toEqual([kept, rule("tool:foo", "allow")]); - }); -}); - -describe("collectDeclaredResources", () => { - test("unions the declared resources across every step", () => { - // Two steps, two distinct inference sources: the union carries both, so a - // parent grant for either survives the filter. - const workflow = defineWorkflow({ - id: "child-wf", - trigger: { type: "manual" }, - steps: { - a: step({ - agent: defineAgent({ - id: "a", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m1" }] }, - }), - }), - b: step({ - agent: defineAgent({ - id: "b", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m2" }] }, - }), - }), - }, - }); - const declared = collectDeclaredResources(workflow, createDefaultDirectorRegistry(), new Map()); - expect(declared.has("inference.source:anthropic:m1")).toBe(true); - expect(declared.has("inference.source:anthropic:m2")).toBe(true); - }); -}); diff --git a/apps/sidecar/src/child-grant-filter.ts b/apps/sidecar/src/child-grant-filter.ts deleted file mode 100644 index 6831d2457f..0000000000 --- a/apps/sidecar/src/child-grant-filter.ts +++ /dev/null @@ -1,60 +0,0 @@ -// Caps a spawned child's inherited grants at what the child body itself -// declares, so a parent-held effect:/credential: grant or a bare-name tool: -// collision never authorizes a capability the child was never written to use. - -import type { DirectorRegistry } from "@intx/agent"; -import { matchPattern } from "@intx/authz"; -import type { WorkflowDefinition } from "@intx/workflow"; -import { walkCapabilities, type PluginToolDefinitions } from "@intx/workflow-deploy"; - -/** Pass the pre-rewrite definition: the walk skips a { ref } body, so a rewritten one would drop grandchild resources. */ -export function collectDeclaredResources( - definition: WorkflowDefinition, - directors: DirectorRegistry, - pluginDefs: PluginToolDefinitions, -): ReadonlySet { - const walk = walkCapabilities(definition, directors, pluginDefs); - const declared = new Set(); - for (const step of walk.perStep.values()) { - for (const grant of step.grants) { - declared.add(grant); - } - } - return declared; -} - -/** - * Only ever removes parent rules, erring toward keeping since dropping is the - * only unsafe direction: deny/ask are kept unconditionally (dropping an ask - * floor would punch through an approval gate), an allow is kept only if its - * pattern covers a declared resource, and action is deliberately not part of - * the coverage test since evaluateGrants re-gates on it at decision time. - */ -export function filterGrantsToDeclaredResources( - parentGrants: readonly unknown[], - declared: ReadonlySet, -): readonly unknown[] { - return parentGrants.filter((grant) => keepGrantForDeclared(grant, declared)); -} - -function keepGrantForDeclared(grant: unknown, declared: ReadonlySet): boolean { - if (!isAllowRuleWithResource(grant)) { - return true; - } - for (const resource of declared) { - if (matchPattern(grant.resource, resource)) { - return true; - } - } - return false; -} - -function isAllowRuleWithResource(grant: unknown): grant is { effect: "allow"; resource: string } { - if (typeof grant !== "object" || grant === null) { - return false; - } - if (!("effect" in grant) || !("resource" in grant)) { - return false; - } - return grant.effect === "allow" && typeof grant.resource === "string"; -} diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index 79a9f47ef1..ff4819958c 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -8,11 +8,34 @@ // resolved value through `DefaultHarnessBuilderConfig` rather than // re-reading env, so the boundary stays at the boot edge. -import { AdapterManifest } from "@intx/inference"; -import { hexDecode } from "@intx/types"; +import { type AdapterManifest, parseAdapterManifestEnv } from "@intx/types/adapter-manifest"; +import { hexDecode } from "@intx/types/hex"; +import { DEFAULT_SIDECAR_CAPACITY, SidecarCapacity } from "@intx/types/sidecar"; +import { type } from "arktype"; const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; +export function parseDeploymentCapacity( + active: string | undefined, + retained: string | undefined, +): SidecarCapacity { + const capacity = SidecarCapacity({ + maxActiveDeployments: + active === undefined || active.trim() === "" + ? DEFAULT_SIDECAR_CAPACITY.maxActiveDeployments + : Number(active), + maxRetainedDeployments: + retained === undefined || retained.trim() === "" + ? DEFAULT_SIDECAR_CAPACITY.maxRetainedDeployments + : Number(retained), + }); + if (capacity instanceof type.errors) + throw new Error( + `Invalid SIDECAR_MAX_ACTIVE_DEPLOYMENTS or SIDECAR_MAX_RETAINED_DEPLOYMENTS: ${capacity.summary}`, + ); + return capacity; +} + // The operator key the sidecar seals its at-rest credential material under // (inference-source apiKeys, and the tool credential material store). A // SEPARATE key from the hub's `CREDENTIAL_ENCRYPTION_KEY`: the sidecar seals a @@ -92,22 +115,13 @@ export function parseReconnectDelayMs(raw: string | undefined): number | undefin // `import(specifier)` is arbitrary code execution, so a specifier must // never originate from deploy or tenant data — the agent deploy tree // carries only a `provider` key, never a specifier. The shape is -// arktype-validated here (and re-validated at the workflow-child spawn -// boundary as defense in depth), but the value itself is trusted. +// validated by the shared reader (and re-validated at the workflow-child +// spawn boundary as defense in depth), but the value itself is trusted. // // Unset or whitespace-only means "no custom adapters", a valid // configuration — the sidecar then resolves only the statically-linked -// built-ins. A present-but-malformed value fails loud at boot: an -// opaque `JSON.parse` SyntaxError is rethrown naming the env key, and -// the parsed value is asserted against `AdapterManifest`. +// built-ins, and the empty manifest is what the workflow child receives. +// A present-but-malformed value fails loud at boot. export function readAdapterManifest(): AdapterManifest { - const raw = process.env["SIDECAR_ADAPTER_MANIFEST"]; - if (raw === undefined || raw.trim() === "") return []; - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new Error("SIDECAR_ADAPTER_MANIFEST is not valid JSON", { cause }); - } - return AdapterManifest.assert(parsed); + return parseAdapterManifestEnv(process.env["SIDECAR_ADAPTER_MANIFEST"]) ?? []; } diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index f13ba28c03..9698ea98f0 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -14,16 +14,27 @@ import { createInboundMailPolicyRegistry, createInboundMailPolicyLookup, createSidecarOrchestrator, + isConnectionLost, + MAX_INLINE_ASSET_PAYLOAD_BYTES, + materializeWorkflowAssets, + resolveInboundMailPolicy, + sourceAssetGitDir, type HubLink, } from "@intx/hub-agent"; -import { hexDecode, hexEncode } from "@intx/types"; -import { createAgentRepoStore } from "@intx/hub-sessions"; -import { createTarballCache } from "@intx/tool-packaging"; +import { hexDecode, hexEncode } from "@intx/types/hex"; +import { createAgentRepoStore } from "@intx/hub-sessions/substrate"; +import { + applyFrozenWorkflowClosure, + createTarballCache, + createWorkflowClosureMaterializer, +} from "@intx/tool-packaging"; +import { deriveWorkflowRunRepoId, inertFlatNamespaceStepIds } from "@intx/workflow-deploy"; import { loadAdapterRegistry } from "@intx/inference/providers"; import { parseReconnectDelayMs, + parseDeploymentCapacity, readAdapterManifest, readCacheMaxBytes, readCredentialEncryptionKey, @@ -35,26 +46,32 @@ import { createDefaultHarnessBuilder } from "./default-harness"; // inbound frame stages through the workflow-run substrate, spawning a // supervised workflow-process child for a workflow deploy. import type { DispatchTimingMark } from "@intx/workflow-host"; - -import { createSidecarDeployRouter, type SidecarDeployRouter } from "./workflow-host-wiring"; import { createDeploymentAddressRegistry, + createMultistepCredentialsRouter, createMultistepDrainRouter, createMultistepGrantsRouter, createMultistepMailRouter, createMultistepSignalRouter, createMultistepSourcesRouter, - createMultistepCredentialsRouter, + createSidecarDeployRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, -} from "./workflow-run-pack-client"; -import { createWorkflowRunPackRestorer } from "./workflow-run-pack-restore"; -import { readRegistries } from "./sidecar-materialization-config"; -import { createWorkflowClosureMaterializer } from "./workflow-closure-materialization"; -import { MAX_INLINE_ASSET_PAYLOAD_BYTES } from "./source-asset-delivery"; -import { createWorkflowProbeExecutor } from "./workflow-probe-handler"; + createWorkflowRunPackRestorer, + WORKFLOW_RUN_RECORD_FILENAME, + removeFileAtomicDurable, + writeFileAtomicDurable, + type SidecarDeployRouter, +} from "@intx/workflow-host/deploy"; +import { createWorkflowProbeExecutor } from "@intx/workflow-host/probe"; + +import { readRegistries, resolveHostPlatform } from "./sidecar-materialization-config"; import { loadOrMintSidecarKeypair } from "./signing-keypair"; -import { removeFileAtomicDurable, writeFileAtomicDurable } from "./atomic-write"; +import { defaultSubprocessSpawner, SIDECAR_WORKFLOW_CHILD_BINARY } from "./workflow-child-spawner"; +import { + defaultProbeChildSpawner, + SIDECAR_WORKFLOW_PROBE_CHILD_BINARY, +} from "./workflow-probe-spawner"; await setup(); @@ -83,6 +100,10 @@ const cacheRoot = ? sidecarCacheDir : path.join(dataDir, "cache", "tarballs"); const cacheMaxBytes = readCacheMaxBytes(); +const capacity = parseDeploymentCapacity( + process.env["SIDECAR_MAX_ACTIVE_DEPLOYMENTS"], + process.env["SIDECAR_MAX_RETAINED_DEPLOYMENTS"], +); const registryMaxTarballBytes = readRegistryMaxTarballBytes(); // Operator-configured custom inference adapters, resolved once at the @@ -186,7 +207,8 @@ if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { // Bound on the child's spawn-time `ready` handshake. Threaded to every // per-deployment supervisor; on expiry the supervisor kills the child and // rejects the spawn, so a child that spawns but never signals ready fails -// the deploy (or is skipped by boot-time restore) instead of hanging it. +// the deploy, or a boot-time restore holds it as stopped, instead of hanging +// it. // Absent, the supervisor applies its 30s default. const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; let readyTimeoutMs: number | undefined; @@ -206,15 +228,20 @@ if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { // absent-means-link-default semantics. const reconnectDelayMs = parseReconnectDelayMs(process.env["SIDECAR_RECONNECT_DELAY_MS"]); +// Probes run concurrently, and the cache's in-use count protects an extraction +// from eviction only among callers of one instance, so every probe shares this +// one. +const probeTarballCache = createTarballCache({ + rootDir: cacheRoot, + maxBytes: cacheMaxBytes, +}); + // Sweep any tmp staging directories left behind by a `put` or // `extractTarball` that crashed between staging and the final rename // on a previous boot. Running here, before the orchestrator starts // accepting apply work, keeps the cache root from accumulating // orphans for the lifetime of the sidecar's data directory. -await createTarballCache({ - rootDir: cacheRoot, - maxBytes: cacheMaxBytes, -}).sweepOrphans(); +await probeTarballCache.sweepOrphans(); // Load or mint the sidecar's local Ed25519 keypair. The supervisor // principal signs every workflow-run commit with this key; the @@ -238,8 +265,8 @@ const agentRepoStore = createAgentRepoStore({ }); // Cache of the hub-vouched public keys of senders this sidecar's deployments -// are authorized to receive mail from. The grants handler writes each key the -// hub co-delivers on a `run.grants` frame; the recipient's inbound-mail verify +// are authorized to receive mail from. The hub-link writes each key the hub +// co-delivers on a `run.grants` frame; the recipient's inbound-mail verify // reads it back. The keyring is loaded here at boot so a restart keeps every // previously-cached key. The durable-write primitive is injected so the cache // write is atomic and fsynced -- at least as durable as the run-grants write it @@ -326,7 +353,12 @@ const multistepSourcesRouter = createMultistepSourcesRouter(); // unrouted. const multistepCredentialsRouter = createMultistepCredentialsRouter(); -const transport = createInMemoryTransport(); +// Deployments register here only to sign their mail; inbound mail reaches +// them from the Hub through their supervisors. Relaying every recipient +// sends mail between two deployments on this sidecar, or to the sender +// itself, through the Hub like any other mail instead of into a local +// inbox nothing reads. +const transport = createInMemoryTransport({ relayOnly: true }); // The pack-push client closes over the substrate (for `createPack`) // and a lazy hub-link binding (for `pushWorkflowRunPack`). The link @@ -354,6 +386,7 @@ const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ // Hub and incorrectly present it as a new supervisor write. substrate: agentRepoStore.repoStore, markRestored: workflowRunPackClient.markRestored, + deriveWorkflowRunRepoId, }); // Wrap the substrate's RepoStore with the boot-edge facade so a @@ -362,9 +395,11 @@ const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ // (today, the agent-state deploy-applier path) flow through // unchanged. const wrappedRepoStore = createWorkflowRunPackPushingRepoStore({ + isConnectionLost, underlying: agentRepoStore.repoStore, packClient: workflowRunPackClient, registry: deploymentAddressRegistry, + deriveWorkflowRunRepoId, }); const hubWsUrl = requireEnv("HUB_WS_URL"); @@ -373,11 +408,14 @@ const sidecarToken = requireEnv("SIDECAR_TOKEN"); // Multi-step substrate-config the deploy router threads into the // workflow-process child's spawn-time env. The child's substrate -// factory consumes these via the typed `SubstrateConfig` validator so -// the per-step pack-push wrap can identify the deployment's hub-side -// trust anchors. Today the IPC bridge in `pack.push.request` carries -// the pack; the WebSocket-connection keys are reserved for a future -// child-local hub link without redoing the boot-edge wiring. +// factory consumes these via the typed `SubstrateConfig` validator. +// The sidecar's Hub credential stays out of it: the child runs +// deployment code and never talks to the Hub itself, and the token +// authenticates the sidecar for every deployment it hosts. That keeps the +// token out of what the child is handed, not out of its reach: a child +// running as the sidecar's OS user can still read the sidecar's own +// environment, token and credential encryption key included (see +// docs/SIDECAR_PLACEMENT.md). // // `PATH`, `HOME`, and `TMPDIR` are propagated from the boot edge's own // environment so the child's `#!/usr/bin/env bun` shebang can resolve @@ -389,9 +427,6 @@ const multistepSubstrateEnv: Record = { SIDECAR_DATA_DIR: dataDir, SIDECAR_SIGNING_PUBLIC_KEY: hexEncode(sidecarSigningKey.publicKey), SIDECAR_SIGNING_PRIVATE_KEY: hexEncode(sidecarSigningKey.privateKey), - HUB_WS_URL: hubWsUrl, - SIDECAR_ID: sidecarId, - SIDECAR_TOKEN: sidecarToken, PATH: requireEnv("PATH"), // Tool-loader caps the child's per-step tool materialization uses. // Resolved once at the boot edge and threaded into the child through @@ -437,13 +472,16 @@ const buildHarness = createDefaultHarnessBuilder({ adapters }); // inline-payload cap is the signal to move the probe's asset delivery to the // streamed transfer the deploy path uses. const workflowProbeExecutor = createWorkflowProbeExecutor({ + binaryPath: SIDECAR_WORKFLOW_PROBE_CHILD_BINARY, + spawnProbeChild: defaultProbeChildSpawner, materialize: createWorkflowClosureMaterializer({ - cacheRoot, - cacheMaxBytes, + cache: probeTarballCache, registryMaxTarballBytes, maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, registries: readRegistries(), scratchRoot: path.join(dataDir, "workflow-probe", "closures"), + host: resolveHostPlatform(), + materializeAssets: materializeWorkflowAssets, }), }); @@ -454,6 +492,7 @@ let sidecarDeployRouter: SidecarDeployRouter | undefined; const orchestrator = createSidecarOrchestrator({ hubURL: hubWsUrl, + capacity, sidecarId, token: sidecarToken, dataDir, @@ -465,7 +504,8 @@ const orchestrator = createSidecarOrchestrator({ resolveSenderCrypto, lookupInboundMailPolicy, // Write peer of `resolveSenderCrypto`: an inbound `sender.key.refresh` frame - // re-pushes a rotated sender key here. Decode the hex and persist through the + // re-pushes a rotated sender key here, and the keys a `run.grants` frame + // carries land here too. Decode the hex and persist through the // same cache the read side serves from; `put` owns the 32-byte length check // and `hexDecode` owns hex validity, so both faults surface to the link's // handler rather than being masked here. @@ -484,19 +524,19 @@ const orchestrator = createSidecarOrchestrator({ // Test-only override of the hub-link reconnect backoff; unset in // production, where the link applies its 3s default. ...(reconnectDelayMs !== undefined ? { reconnectDelayMs } : {}), - // The hub link calls this on every (re)connect to announce the workflow - // deployments this sidecar hosts so the hub re-registers their routes. - // `createDeployRouter` runs synchronously during construction (below), so - // the router is captured before the link ever connects; assert rather than - // optional-chain so a wiring regression fails loud instead of silently - // announcing no deployments. - getWorkflowAddresses: () => { + // The hub link reports these in every `hello` so the Hub reconciles against + // what this sidecar holds, and delivers a Hub frame only to an incarnation + // held live. `createDeployRouter` runs synchronously during construction + // (below), so the router is captured before the link ever connects; assert + // rather than optional-chain so a wiring regression fails loud instead of + // silently reporting no deployments. + getIncarnations: () => { if (sidecarDeployRouter === undefined) { throw new Error( - "sidecar boot: deploy router was not constructed before the hub link requested workflow addresses", + "sidecar boot: deploy router was not constructed before the hub link requested its incarnations", ); } - return sidecarDeployRouter.activeAddresses(); + return sidecarDeployRouter.incarnations(); }, // Report the sidecar's cached rotatable senders on every (re)connect so the // hub re-resolves each current key and re-pushes it, catching a rotation that @@ -505,22 +545,21 @@ const orchestrator = createSidecarOrchestrator({ // workflow_run.public_key); the link stays source-opaque and reports whatever // this returns. getCachedSenderAddresses: () => senderKeyCache.rotatableAddresses(), - // When the hub-link re-announces a deployment address in an authenticated - // reconnect, re-drive any workflow-run pack the disconnect cancelled. The - // link fires this AFTER sending the reconnect frame, so the hub routes the - // address before it sees the re-shipped pack (both frame families queue on - // the hub's per-connection chain). This is the liveness half of - // reconnect recovery: without it, a synchronous single-step run whose only - // pack was interrupted mid-transfer never re-ships, because it has no later - // local write to re-arm the coalescing loop. + // When the Hub's `welcome` routes a held incarnation again, re-drive any + // workflow-run pack the disconnect cancelled. The Hub has routed the + // address by the time it sends `welcome`, so it accepts the re-shipped + // pack. This is the liveness half of reconnect recovery: without it, a + // synchronous single-step run whose only pack was interrupted mid-transfer + // never re-ships, because it has no later local write to re-arm the + // coalescing loop. onWorkflowAddressesRoutable: (addresses) => { // The deploy router is captured synchronously during orchestrator // construction, before the link ever connects, so a routable callback can - // only fire once it exists; assert (like `getWorkflowAddresses`) rather + // only fire once it exists; assert (like `getIncarnations`) rather // than optional-chain so a wiring regression fails loud. if (sidecarDeployRouter === undefined) { throw new Error( - "sidecar boot: deploy router was not constructed before a reconnect made workflow addresses routable", + "sidecar boot: deploy router was not constructed before a welcome made workflow addresses routable", ); } for (const address of addresses) { @@ -534,9 +573,8 @@ const orchestrator = createSidecarOrchestrator({ // at DEPLOY time and persists across the outage -- the reconnect // re-routes the address in the hub's in-memory index but writes no DB // status, so the lookup already resolves. (The frame also lands after - // the address is wire-routable: the link fires this after the - // reconnect frame, and both frames queue on the hub's per-connection - // chain -- see the pack re-ship note above.) + // the address is routable: the link fires this on `welcome`, which + // the Hub sends once it has routed the address.) // - A sidecar restart re-emits the parked set twice (child // re-establishment fires Trigger A too), but the co-write dedups on // the `correlationId` PK/unique constraints via `onConflictDoNothing`, @@ -546,11 +584,12 @@ const orchestrator = createSidecarOrchestrator({ // here. sidecarDeployRouter.reEmitParkedCorrelations(address); } + sidecarDeployRouter.reportStoppedDeployments(); }, - // On disconnect, block the deployment addresses' workflow-run pushes until - // the authenticated reconnect above re-routes them. Without the block, the - // coalescing pusher re-ships onto the fresh, not-yet-registered connection - // and the hub drops the frames as "unrouted". + // When a connection opens or closes, block the deployment addresses' + // workflow-run pushes until the next `welcome` routes them. Without the + // block, the coalescing pusher keeps trying to send packs the link cannot + // carry until then. onWorkflowAddressesUnroutable: (addresses) => { for (const address of addresses) { wrappedRepoStore.markAddressUnroutable(address); @@ -561,23 +600,37 @@ const orchestrator = createSidecarOrchestrator({ keyStore, publishWorkflowInferenceEvent, publishWorkflowSuspension, + publishDeploymentStopped, }) => { const router = createSidecarDeployRouter({ + ...capacity, sessions, keyStore, - senderKeyCache, transport, repoStore: wrappedRepoStore, signingKeySeed: sidecarSigningKey.privateKey, credentialCipher, createAgentCrypto: createEd25519Crypto, assertSourceBuildable: buildHarness.canBuildSource, - registerDeployment: ({ runId, agentAddress }) => { - deploymentAddressRegistry.record(runId, agentAddress); + registerDeployment: ({ runId, agentAddress, generation }) => { + deploymentAddressRegistry.record(runId, { agentAddress, generation }); }, - unregisterDeployment: ({ runId }) => { + unregisterDeployment: ({ runId, agentAddress }) => { deploymentAddressRegistry.unregister(runId); + wrappedRepoStore.forgetDeployment(runId, agentAddress); }, + reportDeploymentRefTips: (incarnation) => + wrappedRepoStore.reportWorkflowRunRefTips(incarnation), + publishDeploymentStopped, + // The run record lives in the repository and goes last, so a crash + // partway through leaves it for the next boot to report. + removeRunRepository: (runId) => + agentRepoStore.repoStore.removeRepo( + { kind: "workflow-run", id: runId }, + { last: WORKFLOW_RUN_RECORD_FILENAME }, + ), + removeAgentStateRepository: (id) => + agentRepoStore.repoStore.removeRepo({ kind: "agent-state", id }), multistepMailRouter, inboundMailPolicyRegistry, multistepSignalRouter, @@ -586,6 +639,18 @@ const orchestrator = createSidecarOrchestrator({ multistepSourcesRouter, multistepCredentialsRouter, multistepSubstrateEnv, + multistepSubprocessSpawner: defaultSubprocessSpawner, + multistepBinaryPath: SIDECAR_WORKFLOW_CHILD_BINARY, + createWorkflowCache: createTarballCache, + applyFrozenWorkflowClosure, + readRegistries, + resolveHostPlatform, + resolveInboundMailPolicy, + materializeWorkflowAssets, + maxInlineAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, + deriveWorkflowRunRepoId, + inertFlatNamespaceStepIds, + sourceAssetGitDir, publishWorkflowInferenceEvent, publishWorkflowSuspension, ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), diff --git a/apps/sidecar/src/run-grants.ts b/apps/sidecar/src/run-grants.ts deleted file mode 100644 index dcc9b1353b..0000000000 --- a/apps/sidecar/src/run-grants.ts +++ /dev/null @@ -1,62 +0,0 @@ -// Shared by workflow-host-wiring.ts and workflow-substrate-factory.ts, both -// of which read runs//grants.json. Kept dependency-narrow (fs/path, -// arktype, RepoStore) so importing this into the child binary doesn't drag -// in the hub-agent deploy-router surface the wiring module also depends on. - -import { readFile } from "node:fs/promises"; -import { join as pathJoin } from "node:path"; - -import { type } from "arktype"; - -import type { RepoStore } from "@intx/hub-sessions"; -import { isErrnoNotFound } from "@intx/workflow-host"; - -/** Sibling to the run's events/ blobs, so grants live and are reclaimed with the rest of the run's state. */ -export function runGrantsPath(runId: string): string { - return `runs/${runId}/grants.json`; -} - -/** Entries stay unknown; the child's authorize layer narrows each against its own grant-rule validator. */ -export const RunGrantsFile = type({ - grants: "unknown[]", -}).onUndeclaredKey("ignore"); - -/** - * Returns undefined (distinct from an empty array) when the file is - * absent, so the caller can tell "no per-run grants file" from "empty - * grants set". A malformed existing file throws rather than defaulting. - */ -export async function readRunGrants(args: { - repoStore: RepoStore; - anchorRunId: string; - runId: string; -}): Promise { - const dir = args.repoStore.getRepoDir({ - kind: "workflow-run", - id: args.anchorRunId, - }); - const filePath = pathJoin(dir, runGrantsPath(args.runId)); - let raw: string; - try { - raw = await readFile(filePath, "utf8"); - } catch (cause) { - if (isErrnoNotFound(cause)) return undefined; - throw cause; - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new Error( - `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} is not valid JSON`, - { cause }, - ); - } - const validated = RunGrantsFile(parsed); - if (validated instanceof type.errors) { - throw new Error( - `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} failed validation: ${validated.summary}`, - ); - } - return validated.grants; -} diff --git a/apps/sidecar/src/sidecar-materialization-config.ts b/apps/sidecar/src/sidecar-materialization-config.ts index ee5646959e..2e81de87f6 100644 --- a/apps/sidecar/src/sidecar-materialization-config.ts +++ b/apps/sidecar/src/sidecar-materialization-config.ts @@ -1,112 +1,22 @@ -// Shared by tool-materialization.ts and workflow-closure-materialization.ts -// so both loader-boundary inputs (registry map, host platform) come from -// one source of truth rather than duplicated env parsing and allowlists. - -import { type } from "arktype"; -import type { HostPlatform, RegistryConfig } from "@intx/tool-packaging"; - -// name rides alongside the config so an operator authors a flat JSON array; -// this collapses it into a Map keyed by name for the loader. -const RegistryConfigEnvEntry = type({ - name: "string", - url: "string", - "auth?": type({ - "token?": "string", - "basic?": type({ user: "string", pass: "string" }), - }), -}); -const RegistryConfigEnvArray = RegistryConfigEnvEntry.array(); +// Process edge for the tool-package materialization boundary. +// +// `parseToolRegistries` and `hostPlatform` in `@intx/tool-packaging` +// own the `SIDECAR_TOOL_REGISTRIES` rules and the npm `os`/`cpu` +// allowlists. This module reads the process and forwards one input +// each. Boot, the deploy router, and the closure materializer call +// these names. + +import { + hostPlatform, + parseToolRegistries, + type HostPlatform, + type RegistryConfig, +} from "@intx/tool-packaging"; export function readRegistries(): ReadonlyMap { - const raw = process.env["SIDECAR_TOOL_REGISTRIES"]; - if (raw === undefined) { - return new Map([["npmjs", { url: "https://registry.npmjs.org" }]]); - } - // An empty string almost always means misconfig; falling through to the - // npmjs default would silently route packages through public npm. - if (raw.trim() === "") { - throw new Error( - "SIDECAR_TOOL_REGISTRIES is set but empty — unset the variable to use the default npmjs registry", - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err) { - throw new Error( - `SIDECAR_TOOL_REGISTRIES is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, - ); - } - const validated = RegistryConfigEnvArray(parsed); - if (validated instanceof type.errors) { - throw new Error(`SIDECAR_TOOL_REGISTRIES failed validation: ${validated.summary}`); - } - const out = new Map(); - for (const entry of validated) { - if (out.has(entry.name)) { - throw new Error( - `SIDECAR_TOOL_REGISTRIES has duplicate registry name ${JSON.stringify(entry.name)}`, - ); - } - const config: RegistryConfig = { - url: entry.url, - ...(entry.auth !== undefined ? { auth: entry.auth } : {}), - }; - out.set(entry.name, config); - } - return out; -} - -// Validated at the boundary so an unknown process.platform token fails -// boot instead of silently mis-routing the loader's os filter. A Node/Bun -// major bump adding a new platform value will fail boot until refreshed. -const KNOWN_PROCESS_PLATFORMS = new Set([ - "aix", - "android", - "darwin", - "freebsd", - "haiku", - "linux", - "openbsd", - "sunos", - "win32", - "cygwin", - "netbsd", -]); - -// Same rationale as KNOWN_PROCESS_PLATFORMS, for process.arch. -const KNOWN_PROCESS_ARCHS = new Set([ - "arm", - "arm64", - "ia32", - "loong64", - "mips", - "mipsel", - "ppc64", - "riscv64", - "s390x", - "x64", -]); - -function assertKnownHostPlatform(platform: NodeJS.Platform): void { - if (!KNOWN_PROCESS_PLATFORMS.has(platform)) { - throw new Error( - `sidecar boot: process.platform ${JSON.stringify(platform)} is not a recognized npm \`os\` token; tool-package platform filtering would be unreliable`, - ); - } -} - -function assertKnownHostArch(arch: NodeJS.Architecture): void { - if (!KNOWN_PROCESS_ARCHS.has(arch)) { - throw new Error( - `sidecar boot: process.arch ${JSON.stringify(arch)} is not a recognized npm \`cpu\` token; tool-package platform filtering would be unreliable`, - ); - } + return parseToolRegistries(process.env["SIDECAR_TOOL_REGISTRIES"]); } -/** Both consumers resolve the host through this one helper so the allowlists and gate live in one place. */ export function resolveHostPlatform(): HostPlatform { - assertKnownHostPlatform(process.platform); - assertKnownHostArch(process.arch); - return { os: process.platform, cpu: process.arch }; + return hostPlatform(process.platform, process.arch); } diff --git a/apps/sidecar/src/signing-keypair.test.ts b/apps/sidecar/src/signing-keypair.test.ts deleted file mode 100644 index fa9527c38a..0000000000 --- a/apps/sidecar/src/signing-keypair.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { loadOrMintSidecarKeypair } from "./signing-keypair"; - -async function makeTempDir(): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), "sidecar-signing-keypair-")); -} - -const PRIVATE_KEY_PATH = (dir: string) => path.join(dir, "ed25519.private"); -const PUBLIC_KEY_PATH = (dir: string) => path.join(dir, "ed25519.public"); - -describe("loadOrMintSidecarKeypair", () => { - test("mints a fresh keypair on first boot and persists both files", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - const minted = await loadOrMintSidecarKeypair(signingDir); - - expect(minted.privateKey).toHaveLength(32); - expect(minted.publicKey).toHaveLength(32); - expect(Array.from(await fs.readFile(PRIVATE_KEY_PATH(signingDir)))).toEqual( - Array.from(minted.privateKey), - ); - expect(Array.from(await fs.readFile(PUBLIC_KEY_PATH(signingDir)))).toEqual( - Array.from(minted.publicKey), - ); - }); - - test("reloads a consistent keypair and derives the public key from the seed", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - const minted = await loadOrMintSidecarKeypair(signingDir); - const reloaded = await loadOrMintSidecarKeypair(signingDir); - - expect(reloaded.privateKey).toEqual(minted.privateKey); - expect(reloaded.publicKey).toEqual(minted.publicKey); - }); - - test("rejects a public key file that does not match the seed", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - const minted = await loadOrMintSidecarKeypair(signingDir); - - const tampered = new Uint8Array(minted.publicKey); - const firstByte = tampered[0]; - if (firstByte === undefined) { - throw new Error("expected a 32-byte public key"); - } - tampered[0] = firstByte ^ 0xff; - await fs.writeFile(PUBLIC_KEY_PATH(signingDir), tampered); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( - /does not match the key derived from the seed/, - ); - }); - - test("rejects a truncated public key file", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - await fs.writeFile(PUBLIC_KEY_PATH(signingDir), new Uint8Array(5)); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( - /does not match the key derived from the seed/, - ); - }); - - test("rejects an empty public key file", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - await fs.writeFile(PUBLIC_KEY_PATH(signingDir), new Uint8Array(0)); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( - /does not match the key derived from the seed/, - ); - }); - - test("rejects a seed that is not a valid Ed25519 seed", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - - await fs.writeFile(PRIVATE_KEY_PATH(signingDir), new Uint8Array(16)); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( - /is not a valid Ed25519 seed/, - ); - }); - - test("rejects a length-valid seed that derives to a different public key", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - - const otherDir = path.join(await makeTempDir(), ".sidecar-signing"); - const other = await loadOrMintSidecarKeypair(otherDir); - await fs.writeFile(PRIVATE_KEY_PATH(signingDir), other.privateKey); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( - /does not match the key derived from the seed/, - ); - }); - - test("rejects a partial keypair where only the private key is present", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - await fs.rm(PUBLIC_KEY_PATH(signingDir)); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/is partial/); - }); - - test("rejects a partial keypair where only the public key is present", async () => { - const dataDir = await makeTempDir(); - const signingDir = path.join(dataDir, ".sidecar-signing"); - - await loadOrMintSidecarKeypair(signingDir); - await fs.rm(PRIVATE_KEY_PATH(signingDir)); - - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/is partial/); - }); -}); diff --git a/apps/sidecar/src/signing-keypair.ts b/apps/sidecar/src/signing-keypair.ts index 0c4eda726a..8d3932dc48 100644 --- a/apps/sidecar/src/signing-keypair.ts +++ b/apps/sidecar/src/signing-keypair.ts @@ -28,10 +28,20 @@ async function exists(filePath: string): Promise { } /** - * The seed file is the sole source of identity; the public file is cross - * checked against it on every load, not trusted as a cache, so a corrupted - * or swapped file fails loud at boot instead of silently failing signature - * verification at the hub later. + * Load the sidecar's persisted Ed25519 signing keypair, minting a fresh + * one on first boot. + * + * The 32-byte seed in `ed25519.private` is the sole source of truth for + * the sidecar's identity; the public key is always derived from it. The + * `ed25519.public` file is an identity anchor, not a cache: on every load + * it is cross-checked against the seed-derived public key. A mismatch + * means either the seed or the public file was corrupted or swapped (a + * bad backup restore, disk bitrot, an operator fat-finger). Rather than + * trusting the file and advertising a public key the sidecar cannot sign + * with -- which would make every signature fail verification at the hub, + * far from the root cause -- we fail loudly here at boot. We cannot tell + * which of the two files rotted, so we halt and let an operator decide + * (restore the seed, or remove the directory to mint a fresh identity). */ export async function loadOrMintSidecarKeypair(signingDir: string): Promise { const privateKeyPath = path.join(signingDir, PRIVATE_KEY_FILENAME); diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts deleted file mode 100644 index 446088e2ce..0000000000 --- a/apps/sidecar/src/step-agent-tools.ts +++ /dev/null @@ -1,444 +0,0 @@ -// The child IS the sidecar binary, so its tool runtime is present in the -// child's address space; this module runs it when the real step-invoker -// builds a step's agent. Wrapping close() to run plugin/bundle disposers -// is what guarantees the LSP subprocess dies with the step's agent on -// every exit path. Lives in apps/sidecar so @intx/workflow-host stays -// free of the tool runtime dependency. - -import path from "node:path"; - -import { - createAgent, - defineAgent, - defineTool, - toolApprovalEffect, - type Agent, - type AgentDefinition, - type AnnotatedPluginFactory, - type AnnotatedToolFactory, - type BaseEnv, - type ToolBundle, -} from "@intx/agent"; -import { readDeployTree, agentDir } from "@intx/hub-agent/paths"; -import { getLogger } from "@intx/log"; -import type { HostCredentialCapability } from "@intx/harness"; -import type { LoadedToolFactory } from "@intx/tool-packaging"; -import { resolveStepAddress } from "@intx/workflow-deploy"; -import { parseRunAddress } from "@intx/types"; -import type { GrantRule } from "@intx/types/authz"; -import { - layerRuntimeCapabilities, - type RuntimeCapabilities, -} from "@intx/types/runtime-capabilities"; -import { baseStepId } from "@intx/workflow"; - -import { - buildCredentialCapabilities, - type StepCredentialWiring, -} from "./step-credential-capabilities"; -import { materializeToolPackages, type StepToolFactory } from "./tool-materialization"; - -const logger = getLogger(["sidecar", "workflow-child", "step-tools"]); - -/** Resolved at the boot edge and threaded into the child so per-step materialization stays bounded. */ -export interface StepToolCacheConfig { - readonly cacheMaxBytes: number; - readonly registryMaxTarballBytes: number; -} - -/** Carried via a symbol-keyed slot on the per-step env so the two step-invoker callbacks cooperate without widening its surface. */ -export interface StepToolMaterialization { - readonly factories: readonly StepToolFactory[]; - readonly pluginFactories: readonly AnnotatedPluginFactory[]; -} - -/** - * The hub's capability walk only reads inline agent.toolFactories, so a - * pinned-package tool never gets a hub-side tool: grant; these - * derived rows supply that floor as additional grants, which an explicit - * deny still overrides by evaluateGrants precedence. The id is - * deterministic (floor:tool:) since evaluateGrants doesn't dedupe by id. - */ -export function deriveToolMarkFloorGrants(factories: readonly LoadedToolFactory[]): GrantRule[] { - const rows: GrantRule[] = []; - for (const factory of factories) { - for (const definition of factory.definitions) { - rows.push({ - id: `floor:tool:${definition.name}`, - resource: `tool:${definition.name}`, - action: "invoke", - effect: toolApprovalEffect(definition), - origin: "creator", - conditions: null, - expiresAt: null, - roleId: null, - principalId: null, - }); - } - } - return rows; -} - -// Symbol survives the step-invoker adapter's `{ ...envBase, authorize }` -// spread, which copies own enumerable symbol-keyed properties. -const STEP_TOOLS = Symbol("intx.sidecar.step-tools"); - -// Read/written via Reflect since BaseEnv has no symbol index signature. -function setStepToolSlot(env: object, materialization: StepToolMaterialization): void { - Reflect.set(env, STEP_TOOLS, materialization); -} - -function getStepToolSlot(env: object): StepToolMaterialization | undefined { - const value: unknown = Reflect.get(env, STEP_TOOLS); - if (value === undefined) return undefined; - if (!isStepToolMaterialization(value)) { - throw new Error( - "sidecar workflow-child step tools: the per-step env's tool slot is not a StepToolMaterialization; the slot is private to this module and must only be set by attachStepTools", - ); - } - return value; -} - -function isStepToolMaterialization(value: unknown): value is StepToolMaterialization { - return ( - typeof value === "object" && - value !== null && - "factories" in value && - "pluginFactories" in value && - Array.isArray(value.factories) && - Array.isArray(value.pluginFactories) - ); -} - -/** Absent for a build with no credential context (e.g. a unit test), in which case bundles keep the base bag. */ -const STEP_CREDENTIAL_WIRING = Symbol("intx.sidecar.step-credential-wiring"); - -function setStepCredentialWiring(env: object, wiring: StepCredentialWiring): void { - Reflect.set(env, STEP_CREDENTIAL_WIRING, wiring); -} - -function getStepCredentialWiring(env: object): StepCredentialWiring | undefined { - const value: unknown = Reflect.get(env, STEP_CREDENTIAL_WIRING); - if (value === undefined) return undefined; - if (!isStepCredentialWiring(value)) { - throw new Error( - "sidecar workflow-child step tools: the per-step env's credential-wiring slot is not a StepCredentialWiring; the slot is private to this module and must only be set by attachStepCredentialWiring", - ); - } - return value; -} - -function isStepCredentialWiring(value: unknown): value is StepCredentialWiring { - return ( - typeof value === "object" && - value !== null && - "materialCell" in value && - "resolveGrants" in value && - "providers" in value - ); -} - -/** Omitted for a build with no credential context, leaving the slot unset. */ -export function attachStepCredentialWiring( - env: Omit, - wiring: StepCredentialWiring, -): void { - setStepCredentialWiring(env, wiring); -} - -/** Throws on a missing base bag: that's a wiring inconsistency (buildEnv sets both), not to paper over. */ -function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { - const value: unknown = Reflect.get(env, "capabilities"); - if ( - typeof value !== "object" || - value === null || - !("resolve" in value) || - typeof (value as { resolve: unknown }).resolve !== "function" - ) { - throw new Error( - "sidecar workflow-child step tools: a credentials capability was assembled for this bundle but the per-step env carries no base capabilities bag to layer it onto; buildEnv must set env.capabilities before assembling credentials", - ); - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated structurally above: an object whose `resolve` is callable is the RuntimeCapabilities resolver buildEnv set - return value as RuntimeCapabilities; -} - -/** - * The deploy tree lands in the legacy per-agent directory keyed by the - * step's mail address, not the substrate's agent-state/ layout — the - * multi-step deploy path never pushes step agent-state packs there. - */ -export function stepDeployTreeDir(args: { - dataDir: string; - mailboxAddress: string; - stepId: string; - stepCount: number; -}): string { - const parsed = parseRunAddress(args.mailboxAddress); - if (parsed === null) { - throw new Error( - `sidecar workflow-child step tools: deployment mailbox address ${JSON.stringify(args.mailboxAddress)} is not a parseable run address; cannot locate the step's deploy tree`, - ); - } - // A map iteration's scoped step id resolves to its base address, since - // deploy stages one tree per base step; baseStepId is identity otherwise. - const stepAddress = resolveStepAddress({ - runId: parsed.runId, - stepId: baseStepId(args.stepId), - domain: parsed.domain, - stepCount: args.stepCount, - }); - return agentDir(args.dataDir, stepAddress); -} - -/** Rooted under storeDir so concurrent steps in one child never collide on cache/apply-state paths. */ -export async function materializeStepTools(args: { - dataDir: string; - mailboxAddress: string; - stepId: string; - stepCount: number; - /** Per-step state root; cache + instance dir + workspace live under it. */ - storeDir: string; - cache: StepToolCacheConfig; -}): Promise { - const deployTreeDir = stepDeployTreeDir({ - dataDir: args.dataDir, - mailboxAddress: args.mailboxAddress, - stepId: args.stepId, - stepCount: args.stepCount, - }); - const deployTree = await readDeployTree(deployTreeDir); - - // Per-step even though content-addressed, so a wedged apply in one step - // cannot corrupt another's view. - const cacheRoot = path.join(args.storeDir, "tarball-cache"); - - // deployTreeDir/workspace (read-only, keyed by base step) and - // storeDir/workspace (read-write, keyed by scoped step) share a leaf name - // but are deliberately different roots — a map iteration reads one shared - // tree while each iteration writes its own scratch. Do not unify them. - const assetRoot = path.join(deployTreeDir, "workspace"); - - const materialized = await materializeToolPackages({ - rawManifestBytes: deployTree.toolPackageManifestRaw, - assetMounts: deployTree.assetMounts, - storeDir: args.storeDir, - assetRoot, - agentAddress: args.mailboxAddress, - cacheRoot, - cacheMaxBytes: args.cache.cacheMaxBytes, - registryMaxTarballBytes: args.cache.registryMaxTarballBytes, - }); - return { - factories: materialized.factories, - pluginFactories: materialized.pluginFactories, - }; -} - -/** Omit because buildEnv yields exactly that shape before the adapter adds authorize. */ -export function attachStepTools( - env: Omit, - materialization: StepToolMaterialization, -): void { - setStepToolSlot(env, materialization); -} - -/** definitions is forwarded verbatim: this wrapper must not rename tools the deploy-time walk enumerated. */ -export function rewrapStepToolFactory( - annotated: AnnotatedToolFactory, - onDispose: (dispose: () => unknown) => void, - credentials: HostCredentialCapability | undefined, -): AnnotatedToolFactory { - return defineTool({ - id: annotated.id, - requires: annotated.requires, - definitions: annotated.definitions, - factory: (factoryEnv: BaseEnv): ToolBundle => { - const env = - credentials === undefined ? factoryEnv : layerCredentialsOntoEnv(factoryEnv, credentials); - const bundle = annotated(env); - if (bundle.dispose !== undefined) { - onDispose(bundle.dispose); - } - return bundle; - }, - }); -} - -/** Spread preserves every other env surface (including the private tool slot); only capabilities is replaced. */ -function layerCredentialsOntoEnv( - factoryEnv: BaseEnv, - credentials: HostCredentialCapability, -): BaseEnv { - const base = requireCapabilitiesBag(factoryEnv); - const layered = layerRuntimeCapabilities(base, { credentials }); - // Reflect.set, not a literal: BaseEnv doesn't type "capabilities", so a - // `{ ...factoryEnv, capabilities }` literal trips the excess-property check. - const layeredEnv: BaseEnv = { ...factoryEnv }; - Reflect.set(layeredEnv, "capabilities", layered); - return layeredEnv; -} - -/** Falls back to createAgent(def, env) unchanged when the env carries no materialized tools (e.g. a bare-factory test). */ -export function createToolBearingAgentFactory(): ( - def: AgentDefinition, - env: EnvReq, -) => Promise { - return async ( - def: AgentDefinition, - env: EnvReq, - ): Promise => { - const materialization = getStepToolSlot(env); - if (materialization === undefined) { - return createAgent(def, env); - } - - // Fails the launch here, loudly, rather than at the tool's first resolve. - const credentialWiring = getStepCredentialWiring(env); - const credentialCapabilities = - credentialWiring === undefined - ? new Map() - : buildCredentialCapabilities(materialization.factories, credentialWiring); - - // Set, not array: dedupes by closure identity so a bundle whose dispose - // is the same on every invocation isn't torn down once per push. - const capturedDisposers = new Set<() => unknown>(); - for (const capability of credentialCapabilities.values()) { - capturedDisposers.add(() => capability.dispose()); - } - const factoriesWithCapture = materialization.factories.map((stf) => - rewrapStepToolFactory( - stf.factory, - (dispose) => { - capturedDisposers.add(dispose); - }, - credentialCapabilities.get(stf.packageName), - ), - ); - - // Shared between success teardown and the construction-failure rollbacks - // below, since credentials capabilities are built before the plugin - // chain and must still be released if a later step throws. - const runCapturedDisposers = async (): Promise => { - const failures: unknown[] = []; - for (const dispose of capturedDisposers) { - try { - await dispose(); - } catch (cause) { - logger.error`step tool bundle dispose failed: ${cause instanceof Error ? cause.message : String(cause)}`; - failures.push(cause); - } - } - return failures; - }; - - // The serialized def.toolFactories carry only { id, requires } metadata - // (the workflow projection strips closures on the wire), so the runnable - // factories come from materialization, not the incoming def. - const toolDef = defineAgent({ - id: def.id, - systemPrompt: def.systemPrompt, - tools: factoriesWithCapture, - capabilities: [...def.capabilities], - inference: { sources: [...def.inference.sources] }, - ...(def.description !== undefined ? { description: def.description } : {}), - ...(def.director !== undefined ? { director: def.director } : {}), - ...(def.tags !== undefined ? { tags: def.tags } : {}), - }); - - // One at a time so each factory sees prior plugins' instances on - // env.plugins (posix reads them; LSP populates them). On a midway - // throw, every constructed instance is released so nothing leaks. - const pluginInstances: unknown[] = []; - let chainEnv: BaseEnv = env; - try { - for (const factory of materialization.pluginFactories) { - const instance = factory(chainEnv); - pluginInstances.push(instance); - chainEnv = { - ...env, - plugins: [...pluginInstances], - }; - } - } catch (err) { - await runCapturedDisposers(); - await disposeAll(pluginInstances, "plugin construction rollback"); - throw err; - } - - let agent: Agent; - try { - agent = await createAgent(toolDef, chainEnv); - } catch (err) { - // createAgent disposes its own tool bundles on failure, but the - // credentials capabilities and plugin instances are this module's to own. - await runCapturedDisposers(); - await disposeAll(pluginInstances, "agent construction failure"); - throw err; - } - - return wrapAgentClose(agent, async () => { - // Disposing the LSP plugin twice is safe (idempotent); running both - // guarantees teardown even for a plugin no tool bundle consumed. - const failures = [ - ...(await runCapturedDisposers()), - ...(await disposeAll(pluginInstances, "step teardown")), - ]; - if (failures.length > 0) { - throw new AggregateError( - failures, - `step agent close: ${String(failures.length)} disposer(s) failed during teardown; an LSP subprocess may be leaked`, - ); - } - }); - }; -} - -/** Teardown runs after the agent's own close so the reactor has stopped issuing tool calls first. */ -function wrapAgentClose(agent: Agent, teardown: () => Promise): Agent { - let tornDown = false; - return { - ...agent, - send: (content, opts) => agent.send(content, opts), - stream: () => agent.stream(), - deliver: (message) => agent.deliver(message), - setSource: (source) => agent.setSource(source), - history: () => agent.history(), - checkpoints: (limit) => agent.checkpoints(limit), - readAt: (hash) => agent.readAt(hash), - blobReader: agent.blobReader, - async close() { - await agent.close(); - if (tornDown) return; - tornDown = true; - await teardown(); - }, - }; -} - -async function disposeAll(instances: readonly unknown[], context: string): Promise { - const failures: unknown[] = []; - for (const instance of instances) { - const dispose = pluginDispose(instance); - if (dispose === undefined) continue; - try { - // `await` accepts non-promise values verbatim, so this works - // whether the disposer is sync or async. - await dispose(); - } catch (cause) { - logger.error`step plugin dispose failed during ${context}: ${cause instanceof Error ? cause.message : String(cause)}`; - failures.push(cause); - } - } - return failures; -} - -/** Plugin instance type is unknown (host-defined shapes the runtime doesn't interpret). */ -function pluginDispose(value: unknown): (() => unknown) | undefined { - if (value === null || typeof value !== "object") return undefined; - if (!("dispose" in value)) return undefined; - const dispose: unknown = value.dispose; - if (typeof dispose !== "function") return undefined; - const fn = dispose; - return () => fn.call(value); -} diff --git a/apps/sidecar/src/step-credential-capabilities.ts b/apps/sidecar/src/step-credential-capabilities.ts deleted file mode 100644 index 6ca875d6b0..0000000000 --- a/apps/sidecar/src/step-credential-capabilities.ts +++ /dev/null @@ -1,180 +0,0 @@ -// One credential capability per package (not one shared per step), since -// sharing across packages would let package A's grant authorize package -// B's resolve. Two fail-closed invariants: material is read from the live -// delivery cell on every use (not snapshotted), so revocation takes effect -// at the material rather than only at the launch-time grant gate; and the -// declared-vs-bound reconcile runs per consumer against its own bound set. - -import { toolConsumer } from "@intx/authz"; -import { - createCredentialCapability, - reconcileDeclaredCredentials, - type CredentialProviderRegistry, - type HostCredentialCapability, - type ResolvedCredentialBinding, -} from "@intx/harness"; -import type { - CredentialMaterial, - CredentialMaterialResolver, - CredentialMaterialSource, -} from "@intx/types"; -import type { GrantRule } from "@intx/types/authz"; -import type { CredentialDelivery } from "@intx/types/sidecar"; - -import type { StepToolFactory } from "./tool-materialization"; - -/** Held structurally so this module stays free of the transport package and a test can use a plain object. */ -export interface CredentialMaterialCell { - readonly current: CredentialDelivery | null; -} - -/** Assembled at the invoke-step boundary: material cell and grants ride in from the run child; providers are sidecar-static. */ -export interface StepCredentialWiring { - readonly materialCell: CredentialMaterialCell; - /** - * A thunk, not a resolved array: a resumed run may not yet have the - * grants snapshot a Gate-2 read needs, and a toolless resume must not - * fault on that. - */ - readonly resolveGrants: () => readonly GrantRule[]; - readonly providers: CredentialProviderRegistry; -} - -/** - * A package with neither declared nor bound credentials gets no entry, so - * resolve("credentials") fails closed rather than handing back an empty - * sub-registry. Every returned capability owns a dispose the caller must run. - */ -export function buildCredentialCapabilities( - factories: readonly StepToolFactory[], - wiring: StepCredentialWiring, -): Map { - const byPackage = new Map(); - const seenPackages = new Set(); - let grants: readonly GrantRule[] | undefined; - - for (const stf of factories) { - if (seenPackages.has(stf.packageName)) continue; - seenPackages.add(stf.packageName); - - const consumer = toolConsumer(stf.packageName); - const bindings = buildConsumerBindings(consumer, wiring.materialCell); - - // Fail closed against this consumer's own bound set only. - reconcileDeclaredCredentials(consumer, stf.declaredCredentials, new Set(bindings.keys())); - - if (stf.declaredCredentials.length === 0 && bindings.size === 0) { - continue; - } - - grants ??= wiring.resolveGrants(); - byPackage.set( - stf.packageName, - createCredentialCapability({ - consumer, - bindings, - providers: wiring.providers, - grants: [...grants], - }), - ); - } - - return byPackage; -} - -/** A descriptor addressed to this consumer with no matching material is a malformed delivery and fails closed. */ -function buildConsumerBindings( - consumer: string, - cell: CredentialMaterialCell, -): Map { - const bindings = new Map(); - const delivery = cell.current; - if (delivery === null) return bindings; - - for (const descriptor of delivery.bindings) { - if (descriptor.consumer !== consumer) continue; - - const material = delivery.materials.find( - (entry) => entry.credentialId === descriptor.credentialId, - ); - if (material === undefined) { - throw new Error( - `credential delivery is malformed: descriptor for handle "${descriptor.handle}" (consumer ${consumer}) references credential ${descriptor.credentialId} but the delivery carries no material for it`, - ); - } - - // provider/origin are pinned at build time; the secret is read live. - bindings.set(descriptor.handle, { - credentialId: descriptor.credentialId, - providerKey: material.providerKey, - origin: material.origin, - readCurrentMaterial: makeReadCurrentMaterial({ - cell, - credentialId: descriptor.credentialId, - providerKey: material.providerKey, - origin: material.origin, - consumer, - }), - }); - } - - return bindings; -} - -/** Reads the live cell on every call so a re-push's revocation takes effect immediately. */ -function makeReadCurrentMaterial(args: { - cell: CredentialMaterialCell; - credentialId: string; - providerKey: string; - origin: string; - consumer: string; -}): CredentialMaterialSource { - const { cell, credentialId, providerKey, origin, consumer } = args; - return () => { - const delivery = cell.current; - if (delivery === null) { - throw new Error( - `credential material for ${credentialId} (consumer ${consumer}) is not available: the delivery cell is empty`, - ); - } - const material = delivery.materials.find((entry) => entry.credentialId === credentialId); - if (material === undefined) { - throw new Error( - `credential material for ${credentialId} (consumer ${consumer}) is no longer delivered: a re-push dropped it (rotated away or revoked)`, - ); - } - // A rotation changes only the secret; a provider/origin drift would mean - // the handle now authenticates somewhere it wasn't pinned to. - if (material.providerKey !== providerKey || material.origin !== origin) { - throw new Error( - `credential ${credentialId} changed provider/origin under an already-shaped handle (${providerKey}@${origin} -> ${material.providerKey}@${material.origin}); a shaped handle cannot follow that change`, - ); - } - return { secret: material.secret }; - }; -} - -/** - * Unlike makeReadCurrentMaterial, pins no provider/origin: an inference - * request authenticates to its own baseURL, so there's no handle to - * protect against origin drift. - */ -export function createInferenceCredentialResolver( - cell: CredentialMaterialCell, -): CredentialMaterialResolver { - return (credentialId: string): CredentialMaterial => { - const delivery = cell.current; - if (delivery === null) { - throw new Error( - `inference credential material for ${credentialId} is not available: the delivery cell is empty`, - ); - } - const material = delivery.materials.find((entry) => entry.credentialId === credentialId); - if (material === undefined) { - throw new Error( - `inference credential material for ${credentialId} is no longer delivered: a re-push dropped it (rotated away or revoked)`, - ); - } - return { secret: material.secret }; - }; -} diff --git a/apps/sidecar/src/step-tool-materialization.ts b/apps/sidecar/src/step-tool-materialization.ts new file mode 100644 index 0000000000..c3f6227e8a --- /dev/null +++ b/apps/sidecar/src/step-tool-materialization.ts @@ -0,0 +1,132 @@ +// Pinned tool-package materialization for one workflow step. +// +// The child factory takes `materializeStepTools` as a required argument. +// This module is the sidecar process's implementation: it reads the step's +// deploy tree and loads the pinned closure. The factory never imports it. + +import path from "node:path"; + +import { agentDir, readDeployTree } from "@intx/hub-agent/paths"; +import { materializeToolPackages } from "@intx/tool-packaging"; +import { parseRunAddress } from "@intx/types/agent-address"; +import { baseStepId } from "@intx/workflow"; +import { resolveStepAddress } from "@intx/workflow-deploy/addresses"; +import type { StepToolCacheConfig, StepToolMaterialization } from "@intx/workflow-host/child"; + +import { readRegistries, resolveHostPlatform } from "./sidecar-materialization-config"; + +/** + * Resolve the on-disk directory holding a step's deploy tree. + * + * The deploy tree (`deploy/prompt.md`, `deploy/tool-packages-manifest.json`, + * `deploy/asset-mounts.json`) is shipped to the sidecar per step by the + * hub's `launchSession` deploy-pack push, which lands it in the LEGACY + * per-agent directory keyed by the step's sanitized mail address (see + * `@intx/hub-agent` `agentDir`). It is NOT in the substrate's + * `agent-state/` layout -- the multi-step deploy path never pushes + * step `agent-state` packs to the child's substrate. + * + * The step's mail address is `resolveStepAddress(...)`, the single owner + * of the head/step collapse: for a single-step deployment the lone step + * IS the head (the deployment mailbox itself), so the tree is read at the + * head; for multi-step it is `deriveStepAddress(runId, stepId, + * domain)`. The `runId`/`domain` are recovered from the deployment mailbox + * address the supervisor threaded into the child as `MAILBOX_ADDRESS` + * (`@`): the local part is the run id and the address domain + * is the deployment domain. `stepCount` is sourced from the host (via + * `substrateEnv`) so producer and consumer never derive divergent addresses. + */ +export function stepDeployTreeDir(args: { + dataDir: string; + mailboxAddress: string; + stepId: string; + stepCount: number; +}): string { + const parsed = parseRunAddress(args.mailboxAddress); + if (parsed === null) { + throw new Error( + `sidecar workflow-child step tools: deployment mailbox address ${JSON.stringify(args.mailboxAddress)} is not a parseable run address; cannot locate the step's deploy tree`, + ); + } + // A `map` iteration runs under a scoped step id `[]`, but + // deploy stages one deploy tree per base step, so the scoped id resolves + // to its base address -- every iteration reads the base step's tree. + // `baseStepId` is the identity on an unscoped id, so a plain step is + // unaffected. + const stepAddress = resolveStepAddress({ + runId: parsed.runId, + stepId: baseStepId(args.stepId), + domain: parsed.domain, + stepCount: args.stepCount, + }); + return agentDir(args.dataDir, stepAddress); +} + +/** + * Read a step's deploy tree and materialize its pinned tool-package + * closure. The tarball cache and the tool instance dir are rooted under + * the supplied per-step `storeDir` (the Phase-1 per-step state root) so + * concurrent steps/agents in one child never collide on cache or + * apply-state paths. + * + * A deploy with no tool-package manifest yields empty factories -- the + * legitimate `rawManifestBytes === undefined` case. A manifest that is + * present but fails to load surfaces loudly through + * `materializeToolPackages` (the throw path), never a silent + * empty-tools fallback that would mask a broken deploy. + */ +export async function materializeStepTools(args: { + dataDir: string; + mailboxAddress: string; + stepId: string; + stepCount: number; + /** Per-step state root; cache + instance dir + workspace live under it. */ + storeDir: string; + cache: StepToolCacheConfig; +}): Promise { + const deployTreeDir = stepDeployTreeDir({ + dataDir: args.dataDir, + mailboxAddress: args.mailboxAddress, + stepId: args.stepId, + stepCount: args.stepCount, + }); + const deployTree = await readDeployTree(deployTreeDir); + + // Root the tarball cache per step so concurrent steps in one child + // do not race on the content-addressable cache root. The cache is + // content-addressed and therefore safe to share globally, but the + // design (§3d, point 3) calls for a per-step cacheRoot so a wedged + // or partially-written apply in one step cannot corrupt another's + // view. + const cacheRoot = path.join(args.storeDir, "tarball-cache"); + + // Asset-mounted tool tarballs are staged by the hub's asset-pack push + // into the step's LEGACY agent dir workspace (the same dir the deploy + // tree lives in), not under the per-step store dir. Point the loader's + // asset resolution there while keeping the apply-state + cache rooted + // per step under `storeDir`. + // + // This `/workspace` (read-only staged assets, keyed by the + // BASE step) and the agent's read-write workdir `/workspace` + // (keyed by the SCOPED step) share a leaf name but are deliberately + // different roots -- a map iteration reads one shared deploy tree while + // each iteration writes its own scratch. Do not unify them. + const assetRoot = path.join(deployTreeDir, "workspace"); + + const materialized = await materializeToolPackages({ + rawManifestBytes: deployTree.toolPackageManifestRaw, + assetMounts: deployTree.assetMounts, + storeDir: args.storeDir, + assetRoot, + agentAddress: args.mailboxAddress, + cacheRoot, + cacheMaxBytes: args.cache.cacheMaxBytes, + registryMaxTarballBytes: args.cache.registryMaxTarballBytes, + registries: readRegistries(), + host: resolveHostPlatform(), + }); + return { + factories: materialized.factories, + pluginFactories: materialized.pluginFactories, + }; +} diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts deleted file mode 100644 index c035cba422..0000000000 --- a/apps/sidecar/src/tool-materialization.ts +++ /dev/null @@ -1,391 +0,0 @@ -// Lives in apps/sidecar (not @intx/workflow-host) to keep that package -// host-agnostic; the child IS the sidecar binary, so its tool runtime is -// already present. Kept free of @intx/harness so the workflow-process -// child's boot graph doesn't pull in its reactor stack (enforced by a -// forbidden-import guard). -// See docs/sidecar-tool-package-durability.md for the deploy-id persist protocol. - -import fs from "node:fs"; -import path from "node:path"; -import { type } from "arktype"; -import { type AnnotatedPluginFactory } from "@intx/agent"; -import { getLogger } from "@intx/log"; -import { - type LoadedToolFactory, - type LoadedToolPackage, - applyAtomic, - createTarballCache, - createToolLoader, -} from "@intx/tool-packaging"; -import { hasCode } from "@intx/types"; -import type { ToolCredentialDeclaration } from "@intx/types/package-json"; -import { ToolPackageManifest } from "@intx/types/tool-packages"; - -import { readRegistries, resolveHostPlatform } from "./sidecar-materialization-config"; - -const logger = getLogger(["sidecar", "harness-builder"]); - -// Sentinel the hub treats as "no prior deploy" rather than a real id. -const NO_PRIOR_DEPLOY_ID = "none"; - -/** - * Keeps package identity attached per factory because the per-bundle - * credential assembly needs the package name and declared handles that a - * bare factory (which carries only its bundle id) cannot supply. - */ -export interface StepToolFactory { - readonly packageName: string; - readonly declaredCredentials: readonly ToolCredentialDeclaration[]; - readonly factory: LoadedToolFactory; -} - -interface MaterializedToolPackages { - readonly factories: readonly StepToolFactory[]; - readonly pluginFactories: readonly AnnotatedPluginFactory[]; -} - -// Exported for direct unit testing of the manifest-invalid gate -// (JSON.parse failure + arktype schema failure). The workflow-process -// child reaches it through the substrate factory's per-step agent -// builder. -export async function materializeToolPackages(args: { - rawManifestBytes: string | undefined; - /** assetId -> workspace-relative mount path from deploy/asset-mounts.json. */ - assetMounts: ReadonlyMap; - storeDir: string; - /** - * Defaults to /workspace. The workflow-process child overrides - * this to keep its asset source and apply-state root as separate directories. - */ - assetRoot?: string; - agentAddress: string; - cacheRoot: string; - cacheMaxBytes: number; - registryMaxTarballBytes: number; -}): Promise { - if (args.rawManifestBytes === undefined) { - return { factories: [], pluginFactories: [] }; - } - // Local capture: readonly-property narrowing doesn't survive the nested - // closure boundary below. - const rawManifestBytes = args.rawManifestBytes; - - // Shares storeDir with the factories' own workspace; the substrate trusts - // factories not to write outside cwd rather than sandboxing this boundary. - const instanceDir = path.join(args.storeDir, "tool-packages"); - await fs.promises.mkdir(instanceDir, { recursive: true }); - - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const activeIdDirtyFile = `${activeIdFile}.dirty`; - // See docs/sidecar-tool-package-durability.md for the dirty-marker protocol. - let previousDeployId = NO_PRIOR_DEPLOY_ID; - try { - const dirtyRaw = (await fs.promises.readFile(activeIdDirtyFile, "utf-8")).trim(); - const dirtyId = parseActiveDeployId(dirtyRaw, activeIdDirtyFile); - logger.warn`active-deploy-id dirty marker present at ${activeIdDirtyFile}; the prior apply could not durably record the committed deploy id and the boot is reconciling from ${dirtyId}`; - previousDeployId = dirtyId; - } catch (err) { - if (!(hasCode(err) && err.code === "ENOENT")) { - throw err; - } - try { - const raw = (await fs.promises.readFile(activeIdFile, "utf-8")).trim(); - previousDeployId = parseActiveDeployId(raw, activeIdFile); - } catch (innerErr) { - if (!(hasCode(innerErr) && innerErr.code === "ENOENT")) { - throw innerErr; - } - } - } - - const attemptId = crypto.randomUUID(); - const newDeployId = crypto.randomUUID(); - - // Returns rather than throws so the caller can `throw await ...`, which - // TypeScript narrows control flow against with no dead-code suffix. - const rejectManifestInvalid = async (reason: string): Promise => { - const occurredAt = new Date().toISOString(); - logger.warn`tool-package apply rejected for ${args.agentAddress}: manifest.invalid — ${reason}`; - const message = `deploy/tool-packages-manifest.json could not be loaded: ${reason}`; - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId, - manifestBytes: rawManifestBytes, - failure: { - attemptId, - previousDeployId, - category: "manifest.invalid", - message, - occurredAt, - }, - }); - return new Error(`tool-package apply rejected (manifest.invalid): ${reason}`); - }; - - let parsedManifest: unknown; - try { - parsedManifest = JSON.parse(rawManifestBytes); - } catch (err) { - throw await rejectManifestInvalid( - `JSON.parse failed: ${err instanceof Error ? err.message : String(err)}`, - ); - } - const validated = ToolPackageManifest(parsedManifest); - if (validated instanceof type.errors) { - throw await rejectManifestInvalid(`schema validation failed: ${validated.summary}`); - } - - const cache = createTarballCache({ - rootDir: args.cacheRoot, - maxBytes: args.cacheMaxBytes, - }); - const loader = createToolLoader({ - cache, - registries: readRegistries(), - host: resolveHostPlatform(), - maxRegistryTarballBytes: args.registryMaxTarballBytes, - }); - const result = await applyAtomic({ - manifest: validated, - loader, - instanceDir, - assetRoot: args.assetRoot ?? path.join(args.storeDir, "workspace"), - assetMounts: args.assetMounts, - // Only workflow-definition closures carry git entries. - gitDirs: new Map(), - attemptId, - previousDeployId, - newDeployId, - }); - if (result.status === "failed") { - logger.warn`tool-package apply rejected for ${args.agentAddress}: ${result.category} — ${result.message}`; - // A failed apply never wrote active-deploy-id, so the prior deploy is - // trivially still live. - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId: result.attemptId, - // Raw bytes from disk, not the arktype-narrowed object — preserves - // whitespace/key order/unknown fields for replay against a newer validator. - manifestBytes: rawManifestBytes, - failure: result, - }); - throw new Error(`tool-package apply rejected (${result.category}): ${result.message}`); - } - - // Persisting the id is the commit. On a degraded persist, throw so the - // harness tears down — see docs/sidecar-tool-package-durability.md. - const persistOutcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - result.activeDeployId, - ); - if (persistOutcome.degraded) { - const err = persistOutcome.error; - const occurredAt = new Date().toISOString(); - const message = `active-deploy-id persist failed after staging deploy: ${err instanceof Error ? err.message : String(err)}`; - logger.error`tool-package apply: ${message}; active deploy ${result.activeDeployId} is live on disk but the recorded id was not durably written`; - await writeRejectedApplyAudit({ - storeDir: args.storeDir, - attemptId, - manifestBytes: rawManifestBytes, - failure: { - attemptId, - previousDeployId: result.activeDeployId, - category: "apply.previous-rotation.failed", - message, - occurredAt, - }, - }); - throw new Error(`tool-package apply rejected (apply.previous-rotation.failed): ${message}`, { - cause: err, - }); - } - return { - factories: collectFactories(result.loaded), - pluginFactories: collectPluginFactories(result.loaded), - }; -} - -/** - * POSIX doesn't guarantee a parent-directory entry is durably linked from a - * file's own fsync alone, so the dir handle is synced separately. Dir-fsync - * itself is best-effort: some filesystems reject it (FAT/exFAT), which - * shouldn't force a restart when the file's own fsync already landed. - */ -const ACTIVE_DEPLOY_ID_VERSION = "v1"; -const ACTIVE_DEPLOY_ID_PREFIX = `${ACTIVE_DEPLOY_ID_VERSION}:`; - -// Exported for unit testing of the version-prefix gate. -export function parseActiveDeployId(raw: string, sourcePath: string): string { - if (raw.length === 0) { - throw new Error( - `active-deploy-id file ${sourcePath} is empty; expected ${ACTIVE_DEPLOY_ID_PREFIX}`, - ); - } - // Accepts pre-versioning unprefixed files so upgrade needs no operator - // rewrite; delete this branch once every sidecar has restarted on a - // version that writes the prefix (see docs/sidecar-tool-package-durability.md). - if (raw.startsWith(ACTIVE_DEPLOY_ID_PREFIX)) { - const id = raw.slice(ACTIVE_DEPLOY_ID_PREFIX.length); - if (id.length === 0) { - throw new Error( - `active-deploy-id file ${sourcePath} carries the ${ACTIVE_DEPLOY_ID_VERSION} prefix but no id`, - ); - } - return id; - } - if (raw.includes(":")) { - // A leading token shaped like a version prefix that we do not - // recognize. Surface loudly rather than treating the whole string - // as the deploy id. - const prefix = raw.slice(0, raw.indexOf(":") + 1); - throw new Error( - `active-deploy-id file ${sourcePath} carries unknown version prefix ${JSON.stringify(prefix)}; this sidecar understands ${JSON.stringify(ACTIVE_DEPLOY_ID_PREFIX)}`, - ); - } - return raw; -} - -function formatActiveDeployId(deployId: string): string { - return `${ACTIVE_DEPLOY_ID_PREFIX}${deployId}`; -} - -async function persistActiveDeployId( - instanceDir: string, - activeIdFile: string, - deployId: string, -): Promise { - const handle = await fs.promises.open(activeIdFile, "w"); - try { - await handle.writeFile(formatActiveDeployId(deployId)); - await handle.sync(); - } finally { - await handle.close(); - } - try { - const dirHandle = await fs.promises.open(instanceDir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`parent-dir fsync failed for ${instanceDir} after deploy-id persist; deploy-id durability is degraded but the committed deploy is staged on disk — ${err instanceof Error ? err.message : String(err)}`; - } - await clearDirtyMarker(activeIdFile, "successful persist"); -} - -/** - * Best-effort: a leftover marker would otherwise shadow the recorded id on - * the next boot, since the boot reader prefers it when present. - */ -export async function clearDirtyMarker(activeIdFile: string, reason: string): Promise { - try { - await fs.promises.unlink(`${activeIdFile}.dirty`); - } catch (err) { - if (!(hasCode(err) && err.code === "ENOENT")) { - logger.warn`failed to clear active-deploy-id dirty marker after ${reason}: ${err instanceof Error ? err.message : String(err)}`; - } - } -} - -// See docs/sidecar-tool-package-durability.md for the degradation ladder. -export async function persistActiveDeployIdWithFallback( - instanceDir: string, - activeIdFile: string, - deployId: string, -): Promise<{ readonly degraded: boolean; readonly error?: unknown }> { - try { - await persistActiveDeployId(instanceDir, activeIdFile, deployId); - return { degraded: false }; - } catch (primary) { - logger.warn`active-deploy-id primary persist failed for ${activeIdFile}: ${primary instanceof Error ? primary.message : String(primary)}; attempting degraded write`; - try { - await fs.promises.writeFile(activeIdFile, formatActiveDeployId(deployId)); - logger.warn`active-deploy-id degraded write succeeded (no fsync); next boot reads the recorded id from disk but a crash before flush may reveal the prior id`; - await clearDirtyMarker(activeIdFile, "degraded persist"); - return { degraded: true, error: primary }; - } catch (fallback) { - logger.error`active-deploy-id degraded write also failed for ${activeIdFile}: ${fallback instanceof Error ? fallback.message : String(fallback)}; writing dirty marker so the next boot can reconcile`; - try { - const dirtyPath = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyPath, formatActiveDeployId(deployId)); - logger.warn`active-deploy-id dirty marker written to ${dirtyPath}; next boot will prefer it over the stale recorded id`; - return { degraded: true, error: primary }; - } catch (marker) { - logger.error`active-deploy-id dirty marker write also failed for ${activeIdFile}.dirty: ${marker instanceof Error ? marker.message : String(marker)}; on-disk state will diverge from the recorded id for one boot cycle`; - return { degraded: true, error: primary }; - } - } - } -} - -// See docs/sidecar-tool-package-durability.md for the audit trail format. -async function writeRejectedApplyAudit(args: { - storeDir: string; - attemptId: string; - manifestBytes: string; - failure: { - attemptId: string; - previousDeployId: string; - category: string; - message: string; - package?: { name: string; version: string }; - occurredAt: string; - }; -}): Promise { - const dir = path.join(args.storeDir, "audit", "rejected-applies", args.attemptId); - await fs.promises.mkdir(dir, { recursive: true }); - // Fsynced before the caller throws so a crash right after doesn't lose - // the evidence; fsync failures downgrade to warnings, not a second failure. - await fsyncWriteFile(path.join(dir, "manifest.json"), args.manifestBytes); - await fsyncWriteFile(path.join(dir, "error.json"), JSON.stringify(args.failure, null, 2)); - try { - const dirHandle = await fs.promises.open(dir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close(); - } - } catch (err) { - logger.warn`audit-dir fsync failed for ${dir}; rejected-apply durability is degraded but the files are written — ${err instanceof Error ? err.message : String(err)}`; - } -} - -async function fsyncWriteFile(filePath: string, contents: string): Promise { - const handle = await fs.promises.open(filePath, "w"); - try { - await handle.writeFile(contents); - try { - await handle.sync(); - } catch (err) { - logger.warn`fsync failed for ${filePath}; durability is degraded but the bytes are written — ${err instanceof Error ? err.message : String(err)}`; - } - } finally { - await handle.close(); - } -} - -function collectFactories(loaded: readonly LoadedToolPackage[]): readonly StepToolFactory[] { - const out: StepToolFactory[] = []; - for (const pkg of loaded) { - for (const f of pkg.factories) { - out.push({ - packageName: pkg.name, - declaredCredentials: pkg.credentials, - factory: f, - }); - } - } - return out; -} - -function collectPluginFactories( - loaded: readonly LoadedToolPackage[], -): readonly AnnotatedPluginFactory[] { - const out: AnnotatedPluginFactory[] = []; - for (const pkg of loaded) { - for (const p of pkg.plugins) out.push(p); - } - return out; -} diff --git a/apps/sidecar/src/workflow-child-bindings.ts b/apps/sidecar/src/workflow-child-bindings.ts new file mode 100644 index 0000000000..fd4452c3ac --- /dev/null +++ b/apps/sidecar/src/workflow-child-bindings.ts @@ -0,0 +1,26 @@ +// Composition edge for the workflow child. +// +// Closes the child factory over the sidecar's tool materializer and the +// grant cap, and re-exports the substrate-config key list the binary +// forwards. The binary imports this module and the workflow-host child entry. + +import { + collectDeclaredCredentialConsumers, + collectDeclaredResources, + filterGrantsToDeclaredResources, +} from "@intx/workflow-deploy/child-grants"; +import { + SIDECAR_SUBSTRATE_CONFIG_KEYS, + createSidecarSubstrateFactory, +} from "@intx/workflow-host/child"; + +import { materializeStepTools } from "./step-tool-materialization"; + +export { SIDECAR_SUBSTRATE_CONFIG_KEYS }; + +export const createSubstrate = createSidecarSubstrateFactory({ + materializeStepTools, + collectDeclaredResources, + collectDeclaredCredentialConsumers, + filterGrantsToDeclaredResources, +}); diff --git a/apps/sidecar/src/workflow-child-spawner.ts b/apps/sidecar/src/workflow-child-spawner.ts new file mode 100644 index 0000000000..e7b0354728 --- /dev/null +++ b/apps/sidecar/src/workflow-child-spawner.ts @@ -0,0 +1,230 @@ +// Bun-backed spawn edge for the workflow child. +// +// The deploy program takes a spawner and a binary path as required +// arguments. This module is the process that owns both: it resolves +// `bin/workflow-child` and launches it in its own process group. + +import { fileURLToPath } from "node:url"; + +import { getLogger } from "@intx/log"; +import type { + FrameReader, + NdjsonReader, + NdjsonWriter, + SubprocessHandle, + SubprocessSpawner, +} from "@intx/workflow-host"; + +const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); + +/** + * Path of this package's `bin/workflow-child`. Resolved at load so the + * boot edge can hand the deploy program a concrete path. + */ +export const SIDECAR_WORKFLOW_CHILD_BINARY: string = (() => { + const url = import.meta.resolve("../bin/workflow-child"); + return fileURLToPath(url); +})(); + +/** + * Child fd the supervisor inherits the event-channel pipe on. The + * supervisor's spawn-time convention is: + * + * fd 0 stdin -- downstream control channel (supervisor -> child) + * fd 1 stdout -- upstream control channel (child -> supervisor) + * fd 2 stderr -- inherited so child diagnostics land on the + * sidecar's stderr + * fd 3 -- event-channel write side (child writes + * HMAC-authenticated InferenceEvent frames here; + * the supervisor reads the parent end as a + * `FrameReader`) + * + * The child opens fd 3 via `EVENT_CHANNEL_FD` in + * `@intx/workflow-host`'s `from-process-env`. The two ends of the + * pipe are provisioned by `Bun.spawn`'s `stdio` slot: setting + * `stdio[3] = "pipe"` makes Bun mint a pipe pair where the child + * inherits the write half at fd 3 and the parent receives the read + * half as a numeric fd at `proc.stdio[3]` in its own address space. + */ +const CHILD_EVENT_CHANNEL_FD = 3; + +/** + * Wrap a Bun `FileSink` as the supervisor's `NdjsonWriter`. The + * supervisor's control-channel sender writes one JSON line per + * frame (already including the trailing newline); the writer is + * responsible for passing the bytes through to the child's stdin + * without buffering across frames so each frame surfaces on the + * far side as soon as `write()` resolves. + */ +function ndjsonWriterFromFileSink(sink: Bun.FileSink): NdjsonWriter { + return { + async write(line: string): Promise { + const result = sink.write(line); + if (typeof result !== "number") await result; + const flushed = sink.flush(); + if (typeof flushed !== "number") await flushed; + }, + }; +} + +/** + * Wrap a Bun stdout `ReadableStream` as the supervisor's + * `NdjsonReader`. The pipe is a byte stream; this reader buffers + * partial chunks and yields one complete line per iteration. The + * receiver's iterator finalises only on EOF, which mirrors the + * `defaultControlReader` shape the child wires for `process.stdin`. + */ +function ndjsonReaderFromReadableStream(stream: ReadableStream): NdjsonReader { + return { + read(): AsyncIterableIterator { + return (async function* () { + const decoder = new TextDecoder("utf-8"); + let pending = ""; + const reader = stream.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (value !== undefined) { + pending += decoder.decode(value, { stream: true }); + let nl = pending.indexOf("\n"); + while (nl >= 0) { + const line = pending.slice(0, nl).replace(/\r$/, ""); + pending = pending.slice(nl + 1); + if (line.length > 0) yield line; + nl = pending.indexOf("\n"); + } + } + if (done) break; + } + if (pending.length > 0) yield pending; + } finally { + reader.releaseLock(); + } + })(); + }, + }; +} + +/** + * Wrap the parent-side read fd of the event-channel pipe as the + * supervisor's `FrameReader`. The child publishes one HMAC- + * authenticated envelope per `FileSink.write()` and the supervisor's + * `receiveEventChannel` parses each yielded `Uint8Array` as one + * complete envelope. The pipe is a byte stream; this reader yields + * each raw chunk the kernel delivers and trusts the sender's + * one-write-per-envelope discipline. The buffer-overflow / framing + * discipline lives in `receiveEventChannel`'s parser. + */ +function frameReaderFromFd(fd: number): FrameReader { + const stream = Bun.file(fd).stream(); + return { + read(): AsyncIterableIterator { + return (async function* () { + const reader = stream.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (value !== undefined && value.byteLength > 0) yield value; + if (done) break; + } + } finally { + reader.releaseLock(); + } + })(); + }, + }; +} + +/** + * Real `Bun.spawn`-backed subprocess spawner. Constructs a fresh env + * carrying exactly the trust anchors and substrate-config keys the + * supervisor passed in (no inheritance of the sidecar's process env); + * inherits stdio 0/1/2 as control + stderr; pipes fd 3 for the event + * channel and surfaces the parent-side read fd as the supervisor's + * `FrameReader`. + * + * A child that fails to launch (binary missing or not executable) + * makes `Bun.spawn` throw, so this spawner throws synchronously and no + * handle exists. If Bun returns no event channel fd, the spawner kills + * the child's process group and throws the same way. The supervisor + * calls the spawner before it enters `starting`, so a first spawn + * rejects and the supervisor stays idle; a recycle that fails this way + * shuts the supervisor down. A child that launches and then dies + * before `ready` ends the control channel, which rejects the ready + * handshake; the ready timeout covers a child that keeps the channel + * open without sending `ready`. + */ +export const defaultSubprocessSpawner: SubprocessSpawner = ({ + binaryPath, + env, +}): SubprocessHandle => { + // The child leads its own process group so a stop reaches the tool + // subprocesses it started, not just the child's own pid. + const proc = Bun.spawn([binaryPath], { + stdio: ["pipe", "pipe", "inherit", "pipe"], + env, + detached: true, + }); + // Once the final group kill has run, the group id may be reused by an + // unrelated process group, so later signals must not reach it. + let groupReleased = false; + function signalGroup(signal: number | NodeJS.Signals): void { + if (groupReleased) return; + try { + process.kill(-proc.pid, signal); + } catch (err) { + if (!(err instanceof Error && "code" in err && err.code === "ESRCH")) throw err; + } + } + const eventFd = proc.stdio[CHILD_EVENT_CHANNEL_FD]; + if (typeof eventFd !== "number") { + // The throw drops the only handle to a child that leads its own group, + // so stop the group here or nothing will. + try { + signalGroup("SIGKILL"); + } catch (err) { + logger.error`Cannot stop workflow child ${String(proc.pid)} that has no event channel: ${err instanceof Error ? err.message : String(err)}`; + } + throw new Error( + `workflow-host-wiring: Bun.spawn did not return a numeric fd at stdio[${String(CHILD_EVENT_CHANNEL_FD)}] for the event channel; got ${typeof eventFd}`, + ); + } + return { + pid: proc.pid, + controlWriter: ndjsonWriterFromFileSink(proc.stdin), + controlReader: ndjsonReaderFromReadableStream(proc.stdout), + eventReader: frameReaderFromFd(eventFd), + kill(signal?: number | string): void { + // The supervisor's `SubprocessHandle.kill` widens the signal + // to `number | string`; `process.kill` accepts + // `number | NodeJS.Signals`. The supervisor's call sites pass + // `"SIGTERM"` / `"SIGKILL"` (recycle path) or no argument + // (shutdown path). Cast at the boundary so the inner call + // matches the narrower type without coercing valid input. + if (signal === undefined) { + signalGroup("SIGTERM"); + return; + } + if (typeof signal === "number") { + signalGroup(signal); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- supervisor's kill widens to `string`; the runtime accepts the same `"SIG*"` strings, narrowed back at the boundary. + signalGroup(signal as NodeJS.Signals); + }, + // Tool subprocesses outlive a child that exits or is killed, so the + // group is killed before `exited` settles. If the group was already + // empty when Bun reaped the child, its id could be reused before this + // kill lands; the window is brief, and closing it needs a process + // handle such as a pidfd, which Bun does not expose. + exited: proc.exited.then((code) => { + try { + signalGroup("SIGKILL"); + groupReleased = true; + } catch (err) { + logger.error`Cannot stop processes left by workflow child ${String(proc.pid)}: ${err instanceof Error ? err.message : String(err)}`; + } + return code; + }), + }; +}; diff --git a/apps/sidecar/src/workflow-closure-apply.ts b/apps/sidecar/src/workflow-closure-apply.ts deleted file mode 100644 index a64d97fe6a..0000000000 --- a/apps/sidecar/src/workflow-closure-apply.ts +++ /dev/null @@ -1,160 +0,0 @@ -// The durable deploy counterpart to the airlocked install-time probe: the -// closure is applied byte-for-byte as the hub froze it, never re-resolved -// against the registry. topLevel is emptied for applyAtomic since a -// workflow-definition package declares interchange.workflow, not -// interchange.tools, which applyAtomic's load phase would otherwise reject -// as package.entry.missing; loadWorkflowDefinitionFromClosure imports the -// workflow entry instead. - -import path from "node:path"; - -import { getLogger } from "@intx/log"; -import { - type RegistryConfig, - type TarballFetcher, - applyAtomic, - createTarballCache, - createToolLoader, - storeEntryDir, -} from "@intx/tool-packaging"; -import type { ToolPackageManifest } from "@intx/types/tool-packages"; -import { getToolPackageSourceContentIdentity } from "@intx/types/tool-packages"; -import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; -import { loadWorkflowDefinitionFromClosure } from "@intx/workflow-host"; -import type { WorkflowDefinition } from "@intx/workflow/definition"; - -const logger = getLogger(["sidecar", "workflow-closure-apply"]); - -export interface ApplyFrozenWorkflowClosureArgs { - /** Names the registry the workflow definition package is published to. */ - readonly source: WorkflowDefinitionSource; - /** Concrete versions and integrity SRIs; applied byte-for-byte, never re-resolved. */ - readonly closure: ToolPackageManifest; - /** Durable per-deployment directory the closure is staged under. */ - readonly instanceDir: string; - /** Content-addressable tarball cache root shared across applies. */ - readonly cacheRoot: string; - /** Byte cap for the tarball cache. */ - readonly cacheMaxBytes: number; - /** Byte cap for a single HTTP-registry tarball fetch. */ - readonly registryMaxTarballBytes: number; - /** Registry identifier -> URL + credentials the loader resolves entries against. */ - readonly registries: ReadonlyMap; - /** Defaults to /workspace; a registry-sourced closure carries no asset entries. */ - readonly assetRoot?: string; - /** assetId -> mount path for tarball asset entries; empty by default. */ - readonly assetMounts?: ReadonlyMap; - /** assetId -> indexed git directory for source-format entries; empty by default. */ - readonly gitDirs?: ReadonlyMap; - /** Test seam; production fetches from the configured registry. */ - readonly fetchTarball?: TarballFetcher; - /** Test seam; production imports the materialized entry natively. */ - readonly importModule?: (importUrl: string) => Promise; -} - -export interface AppliedWorkflowClosure { - /** The validated definition the pinned code evaluated to. */ - readonly definition: WorkflowDefinition; - /** Directory of the materialized workflow package within the closure. */ - readonly packageDir: string; - /** The staged, never-renamed deploy directory the closure was laid out under. */ - readonly deployDir: string; -} - -/** The frozen entries are applied verbatim; no registry re-resolution happens at apply time. */ -export async function applyFrozenWorkflowClosure( - args: ApplyFrozenWorkflowClosureArgs, -): Promise { - if (args.closure.topLevel.length !== 1) { - throw new Error( - `sidecar workflow-closure apply: the frozen closure must carry exactly one top-level pin (the workflow definition package), got ${String(args.closure.topLevel.length)}`, - ); - } - const workflowPin = args.closure.topLevel[0]; - if (workflowPin === undefined) { - throw new Error( - "sidecar workflow-closure apply: the frozen closure's single top-level pin is undefined", - ); - } - - // The never default makes a future source kind a compile error, not a silent fallthrough. - switch (args.source.kind) { - case "registry": - if (!args.registries.has(args.source.registry)) { - throw new Error( - `sidecar workflow-closure apply: source registry ${JSON.stringify(args.source.registry)} is not in the sidecar registry config`, - ); - } - break; - case "asset": - break; - default: { - const _exhaustive: never = args.source; - throw new Error( - `sidecar workflow-closure apply: unhandled workflow source kind ${String(_exhaustive)}`, - ); - } - } - - const cache = createTarballCache({ - rootDir: args.cacheRoot, - maxBytes: args.cacheMaxBytes, - }); - const loader = createToolLoader({ - cache, - registries: args.registries, - host: { os: process.platform, cpu: process.arch }, - maxRegistryTarballBytes: args.registryMaxTarballBytes, - ...(args.fetchTarball !== undefined ? { fetchTarball: args.fetchTarball } : {}), - }); - - const layoutManifest: ToolPackageManifest = { - schemaVersion: args.closure.schemaVersion, - topLevel: [], - entries: args.closure.entries, - }; - - const result = await applyAtomic({ - manifest: layoutManifest, - loader, - instanceDir: args.instanceDir, - assetRoot: args.assetRoot ?? path.join(args.instanceDir, "workspace"), - assetMounts: args.assetMounts ?? new Map(), - gitDirs: args.gitDirs ?? new Map(), - attemptId: crypto.randomUUID(), - // No prior deploy under instanceDir to retain, so this disables the retention window. - previousDeployId: "none", - newDeployId: crypto.randomUUID(), - }); - if (result.status === "failed") { - throw new Error( - `sidecar workflow-closure apply: materializing the frozen closure for ${workflowPin.name}@${workflowPin.version} failed (${result.category}): ${result.message}`, - ); - } - - const packageDir = storeEntryDir( - path.join(result.deployDir, "store"), - workflowPin.name, - workflowPin.version, - ); - - // The package's own integrity is the cache-bust token: a re-apply of - // changed bytes under the same name@version reimports rather than - // resolving to the prior module-cache instance. - const workflowEntry = args.closure.entries.find( - (entry) => entry.name === workflowPin.name && entry.version === workflowPin.version, - ); - - const definition = await loadWorkflowDefinitionFromClosure({ - packageDir, - ...(workflowEntry !== undefined - ? { - importCacheKey: getToolPackageSourceContentIdentity(workflowEntry.source), - } - : {}), - ...(args.importModule !== undefined ? { importModule: args.importModule } : {}), - }); - - logger.debug`applied frozen workflow closure ${workflowPin.name}@${workflowPin.version}: loaded definition ${definition.id}`; - return { definition, packageDir, deployDir: result.deployDir }; -} diff --git a/apps/sidecar/src/workflow-closure-materialization.ts b/apps/sidecar/src/workflow-closure-materialization.ts deleted file mode 100644 index b2f6cee09f..0000000000 --- a/apps/sidecar/src/workflow-closure-materialization.ts +++ /dev/null @@ -1,214 +0,0 @@ -// Materializes a probe's frozen closure on the sidecar host (I/O only, no -// author-code eval) before the airlocked child evaluates it. Lives in -// apps/sidecar so @intx/workflow-host stays free of @intx/tool-packaging. -// loadManifest runs with an emptied topLevel so it lays out every entry -// without importing (evaluating) any of them. - -import { promises as fs } from "node:fs"; -import path from "node:path"; - -import { type } from "arktype"; -import { getLogger } from "@intx/log"; -import { - type RegistryConfig, - type TarballFetcher, - createTarballCache, - createToolLoader, - storeEntryDir, -} from "@intx/tool-packaging"; -import { PackageJSON } from "@intx/types/package-json"; -import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; -import type { ToolPackageManifest } from "@intx/types/tool-packages"; - -import { materializeWorkflowAssets } from "./source-asset-delivery"; -import { resolveHostPlatform } from "./sidecar-materialization-config"; -import type { - MaterializedWorkflowClosure, - MaterializeWorkflowClosure, -} from "./workflow-probe-handler"; - -const logger = getLogger(["sidecar", "workflow-closure-materialization"]); - -export interface WorkflowClosureMaterializerConfig { - /** Content-addressable tarball cache root shared across materializations. */ - readonly cacheRoot: string; - /** Byte cap for the tarball cache. */ - readonly cacheMaxBytes: number; - /** Byte cap for a single HTTP-registry tarball fetch. */ - readonly registryMaxTarballBytes: number; - /** Enforced against the base64 wire length, before decoding, so an oversized frame fails loud. */ - readonly maxAssetPayloadBytes: number; - /** Registry identifier -> URL + credentials the loader resolves entries against. */ - readonly registries: ReadonlyMap; - /** - * Root directory under which each probe's ephemeral closure scratch dir - * is created (one per probe, removed by the returned `cleanup`). - */ - readonly scratchRoot: string; - /** - * Test seam for tarball fetching, forwarded to `createToolLoader`. - * Production omits it and the loader fetches from the configured registry. - */ - readonly fetchTarball?: TarballFetcher; -} - -/** Lays out a probe frame's frozen closure under a fresh scratch dir removed by the returned cleanup. */ -export function createWorkflowClosureMaterializer( - config: WorkflowClosureMaterializerConfig, -): MaterializeWorkflowClosure { - const host = resolveHostPlatform(); - - return async function materialize( - frame: WorkflowProbeRequestFrame, - ): Promise { - // Fail loud rather than silently picking [0]: a 0- or >1-pin closure is - // an incoherent request this boundary owns rejecting. - const topLevel = frame.closure.topLevel; - if (topLevel.length !== 1) { - throw new Error( - `workflow-probe closure materialization: the frozen closure must pin exactly one top-level package (the workflow definition package), got ${String(topLevel.length)}`, - ); - } - const workflowPin = topLevel[0]; - if (workflowPin === undefined) { - throw new Error( - "workflow-probe closure materialization: the frozen closure's single top-level pin is undefined", - ); - } - - // The never default makes a future source kind a compile error, not a silent fallthrough. - switch (frame.source.kind) { - case "registry": - if (!config.registries.has(frame.source.registry)) { - throw new Error( - `workflow-probe closure materialization: source registry ${JSON.stringify(frame.source.registry)} is not in the sidecar registry config`, - ); - } - break; - case "asset": - break; - default: { - const _exhaustive: never = frame.source; - throw new Error( - `workflow-probe closure materialization: unhandled workflow source kind ${String(_exhaustive)}`, - ); - } - } - - const scratchDir = path.join(config.scratchRoot, crypto.randomUUID()); - await fs.mkdir(scratchDir, { recursive: true }); - const cleanup = async (): Promise => { - await fs.rm(scratchDir, { recursive: true, force: true }); - }; - - try { - const cache = createTarballCache({ - rootDir: config.cacheRoot, - maxBytes: config.cacheMaxBytes, - }); - const loader = createToolLoader({ - cache, - registries: config.registries, - host, - maxRegistryTarballBytes: config.registryMaxTarballBytes, - ...(config.fetchTarball !== undefined ? { fetchTarball: config.fetchTarball } : {}), - }); - - // Registry-sourced closures deliver no assets; both maps stay empty - // and the loader fetches over HTTP instead. - const assetRoot = path.join(scratchDir, "workspace"); - const gitDirRoot = path.join(scratchDir, "gitdirs"); - const { assetMounts, gitDirs } = await materializeWorkflowAssets({ - assets: frame.assets ?? [], - closure: frame.closure, - assetRoot, - gitDirRoot, - maxAssetPayloadBytes: config.maxAssetPayloadBytes, - }); - - // The asset the definition is sourced from must be among the delivered - // assets; surface a missing delivery here, not as a downstream failure. - if (frame.source.kind === "asset") { - const delivered = - frame.source.package.format === "source" - ? gitDirs.has(frame.source.assetId) - : assetMounts.has(frame.source.assetId); - if (!delivered) { - throw new Error( - `workflow-probe closure materialization: asset source ${JSON.stringify(frame.source.assetId)} was not among the delivered assets`, - ); - } - } - - // topLevel emptied so the loader's phase-3 import loop runs nothing on - // the host; the airlocked child owns the single import. - const layoutManifest: ToolPackageManifest = { - schemaVersion: frame.closure.schemaVersion, - topLevel: [], - entries: frame.closure.entries, - }; - await loader.loadManifest({ - manifest: layoutManifest, - instanceScratchDir: scratchDir, - assetRoot, - assetMounts, - gitDirs, - }); - - const storeDir = path.join(scratchDir, "store"); - const packageDir = storeEntryDir(storeDir, workflowPin.name, workflowPin.version); - - await assertFrameEntryMatchesPackage(packageDir, frame.entry); - - logger.debug`materialized workflow-probe closure for ${workflowPin.name}@${workflowPin.version} at ${packageDir}`; - return { packageDir, cleanup }; - } catch (err) { - // The executor never sees a cleanup to call before a handle is returned. - await cleanup(); - throw err; - } - }; -} - -/** The child loader ignores frame.entry; comparing it host-side surfaces a tampered request before the child spawns. */ -async function assertFrameEntryMatchesPackage( - packageDir: string, - frameEntry: string, -): Promise { - const pkgJsonPath = path.join(packageDir, "package.json"); - let raw: string; - try { - raw = await fs.readFile(pkgJsonPath, "utf8"); - } catch (cause) { - throw new Error( - `workflow-probe closure materialization: cannot read package.json at ${packageDir} to cross-check the frame entry`, - { cause }, - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new Error( - `workflow-probe closure materialization: malformed package.json at ${packageDir}`, - { cause }, - ); - } - const pkg = PackageJSON(parsed); - if (pkg instanceof type.errors) { - throw new Error( - `workflow-probe closure materialization: package.json at ${packageDir} failed validation: ${pkg.summary}`, - ); - } - const declaredEntry = pkg.interchange?.workflow; - if (declaredEntry === undefined) { - throw new Error( - `workflow-probe closure materialization: workflow package at ${packageDir} declares no "interchange.workflow" entry`, - ); - } - if (declaredEntry !== frameEntry) { - throw new Error( - `workflow-probe closure materialization: probe frame entry ${JSON.stringify(frameEntry)} does not match the materialized package's interchange.workflow ${JSON.stringify(declaredEntry)}`, - ); - } -} diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts deleted file mode 100644 index 2202f747ba..0000000000 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ /dev/null @@ -1,1690 +0,0 @@ -// Thin wiring for createWorkflowSupervisor's sidecar-specific bindings; any -// logic reusable by a future alternative sidecar belongs in @intx/workflow-host, not here. - -import { rm, stat } from "node:fs/promises"; -import { join as pathJoin } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { type } from "arktype"; - -import { derivePublicKeyBytes, signEd25519 } from "@intx/crypto"; -import { getLogger } from "@intx/log"; -import type { HubTransport } from "@intx/mail-memory"; -import { - parseAgentId, - workflowSourceAssetMountPath, - type Principal, - type RepoId, - type RepoStore, - type WorkflowRunSupervisorPrincipal, -} from "@intx/hub-sessions"; -import { - resolveInboundMailPolicy, - type AgentKeyStore, - type DeployRouter, - type DeployRouterResult, - type InboundMailPolicyRegistry, - type SenderKeyCache, - type SessionManager, -} from "@intx/hub-agent"; -import { - createWorkflowSupervisor, - hashGrants, - STEP_GRANTS_PATH, - STEP_GRANTS_REF, - wrapHubTransportAsMailBus, - type CredentialsSnapshot, - type CredentialsSnapshotStep, - type DeriveStepAddress, - type DeriveStepRepoId, - type DispatchTimingMark, - type FrameReader, - type HubTransportMailBusAdapter, - type NdjsonReader, - type NdjsonWriter, - type SpawnOpts, - type SubprocessHandle, - type SubprocessSpawner, - type SuspensionRegistration, - type WorkflowSupervisor, -} from "@intx/workflow-host"; -import { hexDecode, hexEncode, type CredentialCipher, type SignalKind } from "@intx/types"; -import { - parseInferenceEvent, - type ApprovalSnapshot, - type CryptoProvider, - type InferenceEvent, - type InferenceSource, - type KeyPair, -} from "@intx/types/runtime"; -import { - WorkflowProjectionDefinition, - type AgentDeployFrame, - type CredentialDelivery, - type SourceRefPin, -} from "@intx/types/sidecar"; -import { STEP_ID_PATTERN, projectLiveToInert } from "@intx/workflow"; -import { deriveWorkflowRunRepoId, inertFlatNamespaceStepIds } from "@intx/workflow-deploy"; - -import { applyFrozenWorkflowClosure, type AppliedWorkflowClosure } from "./workflow-closure-apply"; -import { - MAX_INLINE_ASSET_PAYLOAD_BYTES, - materializeWorkflowAssets, - sourceAssetGitDir, -} from "./source-asset-delivery"; -import { readRegistries } from "./sidecar-materialization-config"; - -import type { - MultistepDrainRouter, - MultistepGrantsRouter, - MultistepMailRouter, - MultistepSignalRouter, - MultistepSourcesRouter, - MultistepCredentialsRouter, -} from "./workflow-run-pack-client"; -import { - deleteWorkflowRunRecord, - scanWorkflowRunRecords, - writeWorkflowRunRecord, - type WorkflowRunRecord, -} from "./workflow-run-record"; -import { readRunGrants, runGrantsPath } from "./run-grants"; - -const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); - -// A raw Ed25519 public key is 32 bytes. -const ED25519_PUBLIC_KEY_BYTES = 32; - -/** Thin delegator to `@intx/workflow-deploy` so the hub's read routes reconstruct the identical id; named for the deploy-phase routing slug, not a run identity. */ -export function deriveDeploymentId(agentAddress: string): string { - return deriveWorkflowRunRepoId(agentAddress); -} - -/** A sibling of the closure instance dir, never reclaimed by apply/restore, so checked-out assets survive a restart. */ -function deploymentSourceAssetRoot(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-definition-sources", deploymentId); -} - -/** - * The durable indexed-`.git` store root a pinned deployment's source-format - * asset entries are checked out from. Sibling of the plain-file source store; - * both survive restart so re-materialization needs no re-delivery. - */ -function deploymentSourceGitRoot(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-definition-source-gits", deploymentId); -} - -/** - * The `assetId -> mountPath` map a pinned closure's TARBALL `kind:"asset"` - * entries resolve against, derived purely from the pin (via the shared - * mount-path helper) so deploy and restore agree without the frame's delivered - * assets. Source-format entries resolve through `deriveSourceGitDirs` instead. - */ -function deriveSourceAssetMounts(pin: SourceRefPin): Map { - const mounts = new Map(); - for (const entry of pin.closure.entries) { - if (entry.source.kind === "asset" && entry.source.package.format === "tarball") { - mounts.set(entry.source.assetId, workflowSourceAssetMountPath(entry.source.assetId)); - } - } - return mounts; -} - -/** - * The `assetId -> gitDir` map a pinned closure's SOURCE `kind:"asset"` entries - * check subtrees out of, derived purely from the pin so deploy and restore - * agree without re-delivery. - */ -function deriveSourceGitDirs(pin: SourceRefPin, gitRoot: string): Map { - const gitDirs = new Map(); - for (const entry of pin.closure.entries) { - if (entry.source.kind === "asset" && entry.source.package.format === "source") { - gitDirs.set(entry.source.assetId, sourceAssetGitDir(gitRoot, entry.source.assetId)); - } - } - return gitDirs; -} - -async function isExistingDir(dir: string): Promise { - try { - return (await stat(dir)).isDirectory(); - } catch (err) { - if (err instanceof Error && "code" in err && err.code === "ENOENT") { - return false; - } - throw err; - } -} - -/** Cheap early gate for a missing checkout (the loader still SRI-verifies bytes); a missing mount fails loud since the hub must re-drive it. */ -export async function resolveDeploymentAssetMounts( - dataDir: string, - deploymentId: string, - pin: SourceRefPin, -): Promise<{ - assetRoot: string; - assetMounts: ReadonlyMap; - gitDirs: ReadonlyMap; -}> { - const assetRoot = deploymentSourceAssetRoot(dataDir, deploymentId); - const assetMounts = deriveSourceAssetMounts(pin); - for (const [assetId, mountPath] of assetMounts) { - const mountDir = pathJoin(assetRoot, mountPath); - if (!(await isExistingDir(mountDir))) { - throw new Error( - `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} is not present in the durable store at ${mountDir}; the deployment must be re-driven from the hub`, - ); - } - } - const gitRoot = deploymentSourceGitRoot(dataDir, deploymentId); - const gitDirs = deriveSourceGitDirs(pin, gitRoot); - for (const [assetId, gitDir] of gitDirs) { - if (!(await isExistingDir(gitDir))) { - throw new Error( - `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} has no indexed git store at ${gitDir}; the deployment must be re-driven from the hub`, - ); - } - } - return { assetRoot, assetMounts, gitDirs }; -} - -/** A missing or non-numeric byte cap is a boot-edge wiring bug, so it fails loud rather than defaulting. */ -function requireSubstrateByteCap(env: Record, key: string): number { - const raw = env[key]; - if (raw === undefined) { - throw new Error( - `sidecar deploy router: ${key} must be present in the multi-step substrate env to materialize a frozen workflow closure`, - ); - } - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error( - `sidecar deploy router: ${key} must be a positive finite number, got ${JSON.stringify(raw)}`, - ); - } - return parsed; -} - -/** The deploy router is the local stand-in for the hub on the sidecar's disk, so it claims the hub principal for this write since agent-state gates writeTree as hub-only. */ -const GRANTS_WRITE_PRINCIPAL: Principal = { kind: "hub" }; - -/** Single-step and multi-step deploys disagree on how the address/repo id are computed, so the two derivations are minted together to stay in sync. */ -type StepStrategy = { - deriveStepAddress: DeriveStepAddress; - deriveStepRepoId: DeriveStepRepoId; -}; - -/** - * A single-step deploy keeps the deploy's own mail address; any other step - * count derives a `-` address/repo per step. Note this does - * NOT feed step-agent-tools.ts's stepDeployTreeDir, which re-derives the - * step address independently — the deploy tree must still be staged there. - */ -function createStepStrategy(args: { - legacyAddress: string; - stepOrder: readonly string[]; - multistepDeriveStepAddress: DeriveStepAddress; -}): StepStrategy { - if (args.stepOrder.length === 1) { - return { - deriveStepAddress: () => args.legacyAddress, - // Deferred into the closure so a malformed address surfaces at the same point the rest of spawn() would fault. - deriveStepRepoId: () => ({ - kind: "agent-state", - id: parseAgentId(args.legacyAddress), - }), - }; - } - return { - deriveStepAddress: args.multistepDeriveStepAddress, - deriveStepRepoId: ({ runId, stepId }) => ({ - kind: "agent-state", - id: `${runId}-${stepId}`, - }), - }; -} - -/** - * Two destinations selected by `runId`: absent means deploy time (writes - * every step's grants into its own agent-state repo, on the spawn critical - * path so a failure rejects the deploy); present means per-run delivery - * (writes one `runs//grants.json`, ignoring stepOrder/deriveStepRepoId). - */ -async function writeStepGrants(args: { - repoStore: RepoStore; - anchorRunId: string; - stepOrder: readonly string[]; - deriveStepRepoId: DeriveStepRepoId; - grants: readonly unknown[] | undefined; - runId?: string; -}): Promise { - // The deploy frame's validated HarnessConfig always carries a `grants` - // array (possibly empty); an absent array means "no grants", which - // serializes to the same fail-closed empty file the snapshot expects. - // Coerce here so the on-disk envelope is always a valid `{ grants: [] }` - // rather than `{}` (which the snapshot's validator rejects). - const grants = args.grants ?? []; - const serialized = JSON.stringify({ grants }, null, 2); - if (args.runId !== undefined) { - await args.repoStore.writeTree( - GRANTS_WRITE_PRINCIPAL, - { kind: "workflow-run", id: args.anchorRunId }, - STEP_GRANTS_REF, - { - files: { [runGrantsPath(args.runId)]: serialized }, - message: `Write run grants for ${args.runId}`, - }, - ); - return; - } - for (const stepId of args.stepOrder) { - const repoId = args.deriveStepRepoId({ - runId: args.anchorRunId, - stepId, - }); - await args.repoStore.writeTree(GRANTS_WRITE_PRINCIPAL, repoId, STEP_GRANTS_REF, { - files: { [STEP_GRANTS_PATH]: serialized }, - message: `Write step grants for ${stepId}`, - }); - } -} - -export type AssembleRunCredentialsSnapshotOpts = { - /** Substrate handle the sink reads the per-run grants file from. */ - repoStore: RepoStore; - /** Anchor run id keying the workflow-run repo the grants file lives in. */ - anchorRunId: string; - /** Run whose per-run grants file is read. */ - runId: string; - /** - * Every step id the snapshot must cover -- the deployment's flat step-id - * namespace, loop-body step ids included. The per-run grants apply - * uniformly across them. - */ - stepOrder: readonly string[]; - /** Per-step mail-address derivation. */ - deriveStepAddress: DeriveStepAddress; -}; - -/** - * Every legitimate run birth path writes `runs//grants.json` before - * the run dispatches, so a missing file means the run reached its barrier - * with no grants written; this fails closed rather than running - * under-authorized. A malformed file also throws for the same reason. - */ -export async function assembleRunCredentialsSnapshot( - opts: AssembleRunCredentialsSnapshotOpts, -): Promise { - const runGrants = await readRunGrants({ - repoStore: opts.repoStore, - anchorRunId: opts.anchorRunId, - runId: opts.runId, - }); - if (runGrants === undefined) { - throw new Error( - `sidecar onRunStart: run ${opts.runId} has no grants file at ${runGrantsPath(opts.runId)}; refusing to start the run under-authorized`, - ); - } - const contentHash = await hashGrants(runGrants); - const steps: CredentialsSnapshotStep[] = opts.stepOrder.map((stepId) => ({ - stepId, - address: opts.deriveStepAddress({ - runId: opts.anchorRunId, - stepId, - }), - grants: runGrants, - contentHash, - })); - return { steps }; -} - -// Resolved statically at module load so the production spawn surface is -// independent of any runtime env override; tests inject a sentinel path -// via the binaryPath opts override instead. -const SIDECAR_WORKFLOW_CHILD_BINARY: string = (() => { - const url = import.meta.resolve("../bin/workflow-child"); - return fileURLToPath(url); -})(); - -/** fd 0/1 are the control channel, fd 2 is inherited stderr, fd 3 is the event-channel pipe Bun.spawn's stdio[3]="pipe" provisions. */ -const CHILD_EVENT_CHANNEL_FD = 3; - -/** Passes each already-newline-terminated frame through without buffering, so it surfaces as soon as write() resolves. */ -function ndjsonWriterFromFileSink(sink: Bun.FileSink): NdjsonWriter { - return { - async write(line: string): Promise { - const result = sink.write(line); - if (typeof result !== "number") await result; - const flushed = sink.flush(); - if (typeof flushed !== "number") await flushed; - }, - }; -} - -/** - * Wrap a Bun stdout `ReadableStream` as the supervisor's - * `NdjsonReader`. The pipe is a byte stream; this reader buffers - * partial chunks and yields one complete line per iteration. The - * receiver's iterator finalises only on EOF, which mirrors the - * `defaultControlReader` shape the child wires for `process.stdin`. - */ -function ndjsonReaderFromReadableStream(stream: ReadableStream): NdjsonReader { - return { - read(): AsyncIterableIterator { - return (async function* () { - const decoder = new TextDecoder("utf-8"); - let pending = ""; - const reader = stream.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (value !== undefined) { - pending += decoder.decode(value, { stream: true }); - let nl = pending.indexOf("\n"); - while (nl >= 0) { - const line = pending.slice(0, nl).replace(/\r$/, ""); - pending = pending.slice(nl + 1); - if (line.length > 0) yield line; - nl = pending.indexOf("\n"); - } - } - if (done) break; - } - if (pending.length > 0) yield pending; - } finally { - reader.releaseLock(); - } - })(); - }, - }; -} - -/** Yields each raw chunk the kernel delivers and trusts one-write-per-envelope; framing discipline lives in receiveEventChannel's parser. */ -function frameReaderFromFd(fd: number): FrameReader { - const stream = Bun.file(fd).stream(); - return { - read(): AsyncIterableIterator { - return (async function* () { - const reader = stream.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (value !== undefined && value.byteLength > 0) yield value; - if (done) break; - } - } finally { - reader.releaseLock(); - } - })(); - }, - }; -} - -/** - * Constructs a fresh env with no inheritance of the sidecar's process env. - * A launch failure settles `exited` non-zero; the supervisor races that - * against `readyPromise` so a spawn-time crash rejects rather than wedging. - */ -export const defaultSubprocessSpawner: SubprocessSpawner = ({ - binaryPath, - env, -}): SubprocessHandle => { - const proc = Bun.spawn([binaryPath], { - stdio: ["pipe", "pipe", "inherit", "pipe"], - env, - }); - const eventFd = proc.stdio[CHILD_EVENT_CHANNEL_FD]; - if (typeof eventFd !== "number") { - throw new Error( - `workflow-host-wiring: Bun.spawn did not return a numeric fd at stdio[${String(CHILD_EVENT_CHANNEL_FD)}] for the event channel; got ${typeof eventFd}`, - ); - } - return { - pid: proc.pid, - controlWriter: ndjsonWriterFromFileSink(proc.stdin), - controlReader: ndjsonReaderFromReadableStream(proc.stdout), - eventReader: frameReaderFromFd(eventFd), - kill(signal?: number | string): void { - // Bun's kill() takes NodeJS.Signals, narrower than the supervisor's number|string; cast at the boundary. - if (signal === undefined) { - proc.kill(); - return; - } - if (typeof signal === "number") { - proc.kill(signal); - return; - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- supervisor's kill widens to `string`; Bun's runtime accepts the same `"SIG*"` strings, narrowed back at the boundary. - proc.kill(signal as NodeJS.Signals); - }, - exited: proc.exited, - }; -}; - -export type CreateSidecarWorkflowSupervisorOpts = { - /** Sidecar's hub mail transport. */ - transport: HubTransport; - /** Substrate-shaped RepoStore the workflow-host's supervisor reads from. */ - repoStore: RepoStore; - /** Sidecar's 32-byte Ed25519 private key seed for principal signing. */ - signingKeySeed: Uint8Array; - /** Workflow-run repo identity for the deployment. */ - workflowRunRepoId: RepoId; - /** Workflow-run repo ref the supervisor commits events to. */ - workflowRunRef: string; - /** Deployment id baked into principal claims and address derivation. */ - runId: string; - /** Decrypted credential material delivered to the child on the pre-trigger barrier; absent when the deployment binds no credentials. */ - credentialDelivery?: CredentialDelivery; - /** Threaded into the child's spawn-time env so its deploy-tree read collapses onto the head for a single-step deployment. */ - stepCount: number; - /** Every step id (including loop-body ids) the onRunStart grants sink walks to assemble the per-run credentialsSnapshot. */ - stepOrder: readonly string[]; - /** Deployment's mail address. */ - deploymentMailAddress: string; - /** Per-step mail-address derivation. */ - deriveStepAddress: DeriveStepAddress; - /** Overrides the default `-` repo id; single-step deploy returns the legacy agent-state repo instead. */ - deriveStepRepoId?: DeriveStepRepoId; - /** Substrate-config keys propagated to the child via spawn-time env. */ - substrateEnv: Record; - /** Recomputed on every spawn and recycle respawn (e.g. a live-rotated inference-source list); layers over substrateEnv. */ - dynamicSpawnEnv: () => Record; - /** Tests inject a deterministic mock; production defaults to defaultSubprocessSpawner. */ - subprocessSpawner?: SubprocessSpawner; - /** Override the `bin/workflow-child` path. */ - binaryPath?: string; - /** Absent in production; the deploy router wires this (off a benchmark env gate) only for the Phase 4.7 latency gate. */ - onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** Absent in production; the deploy router wires this (off the same benchmark env gate) only for the D2 attribution run. */ - repackEveryMessages?: { everyMessages: number }; - /** Boot edge resolves the operator's CONSUMED_RETENTION_MS; absent, the supervisor applies its own 24h default. */ - consumedRetentionMs?: number; - /** Boot edge resolves the operator's CHILD_READY_TIMEOUT_MS; absent, the supervisor applies its own 30s default. */ - readyTimeoutMs?: number; - /** Invoked when a child reports park.notify; production wiring routes it to the hub link's signal.correlation.register. */ - onSuspensionRegister?: (registration: SuspensionRegistration) => void; - /** Invoked on a terminal phase (crash-loop, channel crash, recycle failure); production routes it to the deploy router's address reclaim. */ - onSelfTerminate?: (info: { phase: "stopped" | "crash-looping"; reason: string }) => void; - /** Lets a run whose grants write is KNOWN to have failed reject with that specific reason instead of a generic missing-file error. */ - isRunPoisoned?: (runId: string) => boolean; -}; - -export type SidecarWorkflowSupervisor = { - supervisor: WorkflowSupervisor; - /** Resolves once durably accepted (so the hub-link can send mail.inbound.ack only then) and rejects otherwise. */ - routeInbound(message: Uint8Array): Promise; - /** Snapshot accessor that proxies the supervisor's credentials view. */ - getCredentialsSnapshot(): CredentialsSnapshot | null; - /** The per-run grants barrier; rejects for a poisoned run, exposed so it can be exercised without driving a full spawn. */ - onRunStart(args: { runId: string; anchorRunId: string }): Promise; -}; - -/** Listed here so the router and the substrate-factory consumer spell this env key the same way without a magic-string trip hazard. */ -export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; - -/** Carries every spawned body's per-step inference-source pins as plaintext JSON so the child resolves them without the sidecar's cipher key. */ -export const WORKFLOW_BODY_SOURCES_ENV_KEY = "WORKFLOW_BODY_SOURCES"; - -/** Below the OS argument-string ceiling (128 KiB on Linux) with headroom, so an over-large deployment is rejected loudly at deploy time, not at a much-later spawn's execve. */ -const WORKFLOW_BODY_SOURCES_MAX_BYTES = 96 * 1024; - -/** `undefined` when the deployment references no bodies, so the record omits the field. */ -function buildBodySourcesMap( - referenced: NonNullable["referencedDefinitions"], -): WorkflowRunRecord["bodySources"] { - if (referenced === undefined || referenced.length === 0) { - return undefined; - } - const map: NonNullable = {}; - for (const ref of referenced) { - map[ref.definition.id] = ref.sources; - } - return map; -} - -/** Takes `unknown` inputs (unlike the arktype AgentDeployFrame validator) so it can also gate callers that bypass the wire boundary. */ -export function validateWorkflowProjection(projection: { - definition: { id: unknown; stepOrder: unknown; steps: unknown }; - sources: unknown; -}): void { - const def = projection.definition; - if (typeof def.id !== "string" || def.id.length === 0) { - throw new Error("sidecar deploy router: workflow.definition.id must be a non-empty string"); - } - if (!Array.isArray(def.stepOrder) || def.stepOrder.length === 0) { - throw new Error( - "sidecar deploy router: workflow.definition.stepOrder must be a non-empty array", - ); - } - if (typeof def.steps !== "object" || def.steps === null) { - throw new Error("sidecar deploy router: workflow.definition.steps must be an object"); - } - if (typeof projection.sources !== "object" || projection.sources === null) { - throw new Error("sidecar deploy router: workflow.sources must be an object"); - } - const steps = def.steps; - const sources = projection.sources; - for (const stepId of def.stepOrder) { - if (typeof stepId !== "string" || stepId.length === 0) { - throw new Error( - "sidecar deploy router: workflow.definition.stepOrder entries must be non-empty strings", - ); - } - if (!STEP_ID_PATTERN.test(stepId)) { - throw new Error( - `sidecar deploy router: stepId ${JSON.stringify(stepId)} must match ${STEP_ID_PATTERN.source}`, - ); - } - if (!Object.prototype.hasOwnProperty.call(steps, stepId)) { - throw new Error( - `sidecar deploy router: workflow.definition.steps is missing entry for stepId ${JSON.stringify(stepId)}`, - ); - } - if (!Object.prototype.hasOwnProperty.call(sources, stepId)) { - throw new Error( - `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, - ); - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- sources is checked to be a non-null object above; this reads a value to re-check its array shape - const stepSources = (sources as Record)[stepId]; - if (!Array.isArray(stepSources) || stepSources.length === 0) { - throw new Error( - `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, - ); - } - } -} - -/** The supervisor signs every workflow-run event with this key; the multi-step branch surfaces it so the hub records the verifying key. */ -async function derivePrincipalPublicKeyHex(signingKeySeed: Uint8Array): Promise { - return hexEncode(await derivePublicKeyBytes(signingKeySeed)); -} - -/** Adds the sidecar-app-only boot-time restore driver on top of the shared `DeployRouter` contract. */ -export interface SidecarDeployRouter extends DeployRouter { - /** Soft-fails per deployment (a record left on disk for a later boot to retry, never deleted) rather than failing the whole boot. */ - restoreWorkflowRuns(): Promise; - /** Addresses this router currently hosts a live supervisor for; re-read per connect since deploy/undeploy/restore change it live. */ - activeAddresses(): string[]; - /** Re-registers parked correlations a long outage may have evicted from the hub's bounded send queue; fire-and-forget. */ - reEmitParkedCorrelations(address: string): void; -} - -export function createSidecarDeployRouter(deps: { - sessions: SessionManager; - keyStore: AgentKeyStore; - /** Grants handler writes each co-delivered senderIdentities entry here before the run's grants land, so a durable grant always has its verify key. */ - senderKeyCache: SenderKeyCache; - transport: HubTransport; - repoStore: RepoStore; - signingKeySeed: Uint8Array; - /** Seals each run record's inference-source apiKeys on persist and unseals them on the boot scan. */ - credentialCipher: CredentialCipher; - /** Registers the spawned agent's signing key on the transport before spawn(), or an outbound send throws "address is not registered". */ - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; - /** Distinct from the operator-approval check: this gates on whether a provider is buildable at all, not whether it was approved. */ - assertSourceBuildable: (source: InferenceSource) => void; - /** Fires once per inbound agent.deploy before the supervisor spawns, so the first pack push sees the mapping. No-op-safe for tests. */ - registerDeployment: (entry: { runId: string; agentAddress: string }) => void; - /** Symmetric removal hook so a stale writeTreePreservingPrefix fails structurally instead of resolving to the prior address. */ - unregisterDeployment: (entry: { runId: string; agentAddress: string }) => void; - /** Defaults to empty so a router built without substrate config (e.g. a test) needs no boot-edge threading. */ - multistepSubstrateEnv?: Record; - /** Defaults to the production Bun.spawn-backed spawner; tests inject a deterministic mock. */ - multistepSubprocessSpawner?: SubprocessSpawner; - /** Production wiring uses the package-local default; tests inject a sentinel value the mock spawner can assert on. */ - multistepBinaryPath?: string; - /** Session id rides alongside the (sessionless) InferenceEvent since it's optional for a headless deployment; defaults to a no-op. */ - publishWorkflowInferenceEvent?: ( - agentAddress: string, - event: InferenceEvent, - sessionId: string | undefined, - ) => void; - /** Registers a parked run's correlation at the hub; defaults to a no-op. */ - publishWorkflowSuspension?: (registration: { - correlationId: string; - runId: string; - anchorRunId: string; - agentAddress: string; - kind: SignalKind; - approvalSnapshot?: ApprovalSnapshot; - }) => void; - /** Defaults to `${runId}-${stepId}@`; tests inject a deterministic factory. */ - multistepDeriveStepAddress?: DeriveStepAddress; - /** Registers wired.routeInbound once spawn succeeds; absent means multi-step inbound mail cannot route until wired. */ - multistepMailRouter?: MultistepMailRouter; - /** Removed in the same teardown as the mail-router registration so a reused address never inherits a stale policy; absent means fully-closed default. */ - inboundMailPolicyRegistry?: InboundMailPolicyRegistry; - /** The child commits the resulting SignalReceived through its own substrate, preserving the single-writer invariant sidecar-side. */ - multistepSignalRouter?: MultistepSignalRouter; - /** Cancel-mode in-flight steps abort child-side; wait-mode steps continue until the drainTimeout deadline commits a CancelRequested. */ - multistepDrainRouter?: MultistepDrainRouter; - /** The write is awaited in tryRoute so the frame's FIFO completion means grants are durable on disk. */ - multistepGrantsRouter?: MultistepGrantsRouter; - /** Only a single-step warm deployment registers a handler; a multi-step deployment has no warm agent to rotate, so its address is unrouted. */ - multistepSourcesRouter?: MultistepSourcesRouter; - /** Every deployment with a supervisor registers one after spawn (material cell is per-child); an unregistered address is unrouted. */ - multistepCredentialsRouter?: MultistepCredentialsRouter; - /** Resolved from the Phase 4.7 latency-gate env; absent in ordinary production. */ - onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** Resolved from the same benchmark env gate; absent in ordinary production. */ - repackEveryMessages?: { everyMessages: number }; - /** Boot edge resolves the operator's CONSUMED_RETENTION_MS; absent, the supervisor applies its own 24h default. */ - consumedRetentionMs?: number; - /** A child that spawns but never signals ready is killed and its spawn rejected rather than hanging the deploy or restore. */ - readyTimeoutMs?: number; - /** - * Run-record writer, injectable so a test can block or fail the - * persist at a controlled point -- the natural seam for exercising a - * recycle that interleaves the source-rotation persist window. Defaults - * to the real `writeWorkflowRunRecord`; production never overrides - * it. - */ - writeWorkflowRunRecord?: typeof writeWorkflowRunRecord; - /** - * Materialize a source-ref deployment's frozen closure. Defaults to the real - * `applyFrozenWorkflowClosure` (registry fetch + SRI verify + layout); - * production never overrides it. A test seam so a unit test can drive the - * deploy/restore source-ref path without a live registry. - */ - applyFrozenWorkflowClosure?: typeof applyFrozenWorkflowClosure; -}): SidecarDeployRouter { - // Validate the signing seed at construction so a malformed key fails - // sidecar boot rather than the first multi-step deploy, where the - // public key is derived from it (`derivePrincipalPublicKeyHex`). The - // seed also signs every workflow-run event via the supervisor. - if (deps.signingKeySeed.length !== 32) { - throw new Error( - `sidecar deploy router: Ed25519 signing seed must be 32 bytes, got ${deps.signingKeySeed.length}`, - ); - } - const publishInferenceEvent = - deps.publishWorkflowInferenceEvent ?? - ((_address: string, _event: InferenceEvent, _sessionId: string | undefined): void => { - /* no-op default: tests and production-without-a-publisher - deployments do not consume events. */ - }); - const publishSuspension = - deps.publishWorkflowSuspension ?? - ((_registration: { - correlationId: string; - runId: string; - anchorRunId: string; - agentAddress: string; - kind: SignalKind; - approvalSnapshot?: ApprovalSnapshot; - }): void => { - /* no-op default: tests and production-without-a-publisher - deployments do not register suspensions. */ - }); - const multistepSubstrateEnv = deps.multistepSubstrateEnv ?? {}; - // Sidecar data dir the deployment's per-step scratch is rooted under - // (`/workflow-step-state//...`). Resolved once - // from the boot-edge substrate env so the undeploy hook can reclaim - // the whole subtree. Absent only when the router is wired without - // substrate config (a test that never spawns a child), in which case - // no child ever rooted scratch and the undeploy reclaim is correctly - // skipped. - const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; - const persistWorkflowRunRecord = deps.writeWorkflowRunRecord ?? writeWorkflowRunRecord; - const applyClosure = deps.applyFrozenWorkflowClosure ?? applyFrozenWorkflowClosure; - const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; - const multistepDeriveStepAddress: DeriveStepAddress = - deps.multistepDeriveStepAddress ?? (({ runId, stepId }) => `${runId}-${stepId}`); - - // Per-deployment supervisor tracking. The multi-step branch - // constructs one `SidecarWorkflowSupervisor` per `agent.deploy` - // frame; the supervisor owns the workflow-process child, its IPC - // pipes, and its event-channel fd. The undeploy hook consults this - // map to call `supervisor.shutdown()` so the child's lifetime ends - // with the deployment. - const activeSupervisors = new Map(); - - // Closes the window before activeSupervisors is populated (only after spawn - // succeeds): a second same-address frame arriving mid-deploy is rejected - // here rather than deleting the winner's live run record on unwind. - const reservingDeployAddresses = new Set(); - - // deriveDeploymentId's char substitution is lossy: two distinct addresses - // can collapse to the same slug, which IS the repoId, so a collision would - // let the second deploy silently overwrite the first's repo state. - const slugClaims = new Map(); - - function claimSlug(runId: string, agentAddress: string): void { - const existing = slugClaims.get(runId); - if (existing !== undefined && existing !== agentAddress) { - throw new Error( - `deriveDeploymentId collision: run addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to runId ${JSON.stringify(runId)}`, - ); - } - // Defensive no-op: activeSupervisors already rejects a live re-deploy, so existing is only ever undefined or a different address here. - slugClaims.set(runId, agentAddress); - } - - function releaseSlug(runId: string, agentAddress: string): void { - const existing = slugClaims.get(runId); - if (existing === agentAddress) slugClaims.delete(runId); - } - - // Reclaims an address whose supervisor drove ITSELF to a terminal phase - // without an operator undeploy. Never deletes the durable run record (so a - // boot restore can re-spawn) and never calls unregisterDeployment (the - // pack-push mapping is still needed by the crash-loop latch's own commit). - function reclaimSelfTerminatedSupervisor(args: { runId: string; agentAddress: string }): void { - if (!activeSupervisors.has(args.agentAddress)) return; - // Drop racing frames at the router boundary first, then unwind the - // underlying registrations -- the same ordering the undeploy hook uses. - deps.multistepMailRouter?.unregister(args.agentAddress); - deps.inboundMailPolicyRegistry?.unregister(args.agentAddress); - deps.multistepSignalRouter?.unregister(args.agentAddress); - deps.multistepDrainRouter?.unregister(args.agentAddress); - deps.multistepGrantsRouter?.unregister(args.agentAddress); - deps.multistepSourcesRouter?.unregister(args.agentAddress); - deps.multistepCredentialsRouter?.unregister(args.agentAddress); - activeSupervisors.delete(args.agentAddress); - deps.transport.unregister(args.agentAddress); - releaseSlug(args.runId, args.agentAddress); - } - - /** Reclaims a pre-sealed-record deploy's legacy plaintext body-source staging, now retired since sources ride sealed in the run record. */ - async function sweepLegacyBodySources( - sidecarDataDir: string | undefined, - definitionId: string, - ): Promise { - if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the legacy body-source staging is reclaimed against this data dir", - ); - } - const bodyAssetDir = pathJoin(sidecarDataDir, "assets", "workflow", definitionId); - await rm(bodyAssetDir, { recursive: true, force: true }); - } - - /** - * The per-deployment inputs the shared spawn core needs to stand up a - * workflow deployment, independent of the live deploy frame. The live - * deploy path builds this from `frame`/`projection`; a boot-time restore - * path builds the same shape from the persisted run record. - */ - interface WorkflowDeploySpec { - agentAddress: string; - /** Always the closure evaluation (source-ref is the only deploy lineage), never a frame-carried inline definition. */ - definition: WorkflowProjectionDefinition; - sources: NonNullable["sources"]; - /** Delivered to the run child as plaintext through the spawn env so it resolves sources without the sidecar's cipher key. */ - bodySources: WorkflowRunRecord["bodySources"]; - /** Delivered to the child on the pre-trigger barrier; undefined when the deployment binds no credentials. */ - credentials: CredentialDelivery | undefined; - /** Sourced from hub authority, never a sidecar recompute; undefined only for a frame that carried no approved hash. */ - approvedWireHash: string | undefined; - /** Correlates the child's inference events to the deploy's session. */ - sessionId: string | undefined; - /** Required for a single-step deployment (head IS the agent identity); undefined for multi-step, which derives per-step addresses. */ - hubPublicKey: string | undefined; - /** Sidecar-local only: never travels on the hub deploy frame and isn't persisted to the record. Always present. */ - closurePackageDir: string; - /** Carried so a boot-time restore can re-run applyFrozenWorkflowClosure; its source carries no secret, its closure is versions + SRIs. */ - sourceRef: NonNullable["sourceRef"]; - } - - /** Table is a parameter (not spec.sources) so both the deploy path and the rotation handler write through one shape. */ - function buildWorkflowRunRecord( - spec: WorkflowDeploySpec, - sources: WorkflowRunRecord["sources"], - ): WorkflowRunRecord { - // A spec missing this is a wiring defect; fail loudly rather than persist a record the boot scan would reject as corrupt. - if (spec.approvedWireHash === undefined) { - throw new Error( - `buildWorkflowRunRecord: a source-ref deployment (${spec.agentAddress}) must carry approvedWireHash`, - ); - } - return { - version: 2 as const, - agentAddress: spec.agentAddress, - definitionId: spec.definition.id, - sources, - ...(spec.bodySources !== undefined ? { bodySources: spec.bodySources } : {}), - ...(spec.credentials !== undefined ? { credentials: spec.credentials } : {}), - ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), - ...(spec.hubPublicKey !== undefined ? { hubPublicKey: spec.hubPublicKey } : {}), - lineage: "source-ref", - // Feeds the restored child's DEFINITION_HASH so it re-verifies the - // evaluated closure against the hub-approved pin, and the source-ref pin a - // restore re-runs applyFrozenWorkflowClosure with. - approvedWireHash: spec.approvedWireHash, - sourceRef: spec.sourceRef, - }; - } - - /** - * Shared plumbing for both the deploy path and boot-time restore. The - * instance dir is force-reclaimed before the apply since deploymentId is - * deterministic per address and a prior soft-failed deploy can leave it - * half-materialized; safe only because no live reader holds the dir when - * this runs, a precondition each caller establishes at its call site. - */ - async function materializeDeploymentClosure( - dataDir: string, - deploymentId: string, - pin: SourceRefPin, - ): Promise { - const instanceDir = pathJoin(dataDir, "workflow-definition-closures", deploymentId); - await rm(instanceDir, { recursive: true, force: true }); - - // Tarball `kind:"asset"` entries read from the durable plain-file store; a - // source-format entry checks its subtree out of the durable indexed git - // store. Deriving and asserting both from the pin alone is what makes this - // symmetric on deploy and restore. - const { assetRoot, assetMounts, gitDirs } = await resolveDeploymentAssetMounts( - dataDir, - deploymentId, - pin, - ); - - return applyClosure({ - source: pin.source, - closure: pin.closure, - instanceDir, - cacheRoot: pathJoin(dataDir, "workflow-definition-closure-cache"), - cacheMaxBytes: requireSubstrateByteCap(multistepSubstrateEnv, "SIDECAR_CACHE_MAX_BYTES"), - registryMaxTarballBytes: requireSubstrateByteCap( - multistepSubstrateEnv, - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - ), - registries: readRegistries(), - assetRoot, - assetMounts, - gitDirs, - }); - } - - /** - * The single owner of the workflow-deployment spawn sequence; its - * try/finally unwinds every piece of partial state if any step throws. - * Both the live deploy path and the boot-time restore path route through - * here so the two can never diverge on how a deployment is stood up. - */ - async function spawnWorkflowRun(spec: WorkflowDeploySpec): Promise { - // Sealed into the record by the sidecar's cipher; restore unseals it from the record. - const credentialDelivery = spec.credentials; - // The primary early guard, ahead of the lower-level transport.register duplicate-throw backstop; guards against a boot restore racing a legacy restore for the same address. - if (activeSupervisors.has(spec.agentAddress)) { - throw new Error( - `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, - ); - } - const runId = deriveDeploymentId(spec.agentAddress); - - const stepStrategy = createStepStrategy({ - legacyAddress: spec.agentAddress, - stepOrder: spec.definition.stepOrder, - multistepDeriveStepAddress, - }); - - // Not stepOrder: a loop body runs in-process inheriting the parent's env, so it must authorize against the same snapshot keyed by its own plain step id. - const credentialStepIds = inertFlatNamespaceStepIds({ - definition: spec.definition, - context: "sidecar deploy router credentials snapshot: ", - }); - - // The caller owns the deployment slug and the deployment-address registration, so neither is touched in this unwind. - let succeeded = false; - let wiredForUnwind: SidecarWorkflowSupervisor | undefined; - let supervisorRegistered = false; - let routersRegistered = false; - let agentTransportRegistered = false; - let hubKeyRecorded = false; - let deploymentRegistered = false; - try { - // A missing hash is a wiring bug, not a legacy case: substituting a sidecar recompute would collapse the child's re-verify to a self-check against its own hash. - if (spec.approvedWireHash === undefined) { - throw new Error( - `workflow deploy spawn (${spec.agentAddress}): the deploy spec carries no approvedWireHash. The hub deploy builder must stamp the hub-approved wire hash and a restore must re-attach it from the persisted record; the sidecar will not recompute it, which would collapse the child's re-verify to a self-check.`, - ); - } - const definitionHash = spec.approvedWireHash; - - // Bodies don't live-rotate, unlike top-level sources; guarded against the OS argument-string ceiling here so an over-large deployment fails at deploy, not at a later child execve. - const bodySourcesEnv = JSON.stringify(spec.bodySources ?? {}); - const bodySourcesBytes = Buffer.byteLength(bodySourcesEnv, "utf8"); - if (bodySourcesBytes > WORKFLOW_BODY_SOURCES_MAX_BYTES) { - throw new Error( - `sidecar deploy router: serialized ${WORKFLOW_BODY_SOURCES_ENV_KEY} is ${String(bodySourcesBytes)} bytes, over the ${String(WORKFLOW_BODY_SOURCES_MAX_BYTES)}-byte spawn-env limit; the deployment has too many spawned-body sources to deliver through the child env`, - ); - } - - const substrateEnv: Record = { - ...multistepSubstrateEnv, - WORKFLOW_RUN_REPO_ID: runId, - WORKFLOW_RUN_REF: "refs/heads/main", - // Thread the materialized closure's sidecar-local package dir so the - // run child EVALUATES the pinned code to a live definition and - // re-verifies by project-then-hash against `DEFINITION_HASH`. Source-ref - // is the only deploy lineage, so this is always present. Sidecar-local; - // carried on the frozen substrate env because the value is fixed for the - // deployment's lifetime. - CLOSURE_PACKAGE_DIR: spec.closurePackageDir, - [WORKFLOW_BODY_SOURCES_ENV_KEY]: bodySourcesEnv, - }; - // Live-rotatable per-step inference sources. Seeded from the deploy - // spec, then revised in place by the single-step sources-rotation - // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen - // `substrateEnv`: it is recomputed on every spawn and recycle respawn - // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of - // reverting to the deploy-time list. - let currentSources = spec.sources; - - // RunIds whose `run.grants` write was attempted and failed. The grants - // handler below records a runId here on a write failure; the grants - // barrier reads it through `isRunPoisoned` and fails the run rather than - // starting it under the empty deploy-time grant set. Scoped to this - // deployment's supervisor; a run that never had a `run.grants` frame is - // never added, so internal runs inherit the deployment's grants normally. - const poisonedRunIds = new Set(); - - const wired = createSidecarWorkflowSupervisor({ - transport: deps.transport, - repoStore: deps.repoStore, - signingKeySeed: deps.signingKeySeed, - workflowRunRepoId: { - kind: "workflow-run", - id: runId, - }, - workflowRunRef: "refs/heads/main", - runId, - stepCount: spec.definition.stepOrder.length, - stepOrder: credentialStepIds, - deploymentMailAddress: spec.agentAddress, - ...(credentialDelivery !== undefined ? { credentialDelivery } : {}), - deriveStepAddress: stepStrategy.deriveStepAddress, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - isRunPoisoned: (runId) => poisonedRunIds.has(runId), - // The supervisor stamps `runId` + `agentAddress` before - // invoking this; forward the fully-stamped registration to the - // hub-link-backed publisher so a `signal.correlation.register` frame - // reaches the hub for the parked run. - onSuspensionRegister: publishSuspension, - // Reclaim the deployment address when the supervisor drives itself to a - // terminal phase (crash-loop latch, channel crash, recycle failure) so - // the dead supervisor is dropped from `activeSupervisors` and the - // address is redeployable without a manual undeploy first. - onSelfTerminate: () => - reclaimSelfTerminatedSupervisor({ - runId, - agentAddress: spec.agentAddress, - }), - substrateEnv, - // Recomputed on every spawn AND recycle respawn. The rotation - // handler below revises `currentSources` in place, so a respawn - // re-serializes the current (possibly rotated) list rather than the - // frozen deploy-time value. - dynamicSpawnEnv: () => ({ - [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), - }), - subprocessSpawner: multistepSpawner, - ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } : {}), - ...(deps.onDispatchTiming !== undefined ? { onDispatchTiming: deps.onDispatchTiming } : {}), - ...(deps.repackEveryMessages !== undefined - ? { repackEveryMessages: deps.repackEveryMessages } - : {}), - ...(deps.consumedRetentionMs !== undefined - ? { consumedRetentionMs: deps.consumedRetentionMs } - : {}), - ...(deps.readyTimeoutMs !== undefined ? { readyTimeoutMs: deps.readyTimeoutMs } : {}), - }); - - // Every step signs outbound sends as spec.agentAddress, so the transport must hold a CryptoProvider for it before spawn() or getTransportFor throws "not registered". - const { keyPair } = await deps.keyStore.loadOrGenerateKey(spec.agentAddress); - deps.transport.register(spec.agentAddress, deps.createAgentCrypto(keyPair)); - agentTransportRegistered = true; - - // Every deployment acks this key, single- and multi-step alike, so the Hub can publish the same identity. - const deploymentPublicKey = hexEncode(keyPair.publicKey); - if (spec.definition.stepOrder.length === 1) { - // Idempotent init; the narrow initRepo (not provisionAgent) is deliberate since the supervised child mints its own keypair. - await deps.sessions.initRepo(spec.agentAddress); - - // Verifier resolves this from the in-memory key store, so a single-step deployment cannot stand up without it. - if (spec.hubPublicKey === undefined) { - throw new Error( - "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", - ); - } - deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); - hubKeyRecorded = true; - } - - // The sole step IS the long-lived agent for single-step; multi-step keeps instantiate-send-teardown per step. - const warmKeep = spec.definition.stepOrder.length === 1; - const spawnOpts: SpawnOpts = { - stepOrder: [...credentialStepIds], - definitionHash, - warmKeep, - onInferenceEvent: (event) => { - // A parse failure means upstream corruption, so drop loudly rather than forwarding an unvalidated payload onto the hub timeline. - const validated = parseInferenceEvent(event); - if (validated instanceof type.errors) { - logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; - return; - } - publishInferenceEvent(spec.agentAddress, validated, spec.sessionId); - }, - }; - - // Recorded before spawn, since spawn's replayProcessingToInbox write resolves this mapping to address the outbound pack frame; recording after loses the race. - deps.registerDeployment({ - runId, - agentAddress: spec.agentAddress, - }); - deploymentRegistered = true; - - // The supervisor is registered against the address only after spawn succeeds, so a spawn-time rejection leaves the registry untouched. - await wired.supervisor.spawn(spawnOpts); - wiredForUnwind = wired; - activeSupervisors.set(spec.agentAddress, wired); - supervisorRegistered = true; - - deps.multistepMailRouter?.register(spec.agentAddress, (message) => - wired.routeInbound(message), - ); - // Resolved once, beside the mail-router registration, so the hub-link seam has the recipient's policy before any inbound frame can route. - deps.inboundMailPolicyRegistry?.register( - spec.agentAddress, - resolveInboundMailPolicy(spec.definition.inboundMailPolicy), - ); - deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.deliverSignal({ - runId: args.runId, - signalName: args.signalName, - signalId: args.signalId, - payload: args.payload, - }); - }); - deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); - }); - // runId selects the per-run destination; the step-fan-out fields are inert here but the shared write machinery still takes them. - deps.multistepGrantsRouter?.register(spec.agentAddress, async (args) => { - try { - // Cached before the grants file lands so "grant durable" implies "key durable"; a malformed key is skipped and logged at ERROR rather than wedging the run. - for (const identity of args.senderIdentities ?? []) { - let publicKey: Uint8Array; - try { - publicKey = hexDecode(identity.publicKey); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`Skipping unparseable sender key for ${identity.address} on run ${args.runId}: ${message}`; - continue; - } - if (publicKey.length !== ED25519_PUBLIC_KEY_BYTES) { - logger.error`Skipping wrong-length sender key for ${identity.address} on run ${args.runId}: got ${String(publicKey.length)} bytes`; - continue; - } - await deps.senderKeyCache.put(identity.address, publicKey); - } - await writeStepGrants({ - repoStore: deps.repoStore, - anchorRunId: runId, - stepOrder: spec.definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: args.stepGrants, - runId: args.runId, - }); - } catch (cause) { - // Poison the runId so the grants barrier fails the run instead of starting it under the deploy-time grant set. - poisonedRunIds.add(args.runId); - throw cause; - } - // Best-effort and non-fatal: refreshes a live child so a standing approval lowers its floor immediately; a skipped push is healed by the next barrier/respawn. - await wired.supervisor.deliverGrants(args.runId); - }); - // Only a single-step warm deployment (one long-lived agent to swap sources on); multi-step registers none, so tryRoute reports its address as unrouted. - if (warmKeep) { - // The layer that owns the single-key invariant; deliverSources stays flat and stepId-agnostic. - const rotationStepId = spec.definition.stepOrder[0]; - if (rotationStepId === undefined) { - throw new Error("single-step deploy has no step id for sources rotation"); - } - deps.multistepSourcesRouter?.register(spec.agentAddress, async (args) => { - const rotated = { [rotationStepId]: args.sources }; - // Swapped before the durable persist so a recycle interleaving the - // persist await respawns on the SAME sources being persisted, not - // the stale prior table; a failed persist rolls the hint back so - // the only residual disagreement is child-ahead-of-durable, which - // the next recycle heals down to the rolled-back durable truth. - const prevSources = currentSources; - currentSources = rotated; - // Skipped when no data dir was wired (a test router that never persists), matching the restore guard. - if (stepStateDataDir !== undefined) { - try { - await persistWorkflowRunRecord( - stepStateDataDir, - runId, - buildWorkflowRunRecord(spec, rotated), - deps.credentialCipher, - ); - } catch (cause) { - // Safe because the sidecar's per-connection inbound-frame queue serializes rotations for one deployment, so no second rotation is in flight. - currentSources = prevSources; - throw cause; - } - } - await wired.supervisor.deliverSources({ - sources: args.sources, - defaultSource: args.defaultSource, - }); - }); - } - - // For EVERY deployment (not only warm single-step): the material cell is per-child. No durable persist; credential material never touches disk. - deps.multistepCredentialsRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.deliverCredentials({ - delivery: args.delivery, - ...(args.revoke !== undefined ? { revoke: args.revoke } : {}), - }); - }); - routersRegistered = true; - - succeeded = true; - return { publicKey: deploymentPublicKey }; - } finally { - if (!succeeded) { - // Unwind in reverse registration order so each step undoes state - // the success path confirmed; ordering matches the `undeploy` hook. - if (routersRegistered) { - deps.multistepMailRouter?.unregister(spec.agentAddress); - deps.inboundMailPolicyRegistry?.unregister(spec.agentAddress); - deps.multistepSignalRouter?.unregister(spec.agentAddress); - deps.multistepDrainRouter?.unregister(spec.agentAddress); - deps.multistepGrantsRouter?.unregister(spec.agentAddress); - // unregister is a no-op for an address that never registered, so a multi-step unwind safely calls this too. - deps.multistepSourcesRouter?.unregister(spec.agentAddress); - deps.multistepCredentialsRouter?.unregister(spec.agentAddress); - } - if (supervisorRegistered) { - activeSupervisors.delete(spec.agentAddress); - } - if (wiredForUnwind !== undefined) { - await wiredForUnwind.supervisor.shutdown().catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`multi-step deploy unwind: supervisor.shutdown failed: ${message}`; - }); - } - if (agentTransportRegistered) { - // Drop the agent's transport registration so a failed deploy does - // not leave the address live with a dangling `CryptoProvider`. - deps.transport.unregister(spec.agentAddress); - } - if (hubKeyRecorded) { - // forgetAgent also drops the keypair cache, safe since a redeploy reloads it from disk. The on-disk deploy-tree repo is deliberately NOT reversed: it holds the durable identity keypair a rerouted head must keep. - deps.keyStore.forgetAgent(spec.agentAddress); - } - if (deploymentRegistered) { - // Reverse the pre-spawn `registerDeployment`: drop the address - // mapping so a failed spawn leaves the boot-edge registry as it - // found it. Registered first (before spawn), unwound last. A - // subsequent stale workflow-run write for the dead deployment then - // surfaces structurally (`registry.resolve` returns null) rather - // than resolving to the address of a deployment that never came up. - deps.unregisterDeployment({ - runId, - agentAddress: spec.agentAddress, - }); - } - } - } - } - - /** - * Provisions one step of a multi-step deploy WITHOUT spawning (same - * initRepo + recordHubKey seam the single-step head uses), since a - * full-closure deploy pack needs an initialized repo before the - * deployment-level workflow frame spawns the child. - */ - async function provisionStep(frame: AgentDeployFrame): Promise { - await deps.sessions.initRepo(frame.agentAddress); - deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - return { - publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), - }; - } - - async function deployMultiStep( - frame: AgentDeployFrame, - projection: NonNullable, - ): Promise { - // activeSupervisors catches a completed deploy; reservingDeployAddresses - // catches one still in flight, closing the window before spawn populates - // the map, where two frames could both pass and the loser deletes the - // winner's live record. - if ( - activeSupervisors.has(frame.agentAddress) || - reservingDeployAddresses.has(frame.agentAddress) - ) { - throw new Error( - `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, - ); - } - - const runId = deriveDeploymentId(frame.agentAddress); - - // The run record, materialized closure, and per-step scratch all root under this. - const dataDir = stepStateDataDir; - if (typeof dataDir !== "string" || dataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the run record and workflow-process child root under it", - ); - } - - // Claimed before any durable write so a colliding runId is rejected before disk is touched; released on failure, kept on success (undeploy releases it). - claimSlug(runId, frame.agentAddress); - reservingDeployAddresses.add(frame.agentAddress); - try { - // Materializes the hub's frozen closure byte-for-byte (never re-resolves the pin); the child's load-boundary re-verify fails closed if it diverges from the hub-approved hash. - // Reclaims the store first so a redeploy drops unreferenced assets; runs only on the deploy path since restore re-reads what deploy persisted, with no re-delivery. - const assetStore = deploymentSourceAssetRoot(dataDir, runId); - const gitStore = deploymentSourceGitRoot(dataDir, runId); - await rm(assetStore, { recursive: true, force: true }); - await rm(gitStore, { recursive: true, force: true }); - if (projection.assets !== undefined && projection.assets.length > 0) { - await materializeWorkflowAssets({ - assets: projection.assets, - closure: projection.sourceRef.closure, - assetRoot: assetStore, - gitDirRoot: gitStore, - maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, - }); - } - // Safe: this deploy is single-flight-guarded and the child is not yet spawned, so no live reader holds the instance dir. - const applied = await materializeDeploymentClosure(dataDir, runId, projection.sourceRef); - const validatedDefinition = WorkflowProjectionDefinition( - projectLiveToInert(applied.definition), - ); - if (validatedDefinition instanceof type.errors) { - throw new Error( - `sidecar deploy router: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`, - ); - } - const effectiveDefinition = validatedDefinition; - - // Checked against the closure-derived definition since the wire arktype doesn't cover it and the frame carries none. Mirrors the restore path. - validateWorkflowProjection({ - definition: effectiveDefinition, - sources: projection.sources, - }); - - // Every source in a step's failover chain must be buildable, since an unbuildable tail would fail only after the reactor failed over onto it. - for (const stepId of effectiveDefinition.stepOrder) { - const chain = projection.sources[stepId]; - if (chain !== undefined) { - for (const source of chain) deps.assertSourceBuildable(source); - } - } - - // Single-agent deploy vs. derived multi-step deploy. A one-step - // definition keeps the deploy's own mail address and its grants in the - // agent-state repo keyed by the run id; a multi-step definition derives - // `-` per step for both the mail address and the - // agent-state repo id, isolating each step's grants in its own repo. - const stepStrategy = createStepStrategy({ - legacyAddress: frame.agentAddress, - stepOrder: effectiveDefinition.stepOrder, - multistepDeriveStepAddress, - }); - - // The spec the shared spawn core consumes, and the durable record that - // lets a boot-time restore rebuild the SAME spec (definition re-evaluated - // from the pinned closure, grants from the step repos, and the record's - // frame/in-memory-only inputs: sources, session id, single-step hub key). - const spec: WorkflowDeploySpec = { - agentAddress: frame.agentAddress, - definition: effectiveDefinition, - sources: projection.sources, - bodySources: buildBodySourcesMap(projection.referencedDefinitions), - // The hub's unified credential-material cell (inference + tool secrets); - // persisted sealed in the record and delivered on the barrier. - credentials: projection.credentials, - approvedWireHash: projection.approvedWireHash, - sessionId: frame.config.sessionId, - hubPublicKey: effectiveDefinition.stepOrder.length === 1 ? frame.hubPublicKey : undefined, - // The sidecar-local dir of the just-materialized closure the spawn core - // threads into the child's env so it re-evaluates the pinned code. - closurePackageDir: applied.packageDir, - // The source-ref pin the record persists so a restore can - // re-materialize the closure. - sourceRef: projection.sourceRef, - }; - const record = buildWorkflowRunRecord(spec, spec.sources); - - // Persist the run record BEFORE the spawn so a crash mid-spawn leaves a - // record the boot scan re-drives (an idempotent re-spawn; the child's - // in-flight-run discovery resumes any run). A soft-failed deploy deletes - // it below, so only a crash-interrupted deploy leaves one. - await persistWorkflowRunRecord(dataDir, runId, record, deps.credentialCipher); - - // Sweep any legacy plaintext body-source file this deployment's bodies - // left behind. Body sources now ride sealed in the run record and reach - // the child through the spawn env, so the `assets/workflow//` - // staging is retired. A deployment first seen on this build never wrote - // one (the rm is a no-op); a redeploy across the upgrade -- or the reconnect - // re-push -- reaches here with its old world-readable plaintext file still - // on disk and removes it now that the record carries the sealed copy. - for (const referenced of projection.referencedDefinitions ?? []) { - await sweepLegacyBodySources(dataDir, referenced.definition.id); - } - - // The spawned child doesn't see the frame, so operator-approved grants are written to the same repo the supervisor reads, before the spawn core. Covers loop body steps too, or an unwritten repo denies the body every resource. - await writeStepGrants({ - repoStore: deps.repoStore, - anchorRunId: runId, - stepOrder: inertFlatNamespaceStepIds({ - definition: effectiveDefinition, - context: "sidecar deploy router grants bridge: ", - }), - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: frame.config.grants, - }); - - // Hand off to the shared spawn core. - return await spawnWorkflowRun(spec); - } catch (cause) { - // A rejecting delete is logged, not thrown, so `cause` still propagates and the slug is still released. - try { - await deleteWorkflowRunRecord(dataDir, runId); - } catch (cleanupError) { - const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); - logger.error`deploy cleanup: deleteWorkflowRunRecord failed for ${runId}: ${message}`; - } - releaseSlug(runId, frame.agentAddress); - throw cause; - } finally { - // On success the address is now in activeSupervisors, which the guard also consults, so a later re-deploy is still rejected. - reservingDeployAddresses.delete(frame.agentAddress); - } - } - - return { - async deploy(frame): Promise { - if (frame.provisionStep === true) { - return await provisionStep(frame); - } - if (frame.workflow !== undefined) { - return await deployMultiStep(frame, frame.workflow); - } - // A frame carrying neither provisionStep nor workflow is an unsupported shape; there is no in-process fall-through. - throw new Error( - `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, - ); - }, - async undeploy(frame): Promise { - // Routers come down before shutdown() so a hub-side frame racing the undeploy is dropped at the boundary rather than dispatched into a supervisor mid-teardown. unregister is idempotent. - const runId = deriveDeploymentId(frame.agentAddress); - deps.multistepMailRouter?.unregister(frame.agentAddress); - deps.inboundMailPolicyRegistry?.unregister(frame.agentAddress); - deps.multistepSignalRouter?.unregister(frame.agentAddress); - deps.multistepDrainRouter?.unregister(frame.agentAddress); - deps.multistepGrantsRouter?.unregister(frame.agentAddress); - deps.multistepSourcesRouter?.unregister(frame.agentAddress); - deps.multistepCredentialsRouter?.unregister(frame.agentAddress); - // Map entry removed before the await so a subsequent re-deploy cannot observe a stale handle even if shutdown() rejects. - const wired = activeSupervisors.get(frame.agentAddress); - if (wired !== undefined) { - activeSupervisors.delete(frame.agentAddress); - await wired.supervisor.shutdown(); - deps.transport.unregister(frame.agentAddress); - // Whole workflow-step-state// subtree goes; agent-conversation-state/ is a different root, deliberately not touched, so a re-deploy restores the prior conversation. - if (stepStateDataDir !== undefined) { - await rm(pathJoin(stepStateDataDir, "workflow-step-state", runId), { - recursive: true, - force: true, - }); - } - } - // Runs on every undeploy, not only when a supervisor was active, so a crash-interrupted deploy's leftover state is reclaimed too. - if (stepStateDataDir !== undefined) { - await deleteWorkflowRunRecord(stepStateDataDir, runId); - await rm(pathJoin(stepStateDataDir, "workflow-definition-closures", runId), { - recursive: true, - force: true, - }); - await rm(deploymentSourceAssetRoot(stepStateDataDir, runId), { - recursive: true, - force: true, - }); - await rm(deploymentSourceGitRoot(stepStateDataDir, runId), { - recursive: true, - force: true, - }); - } - releaseSlug(runId, frame.agentAddress); - deps.unregisterDeployment({ - runId, - agentAddress: frame.agentAddress, - }); - }, - async restoreWorkflowRuns(): Promise { - const dataDir = stepStateDataDir; - if (dataDir === undefined) { - // No substrate config was wired (a test router that never spawns a - // child): nothing was ever persisted under this data dir, so there - // is nothing to restore. - return; - } - - const scanned = await scanWorkflowRunRecords(dataDir, deps.credentialCipher); - // Restore serially, not in parallel: deterministic boot-log ordering, - // one isolable warning per failed record, and no concurrent - // child-spawn / transport-register storm. Restore runs before - // `hubLink.connect()`, so there are no concurrent deploys to contend - // with. Each record's failure is caught so one bad deployment cannot - // strand the rest. - for (const { runId, record } of scanned) { - try { - // Integrity: the stored address must re-derive to its own directory - // name. A mismatch means a corrupt or misplaced record; skip it - // rather than restore a deployment under the wrong slug. (A source-ref - // record missing its source/closure/approvedWireHash is rejected - // earlier, at the scan boundary, by the record schema's discriminated - // union -- so no bespoke source-ref guard is needed here.) - const derived = deriveDeploymentId(record.agentAddress); - if (derived !== runId) { - logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${runId}`; - continue; - } - - // Same computation the deploy path applies; safe to reclaim the instance dir first since the prior process is dead and restore is serial before hubLink.connect(). A cache/store miss soft-fails the record, kept for the next boot. - const applied = await materializeDeploymentClosure(dataDir, runId, record.sourceRef); - const validatedDefinition = WorkflowProjectionDefinition( - projectLiveToInert(applied.definition), - ); - if (validatedDefinition instanceof type.errors) { - logger.warn`skipping workflow deployment restore for ${record.agentAddress}: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`; - continue; - } - const definition: WorkflowProjectionDefinition = validatedDefinition; - const closurePackageDir = applied.packageDir; - - // The closure eval skips the deploy frame's coverage narrow, so this is where it's checked. - validateWorkflowProjection({ definition, sources: record.sources }); - - // Refuses to restore a deployment whose pinned provider this sidecar can no longer build; the record is KEPT so a later boot with the provider restored retries it. - for (const stepId of definition.stepOrder) { - const chain = record.sources[stepId]; - if (chain !== undefined) { - for (const source of chain) deps.assertSourceBuildable(source); - } - } - - const spec: WorkflowDeploySpec = { - agentAddress: record.agentAddress, - definition, - sources: record.sources, - // undefined for a legacy record written before body sources were persisted; the child falls back to the on-disk plaintext file. - bodySources: record.bodySources, - // Unsealed by the boot scan so an offline restart restores credentials without the hub. - credentials: record.credentials, - approvedWireHash: record.approvedWireHash, - sessionId: record.sessionId, - hubPublicKey: record.hubPublicKey, - closurePackageDir, - // Carried so a post-restore rotation re-persists it; otherwise a rotation would silently drop it and wedge the next restart. - sourceRef: record.sourceRef, - }; - - // Unlike deploy's soft-fail, restore does NOT delete the record: a failed restore just warns and leaves it for the next boot (an operator reclaims via undeploy). - // - // Release only a slug THIS pass newly claimed, or freeing it would strand a live deployment's collision guard. - const slugNewlyClaimed = slugClaims.get(runId) !== record.agentAddress; - claimSlug(runId, record.agentAddress); - try { - await spawnWorkflowRun(spec); - logger.info`Restored workflow deployment for ${record.agentAddress}`; - } catch (cause) { - if (slugNewlyClaimed) { - releaseSlug(runId, record.agentAddress); - } - throw cause; - } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`Failed to restore workflow deployment ${runId}: ${reason}`; - } - } - }, - activeAddresses(): string[] { - // `activeSupervisors` is keyed by deployment run address and holds - // exactly the deployments with a live supervisor (deploy and restore - // add; undeploy and spawn-unwind remove), so its keys are the addresses - // this sidecar can currently route mail to. - return [...activeSupervisors.keys()]; - }, - reEmitParkedCorrelations(address: string): void { - const wired = activeSupervisors.get(address); - if (wired === undefined) { - // Edge boundary: the hub reports this address routable but no live - // supervisor owns it -- a deployment torn down, or not yet respawned. - // Re-registering a torn-down deployment would be wrong, so skipping is - // correct. Logged at debug (not warn) so a supervisor unexpectedly - // missing for a live deployment is still traceable without crying wolf - // on the ordinary torn-down case. - logger.debug`re-emit on reconnect: no active supervisor for ${address}; skipping`; - return; - } - // Fire-and-forget: the driver is best-effort and watchdog-bounded inside - // the supervisor, so the reconnect fan-out never awaits it. The promise - // is dropped deliberately; the `.catch` is defense-in-depth since the - // driver already swallows its own query failures. - void wired.supervisor.reEmitParkedCorrelations().catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`re-emit of parked correlations on hub reconnect failed for ${address}: ${message}`; - }); - }, - }; -} - -/** The substrate never dereferences this; it's a host-side pointer the audit consumer joins on, unique per inbound message. */ -export function deriveSidecarMailAuditRef(runId: string): ( - messageId: string, - rawMessage: Uint8Array, -) => { - store: string; - path: string; -} { - return (messageId, _rawMessage) => ({ - store: "sidecar-mail-audit", - path: `${runId}/${messageId}`, - }); -} - -/** - * Construct a per-deployment supervisor with the sidecar's bindings - * pre-wired. The router calls this once per multi-step `agent.deploy` - * frame to stand up the workflow-process child that hosts the - * deployment. - */ -export function createSidecarWorkflowSupervisor( - opts: CreateSidecarWorkflowSupervisorOpts, -): SidecarWorkflowSupervisor { - const mailBus: HubTransportMailBusAdapter = wrapHubTransportAsMailBus(opts.transport); - const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { - kind: "supervisor", - anchorRunId: opts.runId, - }; - // Per-run grants sink. The supervisor awaits this and pushes the - // resulting snapshot to the child before the run's `trigger.fire`; a - // throw propagates and the dispatch barrier fails the run rather than - // firing the trigger against absent grants. The snapshot is the run's - // own per-run grants file, which every legitimate birth path writes - // before dispatch (see `assembleRunCredentialsSnapshot`). - const onRunStart = (args: { - runId: string; - anchorRunId: string; - }): Promise => { - // Fail closed on a run whose `run.grants` write was recorded as failed: - // its per-run grants file never landed. This is the known-failed-write - // case, distinct from the general absent-file backstop in - // `assembleRunCredentialsSnapshot` -- both fail the run closed, but this - // one names the recorded write failure specifically so a poisoned run's - // diagnostics point at the failed push rather than a generic absence. - if (opts.isRunPoisoned?.(args.runId) === true) { - return Promise.reject( - new Error( - `sidecar onRunStart: run ${args.runId} grants write failed; refusing to start the run under-authorized`, - ), - ); - } - return assembleRunCredentialsSnapshot({ - repoStore: opts.repoStore, - anchorRunId: args.anchorRunId, - runId: args.runId, - stepOrder: opts.stepOrder, - deriveStepAddress: opts.deriveStepAddress, - }); - }; - const supervisor = createWorkflowSupervisor({ - repoStore: opts.repoStore, - signAsPrincipal: async (kind, payload) => { - const sig = await signEd25519(opts.signingKeySeed, payload); - return { sig, principalKind: kind }; - }, - mailBus, - subprocessSpawner: opts.subprocessSpawner ?? defaultSubprocessSpawner, - binaryPath: opts.binaryPath ?? SIDECAR_WORKFLOW_CHILD_BINARY, - substrateEnv: opts.substrateEnv, - dynamicSpawnEnv: opts.dynamicSpawnEnv, - workflowRunRepoId: opts.workflowRunRepoId, - workflowRunRef: opts.workflowRunRef, - anchorRunId: opts.runId, - stepCount: opts.stepCount, - deploymentMailAddress: opts.deploymentMailAddress, - readPrincipal: supervisorPrincipal, - deriveStepAddress: opts.deriveStepAddress, - onRunStart, - ...(opts.credentialDelivery !== undefined - ? { credentialDelivery: opts.credentialDelivery } - : {}), - ...(opts.onSuspensionRegister !== undefined - ? { onSuspensionRegister: opts.onSuspensionRegister } - : {}), - ...(opts.onSelfTerminate !== undefined ? { onSelfTerminate: opts.onSelfTerminate } : {}), - ...(opts.deriveStepRepoId !== undefined ? { deriveStepRepoId: opts.deriveStepRepoId } : {}), - deriveMailAuditRef: deriveSidecarMailAuditRef(opts.runId), - ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } : {}), - ...(opts.repackEveryMessages !== undefined - ? { repackEveryMessages: opts.repackEveryMessages } - : {}), - ...(opts.consumedRetentionMs !== undefined - ? { consumedRetentionMs: opts.consumedRetentionMs } - : {}), - ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), - }); - return { - supervisor, - routeInbound(message) { - return mailBus.routeInbound(opts.deploymentMailAddress, message); - }, - getCredentialsSnapshot: () => supervisor.getCredentialsSnapshot(), - onRunStart, - }; -} diff --git a/apps/sidecar/src/workflow-probe-handler.ts b/apps/sidecar/src/workflow-probe-handler.ts deleted file mode 100644 index 90042974a5..0000000000 --- a/apps/sidecar/src/workflow-probe-handler.ts +++ /dev/null @@ -1,541 +0,0 @@ -// Evaluates a code-sourced workflow's entry to inspect it without -// deploying, so it must never run in the sidecar host's address space: -// this spawns a one-shot child behind an IPC airlock. Reaping is -// sidecar-owned with its own deadline, independent of the hub's probe -// timeout (which only rejects the hub-side promise, never kills the child). - -import { fileURLToPath } from "node:url"; - -import { type } from "arktype"; - -import { getLogger } from "@intx/log"; -import { GrantWalkSnapshot, hexDecode, hexEncode } from "@intx/types"; -import type { GrantRequirement } from "@intx/types"; -import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; -import { WorkflowProjectionDefinition } from "@intx/types/sidecar"; -import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; -import { collectDeclaredPluginNames, projectLiveToInert } from "@intx/workflow"; -import { walkCapabilities, type CapabilityWalkResult } from "@intx/workflow-deploy"; -import { - DEFAULT_KILL_TIMEOUT_MS, - MacedEnvelope, - encodeEnvelope, - generateChannelId, - generateHmacKey, - loadWorkflowDefinitionFromClosure, - loadWorkflowDirectorRegistryFromClosure, - loadWorkflowPluginToolDefinitionsFromClosure, - signHmac, - verifyHmac, - type FrameEnvelope, -} from "@intx/workflow-host"; - -const logger = getLogger(["sidecar", "workflow-probe"]); - -const IPC_HMAC_KEY_BYTES = 32; - -/** Independent of the hub's probeTimeoutMs, which only rejects the hub-side promise; this is what kills a wedged child. */ -export const DEFAULT_PROBE_CHILD_TIMEOUT_MS = 30_000; - -/** Mirrors the supervisor's kill timeout so a probe child is force-killed on the same schedule. */ -export const DEFAULT_PROBE_CHILD_KILL_TIMEOUT_MS = DEFAULT_KILL_TIMEOUT_MS; - -// Env keys the host sets on the child's fresh spawn env. The child reads -// exactly these plus PATH/HOME/TMPDIR (for exec + tmp); nothing else -// crosses the airlock. -const PROBE_CHANNEL_ID_ENV = "PROBE_IPC_CHANNEL_ID"; -const PROBE_HMAC_KEY_ENV = "PROBE_IPC_HMAC_KEY"; -const PROBE_PACKAGE_DIR_ENV = "PROBE_PACKAGE_DIR"; - -/** Resolved statically at module load so the spawn surface doesn't depend on a runtime env override. */ -const DEFAULT_PROBE_CHILD_BINARY: string = fileURLToPath( - import.meta.resolve("../bin/workflow-probe-child"), -); - -// --------------------------------------------------------------------------- -// Result payload wire (child -> host) -// --------------------------------------------------------------------------- - -/** ok: false carries the failure reason so the host rejects with a message rather than a bare "child exited". */ -const ProbeResultPayload = type({ - ok: "true", - projection: "unknown", - grants: "string[]", - grantWalkSnapshot: GrantWalkSnapshot, - wireHash: "string > 0", -}).or({ - ok: "false", - error: "string", -}); -type ProbeResultPayload = typeof ProbeResultPayload.infer; - -/** grantWalkSnapshot preserves per-step grouping and effect data the flattened `grants` union discards. */ -export interface WorkflowProbeResult { - readonly projection: WorkflowProjectionDefinition; - readonly grants: string[]; - readonly grantWalkSnapshot: GrantWalkSnapshot; - readonly wireHash: string; -} - -// --------------------------------------------------------------------------- -// Closure materialization seam -// --------------------------------------------------------------------------- - -/** node_modules/ is laid out so the entry's bare-specifier imports resolve; cleanup runs once the child is reaped. */ -export interface MaterializedWorkflowClosure { - readonly packageDir: string; - cleanup(): Promise; -} - -/** Runs on the sidecar host (I/O, not author-code eval), so the airlocked child only does load+evaluate. */ -export type MaterializeWorkflowClosure = ( - frame: WorkflowProbeRequestFrame, -) => Promise; - -// --------------------------------------------------------------------------- -// Child spawn seam -// --------------------------------------------------------------------------- - -/** No control/event channels: the probe carries no bidirectional control traffic. */ -export interface ProbeChildHandle { - readonly pid: number; - readonly stdout: ReadableStream; - kill(signal?: number | string): void; - readonly exited: Promise; -} - -/** Tests inject a spawner that records the pid so they can assert the child was reaped. */ -export type ProbeChildSpawner = (args: { - binaryPath: string; - env: Record; -}) => ProbeChildHandle; - -/** No process.env spread — the caller assembles a fresh env. */ -export const defaultProbeChildSpawner: ProbeChildSpawner = ({ - binaryPath, - env, -}): ProbeChildHandle => { - const proc = Bun.spawn([binaryPath], { - stdio: ["ignore", "pipe", "inherit"], - env, - }); - return { - pid: proc.pid, - stdout: proc.stdout, - kill(signal?: number | string): void { - if (signal === undefined) { - proc.kill(); - return; - } - if (typeof signal === "number") { - proc.kill(signal); - return; - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the probe reaper passes "SIGTERM"/"SIGKILL"; Bun's runtime accepts the same "SIG*" strings, narrowed back at the boundary. - proc.kill(signal as NodeJS.Signals); - }, - exited: proc.exited, - }; -}; - -// --------------------------------------------------------------------------- -// Executor (host side) -// --------------------------------------------------------------------------- - -export interface WorkflowProbeExecutorOpts { - /** Host-side materializer for the frame's frozen closure. */ - materialize: MaterializeWorkflowClosure; - /** Override the child spawner (defaults to the Bun.spawn-backed one). */ - spawnProbeChild?: ProbeChildSpawner; - /** Override the `bin/workflow-probe-child` path. */ - binaryPath?: string; - /** - * Self-owned deadline before the child is reaped and the probe fails. - * Independent of the hub's `probeTimeoutMs`. - */ - childTimeoutMs?: number; - /** SIGTERM->SIGKILL escalation window when reaping. */ - killTimeoutMs?: number; -} - -/** Satisfies the hub-agent WorkflowProbeExecutor seam; throws so the link answers workflow.probe.error. */ -export function createWorkflowProbeExecutor(opts: WorkflowProbeExecutorOpts): { - probe(frame: WorkflowProbeRequestFrame): Promise; -} { - const spawnProbeChild = opts.spawnProbeChild ?? defaultProbeChildSpawner; - const binaryPath = opts.binaryPath ?? DEFAULT_PROBE_CHILD_BINARY; - const childTimeoutMs = opts.childTimeoutMs ?? DEFAULT_PROBE_CHILD_TIMEOUT_MS; - const killTimeoutMs = opts.killTimeoutMs ?? DEFAULT_PROBE_CHILD_KILL_TIMEOUT_MS; - - async function probe(frame: WorkflowProbeRequestFrame): Promise { - const materialized = await opts.materialize(frame); - try { - return await runOneShotProbeChild({ - packageDir: materialized.packageDir, - spawnProbeChild, - binaryPath, - childTimeoutMs, - killTimeoutMs, - }); - } finally { - await materialized.cleanup(); - } - } - - return { probe }; -} - -interface RunOneShotProbeChildArgs { - readonly packageDir: string; - readonly spawnProbeChild: ProbeChildSpawner; - readonly binaryPath: string; - readonly childTimeoutMs: number; - readonly killTimeoutMs: number; -} - -/** The finally guarantees the child is killed on every exit path, including a self-owned deadline firing first. */ -async function runOneShotProbeChild(args: RunOneShotProbeChildArgs): Promise { - const channelId = generateChannelId(); - const hmacKey = generateHmacKey(); - const env = buildProbeChildEnv({ - packageDir: args.packageDir, - channelId, - hmacKey, - }); - const handle = args.spawnProbeChild({ binaryPath: args.binaryPath, env }); - - let reaped = false; - async function reap(): Promise { - if (reaped) return; - reaped = true; - await reapChild(handle, args.killTimeoutMs); - } - - // Attach a catch so a post-reap stdout read error (the kill closes the - // pipe mid-read) resolves to null instead of surfacing as an unhandled - // rejection on the losing race branch. - const linePromise: Promise = readResultLine(handle.stdout).catch( - (err: unknown) => { - logger.debug`probe child ${String(handle.pid)} stdout read errored: ${errorMessage(err)}`; - return null; - }, - ); - - const deadline = createDeadline(args.childTimeoutMs); - try { - // Child exit is deliberately not a race arm: it would risk winning - // against an already-written result and failing the probe spuriously. - // readResultLine already settles on stdout close either way. - const outcome = await Promise.race([ - linePromise.then((line) => ({ kind: "line" as const, line })), - deadline.promise.then(() => ({ kind: "timeout" as const })), - ]); - - if (outcome.kind === "timeout") { - throw new Error( - `workflow probe child ${String(handle.pid)} did not produce a result within ${String(args.childTimeoutMs)}ms`, - ); - } - if (outcome.line === null) { - throw new Error( - `workflow probe child ${String(handle.pid)} closed its output without producing a result`, - ); - } - return await parseProbeResult(outcome.line, channelId, hmacKey); - } finally { - deadline.cancel(); - await reap(); - } -} - -function buildProbeChildEnv(args: { - packageDir: string; - channelId: string; - hmacKey: Uint8Array; -}): Record { - // No process.env spread: no sidecar secret or ambient input crosses the airlock. - const env: Record = { - [PROBE_CHANNEL_ID_ENV]: args.channelId, - [PROBE_HMAC_KEY_ENV]: hexEncode(args.hmacKey), - [PROBE_PACKAGE_DIR_ENV]: args.packageDir, - }; - const path = process.env["PATH"]; - if (path !== undefined) env["PATH"] = path; - const home = process.env["HOME"]; - if (home !== undefined) env["HOME"] = home; - const tmpdir = process.env["TMPDIR"]; - if (tmpdir !== undefined) env["TMPDIR"] = tmpdir; - return env; -} - -/** SIGKILL is unignorable, so `exited` always settles even if the child traps SIGTERM. */ -async function reapChild(handle: ProbeChildHandle, killTimeoutMs: number): Promise { - try { - handle.kill("SIGTERM"); - } catch (err) { - logger.debug`probe child ${String(handle.pid)} SIGTERM raised (already exited?): ${errorMessage(err)}`; - } - const deadline = createDeadline(killTimeoutMs); - const first = await Promise.race([ - handle.exited.then(() => "exited" as const), - deadline.promise.then(() => "deadline" as const), - ]); - deadline.cancel(); - if (first === "exited") return; - logger.warn`workflow probe child ${String(handle.pid)} did not exit on SIGTERM within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; - try { - handle.kill("SIGKILL"); - } catch (err) { - logger.debug`probe child ${String(handle.pid)} SIGKILL raised (already exited?): ${errorMessage(err)}`; - } - await handle.exited.catch(() => { - // A non-zero exit on SIGKILL is the expected outcome; reaping treats - // child exit as success regardless of code. - }); -} - -/** Verifies HMAC before trusting any field, mirroring the event channel's receiver. */ -async function parseProbeResult( - line: string, - channelId: string, - hmacKey: Uint8Array, -): Promise { - let raw: unknown; - try { - raw = JSON.parse(line); - } catch (cause) { - throw new Error("workflow probe child result is not valid JSON", { cause }); - } - const maced = MacedEnvelope(raw); - if (maced instanceof type.errors) { - throw new Error(`workflow probe child result envelope failed validation: ${maced.summary}`); - } - const envelopeBytes = encodeEnvelope(maced.envelope); - const macBytes = hexDecode(maced.mac); - const ok = await verifyHmac(envelopeBytes, macBytes, hmacKey); - if (!ok) { - throw new Error( - `workflow probe child result HMAC did not verify (channelId=${maced.envelope.channelId})`, - ); - } - if (maced.envelope.channelId !== channelId) { - throw new Error( - `workflow probe child result carried a foreign channelId ${JSON.stringify(maced.envelope.channelId)}`, - ); - } - const payload = ProbeResultPayload(maced.envelope.payload); - if (payload instanceof type.errors) { - throw new Error(`workflow probe child result payload failed validation: ${payload.summary}`); - } - if (!payload.ok) { - throw new Error(`workflow probe evaluation failed: ${payload.error}`); - } - const projection = WorkflowProjectionDefinition(payload.projection); - if (projection instanceof type.errors) { - throw new Error(`workflow probe child projection failed validation: ${projection.summary}`); - } - return { - projection, - grants: payload.grants, - grantWalkSnapshot: payload.grantWalkSnapshot, - wireHash: payload.wireHash, - }; -} - -// --------------------------------------------------------------------------- -// Child side -// --------------------------------------------------------------------------- - -/** Bytes are handed to the OS before the child exits so the result isn't truncated. */ -export type ProbeChildLineWriter = (line: string) => Promise; - -export interface RunProbeChildOpts { - /** Raw env the child reads its anchors from (defaults to `process.env`). */ - rawEnv?: Readonly>; - /** Result-line sink (defaults to a drained `process.stdout` write). */ - writeLine?: ProbeChildLineWriter; -} - -/** An eval failure ships as ok: false so the host reaps cleanly instead of seeing a bare crash. */ -export async function runWorkflowProbeChildFromProcessEnv( - opts: RunProbeChildOpts = {}, -): Promise { - const rawEnv = opts.rawEnv ?? process.env; - const writeLine = opts.writeLine ?? defaultStdoutWriteLine; - const { channelId, hmacKey, packageDir } = parseProbeChildEnv(rawEnv); - - let payload: ProbeResultPayload; - try { - payload = await computeProbePayload(packageDir); - } catch (err) { - payload = { ok: false, error: enrichProbeError(err) }; - } - - const envelope: FrameEnvelope = { seq: 0, channelId, payload }; - const envelopeBytes = encodeEnvelope(envelope); - const mac = hexEncode(await signHmac(envelopeBytes, hmacKey)); - await writeLine(`${JSON.stringify({ envelope, mac })}\n`); -} - -async function computeProbePayload(packageDir: string): Promise { - const definition = await loadWorkflowDefinitionFromClosure({ packageDir }); - const projection = projectLiveToInert(definition); - const wireHash = await computeWireDefinitionHash(projection); - // Composed from the SAME closure the run-child will use, so director: - // grants advertised here match what the runtime resolves. - const directors = await loadWorkflowDirectorRegistryFromClosure({ - packageDir, - }); - // A plugin reaches an agent only through env.plugins, so without loading - // its tool definitions here the walk would miss tool: grants and the - // run-child would later fail closed on an un-approved tool. - const pluginToolDefinitions = await loadWorkflowPluginToolDefinitionsFromClosure({ - packageDir, - plugins: collectDeclaredPluginNames(definition), - }); - const walk = walkCapabilities(definition, directors, pluginToolDefinitions); - // Fail closed: the runtime never re-gates director:, so this - // advertisement is the only approval checkpoint for it. - const [unresolved] = walk.unresolvedDirectors; - if (unresolved !== undefined) { - return { ok: false, error: `unresolvable director: ${unresolved}` }; - } - return { - ok: true, - projection, - grants: collectDeploymentGrants(walk), - grantWalkSnapshot: buildGrantWalkSnapshot(walk, definition.grantRequirements), - wireHash, - }; -} - -/** Sorting makes the shipped set order-independent. */ -function collectDeploymentGrants(walk: CapabilityWalkResult): string[] { - const grants = new Set(); - for (const declarations of walk.perStep.values()) { - for (const grant of declarations.grants) { - grants.add(grant); - } - } - return [...grants].sort(); -} - -/** Unlike collectDeploymentGrants, preserves per-step grouping and effect data the flattened set discards. */ -function buildGrantWalkSnapshot( - walk: CapabilityWalkResult, - grantRequirements: readonly GrantRequirement[] | undefined, -): GrantWalkSnapshot { - const perStep = [...walk.perStep].map(([stepId, declarations]) => ({ - stepId, - grants: [...declarations.grants], - grantEffects: Object.fromEntries(declarations.grantEffects), - })); - return { - perStep, - grantRequirements: [...(grantRequirements ?? [])], - }; -} - -interface ProbeChildEnv { - readonly channelId: string; - readonly hmacKey: Uint8Array; - readonly packageDir: string; -} - -const NonEmptyString = type("string > 0"); - -function parseProbeChildEnv(rawEnv: Readonly>): ProbeChildEnv { - const channelId = requireEnv(rawEnv, PROBE_CHANNEL_ID_ENV); - const packageDir = requireEnv(rawEnv, PROBE_PACKAGE_DIR_ENV); - const hmacKeyHex = requireEnv(rawEnv, PROBE_HMAC_KEY_ENV); - const hmacKey = hexDecode(hmacKeyHex); - if (hmacKey.length !== IPC_HMAC_KEY_BYTES) { - throw new Error( - `workflow probe child env: ${PROBE_HMAC_KEY_ENV} must decode to ${String(IPC_HMAC_KEY_BYTES)} bytes, got ${String(hmacKey.length)}`, - ); - } - return { channelId, hmacKey, packageDir }; -} - -function requireEnv(rawEnv: Readonly>, key: string): string { - const value = NonEmptyString(rawEnv[key]); - if (value instanceof type.errors) { - throw new Error(`workflow probe child env: required key ${key} is unset or empty`); - } - return value; -} - -function defaultStdoutWriteLine(line: string): Promise { - return new Promise((resolve, reject) => { - process.stdout.write(line, (err) => { - if (err) reject(err); - else resolve(); - }); - }); -} - -// --------------------------------------------------------------------------- -// Shared helpers -// --------------------------------------------------------------------------- - -/** Resolves null when the stream closes without a complete line (the child exited before writing). */ -async function readResultLine(stream: ReadableStream): Promise { - const reader = stream.getReader(); - const decoder = new TextDecoder("utf-8"); - let pending = ""; - try { - for (;;) { - const { value, done } = await reader.read(); - if (value !== undefined) { - pending += decoder.decode(value, { stream: true }); - const nl = pending.indexOf("\n"); - if (nl >= 0) { - return pending.slice(0, nl).replace(/\r$/, ""); - } - } - if (done) { - const trailing = pending.replace(/\r?\n$/, ""); - return trailing.length > 0 ? trailing : null; - } - } - } finally { - reader.releaseLock(); - } -} - -function createDeadline(ms: number): { - promise: Promise; - cancel: () => void; -} { - let handle: ReturnType | undefined; - const promise = new Promise((resolve) => { - handle = setTimeout(resolve, ms); - }); - return { - promise, - cancel(): void { - if (handle !== undefined) clearTimeout(handle); - }, - }; -} - -function errorMessage(err: unknown): string { - return err instanceof Error ? err.message : String(err); -} - -// Node/Bun's module-not-found message shape. The specifier is the missing -// package the workflow entry imported at evaluation time. -const MISSING_MODULE_RE = /Cannot find (?:module|package) ['"]([^'"]+)['"]/; - -/** Rewrites "Cannot find module" into an actionable hint: the common cause is a devDependencies-only import. */ -export function enrichProbeError(err: unknown): string { - const message = errorMessage(err); - const match = MISSING_MODULE_RE.exec(message); - const specifier = match?.[1]; - if (specifier === undefined) return message; - return ( - `workflow entry could not resolve ${JSON.stringify(specifier)} from its dependency closure; ` + - `if the workflow imports it at run time, declare it under "dependencies" rather than "devDependencies" ` + - `(a devDependencies-only import is not materialized into the closure). ${message}` - ); -} diff --git a/apps/sidecar/src/workflow-probe-spawner.ts b/apps/sidecar/src/workflow-probe-spawner.ts new file mode 100644 index 0000000000..8052197f04 --- /dev/null +++ b/apps/sidecar/src/workflow-probe-spawner.ts @@ -0,0 +1,50 @@ +// Bun-backed spawn edge for the one-shot workflow probe child. +// +// The probe executor takes a spawner and a binary path as required +// arguments. This module is the process that owns both: it resolves +// `bin/workflow-probe-child` and launches it with a fresh env. + +import { fileURLToPath } from "node:url"; + +import type { ProbeChildHandle, ProbeChildSpawner } from "@intx/workflow-host/probe"; + +/** + * Path of this package's `bin/workflow-probe-child`. Resolved at load so + * the boot edge can hand the probe executor a concrete path. + */ +export const SIDECAR_WORKFLOW_PROBE_CHILD_BINARY: string = fileURLToPath( + import.meta.resolve("../bin/workflow-probe-child"), +); + +/** + * Real `Bun.spawn`-backed probe-child spawner. The caller assembles a + * fresh env (no `process.env` spread). Stdout is piped for the result + * line, stdin is ignored, and stderr is inherited so child diagnostics + * land on the sidecar's stderr. + */ +export const defaultProbeChildSpawner: ProbeChildSpawner = ({ + binaryPath, + env, +}): ProbeChildHandle => { + const proc = Bun.spawn([binaryPath], { + stdio: ["ignore", "pipe", "inherit"], + env, + }); + return { + pid: proc.pid, + stdout: proc.stdout, + kill(signal?: number | string): void { + if (signal === undefined) { + proc.kill(); + return; + } + if (typeof signal === "number") { + proc.kill(signal); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the probe reaper passes "SIGTERM"/"SIGKILL"; Bun's runtime accepts the same "SIG*" strings, narrowed back at the boundary. + proc.kill(signal as NodeJS.Signals); + }, + exited: proc.exited, + }; +}; diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts deleted file mode 100644 index 1281994974..0000000000 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ /dev/null @@ -1,566 +0,0 @@ -// Sits between the boot-edge substrate facade and the hub-link's -// pushWorkflowRunPack wire surface, fired after a successful -// writeTreePreservingPrefix against a workflow-run repo. - -import { type } from "arktype"; - -import { getLogger } from "@intx/log"; -import { SourcesUpdatedData } from "@intx/workflow-host"; -import type { InferenceSource } from "@intx/types/runtime"; -import { CredentialDelivery, type SenderIdentity } from "@intx/types/sidecar"; -import type { RepoId, RepoStore, WorkflowRunSupervisorPrincipal } from "@intx/hub-sessions"; -import type { HubLink } from "@intx/hub-agent"; - -const logger = getLogger(["interchange", "sidecar", "workflow-run-pack-client"]); - -export type WorkflowRunPackClient = { - /** Resolves on the hub's repo.pack.ack; rejects loud on reject, disconnect, or a substrate createPack failure. */ - push(opts: { agentAddress: string; repoId: RepoId; ref: string }): Promise; - /** Prevents a reconnect from re-sending an empty delta at the restored tip. */ - markRestored(repoId: RepoId, ref: string, commitSha: string): void; -}; - -export type CreateWorkflowRunPackClientOpts = { - substrate: RepoStore; - hubLink: Pick; -}; - -export function createWorkflowRunPackClient( - opts: CreateWorkflowRunPackClientOpts, -): WorkflowRunPackClient { - const { substrate, hubLink } = opts; - - // Answers a question createPack can't: is the tip already shipped? Skips - // the wire send when it is, since an empty-delta pack the hub rejects as - // sha_mismatch would otherwise turn a re-drive into a spurious rejection. - const lastAckedSha = new Map(); - function ackKey(repoId: RepoId, ref: string): string { - return `${repoId.id}/${ref}`; - } - - return { - markRestored(repoId, ref, commitSha) { - if (repoId.kind !== "workflow-run") { - throw new Error( - `workflow-run pack client: restored repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, - ); - } - substrate.commitPackedTip(repoId, ref, commitSha); - lastAckedSha.set(ackKey(repoId, ref), commitSha); - }, - async push({ agentAddress, repoId, ref }) { - if (repoId.kind !== "workflow-run") { - throw new Error( - `workflow-run pack client: repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, - ); - } - const principal: WorkflowRunSupervisorPrincipal = { - kind: "supervisor", - anchorRunId: repoId.id, - }; - // Nothing to ship if the run's commits already landed on a prior push. - const tip = await substrate.resolveRef(principal, repoId, ref); - if (tip !== null && tip === lastAckedSha.get(ackKey(repoId, ref))) { - return; - } - const { pack, commitSha } = await substrate.createPack(principal, repoId, ref); - await hubLink.pushWorkflowRunPack({ - agentAddress, - repoId, - pack, - ref, - commitSha, - }); - // Advanced only after the ack (never at build time) so a cancelled - // transfer leaves the cursor put and gets re-shipped on retry. - substrate.commitPackedTip(repoId, ref, commitSha); - lastAckedSha.set(ackKey(repoId, ref), commitSha); - }, - }; -} - -/** Resolves repoId.id (the workflow-run runId) back into the agentAddress carried on outbound pack frames. */ -export type DeploymentAddressRegistry = { - record(runId: string, agentAddress: string): void; - resolve(runId: string): string | null; - unregister(runId: string): void; -}; - -export function createDeploymentAddressRegistry(): DeploymentAddressRegistry { - const table = new Map(); - return { - record(runId, agentAddress) { - table.set(runId, agentAddress); - }, - resolve(runId) { - return table.get(runId) ?? null; - }, - unregister(runId) { - table.delete(runId); - }, - }; -} - -/** Dispatches into the workflow-host mail-bus the multi-step child's awaitSignal subscribes against. */ -export type MultistepMailHandler = (message: Uint8Array) => Promise; - -/** - * Kept at the sidecar host layer (not inside workflow-host) since the - * routing decision — legacy single-agent path vs supervisor mail-bus — - * is a concrete host concern the portable package stays agnostic to. - */ -export type MultistepMailRouter = { - register(address: string, handler: MultistepMailHandler): void; - unregister(address: string): void; - /** Returns null when unregistered (caller logs and drops, sends no ack). */ - tryRoute(address: string, message: Uint8Array): Promise | null; -}; - -export function createMultistepMailRouter(): MultistepMailRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - tryRoute(address, message) { - const handler = handlers.get(address); - if (handler === undefined) return null; - return handler(message); - }, - }; -} - -/** Routing every signal through the child preserves the workflow-run repo's single-writer invariant. */ -export type MultistepSignalHandler = (args: { - runId: string; - signalName: string; - signalId: string; - payload: unknown; -}) => Promise; - -// Same boundary reason as MultistepMailRouter. -export type MultistepSignalRouter = { - register(address: string, handler: MultistepSignalHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "signal.deliver"; - agentAddress: string; - runId: string; - signalName: string; - signalId: string; - payload: unknown; - }): Promise; -}; - -export function createMultistepSignalRouter(): MultistepSignalRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) return false; - await handler({ - runId: frame.runId, - signalName: frame.signalName, - signalId: frame.signalId, - payload: frame.payload, - }); - return true; - }, - }; -} - -/** - * The write is awaited so FIFO completion means grants are durable before - * the next frame processes. senderIdentities are cached before the write so - * a durable grant is never missing the key to verify the sender's mail. - */ -export type MultistepGrantsHandler = (args: { - runId: string; - stepGrants: readonly unknown[]; - senderIdentities?: readonly SenderIdentity[]; -}) => Promise; - -// Same boundary reason as MultistepMailRouter / MultistepSignalRouter. -export type MultistepGrantsRouter = { - register(address: string, handler: MultistepGrantsHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "run.grants"; - agentAddress: string; - runId: string; - stepGrants: readonly unknown[]; - senderIdentities?: readonly SenderIdentity[]; - }): Promise; -}; - -export function createMultistepGrantsRouter(): MultistepGrantsRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) return false; - await handler({ - runId: frame.runId, - stepGrants: frame.stepGrants, - ...(frame.senderIdentities !== undefined - ? { senderIdentities: frame.senderIdentities } - : {}), - }); - return true; - }, - }; -} - -/** Each drainTimeout accumulator commits a signed CancelRequested against the workflow-run repo at deadline. */ -export type MultistepDrainHandler = (args: { deadlineMs: number }) => Promise; - -// Same boundary reason as MultistepMailRouter / MultistepSignalRouter. -export type MultistepDrainRouter = { - register(address: string, handler: MultistepDrainHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "drain.deliver"; - agentAddress: string; - deadlineMs: number; - }): Promise; -}; - -export function createMultistepDrainRouter(): MultistepDrainRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) return false; - await handler({ deadlineMs: frame.deadlineMs }); - return true; - }, - }; -} - -/** Only for single-step (warm launched-agent) deployments; multi-step has no single warm agent to rotate. */ -export type MultistepSourcesHandler = (args: { - sources: InferenceSource[]; - defaultSource: string; -}) => Promise; - -// Same boundary reason as the mail/signal/drain routers. -export type MultistepSourcesRouter = { - register(address: string, handler: MultistepSourcesHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "sources.update"; - agentAddress: string; - sources: InferenceSource[]; - defaultSource: string; - }): Promise; -}; - -export function createMultistepSourcesRouter(): MultistepSourcesRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) return false; - // Validated before dispatch, unlike other routers: a bad frame would - // crash the child's control-channel receiver on SourcesUpdatedData's - // narrow. Throwing here turns it into a session.error instead. - const validated = SourcesUpdatedData({ - sources: frame.sources, - defaultSource: frame.defaultSource, - }); - if (validated instanceof type.errors) { - throw new Error(validated.summary); - } - await handler({ - sources: frame.sources, - defaultSource: frame.defaultSource, - }); - return true; - }, - }; -} - -/** - * Unlike sources rotation, registers for any deployment: the material cell - * is per-child. No durable persist — material never touches disk. - */ -export type MultistepCredentialsHandler = (args: { - delivery: CredentialDelivery; - revoke?: string[]; -}) => Promise; - -/** Mirrors MultistepSourcesRouter: request/ack, so a throw surfaces as session.error. */ -export type MultistepCredentialsRouter = { - register(address: string, handler: MultistepCredentialsHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "credentials.update"; - agentAddress: string; - delivery: CredentialDelivery; - revoke?: string[]; - }): Promise; -}; - -export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { - const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) return false; - // Validated before dispatch: a malformed delivery would crash the - // child's control-channel receiver instead of surfacing as session.error. - const validated = CredentialDelivery(frame.delivery); - if (validated instanceof type.errors) { - throw new Error(validated.summary); - } - await handler({ - delivery: frame.delivery, - ...(frame.revoke !== undefined ? { revoke: frame.revoke } : {}), - }); - return true; - }, - }; -} - -/** - * Wraps RepoStore so a workflow-run write schedules a coalesced pack push: - * at most one push per (repoId, ref) in flight, with mid-push writes - * marking the slot dirty for one more run. The cursor advances only after - * the hub's ack, and a failed push latches and re-throws on the next write. - */ -export type WorkflowRunPackPushingRepoStoreOpts = { - underlying: RepoStore; - packClient: Pick; - registry: DeploymentAddressRegistry; -}; - -/** RepoStore shape is unchanged; flushWorkflowRunPushes is opt-in for callers needing hub-side visibility. */ -export type WorkflowRunPackPushingRepoStore = RepoStore & { - /** Resolves once the (repoId.id, ref) pipeline drains; rejects on the same latched error a write would surface. */ - flushWorkflowRunPushes: (repoId: RepoId, ref: string) => Promise; - /** - * Re-arms a slot a disconnect cancelled — the liveness path a synchronous - * single-step run lacks, since it has no later local write to re-set dirty. - * Gated on post-reconnect so the re-ship can't race ahead of hub re-routing. - */ - notifyAddressRoutable: (agentAddress: string) => void; - /** Holds pushes rather than shipping-and-failing, so a WS-drop address waits for reconnect before shipping. */ - markAddressUnroutable: (agentAddress: string) => void; -}; - -export function createWorkflowRunPackPushingRepoStore( - opts: WorkflowRunPackPushingRepoStoreOpts, -): WorkflowRunPackPushingRepoStore { - const { underlying, packClient, registry } = opts; - - type Slot = { - agentAddress: string; - repoId: RepoId; - ref: string; - inFlight: Promise | null; - dirty: boolean; - lastError: Error | null; - settled: (() => void)[]; - }; - const slots = new Map(); - function slotKey(repoId: RepoId, ref: string): string { - return `${repoId.kind}/${repoId.id}/${ref}`; - } - - // Absent means routable. A blocked address's push holds with dirty still - // set, rather than racing ahead of hub re-routing and being dropped. - const blockedAddresses = new Set(); - - function notifySettled(slot: Slot): void { - const callbacks = slot.settled; - slot.settled = []; - for (const cb of callbacks) cb(); - } - - function startLoop(slot: Slot, repoId: RepoId, ref: string): void { - if (slot.inFlight !== null) return; - if (blockedAddresses.has(slot.agentAddress)) return; - slot.inFlight = (async () => { - while (slot.dirty) { - // Re-checked each iteration: a disconnect mid-drain must pause - // rather than push into a severed link. - if (blockedAddresses.has(slot.agentAddress)) break; - slot.dirty = false; - try { - await packClient.push({ - agentAddress: slot.agentAddress, - repoId, - ref, - }); - slot.lastError = null; - } catch (cause) { - const msg = cause instanceof Error ? cause.message : String(cause); - logger.warn`workflow-run pack push failed for deployment ${repoId.id} (${slot.agentAddress}): ${msg}`; - slot.lastError = cause instanceof Error ? cause : new Error(String(cause)); - } - } - slot.inFlight = null; - notifySettled(slot); - })(); - } - - function schedulePush(agentAddress: string, repoId: RepoId, ref: string): void { - const key = slotKey(repoId, ref); - let slot = slots.get(key); - if (slot === undefined) { - slot = { - agentAddress, - repoId, - ref, - inFlight: null, - dirty: false, - lastError: null, - settled: [], - }; - slots.set(key, slot); - } else { - // Refreshed every call, not cached: the contract is "look up at push time". - slot.agentAddress = agentAddress; - } - slot.dirty = true; - startLoop(slot, repoId, ref); - } - - function takeLatchedError(repoId: RepoId, ref: string): Error | null { - const slot = slots.get(slotKey(repoId, ref)); - if (slot === undefined) return null; - const err = slot.lastError; - if (err !== null) slot.lastError = null; - return err; - } - - function markAddressUnroutable(agentAddress: string): void { - blockedAddresses.add(agentAddress); - } - - function notifyAddressRoutable(agentAddress: string): void { - blockedAddresses.delete(agentAddress); - for (const slot of slots.values()) { - if (slot.agentAddress !== agentAddress) continue; - // Skip a clean, already-acked slot to avoid a pointless loop spin. - // Re-arming is safe against double-ship: startLoop no-ops if a push - // is already in flight. - if (!slot.dirty && slot.lastError === null) continue; - slot.dirty = true; - startLoop(slot, slot.repoId, slot.ref); - } - } - - async function flushWorkflowRunPushes(repoId: RepoId, ref: string): Promise { - const slot = slots.get(slotKey(repoId, ref)); - if (slot === undefined) return; - if (slot.inFlight === null && !slot.dirty) { - if (slot.lastError !== null) { - const err = slot.lastError; - slot.lastError = null; - throw err; - } - return; - } - await new Promise((resolve) => { - slot.settled.push(resolve); - }); - if (slot.lastError !== null) { - const err = slot.lastError; - slot.lastError = null; - throw err; - } - } - - const wrapped: WorkflowRunPackPushingRepoStore = { - initRepo: underlying.initRepo.bind(underlying), - writeTree: underlying.writeTree.bind(underlying), - receivePack: underlying.receivePack.bind(underlying), - createPack: underlying.createPack.bind(underlying), - commitPackedTip: underlying.commitPackedTip.bind(underlying), - resolveRef: underlying.resolveRef.bind(underlying), - listRefs: underlying.listRefs.bind(underlying), - resolveHead: underlying.resolveHead.bind(underlying), - getRepoDir: underlying.getRepoDir.bind(underlying), - openCommittedReads: underlying.openCommittedReads.bind(underlying), - openCommittedReadsAtCommit: underlying.openCommittedReadsAtCommit.bind(underlying), - subscribe: underlying.subscribe.bind(underlying), - flushWorkflowRunPushes, - notifyAddressRoutable, - markAddressUnroutable, - async writeTreePreservingPrefix(principal, repoId, ref, args) { - if (repoId.kind === "workflow-run") { - const latched = takeLatchedError(repoId, ref); - if (latched !== null) { - throw latched; - } - } - const result = await underlying.writeTreePreservingPrefix(principal, repoId, ref, args); - if (repoId.kind !== "workflow-run") { - return result; - } - const agentAddress = registry.resolve(repoId.id); - if (agentAddress === null) { - throw new Error( - `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, - ); - } - schedulePush(agentAddress, repoId, ref); - return result; - }, - async writeTreeDelta(principal, repoId, ref, args) { - if (repoId.kind === "workflow-run") { - const latched = takeLatchedError(repoId, ref); - if (latched !== null) { - throw latched; - } - } - const result = await underlying.writeTreeDelta(principal, repoId, ref, args); - if (repoId.kind !== "workflow-run") { - return result; - } - const agentAddress = registry.resolve(repoId.id); - if (agentAddress === null) { - throw new Error( - `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, - ); - } - schedulePush(agentAddress, repoId, ref); - return result; - }, - }; - return wrapped; -} diff --git a/apps/sidecar/src/workflow-run-record.test.ts b/apps/sidecar/src/workflow-run-record.test.ts deleted file mode 100644 index dca3d946d2..0000000000 --- a/apps/sidecar/src/workflow-run-record.test.ts +++ /dev/null @@ -1,471 +0,0 @@ -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { type } from "arktype"; -import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import type { CredentialDelivery } from "@intx/types/sidecar"; - -import { - WorkflowRunRecord, - writeWorkflowRunRecord, - deleteWorkflowRunRecord, - scanWorkflowRunRecords, -} from "./workflow-run-record"; - -// Real ciphers (not a noop) so the tests exercise actual seal/unseal. A second -// cipher under a different key drives the decrypt-failure path (a rotated -// sidecar key). -const CIPHER = createEnvKeyCredentialCipher(new Uint8Array(32).fill(1)); -const OTHER_KEY_CIPHER = createEnvKeyCredentialCipher(new Uint8Array(32).fill(2)); - -async function makeDataDir(): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), "wdr-")); -} - -function recordPath(dataDir: string, anchorRunId: string): string { - return path.join(dataDir, "workflow-runs", anchorRunId, "deployment.json"); -} - -async function fileExists(p: string): Promise { - try { - await fs.access(p); - return true; - } catch { - return false; - } -} - -/** Read the raw on-disk record (still sealed) and validate its schema shape. */ -async function readRawRecord(dataDir: string, anchorRunId: string): Promise { - const raw = await fs.readFile(recordPath(dataDir, anchorRunId), "utf8"); - const parsed = WorkflowRunRecord(JSON.parse(raw)); - if (parsed instanceof type.errors) { - throw new Error(`record failed validation: ${parsed.summary}`); - } - return parsed; -} - -/** Build a credential-material cell (no tool bindings) from id -> secret. */ -function deliveryOf(materials: Record): CredentialDelivery { - return { - bindings: [], - materials: Object.entries(materials).map(([credentialId, secret]) => ({ - credentialId, - providerKey: "anthropic", - origin: "https://api.example/anthropic", - secret, - })), - }; -} - -// Fixtures are the unified format: `sources`/`bodySources` are non-secret config -// carrying a `credentialId` per source, and every secret lives in `credentials` -// (the material cell), sealed to ciphertext on disk by `writeWorkflowRunRecord` -// and unsealed on read by `scanWorkflowRunRecords`. -const SINGLE_STEP: WorkflowRunRecord = { - version: 2, - agentAddress: "run_abc123@tenant.example", - definitionId: "wf_abc123", - sources: { - "step-1": [ - { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - credentialId: "cred-x", - model: "claude-mock", - }, - ], - }, - credentials: deliveryOf({ "cred-x": "sk-x" }), - sessionId: "ses_1", - hubPublicKey: "deadbeef", - // Source-ref is the only lineage: every record carries the pin + approved - // hash the restore re-verifies the re-materialized closure against. - approvedWireHash: "d".repeat(64), - lineage: "source-ref", - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { - schemaVersion: "1", - topLevel: [{ name: "@x/wf", version: "1.0.0" }], - entries: [], - }, - }, -}; - -// A multi-step deployment records no head hub key and may carry no session -// id -- both optional fields absent. -const MULTI_STEP: WorkflowRunRecord = { - version: 2, - agentAddress: "run_xyz@tenant.example", - definitionId: "wf_xyz", - sources: { - plan: [ - { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - credentialId: "cred-y", - model: "claude-mock", - }, - ], - execute: [ - { - id: "openai:mock", - provider: "openai", - baseURL: "https://api.example/openai", - credentialId: "cred-z", - model: "gpt-mock", - }, - ], - }, - credentials: deliveryOf({ "cred-y": "sk-y", "cred-z": "sk-z" }), - approvedWireHash: "e".repeat(64), - lineage: "source-ref", - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { - schemaVersion: "1", - topLevel: [{ name: "@x/wf-multi", version: "2.0.0" }], - entries: [], - }, - }, -}; - -// A source-ref deployment: the record schema's discriminated union REQUIRES -// a sourceRef pin (source + closure) + approvedWireHash for lineage -// "source-ref". -const SOURCE_REF: WorkflowRunRecord = { - version: 2, - agentAddress: "ins_dep_src@tenant.example", - definitionId: "wf_src", - sources: { - "step-1": [ - { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - credentialId: "cred-s", - model: "claude-mock", - }, - ], - }, - credentials: deliveryOf({ "cred-s": "sk-s" }), - approvedWireHash: "c".repeat(64), - lineage: "source-ref", - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { - schemaVersion: "1", - // The workflow-definition package is the single top-level pin the sidecar - // re-materializes; its transitive deps would ride `entries`. - topLevel: [{ name: "@x/wf", version: "1.0.0" }], - entries: [], - }, - }, -}; - -// A deployment that spawns bodies (an onTrigger section and a childWorkflow -// child): the top-level `sources` plus a `bodySources` table keyed by each -// body's definition id, and one `credentials` cell backing every source's -// `credentialId` -- top-level and per-body alike. -const WITH_BODIES: WorkflowRunRecord = { - version: 2, - agentAddress: "ins_dep_bodies@tenant.example", - definitionId: "wf_bodies", - sources: { - "step-1": [ - { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - credentialId: "cred-top", - model: "claude-mock", - }, - ], - }, - bodySources: { - "wf_bodies:onTrigger:0": { - "body-step": [ - { - id: "anthropic:mock", - provider: "anthropic", - baseURL: "https://api.example/anthropic", - credentialId: "cred-body-a", - model: "claude-mock", - }, - ], - }, - "wf_bodies:child:handler": { - plan: [ - { - id: "openai:mock", - provider: "openai", - baseURL: "https://api.example/openai", - credentialId: "cred-body-b", - model: "gpt-mock", - }, - ], - }, - }, - credentials: deliveryOf({ - "cred-top": "sk-top", - "cred-body-a": "sk-body-a", - "cred-body-b": "sk-body-b", - }), - approvedWireHash: "f".repeat(64), - lineage: "source-ref", - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { - schemaVersion: "1", - topLevel: [{ name: "@x/wf", version: "1.0.0" }], - entries: [], - }, - }, -}; - -describe("workflow run record store", () => { - test("seal/unseal round-trips a source-ref record through disk", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "src-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, SOURCE_REF, CIPHER); - - // The scan unseals the credential cell, so the restored record matches the - // original. - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - expect(scanned.map((s) => s.record)).toEqual([SOURCE_REF]); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("seals each credential secret at rest and unseals it on scan", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "abc123-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); - - // The record embeds credential secrets (sealed) plus the deployment's - // identity, so it must not be group/world readable on a shared host. - const stat = await fs.stat(recordPath(dataDir, anchorRunId)); - expect(stat.mode & 0o077).toBe(0); - - // On disk the record is version 2 and each material's secret is ciphertext - // -- NOT the plaintext the caller handed in. The source config carries only - // the non-secret `credentialId`. - const onDisk = await readRawRecord(dataDir, anchorRunId); - expect(onDisk.version).toBe(2); - expect(onDisk.sources["step-1"]?.[0]?.credentialId).toBe("cred-x"); - const sealedSecret = onDisk.credentials?.materials[0]?.secret; - expect(typeof sealedSecret).toBe("string"); - expect(sealedSecret).not.toBe("sk-x"); - - // The scan unseals it back to the plaintext for the restored run. - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - expect(scanned[0]?.record.credentials?.materials[0]?.secret).toBe("sk-x"); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("seals every credential in the cell, top-level and per-body alike", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "bodies-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, WITH_BODIES, CIPHER); - - // On disk every material secret is ciphertext, distinct from its plaintext, - // and the AAD is namespaced by credential id so two materials never seal to - // the same ciphertext even were their plaintext to match. - const onDisk = await readRawRecord(dataDir, anchorRunId); - const sealed = new Map( - (onDisk.credentials?.materials ?? []).map((m) => [m.credentialId, m.secret]), - ); - expect(sealed.get("cred-top")).not.toBe("sk-top"); - expect(sealed.get("cred-body-a")).not.toBe("sk-body-a"); - expect(sealed.get("cred-body-b")).not.toBe("sk-body-b"); - expect(new Set(sealed.values()).size).toBe(3); - - // The scan unseals the whole cell back to the plaintext the caller handed in. - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - expect(scanned).toEqual([{ runId: anchorRunId, record: WITH_BODIES }]); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("soft-skips a credential decrypt failure, dropping the whole run", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "bodies-corrupt-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, WITH_BODIES, CIPHER); - - // Corrupt ONE material's sealed secret in place under the correct key. The - // other materials are intact, so this isolates a single-credential decrypt - // failure: the whole run is dropped rather than half-restored. - const onDisk = await readRawRecord(dataDir, anchorRunId); - const material = onDisk.credentials?.materials[0]; - if (material === undefined) { - throw new Error("fixture is missing a material to corrupt"); - } - material.secret = "not-a-valid-ciphertext"; - await fs.writeFile(recordPath(dataDir, anchorRunId), JSON.stringify(onDisk, null, 2), "utf8"); - - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - expect(scanned).toEqual([]); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("round-trips a record with the optional fields absent (multi-step)", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "run_xyz-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP, CIPHER); - - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - const restored = scanned[0]?.record; - expect(restored).toEqual(MULTI_STEP); - expect(restored !== undefined && "hubPublicKey" in restored).toBe(false); - expect(restored !== undefined && "sessionId" in restored).toBe(false); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("round-trips a record that binds no credentials", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "no-creds-1"; - // A deployment whose sources need no secret carries no `credentials` cell; - // the write seals nothing and the scan unseals nothing. - const { credentials: _omit, ...noCreds } = SOURCE_REF; - void _omit; - await writeWorkflowRunRecord(dataDir, anchorRunId, noCreds, CIPHER); - - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - const restored = scanned[0]?.record; - expect(restored).toEqual(noCreds); - expect(restored !== undefined && "credentials" in restored).toBe(false); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("rejects a source-ref record missing sourceRef / a pin half / approvedWireHash", () => { - // The whole safety argument for deleting the restore loop's hand-rolled - // source-ref guard is that the union rejects EVERY malformed source-ref - // record at the scan boundary -- so check each required piece on its own, - // not just the all-missing shape (a future combinator swap could keep - // all-missing rejecting while silently admitting a partial record). The - // `SourceRefPin` co-requires its `source` + `closure`, so a half-populated - // pin must be rejected too, not just an absent one. - const rejects = (r: unknown): boolean => WorkflowRunRecord(r) instanceof type.errors; - - const base = { - version: 2, - agentAddress: "ins_dep_bad@tenant.example", - definitionId: "wf_bad", - sources: SOURCE_REF.sources, - lineage: "source-ref", - }; - const source = { kind: "registry", registry: "npm" }; - const closure = { schemaVersion: "1", topLevel: [], entries: [] }; - const sourceRef = { source, closure }; - const approvedWireHash = "c".repeat(64); - - // Both required pieces (sourceRef, approvedWireHash) missing. - expect(rejects(base)).toBe(true); - // The pin present but no hash, and the hash present but no pin -- each is - // individually insufficient for a valid source-ref record. - expect(rejects({ ...base, sourceRef })).toBe(true); - expect(rejects({ ...base, approvedWireHash })).toBe(true); - // A half-populated pin (only one of source/closure) is rejected by the - // pin's own co-requirement, even alongside a valid hash. - expect(rejects({ ...base, sourceRef: { source }, approvedWireHash })).toBe(true); - expect(rejects({ ...base, sourceRef: { closure }, approvedWireHash })).toBe(true); - // Full pin + hash -> accepted. - expect(rejects({ ...base, sourceRef, approvedWireHash })).toBe(false); - }); - - test("overwriting a record leaves the new one and no temp orphan", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "rotated-1"; - - // A source rotation overwrites the existing record in place. The - // atomic write must replace it cleanly, leaving only the record and - // no `.tmp` staging file behind. - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); - await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP, CIPHER); - - const dir = path.join(dataDir, "workflow-runs", anchorRunId); - expect(await fs.readdir(dir)).toEqual(["deployment.json"]); - - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - expect(scanned.map((s) => s.record)).toEqual([MULTI_STEP]); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("delete removes the record and is a no-op when absent", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "gone-1"; - - // No-op when the record was never written. - await deleteWorkflowRunRecord(dataDir, anchorRunId); - - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); - expect(await fileExists(recordPath(dataDir, anchorRunId))).toBe(true); - - await deleteWorkflowRunRecord(dataDir, anchorRunId); - expect(await fileExists(recordPath(dataDir, anchorRunId))).toBe(false); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); -}); - -describe("scanWorkflowRunRecords", () => { - test("returns an empty list when the workflow-runs directory is absent", async () => { - const dataDir = await makeDataDir(); - // First boot: nothing has been deployed, so `workflow-runs/` does not - // exist. That is the legitimate empty case, not an error. - expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("returns every schema-valid record keyed by its directory name", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowRunRecord(dataDir, "dep-a", SINGLE_STEP, CIPHER); - await writeWorkflowRunRecord(dataDir, "dep-b", MULTI_STEP, CIPHER); - - const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); - const byId = new Map(scanned.map((s) => [s.runId, s.record])); - expect(byId.size).toBe(2); - expect(byId.get("dep-a")).toEqual(SINGLE_STEP); - expect(byId.get("dep-b")).toEqual(MULTI_STEP); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("soft-skips a record sealed under a different key", async () => { - const dataDir = await makeDataDir(); - // A record whose credential cell was sealed under one key cannot be - // unsealed under a rotated/wrong key: the whole run is dropped as - // corruption, not half-restored. - await writeWorkflowRunRecord(dataDir, "wrong-key", SINGLE_STEP, CIPHER); - expect(await scanWorkflowRunRecords(dataDir, OTHER_KEY_CIPHER)).toEqual([]); - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("soft-skips a record whose JSON is corrupt", async () => { - const dataDir = await makeDataDir(); - const dir = path.join(dataDir, "workflow-runs", "dep-corrupt"); - await fs.mkdir(dir, { recursive: true }); - await fs.writeFile(path.join(dir, "deployment.json"), "{ not json", "utf8"); - expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("soft-skips a record that fails schema validation", async () => { - const dataDir = await makeDataDir(); - const dir = path.join(dataDir, "workflow-runs", "dep-invalid"); - await fs.mkdir(dir, { recursive: true }); - await fs.writeFile(path.join(dir, "deployment.json"), JSON.stringify({ version: 2 }), "utf8"); - expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); - await fs.rm(dataDir, { recursive: true, force: true }); - }); -}); diff --git a/apps/sidecar/src/workflow-run-record.ts b/apps/sidecar/src/workflow-run-record.ts deleted file mode 100644 index 55e363e44e..0000000000 --- a/apps/sidecar/src/workflow-run-record.ts +++ /dev/null @@ -1,219 +0,0 @@ -// Sidecar-local persistence to re-establish a workflow run across a sidecar -// process restart, co-located with the run's substrate so one teardown -// reclaims both. Carries the inputs that are otherwise frame/in-memory only -// (sources, sessionId, hubPublicKey, approvedWireHash, sourceRef); the -// definition and each step's grants are re-materialized elsewhere on restore. - -import { mkdir, readdir, readFile, rm } from "node:fs/promises"; -import { dirname, join as pathJoin } from "node:path"; - -import { type } from "arktype"; - -import { getLogger } from "@intx/log"; -import { credentialAad, type CredentialCipher } from "@intx/types"; -import { InferenceSource } from "@intx/types/runtime"; -import { CredentialDelivery, SourceRefPin } from "@intx/types/sidecar"; - -import { writeFileAtomicDurable } from "./atomic-write"; - -const logger = getLogger(["interchange", "sidecar", "workflow-run-record"]); - -const RECORD_FILENAME = "deployment.json"; - -/** True for a `node:fs` rejection whose `code` is `ENOENT`. */ -function isENOENT(cause: unknown): boolean { - return ( - cause instanceof Error && "code" in cause && (cause as { code: unknown }).code === "ENOENT" - ); -} - -// version 2 unifies credentials: sources/bodySources reference a -// credentialId, and secrets live once in the credentials cell, sealed under -// the sidecar cipher. A pre-unification record is soft-skipped at scan; the -// hub re-pushes its deployment on reconnect. -const workflowRunRecordBase = { - version: "1 | 2", - agentAddress: "string > 0", - definitionId: "string > 0", - // Non-secret: each source carries a credentialId into credentials. - sources: { - "[string]": InferenceSource.array().atLeastLength(1), - }, - // Keyed by spawned-body definition id; a nested loop body folds into its - // container's table instead of a separate entry. - "bodySources?": { - "[string]": { - "[string]": InferenceSource.array().atLeastLength(1), - }, - }, - // The one at-rest home for every secret; only each material's secret is - // sealed under the sidecar cipher (see transformDeliveryMaterials). - "credentials?": CredentialDelivery, - "sessionId?": "string > 0", - "hubPublicKey?": "string > 0", -} as const; - -/** - * Requires sourceRef and approvedWireHash so the restored child re-verifies - * against the hub-approved pin rather than a sidecar recompute — a record - * missing either (including a legacy live-authored one) soft-skips as - * corruption at scan, needing no bespoke source-ref guard. - */ -export const WorkflowRunRecord = type({ - ...workflowRunRecordBase, - lineage: "'source-ref'", - approvedWireHash: "string > 0", - // Re-run through applyFrozenWorkflowClosure on restore; plain strings, no secrets. - sourceRef: SourceRefPin, -}); -export type WorkflowRunRecord = typeof WorkflowRunRecord.infer; - -function recordPath(dataDir: string, runId: string): string { - return pathJoin(dataDir, "workflow-runs", runId, RECORD_FILENAME); -} - -// Binds a sealed secret to its id, so a ciphertext cannot be swapped between credentials or runs and still decrypt. -function credentialSecretColumn(credentialId: string): string { - return `credential:${credentialId}:secret`; -} - -// Returns a fresh delivery; a decrypt failure throws and the scan caller -// treats it as corruption, soft-skipping the whole run. -async function transformDeliveryMaterials( - delivery: CredentialDelivery, - runId: string, - transform: (secret: string, aad: string) => Promise, -): Promise { - return { - bindings: delivery.bindings, - materials: await Promise.all( - delivery.materials.map(async (material) => ({ - ...material, - secret: await transform( - material.secret, - credentialAad(runId, credentialSecretColumn(material.credentialId)), - ), - })), - ), - }; -} - -/** - * Written after the run's slug is claimed and before the child spawns, so a - * crash mid-spawn leaves a record the boot scan re-drives. Idempotent. - * Every credential secret is sealed under the sidecar cipher before disk. - */ -export async function writeWorkflowRunRecord( - dataDir: string, - runId: string, - record: WorkflowRunRecord, - cipher: CredentialCipher, -): Promise { - const path = recordPath(dataDir, runId); - await mkdir(dirname(path), { recursive: true }); - const sealed: WorkflowRunRecord = { - ...record, - version: 2, - ...(record.credentials !== undefined - ? { - credentials: await transformDeliveryMaterials(record.credentials, runId, (secret, aad) => - cipher.encrypt(secret, aad), - ), - } - : {}), - }; - // Atomic + durable since this is the sole restore source; owner-only since - // the record still names each source's provider/baseURL in the clear. - await writeFileAtomicDurable(path, JSON.stringify(sealed, null, 2), { - mode: 0o600, - }); -} - -/** - * Remove a run record. Called on undeploy and on a soft-failed deploy so a - * torn-down or never-completed run is not restored on the next boot. A - * missing record is not an error (`force`). - */ -export async function deleteWorkflowRunRecord(dataDir: string, runId: string): Promise { - await rm(recordPath(dataDir, runId), { force: true }); -} - -/** A restorable run: its directory-derived id plus the validated record. */ -export interface ScannedWorkflowRun { - /** The `workflow-runs/` directory name the record was found under. */ - runId: string; - record: WorkflowRunRecord; -} - -/** - * Soft-fails per record so one corrupt record doesn't strand every other - * run; an absent workflow-runs/ directory is the legitimate first-boot case. - */ -export async function scanWorkflowRunRecords( - dataDir: string, - cipher: CredentialCipher, -): Promise { - const runsDir = pathJoin(dataDir, "workflow-runs"); - let entries; - try { - entries = await readdir(runsDir, { withFileTypes: true }); - } catch (cause) { - if (isENOENT(cause)) return []; - throw cause; - } - - const scanned: ScannedWorkflowRun[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const runId = entry.name; - const path = recordPath(dataDir, runId); - - let raw: string; - try { - raw = await readFile(path, "utf8"); - } catch (cause) { - // A run directory with no record: a crash between mkdir and the record - // write, or a run whose record was already reclaimed. Nothing to - // restore from -- skip. - if (isENOENT(cause)) { - logger.warn`skipping workflow-runs/${runId}: no ${RECORD_FILENAME} to restore from`; - continue; - } - throw cause; - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`skipping workflow-runs/${runId}: ${RECORD_FILENAME} is not valid JSON: ${reason}`; - continue; - } - - const record = WorkflowRunRecord(parsed); - if (record instanceof type.errors) { - logger.warn`skipping workflow-runs/${runId}: ${RECORD_FILENAME} failed validation: ${record.summary}`; - continue; - } - // A decrypt failure is treated as corruption: soft-skip the whole run - // rather than wedge the boot scan. - let restored = record; - if (record.version === 2 && record.credentials !== undefined) { - try { - restored = { - ...record, - credentials: await transformDeliveryMaterials(record.credentials, runId, (secret, aad) => - cipher.decrypt(secret, aad), - ), - }; - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`skipping workflow-runs/${runId}: sealed credential decrypt failed: ${reason}`; - continue; - } - } - scanned.push({ runId, record: restored }); - } - return scanned; -} diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts deleted file mode 100644 index 2270798427..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ /dev/null @@ -1,1673 +0,0 @@ -// Single-writer architecture: the workflow-run repo's ref has exactly one -// writer, the supervisor. The child's bare RepoStore is read-only; its -// proxy RepoStore forwards writeTreePreservingPrefix over control IPC into -// the supervisor's substrate, which is wrapped with the boot-edge -// pack-push facade, so the child never opens its own pack-push pipeline. -// The factory does not read process.env itself; the binary owns that boundary. - -import fs from "node:fs"; -import path from "node:path"; - -import { type } from "arktype"; - -import { InferenceSource } from "@intx/types/runtime"; -import type { - ApprovalSnapshot, - AuditStore, - ContextStore, - InboundMessage, - InferenceEvent, - MessageTransport, - PendingOperation, -} from "@intx/types/runtime"; -import type { RuntimeCapabilities } from "@intx/types/runtime-capabilities"; -import { evaluateGrants } from "@intx/authz"; -import type { GrantRule } from "@intx/authz"; -import { AdapterManifest, createDependencies, type AdapterRegistry } from "@intx/inference"; -import { loadAdapterRegistry } from "@intx/inference/providers"; -import type { AnnotatedPluginFactory, DirectorRegistry } from "@intx/agent"; -import { createDefaultDirectorRegistry } from "@intx/agent"; -import { - builtinCredentialProviders, - createCredentialProviderRegistry, - createHarnessRuntimeCapabilities, - driveConnectorReplies, - type AgentEventStream, - type ConnectorReplyDrain, - type CredentialProviderRegistry, -} from "@intx/harness"; -import { - rawAuthorizationCredentialProvider, - xApiKeyCredentialProvider, -} from "@corbits/credential-header"; -import { createMcpStreamableHttpCredentialProvider } from "@corbits/credential-mcp"; -import { createSSHSignature } from "@intx/crypto"; -import { - createAgentRepoStore, - WORKFLOW_RUN_AGENT_STATE_PREFIX, - type Principal, - type RepoId, - type RepoStore, - type WorkflowRunWorkflowProcessPrincipal, -} from "@intx/hub-sessions/substrate"; -import { createIsogitStore } from "@intx/storage-isogit/node"; -import { - adaptHostScheduler, - createChildMailboxReader, - createCredentialsBackedAuthorize, - createMailboxWatchRegistry, - createProxyWorkflowRunRepoStore, - createSupervisorBackedTransport, - createWorkflowHostScheduler, - createWorkflowRunBlobSubstrate, - createWorkflowRunRepoStore, - createWorkflowHostSignalChannel, - createInMemorySpawnChild, - createInMemorySpawnSuspendableChild, - createWorkflowStepInvoker, - hashGrants, - loadWorkflowLoopFnsFromClosure, - loadWorkflowDirectorRegistryFromClosure, - loadWorkflowPluginFactoriesFromClosure, - loadWorkflowPluginToolDefinitionsFromClosure, - type ChildMailboxReader, - type ChildOutboundMailBridge, - type CredentialsSnapshot, - type CredentialsSnapshotRef, - type GrantEvaluator, - type LoadParkedApproval, - type RunChildWorkflow, - type RunSuspendableChild, - type RunWorkflowChildBindings, - type SourcesSnapshotRef, - type StepEnvBase, - type SubstrateFactory, - type SubstrateFactoryEnv, - type SupervisorBackedTransportInbound, -} from "@intx/workflow-host"; -import { - baseStepId, - collectDeclaredPluginNames, - createLoopIterationHandle, - createNoopDrainController, - createSuspendableChildHandle, - eagerlyResolveLoopFns, - emptyState, - enumerateInlineLoopBodies, - rewriteInlineChildWorkflowBodies, - runtimeRun, - walkWorkflowSteps, - LOOP_BODY_DESCENT, - type LoopFnRegistry, - type ParkedApprovalOp, - type ReadParkedApprovalOps, - type Scheduler, - type StepInvokeRequest, - type StepInvokeResult, - type WorkflowAuthorizeFn, - type WorkflowDefinition, - type WorkflowRuntimeEnv, -} from "@intx/workflow"; -import { type PluginToolDefinitions } from "@intx/workflow-deploy"; - -import { - attachStepCredentialWiring, - attachStepTools, - createToolBearingAgentFactory, - deriveToolMarkFloorGrants, - materializeStepTools, - type StepToolCacheConfig, - type StepToolMaterialization, -} from "./step-agent-tools"; -import { - createInferenceCredentialResolver, - type CredentialMaterialCell, -} from "./step-credential-capabilities"; -import { readRunGrants, runGrantsPath } from "./run-grants"; -import { collectDeclaredResources, filterGrantsToDeclaredResources } from "./child-grant-filter"; -import { - createDurableConversationRegistry, - isErrnoNotFound, - reconstructDurableConversation, - type DurableConversationRegistry, -} from "./conversation-state"; - -/** Enforced presence-and-non-empty before the factory runs; HUB_WS_URL/SIDECAR_ID/SIDECAR_TOKEN are the hub trust anchors. */ -export const SIDECAR_SUBSTRATE_CONFIG_KEYS = [ - "SIDECAR_DATA_DIR", - "WORKFLOW_RUN_REPO_ID", - "WORKFLOW_RUN_REF", - "SIDECAR_SIGNING_PUBLIC_KEY", - "SIDECAR_SIGNING_PRIVATE_KEY", - "HUB_WS_URL", - "SIDECAR_ID", - "SIDECAR_TOKEN", - "STEP_INFERENCE_SOURCES", - "WORKFLOW_BODY_SOURCES", - "SIDECAR_CACHE_MAX_BYTES", - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - "SIDECAR_ADAPTER_MANIFEST", -] as const; - -const SubstrateConfig = type({ - SIDECAR_DATA_DIR: "string > 0", - WORKFLOW_RUN_REPO_ID: "string > 0", - WORKFLOW_RUN_REF: "string > 0", - SIDECAR_SIGNING_PUBLIC_KEY: "string > 0", - SIDECAR_SIGNING_PRIVATE_KEY: "string > 0", - HUB_WS_URL: "string > 0", - SIDECAR_ID: "string > 0", - SIDECAR_TOKEN: "string > 0", - STEP_INFERENCE_SOURCES: "string > 0", - // Always serialized by the deploy router (at least "{}"), so a missing key - // child-side is a serialization bug and must fail loud here. - WORKFLOW_BODY_SOURCES: "string > 0", - SIDECAR_CACHE_MAX_BYTES: "string > 0", - SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", - // Required like the byte-cap fields; re-validated against AdapterManifest - // in parseAdapterManifest before any module is imported. - SIDECAR_ADAPTER_MANIFEST: "string > 0", -}).onUndeclaredKey("ignore"); - -/** Re-parsed at the child boundary rather than trusting the wire, even though the boot edge already validated it. */ -function parseByteCap(raw: string, name: string): number { - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) { - throw new Error( - `sidecar workflow-child substrate config: ${name} must be a positive finite number; got ${JSON.stringify(raw)}`, - ); - } - return n; -} - -/** Each value is the step's ordered failover chain: element 0 active, the tail forward-only targets. */ -const StepInferenceSourceTable = type({ - "[string]": InferenceSource.array().atLeastLength(1), -}); -type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; - -/** Rejected at the boundary with a structured error rather than deferred to a deep-stack buildEnv failure. */ -function parseStepInferenceSources(raw: string): StepInferenceSourceTable { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES is not valid JSON: ${reason}`, - ); - } - const validated = StepInferenceSourceTable(parsed); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** Empty when the deployment spawns no bodies. */ -const BodyInferenceSources = type({ - "[string]": StepInferenceSourceTable, -}); -type BodyInferenceSources = typeof BodyInferenceSources.infer; - -/** Mirrors parseStepInferenceSources: rejected at the boundary rather than deep in a body spawn. */ -function parseBodyInferenceSources(raw: string): BodyInferenceSources { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES is not valid JSON: ${reason}`, - ); - } - const validated = BodyInferenceSources(parsed); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** - * Defense-in-depth re-validation at the deserialization boundary, not a - * trust upgrade. A manifest specifier must resolve from both the sidecar's - * and the child's module-resolution roots, and an adapter module must be - * import-side-effect-free since it's imported once per process. - */ -export function parseAdapterManifest(raw: string): AdapterManifest { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new Error( - "sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST is not valid JSON", - { cause }, - ); - } - const validated = AdapterManifest(parsed); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** A lookup miss here is a supervisor programmer error, not a wire-side failure. A scoped map-iteration id resolves to its base first. */ -function createStepInferenceSourceResolver( - table: StepInferenceSourceTable, -): (stepId: string) => InferenceSource[] { - return (stepId: string): InferenceSource[] => { - const base = baseStepId(stepId); - const sources = table[base]; - if (sources === undefined) { - const scopedNote = - base === stepId ? "" : ` (normalized from scoped invocation id ${JSON.stringify(stepId)})`; - throw new Error( - `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(base)}${scopedNote}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, - ); - } - return sources; - }; -} - -function hexDecode(hex: string, name: string): Uint8Array { - if (hex.length % 2 !== 0) { - throw new Error(`${name} must be even-length hex; got ${String(hex.length)} chars`); - } - const out = new Uint8Array(hex.length / 2); - for (let i = 0; i < out.length; i += 1) { - const byte = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); - if (Number.isNaN(byte)) { - throw new Error(`${name} contains non-hex characters`); - } - out[i] = byte; - } - return out; -} - -/** Production omits these for the default disk-backed store; tests inject an in-memory stub. */ -interface SidecarSubstrateFactoryDeps { - /** Backs the child's read-only operations only; writes go through the proxy RepoStore over IPC instead. */ - createBareRepoStore?: (config: { - dataDir: string; - signingKey: { publicKey: Uint8Array; privateKey: Uint8Array }; - }) => RepoStore; -} - -/** Binds the same Ed25519 keypair the bare RepoStore carries, matching the production signing surface. */ -function createStepStorageSigner(signingKey: { - publicKey: Uint8Array; - privateKey: Uint8Array; -}): (payload: string) => Promise { - return (payload: string) => - Promise.resolve(createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey)); -} - -/** - * Rooted outside the workflow-run working tree to avoid the run-event log. - * A crash-resume must reopen the same attempt-N it suspended under - * (recovered via currentAttempt), or the reactor hangs on an empty gate. - */ -export function stepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "runs", - args.runId, - "steps", - args.stepId, - `attempt-${String(args.attempt)}`, - ); -} - -/** Reclaiming this subtree on run completion drops every step/attempt the run produced in one rm -rf. */ -function runStepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "runs", - args.runId, - ); -} - -/** - * Keyed by step identity, not runId, so the cached warm agent reuses one - * workspace across every message and it survives a child respawn. - * Reclaimed on undeploy under a warm/ sibling of the cold runs/ subtree. - */ -function warmStepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - stepId: string; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "warm", - encodeURIComponent(args.stepId), - ); -} - -async function directoryExists(dir: string): Promise { - try { - return (await fs.promises.stat(dir)).isDirectory(); - } catch (cause) { - if (isErrnoNotFound(cause)) return false; - throw cause; - } -} - -function findApprovalSnapshot( - pendingOperations: readonly PendingOperation[], - correlationId: string, -): ApprovalSnapshot | undefined { - return pendingOperations.find((op) => op.correlationId === correlationId)?.approvalSnapshot; -} - -/** Returns undefined when the parent is not in the mailbox (fresh thread, or an id that never arrived). */ -async function resolveMailboxReferences( - reader: ChildMailboxReader, - inReplyTo: string, -): Promise { - const store = await reader.open(); - const parent = store.messages.find((m) => m.envelope.messageId === inReplyTo); - if (parent === undefined) return undefined; - return [...parent.envelope.references, parent.envelope.messageId]; -} - -/** directoryExists guard keeps this a pure read: createIsogitStore would otherwise mkdir+init a fresh repo. */ -export async function readColdParkedPendingOperations(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; -}): Promise { - const storeDir = stepStorageRoot({ - dataDir: args.dataDir, - workflowRunRepoId: args.workflowRunRepoId, - runId: args.runId, - stepId: args.stepId, - attempt: args.attempt, - }); - if (!(await directoryExists(storeDir))) return []; - const store = await createIsogitStore(storeDir); - const { pendingOperations } = await store.load(); - return pendingOperations; -} - -export async function readColdParkedApprovalSnapshot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; - correlationId: string; -}): Promise { - return findApprovalSnapshot(await readColdParkedPendingOperations(args), args.correlationId); -} - -/** - * Deliberately not through DurableConversationRegistry.acquire, whose first - * acquire would write a substrate restore and front-run the warm agent's own - * restore ordering. - */ -export async function readWarmParkedPendingOperations(args: { - substrate: RepoStore; - workflowRunRepoId: RepoId; - stepId: string; -}): Promise { - const agentStateDir = path.join( - args.substrate.getRepoDir(args.workflowRunRepoId), - WORKFLOW_RUN_AGENT_STATE_PREFIX, - encodeURIComponent(args.stepId), - ); - const reconstructed = await reconstructDurableConversation(agentStateDir, args.stepId); - if (reconstructed === null) return []; - return reconstructed.pendingOperations; -} - -export async function readWarmParkedApprovalSnapshot(args: { - substrate: RepoStore; - workflowRunRepoId: RepoId; - stepId: string; - correlationId: string; -}): Promise { - return findApprovalSnapshot(await readWarmParkedPendingOperations(args), args.correlationId); -} - -/** The runtime reconstructs the lost SignalAwaited from correlationId + deadline alone; it must not see the reactor's internals. */ -export function toParkedApprovalOps(pendingOperations: PendingOperation[]): ParkedApprovalOp[] { - return pendingOperations - .filter((op) => op.kind === "approval") - .map((op) => ({ - correlationId: op.correlationId, - ...(op.timeoutAt !== undefined ? { timeoutAtMs: op.timeoutAt } : {}), - })); -} - -export interface SidecarStepBuildEnvDeps { - dataDir: string; - workflowRunRepoId: RepoId; - signer: (payload: string) => Promise; - /** Locates each step's deploy tree and doubles as the step agent's outbound mail address. */ - mailboxAddress: string; - /** Selects the head/step collapse: single-step reads at the head, multi-step at the per-step address. */ - stepCount: number; - /** Wrapped in a supervisor-backed transport; the step agent never holds the signing key. */ - outboundMailBridge: ChildOutboundMailBridge; - /** Absent for a build with no inbound mailbox (a spawned child or onTrigger body), whose inbound stays inert. */ - inbound?: SupervisorBackedTransportInbound; - /** Per-step tool-loader caps (cache + registry tarball size). */ - cache: StepToolCacheConfig; - /** Built eagerly at boot so a custom-provider step source resolves the same way as the sidecar main path. */ - adapters: AdapterRegistry; - /** Present only for a warm single-step agent; a multi-step deploy's per-step agents need no cross-run durability. */ - durableConversation?: DurableConversationRegistry; - /** - * Recorded by base step id so the grant evaluator can authorize a pinned - * tool (invisible to the hub's capability walk) against its own static - * mark; persists across a warm agent's messages since the env builder - * runs once. - */ - recordToolMarkFloor: (baseStepId: string, grants: GrantRule[]) => void; - /** - * Set for the source-ref lineage: feeds the step agent's own live - * toolFactories in directly, since that lineage stages no - * tool-packages-manifest.json. No floor is recorded here since the - * probe's capability walk already granted the bare tool name. - */ - sourceTools: boolean; - /** Present only on the source-ref lineage, to materialize its declared plugin packages with no re-download. */ - closurePackageDir?: string; -} - -/** Fails closed when plugins are declared but closurePackageDir is absent — that pairing is a wiring bug. */ -async function materializeSourcePluginFactories( - deps: SidecarStepBuildEnvDeps, - req: StepInvokeRequest, -): Promise { - const plugins = req.agent.plugins ?? []; - if (plugins.length === 0) { - return []; - } - if (deps.closurePackageDir === undefined) { - throw new Error( - `sidecar workflow-child step invoker: step agent ${JSON.stringify(req.agent.id)} declares plugins ${JSON.stringify(plugins)} but the source-ref closure package dir is absent; a source workflow's plugin factories can only be materialized from its frozen closure`, - ); - } - return loadWorkflowPluginFactoriesFromClosure({ - packageDir: deps.closurePackageDir, - plugins, - }); -} - -/** Pulled out of createSidecarSubstrateFactory so per-step env construction is observable in isolation. */ -/** Absent when no credential material was threaded, leaving the step's inference reader and tool credentials unwired. */ -interface SidecarStepCredentialContext { - readonly materialCell: CredentialMaterialCell; - /** Typed unknown[] at this boundary; the sidecar casts to GrantRule[] where the grammar is known. */ - readonly resolveStepGrants: (stepId: string) => readonly unknown[]; - readonly providers: CredentialProviderRegistry; -} - -export function createSidecarStepBuildEnv( - deps: SidecarStepBuildEnvDeps, -): ( - req: StepInvokeRequest, - sourcesRef: SourcesSnapshotRef, - credentialContext?: SidecarStepCredentialContext, -) => Promise { - // Local delta: the deploy probe already honours a workflow package's own - // `interchange.directors`, so the run must resolve the same registry or a - // definition naming its director dies at its first step. - let closureDirectors: Promise | undefined; - function directors(): Promise { - closureDirectors ??= - deps.closurePackageDir === undefined - ? Promise.resolve(createDefaultDirectorRegistry()) - : loadWorkflowDirectorRegistryFromClosure({ packageDir: deps.closurePackageDir }); - return closureDirectors; - } - return async ( - req: StepInvokeRequest, - sourcesRef: SourcesSnapshotRef, - credentialContext?: SidecarStepCredentialContext, - ): Promise => { - // Resolved live each build; a warm agent already built doesn't pass - // through here again, so this ref covers only builds not yet done. - const resolveStepInferenceSource = createStepInferenceSourceResolver(sourcesRef.current); - const { stepId, runId, attempt } = req.authzContext; - if (stepId === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.stepId is required for per-step InferenceSource resolution; the workflow runtime must populate stepId on every step-originated invocation", - ); - } - if (runId === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.runId is required to root per-step storage under the run; the workflow runtime must populate runId on every step-originated invocation", - ); - } - if (attempt === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", - ); - } - const sources = resolveStepInferenceSource(stepId); - // The resolver's arktype guarantees a non-empty chain; assert it here so - // the reactor's initial-source pin (element 0) is a checked fact rather - // than an unchecked index. - const activeSource = sources[0]; - if (activeSource === undefined) { - throw new Error( - `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, - ); - } - - // Cold path keys per run/step/attempt; warm path (durableConversation - // present) keys stably per agent. Disjoint runs/ and warm/ sub-roots so - // neither sweep touches the other's tree. - const storeDir = - deps.durableConversation !== undefined - ? warmStepStorageRoot({ - dataDir: deps.dataDir, - workflowRunRepoId: deps.workflowRunRepoId, - stepId, - }) - : stepStorageRoot({ - dataDir: deps.dataDir, - workflowRunRepoId: deps.workflowRunRepoId, - runId, - stepId, - attempt, - }); - // The warm agent's conversation must survive child respawn, so it's - // backed by a per-agent durable store mirrored to the substrate; a - // multi-step deploy keeps the per-run isogit store instead. - const storage: ContextStore & AuditStore = - deps.durableConversation !== undefined - ? (await deps.durableConversation.acquire(stepId)).storage - : await createIsogitStore(storeDir, deps.signer); - - // Cold-path-only guard for the resume-attempt invariant on - // stepStorageRoot: an approval resume must find a suspendedCall-bearing - // pending op for its correlationId, or the runtime reopened the wrong - // attempt's store and the reactor would hang (gateless) or silently - // skip the approved call (matched only an async marker). Loud here at - // the one seam that opened the store and knows the invariant. - if ( - deps.durableConversation === undefined && - req.resume !== undefined && - req.resume.kind === "approval" - ) { - const resumeCorrelationId = req.resume.correlationId; - const loaded = await storage.load(); - const hasPendingGate = loaded.pendingOperations.some( - (op) => op.correlationId === resumeCorrelationId && op.suspendedCall !== undefined, - ); - if (!hasPendingGate) { - throw new Error( - `sidecar workflow-child step invoker buildEnv: resume of step ${JSON.stringify(stepId)} (run ${JSON.stringify(runId)}, attempt ${String(attempt)}) reopened a ContextStore with no re-dispatchable approval gate for correlationId ${JSON.stringify(resumeCorrelationId)}. The cold-path store is keyed by attempt (${storeDir}); a resume that finds no suspendedCall-bearing pending operation here means it reopened the wrong attempt's store (the reactor would come up gateless and the decision would correlate against nothing) or matched only an async pending marker (the reactor would clear the gate without re-running the approved call). This is a keying violation, not a recoverable state.`, - ); - } - } - - const workdir = path.join(storeDir, "workspace"); - await fs.promises.mkdir(workdir, { recursive: true }); - - // sourceTools: run the step agent's own evaluated toolFactories directly - // (the source deploy stages no manifest, so materializeStepTools would - // find nothing); plugin factories come separately from the frozen - // closure into the same pluginFactories slot. Otherwise: - // materializeStepTools loads the pinned tool-package closure, rooted - // per step so concurrent steps never collide on cache/apply-state. - const materialization: StepToolMaterialization = - deps.sourceTools === true - ? { - factories: req.agent.toolFactories.map((factory) => ({ - packageName: factory.id, - declaredCredentials: [], - factory, - })), - pluginFactories: await materializeSourcePluginFactories(deps, req), - } - : await materializeStepTools({ - dataDir: deps.dataDir, - mailboxAddress: deps.mailboxAddress, - stepId, - stepCount: deps.stepCount, - storeDir, - cache: deps.cache, - }); - - // A pinned tool never reached the hub's capability walk, so its floor is - // recorded here for the grant evaluator; skipped on the source-ref - // lineage since the walk already granted the bare tool name directly. - if (deps.sourceTools !== true) { - deps.recordToolMarkFloor( - baseStepId(stepId), - deriveToolMarkFloorGrants(materialization.factories.map((f) => f.factory)), - ); - } - - // Outbound routes over control IPC to the supervisor for the actual - // signed send; inbound (deps.inbound, warm agent only) resolves locally - // against a fresh committed INBOX snapshot. - const transport = createSupervisorBackedTransport( - deps.outboundMailBridge, - deps.mailboxAddress, - deps.inbound, - ); - - const capabilities = createHarnessRuntimeCapabilities({ transport }); - - // Widens StepEnvBase structurally (a wider object is assignable to the - // narrower return type); address is observability-only. - const env: StepEnvBase & { - toolCwd: string; - transport: MessageTransport; - address: string; - capabilities: RuntimeCapabilities; - } = { - // Pins the initial source to element 0 so the reactor fails over - // forward through the full chain. - sources, - defaultSource: activeSource.id, - storage, - workdir, - toolCwd: workdir, - audit: storage, - directors: await directors(), - deps: createDependencies(deps.adapters), - transport, - address: deps.mailboxAddress, - capabilities, - }; - attachStepTools(env, materialization); - // Grants are a thunk so a self-discovery resume preceding the grants - // barrier doesn't fault on a missing snapshot. - if (credentialContext !== undefined) { - // Same live cell tool credentials resolve from, so the child never - // holds the cipher key. - env.readCurrentMaterial = createInferenceCredentialResolver(credentialContext.materialCell); - attachStepCredentialWiring(env, { - materialCell: credentialContext.materialCell, - resolveGrants: () => - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- resolveStepGrants returns unknown[] at the run-child boundary; the sidecar owns the GrantRule grammar - credentialContext.resolveStepGrants(stepId) as readonly GrantRule[], - providers: credentialContext.providers, - }); - } - return env; - }; -} - -/** Widens StepInvoker with the child's authorize, its own per-spawn sourcesRef, and the parent's onEvent funnel. */ -export type SidecarChildStepInvoker = ( - req: StepInvokeRequest, - authorize: WorkflowAuthorizeFn, - sourcesRef: SourcesSnapshotRef, - onEvent: (event: InferenceEvent) => void, - credentialContext?: SidecarStepCredentialContext, -) => Promise; - -/** - * Lifted out of createSidecarSubstrateFactory so it's exercisable in - * isolation. The child runs with runId: childRunId, so its events land at - * runs//... sibling to the parent's own subtree, reusing the - * parent's pack-pushing substrate and signing principal verbatim. - */ -interface SidecarRunChildDeps { - /** Wrapped workflow-run substrate (the factory's `substrate`). */ - substrate: RepoStore; - /** Workflow-run repo identifying the parent's deployment. */ - workflowRunRepoId: RepoId; - /** Workflow-run ref the child reads/writes against. */ - workflowRunRef: string; - /** Principal the child presents on every substrate operation. */ - principal: Principal; - /** Host-process scheduler singleton; shared with the parent. */ - scheduler: Scheduler; - /** - * The child's stepIds are disjoint from the parent's, so routing through - * the parent's STEP_INFERENCE_SOURCES-pinned buildStepEnv would throw a - * misleading error; callers supply a separate invokeStep instead. - */ - invokeStep: SidecarChildStepInvoker; - /** Keyed by definition id so the child never holds the sidecar cipher key; a missing id falls back to dataDir. */ - bodySources: BodyInferenceSources; - /** Backs the legacy on-disk sources.json fallback and per-step storage roots. Required. */ - dataDir?: string; - /** Supplied by the parent factory so the child authorizes against the same evaluator the parent's steps use. */ - evaluateGrants: GrantEvaluator; - /** Combined with the run's live material cell and capped grants to assemble each bundle's credentials capability. */ - credentialProviders: CredentialProviderRegistry; - /** Director registry the child runtime uses; defaults to the canonical built-ins. */ - directors?: DirectorRegistry; - /** Present only on the source-ref lineage, to declare plugin-contributed tools when capping inherited grants. */ - closurePackageDir?: string; - /** Clock for timestamp generation; defaults to `() => new Date()`. */ - clock?: () => Date; - /** - * Random id generator for run ids, signal ids, timer ids; defaults to - * a monotonic counter combined with a random suffix. - */ - newId?: (prefix: string) => string; -} - -/** - * Safe only because the sole caller is write-once, at the run's birth - * before any event append — the shallow runs// prefix rebuild - * would otherwise delete a populated run's events/blobs subtrees. - */ -async function writeChildRunGrants(args: { - substrate: RepoStore; - workflowRunRepoId: RepoId; - principal: Principal; - ref: string; - childRunId: string; - grants: readonly unknown[]; -}): Promise { - const prefix = `runs/${args.childRunId}/`; - const grantsFile = runGrantsPath(args.childRunId); - const serialized = JSON.stringify({ grants: args.grants }, null, 2); - await args.substrate.writeTreePreservingPrefix(args.principal, args.workflowRunRepoId, args.ref, { - preservePrefix: prefix, - merge: async (existing) => { - const files: Record = {}; - for (const [k, v] of existing) files[k] = v; - files[grantsFile] = serialized; - return files; - }, - message: `Write inherited run grants for ${args.childRunId}`, - }); -} - -/** - * No durable CancelRequested: an in-process child aborts through the - * runtime's local-abort seam since it cannot sign the supervisor cancel a - * control-plane cancel would require. The signal channel handle is - * stop()ped in a finally block so no subscribeKind loop outlives the call. - */ -export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkflow { - const directors = deps.directors ?? createDefaultDirectorRegistry(); - const clock = deps.clock ?? defaultClock; - const newId = deps.newId ?? defaultNewId; - // No controlPlanePrincipal: teardown goes through the local-abort seam, not - // a durable CancelRequested. Shared across every child this factory spawns. - const repoStore = createWorkflowRunRepoStore({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - ref: deps.workflowRunRef, - }); - // Self-referential so a child env's recursive spawnChild routes - // grandchild spawns back through this same adapter; recursion bottoms - // out when a rung's definition has no childWorkflow primitive. - const runChild: RunChildWorkflow = async ( - { definition, childRunId, input, parentRunId, signal, depth, maxChildSpawnDepth }, - onEvent, - credentialMaterial, - ) => { - const { - env, - signalChannel, - definition: rewrittenDefinition, - } = await buildChildRunEnv({ - deps, - directors, - clock, - newId, - repoStore, - runChild, - definition, - childRunId, - parentRunId, - onEvent, - // Terminal: the caller awaits this child's terminal rather than driving - // it across parks, and no signal can be addressed to a child run, so a - // park inside it has nothing that could answer it. - hasUpstreamSignalResolver: false, - ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), - }); - try { - // Threads this rung's depth/ceiling so nested spawns keep counting - // against the tree-wide bound rather than resetting to depth 0 per rung. - // A parent abort tears down through the local-abort seam, not a - // control-plane cancel the proxy substrate cannot sign. - // `failed` under its own principal -- no durable cancel, no wedge while the - // parent awaits the terminal below. - const handle = runtimeRun(rewrittenDefinition, env, { - runId: childRunId, - triggerPayload: input, - depth, - maxChildSpawnDepth, - localAbort: signal, - }); - const result = await handle.complete; - return { terminalStatus: result.terminalStatus }; - } finally { - await signalChannel.stop(); - } - }; - return runChild; -} - -/** - * Park-aware analog of {@link createSidecarRunChild}: returns a live handle. - * A body parking on an unserviceable "input" channel is a hard error on - * next() rather than hanging; the signal channel stays alive across every - * park, tied to the run's terminal rather than per-next(). - */ -export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): RunSuspendableChild { - const directors = deps.directors ?? createDefaultDirectorRegistry(); - const clock = deps.clock ?? defaultClock; - const newId = deps.newId ?? defaultNewId; - // No controlPlanePrincipal: teardown goes through the shared handle's - // local-abort seam, not a durable CancelRequested. - const repoStore = createWorkflowRunRepoStore({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - ref: deps.workflowRunRef, - }); - // Body grandchildren spawn terminal-only: buildChildRunEnv wires spawnChild, - // not spawnSuspendableChild, so a nested onTrigger fails loud. - const runChild = createSidecarRunChild(deps); - - return async ( - { - definition, - childRunId, - input, - parentRunId, - signal, - depth, - maxChildSpawnDepth, - resumeFromEvents, - }, - onEvent, - credentialMaterial, - ) => { - const { - env: baseEnv, - signalChannel, - definition: rewrittenDefinition, - } = await buildChildRunEnv({ - deps, - directors, - clock, - newId, - repoStore, - runChild, - definition, - childRunId, - parentRunId, - onEvent, - // The run's live credential-material cell so the body's inference resolves - // its source secret against the parent's current delivery; a grandchild - // spawned from the body inherits it through the recursive spawnChild. - // - // Park-aware: the container drives this body across its parks and relays - // a decision back down onto the body's own channel, so a park here is - // answerable even though the body run carries no address of its own. - hasUpstreamSignalResolver: true, - ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), - }); - - return createSuspendableChildHandle(baseEnv, { - definition: rewrittenDefinition, - childRunId, - input, - depth, - maxChildSpawnDepth, - ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), - signal, - cleanup: () => signalChannel.stop(), - }); - }; -} - -/** - * definition must be the pre-rewrite body (grandchildren still inline), or - * collectDeclaredResources would under-authorize a rewritten { ref } body. - * Write-once per run: a resume re-drive reads back the existing grants - * file rather than rewriting it, since a re-write would race the runtime's - * replay and corrupt the shared repo's event seq. - */ -async function capAndPersistChildGrants(args: { - deps: SidecarRunChildDeps; - directors: ReturnType; - definition: WorkflowDefinition; - childRunId: string; - parentRunId: string; -}): Promise { - const { deps, directors, definition, childRunId, parentRunId } = args; - const existingChildGrants = await readRunGrants({ - repoStore: deps.substrate, - anchorRunId: deps.workflowRunRepoId.id, - runId: childRunId, - }); - if (existingChildGrants !== undefined) return existingChildGrants; - const parentGrants = await readRunGrants({ - repoStore: deps.substrate, - anchorRunId: deps.workflowRunRepoId.id, - runId: parentRunId, - }); - if (parentGrants === undefined) { - throw new Error( - `sidecar runChild: parent run ${parentRunId} has no grants file at ${runGrantsPath(parentRunId)}; refusing to spawn child ${childRunId} under-authorized`, - ); - } - const pluginDefs: PluginToolDefinitions = - deps.closurePackageDir === undefined - ? new Map() - : await loadWorkflowPluginToolDefinitionsFromClosure({ - packageDir: deps.closurePackageDir, - plugins: collectDeclaredPluginNames(definition), - }); - const declaredResources = collectDeclaredResources(definition, directors, pluginDefs); - const childGrants = filterGrantsToDeclaredResources(parentGrants, declaredResources); - await writeChildRunGrants({ - substrate: deps.substrate, - workflowRunRepoId: deps.workflowRunRepoId, - principal: deps.principal, - ref: deps.workflowRunRef, - childRunId, - grants: childGrants, - }); - return childGrants; -} - -/** Shared by both child-drive callers so env construction lives in one place; signal channel returned for caller stop(). */ -async function buildChildRunEnv(args: { - deps: SidecarRunChildDeps; - directors: ReturnType; - clock: () => Date; - newId: (prefix: string) => string; - repoStore: ReturnType; - runChild: RunChildWorkflow; - definition: WorkflowDefinition; - childRunId: string; - parentRunId: string; - /** Required for both paths (childWorkflow and onTrigger body); a missing sink is a wiring defect, not a silent drop. */ - onEvent: (event: InferenceEvent) => void; - /** Absent when a non-sidecar executor carries no credential material, leaving the child's inference reader unset. */ - materialCell?: CredentialMaterialCell; - /** - * Whether a park in this child can be answered from outside it. The two - * seams that share this builder differ precisely here, so neither gets a - * default: a suspendable body is driven across its parks by a container - * that relays a decision back down, while a terminal child has no address - * and no relay above it. - */ - hasUpstreamSignalResolver: boolean; -}): Promise<{ - env: WorkflowRuntimeEnv; - signalChannel: ReturnType; - definition: WorkflowDefinition; -}> { - const { - deps, - directors, - clock, - newId, - repoStore, - runChild, - definition, - childRunId, - parentRunId, - onEvent, - materialCell, - } = args; - // Lifts each inline grandchild childWorkflow to a { ref } and keeps the - // lifted definitions in-memory, so a grandchild spawns with no on-disk read. - const { workflow: rewrittenDefinition, bodies: grandchildBodies } = - rewriteInlineChildWorkflowBodies(definition); - const grandchildMap = new Map(grandchildBodies.map((b) => [b.ref, b.definition])); - // Mints a ref-keyed copy for the loop-iteration host and keeps the - // pre-rewrite form for the per-iteration grant cap. Mirrors the - // top-level loop-body registration in run-child.ts. - const loopBodies = enumerateInlineLoopBodies(definition); - const loopBodiesMap = new Map(); - const loopBodyPreRewrite = new Map(); - for (const loopBody of loopBodies) { - loopBodyPreRewrite.set(loopBody.ref, loopBody.definition); - const bodyRewrite = rewriteInlineChildWorkflowBodies(loopBody.definition); - loopBodiesMap.set(loopBody.ref, bodyRewrite.workflow); - for (const grandchild of bodyRewrite.bodies) { - grandchildMap.set(grandchild.ref, grandchild.definition); - } - } - // A loop declared with no closure wired is a wiring defect; fail loud - // rather than deferring to a mid-run resolve failure. - let loopFns: LoopFnRegistry | undefined; - if (loopBodies.length > 0) { - if (deps.closurePackageDir === undefined) { - throw new Error( - "sidecar child: a loop is nested in this spawned body but deps.closurePackageDir is missing; the loop while/carry fns cannot be resolved", - ); - } - loopFns = await loadWorkflowLoopFnsFromClosure({ - packageDir: deps.closurePackageDir, - }); - eagerlyResolveLoopFns( - [rewrittenDefinition, ...loopBodiesMap.values(), ...grandchildMap.values()], - loopFns, - ); - } - // Persisted as runs//grants.json, the file a grandchild reads as its ceiling. - const childGrants = await capAndPersistChildGrants({ - deps, - directors, - definition, - childRunId, - parentRunId, - }); - // The in-process child has no per-step mail address, so address mirrors - // the step id (createCredentialsBackedAuthorize reads only grants). Walks - // past stepOrder into loop bodies too, or a nested loop's authorize would - // throw on its first tool call for lacking a snapshot entry. - const credentialStepIds = new Set(); - walkWorkflowSteps({ - definition: rewrittenDefinition, - descent: LOOP_BODY_DESCENT, - context: "sidecar child credentials snapshot: ", - visit: ({ stepId }) => { - credentialStepIds.add(stepId); - }, - }); - const contentHash = await hashGrants(childGrants); - const credentialsSnapshot: CredentialsSnapshot = { - steps: [...credentialStepIds].map((stepId) => ({ - stepId, - address: stepId, - grants: childGrants, - contentHash, - })), - }; - const blobs = createWorkflowRunBlobSubstrate({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - runId: childRunId, - ref: deps.workflowRunRef, - }); - const signalChannel = createWorkflowHostSignalChannel({ - repoStore: deps.substrate, - principal: deps.principal, - repoId: deps.workflowRunRepoId, - ref: deps.workflowRunRef, - runId: childRunId, - readState: () => emptyState(childRunId), - newId: () => newId("sig"), - clock, - }); - // Each (resource, action) decision looks up the step's grants and - // delegates to the parent factory's grant evaluator. - const credentialsRef: CredentialsSnapshotRef = { - current: credentialsSnapshot, - }; - const authorize = createCredentialsBackedAuthorize(credentialsRef, deps.evaluateGrants); - const drain = createNoopDrainController(rewrittenDefinition); - // Required for both childWorkflow and onTrigger body paths; a missing - // one is a wiring defect, not a silent drop. - if (onEvent === undefined) { - throw new Error( - "sidecar child: onEvent is missing; child inference events would be silently dropped from the hub stream", - ); - } - const childOnEvent = onEvent; - // Disjoint from the top-level's mutable table, so a source rotation there - // never leaks into a child; sources arrive plaintext, decrypted sidecar-side. - const sourcesRef: SourcesSnapshotRef = { - current: await resolveBodyStepSources(deps, rewrittenDefinition.id), - }; - // Mirrors the rung-0 wrap in run-child.ts so a grandchild's agent steps - // ride this run's event channel. - const spawnHost = createInMemorySpawnChild({ - bodies: grandchildMap, - runChild, - }); - const spawnChild: WorkflowRuntimeEnv["spawnChild"] = (spawnInput) => - spawnHost(spawnInput, childOnEvent, materialCell); - // Absent when no material was threaded (a non-sidecar executor), leaving - // the child's inference reader unset. - const childCredentialContext: SidecarStepCredentialContext | undefined = - materialCell === undefined - ? undefined - : { - materialCell, - resolveStepGrants: (stepId) => { - const entry = credentialsSnapshot.steps.find( - (step) => step.stepId === baseStepId(stepId), - ); - if (entry === undefined) { - throw new Error( - `sidecar child credential wiring: credentials snapshot has no entry for step ${baseStepId(stepId)}`, - ); - } - return entry.grants; - }, - providers: deps.credentialProviders, - }; - // One invoker serves both the childWorkflow and onTrigger body paths. - const invokeStep: WorkflowRuntimeEnv["invokeStep"] = (req) => - deps.invokeStep(req, authorize, sourcesRef, childOnEvent, childCredentialContext); - const env: WorkflowRuntimeEnv = { - repoStore, - scheduler: deps.scheduler, - signalChannel, - blobs, - directors, - authorize, - invokeStep, - spawnChild, - clock, - newId, - drain, - hasUpstreamSignalResolver: args.hasUpstreamSignalResolver, - ...(loopFns !== undefined ? { loopFns } : {}), - }; - // Assigned after the env literal since the iteration host closes over env. - // Deliberately replicates the top-level loop host in run-child.ts rather - // than sharing a helper — extract one only once a deployed loop-in-body - // resume test shows the two paths match. The suspendable-child seam - // services approval parks only. - if (loopFns !== undefined) { - const loopIterationHost = createInMemorySpawnSuspendableChild({ - bodies: loopBodiesMap, - runSuspendableChild: async (loopInput, _onEvent) => { - const preRewriteBody = loopBodyPreRewrite.get(loopInput.definitionRef); - if (preRewriteBody === undefined) { - throw new Error( - `sidecar child: no pre-rewrite loop body for ref ${loopInput.definitionRef}`, - ); - } - // Capped against the body's own grants before the first event. - await capAndPersistChildGrants({ - deps, - directors, - definition: preRewriteBody, - childRunId: loopInput.childRunId, - parentRunId: loopInput.parentRunId, - }); - const iterationSignalChannel = createWorkflowHostSignalChannel({ - repoStore: deps.substrate, - principal: deps.principal, - repoId: deps.workflowRunRepoId, - ref: deps.workflowRunRef, - runId: loopInput.childRunId, - readState: () => emptyState(loopInput.childRunId), - newId: () => newId("sig"), - clock, - }); - return createLoopIterationHandle(env, { - definition: loopInput.definition, - childRunId: loopInput.childRunId, - input: loopInput.input, - depth: loopInput.depth, - maxChildSpawnDepth: loopInput.maxChildSpawnDepth, - ...(loopInput.resumeFromEvents !== undefined - ? { resumeFromEvents: loopInput.resumeFromEvents } - : {}), - signal: loopInput.signal, - signalChannel: iterationSignalChannel, - cleanup: () => iterationSignalChannel.stop(), - }); - }, - }); - env.spawnLoopIteration = (spawnInput) => loopIterationHost(spawnInput, childOnEvent); - } - return { env, signalChannel, definition: rewrittenDefinition }; -} - -/** - * Legacy fallback: a body absent from deps.bodySources (a record predating - * body sources moving into the record) reads its plaintext on-disk - * sources.json instead. Removable once no restorable record predates it. - */ -async function resolveBodyStepSources( - deps: SidecarRunChildDeps, - definitionId: string, -): Promise { - const delivered = deps.bodySources[definitionId]; - if (delivered !== undefined) { - return delivered; - } - if (deps.dataDir === undefined) { - throw new Error( - `sidecar child: body ${definitionId} is absent from the delivered sources and deps.dataDir is missing, so its legacy on-disk sources cannot be read`, - ); - } - return readChildStepInferenceSources(deps.dataDir, definitionId); -} - -/** See resolveBodyStepSources; only reached for a body the delivered set doesn't carry. */ -async function readChildStepInferenceSources( - dataDir: string, - childRef: string, -): Promise { - const sourcesPath = path.join(dataDir, "assets", "workflow", childRef, "sources.json"); - let raw: string; - try { - raw = await fs.promises.readFile(sourcesPath, "utf8"); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar child: failed to read child inference sources at ${sourcesPath}: ${reason}`, - { cause }, - ); - } - return parseStepInferenceSources(raw); -} - -function defaultClock(): Date { - return new Date(); -} - -let runChildIdCounter = 0; -function defaultNewId(prefix: string): string { - runChildIdCounter += 1; - return `${prefix}-${String(runChildIdCounter)}-${Math.random().toString(36).slice(2, 8)}`; -} - -/** - * Opens a bare RepoStore for read-only operations against the shared - * on-disk workflow-run repo, then a proxy RepoStore whose writes forward - * over the control channel into the supervisor's own (pack-push-wrapped) - * substrate. - */ -export function createSidecarSubstrateFactory( - deps: SidecarSubstrateFactoryDeps = {}, -): SubstrateFactory { - const createBareRepoStore = - deps.createBareRepoStore ?? - (({ dataDir, signingKey }) => createAgentRepoStore({ dataDir, signingKey }).repoStore); - - return async (env: SubstrateFactoryEnv) => { - const validated = SubstrateConfig(env.substrateConfig); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config failed validation: ${validated.summary}`, - ); - } - - const stepInferenceSources = parseStepInferenceSources(validated.STEP_INFERENCE_SOURCES); - - // Eager: a bad specifier crashes the child loudly at construction - // rather than silently degrading to built-ins-only at first resolve. - const childAdapterRegistry = await loadAdapterRegistry( - parseAdapterManifest(validated.SIDECAR_ADAPTER_MANIFEST), - ); - - const signingKey = { - publicKey: hexDecode(validated.SIDECAR_SIGNING_PUBLIC_KEY, "SIDECAR_SIGNING_PUBLIC_KEY"), - privateKey: hexDecode(validated.SIDECAR_SIGNING_PRIVATE_KEY, "SIDECAR_SIGNING_PRIVATE_KEY"), - }; - - const bareStore: RepoStore = createBareRepoStore({ - dataDir: validated.SIDECAR_DATA_DIR, - signingKey, - }); - - const workflowRunRepoId = { - kind: "workflow-run" as const, - id: validated.WORKFLOW_RUN_REPO_ID, - }; - const principal: WorkflowRunWorkflowProcessPrincipal = { - kind: "workflow-process", - anchorRunId: env.spawn.anchorRunId, - }; - - // Proxy substrate: writes are forwarded over IPC into the - // supervisor's substrate; reads consult the bare on-disk store. - // The supervisor is the sole writer of the workflow-run ref so - // the child's writes never race against the supervisor's - // claim-check writes (inbox / processing / consumed). - const substrate: RepoStore = createProxyWorkflowRunRepoStore({ - bareStore, - bridge: env.substrateWriteBridge, - workflowRunRepoId, - }); - - // One registry per child, shared by the step agent's transport (mail_wait) - // and the control loop's mailbox.notify firing, so a read after a notify - // observes what the supervisor just committed. - const mailboxWatchRegistry = createMailboxWatchRegistry(); - const transportInbound: SupervisorBackedTransportInbound = { - reader: createChildMailboxReader({ - substrate, - repoId: workflowRunRepoId, - principal, - ref: validated.WORKFLOW_RUN_REF, - }), - watchRegistry: mailboxWatchRegistry, - // No sender-key registry yet, so every message's signature status is "unknown". - getCrypto: () => undefined, - // Write methods route to the supervisor, the sole mailbox writer. - mutationBridge: env.mailboxMutationBridge, - }; - - const hostScheduler = createWorkflowHostScheduler({ - repoStore: substrate, - principal, - listActiveDeployments: () => [workflowRunRepoId], - ref: validated.WORKFLOW_RUN_REF, - clock: () => new Date(), - }); - await hostScheduler.start(); - const scheduler = adaptHostScheduler(hostScheduler); - - const stepToolCache: StepToolCacheConfig = { - cacheMaxBytes: parseByteCap(validated.SIDECAR_CACHE_MAX_BYTES, "SIDECAR_CACHE_MAX_BYTES"), - registryMaxTarballBytes: parseByteCap( - validated.SIDECAR_REGISTRY_MAX_TARBALL_BYTES, - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - ), - }; - - // Built only when warm-kept, since the sole long-lived agent's - // conversation must survive child respawn; on respawn the registry - // rebuilds empty and each store restores from the substrate on first acquire. - const conversationSigner = createStepStorageSigner(signingKey); - const durableConversation: DurableConversationRegistry | undefined = env.spawn.warmKeep - ? createDurableConversationRegistry({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - substrate, - principal, - signer: conversationSigner, - }) - : undefined; - - // Lives for the child's lifetime so a warm agent's floor, recorded on - // its single first build, remains available for every later call. - const toolMarkFloorByStep = new Map(); - - const buildStepEnv = createSidecarStepBuildEnv({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - signer: conversationSigner, - mailboxAddress: env.spawn.mailboxAddress, - stepCount: env.spawn.stepCount, - outboundMailBridge: env.outboundMailBridge, - cache: stepToolCache, - adapters: childAdapterRegistry, - recordToolMarkFloor: (stepId, grants) => { - toolMarkFloorByStep.set(stepId, grants); - }, - // Source-ref is the only deploy lineage: never a pinned tool-package manifest off a deploy tree. - sourceTools: true, - // Always present (source-ref only). - closurePackageDir: env.spawn.closurePackageDir, - // Omitted below for the spawned-child build, which owns no warm inbound mailbox. - inbound: transportInbound, - ...(durableConversation !== undefined ? { durableConversation } : {}), - }); - - // Stateless across steps, so pinned once and shared by every per-step invoker. - const stepAgentFactory = createToolBearingAgentFactory(); - - // Built once from the sidecar-static built-ins plus the header and MCP - // streamable-HTTP presets; per-run material and grants ride in - // separately at each invoke. - const credentialProviders = createCredentialProviderRegistry([ - ...builtinCredentialProviders(), - xApiKeyCredentialProvider(), - rawAuthorizationCredentialProvider(), - createMcpStreamableHttpCredentialProvider(), - ]); - - // Built cold per invocation (no durableConversation/warm hooks, no - // inbound): a spawned child runs its own evaluated toolFactories rather - // than resolving off the deploy tree, which also keeps its tools scoped - // correctly even when a body step id collides with a parent step id. - // The floor recorder throw-asserts no floor is ever recorded here, - // since the source arm's bare tool: grant is already in the snapshot. - const coldChildBuildStepEnv = createSidecarStepBuildEnv({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - signer: conversationSigner, - mailboxAddress: env.spawn.mailboxAddress, - stepCount: env.spawn.stepCount, - outboundMailBridge: env.outboundMailBridge, - cache: stepToolCache, - adapters: childAdapterRegistry, - recordToolMarkFloor: () => { - throw new Error("source-tools child build-env must not record a tool-mark floor"); - }, - sourceTools: true, - closurePackageDir: env.spawn.closurePackageDir, - }); - // Wired as childRunDeps.invokeStep, covering every spawned child's steps. - // Absent credentialContext means a credential-consuming tool fails - // closed and loud at its own resolve("credentials"), never silently. - const childInvokeStep: SidecarChildStepInvoker = ( - req, - authorize, - sourcesRef, - onEvent, - credentialContext, - ) => - createWorkflowStepInvoker({ - workflowAuthorize: authorize, - buildEnv: (buildReq) => coldChildBuildStepEnv(buildReq, sourcesRef, credentialContext), - agentFactory: stepAgentFactory, - sourcesRef, - onEvent, - })(req); - - // Mirrors the warm agent's conversation to the substrate after each - // send settles; absent for a multi-step deploy (no durable registry). - const onRunBoundary: ((key: string) => Promise) | undefined = - durableConversation !== undefined - ? async (key: string) => { - await durableConversation.get(key).mirrorToSubstrate(); - } - : undefined; - - // Routes each mail-derived inbound message onto the warm agent's - // connector thread before send, so the reply path has thread state. - const seedInbound: ((key: string, message: InboundMessage) => Promise) | undefined = - durableConversation !== undefined - ? async (key: string, message: InboundMessage) => { - await durableConversation.get(key).seedInbound(message); - } - : undefined; - - // Composes each connector.reply through the durable thread and sends it - // over the same signed-send path the agent's own transport uses. - const driveReplies: - | ((key: string, stream: AgentEventStream) => ConnectorReplyDrain) - | undefined = - durableConversation !== undefined - ? (key: string, stream: AgentEventStream) => - driveConnectorReplies({ - stream, - composeReply: () => durableConversation.get(key).composeReply(), - send: (message) => env.outboundMailBridge.submit(env.spawn.mailboxAddress, message), - // A parent miss returns undefined and the transport derives [inReplyTo]. - resolveReferences: (inReplyTo) => - resolveMailboxReferences(transportInbound.reader, inReplyTo), - onReplySent: (receipt) => durableConversation.get(key).onReplySent(receipt), - }) - : undefined; - - const invokeStep: RunWorkflowChildBindings["invokeStep"] = async ( - req, - onEvent, - authorize, - warmCache, - sourcesRef, - credentialWiring, - mailPartReader, - ) => - createWorkflowStepInvoker({ - workflowAuthorize: authorize, - // Combines the per-run credential wiring with the sidecar-static providers. - buildEnv: (buildReq) => - buildStepEnv(buildReq, sourcesRef, { - materialCell: credentialWiring.materialRef, - resolveStepGrants: credentialWiring.resolveStepGrants, - providers: credentialProviders, - }), - agentFactory: stepAgentFactory, - onEvent, - sourcesRef, - mailPartReader, - ...(warmCache !== undefined ? { warmCache } : {}), - ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), - ...(seedInbound !== undefined ? { seedInbound } : {}), - ...(driveReplies !== undefined ? { driveReplies } : {}), - })(req); - - const evaluateGrantsAdapter: GrantEvaluator = async ({ resource, action, stepId, grants }) => { - // Additive: evaluateGrants ranks by specificity then effect, so a - // declared deny still beats the derived ask/allow floor. A missing - // entry (?? []) can only fail more closed, never open a hole. - const floor = toolMarkFloorByStep.get(baseStepId(stepId)) ?? []; - const result = await evaluateGrants( - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- credentialsSnapshot.steps[*].grants is typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar - [...(grants as readonly GrantRule[]), ...floor], - resource, - action, - ); - return { - effect: result.effect, - matchingGrants: [], - resolvedBy: null, - }; - }; - - const childRunDeps: SidecarRunChildDeps = { - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - scheduler, - invokeStep: childInvokeStep, - bodySources: parseBodyInferenceSources(validated.WORKFLOW_BODY_SOURCES), - dataDir: validated.SIDECAR_DATA_DIR, - evaluateGrants: evaluateGrantsAdapter, - credentialProviders, - // Source-ref only, so always present here. - closurePackageDir: env.spawn.closurePackageDir, - }; - // run-child builds the in-memory resolver from this plus the lifted-body - // map, so an owned inline child spawns with no on-disk asset read. - const runChild = createSidecarRunChild(childRunDeps); - - // A body resolves in-process with no on-disk read and no separate - // per-body re-verify, since the parent's re-verify covers every inline body. - const runSuspendableChild = createSidecarSpawnSuspendableChild(childRunDeps); - - // Fires once a run reaches terminal, dropping the whole per-run subtree. - // Reclamation keys on the run's terminal status, and a signal-parked step - // keeps the run non-terminal, so this never fires while a suspended - // step's attempt-N store still holds a pending-op resume must reopen. - // Built only for the cold path: a warm deploy's scratch lives under the - // disjoint warm/ sub-root, reclaimed on undeploy instead. - const cleanupRunStorage: ((runId: string) => Promise) | undefined = env.spawn.warmKeep - ? undefined - : (runId: string) => - fs.promises.rm( - runStepStorageRoot({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - }), - { recursive: true, force: true }, - ); - - // Wired unconditionally, unlike cleanupRunStorage: warm parks the same - // way cold does, just reconstructed from the substrate instead of the - // per-attempt isogit store. - const loadParkedApproval: LoadParkedApproval = ({ runId, stepId, attempt, correlationId }) => - env.spawn.warmKeep - ? readWarmParkedApprovalSnapshot({ - substrate, - workflowRunRepoId, - stepId, - correlationId, - }) - : readColdParkedApprovalSnapshot({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - stepId, - attempt, - correlationId, - }); - - // Covers the crash-across-park case where the correlationId never - // reached the log: enumerates pending ops for the resume classifier. - const readParkedApprovalOps: ReadParkedApprovalOps = async ({ runId, stepId, attempt }) => - toParkedApprovalOps( - env.spawn.warmKeep - ? await readWarmParkedPendingOperations({ - substrate, - workflowRunRepoId, - stepId, - }) - : await readColdParkedPendingOperations({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - stepId, - attempt, - }), - ); - - const bindings: RunWorkflowChildBindings = { - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - invokeStep, - initialSources: stepInferenceSources, - runChild, - runSuspendableChild, - // A loop iteration runs under the inherited env, so it's the one body - // birth path that writes no grants file of its own; materialized here instead. - materializeLoopIterationGrants: async ({ parentRunId, childRunId, definition }) => { - await capAndPersistChildGrants({ - deps: childRunDeps, - directors: childRunDeps.directors ?? createDefaultDirectorRegistry(), - definition, - childRunId, - parentRunId, - }); - }, - scheduler, - evaluateGrants: evaluateGrantsAdapter, - loadParkedApproval, - readParkedApprovalOps, - mailboxWatchRegistry, - ...(cleanupRunStorage !== undefined ? { cleanupRunStorage } : {}), - }; - return bindings; - }; -} - -/** The default-deps variant; deployments needing a recording hub sink construct their own via createSidecarSubstrateFactory. */ -export const createSubstrate: SubstrateFactory = createSidecarSubstrateFactory(); diff --git a/apps/web/src/chat/list-chat-agents.test.ts b/apps/web/src/chat/list-chat-agents.test.ts index c28f954327..134c52f09c 100644 --- a/apps/web/src/chat/list-chat-agents.test.ts +++ b/apps/web/src/chat/list-chat-agents.test.ts @@ -81,15 +81,21 @@ describe("listChatAgents self-heal join", () => { expect(redeployMode(await agent())).toBe("manual"); }); - test("a fresh deployment whose run is not addressed yet is starting, not redeployed again", async () => { + test("a pending deployment whose anchor run already carries an address is starting, not live yet", async () => { hub( [ { id: "r2", status: "pending", createdAt: T1H }, { id: "r1", status: "released", createdAt: T0 }, ], - [{ id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H }], + [ + { id: "r2", status: "deployed", createdAt: T1H, updatedAt: T1H }, + { id: "r1", status: "error", createdAt: T0, updatedAt: T1H, endedAt: T1H }, + ], ); - expect(redeployMode(await agent())).toBeUndefined(); + const listed = await agent(); + expect(listed.liveAddress).toBeNull(); + expect(listed.latest).toBe("starting"); + expect(redeployMode(listed)).toBeUndefined(); }); test("a failed redeploy caps the agent", async () => { diff --git a/apps/web/src/chat/threads-api.ts b/apps/web/src/chat/threads-api.ts index e9c67908d7..20ed5b30ba 100644 --- a/apps/web/src/chat/threads-api.ts +++ b/apps/web/src/chat/threads-api.ts @@ -125,15 +125,16 @@ export async function listChatAgents(tenantId: string): Promise 0) entry.addresses.add(address); // Deployments come back newest-first, so the first one seen per asset - // is the latest, and the first live one seen is the current one. A run - // the list does not carry yet is still coming up, unless the deployment - // itself is terminal, in which case it never lived. + // is the latest, and the first live one seen is the current one. A + // pending or recovering deployment has not finished initializing on its + // sidecar yet (`POST /workflows/:runId/mail` rejects with 409 + // `deployment_not_ready`), so only a `"live"` deployment is addressable. const liveness = deploymentLiveness(deployment.status, run?.status); if (entry.latest === undefined) { entry.latest = liveness; if (run === undefined ? liveness === "stopped" : diedQuickly(run)) entry.capped = true; } - if (entry.liveAddress === null && liveness !== "stopped" && address.length > 0) { + if (entry.liveAddress === null && liveness === "live" && address.length > 0) { entry.liveAddress = address; } byAsset.set(deployment.definitionAssetId, entry); diff --git a/bun.lock b/bun.lock index 186d4e8309..ec0fed0849 100644 --- a/bun.lock +++ b/bun.lock @@ -53,13 +53,13 @@ "@corbits/oauth-core": "^0.3.0", "@corbits/webhooks": "github:corbitsdev/webhooks#3f4f83147fd826d3aec5bcac7ef4b0ae7f75c00d", "@corbits/xai-provider": "0.1.2", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", @@ -79,28 +79,26 @@ }, "apps/sidecar": { "name": "@workbench/sidecar", - "version": "0.4.0", + "version": "0.5.0", "bin": { "workflow-child": "./bin/workflow-child", }, "dependencies": { - "@corbits/credential-header": "^0.1.0", - "@corbits/credential-mcp": "^0.1.0", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", - "@intx/tools-lsp": "0.4.0", - "@intx/tools-mail": "0.4.0", - "@intx/tools-posix": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", + "@intx/tools-lsp": "0.5.0", + "@intx/tools-mail": "0.5.0", + "@intx/tools-posix": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -163,7 +161,7 @@ "peerDependencies": { "@intx/agent": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", }, }, @@ -179,7 +177,7 @@ "typescript": "catalog:", }, "peerDependencies": { - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", }, }, "packages/worker": { @@ -190,8 +188,8 @@ "@corbits/deferred-tools": "workspace:*", "@corbits/mcp": "^0.3.0", "@corbits/memory": "github:corbitsdev/corbits-memory#8be608becc749fa0795a899826277823dca6db3e", - "@intx/tools-mail": "0.4.0", - "@intx/tools-posix": "0.4.0", + "@intx/tools-mail": "0.5.0", + "@intx/tools-posix": "0.5.0", }, "devDependencies": { "@types/bun": "catalog:", @@ -205,10 +203,10 @@ }, "vendor/intx/agent": { "name": "@intx/agent", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:", @@ -218,14 +216,25 @@ "typescript": "catalog:", }, }, + "vendor/intx/authz": { + "name": "@intx/authz", + "version": "0.5.0", + "dependencies": { + "@intx/types": "workspace:*", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "vendor/intx/db": { "name": "@intx/db", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", - "@intx/hub-common": "0.4.0", - "@intx/log": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", + "@intx/hub-common": "workspace:*", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", @@ -239,11 +248,11 @@ }, "vendor/intx/harness": { "name": "@intx/harness", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/log": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", }, "devDependencies": { @@ -255,14 +264,14 @@ }, "vendor/intx/hub-agent": { "name": "@intx/hub-agent", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/harness": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", + "@intx/mime": "workspace:*", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", "isomorphic-git": "catalog:", @@ -275,18 +284,18 @@ }, "vendor/intx/hub-api": { "name": "@intx/hub-api", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@hono/standard-validator": "^0.2.2", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.4.0", + "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.4.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:", @@ -308,7 +317,7 @@ }, "vendor/intx/hub-common": { "name": "@intx/hub-common", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/types": "workspace:*", }, @@ -319,17 +328,17 @@ }, "vendor/intx/hub-sessions": { "name": "@intx/hub-sessions", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.4.0", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.4.0", - "@intx/log": "0.4.0", + "@intx/hub-common": "workspace:*", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -345,9 +354,9 @@ }, "vendor/intx/inference": { "name": "@intx/inference", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", }, @@ -358,10 +367,10 @@ }, "vendor/intx/mail-memory": { "name": "@intx/mail-memory", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/crypto": "0.4.0", - "@intx/log": "0.4.0", + "@intx/crypto": "0.5.0", + "@intx/log": "0.5.0", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", @@ -374,10 +383,10 @@ }, "vendor/intx/mailbox": { "name": "@intx/mailbox", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/crypto": "0.4.0", - "@intx/log": "0.4.0", + "@intx/crypto": "0.5.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:", @@ -389,9 +398,9 @@ }, "vendor/intx/mime": { "name": "@intx/mime", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { - "@intx/crypto": "0.4.0", + "@intx/crypto": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", }, @@ -402,7 +411,7 @@ }, "vendor/intx/types": { "name": "@intx/types", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "arktype": "catalog:", "semver": "catalog:", @@ -415,7 +424,7 @@ }, "vendor/intx/workflow": { "name": "@intx/workflow", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/agent": "workspace:*", "@intx/inference": "workspace:*", @@ -429,10 +438,10 @@ }, "vendor/intx/workflow-deploy": { "name": "@intx/workflow-deploy", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/tool-packaging": "0.4.0", + "@intx/authz": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", }, @@ -444,19 +453,24 @@ }, "vendor/intx/workflow-host": { "name": "@intx/workflow-host", - "version": "0.4.0", + "version": "0.5.0", "dependencies": { + "@corbits/credential-header": "^0.1.0", + "@corbits/credential-mcp": "^0.1.0", "@intx/agent": "workspace:*", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", + "@intx/harness": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.4.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", + "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:", }, "devDependencies": { @@ -473,8 +487,8 @@ "@corbits/oauth-core": "^0.3.0", "@corbits/openai-responses": "^0.2.1", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", @@ -482,14 +496,14 @@ "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/inference-catalog": "0.4.0", - "@intx/log": "0.4.0", + "@intx/inference-catalog": "0.5.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -509,7 +523,7 @@ "better-auth": "^1.4.18", "drizzle-orm": "^0.45.2", "hono": "^4.11.9", - "isomorphic-git": "^1.27.2", + "isomorphic-git": "^1.44.3", "postgres": "^3.4.8", "semver": "^7.7.2", "ssri": "^12.0.0", @@ -697,9 +711,9 @@ "@intx/agent": ["@intx/agent@workspace:vendor/intx/agent"], - "@intx/authz": ["@intx/authz@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-A9x4EzcECnFjhDoqj0XjaOb1GaxRyJucXOuIbY+fcPluUuMCGFvExD6DJZwjJmIYdeCvL8i96gPkncu/fAR9qA=="], + "@intx/authz": ["@intx/authz@workspace:vendor/intx/authz"], - "@intx/crypto": ["@intx/crypto@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-4/qWypC90GevbV9u1i/93tn46jsXhj8wE1Pn+awg8/ECpwhHnhPLT5Uy423Arr9eGmRmF6HiMyXcEgH1Z9/Rkw=="], + "@intx/crypto": ["@intx/crypto@0.5.0", "", { "dependencies": { "@intx/types": "0.5.0" } }, "sha512-EX5dAQuazYbsqc51d9e9BkIxUEhGf4+9U3S91XHOg+OrHcjYRxZoFzgcrAHIJVK8Mqh0WzKhGTsY1g2r3iW9nw=="], "@intx/db": ["@intx/db@workspace:vendor/intx/db"], @@ -715,7 +729,7 @@ "@intx/inference": ["@intx/inference@workspace:vendor/intx/inference"], - "@intx/log": ["@intx/log@0.4.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-f1IPZ/9YpEqlA0I2UZPOlRJOTDoOFEZRwZoUVuw8DpgR1WatOMm4hA0E8o6Mtz+pA0ux7gVdls4y/JZcgypsfQ=="], + "@intx/log": ["@intx/log@0.5.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-SxS1WM/wsxzWgAUBDE8RcBpoZSxNnVoRm21YnHr+uZeGzS1bkqc7zW41PzVAW9p6+EsoSZFN6dr5R3jcs4k9RA=="], "@intx/mail-memory": ["@intx/mail-memory@workspace:vendor/intx/mail-memory"], @@ -723,17 +737,17 @@ "@intx/mime": ["@intx/mime@workspace:vendor/intx/mime"], - "@intx/pack-transport": ["@intx/pack-transport@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-I+axd067ZFXaTRtr2zmBXl6/yU253wLc/YWEdXiPiqz06mIspR2BFJMbSXPzV4pM26n+1tv/9E5VNBQtdiYmSA=="], + "@intx/pack-transport": ["@intx/pack-transport@0.5.0", "", { "dependencies": { "@intx/types": "0.5.0" } }, "sha512-Ogxl3oeDEUDsers5lekhVzw2UyhAVrVAVBYZW3UVy7qBzICTFDBD2trAVkeswdBzVaHnvAx0GCwOrfl6Ji19gA=="], - "@intx/storage-isogit": ["@intx/storage-isogit@0.4.0", "", { "dependencies": { "@intx/log": "0.4.0", "@intx/mime": "0.4.0", "@intx/types": "0.4.0", "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "^2.1.29", "buffer": "^6.0.3", "isomorphic-git": "^1.27.2" } }, "sha512-3H4dhUsWmAixLEBp79vq9NVUf6Z4++pKzaODWFEgNfhLX2WeV5J/t6soUamL6T/8eok/SDgQ/sZiZMiJ5kDKOw=="], + "@intx/storage-isogit": ["@intx/storage-isogit@0.5.0", "", { "dependencies": { "@intx/log": "0.5.0", "@intx/mime": "0.5.0", "@intx/types": "0.5.0", "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "^2.1.29", "buffer": "^6.0.3", "isomorphic-git": "^1.27.2" } }, "sha512-9IsTX9zBhvE9U0sSu7rQozn6YU+jKIABnOlpTbqUsgpChyQsM2oq9RTmVm+sacwDrcCkxZFst9egWCH1nA9nkA=="], - "@intx/tool-packaging": ["@intx/tool-packaging@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/log": "0.4.0", "@intx/storage-isogit": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29", "isomorphic-git": "^1.27.2", "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" } }, "sha512-eBZIbGUcEBp0ppMCceA1PPd75LvXpUyWitqTIu7AsLxRNlU9gbE1hymWPXnhUKnt3LgSlHUi959Ehd6jG6mR7g=="], + "@intx/tool-packaging": ["@intx/tool-packaging@0.5.0", "", { "dependencies": { "@intx/agent": "0.5.0", "@intx/log": "0.5.0", "@intx/storage-isogit": "0.5.0", "@intx/types": "0.5.0", "arktype": "^2.1.29", "isomorphic-git": "^1.27.2", "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" } }, "sha512-Jf1XskGsN41q2/mBQNQaf410YeqAXNdXI9+hEP3aq3vmjtSUpNr4juRoRdd9p6otDxMcP9oLLZd8FdfP/dYCrA=="], - "@intx/tools-lsp": ["@intx/tools-lsp@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/log": "0.4.0", "@intx/tools-posix": "0.4.0", "@intx/types": "0.4.0", "vscode-jsonrpc": "^9.0.1", "vscode-languageserver-types": "^3.17.5", "which": "^4.0.0" } }, "sha512-qkrO3dpnPBtyLEEPBrtEzTRhXiKI1mfrQlbehmnq5D7ZcdJ99/FiTlLWzb8WHuzsh7J26xdWOYKzwTp3V/U2Mw=="], + "@intx/tools-lsp": ["@intx/tools-lsp@0.5.0", "", { "dependencies": { "@intx/agent": "0.5.0", "@intx/log": "0.5.0", "@intx/tools-posix": "0.5.0", "@intx/types": "0.5.0", "vscode-jsonrpc": "^9.0.1", "vscode-languageserver-types": "^3.17.5", "which": "^4.0.0" } }, "sha512-opGvja6O8Y95SjfrnDAlWdm/ZRZBat+OpuBAgNoUJUfIEvvZNUvxRzivrekofSG/Gy0p6fPdSd8uUAKBRaixKw=="], - "@intx/tools-mail": ["@intx/tools-mail@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29" } }, "sha512-z7AAXlGEi9Tivf3PPvw8h8Xr9lY4vYidXKX17qQG95iLV/OW8FCmaWXxuUVLBMtlxfXQKpFWKCicdSRbJsbvLQ=="], + "@intx/tools-mail": ["@intx/tools-mail@0.5.0", "", { "dependencies": { "@intx/agent": "0.5.0", "@intx/log": "0.5.0", "@intx/types": "0.5.0", "arktype": "^2.1.29" } }, "sha512-N6iYsyPDnOes3270a74JZHGjSVdWEgSfKc0twbujJhAG63Cjz1FkaQh6VmqxSqboJ5Epku6ixIvUgcYci1GOcA=="], - "@intx/tools-posix": ["@intx/tools-posix@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/types": "0.4.0" } }, "sha512-ZqC3nEDBk5ix+lW9nWAyTynkXOGycsTfCXS24uSqwqLcivucSmGDk6kJ9zs+J0Ybz3tpq7rMmoQcvM3RuZprgQ=="], + "@intx/tools-posix": ["@intx/tools-posix@0.5.0", "", { "dependencies": { "@intx/agent": "0.5.0", "@intx/types": "0.5.0" } }, "sha512-Qv6Wb+4X0+/fOpnfzzkKV4Z9qAVBOSEWKmBd4BsqAO62qAAIMlwg8XHzd3b/DCdY6PorXy5k2GBK1Xq0LeyCzg=="], "@intx/types": ["@intx/types@workspace:vendor/intx/types"], diff --git a/e2e/hub/mailbox-persist.test.ts b/e2e/hub/mailbox-persist.test.ts index f6bcc3696c..e9fd68c995 100644 --- a/e2e/hub/mailbox-persist.test.ts +++ b/e2e/hub/mailbox-persist.test.ts @@ -19,7 +19,11 @@ import { principalMail, type MailboxDb, } from "@corbits/mailbox"; -import { createHubSessionLookups, type AgentRepoStore } from "@intx/hub-sessions"; +import { + createHubSessionLookups, + createWorkflowHistoryReceiveTracker, + type AgentRepoStore, +} from "@intx/hub-sessions"; import { createHubMailboxAuthorizeSender } from "../../apps/hub/src/mailbox-persist"; import { e2eDatabaseUrl } from "../lib/database-url"; import { dbGate } from "../lib/db-gate"; @@ -140,9 +144,10 @@ async function setup(opts: { withSession?: boolean } = {}) { const baseLookups = createHubSessionLookups({ db, // persistMail (the only lookup this suite exercises) never touches - // agentRepoStore -- unlike the pack-receive lookups -- so an - // unimplemented stand-in is safe here. - agentRepoStore: undefined as unknown as AgentRepoStore, + // agentRepoStore or historyReceives -- unlike the pack-receive lookups -- + // so unimplemented stand-ins are safe here. + agentRepoStore: { repoStore: undefined } as unknown as AgentRepoStore, + historyReceives: createWorkflowHistoryReceiveTracker(), }); return { diff --git a/e2e/hub/run-stream.test.ts b/e2e/hub/run-stream.test.ts index e437a0c47b..aeb41db049 100644 --- a/e2e/hub/run-stream.test.ts +++ b/e2e/hub/run-stream.test.ts @@ -41,6 +41,8 @@ describeIfDb("workflow run SSE stream", () => { const sidecarRouter = createSidecarRouter({ authenticateSidecar: async () => null, validateSidecarIdentity: async () => false, + withExecutableWorkflowRun: async () => false, + resolveSidecarBindings: async () => [], }); const now = new Date(); diff --git a/e2e/hub/workflow-run-authenticator.test.ts b/e2e/hub/workflow-run-authenticator.test.ts index 9301b0ca1e..2805544662 100644 --- a/e2e/hub/workflow-run-authenticator.test.ts +++ b/e2e/hub/workflow-run-authenticator.test.ts @@ -83,14 +83,20 @@ describeIfDb("workflow run authenticator", () => { await db.insert(sidecar).values({ id: sidecarId, tokenHashSha256: Buffer.from(await sha256(token)), + provisionerId: "test", + provisionerApiVersion: 2, + provisionerBindingFingerprint: "test", + adapterManifestDigest: "none", }); await db.insert(sidecarAllocation).values({ id: `a-${name}-${suffix}`, anchorRunId: anchorId, tenantId, provisionerId: "test", - provisionerApiVersion: 1, + provisionerApiVersion: 2, provisionerBindingFingerprint: "test", + adapterManifestDigest: "none", + maxDisconnectedMs: 60_000, sidecarId, status: status as "allocated", }); diff --git a/e2e/install/install-package.test.ts b/e2e/install/install-package.test.ts index 465900acba..a99fbe4e1a 100644 --- a/e2e/install/install-package.test.ts +++ b/e2e/install/install-package.test.ts @@ -16,7 +16,7 @@ import { cleanupTenants } from "../lib/tenant-cleanup"; const describeIfDb = dbGate(process.env["DATABASE_URL"] ?? "", import.meta.path); const Id = type({ id: "string" }); -const Deployments = type({ id: "string" }).array(); +const Deployments = type({ id: "string", status: "string" }).array(); const RunList = type({ runIds: "string[]" }); async function bundle(entry: string): Promise { @@ -155,10 +155,19 @@ describeIfDb("installPackage", () => { }); const first = await install(); - const listed = Deployments.assert( - await (await call(`${origin}${t}/workflows/deployments`)).json(), - ); - expect(listed.map((d) => d.id)).toEqual([first.deploymentId]); + // Provisioning completes asynchronously after `POST /workflows/deployments` + // returns 201 (`status: "pending"`); wait for the deployment to reach + // `"deployed"` before mailing it or checking idempotent reinstall. + let firstStatus = ""; + for (let attempt = 0; attempt < 60 && firstStatus !== "deployed"; attempt++) { + const listed = Deployments.assert( + await (await call(`${origin}${t}/workflows/deployments`)).json(), + ); + expect(listed.map((d) => d.id)).toEqual([first.deploymentId]); + firstStatus = listed[0]?.status ?? ""; + if (firstStatus !== "deployed") await Bun.sleep(500); + } + expect(firstStatus).toBe("deployed"); const mailed = await post(`${t}/workflows/${first.deploymentId}/mail`, { content: "hello" }); expect(mailed.status).toBe(202); diff --git a/package.json b/package.json index d3f1c4114e..05a1359fcc 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "better-auth": "^1.4.18", "drizzle-orm": "^0.45.2", "hono": "^4.11.9", - "isomorphic-git": "^1.27.2", + "isomorphic-git": "^1.44.3", "postgres": "^3.4.8", "semver": "^7.7.2", "ssri": "^12.0.0", @@ -82,8 +82,8 @@ "overrides": { "arktype": "^2.2.0", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", @@ -91,13 +91,13 @@ "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/inference-catalog": "0.4.0", - "@intx/log": "0.4.0", + "@intx/inference-catalog": "0.5.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/packages/deferred-tools/package.json b/packages/deferred-tools/package.json index f5ba78d9ef..52f13c4a96 100644 --- a/packages/deferred-tools/package.json +++ b/packages/deferred-tools/package.json @@ -34,7 +34,7 @@ "peerDependencies": { "@intx/agent": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/types": "workspace:*" } } diff --git a/packages/error-sink/package.json b/packages/error-sink/package.json index 748a95c563..6df6a8e1f1 100644 --- a/packages/error-sink/package.json +++ b/packages/error-sink/package.json @@ -33,6 +33,6 @@ "typescript": "catalog:" }, "peerDependencies": { - "@intx/log": "0.4.0" + "@intx/log": "0.5.0" } } diff --git a/packages/worker/package.json b/packages/worker/package.json index 82e8d43011..2e89e08355 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,8 +38,8 @@ "@corbits/deferred-tools": "workspace:*", "@corbits/mcp": "^0.3.0", "@corbits/memory": "github:corbitsdev/corbits-memory#8be608becc749fa0795a899826277823dca6db3e", - "@intx/tools-mail": "0.4.0", - "@intx/tools-posix": "0.4.0" + "@intx/tools-mail": "0.5.0", + "@intx/tools-posix": "0.5.0" }, "devDependencies": { "@types/bun": "catalog:", diff --git a/vendor/intx/agent/README.md b/vendor/intx/agent/README.md index b8420bb3e9..4fe13964c3 100644 --- a/vendor/intx/agent/README.md +++ b/vendor/intx/agent/README.md @@ -4,6 +4,14 @@ In-process agent runtime built on `createReactorAssembly` from `@intx/inference`. Construct an agent, `send()` it a message, get a reply. +Workflow packages can import `defineAgent`, tool and director declarations, +and director registries from `@intx/agent/authoring`. That entry point keeps +the reactor and agent execution runtime out of definition loading. + +`@intx/agent/canonicalize` exports `canonicalizeForHash` and +`CanonicalizationError` for deterministic serialization of deploy-hash inputs +without loading the agent execution runtime. + Use this package when you want an agent you can drive from inside your own program — a CLI, a worker, a test, an embedded assistant. If you want an agent that lives behind a mailbox instead, see diff --git a/vendor/intx/agent/VENDORED-FROM b/vendor/intx/agent/VENDORED-FROM index 7443e8840f..1dec17c15b 100644 --- a/vendor/intx/agent/VENDORED-FROM +++ b/vendor/intx/agent/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/agent) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). No source delta (the CL-7190 ToolBundle.beforeToolExtension seam retired at the prior re-vendor: neither that pin nor HEAD agent/src carries it and nothing in the workbench references it). diff --git a/vendor/intx/agent/package.json b/vendor/intx/agent/package.json index 72c233aec1..6d3db891b0 100644 --- a/vendor/intx/agent/package.json +++ b/vendor/intx/agent/package.json @@ -1,7 +1,7 @@ { "name": "@intx/agent", "description": "In-process agent runtime: construct an agent, send it a message, get a reply", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -12,6 +12,14 @@ "./testing": { "types": "./src/testing/index.ts", "default": "./src/testing/index.ts" + }, + "./authoring": { + "types": "./src/authoring.ts", + "default": "./src/authoring.ts" + }, + "./canonicalize": { + "types": "./src/canonicalize.ts", + "default": "./src/canonicalize.ts" } }, "scripts": { @@ -19,7 +27,7 @@ }, "dependencies": { "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx/agent/src/agent.ts b/vendor/intx/agent/src/agent.ts index 30d462ed92..e7646fb989 100644 --- a/vendor/intx/agent/src/agent.ts +++ b/vendor/intx/agent/src/agent.ts @@ -61,6 +61,7 @@ import type { ConversationTurn, InboundMessage, InferenceSource, + PerCallInferenceOptions, ReactorDirector, ToolCall, ToolDefinition, @@ -114,6 +115,16 @@ export type SendOptions = { signal?: AbortSignal; /** Override the default "from" header on the synthetic inbound message. */ from?: string; + /** + * Inference options for the model calls this send drives. Applies to this + * send alone; a key the director names outright still wins. + */ + inference?: PerCallInferenceOptions; +}; + +type QueuedSend = { + message: InboundMessage; + inference: PerCallInferenceOptions | undefined; }; export type SendResult = @@ -477,7 +488,7 @@ export async function createAgent( // the language allows; the comment block above is what makes // the "assigned before any reachable read" invariant explicit. // eslint-disable-next-line prefer-const -- forward declaration; const cannot express this ordering - let sendQueue: SendQueue; + let sendQueue: SendQueue; // shutdownComplete resolves from the assembly's onShutdown hook // (composed after audit flush by the assembly) or, as a fallback, from @@ -717,11 +728,11 @@ export async function createAgent( ...(env.compactors !== undefined ? { compactors: env.compactors } : {}), }); - sendQueue = createSendQueue({ + sendQueue = createSendQueue({ maxDepth: env.sendQueueMax ?? DEFAULT_SEND_QUEUE_MAX, - start: (message) => { + start: ({ message, inference }) => { activeCycle = { lastAssistantTurn: undefined }; - reactor.deliver(message); + reactor.deliver(message, inference !== undefined ? { inference } : {}); }, }); @@ -760,7 +771,10 @@ export async function createAgent( // and per the design must fail loud. if (closed) return Promise.reject(new AgentClosedError()); const message = buildInboundMessage(content, opts); - return sendQueue.enqueue(message, opts?.signal); + return sendQueue.enqueue( + { message, inference: opts?.inference }, + opts?.signal, + ); } function stream(): AsyncIterable { diff --git a/vendor/intx/agent/src/authoring.ts b/vendor/intx/agent/src/authoring.ts new file mode 100644 index 0000000000..74d53a88e6 --- /dev/null +++ b/vendor/intx/agent/src/authoring.ts @@ -0,0 +1,40 @@ +/** Agent definitions and declarations, without the agent execution runtime. */ +export { + defineAgent, + type AgentDefinition, + type DefineAgentConfig, + type EnvRequiredByAll, + type InferencePreference, +} from "./definition"; +export { + defineTool, + definePlugin, + isAnnotatedPluginFactory, + toolApprovalEffect, + type AnnotatedPluginFactory, + type AnnotatedToolFactory, + type ToolDeclaration, +} from "./tool"; +export { + defineDirector, + isAnnotatedDirectorFactory, + type DefinedDirector, +} from "./director"; +export { + createDefaultDirectorRegistry, + createDirectorRegistry, + createWorkflowDirectorRegistry, + UnknownDirectorIdError, +} from "./director-registry"; +export { + buildDefaultDirectorRef, + defaultDirectorFactory, + type DefaultDirectorConfig, +} from "./default-director"; +export { effectiveDirectorRef } from "./env-validation"; +export type { + AnnotatedDirectorFactory, + DirectorRef, + DirectorRegistry, +} from "./director-types"; +export type { BaseEnv } from "./env"; diff --git a/vendor/intx/agent/src/default-director.ts b/vendor/intx/agent/src/default-director.ts index cce673977e..783f6d6891 100644 --- a/vendor/intx/agent/src/default-director.ts +++ b/vendor/intx/agent/src/default-director.ts @@ -18,7 +18,7 @@ import { type } from "arktype"; import { createDefaultDirector, type DefaultDirectorPolicy, -} from "@intx/inference"; +} from "@intx/inference/default-director"; import { defineDirector } from "./director"; diff --git a/vendor/intx/agent/src/env.ts b/vendor/intx/agent/src/env.ts index d0bb7d8aae..408b96b2b1 100644 --- a/vendor/intx/agent/src/env.ts +++ b/vendor/intx/agent/src/env.ts @@ -137,7 +137,7 @@ export interface BaseEnv { /** * Resolves an inference source's credential secret by `credentialId` from the - * run's credential-material cell at send time -- the same cell tool + * run's credential-material cell at send time -- the same cell binding * credentials resolve from, so the source config carries no inline secret. * * Optional at this boundary only to spare callers whose inference never diff --git a/vendor/intx/agent/src/lock.ts b/vendor/intx/agent/src/lock.ts index a878263e40..4a042e7681 100644 --- a/vendor/intx/agent/src/lock.ts +++ b/vendor/intx/agent/src/lock.ts @@ -8,13 +8,19 @@ // // This is a best-effort in-process check. It does not coordinate across OS // processes, and it compares lexically-resolved absolute paths — two paths -// that point to the same directory through symlinks or `..`/`/./` segments -// are normalized by `path.resolve`, but a hard link or a separately mounted -// bind to the same inode will not be detected. Callers are responsible for +// that differ only in `..`, `.` or repeated `/` segments, a trailing `/`, or +// a relative spelling of the same absolute path are normalized to one key. +// The normalization is lexical: a symlink, a hard link or a separately +// mounted bind to the same directory is not detected, and `/link/..` keys as +// `/` even when `/link` points elsewhere. Callers are responsible for // ensuring `env.workdir` matches the directory backing their `env.storage` // (see `BaseEnv.workdir` for the documented invariant). +// +// The key is computed without `node:path`, so the agent loads on a host with +// no Node APIs. A relative workdir resolves against `process.cwd()` when the +// host has one; a host without one must pass an absolute workdir. -import { resolve } from "node:path"; +import { join, normalize } from "@intx/types/posix-path"; const heldLocks = new Set(); @@ -35,13 +41,32 @@ export type ContextDirLock = { release(): void; }; +/** + * The absolute, normalized form of `workdir`: the lock key. A relative path + * resolves against `process.cwd()` when the host provides it, matching + * `path.resolve`, and is refused otherwise. + */ +function absoluteWorkdir(workdir: string): string { + if (workdir.startsWith("/")) { + return normalize(workdir); + } + const host: { process?: { cwd?: () => string } } = globalThis; + const cwd = host.process?.cwd?.(); + if (cwd === undefined) { + throw new Error( + `workdir ${JSON.stringify(workdir)} is relative, and this host has no working directory to resolve it against; pass an absolute workdir`, + ); + } + return join(cwd, workdir); +} + /** * Acquire the process-wide lock for `workdir`. Throws * `AgentContextLockError` if another agent already holds it. The * returned `release` is idempotent. */ export function acquireContextDirLock(workdir: string): ContextDirLock { - const path = resolve(workdir); + const path = absoluteWorkdir(workdir); if (heldLocks.has(path)) { throw new AgentContextLockError(path); } diff --git a/vendor/intx/agent/src/send-queue.ts b/vendor/intx/agent/src/send-queue.ts index 60774dff01..ad7f96dac0 100644 --- a/vendor/intx/agent/src/send-queue.ts +++ b/vendor/intx/agent/src/send-queue.ts @@ -15,6 +15,9 @@ // queue ordered against actual reactor cycles — two send() promises // cannot interleave at the reactor level. // +// A `start` that throws synchronously rejects that item and pumps the +// next, so a rejected item never occupies the active slot. +// // Queue depth (active + pending) is bounded by `maxDepth`; exceeding it // throws `SendQueueFullError` synchronously from enqueue() so a buggy // caller flooding sends fails loud instead of silently buffering. @@ -106,7 +109,13 @@ export function createSendQueue( continue; } active = next; - opts.start(next.item); + try { + opts.start(next.item); + } catch (cause) { + active = null; + settle(next, "reject", cause); + continue; + } return; } } diff --git a/vendor/intx/agent/src/source.ts b/vendor/intx/agent/src/source.ts index 518f33a128..4d9f5a451b 100644 --- a/vendor/intx/agent/src/source.ts +++ b/vendor/intx/agent/src/source.ts @@ -18,7 +18,7 @@ import { InferenceSource as InferenceSourceValidator, applyInferenceSourceFields, type InferenceSource, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; export class InvalidInferenceSourceError extends Error { constructor(message: string) { diff --git a/vendor/intx/agent/src/tool.ts b/vendor/intx/agent/src/tool.ts index d4467b7c21..ea7b4821b8 100644 --- a/vendor/intx/agent/src/tool.ts +++ b/vendor/intx/agent/src/tool.ts @@ -18,7 +18,7 @@ // handler throws. // // `createToolRunner(tools)` builds a `ToolRunner` that dispatches by tool -// name. Per the ToolRunner contract (packages/types/src/runtime.ts), `run` +// name. Per the ToolRunner contract (packages/types/src/runtime-core.ts), `run` // must not throw -- unknown tool names and handler exceptions are surfaced // as `ToolResult` with `isError: true` so the model sees them and can // recover. diff --git a/vendor/intx/authz/README.md b/vendor/intx/authz/README.md new file mode 100644 index 0000000000..388ff5b12d --- /dev/null +++ b/vendor/intx/authz/README.md @@ -0,0 +1,72 @@ +# @intx/authz + +Grant evaluation engine. Given a principal, a tenant, a resource, +and an action, `authorize` collects the relevant grants from a +`GrantStore`, picks the most specific match, applies condition +evaluators, and returns the resolved effect. + +Pattern matching uses colon-segmented patterns with `*` and `**` +wildcards; specificity scoring picks the most specific matching +grant; condition evaluators gate matches on runtime facts such as +time windows. An in-memory grant store is included for tests and +single-process deployments; `@intx/db` provides the database-backed +implementation. + +Consumed by `@intx/hub-api` for HTTP request authorization and by +`@intx/harness` for tool-call gating. + +```ts +import { authorize, createInMemoryGrantStore } from "@intx/authz"; + +const store = createInMemoryGrantStore([ + { + id: "g1", + principalId: "user-1", + roleId: null, + effect: "allow", + origin: "system", + resource: "tenant:*:agent:*", + action: "read", + conditions: null, + expiresAt: null, + }, +]); + +const result = await authorize( + store, + "user-1", + "acme", + "tenant:acme:agent:abc", + "read", +); + +if (result.effect !== "allow") throw new Error("forbidden"); +``` + +`authorize` returns an `AuthzResult` whose `effect` is one of +`allow`, `deny`, `ask`, or `null` when no grant matches. Callers +translate `null` into a refusal (HTTP 403, tool-call block) because +evaluation is fail-closed. + +## Surface + +- `authorize` — collect grants from a `GrantStore` and resolve a + principal/tenant/resource/action query to an `AuthzResult`. +- `evaluateGrants` — the core resolution logic operating on a supplied + `GrantRule[]` rather than a store, so it can be driven with synthetic + grant lists. `authorize` is a thin wrapper that collects grants and + delegates here. +- `timeWindowEvaluator` — a `ConditionEvaluator` for time-of-day + windows. Register it under the `time_window` key in a + `ConditionRegistry` to gate grants on `{ after, before, timezone }`; + cross-midnight windows are supported. +- `evaluateConditions` — run a grant's `conditions` object against a + `ConditionRegistry`. +- `createInMemoryGrantStore` — a `GrantStore` backed by an in-memory + grant list for tests and single-process deployments. +- `matchPattern`, `patternSpecificity`, `grantSpecificity` — the + colon-segmented wildcard matcher and the specificity scoring used to + pick the winning grant. +- Types: `AuthzResult`, `ConditionContext`, `ConditionEvaluator`, + `ConditionRegistry`, `Effect`, `GrantRule`, `GrantStore`, + `MatchedGrant`. diff --git a/vendor/intx/authz/VENDORED-FROM b/vendor/intx/authz/VENDORED-FROM new file mode 100644 index 0000000000..7d3e38c9f0 --- /dev/null +++ b/vendor/intx/authz/VENDORED-FROM @@ -0,0 +1,4 @@ +Source: https://github.com/faremeter/interchange (packages/authz) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) +License: LGPL-2.1-only (see vendor/intx/LICENSE) +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/authz/package.json b/vendor/intx/authz/package.json new file mode 100644 index 0000000000..1edbfc267b --- /dev/null +++ b/vendor/intx/authz/package.json @@ -0,0 +1,32 @@ +{ + "name": "@intx/authz", + "description": "Grant evaluation engine that authorizes a principal against a resource and action", + "version": "0.5.0", + "license": "LGPL-2.1-only", + "type": "module", + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@intx/types": "workspace:*" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + }, + "files": [ + "src", + "README.md", + "LICENSE" + ], + "sideEffects": false, + "publishConfig": { + "access": "public" + } +} diff --git a/vendor/intx/authz/src/authorize-action.ts b/vendor/intx/authz/src/authorize-action.ts new file mode 100644 index 0000000000..261b523a88 --- /dev/null +++ b/vendor/intx/authz/src/authorize-action.ts @@ -0,0 +1,50 @@ +import { evaluateGrants } from "./evaluate"; +import type { EvalOptions } from "./evaluate"; +import type { GrantRule } from "./types"; + +/** + * A fail-closed authorization decision. `ok: true` authorizes the action; + * `ok: false` carries the structured reason it was withheld. + */ +export type AuthorizeDecision = + | { ok: true } + | { ok: false; reason: "deny" | "ask" | "no_matching_grant" }; + +/** + * Decide whether an action on a resource is authorized against a set of + * already-collected grants, failing closed. + * + * This owns the single fail-closed rule: any effect other than `allow` -- an + * explicit `deny`, an `ask`, or no matching grant at all -- withholds + * authorization. Callers branch on `ok` and never re-derive the rule from a + * raw effect. + * + * Distinct from `authorize`: that one collects grants from a store and returns + * the raw `AuthzResult` for an HTTP caller to interpret. This verb takes + * pre-collected grants and returns the collapsed decision. `opts` is passed to + * `evaluateGrants` verbatim -- this middle layer never defaults a condition + * registry, so a caller that omits one keeps the fail-closed "skip conditioned + * grants" behavior at the edge where that choice belongs. + */ +export async function authorizeAction( + grants: GrantRule[], + resource: string, + action: string, + opts?: EvalOptions, +): Promise { + const result = await evaluateGrants(grants, resource, action, opts); + switch (result.effect) { + case "allow": + return { ok: true }; + case "deny": + return { ok: false, reason: "deny" }; + case "ask": + return { ok: false, reason: "ask" }; + case null: + return { ok: false, reason: "no_matching_grant" }; + default: { + const _exhaustive: never = result.effect; + throw new Error(`unhandled effect: ${String(_exhaustive)}`); + } + } +} diff --git a/vendor/intx/authz/src/conditions.ts b/vendor/intx/authz/src/conditions.ts new file mode 100644 index 0000000000..c641dbc8ee --- /dev/null +++ b/vendor/intx/authz/src/conditions.ts @@ -0,0 +1,36 @@ +import type { ConditionContext, ConditionRegistry } from "./types"; + +/** + * Evaluate whether a grant's conditions are satisfied. + * + * Each key in the conditions object is looked up in the registry. + * All conditions must be met for the grant to apply. If any + * condition key has no registered evaluator, an error is thrown + * to surface misconfiguration immediately. + * + * Null or empty conditions are always met. + */ +export async function evaluateConditions( + conditions: Record | null, + ctx: ConditionContext, + registry: ConditionRegistry = {}, +): Promise { + if (!conditions) return true; + + const keys = Object.keys(conditions); + if (keys.length === 0) return true; + + for (const key of keys) { + const evaluator = registry[key]; + if (!evaluator) { + throw new Error( + `Unknown condition: "${key}". Register an evaluator in the condition registry.`, + ); + } + + const result = await evaluator(conditions[key], ctx); + if (!result) return false; + } + + return true; +} diff --git a/vendor/intx/authz/src/evaluate.ts b/vendor/intx/authz/src/evaluate.ts new file mode 100644 index 0000000000..ac35e66606 --- /dev/null +++ b/vendor/intx/authz/src/evaluate.ts @@ -0,0 +1,133 @@ +import type { + AuthzResult, + ConditionRegistry, + Effect, + GrantRule, + GrantStore, + MatchedGrant, +} from "./types"; +import { matchPattern } from "./patterns"; +import { grantSpecificity } from "./specificity"; +import { evaluateConditions } from "./conditions"; + +const EFFECT_PRIORITY: Record = { + allow: 0, + ask: 1, + deny: 2, +}; + +export type EvalOptions = { + registry?: ConditionRegistry; + principalId?: string; + tenantId?: string; + consumer?: string; +}; + +/** + * Evaluate grants against a resource/action query. + * + * This is the core evaluation logic, separated from grant collection + * so it can be tested with synthetic grant lists. + * + * When a condition registry is provided, grants with non-null + * conditions are evaluated against it. Unknown condition keys + * cause an error. When no registry is provided, grants with + * non-null conditions are skipped (fail-closed). + */ +export async function evaluateGrants( + grants: GrantRule[], + resource: string, + action: string, + opts?: EvalOptions, +): Promise { + const now = new Date(); + const registry = opts?.registry; + const ctx = { + now, + resource, + action, + principalId: opts?.principalId ?? "", + tenantId: opts?.tenantId ?? "", + consumer: opts?.consumer ?? "", + }; + + const matching: MatchedGrant[] = []; + + for (const g of grants) { + if (g.expiresAt !== null && g.expiresAt < now) continue; + if (!matchPattern(g.resource, resource)) continue; + if (!matchPattern(g.action, action)) continue; + + if (g.conditions && Object.keys(g.conditions).length > 0) { + if (!registry) continue; + if (!(await evaluateConditions(g.conditions, ctx, registry))) continue; + } + + matching.push({ + id: g.id, + resource: g.resource, + action: g.action, + effect: g.effect, + origin: g.origin, + specificity: grantSpecificity(g.resource, g.action), + }); + } + + if (matching.length === 0) { + return { effect: null, matchingGrants: [], resolvedBy: null }; + } + + // Sort ascending by specificity, then by effect priority. + // Last element wins -- which is the most specific, and at equal + // specificity the strongest effect (deny > ask > allow). + // + // The `ask > allow` half of that ordering is a load-bearing security + // invariant, not just a convention: a tool's static approval mark is + // materialized as an `ask` floor grant on the run principal, and a + // workflow that declares a competing `allow` grant for the same + // `tool:/invoke` resource lands at equal specificity. `ask` + // outranking `allow` here is what stops a workflow from declaring its + // way below a tool's approval gate. Do not reorder EFFECT_PRIORITY + // without accounting for that floor. + matching.sort((a, b) => { + const specDiff = a.specificity - b.specificity; + if (specDiff !== 0) return specDiff; + return (EFFECT_PRIORITY[a.effect] ?? 0) - (EFFECT_PRIORITY[b.effect] ?? 0); + }); + + const resolvedBy = matching[matching.length - 1]; + if (!resolvedBy) { + return { effect: null, matchingGrants: [], resolvedBy: null }; + } + + return { + effect: resolvedBy.effect, + matchingGrants: matching, + resolvedBy, + }; +} + +/** + * Authorize a principal for a resource/action within a tenant. + * + * Collects all relevant grants via the provided store, matches them + * against the requested resource and action, and returns the resolved + * effect. + * + * Returns null when no grants match (fail-closed). The caller + * interprets the result in context: HTTP routes return 403, the + * agent runtime blocks the tool call. + */ +export async function authorize( + store: GrantStore, + principalId: string, + tenantId: string, + resource: string, + action: string, + registry?: ConditionRegistry, +): Promise { + const grants = await store.collectGrants(principalId, tenantId); + const opts: EvalOptions = { principalId, tenantId }; + if (registry) opts.registry = registry; + return evaluateGrants(grants, resource, action, opts); +} diff --git a/vendor/intx/authz/src/index.ts b/vendor/intx/authz/src/index.ts new file mode 100644 index 0000000000..6deacc7fe2 --- /dev/null +++ b/vendor/intx/authz/src/index.ts @@ -0,0 +1,26 @@ +export { authorize, evaluateGrants } from "./evaluate"; +export type { EvalOptions } from "./evaluate"; +export { authorizeAction } from "./authorize-action"; +export type { AuthorizeDecision } from "./authorize-action"; +export { matchPattern } from "./patterns"; +export { patternSpecificity, grantSpecificity } from "./specificity"; +export { evaluateConditions } from "./conditions"; +export { timeWindowEvaluator } from "./time-window"; +export { + actionConsumer, + credentialBindingConsumer, + toolConsumer, + toolConsumerEvaluator, + CREDENTIAL_USE_CONDITIONS, +} from "./tool-consumer"; +export { createInMemoryGrantStore } from "./memory-store"; +export type { + AuthzResult, + ConditionContext, + ConditionEvaluator, + ConditionRegistry, + Effect, + GrantRule, + GrantStore, + MatchedGrant, +} from "./types"; diff --git a/vendor/intx/authz/src/memory-store.ts b/vendor/intx/authz/src/memory-store.ts new file mode 100644 index 0000000000..fbb20662dd --- /dev/null +++ b/vendor/intx/authz/src/memory-store.ts @@ -0,0 +1,39 @@ +// In-memory grant store for testing and local demos. +// +// The caller provides a pre-scoped array of grants. The store filters by +// principalId on each collectGrants call, matching the DB store's behavior. +// +// Limitations vs. the DB store: +// - tenantId is accepted by the interface but is a no-op here. The caller +// scopes grants to the correct tenant when constructing the store. +// - Role-based grants (principalId: null, roleId set) are not resolved. +// The DB store performs a principalRole join to find role memberships; +// this store has no role data. Grants with principalId: null are never +// returned. If you need role-based grants in tests, set principalId +// directly on each grant. + +import type { GrantRule, GrantStore } from "./types"; + +export function createInMemoryGrantStore(grants: GrantRule[]): GrantStore { + function collect(principalId: string): GrantRule[] { + const now = new Date(); + return grants.filter((g) => { + if (g.principalId !== principalId) return false; + if (g.expiresAt !== null && g.expiresAt <= now) return false; + return true; + }); + } + + // tenantId (and thus the ancestor chain) is a no-op here: the caller + // pre-scopes the grant array, so chain collection returns the same set as + // single-tenant collection. The DB-backed store is where the chain walk + // actually happens. + return { + async collectGrants(principalId: string): Promise { + return collect(principalId); + }, + async collectGrantsInChain(principalId: string): Promise { + return collect(principalId); + }, + }; +} diff --git a/vendor/intx/authz/src/patterns.ts b/vendor/intx/authz/src/patterns.ts new file mode 100644 index 0000000000..11dfe40e20 --- /dev/null +++ b/vendor/intx/authz/src/patterns.ts @@ -0,0 +1,42 @@ +/** + * Match a target string against a glob pattern. + * + * Supports `*` as a wildcard that matches any sequence of characters. + * No other glob syntax is supported (no `?`, `**`, or character classes). + * + * Examples: + * matchPattern("*", "workflow-run:run_abc") => true + * matchPattern("workflow-run:*", "workflow-run:run_abc") => true + * matchPattern("workflow-run:run_abc", "workflow-run:run_abc") => true + * matchPattern("workflow-run:run_abc", "workflow-run:run_xyz") => false + * matchPattern("wallet:wal_*", "wallet:wal_123") => true + * matchPattern("wallet:wal_*", "wallet:xyz") => false + */ +export function matchPattern(pattern: string, target: string): boolean { + if (pattern === "*") return true; + if (pattern === target) return true; + + if (!pattern.includes("*")) return false; + + const parts = pattern.split("*"); + let pos = 0; + + for (let i = 0; i < parts.length; i++) { + const segment = parts[i] ?? ""; + if (segment.length === 0) continue; + + const idx = target.indexOf(segment, pos); + if (idx === -1) return false; + + // First segment must anchor to the start + if (i === 0 && idx !== 0) return false; + + pos = idx + segment.length; + } + + // Last segment must anchor to the end + const lastSegment = parts[parts.length - 1] ?? ""; + if (lastSegment.length > 0 && !target.endsWith(lastSegment)) return false; + + return true; +} diff --git a/vendor/intx/authz/src/specificity.ts b/vendor/intx/authz/src/specificity.ts new file mode 100644 index 0000000000..a8c65a429d --- /dev/null +++ b/vendor/intx/authz/src/specificity.ts @@ -0,0 +1,29 @@ +/** + * Compute a specificity score for a pattern. + * + * More specific patterns get higher scores: + * "*" => 0 (matches everything) + * "wallet:wal_*" => 11 (prefix match, 11 literal chars) + * "workflow-run:*" => 13 (type-level wildcard, 13 literal chars) + * "workflow-run:run_abc" => 1020 (exact match, 20 literal chars, no wildcards) + * + * The score is the count of non-wildcard characters, plus a bonus + * for patterns with no wildcards at all (exact matches). + */ +export function patternSpecificity(pattern: string): number { + if (pattern === "*") return 0; + + const literalLength = pattern.replace(/\*/g, "").length; + const hasWildcard = pattern.includes("*"); + + // Exact matches get a bonus to ensure they always beat prefix globs + // of similar length + return hasWildcard ? literalLength : literalLength + 1000; +} + +/** + * Combined specificity of a resource + action pair. + */ +export function grantSpecificity(resource: string, action: string): number { + return patternSpecificity(resource) + patternSpecificity(action); +} diff --git a/vendor/intx/authz/src/time-window.ts b/vendor/intx/authz/src/time-window.ts new file mode 100644 index 0000000000..e6d77dfa82 --- /dev/null +++ b/vendor/intx/authz/src/time-window.ts @@ -0,0 +1,123 @@ +// Time-window condition evaluator. +// +// Restricts a grant to a specific time-of-day window. The grant's +// conditions object should contain: +// +// { time_window: { after: "09:00", before: "17:00", timezone: "America/Los_Angeles" } } +// +// The evaluator returns true if ctx.now falls within [after, before). +// Cross-midnight windows (after >= before) are supported: +// { after: "22:00", before: "06:00" } → active from 10pm to 6am. +// +// Timezone is required — omitting it is an error. Time strings must +// be HH:MM in 24-hour format. + +import type { ConditionEvaluator } from "./types"; + +type TimeWindowValue = { + after: string; + before: string; + timezone: string; +}; + +function parseHHMM(s: string, field: string): { hour: number; minute: number } { + const match = /^(\d{2}):(\d{2})$/.exec(s); + if (match === null || match[1] === undefined || match[2] === undefined) { + throw new Error( + `time_window: "${field}" must be HH:MM in 24-hour format, got "${s}"`, + ); + } + const hour = parseInt(match[1], 10); + const minute = parseInt(match[2], 10); + if (hour > 23 || minute > 59) { + throw new Error(`time_window: "${field}" is out of range, got "${s}"`); + } + return { hour, minute }; +} + +function validateShape(value: unknown): TimeWindowValue { + if (typeof value !== "object" || value === null) { + throw new Error( + `time_window: expected an object with { after, before, timezone }, got ${typeof value}`, + ); + } + + const after = "after" in value ? value.after : undefined; + const before = "before" in value ? value.before : undefined; + const timezone = "timezone" in value ? value.timezone : undefined; + + if (typeof after !== "string") { + throw new Error( + `time_window: "after" must be a string, got ${typeof after}`, + ); + } + if (typeof before !== "string") { + throw new Error( + `time_window: "before" must be a string, got ${typeof before}`, + ); + } + if (typeof timezone !== "string") { + throw new Error( + `time_window: "timezone" is required and must be a string, got ${typeof timezone}`, + ); + } + + return { after, before, timezone }; +} + +function toMinutes(h: number, m: number): number { + return h * 60 + m; +} + +function getCurrentTimeInZone( + now: Date, + timezone: string, +): { hour: number; minute: number } { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: timezone, + hour: "2-digit", + minute: "2-digit", + hour12: false, + }).formatToParts(now); + + let hour = 0; + let minute = 0; + for (const part of parts) { + if (part.type === "hour") hour = parseInt(part.value, 10); + if (part.type === "minute") minute = parseInt(part.value, 10); + } + return { hour, minute }; +} + +/** + * Condition evaluator for time-of-day windows. + * + * Register as `time_window` in the condition registry: + * `{ time_window: timeWindowEvaluator }` + */ +export const timeWindowEvaluator: ConditionEvaluator = ( + value: unknown, + ctx, +): boolean => { + const tw = validateShape(value); + const after = parseHHMM(tw.after, "after"); + const before = parseHHMM(tw.before, "before"); + const current = getCurrentTimeInZone(ctx.now, tw.timezone); + + const afterMins = toMinutes(after.hour, after.minute); + const beforeMins = toMinutes(before.hour, before.minute); + const currentMins = toMinutes(current.hour, current.minute); + + if (afterMins === beforeMins) { + throw new Error( + `time_window: "after" and "before" are equal ("${tw.after}"), which produces a zero-duration window`, + ); + } + + if (afterMins < beforeMins) { + // Normal window: [after, before) + return currentMins >= afterMins && currentMins < beforeMins; + } + // Cross-midnight window: >= after OR < before + return currentMins >= afterMins || currentMins < beforeMins; +}; diff --git a/vendor/intx/authz/src/tool-consumer.ts b/vendor/intx/authz/src/tool-consumer.ts new file mode 100644 index 0000000000..753cdf74bf --- /dev/null +++ b/vendor/intx/authz/src/tool-consumer.ts @@ -0,0 +1,85 @@ +// Tool-consumer condition evaluator. +// +// Restricts a grant to a specific credential consumer. The grant's +// conditions object should contain: +// +// { tool: "tool:" } // a tool package +// { tool: "action:" } // a workflow action handler +// { tool: "" } // a reserved built-in consumer identity +// +// The condition key is `tool` for every consumer kind; the prefix of the value +// names the kind. +// +// The evaluator returns true only when ctx.consumer is a non-empty identity +// that exactly matches the configured value. An absent (empty) consumer fails +// closed: a consumer-scoped grant never authorizes a caller that supplied no +// consumer identity, and it must be rejected before the equality check so a +// grant value of "" cannot match an empty consumer. A non-string condition +// value is a misconfiguration and throws, mirroring the fail-loud shape of the +// other evaluators rather than silently withholding. + +import type { CredentialBinding } from "@intx/types"; + +import type { ConditionEvaluator, ConditionRegistry } from "./types"; + +/** + * Build the consumer identity for a tool package. This is the value stamped + * onto a `credential:{id}` / `use` grant's `{ tool }` condition at launch, and + * the identical string the runtime gate supplies as `ctx.consumer` when a tool + * in that package uses the credential. Launch and gate MUST derive a consumer + * identity the same way, through the builders in this module, rather than + * inline the format. The granularity is package-level: every tool in a package + * shares one consumer identity. + */ +export function toolConsumer(packageName: string): string { + return `tool:${packageName}`; +} + +/** + * Build the consumer identity for a workflow action handler, keyed by the + * handler's export name. The `action:` prefix keeps it disjoint from every + * tool consumer, so a tool package and an action handler with the same name + * never share a grant. Every action step that runs the handler shares one + * consumer identity. + */ +export function actionConsumer(handler: string): string { + return `action:${handler}`; +} + +/** + * The consumer identity a credential binding names: its tool package or its + * action handler. + */ +export function credentialBindingConsumer(binding: CredentialBinding): string { + if (binding.package !== undefined) return toolConsumer(binding.package); + return actionConsumer(binding.action); +} + +/** + * Condition evaluator for credential consumers. + * + * Register as `tool` in the condition registry: + * `{ tool: toolConsumerEvaluator }` + */ +export const toolConsumerEvaluator: ConditionEvaluator = ( + value: unknown, + ctx, +): boolean => { + if (typeof value !== "string") { + throw new Error( + `tool: condition value must be a string consumer identity, got ${typeof value}`, + ); + } + if (ctx.consumer === "") return false; + return ctx.consumer === value; +}; + +/** + * The condition registry for credential-use authorization. It carries only the + * `tool` evaluator: a `credential:{id}` / `use` grant may narrow to a specific + * consumer, and no other condition key is meaningful on that path. Any + * unrecognized key throws (fail-loud) rather than silently widening the grant. + */ +export const CREDENTIAL_USE_CONDITIONS: ConditionRegistry = { + tool: toolConsumerEvaluator, +}; diff --git a/vendor/intx/authz/src/types.ts b/vendor/intx/authz/src/types.ts new file mode 100644 index 0000000000..639614a660 --- /dev/null +++ b/vendor/intx/authz/src/types.ts @@ -0,0 +1,28 @@ +export type { + ConditionContext, + ConditionEvaluator, + ConditionRegistry, + Effect, + GrantRule, + GrantStore, +} from "@intx/types/authz"; + +import type { Effect } from "@intx/types/authz"; + +export type MatchedGrant = { + id: string; + resource: string; + action: string; + effect: Effect; + origin: "system" | "role" | "creator" | "invoker"; + specificity: number; +}; + +export type AuthzResult = { + /** The resolved effect, or null if no grants matched at all. */ + effect: Effect | null; + /** All grants that matched the resource/action query. */ + matchingGrants: MatchedGrant[]; + /** The specific grant that determined the outcome, if any. */ + resolvedBy: MatchedGrant | null; +}; diff --git a/vendor/intx/authz/tsconfig.json b/vendor/intx/authz/tsconfig.json new file mode 100644 index 0000000000..150fc8dd99 --- /dev/null +++ b/vendor/intx/authz/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.src.json", + "compilerOptions": { + "composite": false, + "noEmit": true, + "disableSourceOfProjectReferenceRedirect": true, + "declaration": false, + "declarationMap": false, + "emitDeclarationOnly": false, + "tsBuildInfoFile": "tsconfig.tsbuildinfo", + "rootDir": "../../.." + }, + "include": [ + "src" + ], + "exclude": [], + "references": [ + { + "path": "../types/tsconfig.src.json" + } + ] +} diff --git a/vendor/intx/authz/tsconfig.src.json b/vendor/intx/authz/tsconfig.src.json new file mode 100644 index 0000000000..cb9a26719f --- /dev/null +++ b/vendor/intx/authz/tsconfig.src.json @@ -0,0 +1,26 @@ +{ + "extends": "../../../tsconfig.base.json", + "include": [ + "src/**/*.ts", + "package.json", + "src/**/*.json" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/*.test.tsx" + ], + "compilerOptions": { + "types": [ + "bun" + ], + "composite": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" + }, + "references": [ + { + "path": "../types/tsconfig.src.json" + } + ] +} diff --git a/vendor/intx/db/README.md b/vendor/intx/db/README.md index d10d719dfe..e82353c95d 100644 --- a/vendor/intx/db/README.md +++ b/vendor/intx/db/README.md @@ -37,6 +37,18 @@ typed values; downstream code uses those values without re-casting. See `CONVENTIONS.md` for the project-wide rule against cast-at-callsite in DB consumers. +`canExecuteWorkflowRun` checks a loaded run's status, cancellation request, +and lifetime deadline. For delivery, use `withExecutableWorkflowRun` with the +authenticated allocation identity and a synchronous send callback. It locks +the anchor run, which cancellation and retirement of a runnable deployment's +allocation also write, then reads the allocation and checks the current time. +It takes no lock on the allocation, so a pack receive holding that row does +not delay delivery. Finish asynchronous +message preparation before calling it; the callback queues frames on the +socket and must not wait for a worker acknowledgement. An ineligible run +throws `WorkflowRunNotExecutableError`; database failures remain errors so +callers can retry them. + Connections created by `createDB` set PostgreSQL's `statement_timeout` to 60,000 milliseconds. Override it with `statementTimeoutMs` in `DBConfig` (a positive integer up to 2,147,483,647 milliseconds). The Hub and the diff --git a/vendor/intx/db/VENDORED-FROM b/vendor/intx/db/VENDORED-FROM index da273b63f4..da10ac2907 100644 --- a/vendor/intx/db/VENDORED-FROM +++ b/vendor/intx/db/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/db) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6324: workflow_definition_version gains a wire_projection jsonb column (migration 0089_workflow_definition_version_wire_projection.sql -- renumbered from 0086 as upstream took 0086/0087/0088; the journal keeps our original `when` values, which still sort after upstream's), schema/workflow-definitions.ts declares it, and workflow-definition-store.ts adds loadFrozenWireProjection (exported from src/index.ts) to read it back beside the approved wire hash. CL-6452: workflow_definition gains an origin column (migration 0090_workflow_definition_origin.sql; declared in schema/workflow-definitions.ts) marking the per-run record a folded run's deploy mints for itself, so launch and refresh resolve the hub-authored definition instead of the newest same-named row. Migration 0091_workflow_definition_version_tolerate_body_failure.sql rewrites the retired `onBodyFailure: "continue"` literal to upstream's `"tolerate"` inside stored wire projections. CL-4455: workflow_definition gains schedule_claimed_minute (migration 0092_workflow_definition_schedule_claimed_minute.sql; declared in schema/workflow-definitions.ts) so the hub's native ScheduleTrigger poller can CAS each UTC minute. CL-8162: the hub-side cron tick is deleted, so schedule_claimed_minute is dropped again (migration 0094_drop_workflow_definition_schedule_claimed_minute.sql, journal idx 94) and schema/workflow-definitions.ts no longer declares it. Workbench applies these files by sorted filename through scripts/db-setup.ts (replay-from-scratch with a filename ledger); the drizzle journal exists for drizzle-kit only. CL-7495 re-pin to c1b5bb73: upstream's 0089_tense_selene.sql (principal_key table) is renumbered to 0093_principal_keys.sql behind our four delta migrations, with journal idx 93 and a 0093_snapshot.json built from upstream's 0089 snapshot reconciled with the wire_projection/origin/schedule_claimed_minute columns; upstream 447ebb08's snapshot-chain repair (relinked 0087 prevId, workflow_run.credential_refs restored) is applied to 0087/0088. Our fork migrations 0089-0092 ship no drizzle snapshots (hand-written SQL, journal only), so there was no fork snapshot chain to repair. CL-8425: the CL-8206 re-pin to 79adc433 moved src/schema (tenant domain lower-unique index, workflow_run_dispatch.sender_address, sidecar_allocation destroy_failed status) without its SQL; upstream's 0090_tenant_domain_lower_unique, 0091_workflow_run_dispatch_sender_address and 0092_sidecar_destroy_failed are vendored verbatim as 0095/0096/0097 (journal idx 95-97, no snapshots). At the prior re-vendor, upstream's 0093_sidecar_initialization.sql was vendored verbatim as 0098 (journal idx 98, no snapshot). Re-vendor to 9febf69: upstream's 0094_dark_old_lace.sql (tenant slug DNS-label check) is vendored verbatim as 0099 (journal idx 99, no snapshot). CL-7505: model-source-resolution.ts gains the serving-time refresh seam (ServingRefresh/ServingRefreshResult, credential_needs_reauth reason; index.ts re-exports the types) so an expiring oauth_token secret can be refreshed in place before serving. CL-7510: schema/catalog.ts admits openai-responses (the workbench's loopback-OAuth providers ride the Responses protocol), in sync with @intx/types modelProviderPlugins. +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6324: workflow_definition_version gains a wire_projection jsonb column (migration 0089_workflow_definition_version_wire_projection.sql -- renumbered from 0086 as upstream took 0086/0087/0088; the journal keeps our original `when` values, which still sort after upstream's), schema/workflow-definitions.ts declares it, and workflow-definition-store.ts adds loadFrozenWireProjection (exported from src/index.ts) to read it back beside the approved wire hash. CL-6452: workflow_definition gains an origin column (migration 0090_workflow_definition_origin.sql; declared in schema/workflow-definitions.ts) marking the per-run record a folded run's deploy mints for itself, so launch and refresh resolve the hub-authored definition instead of the newest same-named row. Migration 0091_workflow_definition_version_tolerate_body_failure.sql rewrites the retired `onBodyFailure: "continue"` literal to upstream's `"tolerate"` inside stored wire projections. CL-4455: workflow_definition gains schedule_claimed_minute (migration 0092_workflow_definition_schedule_claimed_minute.sql; declared in schema/workflow-definitions.ts) so the hub's native ScheduleTrigger poller can CAS each UTC minute. CL-8162: the hub-side cron tick is deleted, so schedule_claimed_minute is dropped again (migration 0094_drop_workflow_definition_schedule_claimed_minute.sql, journal idx 94) and schema/workflow-definitions.ts no longer declares it. Workbench applies these files by sorted filename through scripts/db-setup.ts (replay-from-scratch with a filename ledger); the drizzle journal exists for drizzle-kit only. CL-7495 re-pin to c1b5bb73: upstream's 0089_tense_selene.sql (principal_key table) is renumbered to 0093_principal_keys.sql behind our four delta migrations, with journal idx 93 and a 0093_snapshot.json built from upstream's 0089 snapshot reconciled with the wire_projection/origin/schedule_claimed_minute columns; upstream 447ebb08's snapshot-chain repair (relinked 0087 prevId, workflow_run.credential_refs restored) is applied to 0087/0088. Our fork migrations 0089-0092 ship no drizzle snapshots (hand-written SQL, journal only), so there was no fork snapshot chain to repair. CL-8425: the CL-8206 re-pin to 79adc433 moved src/schema (tenant domain lower-unique index, workflow_run_dispatch.sender_address, sidecar_allocation destroy_failed status) without its SQL; upstream's 0090_tenant_domain_lower_unique, 0091_workflow_run_dispatch_sender_address and 0092_sidecar_destroy_failed are vendored verbatim as 0095/0096/0097 (journal idx 95-97, no snapshots). At earlier re-vendors, upstream's 0093_sidecar_initialization.sql and 0094_dark_old_lace.sql were vendored verbatim as 0098 and 0099 (journal idx 98-99, no snapshots). CL-10976 re-vendor to 0a4eb59: upstream's 0095–0110 migrations are vendored verbatim as 0100–0115 (journal idx 100–115, no snapshots). CL-7510: schema/catalog.ts admits openai-responses (the workbench's loopback-OAuth providers ride the Responses protocol), in sync with @intx/types modelProviderPlugins. Retired at this pin: the CL-7505 ServingRefresh seam on model-source-resolution.ts (unused; hub OAuth refresh pushes updates via pushSourceUpdates). diff --git a/vendor/intx/db/migrations/0100_workflow_lifecycle.sql b/vendor/intx/db/migrations/0100_workflow_lifecycle.sql new file mode 100644 index 0000000000..7a94b6418c --- /dev/null +++ b/vendor/intx/db/migrations/0100_workflow_lifecycle.sql @@ -0,0 +1,6 @@ +ALTER TABLE "workflow_definition" ADD COLUMN "lifecycle_policy" jsonb;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "lifecycle_policy" jsonb;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "expires_at" timestamp;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "cancellation_requested_at" timestamp;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "cancellation_deadline" timestamp;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "capacity_release_at" timestamp; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0101_workflow_pending_projection.sql b/vendor/intx/db/migrations/0101_workflow_pending_projection.sql new file mode 100644 index 0000000000..aeaff4ed99 --- /dev/null +++ b/vendor/intx/db/migrations/0101_workflow_pending_projection.sql @@ -0,0 +1,23 @@ +CREATE TABLE "workflow_pending_projection" ( + "id" text PRIMARY KEY NOT NULL, + "anchor_run_id" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + "history_required" boolean DEFAULT true NOT NULL +); +--> statement-breakpoint +ALTER TABLE "workflow_pending_projection" ADD CONSTRAINT "workflow_pending_projection_anchor_run_id_workflow_run_id_fk" FOREIGN KEY ("anchor_run_id") REFERENCES "public"."workflow_run"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "workflow_pending_projection_anchor_idx" ON "workflow_pending_projection" USING btree ("anchor_run_id","created_at");--> statement-breakpoint +-- Receives before this table existed left no record of a failed projection, so +-- every deployment that still has a live run is recorded as possibly owing one. +-- Such a deployment may never have pushed history, so its row does not require +-- the deployment's Git ref to exist. +INSERT INTO "workflow_pending_projection" ("id", "anchor_run_id", "history_required") +SELECT 'wpp_' || replace(gen_random_uuid()::text, '-', ''), "anchor"."id", false +FROM "workflow_run" AS "anchor" +WHERE "anchor"."id" = "anchor"."anchor_run_id" + AND "anchor"."address" IS NOT NULL + AND EXISTS ( + SELECT 1 FROM "workflow_run" AS "run" + WHERE "run"."anchor_run_id" = "anchor"."id" + AND "run"."status" IN ('deployed', 'running') + ); diff --git a/vendor/intx/db/migrations/0102_workflow_cancellation_reason.sql b/vendor/intx/db/migrations/0102_workflow_cancellation_reason.sql new file mode 100644 index 0000000000..507f1003c4 --- /dev/null +++ b/vendor/intx/db/migrations/0102_workflow_cancellation_reason.sql @@ -0,0 +1 @@ +ALTER TABLE "workflow_run" ADD COLUMN "cancellation_reason" text; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0103_workflow_dispatch_abandonment.sql b/vendor/intx/db/migrations/0103_workflow_dispatch_abandonment.sql new file mode 100644 index 0000000000..ed7d05e447 --- /dev/null +++ b/vendor/intx/db/migrations/0103_workflow_dispatch_abandonment.sql @@ -0,0 +1,5 @@ +ALTER TABLE "workflow_run_dispatch" DROP CONSTRAINT "workflow_run_dispatch_status_check";--> statement-breakpoint +ALTER TABLE "workflow_run_dispatch" DROP CONSTRAINT "workflow_run_dispatch_mail_sender_check";--> statement-breakpoint +CREATE INDEX "workflow_run_dispatch_unresolved_anchor_idx" ON "workflow_run_dispatch" USING btree ("anchor_run_id") WHERE "workflow_run_dispatch"."status" in ('pending', 'acknowledged') or ("workflow_run_dispatch"."status" = 'abandoned' and "workflow_run_dispatch"."next_attempt_at" is not null);--> statement-breakpoint +ALTER TABLE "workflow_run_dispatch" ADD CONSTRAINT "workflow_run_dispatch_status_check" CHECK ("workflow_run_dispatch"."status" in ('pending', 'acknowledged', 'abandoned', 'settled', 'failed'));--> statement-breakpoint +ALTER TABLE "workflow_run_dispatch" ADD CONSTRAINT "workflow_run_dispatch_mail_sender_check" CHECK ("workflow_run_dispatch"."kind" <> 'mail' or "workflow_run_dispatch"."status" in ('abandoned', 'settled', 'failed') or "workflow_run_dispatch"."sender_address" is not null); \ No newline at end of file diff --git a/vendor/intx/db/migrations/0104_workflow_lifecycle_sweep_indexes.sql b/vendor/intx/db/migrations/0104_workflow_lifecycle_sweep_indexes.sql new file mode 100644 index 0000000000..7394b281bd --- /dev/null +++ b/vendor/intx/db/migrations/0104_workflow_lifecycle_sweep_indexes.sql @@ -0,0 +1,3 @@ +CREATE INDEX "sidecar_allocation_dispatchable_anchor_idx" ON "sidecar_allocation" USING btree ("anchor_run_id") WHERE "sidecar_allocation"."status" in ('pending', 'provisioning', 'allocated', 'replacing');--> statement-breakpoint +CREATE INDEX "workflow_run_anchor_idx" ON "workflow_run" USING btree ("id") WHERE "workflow_run"."id" = "workflow_run"."anchor_run_id";--> statement-breakpoint +CREATE INDEX "workflow_run_live_anchor_idx" ON "workflow_run" USING btree ("id") WHERE "workflow_run"."id" = "workflow_run"."anchor_run_id" and "workflow_run"."status" in ('deployed', 'running'); \ No newline at end of file diff --git a/vendor/intx/db/migrations/0105_workflow_infrastructure_failure.sql b/vendor/intx/db/migrations/0105_workflow_infrastructure_failure.sql new file mode 100644 index 0000000000..f8222d8bec --- /dev/null +++ b/vendor/intx/db/migrations/0105_workflow_infrastructure_failure.sql @@ -0,0 +1,7 @@ +ALTER TABLE "workflow_run" ADD COLUMN "infrastructure_failed_at" timestamp;--> statement-breakpoint +CREATE INDEX "workflow_run_infrastructure_failed_idx" ON "workflow_run" USING btree ("id") WHERE "workflow_run"."infrastructure_failed_at" is not null;--> statement-breakpoint +-- The lifecycle sweep relies on the planner choosing its small partial +-- indexes. The new column has no statistics until the next analyze, and +-- without them the planner may walk every deployment instead. +ANALYZE "workflow_run";--> statement-breakpoint +ANALYZE "sidecar_allocation"; diff --git a/vendor/intx/db/migrations/0106_burly_virginia_dare.sql b/vendor/intx/db/migrations/0106_burly_virginia_dare.sql new file mode 100644 index 0000000000..fc6e92318e --- /dev/null +++ b/vendor/intx/db/migrations/0106_burly_virginia_dare.sql @@ -0,0 +1,3 @@ +ALTER TABLE "credential" DROP CONSTRAINT "credential_principal_id_principal_id_fk"; +--> statement-breakpoint +ALTER TABLE "credential" ADD CONSTRAINT "credential_principal_id_principal_id_fk" FOREIGN KEY ("principal_id") REFERENCES "public"."principal"("id") ON DELETE restrict ON UPDATE no action; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0107_sidecar_reuse.sql b/vendor/intx/db/migrations/0107_sidecar_reuse.sql new file mode 100644 index 0000000000..66107d30d5 --- /dev/null +++ b/vendor/intx/db/migrations/0107_sidecar_reuse.sql @@ -0,0 +1,2 @@ +DROP INDEX "sidecar_allocation_active_sidecar_idx";--> statement-breakpoint +CREATE INDEX "workflow_run_sidecar_idx" ON "workflow_run" USING btree ("sidecar_id") WHERE "workflow_run"."sidecar_id" is not null; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0108_workflow_run_failure_reason.sql b/vendor/intx/db/migrations/0108_workflow_run_failure_reason.sql new file mode 100644 index 0000000000..063fc28d73 --- /dev/null +++ b/vendor/intx/db/migrations/0108_workflow_run_failure_reason.sql @@ -0,0 +1,2 @@ +ALTER TABLE "workflow_run" ADD COLUMN "failure_code" text;--> statement-breakpoint +ALTER TABLE "workflow_run" ADD COLUMN "failure_message" text; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0109_sidecar_allocation_max_disconnected.sql b/vendor/intx/db/migrations/0109_sidecar_allocation_max_disconnected.sql new file mode 100644 index 0000000000..8304b1b7fb --- /dev/null +++ b/vendor/intx/db/migrations/0109_sidecar_allocation_max_disconnected.sql @@ -0,0 +1,6 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "max_disconnected_ms" bigint;--> statement-breakpoint +-- Allocations created before deployments had a disconnect limit take the +-- platform default of 15 minutes. On a fresh database this affects zero rows. +UPDATE "sidecar_allocation" SET "max_disconnected_ms" = 900000;--> statement-breakpoint +ALTER TABLE "sidecar_allocation" ALTER COLUMN "max_disconnected_ms" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "sidecar_allocation" ADD CONSTRAINT "sidecar_allocation_max_disconnected_check" CHECK ("sidecar_allocation"."max_disconnected_ms" > 0); diff --git a/vendor/intx/db/migrations/0110_sidecar_allocation_first_deploy_failure.sql b/vendor/intx/db/migrations/0110_sidecar_allocation_first_deploy_failure.sql new file mode 100644 index 0000000000..96dce71b8e --- /dev/null +++ b/vendor/intx/db/migrations/0110_sidecar_allocation_first_deploy_failure.sql @@ -0,0 +1 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "first_deploy_failed_at" timestamp; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0111_sidecar_timestamps_with_time_zone.sql b/vendor/intx/db/migrations/0111_sidecar_timestamps_with_time_zone.sql new file mode 100644 index 0000000000..91f3bcc066 --- /dev/null +++ b/vendor/intx/db/migrations/0111_sidecar_timestamps_with_time_zone.sql @@ -0,0 +1,5 @@ +ALTER TABLE "sidecar" ALTER COLUMN "last_heartbeat" SET DATA TYPE timestamp with time zone;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "created_at" SET DATA TYPE timestamp with time zone;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "created_at" SET DEFAULT now();--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "updated_at" SET DATA TYPE timestamp with time zone;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "updated_at" SET DEFAULT now(); \ No newline at end of file diff --git a/vendor/intx/db/migrations/0112_sidecar_adapter_manifest_digest.sql b/vendor/intx/db/migrations/0112_sidecar_adapter_manifest_digest.sql new file mode 100644 index 0000000000..71dbec4b5d --- /dev/null +++ b/vendor/intx/db/migrations/0112_sidecar_adapter_manifest_digest.sql @@ -0,0 +1,4 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "adapter_manifest_digest" text NOT NULL DEFAULT 'unrecorded';--> statement-breakpoint +ALTER TABLE "sidecar_allocation" ALTER COLUMN "adapter_manifest_digest" DROP DEFAULT;--> statement-breakpoint +ALTER TABLE "workflow_probe" ADD COLUMN "adapter_manifest_digest" text NOT NULL DEFAULT 'unrecorded';--> statement-breakpoint +ALTER TABLE "workflow_probe" ALTER COLUMN "adapter_manifest_digest" DROP DEFAULT; diff --git a/vendor/intx/db/migrations/0113_sidecar_deployment_cleanup_confirmation.sql b/vendor/intx/db/migrations/0113_sidecar_deployment_cleanup_confirmation.sql new file mode 100644 index 0000000000..493516f1d3 --- /dev/null +++ b/vendor/intx/db/migrations/0113_sidecar_deployment_cleanup_confirmation.sql @@ -0,0 +1,8 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "deployment_cleanup_confirmed" boolean DEFAULT false NOT NULL; +--> statement-breakpoint +-- Provisioner API version 2 reports whether destroy confirmed cleanup. The Hub +-- registers only version 2 provisioners, so rows bound under version 1 move to +-- the same binding's version 2 contract. +UPDATE "sidecar_allocation" SET "provisioner_api_version" = 2 WHERE "provisioner_api_version" = 1; +--> statement-breakpoint +UPDATE "workflow_probe" SET "provisioner_api_version" = 2 WHERE "provisioner_api_version" = 1; diff --git a/vendor/intx/db/migrations/0114_sidecar_provider_hold_release.sql b/vendor/intx/db/migrations/0114_sidecar_provider_hold_release.sql new file mode 100644 index 0000000000..a74067fbf1 --- /dev/null +++ b/vendor/intx/db/migrations/0114_sidecar_provider_hold_release.sql @@ -0,0 +1 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "provider_hold_released" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/vendor/intx/db/migrations/0115_sidecar_provisioner_binding.sql b/vendor/intx/db/migrations/0115_sidecar_provisioner_binding.sql new file mode 100644 index 0000000000..c49d39c75e --- /dev/null +++ b/vendor/intx/db/migrations/0115_sidecar_provisioner_binding.sql @@ -0,0 +1,42 @@ +ALTER TABLE "sidecar" ADD COLUMN "provisioner_id" text;--> statement-breakpoint +ALTER TABLE "sidecar" ADD COLUMN "provisioner_api_version" integer;--> statement-breakpoint +ALTER TABLE "sidecar" ADD COLUMN "provisioner_binding_fingerprint" text;--> statement-breakpoint +ALTER TABLE "sidecar" ADD COLUMN "adapter_manifest_digest" text;--> statement-breakpoint +ALTER TABLE "sidecar" ADD COLUMN "retired_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "sidecar" ADD COLUMN "reconnect_window_ended_at" timestamp with time zone;--> statement-breakpoint +UPDATE "sidecar" SET + "provisioner_id" = "placed"."provisioner_id", + "provisioner_api_version" = "placed"."provisioner_api_version", + "provisioner_binding_fingerprint" = "placed"."provisioner_binding_fingerprint", + "adapter_manifest_digest" = "placed"."adapter_manifest_digest" +FROM ( + SELECT DISTINCT ON ("sidecar_id") + "sidecar_id", "provisioner_id", "provisioner_api_version", + "provisioner_binding_fingerprint", "adapter_manifest_digest" + FROM ( + SELECT "sidecar_id", "provisioner_id", "provisioner_api_version", + "provisioner_binding_fingerprint", "adapter_manifest_digest", "created_at" + FROM "sidecar_allocation" WHERE "sidecar_id" IS NOT NULL + UNION ALL + SELECT "sidecar_id", "provisioner_id", "provisioner_api_version", + "provisioner_binding_fingerprint", "adapter_manifest_digest", "created_at" + FROM "workflow_probe" WHERE "sidecar_id" IS NOT NULL + ) AS "work" + ORDER BY "sidecar_id", "created_at" +) AS "placed" +WHERE "sidecar"."id" = "placed"."sidecar_id";--> statement-breakpoint +DELETE FROM "sidecar" WHERE "provisioner_id" IS NULL;--> statement-breakpoint +UPDATE "sidecar" SET "retired_at" = now() +WHERE NOT EXISTS ( + SELECT 1 FROM "sidecar_allocation" + WHERE "sidecar_allocation"."sidecar_id" = "sidecar"."id" + AND "sidecar_allocation"."status" IN ('provisioning', 'allocated', 'replacing', 'releasing', 'destroy_failed') +) AND NOT EXISTS ( + SELECT 1 FROM "workflow_probe" + WHERE "workflow_probe"."sidecar_id" = "sidecar"."id" + AND "workflow_probe"."status" IN ('provisioning', 'probing', 'releasing') +);--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "provisioner_id" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "provisioner_api_version" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "provisioner_binding_fingerprint" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "sidecar" ALTER COLUMN "adapter_manifest_digest" SET NOT NULL; diff --git a/vendor/intx/db/migrations/meta/_journal.json b/vendor/intx/db/migrations/meta/_journal.json index 851cbb894a..c0a34aa873 100644 --- a/vendor/intx/db/migrations/meta/_journal.json +++ b/vendor/intx/db/migrations/meta/_journal.json @@ -694,6 +694,118 @@ "when": 1790350152048, "tag": "0099_dark_old_lace", "breakpoints": true + }, + { + "idx": 100, + "version": "7", + "when": 1790350212048, + "tag": "0100_workflow_lifecycle", + "breakpoints": true + }, + { + "idx": 101, + "version": "7", + "when": 1790350272048, + "tag": "0101_workflow_pending_projection", + "breakpoints": true + }, + { + "idx": 102, + "version": "7", + "when": 1790350332048, + "tag": "0102_workflow_cancellation_reason", + "breakpoints": true + }, + { + "idx": 103, + "version": "7", + "when": 1790350392048, + "tag": "0103_workflow_dispatch_abandonment", + "breakpoints": true + }, + { + "idx": 104, + "version": "7", + "when": 1790358456398, + "tag": "0104_workflow_lifecycle_sweep_indexes", + "breakpoints": true + }, + { + "idx": 105, + "version": "7", + "when": 1790358709307, + "tag": "0105_workflow_infrastructure_failure", + "breakpoints": true + }, + { + "idx": 106, + "version": "7", + "when": 1791488793986, + "tag": "0106_burly_virginia_dare", + "breakpoints": true + }, + { + "idx": 107, + "version": "7", + "when": 1791488793987, + "tag": "0107_sidecar_reuse", + "breakpoints": true + }, + { + "idx": 108, + "version": "7", + "when": 1791488793988, + "tag": "0108_workflow_run_failure_reason", + "breakpoints": true + }, + { + "idx": 109, + "version": "7", + "when": 1791488793989, + "tag": "0109_sidecar_allocation_max_disconnected", + "breakpoints": true + }, + { + "idx": 110, + "version": "7", + "when": 1791488793990, + "tag": "0110_sidecar_allocation_first_deploy_failure", + "breakpoints": true + }, + { + "idx": 111, + "version": "7", + "when": 1791842736947, + "tag": "0111_sidecar_timestamps_with_time_zone", + "breakpoints": true + }, + { + "idx": 112, + "version": "7", + "when": 1791907231430, + "tag": "0112_sidecar_adapter_manifest_digest", + "breakpoints": true + }, + { + "idx": 113, + "version": "7", + "when": 1791968464970, + "tag": "0113_sidecar_deployment_cleanup_confirmation", + "breakpoints": true + }, + { + "idx": 114, + "version": "7", + "when": 1791968762497, + "tag": "0114_sidecar_provider_hold_release", + "breakpoints": true + }, + { + "idx": 115, + "version": "7", + "when": 1792341629593, + "tag": "0115_sidecar_provisioner_binding", + "breakpoints": true } ] } diff --git a/vendor/intx/db/package.json b/vendor/intx/db/package.json index 78259ae748..26e5c0a2da 100644 --- a/vendor/intx/db/package.json +++ b/vendor/intx/db/package.json @@ -1,7 +1,7 @@ { "name": "@intx/db", "description": "Client, schema, migrations, and row parsers for the hub's PostgreSQL database", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -18,10 +18,10 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", - "@intx/hub-common": "0.4.0", - "@intx/log": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", + "@intx/hub-common": "workspace:*", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", diff --git a/vendor/intx/db/src/credential-resolution.ts b/vendor/intx/db/src/credential-resolution.ts index 15817193bf..da47032250 100644 --- a/vendor/intx/db/src/credential-resolution.ts +++ b/vendor/intx/db/src/credential-resolution.ts @@ -1,13 +1,13 @@ import { eq, and, isNull } from "drizzle-orm"; -import { toolConsumer } from "@intx/authz"; +import { credentialBindingConsumer } from "@intx/authz"; import { credentialAad } from "@intx/types"; import type { CredentialBinding, CredentialCipher } from "@intx/types"; import type { CredentialBindingDescriptor, CredentialDelivery, CredentialMaterialEntry, -} from "@intx/types/sidecar"; +} from "@intx/types/credential-delivery"; import type { DB } from "./client"; import { credential } from "./schema/credentials"; @@ -284,8 +284,8 @@ type CredentialRequirement = { /** * Resolves a credential matching an agent definition requirement. * Used at agent launch time by the control plane to satisfy a definition's - * tool and integration credentials (inference sources resolve through the - * catalog, not this path). + * credential bindings (inference sources resolve through the catalog, not this + * path). */ export async function resolveCredentialRequirement( db: DB["db"], @@ -366,14 +366,14 @@ export async function resolveCredentialRequirement( */ export type CredentialDeliveryFailure = { code: "unresolved" | "no_origin" | "ambiguous"; - binding: { provider: string; package: string; handle: string }; + binding: { provider: string; consumer: string; handle: string }; message: string; }; /** * Outcome of `buildCredentialDelivery`. `ok: true` carries the material + - * descriptors delivered to the tools; `delivery` is `undefined` when there are - * no bindings. `ok: false` carries the first launch-blocking configuration + * descriptors delivered to its consumers; `delivery` is `undefined` when there + * are no bindings. `ok: false` carries the first launch-blocking configuration * failure for the caller to map to a fail-closed response. */ export type BuildCredentialDeliveryResult = @@ -385,7 +385,7 @@ export type BuildCredentialDeliveryResult = /** * Resolve a definition's credential bindings into the material + per-handle - * descriptors delivered to its tools. Each credential's secret is decrypted + * descriptors delivered to its consumers. Each credential's secret is decrypted * once, keyed by credentialId (a credential backing several handles is * decrypted once). * @@ -411,9 +411,10 @@ export async function buildCredentialDelivery(args: { const descriptors: CredentialBindingDescriptor[] = []; for (const binding of args.bindings) { + const consumer = credentialBindingConsumer(binding); const context = { provider: binding.provider, - package: binding.package, + consumer, handle: binding.handle, }; @@ -440,7 +441,7 @@ export async function buildCredentialDelivery(args: { reason: { code: "ambiguous", binding: context, - message: `Ambiguous credential for the binding on provider ${binding.provider} (package ${binding.package}, handle ${binding.handle}): ${e.message}`, + message: `Ambiguous credential for the binding on provider ${binding.provider} (consumer ${consumer}, handle ${binding.handle}): ${e.message}`, }, }; } @@ -451,15 +452,15 @@ export async function buildCredentialDelivery(args: { reason: { code: "unresolved", binding: context, - message: `No credential resolves the binding for provider ${binding.provider} (package ${binding.package}, handle ${binding.handle})`, + message: `No credential resolves the binding for provider ${binding.provider} (consumer ${consumer}, handle ${binding.handle})`, }, }; } // The credential is delivered as an origin-pinned http handle, so its // provider must declare an API origin. A provider without one (OAuth-login - // only) cannot back a tool credential; fail closed rather than deliver an - // un-pinnable secret. + // only) cannot back a credential binding; fail closed rather than deliver + // an un-pinnable secret. const providerOrigin = resolved.provider.apiBaseUrl; if (providerOrigin === null || providerOrigin === "") { return { @@ -467,7 +468,7 @@ export async function buildCredentialDelivery(args: { reason: { code: "no_origin", binding: context, - message: `Provider ${binding.provider} has no API base URL; cannot deliver an origin-pinned credential (package ${binding.package}, handle ${binding.handle})`, + message: `Provider ${binding.provider} has no API base URL; cannot deliver an origin-pinned credential (consumer ${consumer}, handle ${binding.handle})`, }, }; } @@ -482,7 +483,7 @@ export async function buildCredentialDelivery(args: { descriptors.push({ handle: binding.handle, credentialId, - consumer: toolConsumer(binding.package), + consumer, }); if (!materials.has(credentialId)) { // Decrypt at the single point of use. A decrypt failure propagates and diff --git a/vendor/intx/db/src/index.ts b/vendor/intx/db/src/index.ts index ebfde66ba7..00d12f47dd 100644 --- a/vendor/intx/db/src/index.ts +++ b/vendor/intx/db/src/index.ts @@ -20,6 +20,17 @@ export { type BackfillPrincipalKeysReport, } from "./backfill-principal-keys"; export { createGrantStore } from "./grant-store"; +export { + loadTenantLifecyclePolicies, + canExecuteWorkflowRun, + workflowRunExecutability, + workflowCapacityReleaseAt, + withExecutableWorkflowRun, + WorkflowRunNotExecutableError, + type WorkflowRunExecutionTarget, + resolveDeploymentLifecyclePolicy, + validateLifecyclePolicyEdit, +} from "./workflow-lifecycle-policy"; export { createPrincipalStore, type PrincipalStore } from "./principal-store"; export { createPrincipalKeyStore, @@ -62,8 +73,19 @@ export { type RetryWorkflowRunDispatchArgs, type WorkflowRunDispatchStore, } from "./workflow-run-dispatch-store"; +export { + createWorkflowPendingProjectionStore, + type WorkflowPendingProjectionStore, +} from "./workflow-pending-projection-store"; export { createSidecarAllocationStore, + SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE, + SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + SIDECAR_PROVISIONER_RELEASE_FAILURE_CODE, + SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE, + SIDECAR_CLEANUP_OPERATOR_RELEASED_FAILURE_CODE, type BeginSidecarReleaseArgs, type BeginSidecarReplacementArgs, type BindInitialSidecarArgs, @@ -72,16 +94,30 @@ export { type ClaimSidecarAllocationArgs, type CreatePendingSidecarAllocationArgs, type FailSidecarAllocationArgs, + type FailStoppedSidecarDeploymentArgs, type MarkSidecarAllocatedArgs, type MarkSidecarConnectionLostArgs, type MarkSidecarConnectionReadyArgs, type MarkSidecarDestroyFailedArgs, type MarkSidecarReleasedArgs, type ParkSidecarReconciliationPolicy, + type ReleaseStoppedSidecarCleanupArgs, + type ReleaseStoppedSidecarCleanupResult, type ScheduleSidecarAllocationRetryArgs, + type ScheduleSidecarReconnectAfterHubStartArgs, type SidecarAllocation, type SidecarAllocationStore, } from "./sidecar-allocation-store"; +export { + SidecarReuseRejectedError, + SidecarInventoryUnavailableError, + type RetainedSidecarIncarnation, + type SidecarInventory, + type SidecarAllocationStoreOptions, + type AllocationConnectionCheck, + type ProvisionerSidecarRetirement, + type SidecarProvisionerBinding, +} from "./sidecar-reuse"; export { createWorkflowProbeStore, type BindWorkflowProbeSidecarArgs, @@ -146,9 +182,6 @@ export { type CatalogSourceResolution, type OfferingSourceResolution, type SourceSkip, - type ServingRefresh, - type ServingRefreshResult, - type ModelSourceResolutionOpts, } from "./model-source-resolution"; export { parseGrantRow, @@ -166,6 +199,7 @@ export { parseCredentialRow, parseProviderRow, parseTenantRow, + TenantConfigInvalidError, parseWalletRow, parseTransactionRow, parseOAuthClientRow, diff --git a/vendor/intx/db/src/model-source-resolution.ts b/vendor/intx/db/src/model-source-resolution.ts index 22bfa078a0..b51518aff8 100644 --- a/vendor/intx/db/src/model-source-resolution.ts +++ b/vendor/intx/db/src/model-source-resolution.ts @@ -9,7 +9,7 @@ import { type ProviderPreference, } from "@intx/types"; import type { InferenceSource } from "@intx/types/runtime"; -import type { CredentialMaterialEntry } from "@intx/types/sidecar"; +import type { CredentialMaterialEntry } from "@intx/types/credential-delivery"; import { listVisibleOfferings, @@ -29,34 +29,8 @@ export type SourceSkip = | { reason: "wallet_backed"; provider: string } | { reason: "credential_unresolved"; provider: string } | { reason: "credential_unauthorized"; provider: string } - | { reason: "credential_needs_reauth"; provider: string } | { reason: "provider_misconfigured"; provider: string }; -/** - * CL-7505 serving-time refresh seam. Before an expiring `oauth_token` - * credential's secret is served, the caller-supplied hook may refresh it in - * place (the hub composes the connections token session). `ok: false` means - * the credential is past saving (refresh failed / re-auth required): the - * offering is skipped, never served with a dead secret. - */ -export type ServingRefreshResult = - | { readonly ok: true } - | { readonly ok: false; readonly message: string }; - -export type ServingRefresh = (credential: { - readonly id: string; - readonly tenantId: string; - readonly type: string; - readonly status: string; - readonly refreshSecret: string | null; - readonly expiresAt: Date | null; -}) => Promise; - -export type ModelSourceResolutionOpts = { - invokerPreferences?: Record; - servingRefresh?: ServingRefresh; -}; - /** * Outcome of resolving an agent's model requirements to an ordered set of * inference sources. The order is the routing order: the head is the @@ -148,7 +122,6 @@ async function buildSource( tenantId: string, resolved: ResolvedOffering, credentialCipher: CredentialCipher, - opts?: { servingRefresh?: ServingRefresh }, ): Promise< | { ok: true; source: InferenceSource; material: CredentialMaterialEntry } | { ok: false; skip: SourceSkip } @@ -174,28 +147,28 @@ async function buildSource( // Resolve the secret through the tenant-scoped credential resolver so a // provider row referencing a credential outside the tenant's ancestor // chain cannot leak that secret (the chain is the authority). - const credentialFound = await resolveCredentialById( + const credential = await resolveCredentialById( db, tenantId, provider.credentialId, ); - if (credentialFound === null) { + if (credential === null) { return { ok: false, skip: { reason: "credential_unresolved", provider: provider.name }, }; } - let credential = credentialFound; // Authority to use a credential through a catalog provider is ownership - // within the tenant hierarchy -- the same rule the tool-binding path resolves - // (`source: "tenant"`). `resolveCredentialById` above already proved the - // credential is reachable in this tenant's ancestor chain, so a tenant-owned - // credential (`principalId IS NULL`) is authorized here by ownership alone, - // with no personal grant and no role grant consulted. This runs live on every - // launch and rotation, so a catalog edit that adds an eligible credential is - // picked up without re-materializing any grant. A principal-owned credential - // is not usable through a shared catalog provider; fail closed. + // within the tenant hierarchy -- the same rule the credential-binding path + // resolves (`source: "tenant"`). `resolveCredentialById` above already proved + // the credential is reachable in this tenant's ancestor chain, so a + // tenant-owned credential (`principalId IS NULL`) is authorized here by + // ownership alone, with no personal grant and no role grant consulted. This + // runs live on every launch and rotation, so a catalog edit that adds an + // eligible credential is picked up without re-materializing any grant. A + // principal-owned credential is not usable through a shared catalog provider; + // fail closed. if (credential.principalId !== null) { return { ok: false, @@ -203,41 +176,6 @@ async function buildSource( }; } - // CL-7505 local delta: serving-time refresh. An expiring `oauth_token` - // credential gets one chance to refresh in place before its secret is - // served; a credential the hook cannot save (grant failed, re-auth - // required) is skipped outright — never served unauthenticated. The hook - // itself owns the expiry decision, so a credential that is not near - // expiry no-ops through it cheaply. - if (opts?.servingRefresh !== undefined && credential.type === "oauth_token") { - const refreshed = await opts.servingRefresh({ - id: credential.id, - tenantId: credential.tenantId, - type: credential.type, - status: credential.status, - refreshSecret: credential.refreshSecret, - expiresAt: credential.expiresAt, - }); - if (!refreshed.ok) { - return { - ok: false, - skip: { reason: "credential_needs_reauth", provider: provider.name }, - }; - } - const reread = await resolveCredentialById( - db, - tenantId, - provider.credentialId, - ); - if (reread === null) { - return { - ok: false, - skip: { reason: "credential_unresolved", provider: provider.name }, - }; - } - credential = reread; - } - // Validate the row once at this DB-to-runtime boundary. This narrows the // jsonb `quirks` from `unknown` to a `Record | null` and checks // `capabilities` against the curated enum. `quirks` is spread in only when @@ -338,7 +276,9 @@ export async function resolveModelSources( // noop fallback for a keyless composition is resolved once at that edge // (`resolveCredentialCipher` in the hub app), never defaulted here. credentialCipher: CredentialCipher, - opts?: ModelSourceResolutionOpts, + opts?: { + invokerPreferences?: Record; + }, ): Promise { if (requirements.length === 0) { return { ok: false, reason: "no_requirements" }; @@ -372,11 +312,12 @@ export async function resolveModelSources( const modelSources: InferenceSource[] = []; const modelMaterials: CredentialMaterialEntry[] = []; for (const candidate of candidates) { - const built = await buildSource(db, tenantId, candidate, credentialCipher, { - ...(opts?.servingRefresh !== undefined - ? { servingRefresh: opts.servingRefresh } - : {}), - }); + const built = await buildSource( + db, + tenantId, + candidate, + credentialCipher, + ); if (built.ok) { modelSources.push(built.source); modelMaterials.push(built.material); @@ -491,7 +432,6 @@ export async function resolveInstanceModelSources( tenantId: string, instance: { definitionId: string; modelPreferences: unknown }, credentialCipher: CredentialCipher, - opts?: { servingRefresh?: ServingRefresh }, ): Promise { // Resolve from the run's own definition by primary key. This is the SAME row // the launch resolves its requirements from, so a rotation or catalog edit @@ -530,8 +470,5 @@ export async function resolveInstanceModelSources( return resolveModelSources(db, tenantId, requirements, credentialCipher, { invokerPreferences, - ...(opts?.servingRefresh !== undefined - ? { servingRefresh: opts.servingRefresh } - : {}), }); } diff --git a/vendor/intx/db/src/parse-row.ts b/vendor/intx/db/src/parse-row.ts index 0cc6c66651..6f66eb7c38 100644 --- a/vendor/intx/db/src/parse-row.ts +++ b/vendor/intx/db/src/parse-row.ts @@ -13,12 +13,15 @@ import { principalStatuses, signalKinds, TenantConfig, + WorkflowLifecyclePolicy, workflowDefinitionStatuses, workflowDefinitionVersionStatuses, } from "@intx/types"; import { WireGrantRule } from "@intx/types/grant-wire"; -import { FrozenApprovalBundle, RepoAction } from "@intx/types/sidecar"; +import { FrozenApprovalBundle } from "@intx/types/sidecar"; +import { RepoAction } from "@intx/types/repo"; import { ToolPackagePinArray } from "@intx/types/tool-packages"; +import { workflowRunDispatchStatuses } from "./schema/workflow-run-dispatch"; import type { approval, @@ -74,10 +77,7 @@ const workflowRunStatuses = [ ] as const; const WorkflowRunStatusValidator = type.enumerated(...workflowRunStatuses); const WorkflowRunDispatchStatusValidator = type.enumerated( - "pending", - "acknowledged", - "settled", - "failed", + ...workflowRunDispatchStatuses, ); const WorkflowRunDispatchKindValidator = type.enumerated("mail", "signal"); @@ -132,12 +132,21 @@ const turnPartTypes = [ ] as const; const TurnPartTypeValidator = type.enumerated(...turnPartTypes); +export function parseWorkflowDefinitionLifecyclePolicy( + raw: unknown, +): WorkflowLifecyclePolicy { + return raw == null ? {} : WorkflowLifecyclePolicy.assert(raw); +} + export function parseWorkflowDefinitionRow( row: typeof workflowDefinition.$inferSelect, ) { return { ...row, status: WorkflowDefinitionStatusValidator.assert(row.status), + lifecyclePolicy: parseWorkflowDefinitionLifecyclePolicy( + row.lifecyclePolicy, + ), grantRequirements: row.grantRequirements !== null ? GrantRequirement.array().assert(row.grantRequirements) @@ -214,6 +223,10 @@ export function parseWorkflowRunRow(row: typeof workflowRun.$inferSelect) { return { ...row, status: WorkflowRunStatusValidator.assert(row.status), + lifecyclePolicy: + row.lifecyclePolicy == null + ? null + : WorkflowLifecyclePolicy.assert(row.lifecyclePolicy), modelPreferences: row.modelPreferences !== null ? InvokerModelPreferences.assert(row.modelPreferences) @@ -307,10 +320,28 @@ export function parseModelOfferingRow(row: typeof modelOffering.$inferSelect) { }; } +export class TenantConfigInvalidError extends Error { + constructor(tenantId: string, summary: string) { + super(`Tenant ${tenantId} has invalid configuration: ${summary}`); + this.name = "TenantConfigInvalidError"; + } +} + +// A config saved before a key was validated can fail validation added since. +export function parseTenantConfig( + tenantId: string, + raw: unknown, +): TenantConfig { + const config = TenantConfig(raw); + if (config instanceof type.errors) + throw new TenantConfigInvalidError(tenantId, config.summary); + return config; +} + export function parseTenantRow(row: typeof tenant.$inferSelect) { return { ...row, - config: row.config !== null ? TenantConfig.assert(row.config) : null, + config: row.config !== null ? parseTenantConfig(row.id, row.config) : null, }; } diff --git a/vendor/intx/db/src/schema/catalog.ts b/vendor/intx/db/src/schema/catalog.ts index 77a0cabc1c..46179f69ff 100644 --- a/vendor/intx/db/src/schema/catalog.ts +++ b/vendor/intx/db/src/schema/catalog.ts @@ -43,18 +43,7 @@ export const modelProvider = pgTable( .notNull() .references(() => tenant.id, { onDelete: "cascade" }), name: text("name").notNull(), - // Local delta (CL-7510): "openai-responses" — the workbench's - // loopback-OAuth providers (Codex, xai-oauth) ride the Responses - // protocol; must stay in sync with @intx/types' modelProviderPlugins. - plugin: text("plugin", { - enum: [ - "anthropic", - "openai", - "openai-compatible", - "google-genai", - "openai-responses", - ], - }).notNull(), + plugin: text("plugin").notNull(), baseURL: text("base_url").notNull(), // A provider authenticates via exactly one of credential or wallet (the // check constraint below enforces the XOR). onDelete is "restrict" so a diff --git a/vendor/intx/db/src/schema/credentials.ts b/vendor/intx/db/src/schema/credentials.ts index cee3b97572..e27396ee5d 100644 --- a/vendor/intx/db/src/schema/credentials.ts +++ b/vendor/intx/db/src/schema/credentials.ts @@ -12,8 +12,11 @@ export const credential = pgTable( tenantId: text("tenant_id") .notNull() .references(() => tenant.id, { onDelete: "cascade" }), + // Null means tenant-owned. `onDelete: "set null"` would hand the secret + // to the tenant. `restrict` keeps the row bound to this principal until + // the credential row itself is deleted. principalId: text("principal_id").references(() => principal.id, { - onDelete: "set null", + onDelete: "restrict", }), providerId: text("provider_id") .notNull() diff --git a/vendor/intx/db/src/schema/index.ts b/vendor/intx/db/src/schema/index.ts index 158e2d8e5d..03049f44c3 100644 --- a/vendor/intx/db/src/schema/index.ts +++ b/vendor/intx/db/src/schema/index.ts @@ -22,6 +22,7 @@ export * from "./git-tokens"; export * from "./workflow-definitions"; export * from "./workflow-run"; export * from "./workflow-run-dispatch"; +export * from "./workflow-pending-projection"; export * from "./workflow-run-launch-spec"; export * from "./workflow-run-execution"; export * from "./workflow-probe"; diff --git a/vendor/intx/db/src/schema/sidecar-allocation.ts b/vendor/intx/db/src/schema/sidecar-allocation.ts index 7e9247695e..1d19c07a31 100644 --- a/vendor/intx/db/src/schema/sidecar-allocation.ts +++ b/vendor/intx/db/src/schema/sidecar-allocation.ts @@ -1,5 +1,7 @@ import { sql } from "drizzle-orm"; import { + bigint, + boolean, check, index, integer, @@ -27,7 +29,7 @@ export const sidecarAllocation = pgTable( .references(() => tenant.id, { onDelete: "restrict" }), provisionerId: text("provisioner_id").notNull(), provisionerApiVersion: integer("provisioner_api_version") - .$type<1>() + .$type<2>() .notNull(), provisionerBindingFingerprint: text( "provisioner_binding_fingerprint", @@ -35,6 +37,11 @@ export const sidecarAllocation = pgTable( sidecarId: text("sidecar_id").references(() => sidecar.id, { onDelete: "restrict", }), + // The digest of the adapter manifest that the Hub sent, or will send, in + // the ensure for this generation. For a row with a sidecar, it is the + // manifest that the sidecar booted with. The Hub uses it to place new + // work only on a sidecar that loaded the manifest the work needs. + adapterManifestDigest: text("adapter_manifest_digest").notNull(), status: text("status") .$type() .notNull() @@ -48,7 +55,21 @@ export const sidecarAllocation = pgTable( initializationLeaseId: text("initialization_lease_id"), ensureAttempts: integer("ensure_attempts").notNull().default(0), destroyAttempts: integer("destroy_attempts").notNull().default(0), + deploymentCleanupConfirmed: boolean("deployment_cleanup_confirmed") + .notNull() + .default(false), + providerHoldReleased: boolean("provider_hold_released") + .notNull() + .default(false), connectDeadline: timestamp("connect_deadline"), + // How long the sidecar may stay disconnected before the Hub fails the + // deployment, from the deployment's lifecycle policy. + maxDisconnectedMs: bigint("max_disconnected_ms", { + mode: "number", + }).notNull(), + // When the deployment's first deploy first failed before its deploy frame + // was sent; the Hub stops retrying it a while after. + firstDeployFailedAt: timestamp("first_deploy_failed_at"), failureCode: text("failure_code"), failureMessage: text("failure_message"), createdAt: timestamp("created_at").notNull().defaultNow(), @@ -56,22 +77,26 @@ export const sidecarAllocation = pgTable( }, (t) => [ uniqueIndex("sidecar_allocation_anchor_run_idx").on(t.anchorRunId), - uniqueIndex("sidecar_allocation_active_sidecar_idx") - .on(t.sidecarId) - .where( - sql`${t.status} in ('provisioning', 'allocated', 'replacing', 'releasing', 'destroy_failed')`, - ), index("sidecar_allocation_sidecar_idx").on(t.sidecarId), index("sidecar_allocation_reconciliation_idx") .on(t.nextAttemptAt, t.createdAt) .where( sql`${t.status} in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing') and ${t.nextAttemptAt} is not null`, ), + index("sidecar_allocation_dispatchable_anchor_idx") + .on(t.anchorRunId) + .where( + sql`${t.status} in ('pending', 'provisioning', 'allocated', 'replacing')`, + ), check( "sidecar_allocation_status_check", sql`${t.status} in ('pending', 'provisioning', 'allocated', 'replacing', 'releasing', 'destroy_failed', 'released', 'failed')`, ), check("sidecar_allocation_generation_check", sql`${t.generation} >= 0`), + check( + "sidecar_allocation_max_disconnected_check", + sql`${t.maxDisconnectedMs} > 0`, + ), check( "sidecar_allocation_accepted_generation_check", sql`${t.ensureAcceptedGeneration} is null or ${t.ensureAcceptedGeneration} <= ${t.generation}`, diff --git a/vendor/intx/db/src/schema/sidecar.ts b/vendor/intx/db/src/schema/sidecar.ts index 7229fbf107..17afbf8bae 100644 --- a/vendor/intx/db/src/schema/sidecar.ts +++ b/vendor/intx/db/src/schema/sidecar.ts @@ -1,4 +1,4 @@ -import { pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { integer, pgTable, text, timestamp } from "drizzle-orm/pg-core"; import { bytea } from "./column-types"; @@ -11,12 +11,36 @@ export const sidecar = pgTable("sidecar", { // WebSocket handshake and matched against this column. Only the digest // is stored; the raw token is never persisted. tokenHashSha256: bytea("token_hash_sha256").notNull().unique(), + // The provisioner binding that minted this identity and the adapter + // manifest its ensure carried. Reuse matches them, including on a sidecar + // that hosts nothing. + provisionerId: text("provisioner_id").notNull(), + provisionerApiVersion: integer("provisioner_api_version") + .$type<2>() + .notNull(), + provisionerBindingFingerprint: text( + "provisioner_binding_fingerprint", + ).notNull(), + adapterManifestDigest: text("adapter_manifest_digest").notNull(), + // Set once the Hub knows no sidecar runs with this identity. Migration 0110 + // also set it on every sidecar then hosting nothing current, some of which + // may still have been running. + retiredAt: timestamp("retired_at", { withTimezone: true }), + // Set when its provisioner ended the reconnect window of this sidecar's + // work, so that work gets none again, even after a Hub restart. + reconnectWindowEndedAt: timestamp("reconnect_window_ended_at", { + withTimezone: true, + }), status: text("status", { enum: ["online", "offline", "error"], }) .notNull() .default("online"), - lastHeartbeat: timestamp("last_heartbeat"), - createdAt: timestamp("created_at").notNull().defaultNow(), - updatedAt: timestamp("updated_at").notNull().defaultNow(), + lastHeartbeat: timestamp("last_heartbeat", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .notNull() + .defaultNow(), }); diff --git a/vendor/intx/db/src/schema/workflow-definitions.ts b/vendor/intx/db/src/schema/workflow-definitions.ts index 57cbdee38a..7dcc570b9d 100644 --- a/vendor/intx/db/src/schema/workflow-definitions.ts +++ b/vendor/intx/db/src/schema/workflow-definitions.ts @@ -63,12 +63,14 @@ export const workflowDefinition = pgTable( // which supplies its sources at deploy time. Validated as ModelRequirements // at parse time. modelRequirements: jsonb("model_requirements"), - // Credential bindings manifest: maps a tool package's declared credential - // handle (package, handle) to a concrete credential resolved fresh at - // launch (by locator + provider + name against the tenant walk-up). + // Credential bindings manifest: maps a consumer's declared credential + // handle -- (package, handle) for a tool package or (action, handle) for an + // action handler -- to a concrete credential resolved fresh at launch (by + // locator + provider + name against the tenant walk-up). // Validated as CredentialBinding[] at parse time. A binding is a request // the launch-time grant gate authorizes; it consents to nothing on its own. credentialBindings: jsonb("credential_bindings"), + lifecyclePolicy: jsonb("lifecycle_policy"), currentVersion: text("current_version").notNull().default("1"), status: text("status", { enum: ["deployed", "stopped"], diff --git a/vendor/intx/db/src/schema/workflow-pending-projection.ts b/vendor/intx/db/src/schema/workflow-pending-projection.ts new file mode 100644 index 0000000000..7e650a3c42 --- /dev/null +++ b/vendor/intx/db/src/schema/workflow-pending-projection.ts @@ -0,0 +1,34 @@ +import { boolean, index, pgTable, text, timestamp } from "drizzle-orm/pg-core"; + +import { workflowRun } from "./workflow-run"; + +/** + * One workflow-run pack receive whose run-status projection is not yet + * confirmed. The row is written before the receive can advance Git, because Git + * acceptance and the `workflow_run` projection are not atomic: a projection that + * fails after the ref advances leaves no other durable trace. The receive + * deletes its row once every run it carried reached a final decision, or when + * it provably left Git unchanged; lifecycle recovery deletes rows it has + * reconciled from Git. While any row exists for a deployment, its + * `workflow_run` rows may disagree with accepted history. + */ +export const workflowPendingProjection = pgTable( + "workflow_pending_projection", + { + id: text("id").primaryKey(), + anchorRunId: text("anchor_run_id") + .notNull() + .references(() => workflowRun.id, { onDelete: "cascade" }), + createdAt: timestamp("created_at").notNull().defaultNow(), + // A receive that reached Git leaves the deployment's ref in place, so a + // missing ref under its row means history is unavailable, not empty. Only + // rows that cannot know whether Git was ever written clear on a missing ref. + historyRequired: boolean("history_required").notNull().default(true), + }, + (t) => [ + index("workflow_pending_projection_anchor_idx").on( + t.anchorRunId, + t.createdAt, + ), + ], +); diff --git a/vendor/intx/db/src/schema/workflow-probe.ts b/vendor/intx/db/src/schema/workflow-probe.ts index 8e6ed77aef..bf9f3d12fd 100644 --- a/vendor/intx/db/src/schema/workflow-probe.ts +++ b/vendor/intx/db/src/schema/workflow-probe.ts @@ -52,7 +52,7 @@ export const workflowProbe = pgTable( .default("pending"), provisionerId: text("provisioner_id").notNull(), provisionerApiVersion: integer("provisioner_api_version") - .$type<1>() + .$type<2>() .notNull(), provisionerBindingFingerprint: text( "provisioner_binding_fingerprint", @@ -60,6 +60,11 @@ export const workflowProbe = pgTable( sidecarId: text("sidecar_id").references(() => sidecar.id, { onDelete: "restrict", }), + // The digest of the adapter manifest that the Hub sends in the ensure for + // this probe. For a probe with a sidecar, it is the manifest that the + // sidecar booted with. The Hub uses it to place new work only on a + // sidecar that loaded the manifest the work needs. + adapterManifestDigest: text("adapter_manifest_digest").notNull(), generation: integer("generation").notNull().default(0), externalRef: text("external_ref"), result: jsonb("result").$type(), diff --git a/vendor/intx/db/src/schema/workflow-run-dispatch.ts b/vendor/intx/db/src/schema/workflow-run-dispatch.ts index f4738000a8..3816b67cba 100644 --- a/vendor/intx/db/src/schema/workflow-run-dispatch.ts +++ b/vendor/intx/db/src/schema/workflow-run-dispatch.ts @@ -16,6 +16,7 @@ import { workflowRun } from "./workflow-run"; export const workflowRunDispatchStatuses = [ "pending", "acknowledged", + "abandoned", "settled", "failed", ] as const; @@ -23,6 +24,14 @@ export const workflowRunDispatchStatuses = [ export type WorkflowRunDispatchStatus = (typeof workflowRunDispatchStatuses)[number]; +// These outcomes can still be resolved from Git. Abandoned deliveries cannot +// be retried; stopping their workflow does not prove whether it accepted them. +export const unresolvedWorkflowRunDispatchStatuses = [ + "pending", + "acknowledged", + "abandoned", +] as const; + export const workflowRunDispatchKinds = ["mail", "signal"] as const; export type WorkflowRunDispatchKind = (typeof workflowRunDispatchKinds)[number]; @@ -74,9 +83,16 @@ export const workflowRunDispatch = pgTable( t.anchorRunId, t.status, ), + // Dispatch outcomes are projected by a sweep that runs every second, and + // settled rows are kept, so it reaches deployments through this index. + index("workflow_run_dispatch_unresolved_anchor_idx") + .on(t.anchorRunId) + .where( + sql`${t.status} in ('pending', 'acknowledged') or (${t.status} = 'abandoned' and ${t.nextAttemptAt} is not null)`, + ), check( "workflow_run_dispatch_status_check", - sql`${t.status} in ('pending', 'acknowledged', 'settled', 'failed')`, + sql`${t.status} in ('pending', 'acknowledged', 'abandoned', 'settled', 'failed')`, ), check( "workflow_run_dispatch_kind_check", @@ -99,13 +115,13 @@ export const workflowRunDispatch = pgTable( sql`${t.status} <> 'pending' or ${t.nextAttemptAt} is not null`, ), // A deliverable mail dispatch must carry the hub-verified sender that - // its inbound frame is stamped with; signals have no sender. Terminal - // (settled/failed) mail rows are exempt because they are never + // its inbound frame is stamped with; signals have no sender. Abandoned + // and terminal (settled/failed) mail rows are exempt because they are never // re-dispatched and so never reconstruct a frame -- this is what lets a // migration fail legacy in-flight mail in place rather than delete it. check( "workflow_run_dispatch_mail_sender_check", - sql`${t.kind} <> 'mail' or ${t.status} in ('settled', 'failed') or ${t.senderAddress} is not null`, + sql`${t.kind} <> 'mail' or ${t.status} in ('abandoned', 'settled', 'failed') or ${t.senderAddress} is not null`, ), ], ); diff --git a/vendor/intx/db/src/schema/workflow-run-launch-spec.ts b/vendor/intx/db/src/schema/workflow-run-launch-spec.ts index 388af0c614..0366b48a38 100644 --- a/vendor/intx/db/src/schema/workflow-run-launch-spec.ts +++ b/vendor/intx/db/src/schema/workflow-run-launch-spec.ts @@ -4,10 +4,9 @@ import { principal } from "./principals"; import { workflowRun } from "./workflow-run"; /** - * Immutable inputs required to recreate an exclusively placed workflow on a - * replacement sidecar. Secrets are deliberately excluded: source offering - * ids are resolved against the current catalog and principal authority when a - * generation is launched. + * Immutable inputs a generation's first deploy launches the workflow from. + * Secrets are deliberately excluded: source offering ids are resolved against + * the current catalog and principal authority when a generation is launched. */ export const workflowRunLaunchSpec = pgTable("workflow_run_launch_spec", { anchorRunId: text("anchor_run_id") diff --git a/vendor/intx/db/src/schema/workflow-run.ts b/vendor/intx/db/src/schema/workflow-run.ts index cbf19fd3a7..166d810779 100644 --- a/vendor/intx/db/src/schema/workflow-run.ts +++ b/vendor/intx/db/src/schema/workflow-run.ts @@ -17,9 +17,9 @@ import { workflowDefinition } from "./workflow-definitions"; // The non-secret projection of a deployment's credential delivery, persisted on // its anchor run so the reconnect resync can re-resolve current materials. // `credentialIds` is every credential the delivery references (inference -// sources have no binding; tool handles do). `bindings` is the tool binding -// descriptors (handle -> credentialId, plus the consumer allowed to use it). -// Never holds a secret. +// sources have no binding; bound handles do). `bindings` is the credential +// binding descriptors (handle -> credentialId, plus the consumer allowed to use +// it). Never holds a secret. export type WorkflowRunCredentialRefs = { credentialIds: string[]; bindings: { handle: string; credentialId: string; consumer: string }[]; @@ -98,13 +98,28 @@ export const workflowRun = pgTable( kernelId: text("kernel_id"), modelPreferences: jsonb("model_preferences"), // The non-secret shape of the credential delivery this deployment resolved - // at deploy: the credentialIds it references and the tool binding + // at deploy: the credentialIds it references and the credential binding // descriptors. Set on a deployment anchor run; null for runs that carry no // credential delivery. Secrets are NEVER stored here -- only ids and // descriptors -- so the reconnect resync can re-resolve each id's CURRENT // material from the credential table (dropping a revoked or deleted id, // picking up a rotated secret) and reconcile the reconnecting child. credentialRefs: jsonb("credential_refs").$type(), + lifecyclePolicy: jsonb("lifecycle_policy"), + expiresAt: timestamp("expires_at"), + cancellationRequestedAt: timestamp("cancellation_requested_at"), + cancellationReason: text("cancellation_reason"), + cancellationDeadline: timestamp("cancellation_deadline"), + capacityReleaseAt: timestamp("capacity_release_at"), + // When unrecoverable capacity loss happened while accepted history could + // still settle some runs. Set on the anchor instead of failing its live + // runs; they are failed at this time once that history is reconciled. + infrastructureFailedAt: timestamp("infrastructure_failed_at"), + // Why the Hub failed this run. Set on the anchor when the Hub decides to + // fail the deployment, deferred or not, then on every live run it fails, + // and kept after the allocation that also records it is released. + failureCode: text("failure_code"), + failureMessage: text("failure_message"), createdAt: timestamp("created_at").notNull().defaultNow(), // Nullable: a run has no end time until it reaches a terminal state. endedAt: timestamp("ended_at"), @@ -120,6 +135,24 @@ export const workflowRun = pgTable( uniqueIndex("workflow_run_address_idx") .on(t.address) .where(sql`${t.address} is not null`), + // The lifecycle sweep runs every second and runs are never deleted, so its + // lookups use these partial indexes instead of walking every run. + index("workflow_run_anchor_idx") + .on(t.id) + .where(sql`${t.id} = ${t.anchorRunId}`), + index("workflow_run_live_anchor_idx") + .on(t.id) + .where( + sql`${t.id} = ${t.anchorRunId} and ${t.status} in ('deployed', 'running')`, + ), + index("workflow_run_infrastructure_failed_idx") + .on(t.id) + .where(sql`${t.infrastructureFailedAt} is not null`), + // Deleting a sidecar nulls this column in every run that names it, so + // without an index each delete walks every run. + index("workflow_run_sidecar_idx") + .on(t.sidecarId) + .where(sql`${t.sidecarId} is not null`), ], ); diff --git a/vendor/intx/db/src/sidecar-allocation-store.ts b/vendor/intx/db/src/sidecar-allocation-store.ts index d95d594493..dc86139849 100644 --- a/vendor/intx/db/src/sidecar-allocation-store.ts +++ b/vendor/intx/db/src/sidecar-allocation-store.ts @@ -5,11 +5,13 @@ import { eq, gt, inArray, + isNotNull, isNull, lte, notInArray, or, sql, + type SQL, } from "drizzle-orm"; import { @@ -18,7 +20,20 @@ import { } from "@intx/types"; import type { DB, DBExecutor } from "./client"; +import { createWorkflowPendingProjectionStore } from "./workflow-pending-projection-store"; +import { + assertSidecarHasRoom, + assertSidecarReusable, + type AllocationConnectionCheck, + lockSidecars, + type ProvisionerSidecarRetirement, + retireProvisionerSidecar, + retireSidecar, + type SidecarAllocationStoreOptions, + type SidecarProvisionerBinding, +} from "./sidecar-reuse"; import { createWorkflowRunDispatchStore } from "./workflow-run-dispatch-store"; +import { canExecuteWorkflowRun } from "./workflow-lifecycle-policy"; import { isLiveWorkflowRunStatus, liveWorkflowRunStatuses, @@ -36,7 +51,7 @@ type SidecarAllocationRow = typeof sidecarAllocation.$inferSelect; const SidecarAllocationStatusValidator = type.enumerated( ...sidecarAllocationStatuses, ); -const SidecarProvisionerApiVersion = type("1"); +const SidecarProvisionerApiVersion = type("2"); const activeStatuses = [ "pending", @@ -46,14 +61,45 @@ const activeStatuses = [ "releasing", ] as const; +// Why the Hub fails a deployment whose sidecar reported it stopped. +export const SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE = + "sidecar_deployment_stopped"; +export const SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE = + "sidecar_cleanup_unconfirmed"; +export const SIDECAR_PROVISIONER_RELEASE_FAILURE_CODE = + "sidecar_provisioner_release_failed"; +export const SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE = + "sidecar_cleanup_retry_exhausted"; +export const SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE = + "sidecar_cleanup_disconnect_timeout"; +export const SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE = + "sidecar_cleanup_preparation_timeout"; +// Records that a person, not cleanup, confirmed the copy and provider hold are gone. +export const SIDECAR_CLEANUP_OPERATOR_RELEASED_FAILURE_CODE = + "sidecar_cleanup_operator_released"; +// Codes that stopped cleanup itself. A completed release clears them, so an +// allocation keeps a workflow failure only if failed cleanup never replaced it; +// the runs record their own failure either way. +const CLEANUP_STOP_FAILURE_CODES: readonly string[] = [ + SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE, +]; + export type SidecarAllocation = { readonly id: string; readonly anchorRunId: string; readonly tenantId: string; readonly provisionerId: string; - readonly provisionerApiVersion: 1; + readonly provisionerApiVersion: 2; readonly provisionerBindingFingerprint: string; readonly sidecarId?: string; + /** + * The digest of the adapter manifest that the Hub sent, or will send, in + * the ensure for the current generation. For a row with a sidecar, it is + * the manifest that the sidecar booted with. + */ + readonly adapterManifestDigest: string; readonly status: SidecarAllocationStatus; readonly generation: number; readonly ensureAcceptedGeneration?: number; @@ -63,9 +109,19 @@ export type SidecarAllocation = { readonly reconciliationLeaseExpiresAt?: Date; /** Outstanding deploy attempt, retained after its reconciliation lease ends. */ readonly initializationLeaseId?: string; + /** Sidecar identities bound in the current provisioning sequence. */ readonly ensureAttempts: number; readonly destroyAttempts: number; + /** Confirmed removal for this cleanup generation, retained across retries. */ + readonly deploymentCleanupConfirmed: boolean; + /** Confirmed provider release for this cleanup generation, retained across retries. */ + readonly providerHoldReleased: boolean; + /** Also bounds waiting for a cleanup connection. */ readonly connectDeadline?: Date; + /** How long the sidecar may stay disconnected before the Hub fails it. */ + readonly maxDisconnectedMs: number; + /** When the first deploy first failed before its deploy frame was sent. */ + readonly firstDeployFailedAt?: Date; readonly failureCode?: string; readonly failureMessage?: string; readonly createdAt: Date; @@ -77,8 +133,10 @@ export type CreatePendingSidecarAllocationArgs = { readonly anchorRunId: string; readonly tenantId: string; readonly provisionerId: string; - readonly provisionerApiVersion: 1; + readonly provisionerApiVersion: 2; readonly provisionerBindingFingerprint: string; + readonly adapterManifestDigest: string; + readonly maxDisconnectedMs: number; readonly now?: Date; }; @@ -111,6 +169,8 @@ export type BindInitialSidecarArgs = { readonly expectedGeneration: number; readonly sidecarId: string; readonly tokenHashSha256: Uint8Array; + /** The adapter manifest the ensure for this sidecar carries. */ + readonly adapterManifestDigest: string; readonly connectDeadline: Date; readonly expectedLeaseId?: string; readonly now?: Date; @@ -121,6 +181,8 @@ export type BindReplacementSidecarArgs = { readonly generation: number; readonly sidecarId: string; readonly tokenHashSha256: Uint8Array; + /** The adapter manifest the ensure for this sidecar carries. */ + readonly adapterManifestDigest: string; readonly connectDeadline: Date; readonly expectedLeaseId?: string; readonly now?: Date; @@ -129,6 +191,11 @@ export type BindReplacementSidecarArgs = { export type MarkSidecarAllocatedArgs = { readonly allocationId: string; readonly generation: number; + /** + * An existing sidecar the provisioner placed this generation on instead of + * starting the one bound to it. The unused bound identity is deleted. + */ + readonly sidecarId?: string; readonly externalRef?: string; readonly expectedLeaseId?: string; readonly now?: Date; @@ -139,14 +206,24 @@ export type ScheduleSidecarAllocationRetryArgs = { readonly expectedStatus: (typeof activeStatuses)[number]; readonly expectedGeneration: number; readonly nextAttemptAt: Date; + /** Minimum wait measured on the database clock, despite Hub clock skew. */ + readonly minimumDelayMs?: number; readonly expectedLeaseId?: string; - readonly attempt?: "ensure" | "destroy"; - readonly failure?: { - readonly code: string; - readonly message: string; - }; + readonly firstDeployFailedAt?: Date; readonly now?: Date; -}; +} & ( + | { + readonly attempt: "destroy"; + readonly failure?: never; + } + | { + readonly attempt?: never; + readonly failure?: { + readonly code: string; + readonly message: string; + }; + } +); export type BeginSidecarReplacementArgs = { readonly allocationId: string; @@ -156,6 +233,8 @@ export type BeginSidecarReplacementArgs = { readonly onlyIfInitializationIncomplete?: boolean; readonly expectedInitializationLeaseId?: string; readonly nextAttemptAt: Date; + /** Schedule a new sequence when the previous worker finished initialization. */ + readonly firstAttemptAt: Date; readonly failureCode: string; readonly failureMessage: string; readonly now?: Date; @@ -175,17 +254,35 @@ export type BeginSidecarReleaseArgs = { readonly now?: Date; }; -export type BeginUnrecoverableSidecarReleaseArgs = { +export type FailStoppedSidecarDeploymentArgs = { readonly allocationId: string; readonly expectedGeneration: number; readonly expectedLeaseId: string; - readonly onlyIfInitializationIncomplete?: boolean; - readonly expectedInitializationLeaseId?: string; readonly failureCode: string; readonly failureMessage: string; readonly now?: Date; }; +export type BeginUnrecoverableSidecarReleaseArgs = { + readonly allocationId: string; + readonly expectedGeneration: number; + readonly expectedLeaseId: string; + readonly failureCode: string; + readonly failureMessage: string; + readonly now?: Date; +} & ( + | { + readonly expectedStatus: "allocated" | "provisioning"; + readonly onlyIfInitializationIncomplete?: boolean; + readonly expectedInitializationLeaseId?: string; + } + | { + readonly expectedStatus: "replacing"; + readonly onlyIfInitializationIncomplete?: never; + readonly expectedInitializationLeaseId?: never; + } +); + export type MarkSidecarReleasedArgs = { readonly allocationId: string; readonly generation: number; @@ -193,30 +290,46 @@ export type MarkSidecarReleasedArgs = { readonly now?: Date; }; -export type MarkSidecarConnectionReadyArgs = { - readonly allocationId: string; +export type ReleaseStoppedSidecarCleanupArgs = { + readonly tenantId: string; + readonly anchorRunId: string; readonly generation: number; - readonly expectedLeaseId?: string; + /** The principal that confirmed the copy and provider hold are gone. */ + readonly releasedBy: string; readonly now?: Date; }; -export type MarkSidecarConnectionLostArgs = { +export type ReleaseStoppedSidecarCleanupResult = + | { readonly kind: "released"; readonly allocation: SidecarAllocation } + | { readonly kind: "already_released" } + | { readonly kind: "not_found" } + | { readonly kind: "not_stopped" } + | { readonly kind: "generation_mismatch" }; + +export type MarkSidecarConnectionReadyArgs = { readonly allocationId: string; readonly generation: number; - readonly connectDeadline: Date; + readonly expectedLeaseId?: string; readonly now?: Date; }; -export type ScheduleSidecarReconnectIfUnscheduledArgs = { +export type MarkSidecarConnectionLostArgs = { readonly allocationId: string; readonly generation: number; - readonly connectDeadline: Date; readonly now?: Date; + /** `now` plus the window a sidecar gets to connect the first time. */ + readonly firstConnectDeadline: Date; }; +export type ScheduleSidecarReconnectAfterHubStartArgs = + MarkSidecarConnectionLostArgs; + +export type ScheduleSidecarReconnectIfUnscheduledArgs = + MarkSidecarConnectionLostArgs; + export type FailSidecarAllocationArgs = { readonly allocationId: string; - readonly expectedStatus: "pending" | "provisioning"; + readonly expectedStatus: "pending" | "provisioning" | "replacing"; readonly expectedGeneration: number; readonly code: string; readonly message: string; @@ -227,7 +340,10 @@ export type FailSidecarAllocationArgs = { export type MarkSidecarDestroyFailedArgs = Omit< FailSidecarAllocationArgs, "expectedStatus" ->; +> & { + /** A connection-wait deadline is not a failed cleanup attempt. */ + readonly countAttempt?: boolean; +}; type InitializationArgs = { readonly allocationId: string; @@ -238,6 +354,18 @@ type InitializationArgs = { readonly signal: AbortSignal; }; +function allocationBinding( + row: SidecarAllocationRow, +): SidecarProvisionerBinding { + return { + provisionerId: row.provisionerId, + provisionerApiVersion: SidecarProvisionerApiVersion.assert( + row.provisionerApiVersion, + ), + provisionerBindingFingerprint: row.provisionerBindingFingerprint, + }; +} + function parseSidecarAllocationRow( row: SidecarAllocationRow, ): SidecarAllocation { @@ -251,6 +379,7 @@ function parseSidecarAllocationRow( ), provisionerBindingFingerprint: row.provisionerBindingFingerprint, ...(row.sidecarId !== null ? { sidecarId: row.sidecarId } : {}), + adapterManifestDigest: row.adapterManifestDigest, status: SidecarAllocationStatusValidator.assert(row.status), generation: row.generation, ...(row.ensureAcceptedGeneration !== null @@ -269,9 +398,15 @@ function parseSidecarAllocationRow( : {}), ensureAttempts: row.ensureAttempts, destroyAttempts: row.destroyAttempts, + deploymentCleanupConfirmed: row.deploymentCleanupConfirmed, + providerHoldReleased: row.providerHoldReleased, ...(row.connectDeadline !== null ? { connectDeadline: row.connectDeadline } : {}), + maxDisconnectedMs: row.maxDisconnectedMs, + ...(row.firstDeployFailedAt !== null + ? { firstDeployFailedAt: row.firstDeployFailedAt } + : {}), ...(row.failureCode !== null ? { failureCode: row.failureCode } : {}), ...(row.failureMessage !== null ? { failureMessage: row.failureMessage } @@ -285,6 +420,50 @@ function databaseTimestamp(override?: Date) { return override ?? sql`now()`; } +function deadlineTimestamp(now?: Date) { + return now === undefined + ? sql`now()` + : sql`${sql.param(now, sidecarAllocation.connectDeadline)}::timestamp`; +} + +// When a sidecar that lost its connection at `now` has stayed away for as +// long as its allocation allows. +function disconnectDeadline(now?: Date) { + return sql`${deadlineTimestamp(now)} + ${sidecarAllocation.maxDisconnectedMs} * interval '1 millisecond'`; +} + +// The disconnect limit covers a deployment once its first deploy completed; +// until then a lost sidecar gets only the first-connect window. Work on a +// sidecar whose provisioner ended its reconnect window gets no window at all. +function reconnectDeadline( + args: MarkSidecarConnectionLostArgs, + beforeFirstDeploy: SQL, +) { + return sql`case when exists (select 1 from ${sidecar} where ${sidecar.id} = ${sidecarAllocation.sidecarId} and ${sidecar.reconnectWindowEndedAt} is not null) then ${deadlineTimestamp(args.now)} when exists (select 1 from ${workflowRun} where ${workflowRun.id} = ${sidecarAllocation.anchorRunId} and ${workflowRun.publicKey} is not null) then ${disconnectDeadline(args.now)} else ${beforeFirstDeploy} end`; +} + +function firstConnectDeadline(args: MarkSidecarConnectionLostArgs) { + return sql`${sql.param(args.firstConnectDeadline, sidecarAllocation.connectDeadline)}::timestamp`; +} + +// A report that ends the sidecar's reconnect window holds its row until it +// commits. Waiting for it lets the deadline written next see that report. +async function shareAllocationSidecar(tx: DBExecutor, allocationId: string) { + await tx + .select({ id: sidecar.id }) + .from(sidecar) + .where( + inArray( + sidecar.id, + tx + .select({ sidecarId: sidecarAllocation.sidecarId }) + .from(sidecarAllocation) + .where(eq(sidecarAllocation.id, allocationId)), + ), + ) + .for("share"); +} + function leaseCondition(expectedLeaseId?: string) { return expectedLeaseId === undefined ? [] @@ -297,8 +476,52 @@ function leaseCondition(expectedLeaseId?: string) { ]; } -export function createSidecarAllocationStore(db: DBHandle) { +// Stores used only for lifecycle updates cannot accidentally place work without +// the connection's current retained-inventory proof. +export function createSidecarAllocationStore( + db: DBHandle, +): Omit; +export function createSidecarAllocationStore( + db: DBHandle, + options: Required, +): SidecarAllocationStore; +export function createSidecarAllocationStore( + db: DBHandle, + options?: Required, +) { + return buildSidecarAllocationStore(db, options); +} + +function buildSidecarAllocationStore( + db: DBHandle, + options: SidecarAllocationStoreOptions = {}, +) { const workflowRunDispatchStore = createWorkflowRunDispatchStore(db); + const pendingProjections = createWorkflowPendingProjectionStore(db); + + async function lockAllocationWithSidecars( + tx: DBExecutor, + condition: SQL | undefined, + destinationSidecarId?: string, + ) { + const [candidate] = await tx + .select({ sidecarId: sidecarAllocation.sidecarId }) + .from(sidecarAllocation) + .where(condition); + if (candidate === undefined) return undefined; + await lockSidecars( + tx, + [candidate.sidecarId, destinationSidecarId].filter((id) => id != null), + ); + // A move changes status or generation, so rechecking the condition rejects + // work whose binding moved while the sidecar lock was awaited. + const [allocation] = await tx + .select() + .from(sidecarAllocation) + .where(condition) + .for("update"); + return allocation; + } function initializationConditions( args: InitializationArgs, @@ -325,7 +548,7 @@ export function createSidecarAllocationStore(db: DBHandle) { readonly publicKey: string; readonly credentialRefs?: WorkflowRunCredentialRefs; }, - ): Promise<{ previousPublicKey: string | null } | null> { + ): Promise { args.signal.throwIfAborted(); return db.transaction(async (tx) => { const condition = initializationConditions( @@ -338,12 +561,16 @@ export function createSidecarAllocationStore(db: DBHandle) { .where(condition) .for("update"); args.signal.throwIfAborted(); - if (allocation === undefined) return null; - let previousPublicKey: string | null = null; - if (completion === undefined) { - const [previous] = await tx - .select({ publicKey: workflowRun.publicKey }) - .from(workflowRun) + if (allocation === undefined) return false; + if (completion !== undefined) { + const [anchor] = await tx + .update(workflowRun) + .set({ + publicKey: completion.publicKey, + ...(completion.credentialRefs !== undefined + ? { credentialRefs: completion.credentialRefs } + : {}), + }) .where( and( eq(workflowRun.id, args.anchorRunId), @@ -351,42 +578,27 @@ export function createSidecarAllocationStore(db: DBHandle) { eq(workflowRun.tenantId, args.tenantId), ), ) - .for("update"); - if (previous === undefined) + .returning({ id: workflowRun.id }); + if (anchor === undefined) throw new Error("Initialization anchor is missing"); - previousPublicKey = previous.publicKey; } - const [anchor] = await tx - .update(workflowRun) - .set({ - publicKey: completion?.publicKey ?? null, - ...(completion?.credentialRefs !== undefined - ? { credentialRefs: completion.credentialRefs } - : {}), - }) - .where( - and( - eq(workflowRun.id, args.anchorRunId), - eq(workflowRun.anchorRunId, args.anchorRunId), - eq(workflowRun.tenantId, args.tenantId), - ), - ) - .returning({ id: workflowRun.id }); - if (anchor === undefined) - throw new Error("Initialization anchor is missing"); - // Check the lease again after waiting on the anchor row. Both writes roll - // back if ownership expired while the transaction held the allocation lock. + // Check the lease again: ownership can expire while the transaction holds + // the allocation lock, such as while a completion waits on the anchor + // row, and the completion's anchor write then rolls back with this one. const [updated] = await tx .update(sidecarAllocation) .set({ initializationLeaseId: completion === undefined ? args.leaseId : null, + // Completion proves the first connection succeeded. A later disconnect + // acquires this same row lock and starts its own deadline. + ...(completion === undefined ? {} : { connectDeadline: null }), }) .where(condition) .returning({ id: sidecarAllocation.id }); args.signal.throwIfAborted(); if (updated === undefined) throw new Error("Initialization lease expired"); - return { previousPublicKey }; + return true; }); } @@ -424,6 +636,8 @@ export function createSidecarAllocationStore(db: DBHandle) { args: { sidecarId: string; tokenHashSha256: Uint8Array; + binding: SidecarProvisionerBinding; + adapterManifestDigest: string; now: Date | ReturnType; }, ): Promise { @@ -431,29 +645,147 @@ export function createSidecarAllocationStore(db: DBHandle) { id: args.sidecarId, url: null, tokenHashSha256: args.tokenHashSha256, + ...args.binding, + adapterManifestDigest: args.adapterManifestDigest, status: "offline", createdAt: args.now, updatedAt: args.now, }); } + // Lifecycle settles a pending cancellation and its dispatches together after + // accepted history is recovered. The release already fences delivery. + async function settleReleasedDispatches( + tx: DBExecutor, + anchorRunId: string, + now: Date | SQL, + ): Promise { + if (await hasPendingCancellation(tx, anchorRunId)) return; + await workflowRunDispatchStore.abandonUnsettled( + anchorRunId, + "workflow_capacity_released", + "Workflow capacity was released before delivery could be confirmed", + now, + tx, + ); + } + + async function hasPendingCancellation( + tx: DBExecutor, + anchorRunId: string, + ): Promise { + const cancellation = await tx.query.workflowRun.findFirst({ + columns: { id: true }, + where: and( + eq(workflowRun.id, anchorRunId), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + isNotNull(workflowRun.cancellationRequestedAt), + ), + }); + return cancellation !== undefined; + } + async function failRunningRuns( tx: DBExecutor, anchorRunId: string, now: Date | ReturnType, + failure: { readonly code: string; readonly message: string }, + ): Promise { + // The first failure of a live deployment is the one its runs failed for. + await tx + .update(workflowRun) + .set({ failureCode: failure.code, failureMessage: failure.message }) + .where( + and( + eq(workflowRun.id, anchorRunId), + isNull(workflowRun.failureCode), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + ), + ); + // A pending projection means accepted history may already hold some of + // these runs' outcomes. Record when the deployment was lost and fail the + // runs once that history is reconciled. + if (await pendingProjections.hasAny(anchorRunId, tx)) { + await tx + .update(workflowRun) + .set({ infrastructureFailedAt: now }) + .where( + and( + eq(workflowRun.id, anchorRunId), + isNull(workflowRun.infrastructureFailedAt), + ), + ); + return; + } + const failed = await failLiveRuns(tx, anchorRunId, now, failure); + // A failure deferred earlier has nothing left to fail, and applying it + // later would drop the reason these runs failed for. + await settleDeferredFailure(tx, anchorRunId, failed); + } + + // History reconciled meanwhile may have settled the anchor, and then the + // deferred failure is not why it ended. + async function settleDeferredFailure( + tx: DBExecutor, + anchorRunId: string, + failed: readonly string[], ): Promise { + await tx + .update(workflowRun) + .set({ + infrastructureFailedAt: null, + ...(failed.includes(anchorRunId) + ? {} + : { failureCode: null, failureMessage: null }), + }) + .where( + and( + eq(workflowRun.id, anchorRunId), + isNotNull(workflowRun.infrastructureFailedAt), + ), + ); + } + + async function failLiveRuns( + tx: DBExecutor, + anchorRunId: string, + now: Date | ReturnType, + failure?: { readonly code: string; readonly message: string }, + ): Promise { + // Every run failed here carries the deployment's reason: the one its anchor + // holds, which a deferred failure kept there, or else the one given. + const [anchor] = await tx + .select({ + code: workflowRun.failureCode, + message: workflowRun.failureMessage, + }) + .from(workflowRun) + .where(eq(workflowRun.id, anchorRunId)); + const reason = + anchor?.code != null + ? { code: anchor.code, message: anchor.message } + : failure; // Fail every live run anchored here -- both "running" runs and a "deployed" // anchor torn down before its first trigger -- so a release settles them. const failedRuns = await tx .update(workflowRun) - .set({ status: "failed", endedAt: now }) + .set({ + status: "failed", + endedAt: now, + ...(reason !== undefined + ? { + failureCode: sql`coalesce(${workflowRun.failureCode}, ${reason.code})`, + failureMessage: sql`coalesce(${workflowRun.failureMessage}, ${reason.message})`, + } + : {}), + }) .where( and( eq(workflowRun.anchorRunId, anchorRunId), inArray(workflowRun.status, [...liveWorkflowRunStatuses]), ), ) - .returning({ principalId: workflowRun.principalId }); + .returning({ id: workflowRun.id, principalId: workflowRun.principalId }); const principalIds = failedRuns.flatMap(({ principalId }) => principalId === null ? [] : [principalId], ); @@ -463,6 +795,7 @@ export function createSidecarAllocationStore(db: DBHandle) { .set({ status: "deactivated", updatedAt: now }) .where(inArray(principal.id, principalIds)); } + return failedRuns.map(({ id }) => id); } async function beginRelease( @@ -475,17 +808,30 @@ export function createSidecarAllocationStore(db: DBHandle) { .set({ status: "releasing", generation: args.expectedGeneration + 1, + destroyAttempts: 0, + // Replacement clears ensure acceptance before removing the old copy. + // Preserve that cleanup obligation when release interrupts replacement. + deploymentCleanupConfirmed: sql`case when ${sidecarAllocation.status} = 'replacing' then ${sidecarAllocation.deploymentCleanupConfirmed} else ${sidecarAllocation.ensureAcceptedGeneration} is null end`, + // Abandoning replacement does not create another provider hold. + providerHoldReleased: sql`case when ${sidecarAllocation.status} = 'replacing' then ${sidecarAllocation.providerHoldReleased} else false end`, initializationLeaseId: null, nextAttemptAt: now, reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, - connectDeadline: null, - ...(args.failureCode !== undefined - ? { failureCode: args.failureCode } - : {}), - ...(args.failureMessage !== undefined - ? { failureMessage: args.failureMessage } - : {}), + // Preserve the disconnect window for an initialized deployment or + // replacement cleanup, which has already cleared the anchor's key. + // A first-connect deadline does not bound subsequent cleanup waiting. + connectDeadline: sql`case when ${sidecarAllocation.status} = 'replacing' or exists (select 1 from ${workflowRun} where ${workflowRun.id} = ${sidecarAllocation.anchorRunId} and ${workflowRun.publicKey} is not null) then ${sidecarAllocation.connectDeadline} else null end`, + // Placement and initialization retry diagnostics are not release causes. + // Preserve a recorded workflow failure, or the reason supplied by release. + failureCode: + args.failureCode ?? + sql`(select ${workflowRun.failureCode} from ${workflowRun} where ${workflowRun.id} = ${sidecarAllocation.anchorRunId})`, + failureMessage: + args.failureMessage ?? + (args.failureCode === undefined + ? sql`(select ${workflowRun.failureMessage} from ${workflowRun} where ${workflowRun.id} = ${sidecarAllocation.anchorRunId})` + : null), updatedAt: now, }) .where( @@ -494,6 +840,21 @@ export function createSidecarAllocationStore(db: DBHandle) { eq(sidecarAllocation.status, args.expectedStatus), eq(sidecarAllocation.generation, args.expectedGeneration), ...leaseCondition(args.expectedLeaseId), + // An ordinary caller cannot interrupt an active reconciler. A + // definitive initialization refusal instead owns the exact attempt + // marker, which outlives a disconnect's reconciliation lease. + ...(args.expectedLeaseId === undefined && + args.expectedInitializationLeaseId === undefined + ? [ + or( + isNull(sidecarAllocation.reconciliationLeaseExpiresAt), + lte( + sidecarAllocation.reconciliationLeaseExpiresAt, + sql`clock_timestamp()`, + ), + ), + ] + : []), ...(args.expectedInitializationLeaseId !== undefined ? [ eq( @@ -508,6 +869,118 @@ export function createSidecarAllocationStore(db: DBHandle) { return updated === undefined ? null : parseSidecarAllocationRow(updated); } + async function releaseFailedAllocation( + tx: DBExecutor, + args: BeginSidecarReleaseArgs & { + readonly failureCode: string; + readonly failureMessage: string; + }, + ): Promise { + const now = databaseTimestamp(args.now); + let releasing: SidecarAllocation | null; + if (args.expectedStatus === "replacing") { + // This generation already fences the old copy for cleanup. Giving up + // replacement must preserve its proofs, attempts and disconnect window. + const [updated] = await tx + .update(sidecarAllocation) + .set({ + status: "releasing", + failureCode: args.failureCode, + failureMessage: args.failureMessage, + nextAttemptAt: now, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + updatedAt: now, + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, args.expectedStatus), + eq(sidecarAllocation.generation, args.expectedGeneration), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .returning(); + releasing = + updated === undefined ? null : parseSidecarAllocationRow(updated); + } else { + releasing = await beginRelease(args, tx); + } + if (releasing === null) return null; + // Confirmed removal leaves the requested cancellation and its dispatches + // to lifecycle after accepted-history recovery. + if ( + releasing.deploymentCleanupConfirmed && + (await hasPendingCancellation(tx, releasing.anchorRunId)) + ) + return releasing; + await failRunningRuns(tx, releasing.anchorRunId, now, { + code: args.failureCode, + message: args.failureMessage, + }); + await workflowRunDispatchStore.abandonUnsettled( + releasing.anchorRunId, + args.failureCode, + args.failureMessage, + now, + tx, + ); + return releasing; + } + + async function insertAdopted( + tx: DBExecutor, + args: CreateAdoptedSidecarAllocationArgs, + ): Promise { + // Adopting a probe places a deployment on its sidecar, so it is held to + // the same reuse and room checks as a placement through `markAllocated`. + await assertSidecarReusable(tx, { + sidecarId: args.sidecarId, + binding: { + provisionerId: args.provisionerId, + provisionerApiVersion: args.provisionerApiVersion, + provisionerBindingFingerprint: args.provisionerBindingFingerprint, + }, + adapterManifestDigest: args.adapterManifestDigest, + }); + await assertSidecarHasRoom(tx, { + ...options, + sidecarId: args.sidecarId, + placing: { allocationId: args.id }, + }); + const createdAt = databaseTimestamp(args.now); + const [inserted] = await tx + .insert(sidecarAllocation) + .values({ + id: args.id, + anchorRunId: args.anchorRunId, + tenantId: args.tenantId, + provisionerId: args.provisionerId, + provisionerApiVersion: args.provisionerApiVersion, + provisionerBindingFingerprint: args.provisionerBindingFingerprint, + sidecarId: args.sidecarId, + adapterManifestDigest: args.adapterManifestDigest, + status: "allocated", + generation: args.generation, + ensureAcceptedGeneration: args.generation, + ...(args.externalRef !== undefined + ? { externalRef: args.externalRef } + : {}), + connectDeadline: args.connectDeadline, + maxDisconnectedMs: args.maxDisconnectedMs, + nextAttemptAt: createdAt, + createdAt, + updatedAt: createdAt, + }) + .returning(); + if (inserted === undefined) { + throw new Error( + `sidecarAllocationStore.createAdopted: insert returned no row for ${args.id}`, + ); + } + return parseSidecarAllocationRow(inserted); + } + return { beginInitialization(args: InitializationArgs) { return writeInitialization(args); @@ -519,11 +992,11 @@ export function createSidecarAllocationStore(db: DBHandle) { readonly credentialRefs?: WorkflowRunCredentialRefs; }, ): Promise { - return (await writeInitialization(args, args)) !== null; + return writeInitialization(args, args); }, async clearUnsentInitialization( - args: InitializationArgs & { readonly previousPublicKey: string | null }, + args: InitializationArgs, ): Promise { // No signal gate: the sole caller invokes this only for a proven-unsent // frame, which is reachable precisely when the attempt was cancelled. @@ -531,40 +1004,44 @@ export function createSidecarAllocationStore(db: DBHandle) { // attempt began, and generation/status equality proves the allocation // did not move on. The reconciliation lease is deliberately not // required: a disconnect nulls the lease while leaving this attempt's - // marker behind, and only this attempt's own clear may remove it. Restore - // its previous key in the same transaction so absence of the marker cannot - // expose a live workflow as keyless or certify a different attempt. + // marker behind, and only this attempt's own clear may remove it. + const [updated] = await db + .update(sidecarAllocation) + .set({ initializationLeaseId: null }) + .where( + initializationConditions(args, args.leaseId, { + requireCurrentLease: false, + }), + ) + .returning({ id: sidecarAllocation.id }); + return updated !== undefined; + }, + + async rejectInitialization( + args: InitializationArgs & { readonly message: string }, + ): Promise { + // Like an unsent rollback, a proven refusal remains valid after lease + // cancellation. Lock the attempt marker before changing either the + // allocation or its runs; a completion or newer generation wins otherwise. return db.transaction(async (tx) => { - const condition = initializationConditions(args, args.leaseId, { - requireCurrentLease: false, - }); const [allocation] = await tx .select({ id: sidecarAllocation.id }) .from(sidecarAllocation) - .where(condition) - .for("update"); - if (allocation === undefined) return false; - const [anchor] = await tx - .update(workflowRun) - .set({ publicKey: args.previousPublicKey }) .where( - and( - eq(workflowRun.id, args.anchorRunId), - eq(workflowRun.anchorRunId, args.anchorRunId), - eq(workflowRun.tenantId, args.tenantId), - ), + initializationConditions(args, args.leaseId, { + requireCurrentLease: false, + }), ) - .returning({ id: workflowRun.id }); - if (anchor === undefined) - throw new Error("Initialization anchor is missing"); - const [updated] = await tx - .update(sidecarAllocation) - .set({ initializationLeaseId: null }) - .where(condition) - .returning({ id: sidecarAllocation.id }); - if (updated === undefined) - throw new Error("Initialization attempt changed before rollback"); - return true; + .for("update"); + if (allocation === undefined) return null; + return releaseFailedAllocation(tx, { + allocationId: args.allocationId, + expectedStatus: "allocated", + expectedGeneration: args.generation, + expectedInitializationLeaseId: args.leaseId, + failureCode: "sidecar_deployment_rejected", + failureMessage: args.message, + }); }); }, @@ -609,7 +1086,7 @@ export function createSidecarAllocationStore(db: DBHandle) { `sidecarAllocationStore.createPending: anchor run ${args.anchorRunId} has no launch specification`, ); } - if (args.provisionerApiVersion !== 1) { + if (args.provisionerApiVersion !== 2) { throw new Error( "sidecarAllocationStore.createPending: unsupported API version", ); @@ -624,8 +1101,10 @@ export function createSidecarAllocationStore(db: DBHandle) { provisionerId: args.provisionerId, provisionerApiVersion: args.provisionerApiVersion, provisionerBindingFingerprint: args.provisionerBindingFingerprint, + adapterManifestDigest: args.adapterManifestDigest, status: "pending", generation: 0, + maxDisconnectedMs: args.maxDisconnectedMs, nextAttemptAt: now, createdAt: now, updatedAt: now, @@ -643,35 +1122,9 @@ export function createSidecarAllocationStore(db: DBHandle) { args: CreateAdoptedSidecarAllocationArgs, tx?: DBExecutor, ): Promise { - const createdAt = databaseTimestamp(args.now); - const [inserted] = await (tx ?? db) - .insert(sidecarAllocation) - .values({ - id: args.id, - anchorRunId: args.anchorRunId, - tenantId: args.tenantId, - provisionerId: args.provisionerId, - provisionerApiVersion: args.provisionerApiVersion, - provisionerBindingFingerprint: args.provisionerBindingFingerprint, - sidecarId: args.sidecarId, - status: "allocated", - generation: args.generation, - ensureAcceptedGeneration: args.generation, - ...(args.externalRef !== undefined - ? { externalRef: args.externalRef } - : {}), - connectDeadline: args.connectDeadline, - nextAttemptAt: createdAt, - createdAt, - updatedAt: createdAt, - }) - .returning(); - if (inserted === undefined) { - throw new Error( - `sidecarAllocationStore.createAdopted: insert returned no row for ${args.id}`, - ); - } - return parseSidecarAllocationRow(inserted); + return tx === undefined + ? db.transaction((inner) => insertAdopted(inner, args)) + : insertAdopted(tx, args); }, async bindInitialSidecar( @@ -702,14 +1155,20 @@ export function createSidecarAllocationStore(db: DBHandle) { await insertSidecarIdentity(tx, { sidecarId: args.sidecarId, tokenHashSha256: args.tokenHashSha256, + binding: allocationBinding(allocation), + adapterManifestDigest: args.adapterManifestDigest, now, }); const [updated] = await tx .update(sidecarAllocation) .set({ sidecarId: args.sidecarId, + adapterManifestDigest: args.adapterManifestDigest, status: "provisioning", generation: args.expectedGeneration + 1, + ensureAttempts: sql`${sidecarAllocation.ensureAttempts} + 1`, + deploymentCleanupConfirmed: false, + providerHoldReleased: false, ensureAcceptedGeneration: null, externalRef: null, connectDeadline: args.connectDeadline, @@ -740,37 +1199,35 @@ export function createSidecarAllocationStore(db: DBHandle) { args: BindReplacementSidecarArgs, ): Promise { return db.transaction(async (tx) => { - const [allocation] = await tx - .select() - .from(sidecarAllocation) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - ...leaseCondition(args.expectedLeaseId), - ), - ) - .limit(1) - .for("update"); - if ( - allocation === undefined || - allocation.status !== "replacing" || - allocation.generation !== args.generation || - (args.expectedLeaseId !== undefined && - allocation.reconciliationLeaseId !== args.expectedLeaseId) - ) { - return null; - } + const condition = and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "replacing"), + eq(sidecarAllocation.generation, args.generation), + eq(sidecarAllocation.deploymentCleanupConfirmed, true), + eq(sidecarAllocation.providerHoldReleased, true), + ...leaseCondition(args.expectedLeaseId), + ); + const allocation = await lockAllocationWithSidecars(tx, condition); + if (allocation === undefined) return null; const now = databaseTimestamp(args.now); await insertSidecarIdentity(tx, { sidecarId: args.sidecarId, tokenHashSha256: args.tokenHashSha256, + binding: allocationBinding(allocation), + adapterManifestDigest: args.adapterManifestDigest, now, }); const [updated] = await tx .update(sidecarAllocation) .set({ sidecarId: args.sidecarId, + adapterManifestDigest: args.adapterManifestDigest, status: "provisioning", + // A delayed cleanup hello must not match the replacement binding. + generation: args.generation + 1, + ensureAttempts: sql`${sidecarAllocation.ensureAttempts} + 1`, + deploymentCleanupConfirmed: false, + providerHoldReleased: false, ensureAcceptedGeneration: null, externalRef: null, connectDeadline: args.connectDeadline, @@ -780,14 +1237,7 @@ export function createSidecarAllocationStore(db: DBHandle) { failureMessage: null, updatedAt: now, }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, "replacing"), - eq(sidecarAllocation.generation, args.generation), - ...leaseCondition(args.expectedLeaseId), - ), - ) + .where(condition) .returning(); if (updated === undefined) { throw new Error( @@ -798,31 +1248,71 @@ export function createSidecarAllocationStore(db: DBHandle) { }); }, + /** + * Records the provisioner's acceptance of a generation. Throws + * `SidecarReuseRejectedError` when it placed the generation on a sidecar + * it cannot reuse, or `SidecarInventoryUnavailableError` while inventory is + * unknown. Either failure leaves the allocation unchanged. + */ async markAllocated( args: MarkSidecarAllocatedArgs, ): Promise { - const [updated] = await db - .update(sidecarAllocation) - .set({ - status: "allocated", - ensureAcceptedGeneration: args.generation, - externalRef: args.externalRef ?? null, - nextAttemptAt: sidecarAllocation.connectDeadline, - ensureAttempts: sql`${sidecarAllocation.ensureAttempts} + 1`, - failureCode: null, - failureMessage: null, - updatedAt: databaseTimestamp(args.now), - }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, "provisioning"), - eq(sidecarAllocation.generation, args.generation), - ...leaseCondition(args.expectedLeaseId), - ), - ) - .returning(); - return updated === undefined ? null : parseSidecarAllocationRow(updated); + return db.transaction(async (tx) => { + const condition = and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "provisioning"), + eq(sidecarAllocation.generation, args.generation), + ...leaseCondition(args.expectedLeaseId), + ); + const allocation = await lockAllocationWithSidecars( + tx, + condition, + args.sidecarId, + ); + if (allocation === undefined) return null; + const boundSidecarId = allocation.sidecarId; + const reusedSidecarId = + args.sidecarId !== undefined && args.sidecarId !== boundSidecarId + ? args.sidecarId + : undefined; + if (reusedSidecarId !== undefined) { + await assertSidecarReusable(tx, { + sidecarId: reusedSidecarId, + binding: allocationBinding(allocation), + adapterManifestDigest: allocation.adapterManifestDigest, + }); + await assertSidecarHasRoom(tx, { + ...options, + sidecarId: reusedSidecarId, + placing: { allocationId: allocation.id }, + }); + } + const [updated] = await tx + .update(sidecarAllocation) + .set({ + status: "allocated", + ...(reusedSidecarId !== undefined + ? { sidecarId: reusedSidecarId } + : {}), + ensureAcceptedGeneration: args.generation, + externalRef: args.externalRef ?? null, + nextAttemptAt: sidecarAllocation.connectDeadline, + failureCode: null, + failureMessage: null, + updatedAt: databaseTimestamp(args.now), + }) + .where(condition) + .returning(); + if (updated === undefined) { + throw new Error( + `sidecarAllocationStore.markAllocated: locked allocation ${args.allocationId} changed before update`, + ); + } + if (reusedSidecarId !== undefined && boundSidecarId !== null) { + await tx.delete(sidecar).where(eq(sidecar.id, boundSidecarId)); + } + return parseSidecarAllocationRow(updated); + }); }, async scheduleRetry( @@ -831,12 +1321,12 @@ export function createSidecarAllocationStore(db: DBHandle) { const [updated] = await db .update(sidecarAllocation) .set({ - nextAttemptAt: args.nextAttemptAt, + nextAttemptAt: + args.minimumDelayMs === undefined + ? args.nextAttemptAt + : sql`greatest(${sql.param(args.nextAttemptAt, sidecarAllocation.nextAttemptAt)}, clock_timestamp() + (${args.minimumDelayMs} * interval '1 millisecond'))`, reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, - ...(args.attempt === "ensure" - ? { ensureAttempts: sql`${sidecarAllocation.ensureAttempts} + 1` } - : {}), ...(args.attempt === "destroy" ? { destroyAttempts: sql`${sidecarAllocation.destroyAttempts} + 1` } : {}), @@ -846,6 +1336,9 @@ export function createSidecarAllocationStore(db: DBHandle) { failureMessage: args.failure.message, } : {}), + ...(args.firstDeployFailedAt !== undefined + ? { firstDeployFailedAt: args.firstDeployFailedAt } + : {}), updatedAt: databaseTimestamp(args.now), }) .where( @@ -860,24 +1353,144 @@ export function createSidecarAllocationStore(db: DBHandle) { return updated === undefined ? null : parseSidecarAllocationRow(updated); }, + async confirmDeploymentCleanup(args: { + readonly allocationId: string; + readonly generation: number; + readonly expectedLeaseId: string; + readonly now?: Date; + }): Promise { + const [updated] = await db + .update(sidecarAllocation) + .set({ + deploymentCleanupConfirmed: true, + connectDeadline: null, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.generation, args.generation), + inArray(sidecarAllocation.status, ["releasing", "replacing"]), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .returning({ id: sidecarAllocation.id }); + return updated !== undefined; + }, + + /** Records that no sidecar runs with `sidecarId`; see `retireSidecar`. */ + async retireSidecar(args: { + readonly sidecarId: string; + readonly now?: Date; + }): Promise { + await db.transaction((tx) => + retireSidecar(tx, { + sidecarId: args.sidecarId, + now: databaseTimestamp(args.now), + }), + ); + }, + + /** See `retireProvisionerSidecar`. */ + async retireProvisionerSidecar(args: { + readonly sidecarId: string; + readonly binding: SidecarProvisionerBinding; + readonly endReconnectWindow: boolean; + readonly isConnected: AllocationConnectionCheck; + readonly now?: Date; + }): Promise { + return db.transaction((tx) => + retireProvisionerSidecar(tx, { + sidecarId: args.sidecarId, + binding: args.binding, + endReconnectWindow: args.endReconnectWindow, + isConnected: args.isConnected, + now: databaseTimestamp(args.now), + }), + ); + }, + + async confirmProviderRelease(args: { + readonly allocationId: string; + readonly generation: number; + readonly expectedLeaseId: string; + readonly now?: Date; + }): Promise { + const [updated] = await db + .update(sidecarAllocation) + .set({ + providerHoldReleased: true, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.generation, args.generation), + inArray(sidecarAllocation.status, ["releasing", "replacing"]), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .returning({ id: sidecarAllocation.id }); + return updated !== undefined; + }, + + /** Record observed cleanup connectivity without releasing the reconciliation lease. */ + async recordCleanupConnection(args: { + readonly allocationId: string; + readonly generation: number; + readonly expectedLeaseId: string; + readonly connected: boolean; + readonly now?: Date; + }): Promise { + const [updated] = await db + .update(sidecarAllocation) + .set({ + connectDeadline: args.connected + ? null + : sql`coalesce(${sidecarAllocation.connectDeadline}, ${disconnectDeadline(args.now)})`, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + inArray(sidecarAllocation.status, ["releasing", "replacing"]), + eq(sidecarAllocation.generation, args.generation), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .returning(); + return updated === undefined ? null : parseSidecarAllocationRow(updated); + }, + + // Provisioning is replaced because nothing has run. An allocated + // generation is replaced only when the caller names that status: a + // parked run, whose committed log is enough to continue. async beginReplacement( args: BeginSidecarReplacementArgs, ): Promise { const now = databaseTimestamp(args.now); return db.transaction(async (tx) => { - if ( - args.onlyIfInitializationIncomplete && - (await initializationCompleted(tx, args)) - ) - return null; + const initialized = await initializationCompleted(tx, args); + if (args.onlyIfInitializationIncomplete && initialized) return null; + // The digest stays: the row still names the sidecar being destroyed, + // which hosts it for reuse until `destroy` returns, and + // `bindReplacementSidecar` records the digest of the next sidecar. const [updated] = await tx .update(sidecarAllocation) .set({ status: "replacing", generation: args.expectedGeneration + 1, + destroyAttempts: 0, + // Provider acceptance alone does not complete a provisioning sequence. + // Preserve its budget until a worker has actually initialized. + ensureAttempts: initialized ? 0 : sidecarAllocation.ensureAttempts, + deploymentCleanupConfirmed: sql`${sidecarAllocation.ensureAcceptedGeneration} is null`, + providerHoldReleased: false, initializationLeaseId: null, ensureAcceptedGeneration: null, - nextAttemptAt: args.nextAttemptAt, + nextAttemptAt: initialized + ? args.firstAttemptAt + : args.nextAttemptAt, reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, connectDeadline: null, @@ -905,12 +1518,14 @@ export function createSidecarAllocationStore(db: DBHandle) { if (updated === undefined) return null; // The anchor public key is durable proof that this allocation - // generation completed restore and deployment. Clear it atomically - // with the generation advance so a replacement cannot mistake stale - // local sidecar state for a successfully restored workflow. + // generation completed restore and deployment. Clear the key and + // the credential refs with the generation advance so a replacement + // cannot mistake stale local sidecar state for a restored + // workflow. Leftover refs are the previous attempt, and a first + // grant commit would freeze them. await tx .update(workflowRun) - .set({ publicKey: null }) + .set({ publicKey: null, credentialRefs: null }) .where(eq(workflowRun.id, updated.anchorRunId)); return parseSidecarAllocationRow(updated); @@ -919,61 +1534,205 @@ export function createSidecarAllocationStore(db: DBHandle) { beginRelease, + /** + * Fail the live runs of a deployment its sidecar reported stopped, and + * keep the capacity: the stopped copy's files stay for inspection until + * the retention for failed runs releases them. The allocation then rests + * on its connection as a ready one does, keeping the reason it failed. + */ + async failStoppedDeployment( + args: FailStoppedSidecarDeploymentArgs, + ): Promise { + const now = databaseTimestamp(args.now); + return db.transaction(async (tx) => { + const [updated] = await tx + .update(sidecarAllocation) + .set({ + failureCode: args.failureCode, + failureMessage: args.failureMessage, + updatedAt: now, + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.expectedGeneration), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .returning({ anchorRunId: sidecarAllocation.anchorRunId }); + if (updated === undefined) return false; + await failRunningRuns(tx, updated.anchorRunId, now, { + code: args.failureCode, + message: args.failureMessage, + }); + await workflowRunDispatchStore.abandonUnsettled( + updated.anchorRunId, + args.failureCode, + args.failureMessage, + now, + tx, + ); + // Settles it as `markConnectionReady` does, under the same conditions. + // An allocation whose ensure is not accepted or whose initialization is + // still in flight keeps its schedule and lease, so the work that owns + // it settles it. + await tx + .update(sidecarAllocation) + .set({ + connectDeadline: null, + nextAttemptAt: null, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + updatedAt: now, + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.expectedGeneration), + eq( + sidecarAllocation.ensureAcceptedGeneration, + args.expectedGeneration, + ), + isNull(sidecarAllocation.initializationLeaseId), + ...leaseCondition(args.expectedLeaseId), + ), + ); + return true; + }); + }, + async beginUnrecoverableRelease( args: BeginUnrecoverableSidecarReleaseArgs, ): Promise { - const now = databaseTimestamp(args.now); return db.transaction(async (tx) => { if ( args.onlyIfInitializationIncomplete && (await initializationCompleted(tx, args)) ) return null; - const releasing = await beginRelease( - { - ...args, - expectedStatus: "allocated", - }, - tx, - ); - if (releasing === null) return null; - - await failRunningRuns(tx, releasing.anchorRunId, now); - await workflowRunDispatchStore.failUnsettled( - releasing.anchorRunId, - args.failureCode, - args.failureMessage, - now, - tx, - ); - return releasing; + return releaseFailedAllocation(tx, args); }); }, async markReleased( args: MarkSidecarReleasedArgs, ): Promise { - const [updated] = await db - .update(sidecarAllocation) - .set({ - status: "released", - nextAttemptAt: null, - reconciliationLeaseId: null, - reconciliationLeaseExpiresAt: null, - connectDeadline: null, - destroyAttempts: sql`${sidecarAllocation.destroyAttempts} + 1`, - updatedAt: databaseTimestamp(args.now), - }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, "releasing"), - eq(sidecarAllocation.generation, args.generation), - ...leaseCondition(args.expectedLeaseId), + const now = databaseTimestamp(args.now); + const cleanupFailure = inArray( + sidecarAllocation.failureCode, + CLEANUP_STOP_FAILURE_CODES, + ); + return db.transaction(async (tx) => { + const condition = and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "releasing"), + eq(sidecarAllocation.generation, args.generation), + // An allocation that never bound capacity has nothing to clean up. + or( + isNull(sidecarAllocation.sidecarId), + and( + eq(sidecarAllocation.deploymentCleanupConfirmed, true), + eq(sidecarAllocation.providerHoldReleased, true), + ), ), + ...leaseCondition(args.expectedLeaseId), + ); + if ((await lockAllocationWithSidecars(tx, condition)) === undefined) + return null; + const [updated] = await tx + .update(sidecarAllocation) + .set({ + status: "released", + deploymentCleanupConfirmed: true, + providerHoldReleased: true, + failureCode: sql`case when ${cleanupFailure} then null else ${sidecarAllocation.failureCode} end`, + failureMessage: sql`case when ${cleanupFailure} then null else ${sidecarAllocation.failureMessage} end`, + nextAttemptAt: null, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + connectDeadline: null, + destroyAttempts: sql`${sidecarAllocation.destroyAttempts} + 1`, + updatedAt: now, + }) + .where(condition) + .returning(); + if (updated === undefined) return null; + await settleReleasedDispatches(tx, updated.anchorRunId, now); + return parseSidecarAllocationRow(updated); + }); + }, + + /** + * Releases an allocation whose automatic cleanup stopped, on a principal's + * word that its copy and provider hold are gone. Nothing here verifies that; + * the caller owns the claim, and the row records who made it. + */ + async releaseStoppedCleanup( + args: ReleaseStoppedSidecarCleanupArgs, + ): Promise { + const now = databaseTimestamp(args.now); + return db.transaction(async (tx) => { + const owned = and( + eq(sidecarAllocation.anchorRunId, args.anchorRunId), + eq(sidecarAllocation.tenantId, args.tenantId), + ); + const condition = and( + owned, + eq(sidecarAllocation.status, "destroy_failed"), + eq(sidecarAllocation.generation, args.generation), + ); + const stopped = await lockAllocationWithSidecars(tx, condition); + if (stopped !== undefined) { + const [updated] = await tx + .update(sidecarAllocation) + .set({ + status: "released", + deploymentCleanupConfirmed: true, + providerHoldReleased: true, + failureCode: SIDECAR_CLEANUP_OPERATOR_RELEASED_FAILURE_CODE, + failureMessage: `Released by ${args.releasedBy} after cleanup stopped with ${stopped.failureCode ?? "no recorded code"}: ${stopped.failureMessage ?? "no recorded message"}`, + nextAttemptAt: null, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + connectDeadline: null, + updatedAt: now, + }) + .where(condition) + .returning(); + if (updated === undefined) + throw new Error( + `Allocation ${stopped.id} changed while locked for release`, + ); + await settleReleasedDispatches(tx, updated.anchorRunId, now); + return { + kind: "released", + allocation: parseSidecarAllocationRow(updated), + }; + } + // Under read committed, this statement sees a release that a + // concurrent request committed while this one waited on the lock. + const [current] = await tx + .select({ + status: sidecarAllocation.status, + generation: sidecarAllocation.generation, + failureCode: sidecarAllocation.failureCode, + }) + .from(sidecarAllocation) + .where(owned); + if (current === undefined) return { kind: "not_found" }; + if (current.generation !== args.generation) + return { kind: "generation_mismatch" }; + // Released is terminal, so a repeated request finds the release + // already made. + if ( + current.status === "released" && + current.failureCode === SIDECAR_CLEANUP_OPERATOR_RELEASED_FAILURE_CODE ) - .returning(); - return updated === undefined ? null : parseSidecarAllocationRow(updated); + return { kind: "already_released" }; + return { kind: "not_stopped" }; + }); }, async markDestroyFailed( @@ -981,6 +1740,25 @@ export function createSidecarAllocationStore(db: DBHandle) { ): Promise { const now = databaseTimestamp(args.now); return db.transaction(async (tx) => { + const condition = and( + eq(sidecarAllocation.id, args.allocationId), + inArray(sidecarAllocation.status, ["replacing", "releasing"]), + eq(sidecarAllocation.generation, args.expectedGeneration), + ...leaseCondition(args.expectedLeaseId), + ); + const allocation = await lockAllocationWithSidecars(tx, condition); + if (allocation === undefined) return null; + // Preserve the workflow's cause before the allocation code becomes + // the cleanup outcome recorded on the allocation. + const workflowFailure = + allocation.status === "releasing" && + allocation.failureCode !== null && + !CLEANUP_STOP_FAILURE_CODES.includes(allocation.failureCode) + ? { + code: allocation.failureCode, + message: allocation.failureMessage ?? allocation.failureCode, + } + : { code: args.code, message: args.message }; const [updated] = await tx .update(sidecarAllocation) .set({ @@ -991,28 +1769,33 @@ export function createSidecarAllocationStore(db: DBHandle) { reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, connectDeadline: null, - destroyAttempts: sql`${sidecarAllocation.destroyAttempts} + 1`, + ...(args.countAttempt === false + ? {} + : { + destroyAttempts: sql`${sidecarAllocation.destroyAttempts} + 1`, + }), updatedAt: now, }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - inArray(sidecarAllocation.status, ["replacing", "releasing"]), - eq(sidecarAllocation.generation, args.expectedGeneration), - ...leaseCondition(args.expectedLeaseId), - ), - ) + .where(condition) .returning(); if (updated === undefined) return null; - await failRunningRuns(tx, updated.anchorRunId, now); - await workflowRunDispatchStore.failUnsettled( - updated.anchorRunId, - args.code, - args.message, - now, - tx, - ); + // Once removal is proven, lifecycle owns the requested cancellation + // and its dispatches after accepted-history recovery. Provider cleanup + // can still fail. + if ( + !updated.deploymentCleanupConfirmed || + !(await hasPendingCancellation(tx, updated.anchorRunId)) + ) { + await failRunningRuns(tx, updated.anchorRunId, now, workflowFailure); + await workflowRunDispatchStore.abandonUnsettled( + updated.anchorRunId, + args.code, + args.message, + now, + tx, + ); + } return parseSidecarAllocationRow(updated); }); }, @@ -1025,6 +1808,22 @@ export function createSidecarAllocationStore(db: DBHandle) { const fail = async ( executor: DBExecutor, ): Promise => { + const condition = and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, args.expectedStatus), + eq(sidecarAllocation.generation, args.expectedGeneration), + ...leaseCondition(args.expectedLeaseId), + ...(args.expectedStatus === "replacing" + ? [ + eq(sidecarAllocation.deploymentCleanupConfirmed, true), + eq(sidecarAllocation.providerHoldReleased, true), + ] + : []), + ); + if ( + (await lockAllocationWithSidecars(executor, condition)) === undefined + ) + return null; const [updated] = await executor .update(sidecarAllocation) .set({ @@ -1037,19 +1836,18 @@ export function createSidecarAllocationStore(db: DBHandle) { connectDeadline: null, updatedAt: now, }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, args.expectedStatus), - eq(sidecarAllocation.generation, args.expectedGeneration), - ...leaseCondition(args.expectedLeaseId), - ), - ) + .where(condition) .returning(); if (updated === undefined) return null; - await failRunningRuns(executor, updated.anchorRunId, now); - await workflowRunDispatchStore.failUnsettled( + // No infrastructure remains to stop; lifecycle can finish cancellation. + if (await hasPendingCancellation(executor, updated.anchorRunId)) + return parseSidecarAllocationRow(updated); + await failRunningRuns(executor, updated.anchorRunId, now, { + code: args.code, + message: args.message, + }); + await workflowRunDispatchStore.abandonUnsettled( updated.anchorRunId, args.code, args.message, @@ -1061,6 +1859,44 @@ export function createSidecarAllocationStore(db: DBHandle) { return tx === undefined ? db.transaction(fail) : fail(tx); }, + /** + * Apply an infrastructure failure deferred while accepted history was + * unreconciled: fail the runs still live at the recorded time. Returns + * false while a pending projection remains. + */ + async applyDeferredInfrastructureFailure( + anchorRunId: string, + tx: DBExecutor, + ): Promise { + const [anchor] = await tx + .select({ failedAt: workflowRun.infrastructureFailedAt }) + .from(workflowRun) + .where(eq(workflowRun.id, anchorRunId)); + if (anchor?.failedAt == null) return true; + if (await pendingProjections.hasAny(anchorRunId, tx)) return false; + const failed = await failLiveRuns(tx, anchorRunId, anchor.failedAt); + await settleDeferredFailure(tx, anchorRunId, failed); + return true; + }, + + async hasRunnableAnchor( + anchorRunId: string, + now = new Date(), + ): Promise { + const run = await db.query.workflowRun.findFirst({ + where: and( + eq(workflowRun.id, anchorRunId), + eq(workflowRun.anchorRunId, anchorRunId), + ), + columns: { + status: true, + expiresAt: true, + cancellationRequestedAt: true, + }, + }); + return run !== undefined && canExecuteWorkflowRun(run, now); + }, + async findById(id: string): Promise { const row = await db.query.sidecarAllocation.findFirst({ where: eq(sidecarAllocation.id, id), @@ -1165,6 +2001,9 @@ export function createSidecarAllocationStore(db: DBHandle) { async markConnectionReady( args: MarkSidecarConnectionReadyArgs, ): Promise { + // A stopped deployment the Hub failed stays connected until retention + // releases it, and a later ready pass must not erase why it failed. + const keepsReason = sql`${sidecarAllocation.failureCode} = ${SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE}`; const [updated] = await db .update(sidecarAllocation) .set({ @@ -1172,8 +2011,8 @@ export function createSidecarAllocationStore(db: DBHandle) { nextAttemptAt: null, reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, - failureCode: null, - failureMessage: null, + failureCode: sql`case when ${keepsReason} then ${sidecarAllocation.failureCode} end`, + failureMessage: sql`case when ${keepsReason} then ${sidecarAllocation.failureMessage} end`, updatedAt: databaseTimestamp(args.now), }) .where( @@ -1193,54 +2032,104 @@ export function createSidecarAllocationStore(db: DBHandle) { async markConnectionLost( args: MarkSidecarConnectionLostArgs, ): Promise { - const [updated] = await db - .update(sidecarAllocation) - .set({ - connectDeadline: args.connectDeadline, - nextAttemptAt: args.connectDeadline, - reconciliationLeaseId: null, - reconciliationLeaseExpiresAt: null, - updatedAt: databaseTimestamp(args.now), - }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, "allocated"), - eq(sidecarAllocation.generation, args.generation), - eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), - ), - ) - .returning(); - return updated === undefined ? null : parseSidecarAllocationRow(updated); + const connectDeadline = reconnectDeadline( + args, + firstConnectDeadline(args), + ); + return db.transaction(async (tx) => { + await shareAllocationSidecar(tx, args.allocationId); + const [updated] = await tx + .update(sidecarAllocation) + .set({ + connectDeadline, + nextAttemptAt: connectDeadline, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), + ), + ) + .returning(); + return updated === undefined + ? null + : parseSidecarAllocationRow(updated); + }); + }, + + // No sidecar could connect while the Hub was down, so every generation gets + // a fresh window from Hub start, unless the deadline it already has before + // its first deploy completed is later. + async scheduleReconnectAfterHubStart( + args: ScheduleSidecarReconnectAfterHubStartArgs, + ): Promise { + const connectDeadline = reconnectDeadline( + args, + sql`greatest(${sidecarAllocation.connectDeadline}, ${firstConnectDeadline(args)})`, + ); + return db.transaction(async (tx) => { + await shareAllocationSidecar(tx, args.allocationId); + const [updated] = await tx + .update(sidecarAllocation) + .set({ + connectDeadline, + nextAttemptAt: connectDeadline, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), + ), + ) + .returning(); + return updated === undefined + ? null + : parseSidecarAllocationRow(updated); + }); }, async scheduleReconnectIfUnscheduled( args: ScheduleSidecarReconnectIfUnscheduledArgs, ): Promise { - const connectDeadline = sql.param( - args.connectDeadline, - sidecarAllocation.connectDeadline, + const connectDeadline = reconnectDeadline( + args, + firstConnectDeadline(args), ); - const [updated] = await db - .update(sidecarAllocation) - .set({ - connectDeadline: sql`coalesce(${sidecarAllocation.connectDeadline}, ${connectDeadline})`, - nextAttemptAt: sql`coalesce(${sidecarAllocation.connectDeadline}, ${connectDeadline})`, - updatedAt: databaseTimestamp(args.now), - }) - .where( - and( - eq(sidecarAllocation.id, args.allocationId), - eq(sidecarAllocation.status, "allocated"), - eq(sidecarAllocation.generation, args.generation), - eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), - isNull(sidecarAllocation.nextAttemptAt), - isNull(sidecarAllocation.reconciliationLeaseId), - isNull(sidecarAllocation.reconciliationLeaseExpiresAt), - ), - ) - .returning(); - return updated === undefined ? null : parseSidecarAllocationRow(updated); + return db.transaction(async (tx) => { + await shareAllocationSidecar(tx, args.allocationId); + const [updated] = await tx + .update(sidecarAllocation) + .set({ + connectDeadline: sql`coalesce(${sidecarAllocation.connectDeadline}, ${connectDeadline})`, + nextAttemptAt: sql`coalesce(${sidecarAllocation.connectDeadline}, ${connectDeadline})`, + updatedAt: databaseTimestamp(args.now), + }) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), + isNull(sidecarAllocation.nextAttemptAt), + isNull(sidecarAllocation.reconciliationLeaseId), + isNull(sidecarAllocation.reconciliationLeaseExpiresAt), + ), + ) + .returning(); + return updated === undefined + ? null + : parseSidecarAllocationRow(updated); + }); }, async parkReconciliation( @@ -1277,14 +2166,95 @@ export function createSidecarAllocationStore(db: DBHandle) { return updated !== undefined; }, - async wakeReconciliation( + /** Resume a timeout or an unchanged parked wait, without reviving its workflow. */ + async resumeCleanupAfterDisconnect( allocationId: string, generation: number, + parked?: { readonly deadline: Date; readonly destroyAttempts: number }, ): Promise { const [updated] = await db .update(sidecarAllocation) .set({ + status: sql`case when ${sidecarAllocation.status} = 'replacing' then 'replacing' else 'releasing' end`, + connectDeadline: null, nextAttemptAt: sql`now()`, + updatedAt: sql`now()`, + }) + .where( + and( + eq(sidecarAllocation.id, allocationId), + eq(sidecarAllocation.generation, generation), + or( + and( + eq(sidecarAllocation.status, "destroy_failed"), + eq( + sidecarAllocation.failureCode, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + ), + ), + parked === undefined + ? undefined + : and( + inArray(sidecarAllocation.status, [ + "releasing", + "replacing", + ]), + eq(sidecarAllocation.connectDeadline, parked.deadline), + eq(sidecarAllocation.nextAttemptAt, parked.deadline), + eq( + sidecarAllocation.destroyAttempts, + parked.destroyAttempts, + ), + or( + isNull(sidecarAllocation.reconciliationLeaseExpiresAt), + lte( + sidecarAllocation.reconciliationLeaseExpiresAt, + sql`clock_timestamp()`, + ), + ), + ), + ), + ), + ) + .returning({ id: sidecarAllocation.id }); + return updated !== undefined; + }, + + async wakeReconciliation( + allocationId: string, + generation: number, + { + connected = false, + preserveRetryBackoff = false, + }: { + readonly connected?: boolean; + readonly preserveRetryBackoff?: boolean; + } = {}, + ): Promise { + const [updated] = await db + .update(sidecarAllocation) + .set({ + // Wake disconnected cleanup immediately, but preserve failure backoff + // and the replacement delay before releasing the old provider hold. + nextAttemptAt: preserveRetryBackoff + ? sql`case + when ${sidecarAllocation.status} in ('releasing', 'replacing') + and (${sidecarAllocation.destroyAttempts} > 0 or + (${sidecarAllocation.status} = 'replacing' and not ${sidecarAllocation.providerHoldReleased})) + and ${sidecarAllocation.nextAttemptAt} is not null + and ${sidecarAllocation.nextAttemptAt} is distinct from ${sidecarAllocation.connectDeadline} + then ${sidecarAllocation.nextAttemptAt} + else now() + end` + : sql`now()`, + // A reconnect ends an initialized copy's disconnect window before + // the readiness pass. Preserve the first-connect limit until deploy + // completes; repeated hellos must not extend initialization forever. + ...(connected + ? { + connectDeadline: sql`case when ${sidecarAllocation.status} in ('releasing', 'replacing') or exists (select 1 from ${workflowRun} where ${workflowRun.id} = ${sidecarAllocation.anchorRunId} and ${workflowRun.publicKey} is not null) then null else ${sidecarAllocation.connectDeadline} end`, + } + : {}), }) .where( and( @@ -1297,16 +2267,42 @@ export function createSidecarAllocationStore(db: DBHandle) { return updated !== undefined; }, + /** Active work and the failed cleanups that connection repair can resume. */ + async listForConnectionRepair(): Promise { + const rows = await db + .select() + .from(sidecarAllocation) + .where( + or( + inArray(sidecarAllocation.status, activeStatuses), + and( + eq(sidecarAllocation.status, "destroy_failed"), + eq( + sidecarAllocation.failureCode, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + ), + ), + ), + ); + return rows.map(parseSidecarAllocationRow); + }, + + /** Includes failed cleanup obligations whose fences must survive a restart. */ async listActive(): Promise { const rows = await db .select() .from(sidecarAllocation) - .where(inArray(sidecarAllocation.status, activeStatuses)); + .where( + or( + inArray(sidecarAllocation.status, activeStatuses), + eq(sidecarAllocation.status, "destroy_failed"), + ), + ); return rows.map(parseSidecarAllocationRow); }, }; } export type SidecarAllocationStore = ReturnType< - typeof createSidecarAllocationStore + typeof buildSidecarAllocationStore >; diff --git a/vendor/intx/db/src/sidecar-reuse.ts b/vendor/intx/db/src/sidecar-reuse.ts new file mode 100644 index 0000000000..ed760c318f --- /dev/null +++ b/vendor/intx/db/src/sidecar-reuse.ts @@ -0,0 +1,291 @@ +import { and, eq, inArray, isNull, ne, or, type SQL } from "drizzle-orm"; + +import type { SidecarCapacity } from "@intx/types/sidecar"; + +import type { DBExecutor } from "./client"; +import { sidecar, sidecarAllocation } from "./schema"; + +/** A provisioner named a sidecar the Hub cannot place its work on. */ +export class SidecarReuseRejectedError extends Error { + readonly sidecarId: string; + + constructor(sidecarId: string, reason: string) { + super(`Cannot place work on sidecar ${sidecarId}: ${reason}`); + this.name = "SidecarReuseRejectedError"; + this.sidecarId = sidecarId; + } +} + +/** Placement must wait for a complete inventory from the current connection. */ +export class SidecarInventoryUnavailableError extends Error { + constructor(readonly sidecarId: string) { + super(`Sidecar ${sidecarId} has no current inventory for placement`); + this.name = "SidecarInventoryUnavailableError"; + } +} + +export type SidecarProvisionerBinding = { + readonly provisionerId: string; + readonly provisionerApiVersion: 2; + readonly provisionerBindingFingerprint: string; +}; + +/** A sidecar's durable confirmation that this incarnation cannot resume. */ +export type RetainedSidecarIncarnation = { + readonly allocationId: string; + readonly generation: number; +}; + +export type SidecarInventory = { + readonly capacity: SidecarCapacity; + readonly retainedIncarnations: readonly RetainedSidecarIncarnation[]; +}; + +export type SidecarAllocationStoreOptions = { + /** Missing or undefined inventory blocks placement. */ + readonly getSidecarInventory?: ( + sidecarId: string, + ) => SidecarInventory | undefined; +}; + +/** Lock sidecars in ID order before locking the probe or allocation that moves. */ +export async function lockSidecars( + tx: DBExecutor, + sidecarIds: readonly string[], +) { + if (sidecarIds.length === 0) return []; + return tx + .select({ id: sidecar.id }) + .from(sidecar) + .where(inArray(sidecar.id, [...sidecarIds])) + .orderBy(sidecar.id) + .for("update"); +} + +/** + * Asserts that a provisioner may place a probe or allocation on `sidecarId`. + * The sidecar must have been minted for the same provisioner binding, so a + * provisioner cannot claim another backend's sidecar, and for an ensure that + * carried the same adapter manifest: a sidecar reads its manifest only at + * start, so work never joins one that loaded another manifest. The sidecar + * may host nothing but must not be retired. Locks the sidecar row. + */ +export async function assertSidecarReusable( + tx: DBExecutor, + args: { + readonly sidecarId: string; + readonly binding: SidecarProvisionerBinding; + readonly adapterManifestDigest: string; + }, +): Promise { + const [row] = await tx + .select({ + provisionerId: sidecar.provisionerId, + provisionerApiVersion: sidecar.provisionerApiVersion, + provisionerBindingFingerprint: sidecar.provisionerBindingFingerprint, + adapterManifestDigest: sidecar.adapterManifestDigest, + retiredAt: sidecar.retiredAt, + }) + .from(sidecar) + .where(eq(sidecar.id, args.sidecarId)) + .for("update"); + if (row === undefined) { + throw new SidecarReuseRejectedError(args.sidecarId, "it does not exist"); + } + if (row.retiredAt !== null) { + throw new SidecarReuseRejectedError( + args.sidecarId, + "it was retired: no sidecar runs with this identity", + ); + } + if ( + row.provisionerId !== args.binding.provisionerId || + row.provisionerApiVersion !== args.binding.provisionerApiVersion || + row.provisionerBindingFingerprint !== + args.binding.provisionerBindingFingerprint + ) { + throw new SidecarReuseRejectedError( + args.sidecarId, + `it was not started for provisioner ${args.binding.provisionerId} with the same binding`, + ); + } + if (row.adapterManifestDigest !== args.adapterManifestDigest) { + throw new SidecarReuseRejectedError( + args.sidecarId, + "it booted with another adapter manifest", + ); + } +} + +/** + * Marks `sidecarId` as gone: reuse stops at once, its token once neither + * current work nor cleanup is left. Call it before the write that ends the + * caller's retries. + */ +export async function retireSidecar( + tx: DBExecutor, + args: { readonly sidecarId: string; readonly now: Date | SQL }, +): Promise { + await tx + .update(sidecar) + .set({ retiredAt: args.now, updatedAt: args.now }) + .where(and(eq(sidecar.id, args.sidecarId), isNull(sidecar.retiredAt))); +} + +/** What the Hub saw of a reported sidecar's work when it retired it. */ +export type ProvisionerSidecarRetirement = { + /** Allocated work on the sidecar still connected, which the report contradicts. */ + readonly connectedAllocationIds: readonly string[]; +}; + +/** Whether this Hub holds a live connection for the allocation's generation. */ +export type AllocationConnectionCheck = (target: { + readonly allocationId: string; + readonly generation: number; +}) => boolean; + +/** + * Retires a sidecar its provisioner reports stopped. Returns null, retiring + * nothing, when no sidecar with that id was minted for `binding`. With + * `endReconnectWindow`, it also records that the sidecar's work gets no + * reconnect window: its allocated work without a live connection reaches its + * connection deadline now, and work that disconnects later gets none either. + */ +export async function retireProvisionerSidecar( + tx: DBExecutor, + args: { + readonly sidecarId: string; + readonly binding: SidecarProvisionerBinding; + readonly endReconnectWindow: boolean; + readonly isConnected: AllocationConnectionCheck; + readonly now: Date | SQL; + }, +): Promise { + const [owned] = await tx + .select({ id: sidecar.id }) + .from(sidecar) + .where( + and( + eq(sidecar.id, args.sidecarId), + eq(sidecar.provisionerId, args.binding.provisionerId), + eq(sidecar.provisionerApiVersion, args.binding.provisionerApiVersion), + eq( + sidecar.provisionerBindingFingerprint, + args.binding.provisionerBindingFingerprint, + ), + ), + ) + .for("update"); + if (owned === undefined) return null; + await retireSidecar(tx, { sidecarId: args.sidecarId, now: args.now }); + const allocated = await tx + .select({ + allocationId: sidecarAllocation.id, + generation: sidecarAllocation.generation, + }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.sidecarId, args.sidecarId), + eq(sidecarAllocation.status, "allocated"), + eq( + sidecarAllocation.ensureAcceptedGeneration, + sidecarAllocation.generation, + ), + ), + ) + .for("update"); + // The deadline cannot tell: work still initializing keeps one while + // connected, and a drop the Hub has not recorded yet has none. + const connectedAllocationIds: string[] = []; + const disconnectedAllocationIds: string[] = []; + for (const target of allocated) + (args.isConnected(target) + ? connectedAllocationIds + : disconnectedAllocationIds + ).push(target.allocationId); + if (args.endReconnectWindow) { + await tx + .update(sidecar) + .set({ reconnectWindowEndedAt: args.now, updatedAt: args.now }) + .where( + and( + eq(sidecar.id, args.sidecarId), + isNull(sidecar.reconnectWindowEndedAt), + ), + ); + // Dropping the lease stops a reconciler still waiting on the old + // deadline, like a lost connection does. + if (disconnectedAllocationIds.length > 0) + await tx + .update(sidecarAllocation) + .set({ + connectDeadline: args.now, + nextAttemptAt: args.now, + reconciliationLeaseId: null, + reconciliationLeaseExpiresAt: null, + updatedAt: args.now, + }) + .where(inArray(sidecarAllocation.id, disconnectedAllocationIds)); + } + return { connectedAllocationIds }; +} + +/** + * Locks the sidecar so placements serialize, counting every reservation + * except cleanup confirmed removed or an incarnation durably retained. + * Unknown inventory blocks placement until the sidecar completes its hello. + */ +export async function assertSidecarHasRoom( + tx: DBExecutor, + args: { + readonly sidecarId: string; + readonly placing: { readonly allocationId: string }; + } & SidecarAllocationStoreOptions, +): Promise { + const [locked] = await lockSidecars(tx, [args.sidecarId]); + if (locked === undefined) { + throw new SidecarReuseRejectedError(args.sidecarId, "it does not exist"); + } + const reservations = await tx + .select({ + id: sidecarAllocation.id, + ensureAcceptedGeneration: sidecarAllocation.ensureAcceptedGeneration, + }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.sidecarId, args.sidecarId), + ne(sidecarAllocation.id, args.placing.allocationId), + or( + inArray(sidecarAllocation.status, ["provisioning", "allocated"]), + and( + inArray(sidecarAllocation.status, [ + "replacing", + "releasing", + "destroy_failed", + ]), + eq(sidecarAllocation.deploymentCleanupConfirmed, false), + ), + ), + ), + ); + // Read after the database wait: the connection may have changed meanwhile. + const inventory = args.getSidecarInventory?.(args.sidecarId); + if (inventory === undefined) + throw new SidecarInventoryUnavailableError(args.sidecarId); + const active = reservations.filter( + (reservation) => + !inventory.retainedIncarnations.some( + (incarnation) => + incarnation.allocationId === reservation.id && + incarnation.generation === reservation.ensureAcceptedGeneration, + ), + ).length; + if (active >= inventory.capacity.maxActiveDeployments) { + throw new SidecarReuseRejectedError( + args.sidecarId, + `it already hosts ${String(active)} deployments occupying all active slots`, + ); + } +} diff --git a/vendor/intx/db/src/tenant-hierarchy.ts b/vendor/intx/db/src/tenant-hierarchy.ts index 3a5691f10a..703213bbb0 100644 --- a/vendor/intx/db/src/tenant-hierarchy.ts +++ b/vendor/intx/db/src/tenant-hierarchy.ts @@ -1,9 +1,9 @@ -import { type } from "arktype"; import { eq, inArray } from "drizzle-orm"; -import { TenantConfig, type TenantSidecarCapabilityPolicy } from "@intx/types"; +import type { TenantSidecarCapabilityPolicy } from "@intx/types"; -import type { DB } from "./client"; +import type { DB, DBExecutor } from "./client"; +import { parseTenantConfig } from "./parse-row"; import { tenant } from "./schema/tenants"; /** @@ -14,7 +14,7 @@ import { tenant } from "./schema/tenants"; * chain that could omit inherited policy or configuration. */ export async function getAncestorChain( - db: DB["db"], + db: DBExecutor, tenantId: string, ): Promise { const chain: string[] = []; @@ -57,12 +57,7 @@ export async function resolveTenantSidecarCapabilityPolicies( const row = rowsById.get(ancestorId); if (row === undefined) continue; if (row.config === null) continue; - const config = TenantConfig(row.config); - if (config instanceof type.errors) { - throw new Error( - `Tenant ${row.id} has invalid configuration: ${config.summary}`, - ); - } + const config = parseTenantConfig(row.id, row.config); const rules = config.sidecarPlacement?.capabilities; if (rules !== undefined && rules.length > 0) { policies.push({ tenantId: row.id, rules }); diff --git a/vendor/intx/db/src/workflow-lifecycle-policy.ts b/vendor/intx/db/src/workflow-lifecycle-policy.ts new file mode 100644 index 0000000000..f261585c25 --- /dev/null +++ b/vendor/intx/db/src/workflow-lifecycle-policy.ts @@ -0,0 +1,222 @@ +import { and, eq, inArray } from "drizzle-orm"; + +import { + WorkflowLifecyclePolicy, + lifecycleDeadline, + clampWorkflowLifecyclePolicy, + resolveWorkflowLifecyclePolicy, + type ResolvedWorkflowLifecyclePolicy, +} from "@intx/types"; + +import type { DB, DBExecutor } from "./client"; +import { + parseTenantConfig, + parseWorkflowDefinitionLifecyclePolicy, +} from "./parse-row"; +import { getAncestorChain } from "./tenant-hierarchy"; +import { tenant } from "./schema/tenants"; +import { workflowDefinition } from "./schema/workflow-definitions"; +import { isLiveWorkflowRunStatus, workflowRun } from "./schema/workflow-run"; +import { sidecarAllocation } from "./schema/sidecar-allocation"; + +export type WorkflowRunExecutionTarget = { + readonly allocationId: string; + readonly generation: number; + readonly sidecarId: string; + readonly tenantId: string; + readonly anchorRunId: string; + readonly workflowRunAddress: string; +}; + +export class WorkflowRunNotExecutableError extends Error { + readonly reason: "stopping" | "terminal"; + constructor(runId: string, reason: "stopping" | "terminal") { + super(`Workflow run ${runId} is ${reason} and no longer accepts work`); + this.name = "WorkflowRunNotExecutableError"; + this.reason = reason; + } +} + +/** + * A live run is stopping once cancellation is requested or its deadline + * passes, before its status leaves the live set. Admission uses the original + * deadline, including before the expiry sweep runs. + */ +export function workflowRunExecutability( + run: { + status: string; + expiresAt: Date | null; + cancellationRequestedAt: Date | null; + }, + now = new Date(), +): "executable" | "stopping" | "terminal" { + if (!isLiveWorkflowRunStatus(run.status)) return "terminal"; + return run.cancellationRequestedAt == null && + (run.expiresAt == null || run.expiresAt > now) + ? "executable" + : "stopping"; +} + +export function canExecuteWorkflowRun( + run: Parameters[0], + now = new Date(), +): boolean { + return workflowRunExecutability(run, now) === "executable"; +} + +/** The saved capacity deadline, or the terminal run's frozen policy deadline. */ +export function workflowCapacityReleaseAt( + run: Pick< + typeof workflowRun.$inferSelect, + "status" | "endedAt" | "capacityReleaseAt" | "lifecyclePolicy" + >, +): Date | null { + if (run.capacityReleaseAt !== null) return run.capacityReleaseAt; + if ( + run.status === "deployed" || + run.status === "running" || + run.endedAt === null || + run.lifecyclePolicy == null + ) + return null; + const retention = WorkflowLifecyclePolicy.assert(run.lifecyclePolicy) + .capacityRetention?.[run.status]; + return retention === undefined + ? null + : lifecycleDeadline(run.endedAt, retention); +} + +/** + * Serialize a synchronous send with cancellation and allocation retirement + * through the anchor run's row lock. Pack ingestion holds only the allocation + * row, so a long receive does not delay delivery. This relies on every path + * that retires the allocation of a run that can still execute also writing the + * anchor row, as unrecoverable release does. Failing a stopped deployment + * writes the anchor row but keeps the allocation, so while that failure is + * deferred, the deployment's dropped route is what refuses its delivery. + */ +export async function withExecutableWorkflowRun( + db: DB["db"], + target: WorkflowRunExecutionTarget, + send: () => boolean, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + return db.transaction(async (tx) => { + const [run] = await tx + .select({ + status: workflowRun.status, + expiresAt: workflowRun.expiresAt, + cancellationRequestedAt: workflowRun.cancellationRequestedAt, + }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, target.anchorRunId), + eq(workflowRun.anchorRunId, target.anchorRunId), + eq(workflowRun.tenantId, target.tenantId), + eq(workflowRun.address, target.workflowRunAddress), + ), + ) + .for("update"); + signal?.throwIfAborted(); + // Read under the anchor lock, so a retirement that committed while this + // send waited is visible. + const [allocation] = await tx + .select({ id: sidecarAllocation.id }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.id, target.allocationId), + eq(sidecarAllocation.generation, target.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, target.generation), + eq(sidecarAllocation.sidecarId, target.sidecarId), + eq(sidecarAllocation.tenantId, target.tenantId), + eq(sidecarAllocation.anchorRunId, target.anchorRunId), + eq(sidecarAllocation.status, "allocated"), + ), + ); + signal?.throwIfAborted(); + // Evaluate the clock after the lock, never before a potentially slow read. + const state = + run === undefined ? "terminal" : workflowRunExecutability(run); + // A run that cannot execute is reported as such even once its allocation + // is retired, so callers drop its work instead of retrying it. + if (state !== "executable") + throw new WorkflowRunNotExecutableError(target.anchorRunId, state); + if (allocation === undefined) + throw new Error( + `Allocation ${target.allocationId} is not available for delivery`, + ); + return send(); + }); +} + +export async function loadTenantLifecyclePolicies( + db: DBExecutor, + tenantId: string, +) { + const chain = await getAncestorChain(db, tenantId); + const rows = await db + .select({ id: tenant.id, config: tenant.config }) + .from(tenant) + .where(inArray(tenant.id, chain)); + const configs = new Map(rows.map((row) => [row.id, row.config])); + return chain.reverse().map((id) => { + const raw = configs.get(id); + return raw == null ? {} : (parseTenantConfig(id, raw).lifecycle ?? {}); + }); +} + +/** + * Validate a policy set below `inheritFrom` against the limits it inherits. + * Ancestors are capped first, so a value a later ancestor edit made stale + * cannot reject an otherwise valid policy. + */ +export async function validateLifecyclePolicyEdit( + db: DBExecutor, + inheritFrom: string | null, + policy: WorkflowLifecyclePolicy, +) { + const inherited = + inheritFrom === null + ? [] + : await loadTenantLifecyclePolicies(db, inheritFrom); + return resolveWorkflowLifecyclePolicy([ + clampWorkflowLifecyclePolicy(inherited), + policy, + ]); +} + +/** + * `defaults` fills fields no tenant or installed workflow sets. Unlike a + * tenant value it is not a ceiling, so either level may set a longer duration. + */ +export async function resolveDeploymentLifecyclePolicy( + db: DBExecutor, + tenantId: string, + definitionId: string, + defaults: ResolvedWorkflowLifecyclePolicy, +): Promise { + const policies = await loadTenantLifecyclePolicies(db, tenantId); + const [definition] = await db + .select({ + tenantId: workflowDefinition.tenantId, + policy: workflowDefinition.lifecyclePolicy, + }) + .from(workflowDefinition) + .where(eq(workflowDefinition.id, definitionId)); + if (definition === undefined || definition.tenantId !== tenantId) { + throw new Error("Workflow definition does not belong to deployment tenant"); + } + policies.push(parseWorkflowDefinitionLifecyclePolicy(definition.policy)); + const policy = clampWorkflowLifecyclePolicy(policies); + return { + maxLifetime: policy.maxLifetime ?? defaults.maxLifetime, + maxDisconnected: policy.maxDisconnected ?? defaults.maxDisconnected, + capacityRetention: { + ...defaults.capacityRetention, + ...policy.capacityRetention, + }, + }; +} diff --git a/vendor/intx/db/src/workflow-pending-projection-store.ts b/vendor/intx/db/src/workflow-pending-projection-store.ts new file mode 100644 index 0000000000..8ed0256f8f --- /dev/null +++ b/vendor/intx/db/src/workflow-pending-projection-store.ts @@ -0,0 +1,69 @@ +import { and, asc, eq, inArray } from "drizzle-orm"; + +import type { DB, DBExecutor } from "./client"; +import { workflowPendingProjection } from "./schema/workflow-pending-projection"; + +type DBHandle = DB["db"]; + +/** + * Store for `workflow_pending_projection`, the durable record that a + * workflow-run pack receive may have advanced Git without its run-status + * projection landing. See the schema for the invariant the rows carry. + */ +export function createWorkflowPendingProjectionStore(db: DBHandle) { + return { + async open(id: string, anchorRunId: string): Promise { + await db.insert(workflowPendingProjection).values({ id, anchorRunId }); + }, + + async close(id: string, tx?: DBExecutor): Promise { + await (tx ?? db) + .delete(workflowPendingProjection) + .where(eq(workflowPendingProjection.id, id)); + }, + + async list( + anchorRunId: string, + tx?: DBExecutor, + ): Promise<{ id: string; createdAt: Date; historyRequired: boolean }[]> { + return (tx ?? db) + .select({ + id: workflowPendingProjection.id, + createdAt: workflowPendingProjection.createdAt, + historyRequired: workflowPendingProjection.historyRequired, + }) + .from(workflowPendingProjection) + .where(eq(workflowPendingProjection.anchorRunId, anchorRunId)) + .orderBy(asc(workflowPendingProjection.createdAt)); + }, + + async hasAny(anchorRunId: string, tx?: DBExecutor): Promise { + const [row] = await (tx ?? db) + .select({ id: workflowPendingProjection.id }) + .from(workflowPendingProjection) + .where(eq(workflowPendingProjection.anchorRunId, anchorRunId)) + .limit(1); + return row !== undefined; + }, + + async claim( + anchorRunId: string, + ids: readonly string[], + tx: DBExecutor, + ): Promise { + if (ids.length === 0) return; + await tx + .delete(workflowPendingProjection) + .where( + and( + eq(workflowPendingProjection.anchorRunId, anchorRunId), + inArray(workflowPendingProjection.id, [...ids]), + ), + ); + }, + }; +} + +export type WorkflowPendingProjectionStore = ReturnType< + typeof createWorkflowPendingProjectionStore +>; diff --git a/vendor/intx/db/src/workflow-probe-store.ts b/vendor/intx/db/src/workflow-probe-store.ts index a968d9fab7..a8330dbb72 100644 --- a/vendor/intx/db/src/workflow-probe-store.ts +++ b/vendor/intx/db/src/workflow-probe-store.ts @@ -1,10 +1,15 @@ -import { and, eq, inArray, sql } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import type { WorkflowProbeResultFrame } from "@intx/types/sidecar"; import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; import type { DB, DBExecutor } from "./client"; import { sidecar, workflowProbe, type WorkflowProbeStatus } from "./schema"; +import { + assertSidecarReusable, + lockSidecars, + retireSidecar, +} from "./sidecar-reuse"; type DBHandle = DB["db"]; type WorkflowProbeResult = Omit; @@ -20,8 +25,10 @@ export type CreateWorkflowProbeArgs = { readonly entry: string; readonly pin?: string; readonly provisionerId: string; - readonly provisionerApiVersion: 1; + readonly provisionerApiVersion: 2; readonly provisionerBindingFingerprint: string; + /** The adapter manifest the probe's ensure carries. */ + readonly adapterManifestDigest: string; readonly now?: Date; }; @@ -44,6 +51,36 @@ function timestamp(now?: Date) { } export function createWorkflowProbeStore(db: DBHandle) { + async function lockProbeWithSidecars( + tx: DBExecutor, + condition: SQL | undefined, + destinationSidecarId?: string, + ) { + const [candidate] = await tx + .select({ sidecarId: workflowProbe.sidecarId }) + .from(workflowProbe) + .where(condition); + if (candidate === undefined) return undefined; + await lockSidecars( + tx, + [candidate.sidecarId, destinationSidecarId].filter((id) => id != null), + ); + // Do not transition a probe that moved to another sidecar while we waited. + const [probe] = await tx + .select() + .from(workflowProbe) + .where( + and( + condition, + candidate.sidecarId === null + ? isNull(workflowProbe.sidecarId) + : eq(workflowProbe.sidecarId, candidate.sidecarId), + ), + ) + .for("update"); + return probe; + } + return { async create( args: CreateWorkflowProbeArgs, @@ -62,6 +99,7 @@ export function createWorkflowProbeStore(db: DBHandle) { provisionerId: args.provisionerId, provisionerApiVersion: args.provisionerApiVersion, provisionerBindingFingerprint: args.provisionerBindingFingerprint, + adapterManifestDigest: args.adapterManifestDigest, createdAt, updatedAt: createdAt, }) @@ -79,7 +117,14 @@ export function createWorkflowProbeStore(db: DBHandle) { const bind = async (executor: DBExecutor) => { const updatedAt = timestamp(args.now); const [probe] = await executor - .select({ status: workflowProbe.status }) + .select({ + status: workflowProbe.status, + provisionerId: workflowProbe.provisionerId, + provisionerApiVersion: workflowProbe.provisionerApiVersion, + provisionerBindingFingerprint: + workflowProbe.provisionerBindingFingerprint, + adapterManifestDigest: workflowProbe.adapterManifestDigest, + }) .from(workflowProbe) .where(eq(workflowProbe.id, args.probeId)) .limit(1) @@ -89,6 +134,10 @@ export function createWorkflowProbeStore(db: DBHandle) { id: args.sidecarId, url: null, tokenHashSha256: args.tokenHashSha256, + provisionerId: probe.provisionerId, + provisionerApiVersion: probe.provisionerApiVersion, + provisionerBindingFingerprint: probe.provisionerBindingFingerprint, + adapterManifestDigest: probe.adapterManifestDigest, status: "offline", createdAt: updatedAt, updatedAt, @@ -107,28 +156,84 @@ export function createWorkflowProbeStore(db: DBHandle) { return tx === undefined ? db.transaction(bind) : bind(tx); }, + /** + * Records the provisioner's acceptance of a probe. `sidecarId` names an + * existing sidecar the provisioner placed the probe on instead of starting + * the one bound to it; the unused bound identity is deleted. Throws + * `SidecarReuseRejectedError` when that sidecar cannot be reused, leaving + * the probe unchanged. + */ async markProbing(args: { probeId: string; + sidecarId?: string; externalRef?: string; now?: Date; }): Promise { - const [updated] = await db - .update(workflowProbe) - .set({ - status: "probing", - ...(args.externalRef !== undefined - ? { externalRef: args.externalRef } - : {}), - updatedAt: timestamp(args.now), - }) - .where( - and( - eq(workflowProbe.id, args.probeId), - eq(workflowProbe.status, "provisioning"), - ), - ) - .returning(); - return updated ?? null; + return db.transaction(async (tx) => { + const condition = and( + eq(workflowProbe.id, args.probeId), + eq(workflowProbe.status, "provisioning"), + ); + const probe = await lockProbeWithSidecars( + tx, + condition, + args.sidecarId, + ); + if (probe === undefined) return null; + const boundSidecarId = probe.sidecarId; + const reusedSidecarId = + args.sidecarId !== undefined && args.sidecarId !== boundSidecarId + ? args.sidecarId + : undefined; + if (reusedSidecarId !== undefined) { + await assertSidecarReusable(tx, { + sidecarId: reusedSidecarId, + binding: { + provisionerId: probe.provisionerId, + provisionerApiVersion: probe.provisionerApiVersion, + provisionerBindingFingerprint: + probe.provisionerBindingFingerprint, + }, + adapterManifestDigest: probe.adapterManifestDigest, + }); + } + const [updated] = await tx + .update(workflowProbe) + .set({ + status: "probing", + ...(reusedSidecarId !== undefined + ? { sidecarId: reusedSidecarId } + : {}), + ...(args.externalRef !== undefined + ? { externalRef: args.externalRef } + : {}), + updatedAt: timestamp(args.now), + }) + .where(condition) + .returning(); + if (updated === undefined) { + throw new Error( + `Workflow probe ${args.probeId} changed before it was marked probing`, + ); + } + if (reusedSidecarId !== undefined && boundSidecarId !== null) { + await tx.delete(sidecar).where(eq(sidecar.id, boundSidecarId)); + } + return updated; + }); + }, + + /** Records that no sidecar runs with `sidecarId`; see `retireSidecar`. */ + async retireSidecar(args: { + sidecarId: string; + now?: Date; + }): Promise { + await db.transaction((tx) => + retireSidecar(tx, { + sidecarId: args.sidecarId, + now: timestamp(args.now), + }), + ); }, async recordResult( @@ -157,29 +262,44 @@ export function createWorkflowProbeStore(db: DBHandle) { failureCode?: string; failureMessage?: string; now?: Date; + /** + * For succeeded/failed, do not already hold the probe row lock unless + * this transaction locked its sidecar first. Acquire multiple sidecar + * locks in ID order before locking probe rows, including through writes. + */ tx?: DBExecutor; }, ): Promise { - const [updated] = await (opts?.tx ?? db) - .update(workflowProbe) - .set({ - status: to, - ...(opts?.failureCode !== undefined - ? { failureCode: opts.failureCode } - : {}), - ...(opts?.failureMessage !== undefined - ? { failureMessage: opts.failureMessage } - : {}), - updatedAt: timestamp(opts?.now), - }) - .where( - and( - eq(workflowProbe.id, probeId), - inArray(workflowProbe.status, [...from]), - ), + const removesHolder = to === "succeeded" || to === "failed"; + const transition = async (tx: DBExecutor) => { + const condition = and( + eq(workflowProbe.id, probeId), + inArray(workflowProbe.status, [...from]), + ); + if ( + removesHolder && + (await lockProbeWithSidecars(tx, condition)) === undefined ) - .returning(); - return updated ?? null; + return null; + const [updated] = await tx + .update(workflowProbe) + .set({ + status: to, + ...(opts?.failureCode !== undefined + ? { failureCode: opts.failureCode } + : {}), + ...(opts?.failureMessage !== undefined + ? { failureMessage: opts.failureMessage } + : {}), + updatedAt: timestamp(opts?.now), + }) + .where(condition) + .returning(); + return updated ?? null; + }; + return opts?.tx === undefined && removesHolder + ? db.transaction(transition) + : transition(opts?.tx ?? db); }, async get(probeId: string): Promise { diff --git a/vendor/intx/db/src/workflow-run-dispatch-store.ts b/vendor/intx/db/src/workflow-run-dispatch-store.ts index 22b3827d2c..ba5a418a10 100644 --- a/vendor/intx/db/src/workflow-run-dispatch-store.ts +++ b/vendor/intx/db/src/workflow-run-dispatch-store.ts @@ -4,6 +4,8 @@ import { and, asc, eq, + exists, + gt, inArray, isNull, lte, @@ -22,7 +24,13 @@ import { import type { DB, DBExecutor } from "./client"; import { parseWorkflowRunDispatchRow } from "./parse-row"; -import { sidecarAllocation, workflowRun, workflowRunDispatch } from "./schema"; +import { + liveWorkflowRunStatuses, + sidecarAllocation, + workflowRun, + workflowRunDispatch, + unresolvedWorkflowRunDispatchStatuses, +} from "./schema"; type DBHandle = DB["db"]; type ParsedDispatch = ReturnType; @@ -286,6 +294,22 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { .where( and( eq(workflowRunDispatch.status, "pending"), + exists( + tx + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, workflowRunDispatch.anchorRunId), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + isNull(workflowRun.cancellationRequestedAt), + or( + isNull(workflowRun.expiresAt), + gt(workflowRun.expiresAt, sql`now()`), + ), + ), + ), + ), lte(workflowRunDispatch.nextAttemptAt, sql`now()`), ...(args.excludedDispatchIds !== undefined && args.excludedDispatchIds.length > 0 @@ -409,7 +433,9 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { and( eq(workflowRunDispatch.anchorRunId, anchorRunId), eq(workflowRunDispatch.messageId, messageId), - inArray(workflowRunDispatch.status, ["pending", "acknowledged"]), + inArray(workflowRunDispatch.status, [ + ...unresolvedWorkflowRunDispatchStatuses, + ]), ), ) .returning(); @@ -437,7 +463,9 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { and( eq(workflowRunDispatch.anchorRunId, args.anchorRunId), eq(workflowRunDispatch.messageId, args.messageId), - inArray(workflowRunDispatch.status, ["pending", "acknowledged"]), + inArray(workflowRunDispatch.status, [ + ...unresolvedWorkflowRunDispatchStatuses, + ]), ), ) .returning(); @@ -446,7 +474,12 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { : parseWorkflowRunDispatchRow(updated); }, - async failUnsettled( + /** + * End attempts alongside the run/allocation transition that fences socket + * delivery. An abandoned row's `nextAttemptAt` marks it as owed an evidence + * scan; delivery never claims it, since claims require `pending`. + */ + async abandonUnsettled( anchorRunId: string, code: string, message: string, @@ -456,8 +489,8 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { const rows = await (tx ?? db) .update(workflowRunDispatch) .set({ - status: "failed", - nextAttemptAt: null, + status: "abandoned", + nextAttemptAt: now, deliveryLeaseId: null, deliveryLeaseExpiresAt: null, failureCode: code, @@ -475,6 +508,7 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { }, async requeueUnsettled(anchorRunId: string): Promise { + // Abandonment ends delivery attempts even if this allocation reconnects. const rows = await db .update(workflowRunDispatch) .set({ @@ -505,6 +539,27 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { return row === undefined ? null : parseWorkflowRunDispatchRow(row); }, + /** Stop scanning abandoned rows once their deployment's history is final. */ + async concludeAbandonedScans( + anchorRunId: string, + messageIds: readonly string[], + now = new Date(), + ): Promise { + if (messageIds.length === 0) return 0; + const rows = await db + .update(workflowRunDispatch) + .set({ nextAttemptAt: null, updatedAt: now }) + .where( + and( + eq(workflowRunDispatch.anchorRunId, anchorRunId), + eq(workflowRunDispatch.status, "abandoned"), + inArray(workflowRunDispatch.messageId, [...messageIds]), + ), + ) + .returning({ id: workflowRunDispatch.id }); + return rows.length; + }, + async listUnsettled(anchorRunId: string): Promise { const rows = await db .select() @@ -512,7 +567,9 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { .where( and( eq(workflowRunDispatch.anchorRunId, anchorRunId), - inArray(workflowRunDispatch.status, ["pending", "acknowledged"]), + inArray(workflowRunDispatch.status, [ + ...unresolvedWorkflowRunDispatchStatuses, + ]), ), ) .orderBy(asc(workflowRunDispatch.createdAt)); diff --git a/vendor/intx/db/tsconfig.json b/vendor/intx/db/tsconfig.json index 150fc8dd99..0d87322d17 100644 --- a/vendor/intx/db/tsconfig.json +++ b/vendor/intx/db/tsconfig.json @@ -15,6 +15,12 @@ ], "exclude": [], "references": [ + { + "path": "../authz/tsconfig.src.json" + }, + { + "path": "../hub-common/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/db/tsconfig.src.json b/vendor/intx/db/tsconfig.src.json index cb9a26719f..0c9c7e8808 100644 --- a/vendor/intx/db/tsconfig.src.json +++ b/vendor/intx/db/tsconfig.src.json @@ -19,6 +19,12 @@ "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" }, "references": [ + { + "path": "../authz/tsconfig.src.json" + }, + { + "path": "../hub-common/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/harness/VENDORED-FROM b/vendor/intx/harness/VENDORED-FROM index 1b518c3534..7c43a79ca6 100644 --- a/vendor/intx/harness/VENDORED-FROM +++ b/vendor/intx/harness/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/harness) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/harness/package.json b/vendor/intx/harness/package.json index 8ead04dc26..8164c21952 100644 --- a/vendor/intx/harness/package.json +++ b/vendor/intx/harness/package.json @@ -1,7 +1,7 @@ { "name": "@intx/harness", "description": "Mail-transport composition layer over @intx/agent adding INBOX watch and connector routing", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -15,8 +15,8 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/log": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/log": "0.5.0", "@intx/types": "workspace:*" }, "devDependencies": { diff --git a/vendor/intx/harness/src/action-credential-capability.ts b/vendor/intx/harness/src/action-credential-capability.ts new file mode 100644 index 0000000000..d8d6929cc4 --- /dev/null +++ b/vendor/intx/harness/src/action-credential-capability.ts @@ -0,0 +1,102 @@ +// The credentials capability a workflow action handler resolves its declared +// handles through. Each invocation builds one capability for the handler's +// consumer identity, `action:`, from the live material cell and the +// action step's grants, through the same builder tool packages use. The action +// invoker disposes it when the handler settles. +// +// The request and result types are structural. `@intx/workflow` owns the +// `ActionCredentialsFactory` type, and this package does not depend on it, so +// the factory here is assignable to that type without an import. A host that +// wires it in checks the assignment where both packages are in scope. + +import { actionConsumer } from "@intx/authz"; +import type { HostCredentialCapability } from "@intx/types"; +import type { GrantRule } from "@intx/types/authz"; + +import { createCredentialCapability } from "./credential-capability"; +import type { CredentialProviderRegistry } from "./credential-providers"; +import { + buildConsumerCredentialCapability, + type CredentialMaterialCell, +} from "./step-credential-capabilities"; + +/** + * The run inputs an action credentials factory builds a capability from: the + * live material cell, the grants of the step an invocation runs in, and the + * provider registry that shapes a credential into a mediated handle. + */ +export interface ActionCredentialWiring { + readonly materialCell: CredentialMaterialCell; + /** + * The grants of the step an invocation runs in, for Gate 2. It receives the + * run id as well as the step id, so one factory can serve a parent run and + * its child runs, whose step ids can repeat. It runs only when the handler + * declares a handle or has a binding. + */ + readonly resolveStepGrants: (step: { + readonly runId?: string; + readonly stepId: string; + }) => readonly GrantRule[]; + readonly providers: CredentialProviderRegistry; +} + +/** One action invocation's request for a credentials capability. */ +export interface ActionCredentialRequest { + readonly handler: string; + readonly credentials: readonly string[]; + readonly authzContext: { + readonly runId?: string; + readonly stepId?: string; + }; +} + +/** + * Build the action credentials factory over a run's credential wiring. + * + * The capability is built per invocation, so each invocation reads its own + * step's grants and the bindings in the cell at that moment, and the invoker + * can dispose it without affecting the next invocation. + * + * The declared-versus-bound reconcile checks one direction: a declared handle + * with no binding fails the build, before the handler runs. A binding for a + * handle no action step with this handler declared is refused by the approval + * gate. The capability serves every handle bound to `action:`, so + * steps that share a handler share its handles. + */ +export function createActionCredentialsFactory( + wiring: ActionCredentialWiring, +): (request: ActionCredentialRequest) => HostCredentialCapability { + return ({ handler, credentials, authzContext }) => { + const { runId, stepId } = authzContext; + if (stepId === undefined) { + throw new Error( + `action ${handler} cannot build its credentials capability: the authz context carries no step id`, + ); + } + const consumer = actionConsumer(handler); + const capability = buildConsumerCredentialCapability( + consumer, + credentials.map((handle) => ({ handle })), + { + materialCell: wiring.materialCell, + resolveGrants: () => + wiring.resolveStepGrants( + runId === undefined ? { stepId } : { runId, stepId }, + ), + providers: wiring.providers, + }, + ); + // No declaration and no binding: there is nothing to resolve, so Gate 2 + // never runs and empty grants are correct. `resolve` fails closed naming + // the consumer and the handle. + return ( + capability ?? + createCredentialCapability({ + consumer, + bindings: new Map(), + providers: wiring.providers, + grants: [], + }) + ); + }; +} diff --git a/vendor/intx/harness/src/credential-capability.ts b/vendor/intx/harness/src/credential-capability.ts index 4705a16b5f..d4196eb894 100644 --- a/vendor/intx/harness/src/credential-capability.ts +++ b/vendor/intx/harness/src/credential-capability.ts @@ -1,5 +1,5 @@ -// The consumer-gated `credentials` capability: the sub-registry a tool queries -// by its declared handle to obtain a mediated credential. It is the runtime +// The consumer-gated `credentials` capability: the sub-registry a consumer (a +// tool package or an action handler) queries by its declared handle to obtain a mediated credential. It is the runtime // gate that enforces the `{ tool }` condition on a materialized // `credential:{id}` / `use` grant -- the check the launch-time grant // materialization sets up but does not itself evaluate. @@ -16,8 +16,8 @@ import { type GrantRule, } from "@intx/authz"; import type { - CredentialCapability, CredentialMaterialSource, + HostCredentialCapability, MediatedCredential, } from "@intx/types"; import type { ToolCredentialDeclaration } from "@intx/types/package-json"; @@ -43,12 +43,12 @@ export interface ResolvedCredentialBinding { } /** - * Reconcile a tool package's declared credential handles (its C5 `interchange. - * credentials`) against the handles a binding actually resolved for it. A - * declared handle with no binding is a launch-blocking misconfiguration -- the - * tool needs a credential the definition never bound -- so this fails the launch - * loudly rather than letting the gap surface as a resolve-time throw at the - * tool's first use. It is the throw-on-missing of `resolve`, pulled earlier to + * Reconcile a consumer's declared credential handles (a tool package's C5 + * `interchange.credentials`, or an action's `credentials`) against the handles + * a binding actually resolved for it. A declared handle with no binding is a + * launch-blocking misconfiguration -- the consumer needs a credential the + * definition never bound -- so this fails the launch loudly rather than letting + * the gap surface as a resolve-time throw at the consumer's first use. It is the throw-on-missing of `resolve`, pulled earlier to * launch where the whole set is known. */ export function reconcileDeclaredCredentials( @@ -68,12 +68,12 @@ export function reconcileDeclaredCredentials( export interface CredentialCapabilityDeps { /** - * The consumer identity of the tool package this capability serves - * (`tool:`, from `toolConsumer`). Gate 2 checks each grant's + * The consumer identity this capability serves (`tool:` from + * `toolConsumer`, or `action:` from `actionConsumer`). Gate 2 checks each grant's * `{ tool }` condition against this value; an empty identity fails closed. */ consumer: string; - /** Resolved bindings keyed by the handle the tool declared. */ + /** Resolved bindings keyed by the handle the consumer declared. */ bindings: ReadonlyMap; /** The registry that shapes a credential into a mediated handle. */ providers: CredentialProviderRegistry; @@ -82,17 +82,7 @@ export interface CredentialCapabilityDeps { } /** - * A `CredentialCapability` plus a host-only `dispose`. The tool sees only - * `resolve`; the host runs `dispose` on teardown to release every handle shaped - * through this capability (an http handle holds nothing; a future key-file / - * socket handle would). - */ -export interface HostCredentialCapability extends CredentialCapability { - dispose(): Promise; -} - -/** - * Build the consumer-gated `credentials` capability for one tool package. + * Build the consumer-gated `credentials` capability for one consumer. * * `resolve(handle)` fails closed at every step: an unbound handle throws; a * handle the consumer is not authorized to use throws (Gate 2 -- the same diff --git a/vendor/intx/harness/src/harness.ts b/vendor/intx/harness/src/harness.ts index 0d9cceece1..4eb117712f 100644 --- a/vendor/intx/harness/src/harness.ts +++ b/vendor/intx/harness/src/harness.ts @@ -31,7 +31,6 @@ import type { InferenceSource, MessageRef, MessageTransport, - Unsubscribe, } from "@intx/types/runtime"; import { createConnectorRouter, type RouteDecision } from "./connector-router"; @@ -312,8 +311,8 @@ export async function createHarness( // From here through the final `return`, the agent is constructed // and the workdir lock is held. Anything that throws -- the - // `driveConnectorReplies` setup, `transport.watch()`, - // anything in the watch callback's synchronous registration -- has + // `driveConnectorReplies` setup, `transport.watch()` rejecting, + // a throw from the watch callback's registration -- has // to release the lock by closing the agent before re-raising; the // caller never sees the agent and cannot do it themselves. // `createAgent` covers its own internal failure paths via its @@ -382,11 +381,12 @@ export async function createHarness( } } - // INBOX watch loop. Subscribe before the agent's reactor is fully - // settled so no message is missed in the window between subscription - // and the first watch callback. + // INBOX watch loop. The transport registers the callback before its + // promise resolves, so a message that arrives during acceptance is + // not missed. A refusal rejects inside this try, and the finally + // closes the agent. let stopped = false; - const unsubscribe: Unsubscribe = transport.watch("INBOX", (event) => { + const unsubscribe = await transport.watch("INBOX", (event) => { if (stopped) return; if (event.type !== "exists") return; @@ -502,7 +502,7 @@ export async function createHarness( } finally { if (!harnessSucceeded) { // Close the agent without waiting on its shutdown timeout so a - // synchronous post-`createAgent` throw does not stall the + // throw after `createAgent` does not stall the // caller's failure path. The `.catch` swallows any rejection // from the close: the caller is already receiving the original // throw, and a noisier-than-original close failure here would diff --git a/vendor/intx/harness/src/index.ts b/vendor/intx/harness/src/index.ts index 4d82eb160e..449a689f4a 100644 --- a/vendor/intx/harness/src/index.ts +++ b/vendor/intx/harness/src/index.ts @@ -27,10 +27,25 @@ export { } from "./credential-capability"; export type { CredentialCapabilityDeps, - HostCredentialCapability, ResolvedCredentialBinding, } from "./credential-capability"; +export { + buildConsumerCredentialCapability, + buildCredentialCapabilities, + createInferenceCredentialResolver, +} from "./step-credential-capabilities"; +export type { + CredentialMaterialCell, + StepCredentialWiring, +} from "./step-credential-capabilities"; + +export { createActionCredentialsFactory } from "./action-credential-capability"; +export type { + ActionCredentialRequest, + ActionCredentialWiring, +} from "./action-credential-capability"; + export { createConnectorRouter, NoActiveConnectorThreadError, diff --git a/vendor/intx/harness/src/reply-drain.ts b/vendor/intx/harness/src/reply-drain.ts index 6cf031a653..234f2c7c61 100644 --- a/vendor/intx/harness/src/reply-drain.ts +++ b/vendor/intx/harness/src/reply-drain.ts @@ -61,9 +61,8 @@ export interface ConnectorReplyDrainOpts { * located (the very first reply on a fresh thread, or a malformed id); the * drain then omits `references` and the transport derives `[inReplyTo]`. * - * Optional: a caller with no mailbox to consult (`createHarness`) omits it, - * leaving the pre-existing single-element threading unchanged. The warm - * workflow-host wiring supplies it from the deployment's committed mailbox. + * Optional. When it is omitted, the drain leaves `references` unset. + * `createHarness` omits it, and the transport then derives `[inReplyTo]`. */ resolveReferences?: (inReplyTo: string) => Promise; /** diff --git a/vendor/intx/harness/src/step-credential-capabilities.ts b/vendor/intx/harness/src/step-credential-capabilities.ts new file mode 100644 index 0000000000..7d50419e89 --- /dev/null +++ b/vendor/intx/harness/src/step-credential-capabilities.ts @@ -0,0 +1,281 @@ +// Per-consumer assembly of the runtime `credentials` capability for a step's +// tool bundles. +// +// A step can carry tool factories from several packages, and each package is a +// distinct credential consumer (`toolConsumer(packageName)`). Gate 2 -- the +// `{ tool }` condition on a `credential:{id}` / `use` grant -- is baked into +// each capability at construction, so a single capability shared across a +// step's packages would be a confused-deputy hole (package A's grant would +// authorize package B's resolve). This module therefore builds ONE capability +// per package, keyed by package name, and the bundle-invocation site layers +// each package's capability onto only that package's bundles. +// +// Two invariants this module owns, both fail-closed: +// 1. Material is read from the LIVE delivery cell on every use, never +// snapshotted at shape time. A rotation updates the cell in place; a +// re-push that revokes a consumer's access DROPS the material, and an +// already-shaped handle must starve the instant that happens. This live +// read is the revocation-enforcement surface -- Gate 2 stays a +// launch/first-resolve gate (Seam A memoizes it), so mid-deploy +// revocation takes effect here, at the material, not at the gate. +// 2. The declared-vs-bound reconcile runs per consumer against that +// consumer's OWN bound set, so package A's declaration can never be +// satisfied by a handle bound for package B. + +import { toolConsumer } from "@intx/authz"; +import type { + CredentialMaterial, + CredentialMaterialResolver, + CredentialMaterialSource, + HostCredentialCapability, +} from "@intx/types"; +import type { GrantRule } from "@intx/types/authz"; +import type { ToolCredentialDeclaration } from "@intx/types/package-json"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; + +import { + createCredentialCapability, + reconcileDeclaredCredentials, + type ResolvedCredentialBinding, +} from "./credential-capability"; +import type { CredentialProviderRegistry } from "./credential-providers"; + +/** + * The mutable cell the control channel writes each credential delivery into. + * Held structurally (rather than importing `CredentialMaterialRef` from + * `@intx/workflow-host`) so this module stays free of the transport package and + * a test can drive it with a plain object. + */ +export interface CredentialMaterialCell { + readonly current: CredentialDelivery | null; +} + +/** + * Everything the per-consumer assembly needs beyond the step's factories: the + * live material cell, the step's grants for Gate 2, and the provider registry + * that shapes a credential into a mediated handle. The sidecar assembles this + * at the invoke-step boundary -- the material cell and grants ride in from the + * run child, the providers are sidecar-static. + */ +export interface StepCredentialWiring { + readonly materialCell: CredentialMaterialCell; + /** + * Resolves the step's grants for Gate 2. A thunk rather than a resolved + * array, so the grants are read only when a capability is actually built: a + * step with no credential consumer never touches them. This matters + * on the resume path -- a run resumed by self-discovery does not pass through + * the pre-trigger grants barrier, so the credentials snapshot a Gate-2 read + * needs may not be present yet; a toolless resume must not fault on that. + */ + readonly resolveGrants: () => readonly GrantRule[]; + readonly providers: CredentialProviderRegistry; +} + +/** + * Build the credential capabilities for a step's tool factories, one per + * distinct package, keyed by package name. + * + * A package with neither a declared nor a bound credential gets no entry: its + * bundles keep the base capabilities bag untouched, so `resolve("credentials")` + * fails closed as "not provided by host" rather than handing back an empty + * sub-registry. Every returned capability owns a `dispose` the caller must run + * on teardown. + * + * The factory list is structural. The body reads only `packageName` and + * `declaredCredentials`. Importing `StepToolFactory` or `@intx/tool-packaging` + * is rejected because `bin/check-deps.ts` counts `import type`, and that + * dependency does not belong on this package. + */ +export function buildCredentialCapabilities( + factories: readonly { + readonly packageName: string; + readonly declaredCredentials: readonly ToolCredentialDeclaration[]; + }[], + wiring: StepCredentialWiring, +): Map { + const byPackage = new Map(); + const seenPackages = new Set(); + // Resolve the step's grants at most once, and only if some package actually + // needs a capability -- a step with no credential-consuming package never + // reads them (see StepCredentialWiring.resolveGrants). + let grants: readonly GrantRule[] | undefined; + const stepWiring: StepCredentialWiring = { + ...wiring, + resolveGrants: () => (grants ??= wiring.resolveGrants()), + }; + + for (const stf of factories) { + if (seenPackages.has(stf.packageName)) continue; + seenPackages.add(stf.packageName); + + const capability = buildConsumerCredentialCapability( + toolConsumer(stf.packageName), + stf.declaredCredentials, + stepWiring, + ); + if (capability !== undefined) { + byPackage.set(stf.packageName, capability); + } + } + + return byPackage; +} + +/** + * Build the credential capability for one consumer identity, from the bindings + * the live delivery addresses to that consumer. + * + * Returns `undefined` when the consumer declares no credential and the delivery + * binds none to it: there is nothing to resolve, so no capability is built and + * the grants are never read. A consumer that declares a handle always has a + * binding here, because the reconcile below throws first when one is missing. + * + * Otherwise the grants are read once for this call and the capability is + * returned. Nothing is cached across calls, so a caller that builds a + * capability per invocation reads that invocation's grants. + */ +export function buildConsumerCredentialCapability( + consumer: string, + declared: readonly ToolCredentialDeclaration[], + wiring: StepCredentialWiring, +): HostCredentialCapability | undefined { + const bindings = buildConsumerBindings(consumer, wiring.materialCell); + + // Fail the launch closed if the consumer declares a handle no binding + // resolves for it -- it needs a credential the definition never bound. + // Checked against THIS consumer's bound set only (invariant 2). + reconcileDeclaredCredentials(consumer, declared, new Set(bindings.keys())); + + if (declared.length === 0 && bindings.size === 0) { + return undefined; + } + + return createCredentialCapability({ + consumer, + bindings, + providers: wiring.providers, + grants: [...wiring.resolveGrants()], + }); +} + +/** + * Join the delivery's descriptors and materials into the binding map one + * consumer's capability is built from: every descriptor addressed to this + * consumer, resolved to the material that backs its credential. A descriptor + * whose material is absent is a malformed delivery and fails closed at build. + */ +function buildConsumerBindings( + consumer: string, + cell: CredentialMaterialCell, +): Map { + const bindings = new Map(); + const delivery = cell.current; + if (delivery === null) return bindings; + + for (const descriptor of delivery.bindings) { + if (descriptor.consumer !== consumer) continue; + + const material = delivery.materials.find( + (entry) => entry.credentialId === descriptor.credentialId, + ); + if (material === undefined) { + // The hub sends a material for every descriptor's credential; a + // descriptor with no material is a delivery bug. Refuse it at build + // rather than let the handle resolve to nothing at first use. + throw new Error( + `credential delivery is malformed: descriptor for handle "${descriptor.handle}" (consumer ${consumer}) references credential ${descriptor.credentialId} but the delivery carries no material for it`, + ); + } + + // Capture the provider key and origin at build time (they identify the + // credential and its pinned origin); the secret is read live per use. + bindings.set(descriptor.handle, { + credentialId: descriptor.credentialId, + providerKey: material.providerKey, + origin: material.origin, + readCurrentMaterial: makeReadCurrentMaterial({ + cell, + credentialId: descriptor.credentialId, + providerKey: material.providerKey, + origin: material.origin, + consumer, + }), + }); + } + + return bindings; +} + +/** + * The rotation/revocation indirection a shaped handle reads through: it looks + * the credential's material up in the LIVE cell on every call (invariant 1), + * failing closed when the material is gone (a re-push dropped it) and asserting + * the provider/origin have not drifted under the already-shaped handle. + */ +function makeReadCurrentMaterial(args: { + cell: CredentialMaterialCell; + credentialId: string; + providerKey: string; + origin: string; + consumer: string; +}): CredentialMaterialSource { + const { cell, credentialId, providerKey, origin, consumer } = args; + return () => { + const delivery = cell.current; + if (delivery === null) { + throw new Error( + `credential material for ${credentialId} (consumer ${consumer}) is not available: the delivery cell is empty`, + ); + } + const material = delivery.materials.find( + (entry) => entry.credentialId === credentialId, + ); + if (material === undefined) { + throw new Error( + `credential material for ${credentialId} (consumer ${consumer}) is no longer delivered: a re-push dropped it (rotated away or revoked)`, + ); + } + // A rotation changes the SECRET, not the provider or origin. A live entry + // whose provider/origin diverged would mean the shaped handle now + // authenticates somewhere it was not pinned to; surface it rather than + // silently follow the change. + if (material.providerKey !== providerKey || material.origin !== origin) { + throw new Error( + `credential ${credentialId} changed provider/origin under an already-shaped handle (${providerKey}@${origin} -> ${material.providerKey}@${material.origin}); a shaped handle cannot follow that change`, + ); + } + return { secret: material.secret }; + }; +} + +/** + * A generic inference credential resolver over the live cell: resolves a + * credential's current secret by `credentialId`, failing closed when the cell is + * empty or the credential is absent (a re-push dropped it -- rotated away or + * revoked). Unlike `makeReadCurrentMaterial` it is keyed by `credentialId` at + * call time -- an inference source's forward-only failover chain carries a + * distinct credential per entry -- and pins no provider/origin: an inference + * request authenticates to the source's own `baseURL` (resolved separately), + * so there is no shaped handle to protect against origin drift. + */ +export function createInferenceCredentialResolver( + cell: CredentialMaterialCell, +): CredentialMaterialResolver { + return (credentialId: string): CredentialMaterial => { + const delivery = cell.current; + if (delivery === null) { + throw new Error( + `inference credential material for ${credentialId} is not available: the delivery cell is empty`, + ); + } + const material = delivery.materials.find( + (entry) => entry.credentialId === credentialId, + ); + if (material === undefined) { + throw new Error( + `inference credential material for ${credentialId} is no longer delivered: a re-push dropped it (rotated away or revoked)`, + ); + } + return { secret: material.secret }; + }; +} diff --git a/vendor/intx/harness/tsconfig.json b/vendor/intx/harness/tsconfig.json index 646ea17781..af5253650c 100644 --- a/vendor/intx/harness/tsconfig.json +++ b/vendor/intx/harness/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/harness/tsconfig.src.json b/vendor/intx/harness/tsconfig.src.json index 19b7665ecc..81e181820f 100644 --- a/vendor/intx/harness/tsconfig.src.json +++ b/vendor/intx/harness/tsconfig.src.json @@ -22,6 +22,9 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/hub-agent/README.md b/vendor/intx/hub-agent/README.md index 9beed284cc..07c8dc64c7 100644 --- a/vendor/intx/hub-agent/README.md +++ b/vendor/intx/hub-agent/README.md @@ -10,7 +10,7 @@ In-process harness construction and agent provisioning are retired: every agent now runs as a supervised workflow-process child on the workflow-run substrate. What remains in `createSessionManager` is a thin serialization layer over the agent repo store (deploy/asset-pack -applies, state-pack reads, deploy-ref reads, directory teardown); +applies and directory teardown); operations run one at a time per agent so a teardown never races an in-flight git op. @@ -20,12 +20,14 @@ process into a host for hub-deployed agents. `createSidecarOrchestrator` takes a `SidecarOrchestratorConfig` specifying how to reach the hub (`hubURL`, `sidecarId`, `token`, `transport`), where to persist agent state (`dataDir`), the sidecar's -crypto operations (`cryptoOps`), and the host-injected deploy-router +crypto operations (`cryptoOps`), the deployment limits every `hello` +advertises (`capacity`), and the host-injected deploy-router factory (`createDeployRouter`) that routes every `agent.deploy` frame on the link. Optional fields supply the multi-step inbound routers (`mailInboundRouter`, `signalInboundRouter`, `drainInboundRouter`, -`sourcesInboundRouter`), the workflow-address announce and routability -hooks, and the reconnect cadence. See `SidecarOrchestratorConfig` in +`sourcesInboundRouter`), the incarnation report the `hello` carries +(`getIncarnations`) and the routability hooks, and the reconnect +cadence. See `SidecarOrchestratorConfig` in `src/sidecar-orchestrator.ts` for the full surface. `HarnessBuilder` is a one-method source-admission seam diff --git a/vendor/intx/hub-agent/VENDORED-FROM b/vendor/intx/hub-agent/VENDORED-FROM index 473ab79149..e1a4c4f49f 100644 --- a/vendor/intx/hub-agent/VENDORED-FROM +++ b/vendor/intx/hub-agent/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-agent) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: SessionEventSink and createSidecarOrchestrator's dispatchEvent/publishWorkflowInferenceEvent take an optional childRunId, and createHubLink's sendEvent stamps it on agent.event so overlapping turns close the right occurrence row. CL-7508: createSidecarOrchestrator takes an optional oauthLoginExecutor and forwards it to createHubLink, which routes every inbound oauth.login.start through it (started arm on authorize URL, exactly one terminal arm); omitted, a fail-closed placeholder answers with an error instead of hanging. Absorbed upstream at this re-vendor: the inbound-mail-policy-registry mismatchedFrom/absentFrom reject defaults, the inbound-signature AuthorControllableOutcome findings, and the sender-key-cache load-fault reporting. +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: SessionEventSink and createSidecarOrchestrator's dispatchEvent/publishWorkflowInferenceEvent take an optional childRunId, and createHubLink's sendEvent stamps it on agent.event so overlapping turns close the right occurrence row. Retired at this pin: the CL-7508 oauthLoginExecutor seam (OAuth loopback login runs in-process on the hub via @corbits/oauth-core/hub). diff --git a/vendor/intx/hub-agent/package.json b/vendor/intx/hub-agent/package.json index 864f9a1c08..2316ce3f00 100644 --- a/vendor/intx/hub-agent/package.json +++ b/vendor/intx/hub-agent/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-agent", "description": "Sidecar-side orchestrator linking per-agent stores and sessions to the hub over WebSocket", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -18,12 +18,12 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/harness": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", + "@intx/mime": "workspace:*", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:", "isomorphic-git": "catalog:" diff --git a/vendor/intx/hub-agent/src/agent-key-store.ts b/vendor/intx/hub-agent/src/agent-key-store.ts index c64900db9c..80ef13c2f1 100644 --- a/vendor/intx/hub-agent/src/agent-key-store.ts +++ b/vendor/intx/hub-agent/src/agent-key-store.ts @@ -12,7 +12,8 @@ // verifyDeployCommit for incoming deploy packs. import fsp from "node:fs/promises"; -import { hasCode, hexDecode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; +import { hexDecode } from "@intx/types/hex"; import type { KeyPair } from "@intx/types/runtime"; import { keysDir, privateKeyPath, publicKeyPath } from "./agent-paths"; diff --git a/vendor/intx/hub-agent/src/agent-repo-store.ts b/vendor/intx/hub-agent/src/agent-repo-store.ts index c593354a76..ae328b1f27 100644 --- a/vendor/intx/hub-agent/src/agent-repo-store.ts +++ b/vendor/intx/hub-agent/src/agent-repo-store.ts @@ -1,16 +1,15 @@ // Per-agent on-disk repository layout. // -// Owns the isogit repo wrapper and the deploy-pack apply / state-pack -// produce flow. Key custody lives in AgentKeyStore alongside this -// store; both share the directory-layout helpers in agent-paths. +// Owns the isogit repo wrapper and the deploy-pack apply flow. Key custody +// lives in AgentKeyStore alongside this store; both share the +// directory-layout helpers in agent-paths. import fsp from "node:fs/promises"; import { getLogger } from "@intx/log"; import { initAgentRepo, applyPack, - createDeployPack, - currentBranch, + syncRemovedPathDurable, type CommitVerifier, } from "@intx/storage-isogit/node"; @@ -36,9 +35,6 @@ export type AgentRepoStore = { getAgentDir(address: string): string; initRepo(address: string): Promise; applyDeployPack(args: ApplyDeployPackArgs): Promise; - createStatePack( - address: string, - ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; remove(address: string): Promise; }; @@ -68,18 +64,9 @@ export function createAgentRepoStore(config: { logger.info`Applied deploy pack for ${address} at ${commitSha.slice(0, 8)}`; } - async function createStatePack( - address: string, - ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }> { - const dir = getAgentDir(address); - const branch = await currentBranch(dir); - const ref = `refs/heads/${branch}`; - const { pack, commitSha } = await createDeployPack(dir, ref); - return { pack, commitSha, ref }; - } - async function remove(address: string): Promise { - await fsp.rm(getAgentDir(address), { recursive: true }); + await fsp.rm(getAgentDir(address), { recursive: true, force: true }); + await syncRemovedPathDurable(getAgentDir(address)); logger.info`Deleted agent directory for ${address}`; } @@ -87,7 +74,6 @@ export function createAgentRepoStore(config: { getAgentDir, initRepo, applyDeployPack: applyDeployPackImpl, - createStatePack, remove, }; } diff --git a/vendor/intx/hub-agent/src/deploy-tree.ts b/vendor/intx/hub-agent/src/deploy-tree.ts index 7afd24c434..3c2aedc9b8 100644 --- a/vendor/intx/hub-agent/src/deploy-tree.ts +++ b/vendor/intx/hub-agent/src/deploy-tree.ts @@ -16,7 +16,7 @@ import fs from "node:fs"; import path from "node:path"; import { type } from "arktype"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; const AssetMountsFile = type({ assetMounts: type({ "[string]": "string" }), diff --git a/vendor/intx/hub-agent/src/index.ts b/vendor/intx/hub-agent/src/index.ts index 1ec88a3839..168f4f0f55 100644 --- a/vendor/intx/hub-agent/src/index.ts +++ b/vendor/intx/hub-agent/src/index.ts @@ -35,6 +35,7 @@ export { } from "./session-manager"; export { createHubLink, + isConnectionLost, type DeployRouter, type DeployRouterResult, type HubLink, @@ -45,6 +46,7 @@ export { type GrantsInboundRouter, type WorkflowRunPackApplier, type ReconnectScheduler, + type WorkflowControlOutcome, } from "./ws/hub-link"; export { createSidecarOrchestrator, @@ -54,5 +56,12 @@ export { type SidecarCryptoOps, } from "./sidecar-orchestrator"; export { applyAssetPack, type ApplyAssetPackArgs } from "./apply-asset-pack"; +export { + assetReferenceFormats, + indexAssetPackIntoGitDir, + materializeWorkflowAssets, + sourceAssetGitDir, + MAX_INLINE_ASSET_PAYLOAD_BYTES, +} from "./source-asset-delivery"; export { readDeployTree, type DeployTree } from "./deploy-tree"; export { agentDir, sanitizeAddress } from "./agent-paths"; diff --git a/vendor/intx/hub-agent/src/sender-key-cache.ts b/vendor/intx/hub-agent/src/sender-key-cache.ts index 03aa7f694a..aa4871168e 100644 --- a/vendor/intx/hub-agent/src/sender-key-cache.ts +++ b/vendor/intx/hub-agent/src/sender-key-cache.ts @@ -41,13 +41,9 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { type } from "arktype"; import { getLogger } from "@intx/log"; -import { - hasCode, - hexDecode, - hexEncode, - isRunAddress, - parseAddress, -} from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; +import { hexDecode, hexEncode } from "@intx/types/hex"; +import { isRunAddress, parseAddress } from "@intx/types/agent-address"; const logger = getLogger(["interchange", "hub-agent", "sender-key-cache"]); @@ -72,6 +68,15 @@ const StoredSenderKey = type({ publicKey: "string", }); +/** Whether `hex` encodes a key the cache can hold: a raw Ed25519 public key. */ +export function isSenderPublicKeyHex(hex: string): boolean { + try { + return hexDecode(hex).length === ED25519_PUBLIC_KEY_BYTES; + } catch { + return false; + } +} + export type SenderKeyCacheDeps = { dataDir: string; /** diff --git a/vendor/intx/hub-agent/src/session-manager.ts b/vendor/intx/hub-agent/src/session-manager.ts index b261e1b79a..38e1d9b022 100644 --- a/vendor/intx/hub-agent/src/session-manager.ts +++ b/vendor/intx/hub-agent/src/session-manager.ts @@ -5,9 +5,9 @@ // retired: every agent now runs as a supervised workflow-process child // on the workflow-run substrate. What remains here is the thin // serialization layer over the agent repo store that the deploy path -// and the hub-link still call: deploy/asset-pack applies, state-pack -// reads, and directory teardown, each run one-at-a-time per agent so a -// teardown never races an in-flight git op. +// and the hub-link still call: deploy/asset-pack applies and directory +// teardown, each run one-at-a-time per agent so a teardown never races an +// in-flight git op. import path from "node:path"; @@ -50,16 +50,7 @@ export type SessionManager = { ref: string, commitSha: string, ): Promise; - createStatePack( - agentAddress: string, - ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }>; deleteAgentDir(agentAddress: string): Promise; - /** - * Session addresses this manager hosts. The in-process session runtime - * is retired, so this is always empty; the hub-link ships it in the - * register frame alongside the sidecar's workflow-deployment addresses. - */ - getAddresses(): string[]; /** * Session id for an address' outbound mail forwarding. Always undefined * now that no in-process sessions exist; the hub-link tolerates a @@ -74,16 +65,15 @@ export function createSessionManager( const { repoStore } = config; // Per-agent promise chain that serializes the operations against an agent's - // on-disk directory -- state-pack reads and deploy/asset-pack applies all run - // one-at-a-time per agent. The chain exists for teardown: - // drainRepoOps awaits it before deleting the directory, so an operation that - // was valid when it started never runs against a path that has since - // vanished underneath it. Serializing additionally avoids corruption for the - // members that share the agent's `.git/` object store (state-pack reads and - // deploy-pack applies), which isogit, lacking a cross-process lock, would - // otherwise let interleave. Asset-pack applies are on the chain only for the - // teardown reason -- they materialize into a workspace subtree, not the agent - // repo's object store. + // on-disk directory -- deploy/asset-pack applies run one-at-a-time per agent. + // The chain exists for teardown: drainRepoOps awaits it before deleting the + // directory, so an operation that was valid when it started never runs + // against a path that has since vanished underneath it. Serializing + // additionally keeps two deploy-pack applies from interleaving in the agent's + // `.git/` object store, which isogit, lacking a cross-process lock, would + // otherwise allow. Asset-pack applies are on the chain only for the teardown + // reason -- they materialize into a workspace subtree, not the agent repo's + // object store. const repoOpQueues = new Map>(); function runRepoOp( @@ -114,10 +104,11 @@ export function createSessionManager( // directory only after in-flight git work finishes. Capturing the tail and // clearing the entry means an op enqueued AFTER this point starts a fresh // chain this drain does not await. That is safe only because every caller - // invokes runRepoOp synchronously, before its first await, inside the - // serialized frame dispatch -- so by the time a later agent.undeploy frame - // reaches deleteAgentDir, every racing op is already on the chain. A handler - // that deferred its runRepoOp call past an await would reopen the + // invokes runRepoOp synchronously, before its first await, while handling a + // frame for the agent's address, and the link handles one address's frames + // one at a time -- so by the time a later agent.undeploy frame for the + // address reaches deleteAgentDir, every racing op is already on the chain. A + // handler that deferred its runRepoOp call past an await would reopen the // delete-under-in-flight-op race. async function drainRepoOps(agentAddress: string): Promise { const inflight = repoOpQueues.get(agentAddress); @@ -169,14 +160,6 @@ export function createSessionManager( ); } - async function createStatePack( - agentAddress: string, - ): Promise<{ pack: Uint8Array; commitSha: string; ref: string }> { - return runRepoOp(agentAddress, () => - repoStore.createStatePack(agentAddress), - ); - } - async function deleteAgentDir(agentAddress: string): Promise { await drainRepoOps(agentAddress); await repoStore.remove(agentAddress); @@ -186,9 +169,7 @@ export function createSessionManager( initRepo: (address: string) => repoStore.initRepo(address), applyDeployPack, applyAssetPack, - createStatePack, deleteAgentDir, - getAddresses: () => [], getSessionId: () => undefined, }; } diff --git a/vendor/intx/hub-agent/src/sidecar-orchestrator.ts b/vendor/intx/hub-agent/src/sidecar-orchestrator.ts index 311b900db5..f34a80b30e 100644 --- a/vendor/intx/hub-agent/src/sidecar-orchestrator.ts +++ b/vendor/intx/hub-agent/src/sidecar-orchestrator.ts @@ -14,6 +14,11 @@ import { getLogger } from "@intx/log"; import type { HubTransport } from "@intx/mail-memory"; import type { SignalKind } from "@intx/types"; +import type { + DeploymentStoppedFrame, + HostedIncarnation, + SidecarCapacity, +} from "@intx/types/sidecar"; import type { ApprovalSnapshot, CryptoProvider, @@ -36,7 +41,6 @@ import { type CredentialsInboundRouter, type WorkflowRunPackApplier, type WorkflowProbeExecutor, - type OAuthLoginExecutor, type ReconnectScheduler, } from "./ws/hub-link"; import type { ResolvedInboundMailPolicy } from "./ws/inbound-signature"; @@ -66,8 +70,8 @@ export type CreateDeployRouter = (deps: { /** * Per-event sink the multi-step branch routes a spawned child's * verified `InferenceEvent`s through, keyed by the deployment's agent - * address and the deploy's session id. Wired to the same hub-link - * `agent.event` sink the in-process path's `onEvent` uses, so a step + * address, the generation of the incarnation that produced them, and the + * deploy's session id. Wired to the hub-link `agent.event` sink, so a step * agent's events reach the hub timeline keyed to the right session. * The `sessionId` is optional because a deploy frame need not carry * one (a headless deployment); the sink drops a sessionless event @@ -75,8 +79,10 @@ export type CreateDeployRouter = (deps: { */ publishWorkflowInferenceEvent: ( agentAddress: string, + generation: number, event: InferenceEvent, sessionId: string | undefined, + childRunId?: string, ) => void; /** * Control-plane suspension sink the multi-step branch routes a @@ -91,9 +97,18 @@ export type CreateDeployRouter = (deps: { runId: string; anchorRunId: string; agentAddress: string; + generation: number; kind: SignalKind; approvalSnapshot?: ApprovalSnapshot; }) => void; + /** + * Sink for a deployment that stopped though the Hub did not stop it. Wired + * to the hub-link's `sendDeploymentStopped`, like + * `publishWorkflowSuspension`. + */ + publishDeploymentStopped: ( + report: Omit, + ) => void; }) => DeployRouter; export type SidecarOrchestratorConfig = { @@ -120,8 +135,8 @@ export type SidecarOrchestratorConfig = { /** * Persists the hub-vouched public key for a sender address. The host builds * it over the same sender-key cache as `resolveSenderCrypto` and the - * orchestrator forwards it unchanged to `createHubLink`, where an inbound - * `sender.key.refresh` frame drives it. + * orchestrator forwards it unchanged to `createHubLink`, where inbound + * `sender.key.refresh` and `run.grants` frames drive it. */ cacheSenderKey: (address: string, publicKey: string) => Promise; /** @@ -181,7 +196,7 @@ export type SidecarOrchestratorConfig = { * `createHubLink`. */ sourcesInboundRouter?: SourcesInboundRouter; - /** Apply Hub-authoritative workflow-run refs before replacement deploy. */ + /** Apply Hub-authoritative workflow-run refs before a first deploy. */ applyWorkflowRunPack: WorkflowRunPackApplier; /** * Optional inbound credential-delivery dispatcher the link consults on every @@ -200,21 +215,13 @@ export type SidecarOrchestratorConfig = { */ workflowProbeExecutor?: WorkflowProbeExecutor; /** - * Optional loopback-login executor. The orchestrator forwards it - * unchanged to `createHubLink`, where it answers every inbound - * `oauth.login.start`. Production wires the sidecar host's local-machine - * PKCE login here; omitted, the link falls back to its rejecting - * placeholder so a login is answered with an error rather than hanging. - */ - oauthLoginExecutor?: OAuthLoginExecutor; - /** - * Returns the workflow-substrate deployment addresses this sidecar - * currently hosts. Forwarded to the hub link, which announces them on - * every (re)connect so the hub re-registers them for routing. - * Production wires this to the deploy router's - * `activeAddresses`; omitted, the link announces none. + * Returns every deployment incarnation this sidecar holds. Forwarded to the + * hub link, which reports them in every `hello` and delivers a Hub frame + * only to one it holds live. Production wires this to the deploy router's + * `incarnations`; omitted, the link holds none. */ - getWorkflowAddresses?: () => string[]; + getIncarnations?: () => HostedIncarnation[]; + capacity: SidecarCapacity; /** * Returns the rotatable (non-run) sender addresses this sidecar holds cached * keys for. Forwarded to the hub link, which reports them on every @@ -224,18 +231,17 @@ export type SidecarOrchestratorConfig = { */ getCachedSenderAddresses?: () => string[]; /** - * Invoked with the workflow-substrate addresses the link just announced in - * an authenticated reconnect. Forwarded to the hub link so the workflow-run - * pack pusher can re-drive a push a disconnect cancelled -- gated on the - * address becoming routable again. - * Production wires this to the boot-edge pack-pushing store's - * "address routable" notifier; omitted, the link fires nothing. + * Invoked on `welcome` with the addresses of the held incarnations the Hub + * routes. Forwarded to the hub link so the workflow-run pack pusher can + * re-drive a push a disconnect cancelled, and parked correlations are + * registered again. Production wires this to the boot-edge pack-pushing + * store's "address routable" notifier; omitted, the link fires nothing. */ onWorkflowAddressesRoutable?: (addresses: string[]) => void; /** - * Invoked on WS disconnect with the workflow-substrate addresses the link - * hosts, so the workflow-run pack pusher blocks their pushes until the - * authenticated reconnect re-routes them. Paired with + * Invoked when a connection opens and when it closes, with the address of + * every incarnation the link holds, so the workflow-run pack pusher blocks + * their pushes until the next `welcome` routes them. Paired with * `onWorkflowAddressesRoutable`. Production wires this to the boot-edge * pack-pushing store's block notifier; omitted, the link fires nothing. */ @@ -280,8 +286,8 @@ export function createSidecarOrchestrator( credentialsInboundRouter, applyWorkflowRunPack, workflowProbeExecutor, - oauthLoginExecutor, - getWorkflowAddresses, + getIncarnations, + capacity, getCachedSenderAddresses, onWorkflowAddressesRoutable, onWorkflowAddressesUnroutable, @@ -303,8 +309,10 @@ export function createSidecarOrchestrator( // sendEvent method. let dispatchEvent: ( agentAddress: string, + generation: number, sessionId: string, event: InferenceEvent, + childRunId?: string, ) => void = () => { /* replaced after HubLink construction */ }; @@ -318,12 +326,19 @@ export function createSidecarOrchestrator( runId: string; anchorRunId: string; agentAddress: string; + generation: number; kind: SignalKind; approvalSnapshot?: ApprovalSnapshot; }) => void = () => { /* replaced after HubLink construction */ }; + let dispatchDeploymentStopped: ( + report: Omit, + ) => void = () => { + /* replaced after HubLink construction */ + }; + const sessions = createSessionManager({ repoStore }); const deployRouter = createDeployRouter({ @@ -337,7 +352,13 @@ export function createSidecarOrchestrator( // is observed. A sessionless event is dropped rather than guessed // onto an arbitrary session -- the hub timeline is session-keyed and // a forged session id would mis-route the event. - publishWorkflowInferenceEvent: (agentAddress, event, sessionId) => { + publishWorkflowInferenceEvent: ( + agentAddress, + generation, + event, + sessionId, + childRunId, + ) => { if (sessionId === undefined) { log.warn( "Dropping workflow inference event for {agentAddress}: deploy carried no sessionId", @@ -345,7 +366,7 @@ export function createSidecarOrchestrator( ); return; } - dispatchEvent(agentAddress, sessionId, event); + dispatchEvent(agentAddress, generation, sessionId, event, childRunId); }, // Route a supervisor's suspension registration up the hub-link so the // hub co-writes the parked run's routing + approval rows. @@ -354,6 +375,9 @@ export function createSidecarOrchestrator( publishWorkflowSuspension: (registration) => { dispatchSuspension(registration); }, + publishDeploymentStopped: (report) => { + dispatchDeploymentStopped(report); + }, }); const hubLink = createHubLink({ @@ -378,8 +402,8 @@ export function createSidecarOrchestrator( ? { credentialsInboundRouter } : {}), ...(workflowProbeExecutor !== undefined ? { workflowProbeExecutor } : {}), - ...(oauthLoginExecutor !== undefined ? { oauthLoginExecutor } : {}), - ...(getWorkflowAddresses !== undefined ? { getWorkflowAddresses } : {}), + ...(getIncarnations !== undefined ? { getIncarnations } : {}), + capacity, ...(getCachedSenderAddresses !== undefined ? { getCachedSenderAddresses } : {}), @@ -396,6 +420,7 @@ export function createSidecarOrchestrator( dispatchEvent = hubLink.sendEvent; dispatchSuspension = hubLink.sendSignalCorrelationRegister; + dispatchDeploymentStopped = hubLink.sendDeploymentStopped; function start(): void { hubLink.connect(); diff --git a/apps/sidecar/src/source-asset-delivery.ts b/vendor/intx/hub-agent/src/source-asset-delivery.ts similarity index 61% rename from apps/sidecar/src/source-asset-delivery.ts rename to vendor/intx/hub-agent/src/source-asset-delivery.ts index 7170834a1f..7c9cd077c4 100644 --- a/apps/sidecar/src/source-asset-delivery.ts +++ b/vendor/intx/hub-agent/src/source-asset-delivery.ts @@ -1,28 +1,46 @@ -// Sidecar-side delivery of a workflow closure's source assets. A tarball -// entry checks out a plain-file mount; a source entry indexes the pack -// into a retained gitDir. One asset can be referenced both ways. +// Sidecar-side delivery of a workflow closure's source assets. +// +// A `WorkflowSourceAssetMount` carries a git pack for one hub asset. How that +// pack is materialized depends on how the closure references the asset: +// - a tarball-format entry reads a `.tgz` blob from a plain-file checkout +// (`applyAssetPack`), keyed by an `assetId -> mountPath` map; and +// - a source-format entry checks a subtree out of the git objects, so the +// pack is indexed into a RETAINED `.git` (a "gitDir"), keyed by an +// `assetId -> gitDir` map the loader hands `materializeGitEntry`. +// One asset can be referenced both ways; a source-format workflow closure +// references only source entries, so it produces only gitDirs. import fsp from "node:fs/promises"; import path from "node:path"; import { getLogger } from "@intx/log"; -import { base64Decode } from "@intx/types"; -import { applyAssetPack } from "@intx/hub-agent"; import { DEFAULT_PACK_MATERIALIZATION_LIMITS, indexPackIntoGitDir, } from "@intx/storage-isogit/node"; +import { base64Decode } from "@intx/types/base64"; import type { WorkflowSourceAssetMount } from "@intx/types/sidecar"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; +import { applyAssetPack } from "./apply-asset-pack"; + const logger = getLogger(["sidecar", "source-asset-delivery"]); const SAFE_ASSET_ID = /^[a-zA-Z0-9_.-]+$/; /** - * Builds into a sibling temp .git and renames into place so the durable - * store is complete-or-absent: a crash mid-materialization never leaves a - * partial gitDir the restore dir-exists check would trust. + * Index a delivered asset pack into `gitDir` and RETAIN the object store, so a + * source subtree can be checked out from it. Builds into a sibling temp `.git` + * and RENAMES it into place, so the durable store is complete-or-absent: a crash + * mid-materialization leaves only the temp, never a partial `gitDir` that the + * dir-exists check `resolveDeploymentAssetMounts` runs on restore would trust. + * The rename is same-filesystem (the temp is a sibling under `gitDir`'s parent). + * + * On a rename conflict (a stale `gitDir` from a torn prior attempt) the freshly + * built store wins: the existing dir is removed and the temp renamed over it, so + * a re-delivery at a new commit never keeps the old content. A secondary rm + * failure is logged so it does not silently mask state; the primary error is + * rethrown. */ export async function indexAssetPackIntoGitDir(args: { pack: Uint8Array; @@ -42,7 +60,12 @@ export async function indexAssetPackIntoGitDir(args: { }; try { - await indexPackIntoGitDir(tempDir, pack, commitSha, DEFAULT_PACK_MATERIALIZATION_LIMITS); + await indexPackIntoGitDir( + tempDir, + pack, + commitSha, + DEFAULT_PACK_MATERIALIZATION_LIMITS, + ); } catch (err) { await cleanupTemp(); throw err; @@ -51,12 +74,16 @@ export async function indexAssetPackIntoGitDir(args: { try { await fsp.rename(tempDir, gitDir); } catch (err) { - // Only "destination exists" means a torn prior attempt to supersede; - // any other rename error must not destroy a possibly-good prior store. + // Only a "destination already exists" failure means a torn prior attempt we + // may supersede; any other rename error (EXDEV, EACCES, ENOSPC, EIO) must + // NOT destroy a possibly-good prior store -- clean up only the fresh temp + // and surface it. if (!isDestinationExistsError(err)) { await cleanupTemp(); throw err; } + // The final path already holds a store (a torn prior attempt): rebuild + // wins, so drop the stale store and rename the fresh one over it. await fsp.rm(gitDir, { recursive: true, force: true }); try { await fsp.rename(tempDir, gitDir); @@ -98,15 +125,26 @@ export function assetReferenceFormats( /** The absolute gitDir a source asset's objects are indexed into. */ export function sourceAssetGitDir(gitDirRoot: string, assetId: string): string { - // SAFE_ASSET_ID permits "." as a character, so an all-dots id must be - // rejected separately or it escapes/aliases the per-asset dir. + // Reject an all-dots assetId (".", "..", ...) before the join. SAFE_ASSET_ID + // permits "." as a character, so a bare ".." would otherwise escape the + // per-asset dir (`path.join(root, "..")` is root's parent) and "." would + // resolve to the shared root itself. Mirrors `applyAssetPack`'s all-dots + // segment guard. if (!SAFE_ASSET_ID.test(assetId) || /^\.+$/.test(assetId)) { - throw new Error(`source-asset delivery: unsafe assetId ${JSON.stringify(assetId)}`); + throw new Error( + `source-asset delivery: unsafe assetId ${JSON.stringify(assetId)}`, + ); } return path.join(gitDirRoot, assetId); } -/** Shared by probe and deploy so the two paths cannot drift; exceeding it signals moving to a streamed transfer. */ +/** + * The single cap on the total inline (base64) source-asset payload a workflow + * closure may deliver in one frame. Both the probe and the deploy pass this to + * `materializeWorkflowAssets`; a git-sourced asset that grows past it is the + * signal to move that path's asset delivery to a streamed transfer. One + * constant so the two paths cannot drift. + */ export const MAX_INLINE_ASSET_PAYLOAD_BYTES = 32 * 1024 * 1024; /** @@ -143,7 +181,10 @@ export async function materializeWorkflowAssets(args: { ); } seen.add(asset.assetId); - // An asset with no closure entry is a hub/frame inconsistency; fail loud. + // The frame delivers one mount per asset the closure references, so a + // delivered asset with no closure entry is a hub/frame inconsistency. + // Fail loud rather than silently ignore it (while still counting its + // payload toward the cap above). const refs = formats.get(asset.assetId); if (refs === undefined) { throw new Error( diff --git a/vendor/intx/hub-agent/src/ws/frame-lanes.ts b/vendor/intx/hub-agent/src/ws/frame-lanes.ts new file mode 100644 index 0000000000..a956cf8242 --- /dev/null +++ b/vendor/intx/hub-agent/src/ws/frame-lanes.ts @@ -0,0 +1,28 @@ +export type FrameLane = string | symbol; + +/** + * Orders the handling of inbound hub frames. Frames that share a lane are + * handled one at a time, in the order they arrived; frames on different lanes + * do not wait on each other. + */ +export interface FrameLanes { + run(lanes: readonly FrameLane[], handle: () => Promise): void; +} + +export function createFrameLanes(onError: (err: unknown) => void): FrameLanes { + const tails = new Map>(); + + function run(lanes: readonly FrameLane[], handle: () => Promise): void { + const handled = Promise.all(lanes.flatMap((lane) => tails.get(lane) ?? [])) + .then(handle) + .catch(onError); + for (const lane of lanes) { + tails.set(lane, handled); + void handled.then(() => { + if (tails.get(lane) === handled) tails.delete(lane); + }); + } + } + + return { run }; +} diff --git a/vendor/intx/hub-agent/src/ws/hub-link.ts b/vendor/intx/hub-agent/src/ws/hub-link.ts index c2a0a684e6..00614bf459 100644 --- a/vendor/intx/hub-agent/src/ws/hub-link.ts +++ b/vendor/intx/hub-agent/src/ws/hub-link.ts @@ -1,6 +1,6 @@ // HubLink: the sidecar-side WebSocket protocol. // -// Connects to the hub, sends the register frame, forwards outbound +// Connects to the hub, sends the hello frame, forwards outbound // mail and inference events, and handles inbound agent lifecycle // commands. Per-agent key material lives on AgentKeyStore; the link calls into // the store for deploy-commit verification and hub-key bookkeeping. The wire layer itself never @@ -10,21 +10,30 @@ import { getLogger } from "@intx/log"; import type { HubTransport } from "@intx/mail-memory"; import { type } from "arktype"; import { + DeploymentRejectedError, + fitDeploymentError, + Generation, HubFrame, MAX_MAIL_OUTBOUND_BODY_BYTES, type SidecarFrame, - type RegisterFrame, - type ReconnectFrame, + type SidecarCapacity, type AgentDeployFrame, - type AgentErrorFrame, + type AgentDeployErrorFrame, + type AgentUndeployErrorFrame, type SessionErrorFrame, type AgentUndeployFrame, + type DeploymentStoppedFrame, + type DeploymentRetention, + type WorkflowControlFrame, + type WorkflowRunRefTips, + type HostedIncarnation, + type IncarnationStamp, + type MailInboundFrame, type PackPushFrame, type PackDoneFrame, type PackAckFrame, type PackRejectFrame, type PackRejectReason, - RepoId, type SignalDeliverFrame, type RunGrantsFrame, type SenderKeyRefreshFrame, @@ -34,14 +43,18 @@ import { type DrainDeliverFrame, type SourcesUpdateFrame, type CredentialsUpdateFrame, - type SyncRequestFrame, + type WelcomeFrame, type WorkflowProbeRequestFrame, type WorkflowProbeResultFrame, - type OAuthLoginResultFrame, - type OAuthLoginTokens, } from "@intx/types/sidecar"; +import { RepoId } from "@intx/types/repo"; import type { SignalKind } from "@intx/types"; -import { createPackReceiver, createPackSender } from "@intx/pack-transport"; +import { + createPackReceiver, + createPackSender, + PackRejectedError, +} from "@intx/pack-transport"; +import { createFrameLanes } from "./frame-lanes"; import { createRegisterAcker, DEFAULT_REGISTER_ACK_MAX_ATTEMPTS, @@ -51,9 +64,11 @@ import { verifyInboundSignature, outcomeForVerdict, decideInboundAdmission, + type InboundSignatureVerdict, type ResolvedInboundMailPolicy, } from "./inbound-signature"; -import { base64Decode, base64Encode } from "@intx/types"; +import { base64Decode, base64Encode } from "@intx/types/base64"; +import { isRunAddress } from "@intx/types/agent-address"; import type { ApprovalSnapshot, CryptoProvider, @@ -61,14 +76,17 @@ import type { } from "@intx/types/runtime"; import type { AgentKeyStore } from "../agent-key-store"; +import { isSenderPublicKeyHex } from "../sender-key-cache"; import type { SessionManager } from "../session-manager"; /** * Sink the link exposes for forwarding a spawned child's verified - * InferenceEvents to the hub timeline, keyed by the deploy's session id. + * InferenceEvents to the hub timeline, keyed by the deploy's session id and + * stamped with the incarnation that produced them. */ export type SessionEventSink = ( agentAddress: string, + generation: number, sessionId: string, event: InferenceEvent, childRunId?: string, @@ -88,11 +106,12 @@ const MalformedRequestEnvelope = type({ "requestId?": "string", "agentAddress?": "string", "transferId?": "string", - // `repoId` is carried as `unknown` and validated only inside the pack - // branch below. Validating it here would fail the whole envelope for a - // non-pack frame that happens to carry a malformed `repoId`-shaped field, + // `repoId` and `generation` are carried as `unknown` and validated only + // inside the branches that answer with them. Validating them here would fail + // the whole envelope for a frame whose malformation is in one of them, // sinking its recovery through its own correlation key. "repoId?": "unknown", + "generation?": "unknown", }); /** @@ -100,8 +119,8 @@ const MalformedRequestEnvelope = type({ * correlates by `requestId`, whose failure reply is a `session.error`. * `sources.update` and `credentials.update` qualify -- both are answered with a * `session.error`. Frames answered through the other correlation keys live in - * `AGENT_ERROR_REQUEST_TYPES` and `PACK_REJECT_REQUEST_TYPES`; a request-shaped - * frame in none of the three sets has no requester to answer and is dropped. + * `LIFECYCLE_ERROR_TYPES` and `PACK_REJECT_REQUEST_TYPES`; a malformed frame + * of any other type is dropped unanswered. */ const SESSION_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ "sources.update", @@ -109,13 +128,15 @@ const SESSION_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ ]); /** - * Inbound request/ack frames the hub correlates by `agentAddress` and - * whose failure reply is an `agent.error` -- the frames the hub tracks in - * its per-address pending-deploy / pending-undeploy maps. + * Lifecycle requests the hub correlates by `requestId`, each answered with + * its own typed error, which also names the incarnation. */ -const AGENT_ERROR_REQUEST_TYPES: ReadonlySet = new Set([ - "agent.deploy", - "agent.undeploy", +const LIFECYCLE_ERROR_TYPES: ReadonlyMap< + string, + AgentDeployErrorFrame["type"] | AgentUndeployErrorFrame["type"] +> = new Map([ + ["agent.deploy", "agent.deploy.error"], + ["agent.undeploy", "agent.undeploy.error"], ]); /** @@ -132,10 +153,10 @@ const PACK_REJECT_REQUEST_TYPES: ReadonlySet = new Set([ /** * Answer a malformed inbound request/ack control frame with an error reply * so the hub's request does not hang to its timeout. Two control-frame - * families answer through their correlation key: the `requestId`-correlated - * frame (sources.update) replies `session.error`; the - * `agentAddress`-correlated frames (agent.deploy, agent.undeploy) reply - * `agent.error`. The fire-and-forget frames + * families answer through their `requestId`: sources.update and + * credentials.update reply `session.error`; agent.deploy and agent.undeploy + * reply their typed error, which also needs the frame's address and + * generation. The fire-and-forget frames * (mail/signal/drain/...) have no requester waiting on a reply, so a * malformed one is correctly left to be logged and dropped by the caller. * @@ -156,14 +177,16 @@ const PACK_REJECT_REQUEST_TYPES: ReadonlySet = new Set([ * Classify an `applyAssetPack` failure message into a `repo.pack.reject` reason. * A structural rejection -- a symlink or submodule the checkout cannot reproduce * faithfully, or a mountPath that escapes -- is a `path_violation`, distinct from - * `corrupt` (bad or incomplete bytes). The match is on the messages - * `writeTreeToDisk` and the mountPath guard raise; a wording drift only reverts - * the reason to `corrupt`, never misclassifies bytes as a path issue. The raw - * message rides on the frame's `detail` regardless, so the operator always sees - * the specific cause. + * `corrupt` (bad or incomplete bytes), and a workflow-run seed that would move + * history the sidecar already has is a `conflict`. The match is on the messages + * `writeTreeToDisk`, the mountPath guard and the seed guard raise; a wording + * drift only reverts the reason to `corrupt`, never misclassifies bytes as a + * path issue. The raw message rides on the frame's `detail` regardless, so the + * operator always sees the specific cause. */ export function classifyAssetPackRejectReason(msg: string): PackRejectReason { if (msg.startsWith("sha_mismatch")) return "sha_mismatch"; + if (msg.startsWith("workflow_run_restore_conflict")) return "conflict"; if ( msg.startsWith("signature_invalid") || msg.startsWith("signature_unsigned") @@ -179,7 +202,13 @@ export function classifyAssetPackRejectReason(msg: string): PackRejectReason { export function answerMalformedRequestFrame( raw: unknown, summary: string, - send: (frame: SessionErrorFrame | AgentErrorFrame | PackRejectFrame) => void, + send: ( + frame: + | SessionErrorFrame + | AgentDeployErrorFrame + | AgentUndeployErrorFrame + | PackRejectFrame, + ) => void, ): boolean { const envelope = MalformedRequestEnvelope(raw); if (envelope instanceof type.errors) return false; @@ -197,16 +226,33 @@ export function answerMalformedRequestFrame( }); return true; } + const lifecycleError = LIFECYCLE_ERROR_TYPES.get(frameType); if ( - AGENT_ERROR_REQUEST_TYPES.has(frameType) && + lifecycleError !== undefined && + envelope.requestId !== undefined && + envelope.requestId.length > 0 && envelope.agentAddress !== undefined && envelope.agentAddress.length > 0 ) { - send({ - type: "agent.error", + const generation = Generation(envelope.generation); + if (generation instanceof type.errors) return false; + const correlation = { + requestId: envelope.requestId, agentAddress: envelope.agentAddress, - error: `malformed ${frameType} frame: ${summary}`, - }); + generation, + }; + const message = fitDeploymentError( + `malformed ${frameType} frame: ${summary}`, + ); + send( + lifecycleError === "agent.deploy.error" + ? { + type: lifecycleError, + ...correlation, + error: { code: "deployment_failed", message }, + } + : { type: lifecycleError, ...correlation, error: message }, + ); return true; } if ( @@ -238,17 +284,35 @@ export function answerMalformedRequestFrame( const DEFAULT_PING_INTERVAL_MS = 30_000; const DEFAULT_RECONNECT_DELAY_MS = 3_000; +const DEFAULT_WELCOME_TIMEOUT_MS = 30_000; + +const SENDER_KEYS = Symbol("sender keys"); /** - * The reason string `packSender.cancelAll` rejects an in-flight transfer with - * when the link cycles on the reconnect `open` handler. A push that fails with - * this is a dropped connection, not a receiver-side rejection, so the - * workflow-run push path must not fast-retry it (see `runWithBootstrap`); the - * pushing store's post-reconnect re-drive owns reconnect recovery. + * The reason a workflow-run push fails when no welcomed connection can carry + * it: the one that carried it closed, or none is welcomed yet. It is not a + * rejection by the Hub, so the push path does not fast-retry it (see + * `runWithBootstrap`), and the pushing store keeps the push for the re-drive + * on the next `welcome` instead of failing a write with it. */ const CONNECTION_LOST_REASON = "Connection lost"; -function isConnectionLost(err: unknown): boolean { +/** The incarnation a queued report was sent for. */ +function reportingIncarnation( + frame: SidecarFrame, +): { address: string; generation: number } | undefined { + switch (frame.type) { + case "mail.outbound": + return { address: frame.senderAddress, generation: frame.generation }; + case "signal.correlation.register": + return { address: frame.agentAddress, generation: frame.generation }; + default: + return undefined; + } +} + +/** Whether a push failed because no welcomed connection could carry it. */ +export function isConnectionLost(err: unknown): boolean { return err instanceof Error && err.message === CONNECTION_LOST_REASON; } @@ -290,18 +354,36 @@ export type DeployRouterResult = { * `@intx/workflow-host`). */ export interface DeployRouter { + /** + * Stage the deploy of one incarnation. The router owns what the sidecar + * hosts, which the link reads through `getIncarnations`. An address is + * deployed once, so the router refuses a deploy of an address it already + * holds. A throw is answered with `agent.deploy.error`. + */ deploy(frame: AgentDeployFrame): Promise; /** * Symmetric teardown for `deploy`. The link invokes this when an * `agent.undeploy` frame lands so the router can release any - * per-deployment registrations the deploy path installed + * per-deployment registrations and persistent state the deploy path installed * (`MultistepMailRouter`, `MultistepSignalRouter`, - * `MultistepDrainRouter`, `DeploymentAddressRegistry`). Optional - * so test routers can omit the implementation. + * `MultistepDrainRouter`, `DeploymentAddressRegistry`). It leaves an + * incarnation with a higher generation than the frame's in place. A throw + * is answered with `agent.undeploy.error`. The router deletes the agent + * directory before discarding its durable teardown record. Optional so test + * routers can omit the implementation; the link then deletes the directory. */ undeploy?: (frame: AgentUndeployFrame) => Promise; + /** Cancel, stop, or request durable retention of a workflow's local inspection state. */ + control?: (frame: WorkflowControlFrame) => Promise; } +/** What a handled workflow control reports back to the Hub. */ +export type WorkflowControlOutcome = { + retention?: DeploymentRetention; + /** Final history for a live stop, which the Hub must confirm receiving. */ + refTips?: WorkflowRunRefTips; +}; + /** * Per-address mail handler registry the link consults on every * `mail.inbound` frame. Production wires this against the sidecar's @@ -399,8 +481,13 @@ export interface GrantsInboundRouter { * grants write itself throws. The link surfaces a rejection through a * logged warning -- a structured failure-reply frame for run grants * does not exist on the wire today. + * + * The link caches the sender keys the frame carries before it dispatches + * the frame. `senderKeysCached` is false when one of them did not land, and + * the handler must then not let the run start, since its recipient could + * not verify that sender's mail. */ - tryRoute(frame: RunGrantsFrame): Promise; + tryRoute(frame: RunGrantsFrame, senderKeysCached: boolean): Promise; } /** @@ -447,7 +534,7 @@ export interface CredentialsInboundRouter { } /** - * Applies one Hub-authoritative workflow-run ref before a replacement + * Applies one Hub-authoritative workflow-run ref before a deployment's first * supervisor is allowed to spawn. The host owns the workflow substrate, so * the websocket layer validates and assembles the transfer but delegates the * actual ref update through this boundary. @@ -500,33 +587,6 @@ const defaultWorkflowProbeExecutor: WorkflowProbeExecutor = { }, }; -/** - * Executor for sidecar-hosted loopback OAuth logins (CL-7508). Runs the - * connector's PKCE login on this machine: binds the pinned loopback callback - * server, returns the authorize URL for the hub to ship to the web UI, and - * completes with the staged tokens once the user finishes consent. The PKCE - * verifier stays inside the executor — only the tokens cross the wire. A - * rejection (including a pinned port already in use) rides back as the - * error arm of the `oauth.login.result` reply. - */ -export type OAuthLoginExecutor = ( - connectorId: "codex" | "xai-oauth", -) => Promise<{ - authorizeUrl: string; - completed: Promise; - /** Closes the staged login's pinned-port callback listener. Called when - * the hub cancels the login (`oauth.login.cancel`) or the link - * disconnects, so an abandoned bind cannot wedge every retry of the - * connector until the sidecar restarts. */ - cancel?: () => void; -}>; - -/** Fail-closed placeholder: an `oauth.login.start` still gets an error - * reply instead of hanging the hub's request. */ -const defaultOAuthLoginExecutor: OAuthLoginExecutor = () => { - throw new Error("oauth login execution is not implemented on this sidecar"); -}; - export type HubLinkConfig = { hubURL: string; sidecarId: string; @@ -560,13 +620,14 @@ export type HubLinkConfig = { lookupInboundMailPolicy: (address: string) => ResolvedInboundMailPolicy; /** * Persists the hub-vouched public key for a sender address, overwriting any - * previously cached key. The link calls it on an inbound `sender.key.refresh` - * frame, passing the frame's hex-encoded key through unchanged. Source-opaque, - * the write peer of `resolveSenderCrypto`: the host builds it over the - * sidecar's sender-key cache (decode the hex, `put` the bytes) so the link - * never touches key material or decoding. Required, not optional, because its - * read peer is required and every composition that runs the link already holds - * the same cache. + * previously cached key. The link calls it for an inbound `sender.key.refresh` + * frame and for each key a `run.grants` frame carries, passing the + * hex-encoded key through unchanged. Source-opaque, the write peer of + * `resolveSenderCrypto`: the host builds it over the sidecar's sender-key + * cache (decode the hex, `put` the bytes) so the link never touches key + * material or decoding. Required, not optional, because its read peer is + * required and every composition that runs the link already holds the same + * cache. */ cacheSenderKey: (address: string, publicKey: string) => Promise; /** @@ -664,60 +725,57 @@ export type HubLinkConfig = { */ workflowProbeExecutor?: WorkflowProbeExecutor; /** - * Optional loopback-login executor (CL-7508). When present, the link routes - * every inbound `oauth.login.start` frame through it and answers with - * `oauth.login.result` frames: a `started` arm as soon as the executor - * returns the authorize URL, then exactly one terminal arm (`completed` - * with the staged tokens, or `error`). Absent, the link wires a - * placeholder that always errors — required because a request frame with - * no reply hangs the hub's login request. + * Every deployment incarnation this sidecar holds, with what it is doing + * with each. The `hello` frame reports all of them so the Hub reconciles + * against them, and the link delivers mail, signals, drains, grants and + * updates only to an incarnation it holds live. Defaults to none when omitted + * (tests / deployments with no workflow substrate). */ - oauthLoginExecutor?: OAuthLoginExecutor; - /** - * Returns the workflow-substrate deployment addresses this sidecar - * currently hosts a live supervisor for. Called on every (re)connect to - * announce them to the Hub in the allocation-authenticated reconnect frame. - * Without this announcement the Hub drops the deployment's route on a WS - * reconnect. - * Defaults to none when omitted (tests / deployments with no workflow - * substrate). - */ - getWorkflowAddresses?: () => string[]; + getIncarnations?: () => HostedIncarnation[]; + capacity: SidecarCapacity; /** * Returns the rotatable (non-run) sender addresses this sidecar holds cached - * keys for. Read at each (re)connect and reported on the register/reconnect - * frame so the Hub re-resolves and re-pushes each key, catching a rotation - * that landed while the sidecar was disconnected. + * keys for. Read at each (re)connect and reported on the hello frame so the + * Hub re-resolves and re-pushes each key, catching a rotation that landed + * while the sidecar was disconnected. * * Optional with an empty default, UNLIKE the required `cacheSenderKey`: this * is a report-path reader whose empty result is a legitimate steady state (no * cached senders, or a host with no sender substrate), the same shape as - * `getWorkflowAddresses`. `cacheSenderKey` sits on the receive path a sidecar + * `getIncarnations`. `cacheSenderKey` sits on the receive path a sidecar * must always be able to serve, so it is required; this one is not. */ getCachedSenderAddresses?: () => string[]; /** - * Invoked after the allocation-authenticated reconnect frame is written. - * The Hub serializes that frame ahead of later pack frames on the same - * socket, so the workflow-run pack pusher can safely re-drive a cancelled - * push without overtaking route restoration. + * Invoked on every `welcome` with the addresses of the held incarnations + * the Hub routes on the new connection, possibly none. Everything owed to + * the Hub is re-driven from here: the workflow-run pack pusher re-ships + * what the Hub has not acknowledged for those addresses, their parked + * correlations are registered again, and deployments that stopped on their + * own are reported again. */ onWorkflowAddressesRoutable?: (addresses: string[]) => void; /** - * Invoked on WS disconnect with the workflow-substrate addresses this link - * hosts (`getWorkflowAddresses()`). Their Hub route is gone until the next - * allocation-authenticated reconnect is processed, so the workflow-run pack - * pusher blocks their pushes in the interim. Paired with - * `onWorkflowAddressesRoutable`, which lifts the block after re-announcement. + * Invoked when a connection opens and when it closes, with the address of + * every incarnation this link holds. None of them is routed until the next + * `welcome`, so the workflow-run pack pusher blocks their pushes in the + * interim. Paired with `onWorkflowAddressesRoutable`, which lifts the block. */ onWorkflowAddressesUnroutable?: (addresses: string[]) => void; pingIntervalMs?: number; reconnectDelayMs?: number; + /** + * How long a connection waits for the Hub's `welcome` after its `hello` + * before the link gives up on it and reconnects. + */ + welcomeTimeoutMs?: number; /** Per-attempt watchdog before re-sending an unacked correlation register. */ registerAckTimeoutMs?: number; /** Total register sends (initial + retries) before giving up. */ registerAckMaxAttempts?: number; scheduleReconnect?: ReconnectScheduler; + /** Arms each correlation-register retry; defaults to a real timer. */ + scheduleRegisterRetry?: ReconnectScheduler; }; export type HubLink = { @@ -727,6 +785,10 @@ export type HubLink = { */ connect(): void; close(): void; + /** + * Best-effort: an event produced while the link is not welcomed is dropped + * rather than queued, so an outage cannot fill the queue with events. + */ sendEvent: SessionEventSink; /** * Register a control-plane suspension with the hub. Sends a @@ -734,14 +796,15 @@ export type HubLink = { * routing + approval rows. Fired by the sidecar's supervisor when a workflow * agent step parks on a reserved correlation channel; the fields converge at * this seam (`correlationId`/`runId`/`kind` from the child, `anchorRunId`/ - * `agentAddress` stamped by the supervisor). Mirrors `sendEvent`: a - * fire-and-forget hub-bound send that queues while disconnected. + * `agentAddress`/`generation` stamped by the deployment that parked). Queues + * while the link is not welcomed. */ sendSignalCorrelationRegister: (registration: { correlationId: string; runId: string; anchorRunId: string; agentAddress: string; + generation: number; kind: SignalKind; approvalSnapshot?: ApprovalSnapshot; }) => void; @@ -751,15 +814,25 @@ export type HubLink = { * resolves on the matching `repo.pack.ack` (rejects on * `repo.pack.reject` with the carried reason). The hub routes the * pack to its `workflow-run` receiver because `repoId.kind` is - * `"workflow-run"`. + * `"workflow-run"`, and refuses it unless `generation` is the + * incarnation the connection hosts for the address, routed or not. Rejects + * at once while the link is not welcomed; the pusher re-drives from + * `welcome`. */ pushWorkflowRunPack: (opts: { agentAddress: string; + generation: number; repoId: RepoId; pack: Uint8Array; ref: string; commitSha: string; }) => Promise; + /** + * Report a deployment that stopped though the Hub did not stop it. Dropped + * while the link is not welcomed: the sidecar reports every such deployment + * again after each `welcome`. + */ + sendDeploymentStopped: (report: Omit) => void; }; const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]); @@ -807,16 +880,18 @@ export function createHubLink(config: HubLinkConfig): HubLink { credentialsInboundRouter, applyWorkflowRunPack, workflowProbeExecutor = defaultWorkflowProbeExecutor, - oauthLoginExecutor = defaultOAuthLoginExecutor, - getWorkflowAddresses = () => [], + getIncarnations = () => [], + capacity, getCachedSenderAddresses = () => [], onWorkflowAddressesRoutable, onWorkflowAddressesUnroutable, pingIntervalMs = DEFAULT_PING_INTERVAL_MS, reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS, + welcomeTimeoutMs = DEFAULT_WELCOME_TIMEOUT_MS, registerAckTimeoutMs = DEFAULT_REGISTER_ACK_TIMEOUT_MS, registerAckMaxAttempts = DEFAULT_REGISTER_ACK_MAX_ATTEMPTS, scheduleReconnect = defaultScheduleReconnect, + scheduleRegisterRetry, } = config; const cleartextWarning = cleartextTransportWarning(hubURL); @@ -827,73 +902,164 @@ export function createHubLink(config: HubLinkConfig): HubLink { let ws: WebSocket | null = null; let closed = false; let pingTimer: ReturnType | null = null; + let welcomeTimer: ReturnType | null = null; let cancelReconnect: (() => void) | null = null; let lastPongAt = 0; - // An OPEN socket is not application-ready until its one authoritative - // register/reconnect frame is on the wire. Async deploy-ref reads can keep - // that handshake pending briefly, so ordinary outbound traffic remains in - // the existing bounded queue until the handshake has been sent. - let handshakePending = true; + // Set by the Hub's `welcome` once it has reconciled this connection's + // `hello`. Until then only `hello`, pings, and replies to the Hub's own + // requests go out, so nothing that must be delivered reaches a Hub that has + // not yet decided which of this sidecar's incarnations it keeps. + let welcomed = false; const packReceiver = createPackReceiver(); - // Staged loopback logins by requestId (see handleOAuthLoginStart). - const activeOAuthLogins = new Map< - string, - Awaited> - >(); - // One sender owns the agent-state push path (`handleSyncRequest`, - // `handleAgentUndeploy`) and the workflow-run push path - // (`pushWorkflowRunPack`). transferIds for the two flows live in - // disjoint namespaces (`undeploy-*` / sync-supplied / `workflow-run-*`), - // so a single pending-id map is unambiguous; the protocol logic - // (chunking, ack-handshake) lives once in `@intx/pack-transport`. - const packSender = createPackSender({ sendFrame: (frame) => send(frame) }); + // Sidecar-initiated workflow-run pushes (`pushWorkflowRunPack`). A transfer + // belongs to the connection that carried it: it is refused while the link + // is not welcomed and cancelled when its connection closes, and the pusher + // re-drives it from the next `welcome`. + const packSender = createPackSender({ + sendFrame: (frame) => { + const socket = welcomedSocket(); + if (socket === null) throw new Error(CONNECTION_LOST_REASON); + socket.send(JSON.stringify(frame)); + }, + }); // Retry `signal.correlation.register` until the hub acks it. A register is // fire-and-forget on the wire and can be lost on an open socket or evicted // from the bounded queue below; the acker re-sends on a tight watchdog while - // the link is open and gives up on disconnect, leaving the reconnect re-emit - // as the backstop. `isOpen` tracks the transport lifetime; `send` separately - // holds retries in the queue until the initial handshake is on the wire. + // the link is welcomed and gives up otherwise, leaving the re-emit on + // `welcome` as the backstop. A retry before the welcome would only queue a + // copy behind the one `report` already holds there. const registerAcker = createRegisterAcker({ - sendFrame: (frame) => send(frame), - isOpen: () => ws !== null && ws.readyState === WebSocket.OPEN, + sendFrame: (frame) => { + report(frame); + }, + isOpen: () => welcomedSocket() !== null, timeoutMs: registerAckTimeoutMs, maxAttempts: registerAckMaxAttempts, + ...(scheduleRegisterRetry !== undefined + ? { scheduleRetry: scheduleRegisterRetry } + : {}), }); - // Serialize frame processing so async handlers (deploy, undeploy, abort) - // cannot race against each other. - let messageQueue: Promise = Promise.resolve(); + // Frames for one address start their handling in the order they arrive, + // while frames for different addresses do not wait on each other. A mail + // delivery leaves the lane once routed and a workflow control once started, + // so a later undeploy can run while either is still settling. Writes to the + // shared sender-key cache stay in arrival order across addresses: the + // refresh and evict frames are barriers, and `run.grants`, which caches the + // keys of its run's senders, also takes the sender-key lane. + function logFrameError(err: unknown): void { + const msg = err instanceof Error ? err.message : String(err); + logger.warn`Unhandled error handling a hub frame: ${msg}`; + } + const frameLanes = createFrameLanes(logFrameError); + // Consecutive requests for the same generation share teardown, including + // across reconnects. An intervening address command ends that group. + const pendingUndeploys = new Map< + string, + { + generation: number; + replies: { connection: WebSocket; requestId: string }[]; + } + >(); + + function incarnationOf(address: string): HostedIncarnation | undefined { + return getIncarnations().find((held) => held.address === address); + } + + // Why a Hub frame for one incarnation cannot be delivered, or null when it + // can. A frame for an incarnation this sidecar does not hold live belongs to + // one that was undeployed or never deployed here, and delivering it to + // whatever holds the address now would credit the wrong deployment. + function refusal(frame: { + agentAddress: string; + generation: number; + }): string | null { + const held = incarnationOf(frame.agentAddress); + if (held === undefined) return `${frame.agentAddress} is not hosted here`; + if (held.generation !== frame.generation) { + return `${frame.agentAddress} generation ${String(held.generation)} is hosted here, not generation ${String(frame.generation)}`; + } + if (held.state !== "live") { + return `${frame.agentAddress} generation ${String(held.generation)} is ${held.state}`; + } + return null; + } - // Outbound frames queued while disconnected. + // Reports queued while the link is not welcomed: frames the sidecar owes the + // Hub that nothing re-sends from durable state. Every deployment on the + // sidecar shares the bound, so an outage that outlasts it drops, loudly and + // oldest first, what delivers no mail: a correlation registration, which the + // re-emit on `welcome` sends again, then the audit copy the Hub records sent + // mail from. Mail still to be routed is never dropped: its send fails + // instead, and the sender sees the error. const MAX_QUEUE = 1024; const queue: SidecarFrame[] = []; - function send(frame: SidecarFrame): void { - if ( - ws !== null && - ws.readyState === WebSocket.OPEN && - (!handshakePending || frame.type === "ping") - ) { - ws.send(JSON.stringify(frame)); + function isMailToRoute(frame: SidecarFrame): boolean { + return frame.type === "mail.outbound" && frame.delivered !== true; + } + + // The queued frame an outage drops to make room, or -1 when all of it is + // mail still to be routed. + function droppableIndex(): number { + const register = queue.findIndex( + (queued) => queued.type === "signal.correlation.register", + ); + if (register !== -1) return register; + return queue.findIndex( + (queued) => queued.type === "mail.outbound" && !isMailToRoute(queued), + ); + } + + function welcomedSocket(): WebSocket | null { + return ws !== null && welcomed && ws.readyState === WebSocket.OPEN + ? ws + : null; + } + + function report(frame: SidecarFrame): void { + const socket = welcomedSocket(); + if (socket !== null) { + socket.send(JSON.stringify(frame)); return; } if (queue.length >= MAX_QUEUE) { - logger.warn`Outbound queue full, dropping oldest frame`; - queue.shift(); + const index = droppableIndex(); + // A registration goes first, so one that finds no other queued is the + // one dropped; the re-emit on `welcome` sends it again. + if ( + frame.type === "signal.correlation.register" && + queue[index]?.type !== "signal.correlation.register" + ) { + logger.warn`Outbound queue full, dropping ${frame.type}`; + return; + } + if (index === -1) { + if (isMailToRoute(frame)) { + throw new Error( + `The outbound queue is full of mail waiting for the Hub; ${frame.type} was not queued`, + ); + } + logger.warn`Outbound queue full of mail waiting for the Hub, dropping ${frame.type}`; + return; + } + const [dropped] = queue.splice(index, 1); + logger.warn`Outbound queue full, dropping the oldest queued ${dropped?.type ?? "frame"}`; } queue.push(frame); } function flush(): void { - while ( - queue.length > 0 && - ws !== null && - ws.readyState === WebSocket.OPEN && - !handshakePending + for ( + let socket = welcomedSocket(); + socket !== null; + socket = welcomedSocket() ) { - ws.send(JSON.stringify(queue.shift())); + const frame = queue.shift(); + if (frame === undefined) return; + socket.send(JSON.stringify(frame)); } } @@ -908,35 +1074,26 @@ export function createHubLink(config: HubLinkConfig): HubLink { return true; } - /** - * Send the initial handshake only if `connection` is still the active - * socket. Deploy-ref collection is asynchronous; this attempt fence keeps a - * late completion from sending onto (or flushing the queue through) a newer - * reconnect attempt. - */ - function completeHandshake( - connection: WebSocket, - frame: RegisterFrame | ReconnectFrame, - ): void { - if (!sendOnConnection(connection, frame)) return; - handshakePending = false; - flush(); - if ( - frame.type === "reconnect" && - frame.agentAddresses.length > 0 && - onWorkflowAddressesRoutable !== undefined - ) { - // Allocation authentication binds the whole connection to one workflow - // generation. The reconnect frame is processed ahead of later frames on - // the Hub's per-socket queue, so pushes triggered here cannot overtake - // the route restoration. - onWorkflowAddressesRoutable(frame.agentAddresses); - } + // Replies answer a request on the connection that carried it. The Hub gave + // up on the request when that connection closed, so a reply produced later + // is dropped rather than sent on a newer connection, where it could match an + // unrelated request. + type Reply = (frame: SidecarFrame) => void; + + function replyOn(connection: WebSocket): Reply { + return (frame) => { + if (!sendOnConnection(connection, frame)) { + logger.debug`Dropping ${frame.type}: the connection that carried its request closed`; + } + }; } // Wire the transport's remote send handler to push mail.outbound frames // for routing. The sender comes from the transport's registered-address - // check so the hub can bind the claim to this authenticated connection. + // check so the hub can bind the claim to this authenticated connection, and + // the frame names the sender's incarnation. A deployment finishes sending + // before its teardown completes, so the incarnation held for the sender + // address while it sends is the one sending. transport.setRemoteSendHandler( async (rawMessage, recipients, senderAddress) => { // Derive the base64 payload length arithmetically (4 output characters @@ -955,12 +1112,28 @@ export function createHubLink(config: HubLinkConfig): HubLink { `refusing to send mail.outbound from ${senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`, ); } + const sender = incarnationOf(senderAddress); + if (sender === undefined) { + throw new Error( + `refusing to send mail.outbound from ${senderAddress}: no incarnation of it is hosted here`, + ); + } + // The Hub refuses a mail naming more than one workflow deployment and + // stays the authority on that; refusing here gives the sender the error + // the Hub can only log. + const deployments = recipients.filter(isRunAddress); + if (deployments.length > 1) { + throw new Error( + `refusing to send mail.outbound from ${senderAddress}: it names ${String(deployments.length)} workflow deployments (${deployments.join(", ")}), and a mail may name only one`, + ); + } const encoded = base64Encode(rawMessage); - send({ + report({ type: "mail.outbound", rawMessage: encoded, recipients, senderAddress, + generation: sender.generation, }); }, ); @@ -983,13 +1156,19 @@ export function createHubLink(config: HubLinkConfig): HubLink { logger.error`Skipping delivered mail.outbound audit frame from ${ctx.senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`; return; } + const sender = incarnationOf(ctx.senderAddress); + if (sender === undefined) { + logger.error`Skipping delivered mail.outbound audit frame from ${ctx.senderAddress}: no incarnation of it is hosted here`; + return; + } const encoded = base64Encode(ctx.rawMessage); const sessionId = sessions.getSessionId(ctx.senderAddress); - send({ + report({ type: "mail.outbound", rawMessage: encoded, recipients: ctx.recipients, senderAddress: ctx.senderAddress, + generation: sender.generation, ...(sessionId !== undefined ? { sessionId } : {}), messageId: ctx.messageId, to: ctx.to, @@ -998,47 +1177,29 @@ export function createHubLink(config: HubLinkConfig): HubLink { }); }); - async function handleAgentDeploy(frame: AgentDeployFrame): Promise { - try { - // The deploy router (production: the sidecar's workflow-run deploy - // router) stages the deploy through the substrate and returns the - // deploy public key the link folds into the outbound ack. The link - // itself does not re-decide; routing lives on the router side of - // the seam. - const result = await deployRouter.deploy(frame); - send({ - type: "agent.deploy.ack", - agentAddress: frame.agentAddress, - publicKey: result.publicKey, - }); - logger.info`Deployed agent ${frame.agentAddress}`; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - send({ - type: "agent.error", - agentAddress: frame.agentAddress, - error: message, - }); - } - } - - async function handleAgentUndeploy(frame: AgentUndeployFrame): Promise { - let statePushed = false; - - // Release per-deployment routing state the deploy router installed - // for this address (multi-step mail/signal/drain handlers and the - // deployment-address mapping) before the session tears down. With - // the registrations released, any in-flight `signal.deliver` / - // `drain.deliver` / `mail.inbound` frame that lands during teardown - // is rejected by the router rather than dispatched into a - // soon-to-be-orphaned supervisor handler. Test stubs omit the hook; - // an absent hook means there was nothing to release. + // Tear one incarnation down: the router releases the deployment and the + // link drops what it keeps for the address. Every step runs even when an + // earlier one fails, and the failures are thrown together, so a partial + // teardown is reported rather than acknowledged. A pack transfer still in + // flight is left for the Hub to answer, as it answers every transfer. + async function tearDown(frame: AgentUndeployFrame): Promise { + const failures: Error[] = []; if (deployRouter.undeploy !== undefined) { try { await deployRouter.undeploy(frame); + } catch (err) { + failures.push(err instanceof Error ? err : new Error(String(err))); + } + } else { + try { + await sessions.deleteAgentDir(frame.agentAddress); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - logger.warn`Deploy router undeploy hook failed for ${frame.agentAddress}: ${msg}`; + failures.push( + new Error(`deleting its agent directory failed: ${msg}`, { + cause: err, + }), + ); } } @@ -1060,63 +1221,100 @@ export function createHubLink(config: HubLinkConfig): HubLink { workflowRunPackBootstrappedByAddress.delete(frame.agentAddress); } - // Best-effort state push to the hub before deleting the directory. - // statePushed reflects whether we sent the pack frames, not whether - // the hub acknowledged them. We intentionally skip waiting for - // repo.pack.ack here to avoid blocking the undeploy on a round-trip - // that may never complete if the hub is shutting down -- so the - // pending Promise's rejection on disconnect is intentionally - // swallowed below. - try { - const { pack, commitSha, ref } = await sessions.createStatePack( - frame.agentAddress, + keyStore.forgetAgent(frame.agentAddress); + forgetIncarnation(frame.agentAddress, frame.generation); + + if (failures.length > 0) { + throw new AggregateError( + failures, + `Undeploying ${frame.agentAddress} generation ${String(frame.generation)} failed: ${failures.map((failure) => failure.message).join("; ")}`, ); - const repoId: RepoId = { - kind: "agent-state", - id: frame.agentAddress, - }; + } + } - void packSender - .send({ - agentAddress: frame.agentAddress, - repoId, - transferId: `undeploy-${frame.agentAddress}`, - pack, - ref, - commitSha, - }) - .catch(() => { - // Intentional: undeploy's pack push is best-effort. See above. - }); + // Nothing an undeployed incarnation still owed the Hub goes out after its + // teardown: the Hub no longer routes it. + function forgetIncarnation(agentAddress: string, generation: number): void { + const owedBy = (incarnation: { address: string; generation: number }) => + incarnation.address === agentAddress && + incarnation.generation <= generation; + let dropped = 0; + for (let index = queue.length - 1; index >= 0; index -= 1) { + const queued = queue[index]; + const owing = + queued === undefined ? undefined : reportingIncarnation(queued); + if (owing !== undefined && owedBy(owing)) { + queue.splice(index, 1); + dropped += 1; + } + } + registerAcker.cancelWhere((frame) => + owedBy({ address: frame.agentAddress, generation: frame.generation }), + ); + if (dropped > 0) { + logger.warn`Dropping ${String(dropped)} queued report(s) of ${agentAddress} generation ${String(generation)}: it was undeployed`; + } + } - statePushed = true; + async function handleAgentDeploy( + frame: AgentDeployFrame, + reply: Reply, + ): Promise { + const answering = { + requestId: frame.requestId, + agentAddress: frame.agentAddress, + generation: frame.generation, + }; + try { + // The deploy router (production: the sidecar's workflow-run deploy + // router) stages the deploy through the substrate and returns the + // deploy public key the link folds into the outbound ack. It refuses a + // deploy of an address it already holds an incarnation of. + const result = await deployRouter.deploy(frame); + reply({ + type: "agent.deploy.ack", + ...answering, + publicKey: result.publicKey, + }); + logger.info`Deployed ${frame.agentAddress} generation ${String(frame.generation)}`; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`State push failed for ${frame.agentAddress}: ${msg}`; + const message = err instanceof Error ? err.message : String(err); + reply({ + type: "agent.deploy.error", + ...answering, + error: { + code: + err instanceof DeploymentRejectedError + ? err.code + : "deployment_failed", + message: fitDeploymentError(message), + }, + }); } + } - // Delete the agent directory. + async function handleAgentUndeploy( + frame: AgentUndeployFrame, + ): Promise { + // A sidecar holds one incarnation of an address, so with a newer one held + // nothing of the generation named is left here, and the newer one is not + // the Hub's to remove through this frame. try { - await sessions.deleteAgentDir(frame.agentAddress); + const held = incarnationOf(frame.agentAddress); + if (held !== undefined && held.generation > frame.generation) return; + await tearDown(frame); + logger.info`Undeployed ${frame.agentAddress} generation ${String(frame.generation)}: ${frame.reason}`; } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Failed to delete agent directory for ${frame.agentAddress}: ${msg}`; + const message = err instanceof Error ? err.message : String(err); + logger.error`${message}`; + return fitDeploymentError(message); } - - keyStore.forgetAgent(frame.agentAddress); - - send({ - type: "agent.undeploy.ack", - agentAddress: frame.agentAddress, - statePushed, - }); - logger.info`Undeployed agent ${frame.agentAddress}: ${frame.reason}`; } - function handlePackPush(frame: PackPushFrame): void { + function handlePackPush(frame: PackPushFrame, reply: Reply): void { const reason = packReceiver.handlePush(frame); if (reason !== null) { - send({ + reply({ type: "repo.pack.reject", agentAddress: frame.agentAddress, repoId: frame.repoId, @@ -1126,10 +1324,13 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handlePackDone(frame: PackDoneFrame): Promise { + async function handlePackDone( + frame: PackDoneFrame, + reply: Reply, + ): Promise { const result = packReceiver.handleDone(frame); if (result === null) { - send({ + reply({ type: "repo.pack.reject", agentAddress: frame.agentAddress, repoId: frame.repoId, @@ -1139,6 +1340,22 @@ export function createHubLink(config: HubLinkConfig): HubLink { return; } + // A Hub pack stages a step or seeds a deployment's history before the + // deployment is deployed. Applying one over a held incarnation would + // rewrite what that deployment is running on. + const held = incarnationOf(frame.agentAddress); + if (held !== undefined) { + reply({ + type: "repo.pack.reject", + agentAddress: frame.agentAddress, + repoId: frame.repoId, + transferId: frame.transferId, + reason: "conflict", + detail: `${frame.agentAddress} generation ${String(held.generation)} is hosted here`, + }); + return; + } + try { if (frame.repoId.kind === "workflow-run") { if (frame.mountPath !== undefined) { @@ -1183,7 +1400,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { verifyCommit, ); } - send({ + reply({ type: "repo.pack.ack", agentAddress: frame.agentAddress, repoId: frame.repoId, @@ -1193,7 +1410,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { const msg = err instanceof Error ? err.message : String(err); const reason = classifyAssetPackRejectReason(msg); logger.warn`Pack apply failed for ${frame.agentAddress}: ${msg}`; - send({ + reply({ type: "repo.pack.reject", agentAddress: frame.agentAddress, repoId: frame.repoId, @@ -1204,12 +1421,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - // Counter the boot edge consumes via `pushWorkflowRunPack` to mint - // collision-free transferIds. Lives on the link so undeploy / - // sync-request / workflow-run all share one monotonically increasing - // sequence space. - let workflowRunPackCounter = 0; - // Per-(repoId.id, ref) flag tracking whether at least one workflow-run // pack push has been accepted by the hub, and a per-(repoId.id, ref) // serialization queue. Both are needed because the hub's @@ -1260,29 +1471,6 @@ export function createHubLink(config: HubLinkConfig): HubLink { return `${repoId.kind}:${repoId.id}:${ref}`; } - async function handleSyncRequest(frame: SyncRequestFrame): Promise { - const { agentAddress, transferId } = frame; - try { - const { pack, commitSha, ref } = - await sessions.createStatePack(agentAddress); - const repoId: RepoId = { kind: "agent-state", id: agentAddress }; - - await packSender.send({ - agentAddress, - repoId, - transferId, - pack, - ref, - commitSha, - }); - - logger.info`State push complete for ${agentAddress} (${commitSha.slice(0, 8)})`; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`State push failed for ${agentAddress}: ${msg}`; - } - } - function handlePackAck(frame: PackAckFrame): void { if (!packSender.handleAck(frame)) { logger.warn`Received repo.pack.ack for unknown transferId ${frame.transferId}`; @@ -1306,7 +1494,14 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleSignalDeliver(frame: SignalDeliverFrame): Promise { + async function handleSignalDeliver( + frame: SignalDeliverFrame & IncarnationStamp, + ): Promise { + const refused = refusal(frame); + if (refused !== null) { + logger.warn`Dropping signal.deliver: ${refused}`; + return; + } if (signalInboundRouter === undefined) { logger.warn`Received signal.deliver for ${frame.agentAddress} but no signalInboundRouter is wired; dropping`; return; @@ -1323,6 +1518,11 @@ export function createHubLink(config: HubLinkConfig): HubLink { } async function handleDrainDeliver(frame: DrainDeliverFrame): Promise { + const refused = refusal(frame); + if (refused !== null) { + logger.warn`Dropping drain.deliver: ${refused}`; + return; + } if (drainInboundRouter === undefined) { logger.warn`Received drain.deliver for ${frame.agentAddress} but no drainInboundRouter is wired; dropping`; return; @@ -1338,13 +1538,124 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleRunGrants(frame: RunGrantsFrame): Promise { + async function handleMailInbound( + frame: MailInboundFrame, + { + rawBytes, + verdict, + }: { rawBytes: Uint8Array; verdict: InboundSignatureVerdict }, + reply: Reply, + ): Promise { + const refused = refusal(frame); + if (refused !== null) { + // Not delivered and not acknowledged: the Hub's retry or the + // durable dispatch row delivers it to the incarnation it routes. + logger.warn`Dropping mail.inbound ${frame.messageId ?? "(no messageId)"}: ${refused}`; + return; + } + // The recipient deployment's resolved admission policy. `frame. + // agentAddress` is the mail-router registration key, so an address + // with no registered deployment resolves to the fully-closed policy + // and rejects every outcome. + const policy = lookupInboundMailPolicy(frame.agentAddress); + const admission = decideInboundAdmission(verdict, policy); + if (admission.rejectedBy !== null) { + logger.warn( + "Rejecting inbound mail for {agentAddress}: {rejectedBy} is not admitted (findings {findings}, headline outcome {outcome}, authenticatedSender {authenticatedSender}, messageId {messageId})", + { + agentAddress: frame.agentAddress, + rejectedBy: admission.rejectedBy, + findings: admission.findings, + outcome: outcomeForVerdict(verdict), + authenticatedSender: frame.authenticatedSender, + messageId: frame.messageId ?? null, + }, + ); + // A rejected mail is simply not delivered -- the same drop as the + // no-handler path below: no ack, no reply frame, no dispatch-fail. + // The hub's redelivery and expiry machinery handles the rest. + return; + } + // Admitted: deliver exactly as an admitted frame always has, keeping + // the detached durable settlement and detached ack below off the + // address's frame lane. + // + // Supervised deployments register the deployment-level mail + // address on `mailInboundRouter` once their supervisor spawns; + // that handler delivers the bytes to the supervisor's mail-bus + // subscription, which is what the workflow-host's `awaitSignal` + // listens on. Mail for an address with no registered handler has + // no receiver -- the in-process session runtime that once backed + // it is retired -- so it is logged and dropped. + // + // Guard the router call with try/catch so a synchronous throw is + // logged against this address and the mail is dropped like mail with + // no handler. The durable settlement is observed off the lane (below). + let durable: Promise | null = null; + if (mailInboundRouter !== undefined) { + try { + durable = mailInboundRouter.tryRoute(frame.agentAddress, rawBytes); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn`mail.inbound router threw for ${frame.agentAddress}: ${msg}`; + } + } + if (durable === null) { + logger.warn`Dropping mail.inbound for ${frame.agentAddress}: no registered handler`; + return; + } + // Acknowledge durable receipt only AFTER the inbox write settles, and + // only for hub-originated mail carrying a hub-minted messageId (the + // ack handshake). Observe the settlement DETACHED from the address's + // frame lane so a slow or failing inbox write never holds up its later + // frames; on rejection (transient failure, stale refusal, or a + // tearing-down phase) no ack is sent, so the hub redelivers. The ack + // names the incarnation the mail was delivered to, so the Hub credits + // it to no other. + const ackMessageId = frame.messageId; + if (ackMessageId !== undefined) { + void durable + .then(() => { + reply({ + type: "mail.inbound.ack", + agentAddress: frame.agentAddress, + generation: frame.generation, + messageId: ackMessageId, + }); + }) + .catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + logger.warn`Withholding mail.inbound.ack for ${frame.agentAddress} ${ackMessageId}; hub will redeliver: ${msg}`; + }); + } else { + // Relayed agent-to-agent mail carries no hub-minted messageId and + // does not participate in the ack handshake. Still observe the + // settlement so a rejection is logged, not left unhandled. + void durable.catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + logger.warn`Inbound mail delivery failed for ${frame.agentAddress}: ${msg}`; + }); + } + } + + async function handleRunGrants( + frame: RunGrantsFrame & IncarnationStamp, + senderKeysCached: boolean, + ): Promise { + const refused = refusal(frame); + if (refused !== null) { + logger.warn`Dropping run.grants for run ${frame.runId}: ${refused}`; + return; + } if (grantsInboundRouter === undefined) { logger.warn`Received run.grants for ${frame.agentAddress} but no grantsInboundRouter is wired; dropping`; return; } try { - const routed = await grantsInboundRouter.tryRoute(frame); + const routed = await grantsInboundRouter.tryRoute( + frame, + senderKeysCached, + ); if (!routed) { logger.warn`run.grants for ${frame.agentAddress} did not match any registered deployment; dropping`; } @@ -1354,6 +1665,88 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } + // Caches the sender keys a run's grants carry on the sender-key lane, so they + // land in the order the Hub sent them along with refreshes and evictions. + // Resolves to whether every well-formed key landed. Anything that goes wrong + // counts as not landed, so the run's grants fail instead of waiting forever. + function cacheSenderKeys(frame: RunGrantsFrame): Promise { + const landed = Promise.withResolvers(); + frameLanes.run([SENDER_KEYS], async () => { + let allLanded = false; + try { + allLanded = await cacheEachSenderKey(frame); + } finally { + landed.resolve(allLanded); + } + }); + return landed.promise; + } + + // A malformed key is a Hub defect the sidecar cannot repair, so it is + // skipped as if the Hub had left it out, rather than fail every replay of + // the run's grants. + async function cacheEachSenderKey(frame: RunGrantsFrame): Promise { + let allLanded = true; + for (const identity of frame.senderIdentities ?? []) { + if (!isSenderPublicKeyHex(identity.publicKey)) { + logger.error`Skipping a malformed sender key for ${identity.address} in the grants of run ${frame.runId}`; + continue; + } + try { + await cacheSenderKey(identity.address, identity.publicKey); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error`Caching the sender key for ${identity.address} in the grants of run ${frame.runId} failed: ${msg}`; + allLanded = false; + } + } + return allLanded; + } + + // Checks an inbound mail's signature on the sender-key lane, so it is + // verified against exactly the keys the Hub sent before it, never one a + // later refresh, eviction or grant wrote, and without waiting for its + // deployment's earlier frames. This is the one place raw inbound bytes meet + // the hub-verified sender identity, and every producer -- relay, trigger, + // durable dispatch -- converges here. The verify is CPU-bound, a cache read, + // an Ed25519 verify and a MIME re-parse with no I/O, so it holds the lane + // only briefly. The signature check never throws: a fault degrades to an + // `error` verdict. + // The in-process mail-memory transport does not deliver through this seam + // and carries no hub-verified sender, so it is outside this path. + function verifyInArrivalOrder( + frame: MailInboundFrame, + ): Promise<{ rawBytes: Uint8Array; verdict: InboundSignatureVerdict }> { + const verified = Promise.withResolvers<{ + rawBytes: Uint8Array; + verdict: InboundSignatureVerdict; + }>(); + frameLanes.run([SENDER_KEYS], async () => { + try { + const rawBytes = base64Decode(frame.rawMessage); + const verdict = await verifyInboundSignature( + { + raw: rawBytes, + authenticatedSender: frame.authenticatedSender, + messageId: frame.messageId, + agentAddress: frame.agentAddress, + }, + resolveSenderCrypto, + ); + verified.resolve({ rawBytes, verdict }); + } catch (err) { + verified.reject(err); + } + }); + // A body that is not base64 rejects when the mail reaches its turn on this + // lane, but its delivery awaits that only at its own turn, and never when + // its connection closes first. A rejection nothing has observed ends the + // process, so it is marked observed here; the delivery that awaits it + // still sees it. + void verified.promise.catch(() => undefined); + return verified.promise; + } + async function handleSenderKeyRefresh( frame: SenderKeyRefreshFrame, ): Promise { @@ -1366,14 +1759,11 @@ export function createHubLink(config: HubLinkConfig): HubLink { // until the next reconnect re-pushes, so this push is best-effort, not // delivery-guaranteed. // - // Awaited inline on the message chain, NOT detached the way the - // `mail.inbound` durable write is: the co-resident `run.grants` handler also - // caches sender keys on this same chain, so serializing the refresh write - // keeps last-write-wins deterministic against a concurrent grants write for - // the same address. The fan-out is bounded (one frame per rotatable cached - // sender, once per reconnect), so the head-of-line cost stays far short of - // the heartbeat window; a detached write would trade that determinism for - // latency this low-volume path does not need. + // Awaited on the sender-key lane, NOT detached the way the `mail.inbound` + // durable write is: the keys `run.grants` frames carry are cached on the + // same lane, so writes for the same address land in the order the Hub sent + // them, and inbound mail is verified on it too, against exactly the keys + // that arrived before it. try { await cacheSenderKey(frame.address, frame.publicKey); } catch (err) { @@ -1386,9 +1776,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { frame: SenderKeyEvictFrame, ): Promise { // Mirror of `handleSenderKeyRefresh` for the evict direction: address-keyed, - // cross-run, no reply channel, awaited inline on the message chain so it - // serializes deterministically against a concurrent refresh/grants write for - // the same address. A fault swallowed after logging at ERROR keeps the STALE + // cross-run, no reply channel, awaited on the sender-key lane so it orders + // deterministically against refresh and grants writes for the same + // address. A fault swallowed after logging at ERROR keeps the STALE // key cached until the next reconnect re-evicts -- the evict is best-effort, // not delivery-guaranteed, exactly like the refresh it complements. try { @@ -1399,13 +1789,25 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleSourcesUpdate(frame: SourcesUpdateFrame): Promise { + async function handleSourcesUpdate( + frame: SourcesUpdateFrame, + reply: Reply, + ): Promise { // `sources.update` is request/ack (the hub awaits a reply within its // request timeout), so every path answers `session.ack` or // `session.error` -- unlike the fire-and-forget signal/drain frames // that log and drop. A missing router still answers, or the hub hangs. + const refused = refusal(frame); + if (refused !== null) { + reply({ + type: "session.error", + requestId: frame.requestId, + error: refused, + }); + return; + } if (sourcesInboundRouter === undefined) { - send({ + reply({ type: "session.error", requestId: frame.requestId, error: "no sourcesInboundRouter is wired", @@ -1415,9 +1817,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { try { const routed = await sourcesInboundRouter.tryRoute(frame); if (routed) { - send({ type: "session.ack", requestId: frame.requestId }); + reply({ type: "session.ack", requestId: frame.requestId }); } else { - send({ + reply({ type: "session.error", requestId: frame.requestId, error: `no deployment registered for ${frame.agentAddress}`, @@ -1429,7 +1831,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { // `deliverSources` throwing (e.g. a recycling phase). The reason // rides back verbatim so the hub sees why the rotation failed. const msg = err instanceof Error ? err.message : String(err); - send({ + reply({ type: "session.error", requestId: frame.requestId, error: msg, @@ -1439,12 +1841,22 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleCredentialsUpdate( frame: CredentialsUpdateFrame, + reply: Reply, ): Promise { // `credentials.update` is request/ack, exactly like `sources.update`: every // path answers `session.ack` or `session.error`. A missing router still // answers, or the hub hangs. + const refused = refusal(frame); + if (refused !== null) { + reply({ + type: "session.error", + requestId: frame.requestId, + error: refused, + }); + return; + } if (credentialsInboundRouter === undefined) { - send({ + reply({ type: "session.error", requestId: frame.requestId, error: "no credentialsInboundRouter is wired", @@ -1454,9 +1866,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { try { const routed = await credentialsInboundRouter.tryRoute(frame); if (routed) { - send({ type: "session.ack", requestId: frame.requestId }); + reply({ type: "session.ack", requestId: frame.requestId }); } else { - send({ + reply({ type: "session.error", requestId: frame.requestId, error: `no deployment registered for ${frame.agentAddress}`, @@ -1468,7 +1880,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { // `deliverCredentials` throwing (e.g. a recycling phase). The reason // rides back verbatim so the hub sees why the delivery failed. const msg = err instanceof Error ? err.message : String(err); - send({ + reply({ type: "session.error", requestId: frame.requestId, error: msg, @@ -1478,6 +1890,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { async function handleWorkflowProbeRequest( frame: WorkflowProbeRequestFrame, + reply: Reply, ): Promise { // `workflow.probe.request` is request/response (the hub awaits a reply // within its probe timeout), so every path answers `workflow.probe.result` @@ -1487,7 +1900,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { // fails fast instead of hanging. try { const result = await workflowProbeExecutor.probe(frame); - send({ + reply({ type: "workflow.probe.result", requestId: frame.requestId, projection: result.projection, @@ -1497,7 +1910,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - send({ + reply({ type: "workflow.probe.error", requestId: frame.requestId, error: msg, @@ -1505,69 +1918,9 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleOAuthLoginStart( - frame: Extract, - ): Promise { - // `oauth.login.start` is request/response: every path answers - // `oauth.login.result` — `started` once the executor returns the - // authorize URL, then exactly one terminal arm — never a log-and-drop, - // or the hub's login request hangs. - let handle: Awaited>; - try { - handle = await oauthLoginExecutor(frame.connectorId); - } catch (err) { - send({ - type: "oauth.login.result", - requestId: frame.requestId, - outcome: { - status: "error", - message: err instanceof Error ? err.message : String(err), - }, - }); - return; - } - activeOAuthLogins.set(frame.requestId, handle); - send({ - type: "oauth.login.result", - requestId: frame.requestId, - outcome: { status: "started", authorizeUrl: handle.authorizeUrl }, - }); - try { - const tokens = await handle.completed; - send({ - type: "oauth.login.result", - requestId: frame.requestId, - outcome: { status: "completed", tokens }, - }); - } catch (err) { - send({ - type: "oauth.login.result", - requestId: frame.requestId, - outcome: { - status: "error", - message: err instanceof Error ? err.message : String(err), - }, - }); - } finally { - activeOAuthLogins.delete(frame.requestId); - } - } - - /** In-flight staged logins by requestId, so a hub `oauth.login.cancel` — - * or a link disconnect, where the hub will never see the terminal frame — - * closes the staged login's pinned-port listener. */ - function cancelOAuthLogin(requestId: string): void { - activeOAuthLogins.get(requestId)?.cancel?.(); - activeOAuthLogins.delete(requestId); - } - - function cancelAllOAuthLogins(): void { - for (const handle of activeOAuthLogins.values()) handle.cancel?.(); - activeOAuthLogins.clear(); - } - async function pushWorkflowRunPack(opts: { agentAddress: string; + generation: number; repoId: RepoId; pack: Uint8Array; ref: string; @@ -1576,11 +1929,11 @@ export function createHubLink(config: HubLinkConfig): HubLink { const key = workflowRunPackKey(opts.repoId, opts.ref); async function sendOnce(): Promise { - const transferId = `workflow-run-${++workflowRunPackCounter}-${opts.repoId.id}`; await packSender.send({ agentAddress: opts.agentAddress, + generation: opts.generation, repoId: opts.repoId, - transferId, + transferId: `workflow-run-${crypto.randomUUID()}`, pack: opts.pack, ref: opts.ref, commitSha: opts.commitSha, @@ -1595,16 +1948,16 @@ export function createHubLink(config: HubLinkConfig): HubLink { try { await sendOnce(); } catch (first) { - // A disconnect that cancelled the transfer (`cancelAll` on the - // link's reconnect `open`) is NOT the initRepo bootstrap race: the - // link just cycled, and re-sending on the fresh, not-yet-registered - // connection would ship to a hub that has dropped this address's - // route (the frames land "unrouted"). Reconnect recovery is owned by - // the pushing store's post-reconnect re-drive, not by this - // fast-retry, so re-throw and let the caller latch the failure. Only - // the genuine bootstrap race -- a receiver reject against an - // uninitialised hub repo -- retries here. - if (isConnectionLost(first)) { + // Only the genuine bootstrap race retries here: the Hub rejects the + // push that raced its repository's creation as `corrupt`. Anything + // else re-throws for the caller to latch, including a transfer refused + // or cancelled because the link is not welcomed (the pushing store + // re-drives it on `welcome`) and a push the Hub does not route, which + // a resend would only have rejected again. Real corruption is also + // `corrupt`, so it gets this one retry too. + if ( + !(first instanceof PackRejectedError && first.reason === "corrupt") + ) { throw first; } // First push to a never-bootstrapped (repoId, ref) lost the @@ -1651,7 +2004,67 @@ export function createHubLink(config: HubLinkConfig): HubLink { } } - async function handleMessage(data: string): Promise { + async function handleWorkflowControl( + frame: WorkflowControlFrame, + reply: Reply, + ): Promise { + let error: string | undefined; + let refTips: WorkflowRunRefTips | undefined; + let retention: DeploymentRetention | undefined; + try { + if (deployRouter.control === undefined) + throw new Error("Workflow control is not supported by this sidecar"); + ({ refTips, retention } = await deployRouter.control(frame)); + } catch (cause) { + error = cause instanceof Error ? cause.message : String(cause); + } + reply({ + type: "workflow.control.ack", + requestId: frame.requestId, + ...(error !== undefined ? { error } : {}), + ...(refTips !== undefined ? { refTips } : {}), + ...(retention !== undefined ? { retention } : {}), + }); + } + + // Nothing held is routed on a connection until its `welcome` says so, so + // the workflow-run pusher keeps the pushes of every held incarnation until + // then instead of sending them where nothing receives them. + function blockHeldAddresses(): void { + if (onWorkflowAddressesUnroutable === undefined) return; + const held = getIncarnations().map((incarnation) => incarnation.address); + if (held.length > 0) onWorkflowAddressesUnroutable(held); + } + + function stopWelcomeTimer(): void { + if (welcomeTimer !== null) { + clearTimeout(welcomeTimer); + welcomeTimer = null; + } + } + + function handleWelcome(frame: WelcomeFrame, connection: WebSocket): void { + if (ws !== connection || welcomed) return; + welcomed = true; + stopWelcomeTimer(); + flush(); + if (onWorkflowAddressesRoutable === undefined) return; + const held = new Map( + getIncarnations().map((incarnation) => [ + incarnation.address, + incarnation.generation, + ]), + ); + onWorkflowAddressesRoutable( + frame.routed.flatMap((incarnation) => + held.get(incarnation.address) === incarnation.generation + ? [incarnation.address] + : [], + ), + ); + } + + function receiveFrame(data: string, connection: WebSocket): void { let raw: unknown; try { raw = JSON.parse(data) as unknown; @@ -1659,6 +2072,28 @@ export function createHubLink(config: HubLinkConfig): HubLink { logger.warn`Received unparseable frame from hub`; return; } + const reply = replyOn(connection); + // A frame is handled only while the connection that carried it is still + // the link's. Once it closed, the Hub re-drives whatever it still wants + // after the next `hello`, and a frame handled later could undo what that + // `hello` reconciled. + function onLanes( + frameType: string, + lanes: Parameters[0], + handle: () => Promise, + ): void { + for (const lane of lanes) { + if (typeof lane === "string") pendingUndeploys.delete(lane); + } + frameLanes.run(lanes, async () => { + if (ws !== connection) { + logger.debug`Dropping ${frameType}: the connection that carried it closed`; + return; + } + await handle(); + }); + } + const validated = HubFrame(raw); if (validated instanceof type.errors) { // A malformed request/ack frame must still be answered, or the hub's @@ -1666,144 +2101,148 @@ export function createHubLink(config: HubLinkConfig): HubLink { // usually keep an intact correlation key even when a nested field is // malformed, so reply with the matching error frame; a fire-and-forget // frame (or one with no recoverable key) is only logged and dropped. - answerMalformedRequestFrame(raw, validated.summary, send); - logger.warn`Invalid hub frame: ${validated.summary}`; + // The answer waits for the frames its address received before it. + const summary = validated.summary; + const answer = async (): Promise => { + answerMalformedRequestFrame(raw, summary, reply); + logger.warn`Invalid hub frame: ${summary}`; + }; + const envelope = MalformedRequestEnvelope(raw); + const address = + envelope instanceof type.errors ? undefined : envelope.agentAddress; + if (address === undefined || address.length === 0) { + void answer().catch(logFrameError); + } else { + onLanes("an invalid frame", [address], answer); + } return; } const frame = validated; switch (frame.type) { + case "pong": + if (ws === connection) lastPongAt = Date.now(); + return; + case "welcome": + handleWelcome(frame, connection); + return; + case "workflow.probe.request": + void handleWorkflowProbeRequest(frame, reply).catch(logFrameError); + return; + case "repo.pack.ack": + case "repo.pack.reject": + case "signal.correlation.register.ack": + // Answers to the sidecar's own requests only settle the transfer or + // registration they answer, so their order among the address's frames + // does not matter and they are handled as they arrive. + if (ws === connection) { + void handleFrame(frame, reply).catch(logFrameError); + } + return; + case "sender.key.refresh": + case "sender.key.evict": + onLanes(frame.type, [SENDER_KEYS], () => handleFrame(frame, reply)); + return; + case "run.grants": { + const senderKeysCached = cacheSenderKeys(frame); + onLanes(frame.type, [frame.agentAddress], async () => + handleRunGrants(frame, await senderKeysCached), + ); + return; + } case "mail.inbound": { - const rawBytes = base64Decode(frame.rawMessage); - // This ingress is the one place raw inbound bytes meet the hub-verified - // sender identity (`authenticatedSender` + its resolved key), and every - // producer -- relay, trigger, durable dispatch -- converges here, so the - // inbound-signature verify gates delivery here. Await it INLINE on the - // messageQueue chain: the verdict decides admission, so delivery must - // not race ahead of it. The verify is CPU-bound -- a synchronous cache - // read, an Ed25519 verify, and a MIME re-parse, with no I/O and no lock - // -- so it cannot hang and needs no timeout race around it. It never - // throws: a fault degrades to an `error` verdict. The in-process - // mail-memory transport (the standalone harness path) does not deliver - // through this seam and carries no hub-verified sender, so it is outside - // this path. - const verdict = await verifyInboundSignature( - { - raw: rawBytes, - authenticatedSender: frame.authenticatedSender, - messageId: frame.messageId, - agentAddress: frame.agentAddress, - }, - resolveSenderCrypto, + const verified = verifyInArrivalOrder(frame); + onLanes(frame.type, [frame.agentAddress], async () => + handleMailInbound(frame, await verified, reply), ); - // The recipient deployment's resolved admission policy. `frame. - // agentAddress` is the mail-router registration key, so an address - // with no registered deployment resolves to the fully-closed policy - // and rejects every outcome. - const policy = lookupInboundMailPolicy(frame.agentAddress); - const admission = decideInboundAdmission(verdict, policy); - if (admission.rejectedBy !== null) { - logger.warn( - "Rejecting inbound mail for {agentAddress}: {rejectedBy} is not admitted (findings {findings}, headline outcome {outcome}, authenticatedSender {authenticatedSender}, messageId {messageId})", - { - agentAddress: frame.agentAddress, - rejectedBy: admission.rejectedBy, - findings: admission.findings, - outcome: outcomeForVerdict(verdict), - authenticatedSender: frame.authenticatedSender, - messageId: frame.messageId ?? null, - }, + return; + } + case "workflow.control": + // Start in the address's order, but leave its lane free for a forced + // stop and the deployment's undeploy while cooperative cancellation + // waits for the child. + onLanes(frame.type, [frame.agentAddress], async () => { + void handleWorkflowControl(frame, reply).catch((cause: unknown) => { + logger.warn`Workflow control reply failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + }); + return; + case "agent.undeploy": { + // Runs even once its connection closed; only the answer goes with the + // connection. The Hub sends it when it will not route that generation + // again, and may close the socket right behind it, so dropping it + // would leave the deployment running with nothing left to remove it. + const pending = pendingUndeploys.get(frame.agentAddress); + const answering = { connection, requestId: frame.requestId }; + if (pending?.generation === frame.generation) { + // A closed connection cannot receive an answer. Keep only the + // current socket's waiters while teardown spans reconnects. + pending.replies = pending.replies.filter( + (waiting) => waiting.connection === connection, ); - // A rejected mail is simply not delivered -- the same drop as the - // no-handler path below: no ack, no reply frame, no dispatch-fail. - // The hub's redelivery and expiry machinery handles the rest. - break; + pending.replies.push(answering); + return; } - // Admitted: deliver exactly as an admitted frame always has, keeping - // the detached durable settlement and detached ack below off the - // messageQueue chain -- only the verify above is awaited inline. - // - // Supervised deployments register the deployment-level mail - // address on `mailInboundRouter` once their supervisor spawns; - // that handler delivers the bytes to the supervisor's mail-bus - // subscription, which is what the workflow-host's `awaitSignal` - // listens on. Mail for an address with no registered handler has - // no receiver -- the in-process session runtime that once backed - // it is retired -- so it is logged and dropped. - // - // Guard the router call with try/catch so a synchronous throw does - // not reject this `handleMessage` promise and wedge the per-connection - // `messageQueue` chain. A rejected chain would silently drop every - // subsequent frame -- including the heartbeat `pong` -- and stall the - // link. The durable settlement is observed off the chain (below). - let durable: Promise | null = null; - if (mailInboundRouter !== undefined) { + const cleanup = { generation: frame.generation, replies: [answering] }; + pendingUndeploys.set(frame.agentAddress, cleanup); + frameLanes.run([frame.agentAddress], async () => { try { - durable = mailInboundRouter.tryRoute(frame.agentAddress, rawBytes); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`mail.inbound router threw for ${frame.agentAddress}: ${msg}`; - } - } - if (durable === null) { - logger.warn`Dropping mail.inbound for ${frame.agentAddress}: no registered handler`; - break; - } - // Acknowledge durable receipt only AFTER the inbox write settles, and - // only for hub-originated mail carrying a hub-minted messageId (the - // ack handshake). Observe the settlement DETACHED from the - // `messageQueue` chain so a slow or failing inbox write never wedges - // frame processing; on rejection (transient failure, stale refusal, or - // a tearing-down phase) no ack is sent, so the hub redelivers. - const ackMessageId = frame.messageId; - if (ackMessageId !== undefined) { - void durable - .then(() => { - send({ - type: "mail.inbound.ack", + const error = await handleAgentUndeploy(frame); + for (const waiting of cleanup.replies) { + const response = { + requestId: waiting.requestId, agentAddress: frame.agentAddress, - messageId: ackMessageId, - }); - }) - .catch((err: unknown) => { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Withholding mail.inbound.ack for ${frame.agentAddress} ${ackMessageId}; hub will redeliver: ${msg}`; - }); - } else { - // Relayed agent-to-agent mail carries no hub-minted messageId and - // does not participate in the ack handshake. Still observe the - // settlement so a rejection is logged, not left unhandled. - void durable.catch((err: unknown) => { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Inbound mail delivery failed for ${frame.agentAddress}: ${msg}`; - }); - } - break; + generation: frame.generation, + }; + replyOn(waiting.connection)( + error === undefined + ? { type: "agent.undeploy.ack", ...response } + : { type: "agent.undeploy.error", ...response, error }, + ); + } + } finally { + if (pendingUndeploys.get(frame.agentAddress) === cleanup) + pendingUndeploys.delete(frame.agentAddress); + } + }); + return; } + default: + onLanes(frame.type, [frame.agentAddress], () => + handleFrame(frame, reply), + ); + } + } + + async function handleFrame( + frame: Exclude< + HubFrame, + { + type: + | "pong" + | "welcome" + | "agent.undeploy" + | "workflow.probe.request" + | "workflow.control" + | "run.grants" + | "mail.inbound"; + } + >, + reply: Reply, + ): Promise { + switch (frame.type) { case "agent.deploy": - await handleAgentDeploy(frame); - break; - case "agent.undeploy": - await handleAgentUndeploy(frame); - break; - case "pong": - lastPongAt = Date.now(); + await handleAgentDeploy(frame, reply); break; case "repo.pack.push": - handlePackPush(frame); + handlePackPush(frame, reply); break; case "repo.pack.done": - await handlePackDone(frame); - break; - case "sync.request": - void handleSyncRequest(frame); + await handlePackDone(frame, reply); break; case "signal.deliver": await handleSignalDeliver(frame); break; - case "run.grants": - await handleRunGrants(frame); - break; case "sender.key.refresh": await handleSenderKeyRefresh(frame); break; @@ -1814,19 +2253,10 @@ export function createHubLink(config: HubLinkConfig): HubLink { await handleDrainDeliver(frame); break; case "sources.update": - await handleSourcesUpdate(frame); + await handleSourcesUpdate(frame, reply); break; case "credentials.update": - await handleCredentialsUpdate(frame); - break; - case "workflow.probe.request": - await handleWorkflowProbeRequest(frame); - break; - case "oauth.login.start": - void handleOAuthLoginStart(frame).catch(() => undefined); - break; - case "oauth.login.cancel": - cancelOAuthLogin(frame.requestId); + await handleCredentialsUpdate(frame, reply); break; case "repo.pack.ack": handlePackAck(frame); @@ -1850,7 +2280,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { throw new Error("HubLink.connect called after close"); } - handshakePending = true; + welcomed = false; const connection = new WebSocket(hubURL); ws = connection; @@ -1872,77 +2302,37 @@ export function createHubLink(config: HubLinkConfig): HubLink { connection.close(); return; } - send({ type: "ping" }); + sendOnConnection(connection, { type: "ping" }); }, pingIntervalMs); packReceiver.reset(); - packSender.cancelAll(CONNECTION_LOST_REASON); - // Abandon register retries armed against the prior connection. Any still - // parked correlation is re-registered by the reconnect re-emit once the - // handshake below re-routes the addresses, so a stale retry firing onto - // this fresh, not-yet-registered socket would only land unrouted. - registerAcker.cancelAll(); + blockHeldAddresses(); - // The first handshake is the sidecar's complete hosted-address - // announcement. A fresh sidecar sends register; one that restored a - // deployment sends reconnect instead. Sending an empty register before - // reconnect would expose a false empty inventory and let allocation - // reconciliation restore Hub state over the live workflow. - const restoredAddresses = getWorkflowAddresses(); - // Report the sidecar's cached rotatable senders on both frames: the - // register-vs-reconnect choice turns on workflow-address presence, so a - // sidecar with cached senders but no restored workflow substrate still - // announces them on a register frame. Omit the field when empty to honor - // its additive-optional wire shape. + // The hello reports every incarnation this sidecar holds, including one + // still deploying or tearing down, so the Hub decides from what is + // actually here which to keep. Nothing that must be delivered goes out + // until its `welcome`. const cachedSenderAddresses = getCachedSenderAddresses(); - const senderReport = - cachedSenderAddresses.length > 0 ? { cachedSenderAddresses } : {}; - if (restoredAddresses.length === 0) { - completeHandshake(connection, { - type: "register", - sidecarId, - token, - agentAddresses: [], - ...senderReport, - }); - } else { - completeHandshake(connection, { - type: "reconnect", - sidecarId, - token, - agentAddresses: restoredAddresses, - ...senderReport, - }); - } + sendOnConnection(connection, { + type: "hello", + sidecarId, + token, + capacity, + incarnations: getIncarnations(), + ...(cachedSenderAddresses.length > 0 ? { cachedSenderAddresses } : {}), + }); + // A Hub that never answers would otherwise hold back everything this + // sidecar must deliver for as long as the socket stays up. + welcomeTimer = setTimeout(() => { + welcomeTimer = null; + if (ws !== connection || welcomed) return; + logger.error`The Hub did not answer hello within ${String(welcomeTimeoutMs)}ms; reconnecting`; + connection.close(); + }, welcomeTimeoutMs); }); connection.addEventListener("message", (event) => { - if (typeof event.data === "string") { - // Attach a tail `.catch` to the chained handler so any - // unhandled throw inside `handleMessage` is observed and - // surfaces as a logged warning rather than rejecting the - // shared `messageQueue` chain. A rejected chain wedges every - // subsequent `messageQueue.then(...)` -- including the - // heartbeat `pong` path -- and silently stalls the link. - // Per-arm guards (mail/signal/drain) are the primary defence; - // this catch is the belt-and-braces guarantee that no future - // unguarded arm can wedge the link. - // - // This chain also serializes inbound frames: each frame's - // handler runs to completion before the next begins. A downstream - // invariant depends on that ordering -- the workflow - // source-rotation persist rolls back on failure assuming no second - // rotation is in flight, which holds only because sources.update - // frames are processed one at a time here. Parallelizing this - // dispatch would break that rollback. - const data = event.data; - messageQueue = messageQueue.then(() => - handleMessage(data).catch((err: unknown) => { - const msg = err instanceof Error ? err.message : String(err); - logger.warn`Unhandled error in handleMessage: ${msg}`; - }), - ); - } + if (typeof event.data === "string") receiveFrame(event.data, connection); }); connection.addEventListener("close", () => { @@ -1952,31 +2342,19 @@ export function createHubLink(config: HubLinkConfig): HubLink { if (ws !== connection) return; logger.info`Disconnected from hub`; ws = null; - handshakePending = true; + welcomed = false; + stopWelcomeTimer(); if (pingTimer !== null) { clearInterval(pingTimer); pingTimer = null; } - // Abandon in-flight register retries: the link is down, so recovery - // belongs to the reconnect re-emit, and a lingering watchdog would only - // fire onto a closed socket. + // Transfers and register retries belong to the connection that carried + // them. Recovery belongs to the re-drive on the next `welcome`, and a + // lingering watchdog would only fire onto a closed socket. + packSender.cancelAll(CONNECTION_LOST_REASON); registerAcker.cancelAll(); - // The hub will never see a terminal frame for logins staged on this - // dead connection, and the sidecar-side listener outlives it — close - // those listeners so a retry can rebind the pinned ports. - cancelAllOAuthLogins(); - // The hub dropped every route this link held. Block workflow-run pushes - // for the deployments it hosts until the authenticated reconnect - // re-routes them, so the coalescing pusher does not re-ship onto the - // fresh, not-yet-registered connection (which the hub drops as - // "unrouted"). `onWorkflowAddressesRoutable`, fired after the reconnect - // frame is sent, lifts the block and re-drives. - if (onWorkflowAddressesUnroutable !== undefined) { - const hosted = getWorkflowAddresses(); - if (hosted.length > 0) { - onWorkflowAddressesUnroutable(hosted); - } - } + // The Hub dropped every route this link held. + blockHeldAddresses(); if (!closed) { cancelReconnect = scheduleReconnect(() => { cancelReconnect = null; @@ -1998,6 +2376,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { function close(): void { closed = true; + stopWelcomeTimer(); if (cancelReconnect !== null) { cancelReconnect(); cancelReconnect = null; @@ -2015,17 +2394,21 @@ export function createHubLink(config: HubLinkConfig): HubLink { const sendEvent: SessionEventSink = ( agentAddress, + generation, sessionId, event, childRunId, ) => { - send({ - type: "agent.event", - agentAddress, - sessionId, - event, - ...(childRunId !== undefined ? { childRunId } : {}), - }); + welcomedSocket()?.send( + JSON.stringify({ + type: "agent.event", + agentAddress, + generation, + sessionId, + event, + ...(childRunId !== undefined ? { childRunId } : {}), + } satisfies SidecarFrame), + ); }; const sendSignalCorrelationRegister: HubLink["sendSignalCorrelationRegister"] = @@ -2045,6 +2428,7 @@ export function createHubLink(config: HubLinkConfig): HubLink { runId: registration.runId, anchorRunId: registration.anchorRunId, agentAddress: registration.agentAddress, + generation: registration.generation, kind: registration.kind, snapshot: registration.approvalSnapshot, }; @@ -2052,11 +2436,21 @@ export function createHubLink(config: HubLinkConfig): HubLink { registerAcker.send(frame); }; + const sendDeploymentStopped: HubLink["sendDeploymentStopped"] = (report) => { + welcomedSocket()?.send( + JSON.stringify({ + type: "deployment.stopped", + ...report, + } satisfies SidecarFrame), + ); + }; + return { connect, close, sendEvent, sendSignalCorrelationRegister, pushWorkflowRunPack, + sendDeploymentStopped, }; } diff --git a/vendor/intx/hub-agent/src/ws/inbound-signature.ts b/vendor/intx/hub-agent/src/ws/inbound-signature.ts index e19f3e184d..356fae6fa1 100644 --- a/vendor/intx/hub-agent/src/ws/inbound-signature.ts +++ b/vendor/intx/hub-agent/src/ws/inbound-signature.ts @@ -1,10 +1,10 @@ import { getLogger } from "@intx/log"; +import type { CryptoProvider } from "@intx/types/runtime"; import type { AuthorControllableOutcome, - CryptoProvider, InboundMailOutcome, InboundMailPolicy, -} from "@intx/types/runtime"; +} from "@intx/types/inbound-mail-policy"; import { verifyMimeSignature } from "@intx/mailbox"; import { parseHeaderSection, extractAddrSpec } from "@intx/mime"; diff --git a/vendor/intx/hub-agent/src/ws/register-acker.ts b/vendor/intx/hub-agent/src/ws/register-acker.ts index fc9ee0f934..deede70efc 100644 --- a/vendor/intx/hub-agent/src/ws/register-acker.ts +++ b/vendor/intx/hub-agent/src/ws/register-acker.ts @@ -21,7 +21,7 @@ export const DEFAULT_REGISTER_ACK_MAX_ATTEMPTS = 3; type PendingRegister = { frame: SignalCorrelationRegisterFrame; attempts: number; - timer: ReturnType; + cancelRetry: () => void; }; export type RegisterAckerConfig = { @@ -32,14 +32,15 @@ export type RegisterAckerConfig = { */ sendFrame: (frame: SignalCorrelationRegisterFrame) => void; /** - * True only when the link is OPEN. The acker abandons a pending retry the - * moment the link is not open: re-sending onto a fresh, not-yet-registered - * socket would land "unrouted", and the reconnect re-emit re-registers the - * whole parked set anyway. + * True only when the link is open and welcomed. The acker abandons a pending + * retry the moment it is not: before a `welcome` a resend would only queue a + * duplicate of the registrations re-sent on `welcome`. */ isOpen: () => boolean; timeoutMs?: number; maxAttempts?: number; + /** Arms a retry and returns its canceller; tests drive it instead of time. */ + scheduleRetry?: (callback: () => void, delayMs: number) => () => void; }; /** @@ -65,12 +66,19 @@ export interface RegisterAcker { /** Settle the pending retry for this correlationId; false if none was pending. */ handleAck(correlationId: string): boolean; /** - * Abandon every pending retry without re-sending or re-acking. Called on link - * close and on the reconnect open edge, symmetric with the ping timer and the - * pack sender's `cancelAll`, so no timer leaks and no retry fires onto a dead - * or not-yet-registered socket. + * Abandon every pending retry without re-sending or re-acking. Called when + * the socket closes and on link close, symmetric with the ping timer and the + * pack sender's `cancelAll`, so no timer leaks and no retry fires onto a + * dead socket. */ cancelAll(): void; + /** + * Abandon the pending retries whose frame `matches`, as `cancelAll` does for + * all of them. Called when the incarnation a register belongs to is gone. + */ + cancelWhere( + matches: (frame: SignalCorrelationRegisterFrame) => boolean, + ): void; } export function createRegisterAcker( @@ -78,10 +86,20 @@ export function createRegisterAcker( ): RegisterAcker { const timeoutMs = config.timeoutMs ?? DEFAULT_REGISTER_ACK_TIMEOUT_MS; const maxAttempts = config.maxAttempts ?? DEFAULT_REGISTER_ACK_MAX_ATTEMPTS; + const scheduleRetry = + config.scheduleRetry ?? + ((callback: () => void, delayMs: number) => { + const timer = setTimeout(callback, delayMs); + return () => { + clearTimeout(timer); + }; + }); const pending = new Map(); - function schedule(correlationId: string): ReturnType { - return setTimeout(() => onTimeout(correlationId), timeoutMs); + function schedule(correlationId: string): () => void { + return scheduleRetry(() => { + onTimeout(correlationId); + }, timeoutMs); } function onTimeout(correlationId: string): void { @@ -100,36 +118,46 @@ export function createRegisterAcker( } entry.attempts += 1; config.sendFrame(entry.frame); - entry.timer = schedule(correlationId); + entry.cancelRetry = schedule(correlationId); } function send(frame: SignalCorrelationRegisterFrame): void { const existing = pending.get(frame.correlationId); if (existing !== undefined) { - clearTimeout(existing.timer); + existing.cancelRetry(); } config.sendFrame(frame); pending.set(frame.correlationId, { frame, attempts: 1, - timer: schedule(frame.correlationId), + cancelRetry: schedule(frame.correlationId), }); } function handleAck(correlationId: string): boolean { const entry = pending.get(correlationId); if (entry === undefined) return false; - clearTimeout(entry.timer); + entry.cancelRetry(); pending.delete(correlationId); return true; } function cancelAll(): void { for (const entry of pending.values()) { - clearTimeout(entry.timer); + entry.cancelRetry(); } pending.clear(); } - return { send, handleAck, cancelAll }; + function cancelWhere( + matches: (frame: SignalCorrelationRegisterFrame) => boolean, + ): void { + for (const [correlationId, entry] of pending) { + if (!matches(entry.frame)) continue; + entry.cancelRetry(); + pending.delete(correlationId); + } + } + + return { send, handleAck, cancelAll, cancelWhere }; } diff --git a/vendor/intx/hub-agent/tsconfig.json b/vendor/intx/hub-agent/tsconfig.json index 14eedeec17..e34754c1df 100644 --- a/vendor/intx/hub-agent/tsconfig.json +++ b/vendor/intx/hub-agent/tsconfig.json @@ -10,18 +10,20 @@ "tsBuildInfoFile": "tsconfig.tsbuildinfo", "rootDir": "../../.." }, - "include": ["src"], + "include": [ + "src" + ], "exclude": [], "references": [ - { - "path": "../harness/tsconfig.src.json" - }, { "path": "../mail-memory/tsconfig.src.json" }, { "path": "../mailbox/tsconfig.src.json" }, + { + "path": "../mime/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/hub-agent/tsconfig.src.json b/vendor/intx/hub-agent/tsconfig.src.json index 6a338ef0a9..9a300edb20 100644 --- a/vendor/intx/hub-agent/tsconfig.src.json +++ b/vendor/intx/hub-agent/tsconfig.src.json @@ -1,24 +1,33 @@ { "extends": "../../../tsconfig.base.json", - "include": ["src/**/*.ts", "package.json", "src/**/*.json"], - "exclude": ["src/**/*.test.ts", "src/**/*.test.tsx"], + "include": [ + "src/**/*.ts", + "package.json", + "src/**/*.json" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/*.test.tsx" + ], "compilerOptions": { - "types": ["bun"], + "types": [ + "bun" + ], "composite": true, "emitDeclarationOnly": true, "outDir": "dist", "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" }, "references": [ - { - "path": "../harness/tsconfig.src.json" - }, { "path": "../mail-memory/tsconfig.src.json" }, { "path": "../mailbox/tsconfig.src.json" }, + { + "path": "../mime/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" } diff --git a/vendor/intx/hub-api/VENDORED-FROM b/vendor/intx/hub-api/VENDORED-FROM index 9f625e417d..de9cc8d9af 100644 --- a/vendor/intx/hub-api/VENDORED-FROM +++ b/vendor/intx/hub-api/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-api) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-workflow-deploy-bearer: middleware/workflow-run-deploy-auth.ts is a bearer-authenticated mirror onto the session-cookie POST/GET .../workflows/deployments route; MountHubRoutesDeps and CreateAppOpts gain an optional workflowRunAuthenticator that mounts it ahead of resolveTenant, and src/index.ts exports the middleware and its types. CL-7505: run-source-resolution.ts threads an optional servingRefresh hook into buildSource, so an expiring oauth_token credential refreshes in place (or the offering is skipped) before its secret is served. +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-workflow-deploy-bearer: middleware/workflow-run-deploy-auth.ts is a bearer-authenticated mirror onto the session-cookie POST/GET .../workflows/deployments route; MountHubRoutesDeps and CreateAppOpts gain an optional workflowRunAuthenticator that mounts it ahead of resolveTenant, and src/index.ts exports the middleware and its types. Retired at this pin: the CL-7505 servingRefresh hook on run-source-resolution.ts (unused; hub OAuth refresh pushes updates via pushSourceUpdates). diff --git a/vendor/intx/hub-api/package.json b/vendor/intx/hub-api/package.json index 117a24be8d..18e16de524 100644 --- a/vendor/intx/hub-api/package.json +++ b/vendor/intx/hub-api/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-api", "description": "The hub's HTTP API: routes, middleware, authentication, and served OpenAPI spec", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -16,14 +16,14 @@ "dependencies": { "@hono/standard-validator": "^0.2.2", "@intx/agent": "workspace:*", - "@intx/authz": "0.4.0", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.4.0", + "@intx/hub-common": "workspace:*", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.4.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:", diff --git a/vendor/intx/hub-api/src/app.ts b/vendor/intx/hub-api/src/app.ts index 4d71312891..0a78b77e50 100644 --- a/vendor/intx/hub-api/src/app.ts +++ b/vendor/intx/hub-api/src/app.ts @@ -21,7 +21,11 @@ import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import type { AppEnv } from "./context"; import { createSessionMiddleware } from "./middleware/session"; -import { createRequireGrant, type RequireGrant } from "./middleware/grant"; +import { + createAuthorizeGrant, + createRequireGrant, + type RequireGrant, +} from "./middleware/grant"; import { createResolveTenant, requireAuth } from "./middleware/tenant"; import type { GetSession } from "./session"; import type { @@ -32,6 +36,7 @@ import type { SidecarRouter, WorkflowAllocationService, WorkflowDispatchService, + WorkflowLifecycleService, } from "@intx/hub-sessions"; import { createMeRoutes } from "./routes/me"; @@ -140,6 +145,7 @@ export type MountHubRoutesDeps = { sessionService: SessionService; workflowAllocationService?: WorkflowAllocationService; workflowDispatchService?: WorkflowDispatchService; + workflowLifecycleService?: WorkflowLifecycleService; eventCollectors: EventCollectorRegistry; /** * Encrypts credential secrets at rest on the credential/oauth write paths. @@ -212,6 +218,7 @@ export function mountHubRoutes( sessionService, workflowAllocationService, workflowDispatchService, + workflowLifecycleService, eventCollectors, sidecarWsHandler, assetService, @@ -229,6 +236,10 @@ export function mountHubRoutes( const conditionRegistry: ConditionRegistry = opts.conditionRegistry ?? { time_window: timeWindowEvaluator, }; + const authorizeGrant = createAuthorizeGrant({ + grantStore, + conditionRegistry, + }); const requireGrant: RequireGrant = createRequireGrant({ grantStore, conditionRegistry, @@ -310,8 +321,19 @@ export function mountHubRoutes( // so Hono includes it in the middleware chain. app.use("/api/tenants/:tenantId/*", resolveTenant); - // Global tenant routes (create needs auth, detail/update handle auth inline) - app.route("/api/tenants", createTenantRoutes({ db, principalKeyStore })); + // Tenant create, detail and update. Detail and update mount resolveTenant + // and requireGrant on their own handlers, so their gate does not depend on + // the wildcard above also matching the bare tenant path. + app.route( + "/api/tenants", + createTenantRoutes({ + db, + principalKeyStore, + authorizeGrant, + resolveTenant, + requireGrant, + }), + ); // Tenant-scoped routes app.route( @@ -355,6 +377,9 @@ export function mountHubRoutes( ...(workflowDispatchService !== undefined ? { workflowDispatchService } : {}), + ...(workflowLifecycleService !== undefined + ? { workflowLifecycleService } + : {}), grantStore, conditionRegistry, requireGrant, @@ -370,7 +395,12 @@ export function mountHubRoutes( // `/:runId` patterns, so this ordering is belt-and-suspenders. app.route( "/api/tenants/:tenantId/workflows/definitions", - createWorkflowDefinitionRoutes({ db, requireGrant }), + createWorkflowDefinitionRoutes({ + db, + requireGrant, + grantStore, + conditionRegistry, + }), ); // The workflow deploy + signal + listing surface reads the workflow-run @@ -391,6 +421,7 @@ export function mountHubRoutes( sidecarRouter, repoStore, grantStore, + authorizeGrant, requireGrant, }), ); @@ -562,6 +593,7 @@ export type CreateAppOpts = { sessionService: SessionService; workflowAllocationService?: WorkflowAllocationService; workflowDispatchService?: WorkflowDispatchService; + workflowLifecycleService?: WorkflowLifecycleService; eventCollectors: EventCollectorRegistry; /** * Encrypts credential secrets at rest on the credential/oauth write paths. @@ -577,7 +609,14 @@ export type CreateAppOpts = { * `PRINCIPAL_KEY_ENCRYPTION_KEY`. */ principalKeyStore?: PrincipalKeyStore; - /** See `MountHubRoutesDeps.workflowRunAuthenticator`. */ + /** + * Resolves a sidecar bearer token + run address to the tenant/principal + * it acts as. When supplied, `POST/GET .../workflows/deployments` + * additionally accepts a bearer-authenticated workflow-run caller (see + * `./middleware/workflow-run-deploy-auth.ts`), mirroring every other + * workflow-run write surface. Omitted in tests that have no reason to + * exercise that path -- the session-cookie path is unaffected either way. + */ workflowRunAuthenticator?: WorkflowRunAuthenticator; grantStore?: GrantStore; approvalStore?: ApprovalStore; @@ -602,6 +641,7 @@ export function createApp({ sessionService, workflowAllocationService, workflowDispatchService, + workflowLifecycleService, eventCollectors, credentialCipher, principalKeyStore, @@ -638,12 +678,13 @@ export function createApp({ ...(workflowDispatchService !== undefined ? { workflowDispatchService } : {}), + ...(workflowLifecycleService !== undefined + ? { workflowLifecycleService } + : {}), eventCollectors, ...(credentialCipher ? { credentialCipher } : {}), ...(principalKeyStore ? { principalKeyStore } : {}), - ...(workflowRunAuthenticator !== undefined - ? { workflowRunAuthenticator } - : {}), + ...(workflowRunAuthenticator ? { workflowRunAuthenticator } : {}), assetService, repoStore, maxTarballBytes, diff --git a/vendor/intx/hub-api/src/attachment-validation.ts b/vendor/intx/hub-api/src/attachment-validation.ts deleted file mode 100644 index 766119f715..0000000000 --- a/vendor/intx/hub-api/src/attachment-validation.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { - base64Decode, - isAllowedMimeType, - PER_ATTACHMENT_LIMIT_BYTES, - PER_MESSAGE_TOTAL_LIMIT_BYTES, - type AttachmentError, -} from "@intx/types"; -import type { MessageAttachment } from "@intx/types/runtime"; - -/** - * A single attachment as it arrives on the request body: a MIME type, the - * base64-encoded bytes, and an optional filename. - */ -export type AttachmentInput = { - mimeType: string; - data: string; - name?: string; -}; - -/** - * The attachment policy the route validates against. Defaults to the - * system-level allowlist and limits; a future per-agent or per-workflow - * lookup substitutes a narrowed policy here (the injection seam — no - * validation logic changes). - */ -export type AttachmentPolicy = { - isAllowed: (mimeType: string) => boolean; - perAttachmentLimitBytes: number; - perMessageTotalLimitBytes: number; -}; - -export const DEFAULT_ATTACHMENT_POLICY: AttachmentPolicy = { - isAllowed: isAllowedMimeType, - perAttachmentLimitBytes: PER_ATTACHMENT_LIMIT_BYTES, - perMessageTotalLimitBytes: PER_MESSAGE_TOTAL_LIMIT_BYTES, -}; - -// The error shape is the wire contract `AttachmentError` from @intx/types: -// a machine-actionable `code`, the fields needed to locate the rejection, -// and a human-readable `message`. Defining it once in @intx/types lets the -// route document the structured 400 in its OpenAPI surface. -export type AttachmentValidationError = AttachmentError; - -export type AttachmentValidationResult = - | { ok: true; attachments: MessageAttachment[] } - | { ok: false; error: AttachmentError }; - -function decode(data: string): Uint8Array | null { - try { - return base64Decode(data.replace(/\s+/g, "")); - } catch { - return null; - } -} - -/** - * Validate and decode request-body attachments against a policy. - * - * The most specific error wins, in this priority order so the caller sees - * the most actionable failure: per-attachment oversize, then disallowed - * MIME type, then invalid name, then malformed base64, then per-message - * total oversize. On success the decoded `MessageAttachment[]` is returned - * with names defaulted to `attachment-{index}` by request-body position. - */ -export function validateAttachments( - inputs: readonly AttachmentInput[], - policy: AttachmentPolicy = DEFAULT_ATTACHMENT_POLICY, -): AttachmentValidationResult { - let oversize: AttachmentValidationError | undefined; - let disallowed: AttachmentValidationError | undefined; - let invalidName: AttachmentValidationError | undefined; - let malformed: AttachmentValidationError | undefined; - const decoded: MessageAttachment[] = []; - - for (const [index, input] of inputs.entries()) { - const bytes = decode(input.data); - if (bytes === null) { - malformed ??= { - code: "malformed_base64", - message: `attachment ${index} is not valid base64`, - attachmentIndex: index, - }; - continue; - } - if (bytes.length > policy.perAttachmentLimitBytes) { - oversize ??= { - code: "oversize_attachment", - message: `attachment ${index} is ${bytes.length} bytes, over the ${policy.perAttachmentLimitBytes}-byte limit`, - attachmentIndex: index, - byteLength: bytes.length, - limitBytes: policy.perAttachmentLimitBytes, - }; - continue; - } - if (!policy.isAllowed(input.mimeType)) { - disallowed ??= { - code: "disallowed_mime_type", - message: `attachment ${index} has unsupported content type "${input.mimeType}"`, - attachmentIndex: index, - mimeType: input.mimeType, - }; - continue; - } - // A user-supplied name becomes the MIME part's quoted filename, so it - // must not contain characters that would break out of the header - // (line breaks or a double quote). The default name is always safe. - if (input.name !== undefined && /[\r\n"]/.test(input.name)) { - invalidName ??= { - code: "invalid_attachment_name", - message: `attachment ${index} has a name with invalid characters (no quotes or line breaks)`, - attachmentIndex: index, - }; - continue; - } - decoded.push({ - name: input.name ?? `attachment-${index}`, - contentType: input.mimeType, - data: bytes, - }); - } - - if (oversize !== undefined) return { ok: false, error: oversize }; - if (disallowed !== undefined) return { ok: false, error: disallowed }; - if (invalidName !== undefined) return { ok: false, error: invalidName }; - if (malformed !== undefined) return { ok: false, error: malformed }; - - const totalBytes = decoded.reduce((sum, a) => sum + a.data.length, 0); - if (totalBytes > policy.perMessageTotalLimitBytes) { - return { - ok: false, - error: { - code: "oversize_total", - message: `attachments total ${totalBytes} bytes, over the ${policy.perMessageTotalLimitBytes}-byte limit`, - totalBytes, - limitBytes: policy.perMessageTotalLimitBytes, - }, - }; - } - - return { ok: true, attachments: decoded }; -} diff --git a/vendor/intx/hub-api/src/error-response.ts b/vendor/intx/hub-api/src/error-response.ts index d6d2fd7191..b6d8ac5e7f 100644 --- a/vendor/intx/hub-api/src/error-response.ts +++ b/vendor/intx/hub-api/src/error-response.ts @@ -1,6 +1,11 @@ import type { Context } from "hono"; import type { ContentfulStatusCode } from "hono/utils/http-status"; +import type { TenantConfigInvalidError } from "@intx/db"; +import { getLogger } from "@intx/log"; + +const log = getLogger(["hub", "errors"]); + // The hub's canonical error envelope: `{ error: { code, message } }`, with // the HTTP status paired to the machine-readable code. Every route returns // this shape for non-2xx responses so clients can switch on `code` without @@ -14,6 +19,7 @@ const STATUS_BY_CODE: Readonly> = { forbidden: 403, payload_too_large: 413, invalid_workflow: 409, + invalid_tenant_config: 409, sidecar_unavailable: 502, workflow_run_not_running: 409, deployment_unreachable: 409, @@ -22,6 +28,7 @@ const STATUS_BY_CODE: Readonly> = { invalid_request: 400, unavailable: 503, unsupported_source: 400, + asset_kind_mismatch: 409, anchor_run_missing: 500, reserved_signal_name: 400, unaddressable_run: 400, @@ -49,3 +56,16 @@ export function errorResponse( } return c.json({ error: { code, message } }, status); } + +/** Inherited config can belong to a tenant the caller cannot read. */ +export function tenantConfigErrorResponse( + c: Context, + error: TenantConfigInvalidError, +): Response { + log.warn`${error.message}`; + return errorResponse( + c, + "invalid_tenant_config", + "The tenant or an ancestor has invalid configuration", + ); +} diff --git a/vendor/intx/hub-api/src/git-http/advertise-refs.ts b/vendor/intx/hub-api/src/git-http/advertise-refs.ts index c431f76abd..7a3c283ee4 100644 --- a/vendor/intx/hub-api/src/git-http/advertise-refs.ts +++ b/vendor/intx/hub-api/src/git-http/advertise-refs.ts @@ -27,7 +27,7 @@ * that stock `git clone` and `git ls-remote` accept the response. */ -import type { RepoId } from "@intx/types/sidecar"; +import type { RepoId } from "@intx/types/repo"; import { glob } from "@intx/hub-common"; import { writePktLine, writeFlush } from "./pkt-line"; diff --git a/vendor/intx/hub-api/src/git-http/upload-pack.ts b/vendor/intx/hub-api/src/git-http/upload-pack.ts index 7d78dd7dd7..4a9c975a48 100644 --- a/vendor/intx/hub-api/src/git-http/upload-pack.ts +++ b/vendor/intx/hub-api/src/git-http/upload-pack.ts @@ -16,7 +16,7 @@ import fs from "node:fs"; import git from "isomorphic-git"; -import type { RepoId } from "@intx/types/sidecar"; +import type { RepoId } from "@intx/types/repo"; import { glob } from "@intx/hub-common"; import { createNegotiatedPack, diff --git a/vendor/intx/hub-api/src/middleware/git-token-auth.ts b/vendor/intx/hub-api/src/middleware/git-token-auth.ts index 0b57a1c84a..478b3deebc 100644 --- a/vendor/intx/hub-api/src/middleware/git-token-auth.ts +++ b/vendor/intx/hub-api/src/middleware/git-token-auth.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { createMiddleware } from "hono/factory"; import type { Context, Env, MiddlewareHandler } from "hono"; @@ -13,6 +13,7 @@ import { base64Decode } from "@intx/types"; import type { AppEnv, PrincipalRow, TenantRow } from "../context"; import { errorResponse } from "../error-response"; +import { findUserPrincipal } from "../principal-lookup"; const log = getLogger(["hub", "git-token"]); @@ -282,12 +283,9 @@ async function resolvePrincipal( }); } - return await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, resolvedTenantId), - eq(principal.kind, "user"), - eq(principal.refId, tokenRow.userId), - ), + return await findUserPrincipal(db, { + tenantId: resolvedTenantId, + userId: tokenRow.userId, }); } diff --git a/vendor/intx/hub-api/src/middleware/grant.ts b/vendor/intx/hub-api/src/middleware/grant.ts index b741d2467a..35ad145a61 100644 --- a/vendor/intx/hub-api/src/middleware/grant.ts +++ b/vendor/intx/hub-api/src/middleware/grant.ts @@ -1,5 +1,5 @@ import { createMiddleware } from "hono/factory"; -import type { MiddlewareHandler } from "hono"; +import type { Context, MiddlewareHandler } from "hono"; import { authorize } from "@intx/authz"; import { getLogger } from "@intx/log"; @@ -30,6 +30,55 @@ export type CreateRequireGrantDeps = { conditionRegistry: ConditionRegistry; }; +export type AuthorizeGrantCheck = { + principalId: string; + tenantId: string; + resource: string; + action: string; +}; + +/** + * Bound grant check for a caller that names the principal and tenant itself, + * such as a route whose tenant comes from the request body rather than the + * path. Logs a denial and returns whether the grant allows the action. + */ +export type AuthorizeGrant = (check: AuthorizeGrantCheck) => Promise; + +export function createAuthorizeGrant({ + grantStore, + conditionRegistry, +}: CreateRequireGrantDeps): AuthorizeGrant { + return async function authorizeGrant({ + principalId, + tenantId, + resource, + action, + }) { + const result = await authorize( + grantStore, + principalId, + tenantId, + resource, + action, + conditionRegistry, + ); + if (result.effect === "allow") return true; + + log.info( + "Authorization denied for {principalId}: {resource} {action} -> {effect}", + { + principalId, + tenantId, + resource, + action, + effect: result.effect ?? "no_match", + resolvedBy: result.resolvedBy?.id ?? null, + }, + ); + return false; + }; +} + /** * Builds a `requireGrant` middleware factory bound to the application's * grant store and condition registry. Usage: @@ -37,45 +86,26 @@ export type CreateRequireGrantDeps = { * const requireGrant = createRequireGrant({ grantStore, conditionRegistry }); * app.get("/", requireGrant("workflow-run:*", "read"), handler); */ -export function createRequireGrant({ - grantStore, - conditionRegistry, -}: CreateRequireGrantDeps): RequireGrant { +export function createRequireGrant(deps: CreateRequireGrantDeps): RequireGrant { + const authorizeGrant = createAuthorizeGrant(deps); return function requireGrant(resource, action) { return createMiddleware(async (c, next) => { - const principal = c.get("principal"); - const tenant = c.get("tenant"); - const resolvedResource = typeof resource === "function" ? resource({ param: (name) => c.req.param(name) }) : resource; - const result = await authorize( - grantStore, - principal.id, - tenant.id, - resolvedResource, + const allowed = await authorizeGrant({ + principalId: c.get("principal").id, + tenantId: c.get("tenant").id, + resource: resolvedResource, action, - conditionRegistry, - ); - - if (result.effect === "allow") { + }); + if (allowed) { await next(); return; } - log.info( - "Authorization denied for {principalId}: {resource} {action} -> {effect}", - { - principalId: principal.id, - resource: resolvedResource, - action, - effect: result.effect ?? "no_match", - resolvedBy: result.resolvedBy?.id ?? null, - }, - ); - return errorResponse( c, "forbidden", @@ -101,3 +131,29 @@ export function idResource( return id ? `${resourceType}:${id}` : `${resourceType}:*`; }; } + +/** + * In-handler grant check for an asset named in a request body. `idResource` + * only reads URL params and `requireGrant` runs before the body validator, so + * a body-sourced `asset:` cannot be gated in middleware. Returns the same + * 403 envelope as `requireGrant` when the principal lacks `read`, or `null` + * when the caller may proceed. + */ +export async function denyUnlessAssetGrant(args: { + c: Context; + authorizeGrant: AuthorizeGrant; + assetId: string; +}): Promise { + const allowed = await args.authorizeGrant({ + principalId: args.c.get("principal").id, + tenantId: args.c.get("tenant").id, + resource: `asset:${args.assetId}`, + action: "read", + }); + if (allowed) return null; + return errorResponse( + args.c, + "forbidden", + "You do not have permission to perform this action", + ); +} diff --git a/vendor/intx/hub-api/src/middleware/tenant.ts b/vendor/intx/hub-api/src/middleware/tenant.ts index 0fb4595b22..2710b496b8 100644 --- a/vendor/intx/hub-api/src/middleware/tenant.ts +++ b/vendor/intx/hub-api/src/middleware/tenant.ts @@ -1,12 +1,13 @@ -import { eq, and } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import type { Context, MiddlewareHandler, Next } from "hono"; -import { tenant, principal } from "@intx/db/schema"; +import { tenant } from "@intx/db/schema"; import type { DB } from "@intx/db"; import { getLogger } from "@intx/log"; import { unauthorizedResponse, type AppEnv, type TenantEnv } from "../context"; import { errorResponse } from "../error-response"; +import { findUserPrincipal } from "../principal-lookup"; const log = getLogger(["hub", "middleware", "tenant"]); @@ -46,12 +47,9 @@ export function createResolveTenant({ return errorResponse(c, "not_found", "Tenant not found"); } - const principalRow = await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, tenantId), - eq(principal.kind, "user"), - eq(principal.refId, user.id), - ), + const principalRow = await findUserPrincipal(db, { + tenantId, + userId: user.id, }); if (!principalRow) { diff --git a/vendor/intx/hub-api/src/principal-lookup.ts b/vendor/intx/hub-api/src/principal-lookup.ts new file mode 100644 index 0000000000..e9063a8443 --- /dev/null +++ b/vendor/intx/hub-api/src/principal-lookup.ts @@ -0,0 +1,26 @@ +import { and, eq } from "drizzle-orm"; + +import { principal } from "@intx/db/schema"; +import type { DB } from "@intx/db"; + +import type { PrincipalRow } from "./context"; + +export type FindUserPrincipalArgs = { + tenantId: string; + userId: string; +}; + +// Returns the row whatever its status; each caller owns its decision on a +// principal that is not active. +export async function findUserPrincipal( + db: DB["db"], + { tenantId, userId }: FindUserPrincipalArgs, +): Promise { + return await db.query.principal.findFirst({ + where: and( + eq(principal.tenantId, tenantId), + eq(principal.kind, "user"), + eq(principal.refId, userId), + ), + }); +} diff --git a/vendor/intx/hub-api/src/routes/agent-state-git.ts b/vendor/intx/hub-api/src/routes/agent-state-git.ts index f45aeb4606..f5b93436bf 100644 --- a/vendor/intx/hub-api/src/routes/agent-state-git.ts +++ b/vendor/intx/hub-api/src/routes/agent-state-git.ts @@ -37,7 +37,7 @@ import { workflowRun } from "@intx/db/schema"; import type { DB } from "@intx/db"; import { getLogger } from "@intx/log"; import type { RepoId, RepoStore, UserPrincipal } from "@intx/hub-sessions"; -import type { RepoAction } from "@intx/types/sidecar"; +import type { RepoAction } from "@intx/types/repo"; import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import type { diff --git a/vendor/intx/hub-api/src/routes/approvals.ts b/vendor/intx/hub-api/src/routes/approvals.ts index 50df0fab7c..92ed42c571 100644 --- a/vendor/intx/hub-api/src/routes/approvals.ts +++ b/vendor/intx/hub-api/src/routes/approvals.ts @@ -3,7 +3,6 @@ import { Hono } from "hono"; import type { Context } from "hono"; import { describeRoute, validator } from "hono-openapi"; -import { authorize } from "@intx/authz"; import type { DB, ApprovalStore, SignalCorrelationStore } from "@intx/db"; import { approval, sidecarAllocation } from "@intx/db/schema"; import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; @@ -29,6 +28,7 @@ import { getLogger } from "@intx/log"; import type { TenantEnv } from "../context"; import { errorResponse } from "../error-response"; +import { createAuthorizeGrant } from "../middleware/grant"; import { ts } from "../format"; import { approvalToolName, @@ -195,15 +195,17 @@ export async function resolveApproval( return { kind: "not_found" }; } - const authz = await authorize( + const authorizeGrant = createAuthorizeGrant({ grantStore, - args.principalId, - args.tenantId, - `approval:${approval.anchorRunId}`, - "resolve", conditionRegistry, - ); - if (authz.effect !== "allow") { + }); + const allowed = await authorizeGrant({ + principalId: args.principalId, + tenantId: args.tenantId, + resource: `approval:${approval.anchorRunId}`, + action: "resolve", + }); + if (!allowed) { return { kind: "forbidden" }; } @@ -384,7 +386,7 @@ export async function resolveApproval( return { kind: "resolved", approval: claimed.resolved }; } - sidecarRouter.sendSignalDeliver({ + await sidecarRouter.sendSignalDeliver({ agentAddress: claimed.claim.agentAddress, runId: claimed.claim.runId, signalName: signalName(approval.correlationId), @@ -399,6 +401,7 @@ export function createApprovalRoutes( deps: CreateApprovalRoutesDeps, ): Hono { const app = new Hono(); + const authorizeGrant = createAuthorizeGrant(deps); app.get( "/", @@ -429,15 +432,13 @@ export function createApprovalRoutes( // approvals by id via the detail route. The action is `resolve`, the // same capability the approve/reject routes gate on -- reading an // approval is part of being able to resolve it. - const authz = await authorize( - deps.grantStore, - principal.id, - tenant.id, - "approval:*", - "resolve", - deps.conditionRegistry, - ); - if (authz.effect !== "allow") { + const allowed = await authorizeGrant({ + principalId: principal.id, + tenantId: tenant.id, + resource: "approval:*", + action: "resolve", + }); + if (!allowed) { return errorResponse( c, "forbidden", @@ -502,15 +503,13 @@ export function createApprovalRoutes( // Same grant the approve/reject routes require: whoever can resolve this // approval can read it. No status filter -- a resolved approval is still // fetchable so the UI can show its terminal state. - const authz = await authorize( - deps.grantStore, - principal.id, - tenant.id, - `approval:${row.anchorRunId}`, - "resolve", - deps.conditionRegistry, - ); - if (authz.effect !== "allow") { + const allowed = await authorizeGrant({ + principalId: principal.id, + tenantId: tenant.id, + resource: `approval:${row.anchorRunId}`, + action: "resolve", + }); + if (!allowed) { return errorResponse( c, "forbidden", diff --git a/vendor/intx/hub-api/src/routes/assets.ts b/vendor/intx/hub-api/src/routes/assets.ts index 3a3bf90ce8..58bf45d55c 100644 --- a/vendor/intx/hub-api/src/routes/assets.ts +++ b/vendor/intx/hub-api/src/routes/assets.ts @@ -51,7 +51,7 @@ import { type RepoStore, type UserPrincipal, } from "@intx/hub-sessions"; -import type { RepoAction, RepoKind } from "@intx/types/sidecar"; +import type { RepoAction, RepoKind } from "@intx/types/repo"; import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import { AssetResponse, diff --git a/vendor/intx/hub-api/src/routes/git-tokens.ts b/vendor/intx/hub-api/src/routes/git-tokens.ts index b22bef2c40..f2b2ac3257 100644 --- a/vendor/intx/hub-api/src/routes/git-tokens.ts +++ b/vendor/intx/hub-api/src/routes/git-tokens.ts @@ -16,7 +16,7 @@ import { SVC_PREFIX, } from "@intx/hub-common"; import { getLogger } from "@intx/log"; -import type { RepoAction } from "@intx/types/sidecar"; +import type { RepoAction } from "@intx/types/repo"; import { base64urlEncode, paginatedSchema, ErrorResponse } from "@intx/types"; import { unauthorizedResponse, type AppEnv, type TenantEnv } from "../context"; diff --git a/vendor/intx/hub-api/src/routes/git-user-principal.ts b/vendor/intx/hub-api/src/routes/git-user-principal.ts index 5a2a4f549a..a6dfd7545d 100644 --- a/vendor/intx/hub-api/src/routes/git-user-principal.ts +++ b/vendor/intx/hub-api/src/routes/git-user-principal.ts @@ -22,7 +22,7 @@ import { authorize } from "@intx/authz"; import { repoActionToGrantVerb } from "@intx/hub-common"; import type { RefEntry, RepoStore, UserPrincipal } from "@intx/hub-sessions"; -import type { RepoAction } from "@intx/types/sidecar"; +import type { RepoAction } from "@intx/types/repo"; import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import type { GitTokenClaims } from "../middleware/git-token-auth"; diff --git a/vendor/intx/hub-api/src/routes/principals.ts b/vendor/intx/hub-api/src/routes/principals.ts index 74624f3a9c..31f4e56f00 100644 --- a/vendor/intx/hub-api/src/routes/principals.ts +++ b/vendor/intx/hub-api/src/routes/principals.ts @@ -15,6 +15,8 @@ import { import type { TenantEnv } from "../context"; import { errorResponse } from "../error-response"; +import { isReferencedRowViolation } from "../pg-errors"; +import { findUserPrincipal } from "../principal-lookup"; import { ts } from "../format"; import { generateId } from "@intx/hub-common"; import { idResource } from "../middleware/grant"; @@ -307,21 +309,46 @@ export function createPrincipalRoutes({ description: "Principal removed", }, 403: jsonResponse("Insufficient grants", ErrorResponse), + 409: jsonResponse("Principal is still referenced", ErrorResponse), }, }), async (c) => { const tenantCtx = c.get("tenant"); const principalId = c.req.param("principalId"); - const deleted = await db - .delete(principal) - .where( - and( - eq(principal.id, principalId), - eq(principal.tenantId, tenantCtx.id), - ), - ) - .returning(); + // Two invariants, each owned by one layer: + // - Existence within the tenant is owned by the delete's WHERE clause + // (`id` AND `tenantId`). authz cannot own it: the resource string + // `principal:{id}` is opaque, so a wildcard grant matches an id in any + // tenant. A foreign or unknown id matches zero rows -> 404, disclosing + // nothing across the tenant boundary. + // - "Cannot delete a principal a row still references" is owned by the + // foreign keys that point at principal.id. The handler does not query + // those dependents. A credential pre-check would miss + // agent_session.principal_id (NO ACTION), + // workflow_run_launch_spec.source_authority_principal_id (restrict), + // and workflow_definition.creator_principal_id (NO ACTION). The delete + // fires a violation only for a row actually being deleted, so the + // catch maps it to a 409 instead of a 500. + // Grants, role assignments, and principal keys cascade from the principal, + // so this delete does not clean them up and needs no transaction. + let deleted; + try { + deleted = await db + .delete(principal) + .where( + and( + eq(principal.id, principalId), + eq(principal.tenantId, tenantCtx.id), + ), + ) + .returning(); + } catch (err) { + if (!isReferencedRowViolation(err)) { + throw err; + } + return errorResponse(c, "conflict", "Principal is still referenced"); + } if (deleted.length === 0) { return errorResponse(c, "not_found", "Principal not found"); @@ -375,12 +402,9 @@ export function createInviteRoutes({ return errorResponse(c, "not_found", "No user found with that email"); } - const existing = await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, tenantCtx.id), - eq(principal.kind, "user"), - eq(principal.refId, invitedUser.id), - ), + const existing = await findUserPrincipal(db, { + tenantId: tenantCtx.id, + userId: invitedUser.id, }); if (existing) { diff --git a/vendor/intx/hub-api/src/routes/run-view.ts b/vendor/intx/hub-api/src/routes/run-view.ts index a4c1902504..6c6a64f239 100644 --- a/vendor/intx/hub-api/src/routes/run-view.ts +++ b/vendor/intx/hub-api/src/routes/run-view.ts @@ -54,6 +54,8 @@ export function formatRunView( createdAt: ts(record.createdAt), updatedAt: ts(record.updatedAt), endedAt: record.endedAt ? ts(record.endedAt) : null, + failureCode: record.failureCode, + failureMessage: record.failureMessage, ...(runtimeStatus !== undefined ? { runtimeStatus } : {}), }; } diff --git a/vendor/intx/hub-api/src/routes/runs.ts b/vendor/intx/hub-api/src/routes/runs.ts index a793494fb7..b13e9570f0 100644 --- a/vendor/intx/hub-api/src/routes/runs.ts +++ b/vendor/intx/hub-api/src/routes/runs.ts @@ -10,12 +10,12 @@ import { workflowRun, } from "@intx/db/schema"; import type { DB, ApprovalStore, PrincipalKeyStore } from "@intx/db"; -import { authorize } from "@intx/authz"; import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import { extractPartByPath } from "@intx/mime"; import { WorkflowRunResponse, + WorkflowLifecycleResponse, ErrorResponse, WorkflowRunHealth, RunAuthorizationResponse, @@ -23,10 +23,12 @@ import { OfferingDetail, SendMessage, MailResponse, + RecoverWorkflowCapacityRequest, paginatedSchema, } from "@intx/types"; import { createWorkflowRunReader, + type WorkflowLifecycleService, findRoutableById, resolveRunIdForSession, runRowToRoutableRecord, @@ -36,6 +38,7 @@ import { type SessionService, type SidecarRouter, type WorkflowDispatchService, + WORKFLOW_RUN_REF, } from "@intx/hub-sessions"; import { formatOffering } from "./offerings"; import { formatApproval } from "./approvals"; @@ -48,10 +51,10 @@ import { WorkflowRunEventsResponse, formatRunEvent } from "./run-events-view"; import type { TenantEnv } from "../context"; import { errorResponse } from "../error-response"; -import { idResource } from "../middleware/grant"; +import { createAuthorizeGrant, idResource } from "../middleware/grant"; import type { RequireGrant } from "../middleware/grant"; import { loadCommittedRunGrants } from "../run-grant-materialization"; -import { workflowRunRepoId, WORKFLOW_RUN_REF } from "../workflow-run-lifecycle"; +import { workflowRunRepoId } from "../workflow-run-lifecycle"; import { createWorkflowRunTrigger, MAX_MAIL_BODY_BYTES, @@ -66,13 +69,7 @@ import { } from "../pagination"; import { jsonResponse } from "../openapi"; -// Stop and mail history are not yet wired onto a workflow (anchor) run. They -// were built for the retired folded-launch surface and need genuinely new -// backing (INTR-454): stop is a forced early-cancel with no author yet, and a -// run has no durable mail archive. Each such route returns this not-implemented -// signal (kept mounted, not 404, so the admin UI gets a clean answer) until its -// mapping lands. Mail SEND is wired: it routes through the run's workflow-native -// Trigger path. +// Durable workflow mail history is not implemented on this route yet. function workflowRunOperationUnsupported( c: Context, operation: string, @@ -162,6 +159,7 @@ export type CreateRunRoutesDeps = { // Absent when the hub runs without durable dispatch; the trigger then 503s a // provisioned send, exactly as the deployment Trigger route does. workflowDispatchService?: WorkflowDispatchService; + workflowLifecycleService?: WorkflowLifecycleService; grantStore: GrantStore; conditionRegistry: ConditionRegistry; requireGrant: RequireGrant; @@ -176,18 +174,24 @@ export function createRunRoutes({ eventCollectors, repoStore, workflowDispatchService, + workflowLifecycleService, grantStore, conditionRegistry, requireGrant, approvalStore, }: CreateRunRoutesDeps): Hono { const app = new Hono(); + const authorizeGrant = createAuthorizeGrant({ + grantStore, + conditionRegistry, + }); // The run-event log reader is available only when the workflow-run substrate // is; a null reader makes the turns/events routes answer 503, never an empty // log masquerading as real state. const runReader = repoStore !== null ? createWorkflowRunReader(repoStore) : null; + const lifecycleService = workflowLifecycleService ?? null; // The mail-send trigger fires the run through its workflow-native Trigger // path. It needs the run-event substrate (terminal-state read), so a null @@ -395,16 +399,14 @@ export function createRunRoutes({ const principal = c.get("principal"); - const authResult = await authorize( - grantStore, - principal.id, - tenant.id, - `workflow-run:${resolvedRunId}`, - "read", - conditionRegistry, - ); + const allowed = await authorizeGrant({ + principalId: principal.id, + tenantId: tenant.id, + resource: `workflow-run:${resolvedRunId}`, + action: "read", + }); - if (authResult.effect !== "allow") { + if (!allowed) { return errorResponse( c, "forbidden", @@ -632,6 +634,173 @@ export function createRunRoutes({ }, ); + app.get( + "/:runId/lifecycle", + requireGrant(idResource("workflow-run", "runId"), "read"), + describeRoute({ + tags: ["Runs"], + summary: "Get workflow lifecycle status", + description: + "Returns the deployment's saved policy, deadlines, and allocation cleanup status. Policy edits apply to new deployments.", + responses: { + 200: jsonResponse("Lifecycle status", WorkflowLifecycleResponse), + 404: jsonResponse("Run not found", ErrorResponse), + 503: jsonResponse( + "Workflow lifecycle service unavailable", + ErrorResponse, + ), + }, + }), + async (c) => { + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const status = await lifecycleService.getStatus( + c.get("tenant").id, + c.req.param("runId"), + ); + if (status === null) + return errorResponse(c, "not_found", "Run not found"); + return c.json(status); + }, + ); + + app.post( + "/:runId/capacity/release", + requireGrant(idResource("workflow-run", "runId"), "manage"), + describeRoute({ + tags: ["Runs"], + summary: "Release workflow capacity", + description: + "Durably requests release of a terminal top-level run's allocation. Poll the Location header for cleanup status. Stopped cleanup returns 409. After a cleanup disconnect timeout, automatic cleanup resumes only if the sidecar reconnects. If the sidecar has been destroyed, operator recovery is required. Exhausted retries, permanent provider rejections, `provisioner_unavailable`, and `sidecar_cleanup_preparation_timeout` require operator recovery. Reconnects and this request do not restart exhausted cleanup. Once removal is confirmed outside the Hub, `POST .../capacity/recover` releases the allocation. Returns 503 while accepted run history cannot yet be reconciled; retry later.", + responses: { + 202: { description: "Release requested" }, + 204: { description: "Capacity already released or absent" }, + 404: jsonResponse("Run not found", ErrorResponse), + 409: jsonResponse( + "Run is live or automatic cleanup has stopped", + ErrorResponse, + ), + 503: jsonResponse( + "Workflow lifecycle service or run history unavailable", + ErrorResponse, + ), + }, + }), + async (c) => { + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const tenantId = c.get("tenant").id; + const runId = c.req.param("runId"); + const result = await lifecycleService.releaseCapacity(tenantId, runId); + switch (result) { + case "not_found": + return errorResponse(c, "not_found", "Run not found"); + case "live": + return errorResponse( + c, + "conflict", + "Only terminal top-level runs can release their allocation", + ); + case "history_pending": + return errorResponse( + c, + "unavailable", + "The run's accepted history is not reconciled yet; retry later", + ); + case "cleanup_exhausted": + return errorResponse( + c, + "conflict", + "Automatic cleanup retries are exhausted; operator intervention is required", + ); + case "cleanup_failed": + return errorResponse( + c, + "conflict", + "Capacity cleanup failed permanently; operator intervention is required", + ); + case "cleanup_disconnected": + return errorResponse( + c, + "conflict", + "Cleanup timed out waiting for the sidecar; capacity remains reserved. Automatic cleanup resumes only if the sidecar reconnects. If the sidecar has been destroyed, operator recovery is required.", + ); + case "released": + return c.body(null, 204); + case "pending": + c.header( + "Location", + `/api/tenants/${tenantId}/workflows/runs/${runId}/lifecycle`, + ); + return c.body(null, 202); + } + }, + ); + + app.post( + "/:runId/capacity/recover", + requireGrant(idResource("workflow-run", "runId"), "recover"), + describeRoute({ + tags: ["Runs"], + summary: "Release capacity whose cleanup stopped", + description: + "Releases a `destroy_failed` allocation on the caller's word that its deployment copy and provider hold are gone. The Hub does not verify this. Use it only after confirming removal outside the Hub, since a copy that is still running keeps using the sidecar while the Hub places other work there. Send the allocation generation read from the lifecycle status. Requires the `recover` action on the run, which only the owner role holds by default. Repeating a successful request returns 204. The allocation keeps `sidecar_cleanup_operator_released` with the caller and the stopped cleanup's reason as its failure record.", + responses: { + 204: { description: "Capacity released" }, + 404: jsonResponse("Run or allocation not found", ErrorResponse), + 409: jsonResponse( + "Cleanup has not stopped, or the allocation generation changed", + ErrorResponse, + ), + 503: jsonResponse( + "Workflow lifecycle service unavailable", + ErrorResponse, + ), + }, + }), + validator("json", RecoverWorkflowCapacityRequest), + async (c) => { + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const result = await lifecycleService.releaseStoppedCleanup( + c.get("tenant").id, + c.req.param("runId"), + c.req.valid("json").generation, + c.get("principal").id, + ); + switch (result) { + case "not_found": + return errorResponse(c, "not_found", "Run or allocation not found"); + case "not_stopped": + return errorResponse( + c, + "conflict", + "Only an allocation whose automatic cleanup stopped can be recovered", + ); + case "generation_mismatch": + return errorResponse( + c, + "conflict", + "The allocation generation changed; read the lifecycle status again", + ); + case "released": + return c.body(null, 204); + } + }, + ); + app.delete( "/:runId", requireGrant(idResource("workflow-run", "runId"), "manage"), @@ -639,24 +808,40 @@ export function createRunRoutes({ tags: ["Runs"], summary: "Stop a run", description: - "Stops a live workflow run and releases its sidecar allocation.", + "Durably requests cancellation of a live top-level run. Its saved cancellation retention policy controls subsequent capacity release. Poll the Location header for progress.", responses: { - 204: { - description: "Run stopped", - }, + 202: { description: "Cancellation requested" }, 404: jsonResponse("Run not found", ErrorResponse), - 409: jsonResponse("Run already stopped", ErrorResponse), - 502: jsonResponse("Sidecar unavailable", ErrorResponse), + 409: jsonResponse("Run is already terminal", ErrorResponse), + 503: jsonResponse( + "Workflow lifecycle service unavailable", + ErrorResponse, + ), }, }), async (c) => { - // Stopping a run is a forced early-cancel that a workflow (anchor) run - // has no backing for yet -- it needs a hub-side CancelRequested author - // and a lease-free allocation-release seam. The stop is gated to - // not-implemented until that mapping lands (tracked as INTR-454). A - // `markTerminal`-only stop would lie -- it would desync the child and its - // allocation -- so no partial stop is shipped. - return workflowRunOperationUnsupported(c, "Stopping a run"); + if (lifecycleService === null) + return errorResponse( + c, + "unavailable", + "Workflow lifecycle service unavailable", + ); + const tenantId = c.get("tenant").id; + const runId = c.req.param("runId"); + const result = await lifecycleService.requestCancellation( + tenantId, + runId, + `Cancellation requested by ${c.get("principal").id}`, + ); + if (result === "not_found") + return errorResponse(c, "not_found", "Run not found"); + if (result === "terminal") + return errorResponse(c, "conflict", "Run is already terminal"); + c.header( + "Location", + `/api/tenants/${tenantId}/workflows/runs/${runId}/lifecycle`, + ); + return c.body(null, 202); }, ); @@ -691,12 +876,12 @@ export function createRunRoutes({ WorkflowRunTriggerResponse, ), 400: jsonResponse( - "Attachment validation error. Each variant carries a structured code (oversize_attachment, disallowed_mime_type, malformed_base64, oversize_total) with the offending index and limits. A malformed request body that fails SendMessage validation returns the generic error shape instead.", + "Attachment validation error. Each variant carries a structured code (too_many_attachments, oversize_attachment, disallowed_mime_type, invalid_attachment_name, malformed_base64, oversize_total) with the offending index and limits. A malformed request body that fails SendMessage validation returns the generic error shape instead.", ErrorResponse, ), 404: jsonResponse("Run not found", ErrorResponse), 409: jsonResponse( - "Run address is not routable, its allocation is no longer active, or the run is terminal", + "Run address is not routable, its allocation is no longer active, or the run is stopping or terminal", ErrorResponse, ), 413: jsonResponse( diff --git a/vendor/intx/hub-api/src/routes/tenants.ts b/vendor/intx/hub-api/src/routes/tenants.ts index e46ed43304..d905158b35 100644 --- a/vendor/intx/hub-api/src/routes/tenants.ts +++ b/vendor/intx/hub-api/src/routes/tenants.ts @@ -1,25 +1,50 @@ -import { eq, and, sql } from "drizzle-orm"; -import { Hono } from "hono"; +import { type } from "arktype"; +import { eq, sql } from "drizzle-orm"; +import { Hono, type Context, type MiddlewareHandler } from "hono"; import { describeRoute, validator } from "hono-openapi"; +import { getLogger } from "@intx/log"; -import { tenant, principal, role, principalRole, grant } from "@intx/db/schema"; -import { createPrincipalStore, parseTenantRow } from "@intx/db"; +import { tenant, role, principalRole, grant } from "@intx/db/schema"; +import { + createPrincipalStore, + parseTenantRow, + TenantConfigInvalidError, + validateLifecyclePolicyEdit, +} from "@intx/db"; import type { DB, PrincipalKeyStore } from "@intx/db"; import { CreateTenant, ErrorResponse, + TenantConfig, UpdateTenant, TenantResponse, + type TenantConfigPatch, } from "@intx/types"; -import { unauthorizedResponse, type AppEnv } from "../context"; -import { errorResponse } from "../error-response"; +import { unauthorizedResponse, type AppEnv, type TenantEnv } from "../context"; +import { errorResponse, tenantConfigErrorResponse } from "../error-response"; +import { + idResource, + type AuthorizeGrant, + type RequireGrant, +} from "../middleware/grant"; import { first, ts } from "../format"; +import { findUserPrincipal } from "../principal-lookup"; import { jsonResponse } from "../openapi"; import { generateId } from "@intx/hub-common"; +const log = getLogger(["hub", "tenants"]); + const SYSTEM_ROLES = ["owner", "admin", "member"] as const; +function forbiddenUnderParent(c: Context): Response { + return errorResponse( + c, + "forbidden", + "You do not have permission to create a tenant under this parent", + ); +} + function formatTenant(row: typeof tenant.$inferSelect) { const parsed = parseTenantRow(row); return { @@ -34,14 +59,35 @@ function formatTenant(row: typeof tenant.$inferSelect) { }; } +// Separate policies share `config`, so an update must keep the keys it omits. +function mergeTenantConfig( + stored: unknown, + patch: TenantConfigPatch, +): Record { + const merged = new Map( + typeof stored === "object" && stored !== null ? Object.entries(stored) : [], + ); + for (const [key, value] of Object.entries(patch)) { + if (value === null) merged.delete(key); + else merged.set(key, value); + } + return Object.fromEntries(merged); +} + export type CreateTenantRoutesDeps = { db: DB["db"]; principalKeyStore: PrincipalKeyStore; + authorizeGrant: AuthorizeGrant; + resolveTenant: MiddlewareHandler; + requireGrant: RequireGrant; }; export function createTenantRoutes({ db, principalKeyStore, + authorizeGrant, + resolveTenant, + requireGrant, }: CreateTenantRoutesDeps): Hono { const app = new Hono(); const principalStore = createPrincipalStore(db, principalKeyStore); @@ -52,10 +98,16 @@ export function createTenantRoutes({ tags: ["Tenants"], summary: "Create a tenant", description: - "Creates a new tenant. The authenticated user becomes the owner with a principal and default owner role.", + "Creates a new tenant. The authenticated user becomes the owner with a principal and default owner role. A tenant created under `parentId` inherits the parent's credentials, providers, catalogs, assets and OAuth clients, so the caller must be an active member of the parent and hold `create` on `tenant:*` there.", responses: { 201: jsonResponse("Tenant created", TenantResponse), 400: jsonResponse("Validation error", ErrorResponse), + 401: jsonResponse("Authentication required", ErrorResponse), + 403: jsonResponse( + "Not permitted to create a tenant under the parent", + ErrorResponse, + ), + 409: jsonResponse("Slug already taken", ErrorResponse), }, }), validator("json", CreateTenant), @@ -67,6 +119,36 @@ export function createTenantRoutes({ const body = c.req.valid("json"); + // A child inherits the parent's tenant-owned surface, so attaching one + // takes authority in the parent. Every refusal returns the same 403, so + // the route does not reveal whether a tenant id exists or who belongs to + // it. A principal row cannot outlive its tenant, so a missing parent lands + // in the same branch as a non-member. The check stays outside the insert + // transaction because the grant store does not read through it; only a + // lock on the parent row would close the window, and the cost of the race + // is a parent deleted in between failing the insert's foreign key as a 500. + if (body.parentId != null) { + const parentId = body.parentId; + const parentPrincipal = await findUserPrincipal(db, { + tenantId: parentId, + userId: user.id, + }); + if (parentPrincipal?.status !== "active") { + log.info( + "Tenant creation denied for {userId}: no active membership in parent {parentId}", + { userId: user.id, parentId }, + ); + return forbiddenUnderParent(c); + } + const allowed = await authorizeGrant({ + principalId: parentPrincipal.id, + tenantId: parentId, + resource: "tenant:*", + action: "create", + }); + if (!allowed) return forbiddenUnderParent(c); + } + const tenantId = generateId("tenant"); // Derive the domain from a lowercased slug and check the slug conflict // case-insensitively. An inbound mail `From` is lowercased when parsed, so @@ -221,37 +303,20 @@ export function createTenantRoutes({ summary: "Get tenant details", responses: { 200: jsonResponse("Tenant details", TenantResponse), - 403: jsonResponse("Not a member of this tenant", ErrorResponse), + 403: jsonResponse("Not an active member of this tenant", ErrorResponse), 404: jsonResponse("Tenant not found", ErrorResponse), + 409: jsonResponse("Stored tenant config is invalid", ErrorResponse), }, }), + resolveTenant, async (c) => { - const user = c.get("user"); - if (!user) { - return unauthorizedResponse(c); - } - - const tenantId = c.req.param("tenantId"); - - const tenantRow = await db.query.tenant.findFirst({ - where: eq(tenant.id, tenantId), - }); - if (!tenantRow) { - return errorResponse(c, "not_found", "Tenant not found"); + const tenantRow = c.get("tenant"); + try { + return c.json(formatTenant(tenantRow)); + } catch (err) { + if (!(err instanceof TenantConfigInvalidError)) throw err; + return errorResponse(c, "invalid_tenant_config", err.message); } - - const membership = await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, tenantId), - eq(principal.kind, "user"), - eq(principal.refId, user.id), - ), - }); - if (!membership) { - return errorResponse(c, "forbidden", "Not a member of this tenant"); - } - - return c.json(formatTenant(tenantRow)); }, ); @@ -260,48 +325,88 @@ export function createTenantRoutes({ describeRoute({ tags: ["Tenants"], summary: "Update tenant config", - description: "Requires admin or higher grant within the tenant.", + description: + "Merges `config` by top-level key: keys the request omits keep their stored values, and `null` removes a key. Requires admin or higher grant within the tenant.", responses: { 200: jsonResponse("Tenant updated", TenantResponse), + 400: jsonResponse( + "Invalid config or inherited limit exceeded", + ErrorResponse, + ), 403: jsonResponse("Insufficient grants", ErrorResponse), + 404: jsonResponse("Tenant not found", ErrorResponse), + 409: jsonResponse( + "A stored config the tenant inherits is invalid", + ErrorResponse, + ), }, }), + resolveTenant, + requireGrant(idResource("tenant", "tenantId"), "manage"), validator("json", UpdateTenant), async (c) => { - const user = c.get("user"); - if (!user) { - return unauthorizedResponse(c); - } - - const tenantId = c.req.param("tenantId"); + const tenantId = c.get("tenant").id; const body = c.req.valid("json"); - const membership = await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, tenantId), - eq(principal.kind, "user"), - eq(principal.refId, user.id), - ), + const result = await db.transaction(async (tx) => { + // A concurrent update waits here, then merges into the committed result. + const [current] = await tx + .select({ parentId: tenant.parentId, config: tenant.config }) + .from(tenant) + .where(eq(tenant.id, tenantId)) + .for("no key update"); + if (current === undefined) return { kind: "not_found" } as const; + + const updates: Record = { updatedAt: new Date() }; + if (body.name !== undefined) updates["name"] = body.name; + const lifecycle = body.config?.lifecycle; + if (lifecycle != null) { + let resolved; + try { + resolved = await validateLifecyclePolicyEdit( + tx, + current.parentId, + lifecycle, + ); + } catch (err) { + if (!(err instanceof TenantConfigInvalidError)) throw err; + return { kind: "inherited_invalid", error: err } as const; + } + if (!resolved.ok) + return { + kind: "invalid", + message: `${resolved.field} exceeds inherited limit`, + } as const; + } + // A stored key this update leaves in place can predate validation, + // and the response returns the whole config. + const config = TenantConfig( + mergeTenantConfig(current.config, body.config ?? {}), + ); + if (config instanceof type.errors) + return { + kind: "invalid", + message: `Tenant config would be invalid after this update: ${config.summary}`, + } as const; + if (body.config !== undefined) updates["config"] = config; + + const [updated] = await tx + .update(tenant) + .set(updates) + .where(eq(tenant.id, tenantId)) + .returning(); + return updated === undefined + ? ({ kind: "not_found" } as const) + : ({ kind: "updated", row: updated } as const); }); - if (!membership) { - return errorResponse(c, "forbidden", "Not a member of this tenant"); - } - - const updates: Record = { updatedAt: new Date() }; - if (body.name !== undefined) updates["name"] = body.name; - if (body.config !== undefined) updates["config"] = body.config; - - const [updated] = await db - .update(tenant) - .set(updates) - .where(eq(tenant.id, tenantId)) - .returning(); - if (!updated) { + if (result.kind === "not_found") return errorResponse(c, "not_found", "Tenant not found"); - } - - return c.json(formatTenant(updated)); + if (result.kind === "invalid") + return errorResponse(c, "bad_request", result.message); + if (result.kind === "inherited_invalid") + return tenantConfigErrorResponse(c, result.error); + return c.json(formatTenant(result.row)); }, ); diff --git a/vendor/intx/hub-api/src/routes/workflow-definitions.ts b/vendor/intx/hub-api/src/routes/workflow-definitions.ts index 0bc64b8823..5b20d72194 100644 --- a/vendor/intx/hub-api/src/routes/workflow-definitions.ts +++ b/vendor/intx/hub-api/src/routes/workflow-definitions.ts @@ -1,24 +1,33 @@ -import { eq, and } from "drizzle-orm"; +import { eq, and, inArray } from "drizzle-orm"; import { Hono } from "hono"; import { describeRoute, validator } from "hono-openapi"; -import { workflowDefinition, workflowDefinitionVersion } from "@intx/db/schema"; +import { authorizeAction } from "@intx/authz"; +import { + asset, + workflowDefinition, + workflowDefinitionVersion, +} from "@intx/db/schema"; import { parseWorkflowDefinitionRow, parseWorkflowDefinitionVersionRow, createWorkflowDefinitionStore, + TenantConfigInvalidError, + validateLifecyclePolicyEdit, } from "@intx/db"; import type { DB } from "@intx/db"; +import type { ConditionRegistry, GrantStore } from "@intx/types/authz"; import { WorkflowDefinitionVersion, ErrorResponse, WorkflowDefinitionResponse, WorkflowRollbackRequest, + UpdateWorkflowDefinitionLifecycle, paginatedSchema, } from "@intx/types"; import type { TenantEnv } from "../context"; -import { errorResponse } from "../error-response"; +import { errorResponse, tenantConfigErrorResponse } from "../error-response"; import { ts } from "../format"; import { idResource } from "../middleware/grant"; import type { RequireGrant } from "../middleware/grant"; @@ -34,11 +43,15 @@ import { jsonResponse } from "../openapi"; export type CreateWorkflowDefinitionRoutesDeps = { db: DB["db"]; requireGrant: RequireGrant; + grantStore: GrantStore; + conditionRegistry: ConditionRegistry; }; export function createWorkflowDefinitionRoutes({ db, requireGrant, + grantStore, + conditionRegistry, }: CreateWorkflowDefinitionRoutesDeps): Hono { const app = new Hono(); const definitionStore = createWorkflowDefinitionStore(db); @@ -91,6 +104,7 @@ export function createWorkflowDefinitionRoutes({ name: def.name, description: def.description ?? null, currentVersion: def.currentVersion, + lifecycle: def.lifecyclePolicy, status: def.status, createdAt: ts(def.createdAt), updatedAt: ts(def.updatedAt), @@ -176,6 +190,140 @@ export function createWorkflowDefinitionRoutes({ }, ); + app.patch( + "/:definitionId/lifecycle", + requireGrant(idResource("workflow-definition", "definitionId"), "manage"), + describeRoute({ + tags: ["Workflow Definitions"], + summary: "Update workflow lifecycle policy", + description: + "Replaces the installed workflow's lifecycle overrides for future deployments. The overrides cover every revision of the definition's workflow asset, including revisions deployed later, so the caller needs manage on each existing revision. Omitted fields inherit tenant limits.", + responses: { + 200: jsonResponse("Policy updated", WorkflowDefinitionResponse), + 400: jsonResponse( + "Invalid policy or inherited limit exceeded", + ErrorResponse, + ), + 403: jsonResponse( + "Caller cannot manage every revision of the workflow", + ErrorResponse, + ), + 404: jsonResponse("Definition not found", ErrorResponse), + 409: jsonResponse( + "The workflow's revisions changed during the update, or a stored tenant config it inherits is invalid", + ErrorResponse, + ), + }, + }), + validator("json", UpdateWorkflowDefinitionLifecycle), + async (c) => { + const tenantId = c.get("tenant").id; + const definitionId = c.req.param("definitionId"); + const lifecycle = c.req.valid("json").lifecycle; + const definition = await db.query.workflowDefinition.findFirst({ + where: and( + eq(workflowDefinition.id, definitionId), + eq(workflowDefinition.tenantId, tenantId), + ), + }); + if (definition === undefined) + return errorResponse(c, "not_found", "Definition not found"); + // Each revision of a workflow asset is its own definition, so the + // override is written to all of them, and the caller must be able to + // manage each one. + const assetId = definition.assetId; + const revisionFilter = and( + eq(workflowDefinition.tenantId, tenantId), + assetId === null + ? eq(workflowDefinition.id, definitionId) + : eq(workflowDefinition.assetId, assetId), + ); + const revisionIds = ( + await db + .select({ id: workflowDefinition.id }) + .from(workflowDefinition) + .where(revisionFilter) + ).map((row) => row.id); + if (!revisionIds.includes(definitionId)) + return errorResponse(c, "not_found", "Definition not found"); + const principalId = c.get("principal").id; + const grants = await grantStore.collectGrants(principalId, tenantId); + for (const revisionId of revisionIds) { + const decision = await authorizeAction( + grants, + `workflow-definition:${revisionId}`, + "manage", + { registry: conditionRegistry, principalId, tenantId }, + ); + if (!decision.ok) + return errorResponse( + c, + "forbidden", + "You do not have permission to manage every revision of this workflow", + ); + } + let resolved; + try { + resolved = await validateLifecyclePolicyEdit(db, tenantId, lifecycle); + } catch (err) { + if (!(err instanceof TenantConfigInvalidError)) throw err; + return tenantConfigErrorResponse(c, err); + } + if (!resolved.ok) + return errorResponse( + c, + "bad_request", + `${resolved.field} exceeds inherited limit`, + ); + // A revision created later copies the override under the asset lock + // (see ensureWorkflowDefinitionForAsset). One created since the check + // above is not authorized, so the update is refused rather than + // leaving it on the old override. + const updated = await db.transaction(async (tx) => { + if (assetId !== null) + await tx + .select({ id: asset.id }) + .from(asset) + .where(eq(asset.id, assetId)) + .for("update"); + const current = await tx + .select({ id: workflowDefinition.id }) + .from(workflowDefinition) + .where(revisionFilter); + if ( + current.length !== revisionIds.length || + current.some((row) => !revisionIds.includes(row.id)) + ) + return "revisions_changed" as const; + const rows = await tx + .update(workflowDefinition) + .set({ lifecyclePolicy: lifecycle, updatedAt: new Date() }) + .where(inArray(workflowDefinition.id, revisionIds)) + .returning(); + return rows.find((row) => row.id === definitionId); + }); + if (updated === "revisions_changed") + return errorResponse( + c, + "conflict", + "The workflow's revisions changed during the update; retry", + ); + if (updated === undefined) + return errorResponse(c, "not_found", "Definition not found"); + return c.json({ + id: updated.id, + tenantId: updated.tenantId, + name: updated.name, + description: updated.description, + currentVersion: updated.currentVersion, + status: parseWorkflowDefinitionRow(updated).status, + lifecycle, + createdAt: ts(updated.createdAt), + updatedAt: ts(updated.updatedAt), + }); + }, + ); + app.post( "/:definitionId/rollback", requireGrant(idResource("workflow-definition", "definitionId"), "manage"), @@ -216,6 +364,7 @@ export function createWorkflowDefinitionRoutes({ name: def.name, description: def.description ?? null, currentVersion: def.currentVersion, + lifecycle: def.lifecyclePolicy, status: def.status, createdAt: ts(def.createdAt), updatedAt: ts(def.updatedAt), diff --git a/vendor/intx/hub-api/src/routes/workflows.ts b/vendor/intx/hub-api/src/routes/workflows.ts index 7a9878e903..8b6dff2748 100644 --- a/vendor/intx/hub-api/src/routes/workflows.ts +++ b/vendor/intx/hub-api/src/routes/workflows.ts @@ -13,6 +13,7 @@ import { workflowRun, } from "@intx/db/schema"; import { + TenantConfigInvalidError, WorkflowRunDispatchPayloadConflictError, type DB, type PrincipalKeyStore, @@ -36,6 +37,8 @@ import { type WorkflowAllocationService, type WorkflowDispatchService, WorkflowProvisioningError, + WORKFLOW_RUN_REF, + workflowSourceRepoKind, } from "@intx/hub-sessions"; import { generateId } from "@intx/hub-common"; import { @@ -44,8 +47,13 @@ import { } from "@intx/workflow-deploy"; import type { TenantEnv } from "../context"; -import { errorResponse } from "../error-response"; -import { idResource, type RequireGrant } from "../middleware/grant"; +import { errorResponse, tenantConfigErrorResponse } from "../error-response"; +import { + denyUnlessAssetGrant, + idResource, + type AuthorizeGrant, + type RequireGrant, +} from "../middleware/grant"; import { lockDispatchableAllocation, lockWorkflowRunState, @@ -55,7 +63,6 @@ import { WorkflowRunEventsResponse, formatRunEvent } from "./run-events-view"; import { readDurableWorkflowRunLifecycle, workflowRunRepoId, - WORKFLOW_RUN_REF, } from "../workflow-run-lifecycle"; import { createWorkflowRunTrigger, @@ -98,6 +105,9 @@ const DeliverSignal = type({ signalName: "string > 0", signalId: "string > 0", "payload?": "unknown", + // Addresses one onTrigger relay park. Absent, delivery stays + // name-FIFO and does not complete a correlated park. + "correlationId?": "string > 0", }); const WorkflowRunListResponse = type({ @@ -195,6 +205,7 @@ export type CreateWorkflowRoutesDeps = { sidecarRouter: SidecarRouter; repoStore: RepoStore; grantStore: GrantStore; + authorizeGrant: AuthorizeGrant; requireGrant: RequireGrant; }; @@ -206,6 +217,7 @@ export function createWorkflowRoutes({ sidecarRouter, repoStore, grantStore, + authorizeGrant, requireGrant, }: CreateWorkflowRoutesDeps): Hono { const app = new Hono(); @@ -246,9 +258,13 @@ export function createWorkflowRoutes({ "Workflow deployment accepted for provisioning", WorkflowDeploymentResponse, ), - 404: jsonResponse("Workflow asset not found", ErrorResponse), + 403: jsonResponse( + "Caller lacks the read grant on the definition asset", + ErrorResponse, + ), + 404: jsonResponse("Definition asset not found", ErrorResponse), 409: jsonResponse( - "Workflow definition or source offering chain invalid, workflow provisioning unavailable, or provisioner selection failed", + "Asset kind does not match the source's package format, workflow definition, source offering chain, or stored tenant config invalid, workflow provisioning unavailable, or provisioner selection failed", ErrorResponse, ), 500: jsonResponse( @@ -263,11 +279,10 @@ export function createWorkflowRoutes({ const tenant = c.get("tenant"); const body = c.req.valid("json"); - // The deployment anchors its frozen `workflow_definition` to a - // `workflow`-kind asset. An asset-sourced deploy projects the definition - // over the very asset it sources from; a registry-sourced deploy has no - // backing asset for the definition, so this route (which anchors every - // deployment to a workflow asset) does not support it yet. + // The deployment anchors its frozen `workflow_definition` to the asset + // it sources from, whichever kind the package format lives in. A + // registry-sourced deploy has no backing asset for the definition, so + // this route does not support it yet. if (body.source.kind !== "asset") { return errorResponse( c, @@ -277,15 +292,31 @@ export function createWorkflowRoutes({ } const definitionAssetId = body.source.assetId; + const denied = await denyUnlessAssetGrant({ + c, + authorizeGrant, + assetId: definitionAssetId, + }); + if (denied !== null) { + return denied; + } + const assetRow = await db.query.asset.findFirst({ where: and( eq(asset.id, definitionAssetId), eq(asset.tenantId, tenant.id), - eq(asset.kind, "workflow"), ), }); if (!assetRow) { - return errorResponse(c, "not_found", "Workflow asset not found"); + return errorResponse(c, "not_found", "Definition asset not found"); + } + const expectedKind = workflowSourceRepoKind(body.source); + if (assetRow.kind !== expectedKind) { + return errorResponse( + c, + "asset_kind_mismatch", + `A ${body.source.package.format}-format workflow source must name a ${expectedKind} asset; ${assetRow.id} is a ${assetRow.kind} asset`, + ); } if (workflowAllocationService === undefined) { @@ -325,6 +356,9 @@ export function createWorkflowRoutes({ if (err instanceof WorkflowDefinitionInvalidError) { return errorResponse(c, "invalid_workflow", err.message); } + if (err instanceof TenantConfigInvalidError) { + return tenantConfigErrorResponse(c, err); + } if (err instanceof WorkflowProvisioningError) { return c.json( { error: { code: err.code, message: err.message } }, @@ -432,7 +466,7 @@ export function createWorkflowRoutes({ tags: ["Workflows"], summary: "Deliver a signal to a workflow run", description: - "Delivers a caller-supplied, stable signal to the named run of a workflow deployment. The signalId must be supplied by the caller; the run state machine dedups on it.", + "Delivers a caller-supplied, stable signal to the named run of a workflow deployment. The signalId must be supplied by the caller; the run state machine dedups on it. An onTrigger body await is addressed by correlationId from that relay's SignalAwaited together with the author signalName. Omit correlationId to address a name-FIFO await, such as a loop or a top-level gate.", responses: { 202: { description: "Signal accepted for delivery", @@ -600,6 +634,9 @@ export function createWorkflowRoutes({ signalName: body.signalName, signalId: body.signalId, payload: body.payload ?? null, + ...(body.correlationId !== undefined + ? { correlationId: body.correlationId } + : {}), }, }, tx, @@ -639,12 +676,15 @@ export function createWorkflowRoutes({ } try { - sidecarRouter.sendSignalDeliver({ + await sidecarRouter.sendSignalDeliver({ agentAddress, runId: body.runId, signalName: body.signalName, signalId: body.signalId, payload: body.payload, + ...(body.correlationId !== undefined + ? { correlationId: body.correlationId } + : {}), }); } catch (err) { return errorResponse( @@ -674,12 +714,12 @@ export function createWorkflowRoutes({ WorkflowRunTriggerResponse, ), 400: jsonResponse( - "Attachment validation error. Each variant carries a structured code (oversize_attachment, disallowed_mime_type, malformed_base64, oversize_total) with the offending index and limits. A malformed request body that fails SendMessage validation returns the generic error shape instead.", + "Attachment validation error. Each variant carries a structured code (too_many_attachments, oversize_attachment, disallowed_mime_type, invalid_attachment_name, malformed_base64, oversize_total) with the offending index and limits. A malformed request body that fails SendMessage validation returns the generic error shape instead.", ErrorResponse, ), 404: jsonResponse("Workflow deployment not found", ErrorResponse), 409: jsonResponse( - "Deployment address is not routable, its allocation is no longer active, or its top-level run is terminal", + "Deployment address is not routable, its allocation is no longer active, or its top-level run is stopping or terminal", ErrorResponse, ), 413: jsonResponse( diff --git a/vendor/intx/hub-api/src/run-grant-materialization.ts b/vendor/intx/hub-api/src/run-grant-materialization.ts index c1592351e5..eaa444de69 100644 --- a/vendor/intx/hub-api/src/run-grant-materialization.ts +++ b/vendor/intx/hub-api/src/run-grant-materialization.ts @@ -4,10 +4,14 @@ // cannot drift. // // A run's grant set is the definition-pure runtime grants (the capability -// walk's `tool:`/`effect:` rows) plus the resolved declared grant -// requirements (creator- and invoker-sourced). This module stages those -// rows and commits them idempotently on the run id, minting the run -// principal and anchoring the run row in one transaction. +// walk's `tool:`/`effect:` rows), the resolved declared grant requirements +// (creator- and invoker-sourced), and one system-origin `credential:{id}` / +// `use` row per distinct credential and consumer on the anchor's resolved +// bindings. Those binding rows are ownership, not delegation: deploy already +// resolved the id and the consumer. A first commit waits until that +// resolution is recorded. This +// module stages those rows and commits them idempotently on the run id, +// minting the run principal and anchoring the run row in one transaction. // // Delivery (`run.grants` frame, trigger mail / inbound mail forwarding) is // NOT owned here: the two call sites order those differently for their @@ -25,62 +29,42 @@ import { sidecarAllocation, workflowDefinition, workflowRun, + type WorkflowRunCredentialRefs, } from "@intx/db/schema"; import type { DB, DBExecutor, PrincipalKeyStore } from "@intx/db"; import { createPrincipalStore, createWorkflowRunStore, + workflowRunExecutability, loadFrozenGrantSnapshot, } from "@intx/db"; import type { GrantStore, GrantRule } from "@intx/types/authz"; import { isSidecarAllocationDispatchable, - type GrantEffect, type GrantRequirement, type GrantWalkSnapshot, } from "@intx/types"; import { RunGrantsFrame } from "@intx/types/sidecar"; import { ToolDefinition } from "@intx/types/runtime"; import { type MailTriggeredRunGrantsResult } from "@intx/hub-sessions"; -import { deriveRunPrincipalId, generateId } from "@intx/hub-common"; +import { + credentialUseGrantRules, + runtimeGrantRules, +} from "@intx/workflow-deploy/capabilities"; +import { deriveRunPrincipalId } from "@intx/hub-common"; import { + makeGrantRow, resolveGrantMaterialization, type MaterializedGrantRow, } from "./grant-materialization"; -// The `tool:` rows carry BARE tool names: the walk reads inline -// `agent.toolFactories`, which have no bundle context. A workflow child gates -// each tool call on `tool:`, and every runnable step tool is a -// pinned package the loader namespaces to `:`, so the child -// queries `tool::`. These bare rows therefore never address a -// pinned tool's runtime gate; they are inert against a pinned call. A pinned -// tool's authority (including its `ask` mark) is supplied instead by the -// sidecar tool-mark floor (`deriveToolMarkFloorGrants`), derived from the -// loaded factory's already-namespaced definitions. The `effect:` rows are -// different: an action's EffectContext authorizes the bare `effect:` on -// both sides, so those rows ARE name-matched and operative at run time. const TOOL_GRANT_PREFIX = "tool:"; -const EFFECT_GRANT_PREFIX = "effect:"; /** - * Project the frozen grant-walk snapshot into the run's runtime grant rows -- - * the `tool:` and `effect:` grants the runtime enforces fail-closed. - * Every distinct grant string across all steps becomes one creator-origin - * `grant` row with `action: invoke`. The run's runtime authority is - * definition-pure for the deployment's stable top-level run, so the snapshot - * alone determines it. - * - * Tool grants carry the effect the tool's static declaration requested (`ask` - * for approval-gated tools, `allow` otherwise) via each step's `grantEffects` - * record. A tool in more than one step is emitted once; when two steps disagree - * on its effect, `ask` wins over `allow` so an approval-gated declaration is - * never silently downgraded. - * - * Effect grants are always `allow` -- the `effect.requires` set names the - * capability floor an action needs, with no per-effect ask/allow distinction, - * so they are NOT routed through the `grantEffects` record (which covers tool - * grants only). An `effect:` in more than one step is emitted once. + * Project the frozen grant-walk snapshot into the run's runtime grant rows: + * the rules `runtimeGrantRules` derives, stored as the run principal's + * creator-origin grants. */ export function deriveRunRuntimeGrantRows( snapshot: GrantWalkSnapshot, @@ -88,60 +72,135 @@ export function deriveRunRuntimeGrantRows( runPrincipalId: string, now: Date, ): MaterializedGrantRow[] { - const effectByResource = new Map(); - for (const step of snapshot.perStep) { - // The snapshot serializes each step's tool-grant-to-effect map as a plain - // object; rehydrate it to a `Map` so the lookup below matches the walk's - // original access pattern. - const grantEffects = new Map( - Object.entries(step.grantEffects), - ); - for (const grant of step.grants) { - if (grant.startsWith(TOOL_GRANT_PREFIX)) { - // Every `tool:` grant the snapshot emits carries a `grantEffects` - // entry (the tool-mark floor: `ask` for an approval-gated tool, - // `allow` otherwise). A missing entry means the snapshot's `grants` - // and `grantEffects` maps have diverged -- a defaulted `allow` - // here would silently DOWNGRADE an `ask` tool below its floor, - // defeating the approval gate. Fail loudly instead. - const effect = grantEffects.get(grant); - if (effect === undefined) { - throw new Error( - `deriveRunRuntimeGrantRows: tool grant ${JSON.stringify(grant)} has no grantEffects entry; the grant-walk snapshot must carry an effect for every tool grant`, - ); - } - const existing = effectByResource.get(grant); - if (existing === "ask" || effect === "ask") { - effectByResource.set(grant, "ask"); - } else if (existing === undefined) { - effectByResource.set(grant, effect); - } - } else if (grant.startsWith(EFFECT_GRANT_PREFIX)) { - // Effect grants are always allow; a repeat across steps is idempotent. - if (!effectByResource.has(grant)) { - effectByResource.set(grant, "allow"); - } - } - } - } + return runtimeGrantRules(snapshot).map((rule) => + makeGrantRow({ tenantId, principalId: runPrincipalId, ...rule, now }), + ); +} - const rows: MaterializedGrantRow[] = []; - for (const [resource, effect] of effectByResource) { - rows.push({ - id: generateId("grant"), - tenantId, - principalId: runPrincipalId, - resource, - action: "invoke", - effect, - conditions: null, - origin: "creator", - expiresAt: null, - createdAt: now, - updatedAt: now, - }); +/** + * One system-origin `credential:{id}` / `use` row per distinct `(credentialId, + * consumer)` on the anchor's resolved credential bindings. The consumer string + * is copied onto `{ tool }` as deploy stored it. Inference credentials are not + * in this list: they have no binding and no consumer. + */ +export function deriveCredentialUseGrantRows(args: { + bindings: readonly { credentialId: string; consumer: string }[]; + tenantId: string; + runPrincipalId: string; + now: Date; +}): MaterializedGrantRow[] { + return credentialUseGrantRules(args.bindings).map((rule) => + makeGrantRow({ + tenantId: args.tenantId, + principalId: args.runPrincipalId, + ...rule, + now: args.now, + }), + ); +} + +/** + * Credential resolution is unrecorded while initialization has not published + * it. That is an anchor with neither a public key nor credential refs, or an + * anchor whose allocation holds an initialization lease. The lease means a + * deploy has cleared the key and the refs still on the row are the previous + * attempt's. A key with null refs and no lease means initialization finished + * and there was no delivery. + */ +export function credentialResolutionUnrecorded(resolution: { + credentialRefs: WorkflowRunCredentialRefs | null; + publicKey: string | null; + initializationLeaseId: string | null; +}): boolean { + if (resolution.initializationLeaseId !== null) return true; + return resolution.credentialRefs === null && resolution.publicKey === null; +} + +/** + * Lock the anchor and read the credential resolution a first grant commit + * must stamp from. The pre-transaction read races initialization, so the + * commit stages from this row. + */ +export async function lockAnchorCredentialResolution( + tx: DBExecutor, + anchorRunId: string, +): Promise<{ + credentialRefs: WorkflowRunCredentialRefs | null; + publicKey: string | null; + initializationLeaseId: string | null; +} | null> { + // Lock the anchor only. The allocation lease is read afterwards and is not + // locked: beginInitialization takes the allocation lock before the anchor, + // so locking it here would deadlock. The lease and the key are written in + // that same transaction, which cannot commit while this one holds the anchor. + const [run] = await tx + .select({ + credentialRefs: workflowRun.credentialRefs, + publicKey: workflowRun.publicKey, + }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, anchorRunId), + eq(workflowRun.anchorRunId, anchorRunId), + ), + ) + .limit(1) + .for("update"); + if (run === undefined) return null; + const [allocation] = await tx + .select({ + initializationLeaseId: sidecarAllocation.initializationLeaseId, + }) + .from(sidecarAllocation) + .where(eq(sidecarAllocation.anchorRunId, anchorRunId)) + .limit(1); + return { + credentialRefs: run.credentialRefs, + publicKey: run.publicKey, + initializationLeaseId: + allocation === undefined ? null : allocation.initializationLeaseId, + }; +} + +/** + * Replace any pre-staged credential-use rows with the rows implied by the + * locked anchor. Returns `not-ready` when resolution is still unrecorded, + * so the caller inserts no grant rows. + */ +export function grantRowsWithLockedCredentialResolution( + rows: readonly MaterializedGrantRow[], + resolution: { + credentialRefs: WorkflowRunCredentialRefs | null; + publicKey: string | null; + initializationLeaseId: string | null; + }, + args: { + tenantId: string; + runPrincipalId: string; + now: Date; + }, +): MaterializedGrantRow[] | "not-ready" { + if (credentialResolutionUnrecorded(resolution)) { + return "not-ready"; } - return rows; + const base = rows.filter( + (row) => + !( + row.origin === "system" && + row.action === "use" && + row.resource.startsWith("credential:") + ), + ); + return [ + ...base, + ...deriveCredentialUseGrantRows({ + bindings: resolution.credentialRefs?.bindings ?? [], + tenantId: args.tenantId, + runPrincipalId: args.runPrincipalId, + now: args.now, + }), + ]; } /** @@ -183,7 +242,7 @@ export type StageRunGrantsFromSnapshotArgs = { * wire); the external trigger route passes the caller's grants. */ invokerGrants: GrantRule[]; - /** Declared creator grants resolved against the workflow asset's creator. */ + /** Declared creator grants resolved against the definition asset's creator. */ creatorGrants: GrantRule[]; /** * Grant requirements to resolve. The mail path pre-filters the snapshot's @@ -191,6 +250,12 @@ export type StageRunGrantsFromSnapshotArgs = { * passes the snapshot's requirements unfiltered. */ grantRequirements: readonly GrantRequirement[]; + /** + * Resolved credential-binding descriptors from the anchor run. Each distinct + * `(credentialId, consumer)` becomes one system-origin credential-use grant. + * The caller passes an empty array when the anchor has no refs. + */ + credentialBindings: readonly { credentialId: string; consumer: string }[]; }; export type StageRunGrantsResult = @@ -205,13 +270,15 @@ export type StageRunGrantsResult = }; /** - * Stage a run's grant rows from the deploy-approved grant-walk snapshot plus - * the resolved declared requirements. The snapshot's per-step grants project - * the run's runtime `tool:`/`effect:` rows; the mail-triggered materializer and - * the external trigger route both drive this one tail. Returns the staged rows - * and their wire projection, or a rejection when a declared requirement's - * authority is insufficient. No database write happens here -- - * `commitRunGrants` performs it once the caller has accepted delivery. + * Stage a run's grant rows from the deploy-approved grant-walk snapshot, the + * resolved declared requirements, and the anchor's resolved credential + * bindings. The snapshot's per-step grants project the run's runtime + * `tool:`/`effect:` rows. Each distinct credential and consumer becomes one + * system-origin `credential:{id}` / `use` row. The mail-triggered + * materializer and the external trigger route both drive this one tail. + * Returns the staged rows and their wire projection, or a rejection when a + * declared requirement's authority is insufficient. No database write happens + * here -- `commitRunGrants` performs it once the caller has accepted delivery. */ export async function stageRunGrantsFromSnapshot( args: StageRunGrantsFromSnapshotArgs, @@ -236,13 +303,24 @@ export async function stageRunGrantsFromSnapshot( return { ok: false, rejection: materialization.rejection }; } - const grantRows = [...runtimeGrantRows, ...materialization.grantRows]; + const credentialGrantRows = deriveCredentialUseGrantRows({ + bindings: args.credentialBindings, + tenantId: args.tenantId, + runPrincipalId: args.runPrincipalId, + now: args.now, + }); + + const grantRows = [ + ...runtimeGrantRows, + ...materialization.grantRows, + ...credentialGrantRows, + ]; const stepGrants = grantRows.map((g) => runGrantToWire(g)); return { ok: true, grantRows, stepGrants }; } /** - * Load a workflow asset's `creatorPrincipalId` -- the creator whose + * Load a definition asset's `creatorPrincipalId` -- the creator whose * authority creator-sourced grant requirements resolve against. Returns * `null` when the asset records no creator (the FK is `set null` on * principal deletion) or the asset row is absent. @@ -253,11 +331,7 @@ export async function loadAssetCreatorPrincipalId( definitionAssetId: string, ): Promise { const assetRow = await db.query.asset.findFirst({ - where: and( - eq(asset.id, definitionAssetId), - eq(asset.tenantId, tenantId), - eq(asset.kind, "workflow"), - ), + where: and(eq(asset.id, definitionAssetId), eq(asset.tenantId, tenantId)), }); return assetRow?.creatorPrincipalId ?? null; } @@ -304,16 +378,21 @@ export type CommittedRunGrants = { /** * Lock and classify one run row owned by a deployment. A live run -- a * "deployed" anchor in its pre-trigger window or a "running" run -- classifies - * as "running"; the started-vs-not distinction is owned by the durable - * lifecycle, not this status axis. + * as "running" while it accepts work and "stopping" once cancellation is + * requested or its deadline passes; the started-vs-not distinction is owned by + * the durable lifecycle, not this status axis. */ export async function lockWorkflowRunState( tx: DBExecutor, anchorRunId: string, runId: string, -): Promise<"absent" | "running" | "terminal"> { +): Promise<"absent" | "running" | "stopping" | "terminal"> { const [run] = await tx - .select({ status: workflowRun.status }) + .select({ + status: workflowRun.status, + expiresAt: workflowRun.expiresAt, + cancellationRequestedAt: workflowRun.cancellationRequestedAt, + }) .from(workflowRun) .where( and(eq(workflowRun.id, runId), eq(workflowRun.anchorRunId, anchorRunId)), @@ -321,7 +400,21 @@ export async function lockWorkflowRunState( .limit(1) .for("update"); if (run === undefined) return "absent"; - return isLiveWorkflowRunStatus(run.status) ? "running" : "terminal"; + const state = workflowRunExecutability(run); + return state === "executable" ? "running" : state; +} + +function unavailableRunResult( + runId: string, + state: "stopping" | "terminal", +): MailTriggeredRunGrantsResult { + return { + outcome: "rejected", + status: 409, + code: + state === "stopping" ? "workflow_run_stopping" : "workflow_run_terminal", + message: `Workflow run ${runId} is ${state} and cannot receive more mail`, + }; } /** @@ -569,8 +662,15 @@ type FrozenRunGrantBasis = { * `mail.outbound` handler invokes for each workflow-deployment recipient. * * A mail-triggered run derives its grants from the RECEIVING deployment's - * frozen snapshot: the snapshot's `tool:`/`effect:` runtime grants plus the - * CREATOR-resolved declared requirements. Invoker-sourced requirements are + * frozen snapshot and from that deployment's anchor: the snapshot's + * `tool:`/`effect:` runtime grants, the CREATOR-resolved declared + * requirements, and the anchor's resolved credential-binding grants. + * A first commit returns `notReady` and writes nothing while initialization + * has not published credential resolution. That is an anchor with neither a + * public key nor credential refs, or one whose allocation holds an + * initialization lease. Committing during the lease would freeze the + * previous bindings. + * Invoker-sourced requirements are * NOT materialized -- no invoker is on the wire -- and the run still * launches: a step that needs an invoker grant fails closed at its own * authz check. The snapshot's requirements are pre-filtered to @@ -610,7 +710,10 @@ export function createMailTriggeredRunGrantsMaterializer( definitionId: workflowRun.definitionId, definitionAssetId: workflowDefinition.assetId, anchorStatus: workflowRun.status, + anchorExpiresAt: workflowRun.expiresAt, + anchorCancellationRequestedAt: workflowRun.cancellationRequestedAt, topLevelRunStatus: topLevelRun.status, + credentialRefs: workflowRun.credentialRefs, }) .from(workflowRun) .innerJoin( @@ -630,26 +733,26 @@ export function createMailTriggeredRunGrantsMaterializer( // A "deployed" anchor is live: mail-triggering it IS its first trigger, so // it must not be rejected as terminal here. // - // This preflight inspects only the workflow_run.status column and omits the - // durable-lifecycle terminal check that the HTTP trigger route in - // workflow-run-trigger.ts applies. The supervisor's durable run-ref guard + // This preflight inspects the anchor row's status, cancellation, and + // deadline, and omits the durable-lifecycle terminal check that the HTTP + // trigger route in workflow-run-trigger.ts applies. The supervisor's durable run-ref guard // (rejectTerminalRun / readWorkflowRunLifecycle in supervisor.ts) is the // fired/not-fired authority and never re-fires a durably-settled run, so // this status check is a lagging fast-fail only. The two preflights diverge // inside the status-flip lag window; that asymmetry is tolerable and is // tracked for unification in INTR-456. + const anchorState = workflowRunExecutability({ + status: anchor.anchorStatus, + expiresAt: anchor.anchorExpiresAt, + cancellationRequestedAt: anchor.anchorCancellationRequestedAt, + }); + if (anchorState !== "executable") + return unavailableRunResult(runId, anchorState); if ( - !isLiveWorkflowRunStatus(anchor.anchorStatus) || - (anchor.topLevelRunStatus !== null && - !isLiveWorkflowRunStatus(anchor.topLevelRunStatus)) - ) { - return { - outcome: "rejected", - status: 409, - code: "workflow_run_terminal", - message: `Workflow run ${runId} is terminal and cannot receive more mail`, - }; - } + anchor.topLevelRunStatus !== null && + !isLiveWorkflowRunStatus(anchor.topLevelRunStatus) + ) + return unavailableRunResult(runId, "terminal"); if (anchor.definitionAssetId === null) { throw new Error( `mail-triggered run ${runId} for ${agentAddress}: anchor run's definition has no asset`, @@ -724,6 +827,7 @@ export function createMailTriggeredRunGrantsMaterializer( invokerGrants: [], creatorGrants, grantRequirements: creatorRequirements, + credentialBindings: anchor.credentialRefs?.bindings ?? [], }); if (!staged.ok) { return { @@ -734,14 +838,28 @@ export function createMailTriggeredRunGrantsMaterializer( }; } - const stepGrants = await deps.db.transaction(async (tx) => { - if ( - (await lockWorkflowRunState(tx, anchorRunId, anchorRunId)) !== - "running" || - (await lockWorkflowRunState(tx, anchorRunId, runId)) === "terminal" - ) { - return null; + const reserved = await deps.db.transaction(async (tx) => { + const anchorState = await lockWorkflowRunState( + tx, + anchorRunId, + anchorRunId, + ); + if (anchorState !== "running") + return anchorState === "stopping" ? anchorState : "terminal"; + const runState = await lockWorkflowRunState(tx, anchorRunId, runId); + if (runState === "stopping" || runState === "terminal") return runState; + const resolution = await lockAnchorCredentialResolution(tx, anchorRunId); + if (resolution === null) { + throw new Error( + `mail-triggered run ${runId} for ${agentAddress}: anchor run ${anchorRunId} disappeared while locking credential resolution`, + ); } + const grantRows = grantRowsWithLockedCredentialResolution( + staged.grantRows, + resolution, + { tenantId, runPrincipalId, now }, + ); + if (grantRows === "not-ready") return grantRows; return commitRunGrants( { db: deps.db, @@ -752,22 +870,17 @@ export function createMailTriggeredRunGrantsMaterializer( runId, runPrincipalId, now, - grantRows: staged.grantRows, + grantRows, }, tx, ); }); - if (stepGrants === null) { - return { - outcome: "rejected", - status: 409, - code: "workflow_run_terminal", - message: `Workflow run ${runId} is terminal and cannot receive more mail`, - }; - } + if (reserved === "not-ready") return { outcome: "notReady" }; + if (reserved === "stopping" || reserved === "terminal") + return unavailableRunResult(runId, reserved); return { outcome: "materialized", - stepGrants, + stepGrants: reserved, }; }; } diff --git a/vendor/intx/hub-api/src/run-source-resolution.ts b/vendor/intx/hub-api/src/run-source-resolution.ts index 1c6e1bf278..a1e9ab5511 100644 --- a/vendor/intx/hub-api/src/run-source-resolution.ts +++ b/vendor/intx/hub-api/src/run-source-resolution.ts @@ -6,18 +6,14 @@ // credential-bearing source chain a run launches against, so the launch route // and (later) the unified multi-step path cannot drift. -import { - resolveModelSources, - type DB, - type ServingRefresh, -} from "@intx/db"; +import { resolveModelSources, type DB } from "@intx/db"; import type { CredentialCipher, ModelRequirement, ProviderPreference, } from "@intx/types"; import type { InferenceSource } from "@intx/types/runtime"; -import type { CredentialMaterialEntry } from "@intx/types/sidecar"; +import type { CredentialMaterialEntry } from "@intx/types/credential-delivery"; export type DefinitionSourceResolution = | { @@ -25,7 +21,7 @@ export type DefinitionSourceResolution = sources: InferenceSource[]; // The credential material backing the resolved chain, deduped by // `credentialId`. Forwarded to the deploy so it reaches the unified - // credential-material cell alongside any tool material. + // credential-material cell alongside any binding material. materials: CredentialMaterialEntry[]; defaultSource: string; } @@ -53,10 +49,6 @@ export async function resolveDefinitionSources(args: { // real cipher (resolved to a noop only at that edge for a keyless dev/test // composition). credentialCipher: CredentialCipher; - // CL-7505 local delta: optional serving-time refresh hook, threaded to - // buildSource so an expiring `oauth_token` credential refreshes in place - // (or the offering is skipped) before its secret is served. - servingRefresh?: ServingRefresh; }): Promise { const requirements: ModelRequirement[] = args.modelRequirements !== null @@ -70,12 +62,7 @@ export async function resolveDefinitionSources(args: { args.tenantId, requirements, args.credentialCipher, - { - invokerPreferences: args.invokerPreferences, - ...(args.servingRefresh !== undefined - ? { servingRefresh: args.servingRefresh } - : {}), - }, + { invokerPreferences: args.invokerPreferences }, ); if (!resolution.ok) { const message = diff --git a/vendor/intx/hub-api/src/timeline-reconstruction.ts b/vendor/intx/hub-api/src/timeline-reconstruction.ts index 9fd754bcb9..fd82b3166c 100644 --- a/vendor/intx/hub-api/src/timeline-reconstruction.ts +++ b/vendor/intx/hub-api/src/timeline-reconstruction.ts @@ -6,10 +6,8 @@ import { listMail, type MailDirection, } from "@intx/storage-isogit/node"; -import { - formatSafetyRatingText, - type ConversationTurn, -} from "@intx/types/runtime"; +import type { ConversationTurn } from "@intx/types/runtime"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import { ErrorRecord, type ErrorRecord as ErrorRecordType, diff --git a/vendor/intx/hub-api/src/workflow-run-lifecycle.ts b/vendor/intx/hub-api/src/workflow-run-lifecycle.ts index 5ae2741492..cb47dccb9e 100644 --- a/vendor/intx/hub-api/src/workflow-run-lifecycle.ts +++ b/vendor/intx/hub-api/src/workflow-run-lifecycle.ts @@ -1,24 +1,12 @@ import { readCommittedWorkflowRunLifecycle, + workflowRunRepoIdForAddress, + WORKFLOW_RUN_REF, type RepoId, type RepoStore, type WorkflowRunLifecycle, } from "@intx/hub-sessions"; -import { - deriveRunAddress, - deriveWorkflowRunRepoId, -} from "@intx/workflow-deploy"; - -// Workflow-run events commit on the substrate's default branch; the -// supervisor wires the workflow-process child against this ref. -export const WORKFLOW_RUN_REF = "refs/heads/main"; - -export function workflowRunRepoIdForAddress(agentAddress: string): RepoId { - return { - kind: "workflow-run", - id: deriveWorkflowRunRepoId(agentAddress), - }; -} +import { deriveRunAddress } from "@intx/workflow-deploy"; // The workflow-run repo that holds a top-level run's committed event log, // addressed by the run's own id and the tenant's domain. The sidecar's deploy diff --git a/vendor/intx/hub-api/src/workflow-run-trigger.ts b/vendor/intx/hub-api/src/workflow-run-trigger.ts index 6d747fac08..18c3a164ae 100644 --- a/vendor/intx/hub-api/src/workflow-run-trigger.ts +++ b/vendor/intx/hub-api/src/workflow-run-trigger.ts @@ -22,8 +22,12 @@ import { workflowDefinition, workflowRun, } from "@intx/db/schema"; -import type { DB, PrincipalKeyStore } from "@intx/db"; -import { loadFrozenGrantSnapshot, resolveFrameSenderKey } from "@intx/db"; +import type { DB, DBExecutor, PrincipalKeyStore } from "@intx/db"; +import { + loadFrozenGrantSnapshot, + resolveFrameSenderKey, + WorkflowRunNotExecutableError, +} from "@intx/db"; import type { GrantStore } from "@intx/types/authz"; import { assembleSignedContent, @@ -36,6 +40,7 @@ import { deriveWorkflowRunId, isSidecarAllocationDispatchable, SendMessage, + validateAttachments, type AttachmentError, } from "@intx/types"; import type { RunGrantsFrame } from "@intx/types/sidecar"; @@ -52,12 +57,13 @@ import { collectCreatorGrants, commitRunGrants, loadCommittedRunGrants, + grantRowsWithLockedCredentialResolution, + lockAnchorCredentialResolution, lockDispatchableAllocation, lockWorkflowRunState, stageRunGrantsFromSnapshot, } from "./run-grant-materialization"; import type { MaterializedGrantRow } from "./grant-materialization"; -import { validateAttachments } from "./attachment-validation"; import { readDurableWorkflowRunLifecycle } from "./workflow-run-lifecycle"; // DoS guard on the trigger route body. Sized identically to the agent @@ -108,6 +114,25 @@ export type TriggerWorkflowRunResult = } | { ok: false; status: 400 | 403 | 404 | 409 | 503; body: TriggerErrorBody }; +function runUnavailable( + runId: string, + state: "stopping" | "terminal", +): TriggerWorkflowRunResult { + return { + ok: false, + status: 409, + body: { + error: { + code: + state === "stopping" + ? "workflow_run_stopping" + : "workflow_run_terminal", + message: `Workflow run ${runId} is ${state} and cannot receive more mail`, + }, + }, + }; +} + /** * Build the workflow-run trigger over its stable per-app dependencies. The * returned function fires one trigger occurrence per call: it validates the @@ -183,9 +208,9 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { const messageAttachments = attachmentResult.attachments; // Resolve the deployment's anchor run and, through its definition, the - // workflow asset the trigger's grants derive from. The inner join to the - // definition yields the asset id and the definition id in one read, off - // the run rather than the deployment projection. + // definition asset the trigger's grants derive from. The inner join to + // the definition yields the asset id and the definition id in one read, + // off the run rather than the deployment projection. const [anchor] = await db .select({ definitionId: workflowRun.definitionId, @@ -194,6 +219,7 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { allocationStatus: sidecarAllocation.status, anchorStatus: workflowRun.status, runStatus: topLevelRun.status, + credentialRefs: workflowRun.credentialRefs, }) .from(workflowRun) .innerJoin( @@ -246,16 +272,7 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { !isLiveWorkflowRunStatus(anchor.runStatus)) || durableLifecycle === "terminal" ) { - return { - ok: false, - status: 409, - body: { - error: { - code: "workflow_run_terminal", - message: `Workflow run ${runId} is terminal and cannot receive more mail`, - }, - }, - }; + return runUnavailable(runId, "terminal"); } if ( anchor.allocationStatus !== null && @@ -330,7 +347,6 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { where: and( eq(asset.id, definitionAssetId), eq(asset.tenantId, tenant.id), - eq(asset.kind, "workflow"), ), }); if (!assetRow) { @@ -340,7 +356,7 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { body: { error: { code: "invalid_workflow", - message: `Workflow asset ${definitionAssetId} not found`, + message: `Definition asset ${definitionAssetId} not found`, }, }, }; @@ -370,6 +386,7 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { invokerGrants, creatorGrants, grantRequirements: declaredGrantRequirements, + credentialBindings: anchor.credentialRefs?.bindings ?? [], }); if (!staged.ok) { const { status, code, message } = staged.rejection; @@ -419,6 +436,43 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { const rawMessage = assembleMessage(headers, signedContent, signature); const base64 = base64Encode(rawMessage); + // The pre-transaction binding read races initialization. A first commit + // re-reads the anchor under its row lock and stamps from that row, or + // inserts nothing when resolution is still unrecorded. + const commitFirstOrCanonical = async ( + tx: DBExecutor, + ): Promise => { + const grantArgs = { + db, + principalKeyStore, + tenantId: tenant.id, + anchorRunId, + definitionId: anchor.definitionId, + runId, + runPrincipalId, + now, + }; + if (committedRunGrants !== null) { + return commitRunGrants( + { ...grantArgs, grantRows: stagedGrantRows }, + tx, + ); + } + const resolution = await lockAnchorCredentialResolution(tx, anchorRunId); + if (resolution === null) { + throw new Error( + `trigger run ${runId}: anchor run ${anchorRunId} disappeared while locking credential resolution`, + ); + } + const grantRows = grantRowsWithLockedCredentialResolution( + stagedGrantRows, + resolution, + { tenantId: tenant.id, runPrincipalId, now }, + ); + if (grantRows === "not-ready") return grantRows; + return commitRunGrants({ ...grantArgs, grantRows }, tx); + }; + const allocationId = anchor.allocationId; if (allocationId !== null) { if (workflowDispatchService === undefined) { @@ -448,26 +502,15 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { ) { return "run-terminal" as const; } - if ( - (await lockWorkflowRunState(tx, anchorRunId, anchorRunId)) !== - "running" - ) { - return "run-terminal" as const; - } - const canonicalStepGrants = await commitRunGrants( - { - db, - principalKeyStore, - tenantId: tenant.id, - anchorRunId, - definitionId: anchor.definitionId, - runId, - runPrincipalId, - now, - grantRows: stagedGrantRows, - }, + const anchorState = await lockWorkflowRunState( tx, + anchorRunId, + anchorRunId, ); + if (anchorState === "stopping") return "run-stopping" as const; + if (anchorState !== "running") return "run-terminal" as const; + const canonicalStepGrants = await commitFirstOrCanonical(tx); + if (canonicalStepGrants === "not-ready") return "not-ready" as const; await workflowDispatchService.enqueue( { id: `dispatch:${anchorRunId}:${messageId}`, @@ -482,20 +525,31 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { ); return "committed" as const; }); - if (committed !== "committed") { + if (committed === "not-ready") { return { ok: false, status: 409, body: { error: { - code: - committed === "run-terminal" - ? "workflow_run_terminal" - : "deployment_unreachable", + code: "deployment_not_ready", message: - committed === "run-terminal" - ? `Workflow run ${runId} is terminal and cannot receive more mail` - : "Workflow deployment allocation is no longer active", + "Workflow deployment has not recorded credential resolution", + }, + }, + }; + } + if (committed === "run-terminal") + return runUnavailable(runId, "terminal"); + if (committed === "run-stopping") + return runUnavailable(runId, "stopping"); + if (committed !== "committed") { + return { + ok: false, + status: 409, + body: { + error: { + code: "deployment_unreachable", + message: "Workflow deployment allocation is no longer active", }, }, }; @@ -510,38 +564,30 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { } const reserved = await db.transaction(async (tx) => { - if ( - (await lockWorkflowRunState(tx, anchorRunId, anchorRunId)) !== "running" - ) { - return null; - } - return commitRunGrants( - { - db, - principalKeyStore, - tenantId: tenant.id, - anchorRunId, - definitionId: anchor.definitionId, - runId, - runPrincipalId, - now, - grantRows: stagedGrantRows, - }, + const anchorState = await lockWorkflowRunState( tx, + anchorRunId, + anchorRunId, ); + if (anchorState !== "running") + return anchorState === "stopping" ? anchorState : "terminal"; + return commitFirstOrCanonical(tx); }); - if (reserved === null) { + if (reserved === "not-ready") { return { ok: false, status: 409, body: { error: { - code: "workflow_run_terminal", - message: `Workflow run ${runId} is terminal and cannot receive more mail`, + code: "deployment_not_ready", + message: + "Workflow deployment has not recorded credential resolution", }, }, }; } + if (reserved === "stopping" || reserved === "terminal") + return runUnavailable(runId, reserved); stepGrants = reserved; // Stamp the hub-verified principal address (fromAddr, the address the @@ -563,38 +609,25 @@ export function createWorkflowRunTrigger(deps: TriggerWorkflowRunDeps) { ? [{ address: fromAddr, publicKey: authenticatedSenderPublicKey }] : undefined; - // Send the run's grants BEFORE the trigger mail. Both frames route - // through the same per-address channel, so same-websocket FIFO - // ordering guarantees the grants land at the sidecar before the mail - // that dispatches the run -- no ack round-trip is needed. Reserving the - // grants first makes concurrent deliveries converge on this exact - // snapshot. If routing fails, the grants-only run remains eligible for - // its first fire because Git has no RunStarted event yet. - const grantsDelivered = sidecarRouter.sendRunGrants( - address, - runId, - stepGrants, - senderIdentities, - ); - if (!grantsDelivered) { - return { - ok: false, - status: 409, - body: { - error: { - code: "deployment_unreachable", - message: `Deployment address ${address} is not routable`, - }, + // Admit grants and their mail together so a concurrent delivery cannot + // replace the sender's key between the two frames. + let delivered: boolean; + try { + delivered = await sidecarRouter.routeMail( + address, + base64, + fromAddr, + messageId, + { + runId, + stepGrants, + ...(senderIdentities !== undefined ? { senderIdentities } : {}), }, - }; + ); + } catch (error) { + if (!(error instanceof WorkflowRunNotExecutableError)) throw error; + return runUnavailable(runId, error.reason); } - - const delivered = sidecarRouter.routeMail( - address, - base64, - fromAddr, - messageId, - ); if (!delivered) { return { ok: false, diff --git a/vendor/intx/hub-api/tsconfig.json b/vendor/intx/hub-api/tsconfig.json index eea7976730..144a44ac14 100644 --- a/vendor/intx/hub-api/tsconfig.json +++ b/vendor/intx/hub-api/tsconfig.json @@ -18,9 +18,15 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../db/tsconfig.src.json" }, + { + "path": "../hub-common/tsconfig.src.json" + }, { "path": "../hub-sessions/tsconfig.src.json" }, diff --git a/vendor/intx/hub-api/tsconfig.src.json b/vendor/intx/hub-api/tsconfig.src.json index 9a57039b17..aae478d5a9 100644 --- a/vendor/intx/hub-api/tsconfig.src.json +++ b/vendor/intx/hub-api/tsconfig.src.json @@ -22,9 +22,15 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../db/tsconfig.src.json" }, + { + "path": "../hub-common/tsconfig.src.json" + }, { "path": "../hub-sessions/tsconfig.src.json" }, diff --git a/vendor/intx/hub-common/VENDORED-FROM b/vendor/intx/hub-common/VENDORED-FROM index eac425a59b..cd0fc6098f 100644 --- a/vendor/intx/hub-common/VENDORED-FROM +++ b/vendor/intx/hub-common/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-common) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. First vendored in CL-7495: the hub's signing-key series needs generateId's pky_ principal-key prefix (npm 0.3.0 predates it), and @intx/db's principal-key store depends on this package. Only src/index.ts, ids.ts, action-mapping.ts, and glob.ts are consumed; the package depends on @intx/types via workspace: so the vendored types source joins the composite project-reference graph. diff --git a/vendor/intx/hub-common/package.json b/vendor/intx/hub-common/package.json index 0957c6ea0c..3eb39883b8 100644 --- a/vendor/intx/hub-common/package.json +++ b/vendor/intx/hub-common/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-common", "description": "Shared utilities for the Interchange hub packages", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { diff --git a/vendor/intx/hub-common/src/action-mapping.ts b/vendor/intx/hub-common/src/action-mapping.ts index dae9712348..6b1f2a511d 100644 --- a/vendor/intx/hub-common/src/action-mapping.ts +++ b/vendor/intx/hub-common/src/action-mapping.ts @@ -1,4 +1,4 @@ -import type { RepoAction } from "@intx/types/sidecar"; +import type { RepoAction } from "@intx/types/repo"; /** * Translations between the HTTP request shape, the `RepoAction` diff --git a/vendor/intx/hub-common/src/ids.ts b/vendor/intx/hub-common/src/ids.ts index 3e04db7939..cb8b420092 100644 --- a/vendor/intx/hub-common/src/ids.ts +++ b/vendor/intx/hub-common/src/ids.ts @@ -1,4 +1,4 @@ -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; const PREFIXES = { tenant: "tnt_", @@ -28,6 +28,7 @@ const PREFIXES = { signal: "sig_", workflowDefinition: "wfd_", workflowDefinitionVersion: "wdv_", + workflowPendingProjection: "wpp_", } as const; type IDKind = keyof typeof PREFIXES; diff --git a/vendor/intx/hub-sessions/VENDORED-FROM b/vendor/intx/hub-sessions/VENDORED-FROM index 430bf7aa71..374237732b 100644 --- a/vendor/intx/hub-sessions/VENDORED-FROM +++ b/vendor/intx/hub-sessions/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-sessions) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: sidecar-events.ts carries an optional childRunId on the agent.event frame so the hub can close the matching occurrence row. CL-7505: credential-push.ts threads an optional servingRefresh hook into resolveInstanceModelSources, so an expiring oauth_token credential refreshes in place before the push delivers its material. OAuth loopback login: sidecar-handler.ts hosts the loopback login flow (OAuthLoginFinalOutcome/Gate/RequestOutcome, DEFAULT_OAUTH_LOGIN_TIMEOUT_MS; index.ts and ws/index.ts re-export the types). CL-7508: sidecar-handler.ts settles the login caller promise as an error when a terminal outcome arrives before the started acknowledgement, instead of hanging until the whole-login timeout. corbitsdev/workbench#709: session-service.ts fans every top-level prepared-deploy step out through stageWorkflowStep (pure preparedStepStageArgs maps one step address via resolveStepAddress and agent id via deriveStepAgentId); workflow-allocation-service.ts rehydrates both from the frozen launch spec (preparedDeployContent fails closed without a string systemPrompt). Upstream now threads deployContent/toolPackagePins natively; the address/id mapping and the fail-closed narrowing stay local. Retired at earlier pins (prose trimmed at this re-vendor): the terminal-anchor pack acceptance (ownsWorkflowRunRepo), anchorAddressForPackSource, decideTerminalRunFlip, the PersistFrozenApprovalFn wire_projection stamp, the encodeToolName/MALFORMED_TOOL_NAME gate (and the @intx/inference dependency it needed, dropped), the readCommittedWorkflowRunTerminalStatus backfill, and the registerSignalCorrelation kind check. +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: sidecar-events.ts carries an optional childRunId on the agent.event frame so the hub can close the matching occurrence row. corbitsdev/workbench#709: session-service.ts fans every top-level prepared-deploy step out through stageWorkflowStep (pure preparedStepStageArgs maps one step address via resolveStepAddress and agent id via deriveStepAgentId); workflow-allocation-service.ts rehydrates both from the frozen launch spec (preparedDeployContent fails closed without a string systemPrompt). Upstream now threads deployContent/toolPackagePins natively; the address/id mapping and the fail-closed narrowing stay local. Retired at this pin: the CL-7505 servingRefresh hook in credential-push.ts and the CL-7508 OAuth loopback login flow in sidecar-handler.ts (both replaced by @corbits/oauth-core/hub). diff --git a/vendor/intx/hub-sessions/package.json b/vendor/intx/hub-sessions/package.json index af030169e8..1053caae47 100644 --- a/vendor/intx/hub-sessions/package.json +++ b/vendor/intx/hub-sessions/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-sessions", "description": "Session-orchestration substrate for the hub: sidecar router, session, and asset services", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -19,14 +19,14 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.4.0", + "@intx/crypto": "0.5.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.4.0", - "@intx/log": "0.4.0", + "@intx/hub-common": "workspace:*", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.4.0", - "@intx/storage-isogit": "0.4.0", - "@intx/tool-packaging": "0.4.0", + "@intx/pack-transport": "0.5.0", + "@intx/storage-isogit": "0.5.0", + "@intx/tool-packaging": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/vendor/intx/hub-sessions/src/agent-repo.ts b/vendor/intx/hub-sessions/src/agent-repo.ts index b8170fbbab..7f69f540b2 100644 --- a/vendor/intx/hub-sessions/src/agent-repo.ts +++ b/vendor/intx/hub-sessions/src/agent-repo.ts @@ -5,9 +5,9 @@ import type { ToolPackageManifest } from "@intx/types/tool-packages"; import { createRepoStore } from "./repo-store"; import type { AuthorizeFn, + LocalRepoStore, NewlyTerminalRun, RepoId, - RepoStore, } from "./repo-store"; import { agentStateKindHandler, @@ -116,7 +116,7 @@ export type AgentRepoStore = { * skill repos) can share the same on-disk root and signing key * without spinning up a parallel RepoStore. */ - readonly repoStore: RepoStore; + readonly repoStore: LocalRepoStore; }; /** diff --git a/vendor/intx/hub-sessions/src/anchor-run-sweep.ts b/vendor/intx/hub-sessions/src/anchor-run-sweep.ts new file mode 100644 index 0000000000..c856f29f66 --- /dev/null +++ b/vendor/intx/hub-sessions/src/anchor-run-sweep.ts @@ -0,0 +1,72 @@ +export type AnchorRunSweepQueries = { + /** The highest run id this pass visits, or `undefined` when none qualify. */ + findPassEnd(activeRunIds: string[]): Promise; + /** The lowest eligible run after `afterRunId`, up to and including `passEnd`. */ + findNext(range: { + afterRunId: string | undefined; + passEnd: string; + activeRunIds: string[]; + }): Promise; +}; + +/** + * Visit top-level runs in id order, one bounded pass at a time, so a stream of + * new runs cannot postpone revisiting earlier ones. A selected run stays + * excluded from later selections until it is released. + */ +export function createAnchorRunSweep( + queries: AnchorRunSweepQueries, + pause?: { intervalMs: number; now: () => Date }, +) { + const activeRuns = new Set(); + let selectionTail: Promise = Promise.resolve(); + let afterRunId: string | undefined; + let passEnd: string | undefined; + let nextPassAt = 0; + + function finishPass(): void { + afterRunId = undefined; + passEnd = undefined; + if (pause !== undefined) + nextPassAt = pause.now().getTime() + pause.intervalMs; + } + + function select(): Promise { + // Only selection is serialized. Each caller owns its selected run until it + // releases it, so work on different runs proceeds concurrently. + const selecting = selectionTail.then(async () => { + if (pause !== undefined && pause.now().getTime() < nextPassAt) + return null; + const activeRunIds = [...activeRuns]; + if (passEnd === undefined) { + const end = await queries.findPassEnd(activeRunIds); + if (end === undefined) { + finishPass(); + return null; + } + passEnd = end; + } + const candidate = await queries.findNext({ + afterRunId, + passEnd, + activeRunIds, + }); + if (candidate === undefined || candidate.id === passEnd) finishPass(); + else afterRunId = candidate.id; + if (candidate === undefined) return null; + activeRuns.add(candidate.id); + return candidate; + }); + selectionTail = selecting.then( + () => undefined, + () => undefined, + ); + return selecting; + } + + function release(runId: string): void { + activeRuns.delete(runId); + } + + return { select, release }; +} diff --git a/vendor/intx/hub-sessions/src/asset-service.ts b/vendor/intx/hub-sessions/src/asset-service.ts index 83ee588c0e..f26488350e 100644 --- a/vendor/intx/hub-sessions/src/asset-service.ts +++ b/vendor/intx/hub-sessions/src/asset-service.ts @@ -20,7 +20,7 @@ import { type DB, pgErrorCode, PG_UNIQUE_VIOLATION } from "@intx/db"; import { asset as assetTable } from "@intx/db/schema"; import { generateId } from "@intx/hub-common"; import { getLogger } from "@intx/log"; -import type { RepoKind } from "@intx/types/sidecar"; +import type { RepoKind } from "@intx/types/repo"; import type { InitRepoOpts, diff --git a/vendor/intx/hub-sessions/src/credential-push.ts b/vendor/intx/hub-sessions/src/credential-push.ts index d17e21daa0..4cf5659cbb 100644 --- a/vendor/intx/hub-sessions/src/credential-push.ts +++ b/vendor/intx/hub-sessions/src/credential-push.ts @@ -19,11 +19,10 @@ import { resolveInstanceModelSources, getDescendantTenants, reresolveCurrentMaterials, - type ServingRefresh, } from "@intx/db"; import type { DB } from "@intx/db"; import type { CredentialCipher } from "@intx/types"; -import type { CredentialDelivery } from "@intx/types/sidecar"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; import type { SidecarRouter } from "./ws/sidecar-handler"; @@ -52,17 +51,12 @@ export async function pushInstanceSourceUpdate( modelPreferences: unknown; }, credentialCipher: CredentialCipher, - // CL-7505 local delta: optional serving-time refresh hook threaded to - // `resolveInstanceModelSources`, so an expiring `oauth_token` credential - // refreshes in place before this push delivers its material. - servingRefresh?: ServingRefresh, ): Promise { const resolution = await resolveInstanceModelSources( db, instance.tenantId, instance, credentialCipher, - ...(servingRefresh !== undefined ? [{ servingRefresh }] : []), ); if (!resolution.ok) return; const [head] = resolution.sources; @@ -101,7 +95,6 @@ async function pushSourceUpdatesToTenants( sidecarRouter: SidecarRouter, tenantIds: string[], credentialCipher: CredentialCipher, - servingRefresh?: ServingRefresh, ): Promise { if (tenantIds.length === 0) return; @@ -152,7 +145,6 @@ async function pushSourceUpdatesToTenants( modelPreferences: instance.modelPreferences, }, credentialCipher, - servingRefresh, ); }), ); @@ -178,14 +170,12 @@ export async function pushSourceUpdates( sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher, - servingRefresh?: ServingRefresh, ): Promise { await pushSourceUpdatesToTenants( db, sidecarRouter, [tenantId], credentialCipher, - servingRefresh, ); } @@ -200,7 +190,6 @@ export async function pushSourceUpdatesSubtree( sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher, - servingRefresh?: ServingRefresh, ): Promise { let tenants: string[]; try { @@ -214,7 +203,6 @@ export async function pushSourceUpdatesSubtree( sidecarRouter, tenants, credentialCipher, - servingRefresh, ); } diff --git a/vendor/intx/hub-sessions/src/hub-session-lookups.ts b/vendor/intx/hub-sessions/src/hub-session-lookups.ts index c4d0a46859..578542315a 100644 --- a/vendor/intx/hub-sessions/src/hub-session-lookups.ts +++ b/vendor/intx/hub-sessions/src/hub-session-lookups.ts @@ -11,36 +11,36 @@ import type { DB } from "@intx/db"; import { createApprovalStore, createSignalCorrelationStore, - createWorkflowRunDispatchStore, + createWorkflowPendingProjectionStore, createWorkflowRunStore, } from "@intx/db"; import { agentSession, liveWorkflowRunStatuses, - principal, + isLiveWorkflowRunStatus, sessionMail, sidecarAllocation, workflowRun, } from "@intx/db/schema"; import { getLogger } from "@intx/log"; -import { parseRunAddress, signalName } from "@intx/types"; -import { SignalDeliverFrame } from "@intx/types/sidecar"; +import { signalName } from "@intx/types/signals"; +import { deriveWorkflowRunId } from "@intx/types/workflow-run-id"; import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; import type { AgentRepoStore } from "./agent-repo"; import { generateId } from "@intx/hub-common"; import type { SidecarLookups } from "./ws/sidecar-events"; -import { - listAcceptedWorkflowDispatches, - listConsumedWorkflowDispatches, -} from "./workflow-dispatch-settlement"; -import { readCommittedWorkflowRunLifecycle } from "./workflow-run-kind"; +import { createWorkflowDispatchProjection } from "./workflow-dispatch-projection"; +import type { WorkflowHistoryReceiveTracker } from "./workflow-history-receives"; +import { readWorkflowRunRefTips } from "./workflow-run-restore"; +import { projectTerminalRun } from "./workflow-run-terminal-projection"; const logger = getLogger(["hub", "lookups"]); export type HubSessionLookupsDeps = { db: DB["db"]; agentRepoStore: AgentRepoStore; + historyReceives: WorkflowHistoryReceiveTracker; }; export function createHubSessionLookups( @@ -54,12 +54,16 @@ export function createHubSessionLookups( | "resolveSenderKeyStrict" > > { - const { db, agentRepoStore } = deps; + const { db, agentRepoStore, historyReceives } = deps; const signalCorrelationStore = createSignalCorrelationStore(db); const approvalStore = createApprovalStore(db); const workflowRunStore = createWorkflowRunStore(db); - const workflowRunDispatchStore = createWorkflowRunDispatchStore(db); + const pendingProjections = createWorkflowPendingProjectionStore(db); + const dispatchProjection = createWorkflowDispatchProjection({ + db, + repoStore: agentRepoStore.repoStore, + }); return { async lookupDeployRef() { @@ -69,6 +73,45 @@ export function createHubSessionLookups( return null; }, + readWorkflowRunRefTips(agentAddress) { + return readWorkflowRunRefTips(agentRepoStore.repoStore, agentAddress); + }, + + async isWorkflowRunHistoryFinal(source) { + return db.transaction(async (tx) => { + // Match receive admission: the allocation lock waits for any writer + // already advancing Git; a terminal anchor rejects every later receive. + const [allocation] = await tx + .select() + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.id, source.allocationId), + eq(sidecarAllocation.anchorRunId, source.anchorRunId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, source.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, source.generation), + ), + ) + .for("update"); + if (allocation === undefined) return false; + const [anchor] = await tx + .select({ status: workflowRun.status }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, source.anchorRunId), + eq(workflowRun.anchorRunId, source.anchorRunId), + eq(workflowRun.address, source.agentAddress), + ), + ); + if (anchor === undefined || isLiveWorkflowRunStatus(anchor.status)) + return false; + const pending = await pendingProjections.list(source.anchorRunId, tx); + return !pending.some(({ id }) => historyReceives.isInFlight(id)); + }); + }, + async persistMail({ senderAddress, recipients, raw }) { // The sender and recipients are run addresses, each backed by its // self-anchored workflow_run; resolve each through the resolver. A mail @@ -296,7 +339,7 @@ export function createHubSessionLookups( ); } const agentAddress = repoId.id; - const agentId = parseAgentId(agentAddress); + const agentId = deriveWorkflowRunId(agentAddress); try { await agentRepoStore.receiveAgentStatePack( { kind: "agent-state", id: agentId }, @@ -345,6 +388,7 @@ export function createHubSessionLookups( anchorRunId: workflowRun.anchorRunId, tenantId: workflowRun.tenantId, definitionId: workflowRun.definitionId, + createdAt: workflowRun.createdAt, }) .from(workflowRun) .where( @@ -363,10 +407,36 @@ export function createHubSessionLookups( logger.warn`Workflow-run pack rejected for ${workflowRunRepoId}: source address has no live deployment anchor`; return { accepted: false, reason: "path_violation" as const }; } - const anchorAddress = anchor.address; + // The repository must exist before the row does. Recovery reads a + // missing repository as lost history and a ref-less one as a receive + // that never advanced Git, so a crash between the two must leave the + // latter. + try { + await agentRepoStore.repoStore.initRepo(repoId); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error`Workflow-run pack receive failed for ${workflowRunRepoId}: cannot initialize repository: ${msg}`; + return { accepted: false, reason: "corrupt" as const }; + } + // Recorded before Git can advance: Git acceptance and the run-status + // projection below are not atomic, and this row is the only durable + // trace of a projection that fails after the ref moves. It is removed + // here only when the receive provably left Git unchanged or every run + // reached a final decision; otherwise lifecycle recovery reconciles the + // deployment from Git and removes it. + const pendingId = generateId("workflowPendingProjection"); + historyReceives.begin(pendingId); let newlyTerminalRuns; + let reachedGit = false; try { - newlyTerminalRuns = await db.transaction(async (tx) => { + try { + await pendingProjections.open(pendingId, anchor.id); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.error`Workflow-run pack receive failed for ${workflowRunRepoId}: cannot record pending projection: ${msg}`; + return { accepted: false, reason: "corrupt" as const }; + } + const received = await db.transaction(async (tx) => { const [allocation] = await tx .select() .from(sidecarAllocation) @@ -382,24 +452,58 @@ export function createHubSessionLookups( allocation.generation !== source.generation || allocation.ensureAcceptedGeneration !== source.generation ) { - return null; + await pendingProjections.close(pendingId, tx); + return { + rejected: + "source connection does not own the deployment's current allocation", + } as const; + } + // Recheck under the lock: the lifecycle service records a run's + // outcome while holding this row, and history accepted after that + // would contradict the recorded outcome. + const [live] = await tx + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, anchor.id), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + ), + ) + .limit(1); + if (live === undefined) { + await pendingProjections.close(pendingId, tx); + return { + rejected: `workflow run ${anchor.id} is no longer live`, + } as const; } - // Replacement advances this same row. Keep its lock until the + // A release advances this same row. Keep its lock until the // repository ref has advanced so ownership cannot change after // validation but before the old worker's pack becomes authoritative. - return agentRepoStore.receiveWorkflowRunPack( - { kind: "workflow-run", id: workflowRunRepoId }, - pack, - ref, - commitSha, - ); + reachedGit = true; + return { + runs: await agentRepoStore.receiveWorkflowRunPack( + { kind: "workflow-run", id: workflowRunRepoId }, + pack, + ref, + commitSha, + ), + } as const; }); - if (newlyTerminalRuns === null) { - logger.warn`Workflow-run pack rejected for ${workflowRunRepoId}: source connection does not own the deployment's current allocation`; + if ("rejected" in received) { + logger.warn`Workflow-run pack rejected for ${workflowRunRepoId}: ${received.rejected}`; return { accepted: false, reason: "path_violation" as const }; } + newlyTerminalRuns = received.runs; } catch (err) { + // A receive that reached Git keeps its pending row: the ref may have + // advanced before the failure, so only a Git read can prove nothing was + // lost. One that failed earlier provably left Git unchanged. + if (!reachedGit) + await pendingProjections.close(pendingId).catch((cause: unknown) => { + logger.warn`Cannot clear pending projection for ${anchor.id}: ${cause instanceof Error ? cause.message : String(cause)}`; + }); const msg = err instanceof Error ? err.message : String(err); if (msg.startsWith("path_violation")) { logger.warn`Workflow-run pack rejected for ${workflowRunRepoId}: ${msg}`; @@ -413,220 +517,58 @@ export function createHubSessionLookups( // side. logger.error`Workflow-run pack receive failed for ${workflowRunRepoId}: ${msg}`; return { accepted: false, reason: "corrupt" as const }; + } finally { + historyReceives.end(pendingId); } // The substrate has already durably advanced the git ref by the time it // returns, so the pack is accepted regardless of what happens below. The - // per-run status flip and principal deactivation are a best-effort - // downstream side effect of that durable advance, not part of accepting - // the pack. A failure here leaves the run "running" in the DB with its - // principal still active; there is no automatic re-fire, because a - // redelivery of the same durable tip produces no newly-terminal signal - // (the substrate's per-commit walk short-circuits on an already-present - // tip). The failure is therefore logged at ERROR as the only record that - // the row needs a manual flip, and the pack verdict stays accepted so the - // sidecar is acked and does not wedge re-pushing a pack that already - // landed. + // per-run status flip and principal deactivation are a downstream side + // effect of that durable advance, not part of accepting the pack, so the + // verdict stays accepted and the sidecar does not wedge re-pushing a pack + // that already landed. A redelivery of the same durable tip produces no + // newly-terminal signal, so a failed flip is not retried here; the + // pending row stays and lifecycle recovery projects the run from Git. const now = new Date(); - for (const { runId, status } of newlyTerminalRuns) { + let decided = true; + for (const { runId, status, terminalEventJson } of newlyTerminalRuns) { try { - await db.transaction(async (tx) => { - // Lazily anchor the run before settling it. An internal run that - // parks only on a plain signal gate never reaches - // `registerSignalCorrelation`, the sole other path that mints an - // internal run row, so its terminal event can be the first the hub - // sees of the run. A never-minted row is ordinary bookkeeping, not - // a deployment-boundary violation, so mint it here against this - // deployment's anchor rather than letting the ownership guard below - // mistake absence for foreignness. The insert no-ops when any row - // already exists, which keeps that guard authoritative for a row - // that exists and anchors elsewhere. The principal is null: an - // internal run inherits its deployment's grants and has none of its - // own. - // - // The mint necessarily precedes the ownership guard, so an id the - // hub has never seen is claimed under THIS anchor before anything - // establishes it belongs here. That ordering is required -- the - // guard reads the row the mint may have to create -- and it is - // bounded rather than unbounded: internal run ids are supplied by - // the sidecar and accepted verbatim, so the value is - // caller-influenced, but it is a different population from the - // anchor ids the hub mints itself, and nothing resolves an - // internal id without also constraining the anchor or the tenant. - // The insert cannot take a row away from another deployment; the - // worst it does is create one for an id that deployment would - // otherwise have created later. - await workflowRunStore.createIfAbsent( - { - id: runId, - anchorRunId: anchor.id, - definitionId: anchor.definitionId, - tenantId: anchor.tenantId, - principalId: null, - status: "running", - }, - tx, - ); - const [ownedRun] = await tx - .select({ anchorRunId: workflowRun.anchorRunId }) - .from(workflowRun) - .where(eq(workflowRun.id, runId)) - .limit(1); - if (ownedRun?.anchorRunId !== anchor.id) { - logger.error`Ignoring terminal event for run ${runId}: it does not belong to source deployment ${anchor.id}`; - return; - } - const won = await workflowRunStore.markTerminal( + const outcome = await db.transaction((tx) => + projectTerminalRun(tx, workflowRunStore, { + anchor, runId, status, + terminalEvent: JSON.parse(terminalEventJson) as unknown, now, - tx, - ); - if (won === null) { - // The row exists (the mint above guarantees it) and belongs to - // this deployment (the guard above), so no running row matched - // only because the run is already terminal -- a benign replay - // against an already-settled row. Leave its settled status and - // `endedAt` alone. - return; - } - // Deactivate the run's own principal, if it has one. Externally- - // triggered runs carry a principal; internal, workflow-spawned runs - // have `principalId = null` and inherit the deployment's grants, so - // there is nothing to deactivate. Deactivation is gated on winning - // the flip -- the single claim point -- not on the principal's own - // status. - if (won.principalId !== null) { - await tx - .update(principal) - .set({ status: "deactivated", updatedAt: now }) - // The `refId` clause is a defensive mirror of the per-instance - // teardown in instances.ts: `won.principalId` is already this - // run's own principal, and `principal.id` is the primary key, - // so the `refId` match is belt-and-suspenders that the id we - // won belongs to this run. - .where( - and( - eq(principal.id, won.principalId), - eq(principal.refId, runId), - ), - ); - } - }); + }), + ); + if (outcome === "foreign") + logger.error`Ignoring terminal event for run ${runId}: it does not belong to source deployment ${anchor.id}`; } catch (err) { // Per-run isolation: a failed flip for one run must not abort the // rest of the batch, and must not throw out of this method -- a throw // would leave the sidecar with neither an ack nor a reject for a pack - // the substrate already accepted. This ERROR is the only signal that - // the run is stuck "running" in the DB with its principal active, so - // it carries enough to find and flip the row by hand. + // the substrate already accepted. + decided = false; const msg = err instanceof Error ? err.message : String(err); - logger.error`Terminal DB flip failed for run ${runId} (deployment ${anchor.id}, target status ${status}); run left running in the DB: ${msg}`; + logger.error`Terminal DB flip failed for run ${runId} (deployment ${anchor.id}, target status ${status}); its projection stays pending: ${msg}`; } } - - // A sidecar-local mail ack is only receipt. The raw trigger remains in - // workflow_run_dispatch until the accepted Git tip proves either that - // the run recorded it (RunStarted / SignalReceived), or that the - // supervisor consumed it with an explicit rejection. Rescan the bounded - // retained claim-check index after every accepted pack, and search the - // stable run log newest-first only for currently-unsettled ids. Settlement - // is idempotent, and a later pack naturally retries a transient database - // failure here. - let topLevelTerminalSettlementProjected = false; - try { - const reads = await agentRepoStore.repoStore.openCommittedReads( - { kind: "hub" }, - repoId, - ref, - ); - if (reads !== null) { - const unsettledDispatches = - await workflowRunDispatchStore.listUnsettled(anchor.id); - const unsettledByMessageId = new Map( - unsettledDispatches.map((dispatch) => [ - dispatch.messageId, - dispatch, - ]), - ); - for (const consumed of await listConsumedWorkflowDispatches(reads)) { - if (consumed.address !== anchorAddress) continue; - const persisted = unsettledByMessageId.get(consumed.messageId); - if (persisted?.kind !== "mail") continue; - if (consumed.rejection === undefined) { - await workflowRunDispatchStore.settle( - anchor.id, - consumed.messageId, - now, - ); - } else { - await workflowRunDispatchStore.fail({ - anchorRunId: anchor.id, - messageId: consumed.messageId, - code: consumed.rejection.code, - message: consumed.rejection.message, - now, - }); - } - unsettledByMessageId.delete(consumed.messageId); - } - - // Mail is recorded on the stable deployment run, while a signal is - // recorded on the exact run named by its durable delivery frame. - // Group retained dispatches by that Git run before scanning so an - // internal run's SignalReceived evidence settles its own dispatch. - // The `runs//` log keys on the run id (the address local - // part), NOT the full address: post-collapse the top-level run's id - // IS `anchor.id`, so mail keys on `anchor.id`; a signal keys on its - // frame's own run id. (The `addresses/
/` consumed subtree - // above keys on the full address -- a different subtree.) - const messageIdsByRun = new Map>(); - for (const dispatch of unsettledByMessageId.values()) { - const runId = - dispatch.kind === "mail" - ? anchor.id - : SignalDeliverFrame.assert( - JSON.parse(new TextDecoder().decode(dispatch.rawMessage)), - ).runId; - const messageIds = messageIdsByRun.get(runId) ?? new Set(); - messageIds.add(dispatch.messageId); - messageIdsByRun.set(runId, messageIds); - } - for (const [runId, messageIds] of messageIdsByRun) { - for (const accepted of await listAcceptedWorkflowDispatches( - reads, - runId, - messageIds, - )) { - const persisted = unsettledByMessageId.get(accepted.messageId); - if (persisted?.kind !== accepted.kind) continue; - await workflowRunDispatchStore.settle( - anchor.id, - accepted.messageId, - now, - ); - unsettledByMessageId.delete(accepted.messageId); - } - } - topLevelTerminalSettlementProjected = - (await readCommittedWorkflowRunLifecycle(reads, anchor.id)) === - "terminal"; + if (decided) { + try { + await pendingProjections.close(pendingId); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn`Cannot clear pending projection for ${anchor.id}: ${msg}`; } - } catch (error) { - logger.error`Workflow dispatch settlement failed for ${anchor.id}; accepted Git state remains authoritative and the retained payload will be retried: ${error instanceof Error ? error.message : String(error)}`; } - if (topLevelTerminalSettlementProjected) { - try { - await workflowRunDispatchStore.failUnsettled( - anchor.id, - "workflow_run_terminal", - `Workflow run ${anchorAddress} is terminal and cannot accept this dispatch`, - now, - ); - } catch (error) { - logger.error`Failed to close unsettled workflow dispatches for terminal run ${anchorAddress}: ${error instanceof Error ? error.message : String(error)}`; - } + // Git acceptance survives projection failures. The terminal recovery pass + // retries these outcomes even after the allocation has been released. + try { + await dispatchProjection.project(anchor.id); + } catch (error) { + logger.error`Workflow dispatch settlement failed for ${anchor.id}: ${error instanceof Error ? error.message : String(error)}`; } return { accepted: true }; @@ -634,20 +576,6 @@ export function createHubSessionLookups( }; } -/** - * Extract the run id from an `@` run address. - * Throws on any input the `@intx/types`-owned `parseRunAddress` - * rejects: missing or leading `@`, empty domain, or a run id - * without the canonical `run_` prefix. - */ -export function parseAgentId(agentAddress: string): string { - const parsed = parseRunAddress(agentAddress); - if (parsed === null) { - throw new Error(`Invalid run address: "${agentAddress}"`); - } - return parsed.runId; -} - /** * A live routing endpoint backing a run address. Each address names one * self-anchored `workflow_run`, so the endpoint is always that run. @@ -809,6 +737,9 @@ export interface RoutableRecord { readonly principalId: string | null; readonly kernelId: string | null; readonly sidecarId: string | null; + /** Why the Hub failed the deployment when it lost its capacity. */ + readonly failureCode: string | null; + readonly failureMessage: string | null; } /** @@ -830,9 +761,17 @@ export function runRowToRoutableRecord( principalId: string | null; kernelId: string | null; sidecarId: string | null; + failureCode: string | null; + failureMessage: string | null; + infrastructureFailedAt: Date | null; }, address: string, ): RoutableRecord { + // A failure the Hub decided waits on history still being reconciled, and is + // dropped if that history ends the run first, so only a failed run whose + // failure is no longer waiting reports it. + const reportsFailure = + run.status === "failed" && run.infrastructureFailedAt === null; return { id: run.id, tenantId: run.tenantId, @@ -846,6 +785,8 @@ export function runRowToRoutableRecord( principalId: run.principalId, kernelId: run.kernelId, sidecarId: run.sidecarId, + failureCode: reportsFailure ? run.failureCode : null, + failureMessage: reportsFailure ? run.failureMessage : null, }; } @@ -888,6 +829,9 @@ export async function findRoutableById( kernelId: workflowRun.kernelId, sidecarId: workflowRun.sidecarId, definitionId: workflowRun.definitionId, + failureCode: workflowRun.failureCode, + failureMessage: workflowRun.failureMessage, + infrastructureFailedAt: workflowRun.infrastructureFailedAt, }) .from(workflowRun) .where(and(eq(workflowRun.id, id), eq(workflowRun.tenantId, tenantId))) diff --git a/vendor/intx/hub-sessions/src/hub-session-orchestrator.ts b/vendor/intx/hub-sessions/src/hub-session-orchestrator.ts index c4191a7378..333dfa28cc 100644 --- a/vendor/intx/hub-sessions/src/hub-session-orchestrator.ts +++ b/vendor/intx/hub-sessions/src/hub-session-orchestrator.ts @@ -81,8 +81,8 @@ export function createHubSessionOrchestrator( unsubscribers.push( events.on("mail.outbound.undelivered", ({ recipients }) => { // The hub has no external mail transport today. Anything that - // could not be delivered locally or queued for a disconnected - // agent is dropped; log so operators can see it. + // could not be delivered locally is dropped; log so operators can + // see it. log.warn("Dropping mail with no local recipient: {recipients}", { recipients: recipients.join(", "), }); diff --git a/vendor/intx/hub-sessions/src/index.ts b/vendor/intx/hub-sessions/src/index.ts index 7944bcd64b..8b9c9e0ade 100644 --- a/vendor/intx/hub-sessions/src/index.ts +++ b/vendor/intx/hub-sessions/src/index.ts @@ -1,3 +1,16 @@ +export { + createWorkflowLifecycleService, + type WorkflowLifecycleService, + type WorkflowLifecycleServiceDeps, +} from "./workflow-lifecycle-service"; +export { + createWorkflowHistoryReceiveTracker, + type WorkflowHistoryReceiveTracker, +} from "./workflow-history-receives"; +export { + createWorkflowDispatchProjection, + DEFAULT_WORKFLOW_PROJECTION_CONCURRENCY, +} from "./workflow-dispatch-projection"; export { createAgentRepoStore, type AgentRepoStore, @@ -37,16 +50,18 @@ export { type SidecarRouter, type SidecarRouterConfig, type SenderDeploySettledOutcome, - type OAuthLoginFinalOutcome, - type OAuthLoginGateOutcome, - type OAuthLoginRequestOutcome, - DEFAULT_OAUTH_LOGIN_TIMEOUT_MS, type AllocatedSenderDeployAttempt, type SidecarAuthIdentity, type SidecarAuthenticator, type AllocatedSidecarTarget, type SidecarAllocationRouter, SidecarIdentityValidationError, + WorkflowControlHistoryPendingError, + WorkflowControlInitializingError, + WorkflowControlRejectedError, + WorkflowControlTimeoutError, + WorkflowControlUnconfirmedError, + WorkflowControlUnreachableError, createSidecarCredentialResolver, createSidecarTokenAuthenticator, type CreateSidecarTokenAuthenticatorDeps, @@ -65,7 +80,6 @@ export { export { createHubSessionLookups, findRoutableById, - parseAgentId, resolveRoutableAddress, resolveRunIdForSession, resolveRunSessionId, @@ -87,6 +101,8 @@ export { pushCredentialReconcile, } from "./credential-push"; export { + adapterManifestDigest, + attachSidecarProvisioners, chooseFirstSidecarProvisioner, createSidecarPluginRegistry, createSidecarAllocationReconciler, @@ -97,16 +113,27 @@ export { type EnsureSidecarResult, type SidecarCredentialIdentity, type SidecarCredentialResolver, + type SidecarCredentials, + type SidecarIdentityUse, type SidecarOperationFailure, type SidecarPluginRegistry, type SidecarProvisioner, type SidecarProvisionerChooser, + type SidecarProvisionerHost, + type SidecarStoppedOptions, + type SidecarStoppedResult, type SidecarAllocationReconciler, type SidecarAllocationReconcilerDeps, type SidecarReconciliationContext, } from "./sidecar-allocation"; +export { + createParkedRunClassifier, + type ParkedRunClassification, + type ParkedRunClassifier, + type ParkedRunVerdict, +} from "./parked-run"; export { ensureWorkflowDefinitionForAsset } from "./workflow-definition-ensure"; -export { workflowSourceAssetMountPath } from "./workflow-closure-resolution"; +export { workflowSourceRepoKind } from "./workflow-source-repo-kind"; export { createReconciliationScheduler, DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, @@ -158,7 +185,6 @@ export { export { workflowKindHandler, workflowAuthorize, - workflowDefinitionEnvelopeSchema, WORKFLOW_JSON_PATH, CAPABILITY_DECLARATIONS_JSON_PATH, type WorkflowPrincipal, @@ -177,6 +203,8 @@ export { scanRunsForBoot, readCommittedWorkflowRunLifecycle, readWorkflowRunLifecycle, + workflowRunRepoIdForAddress, + WORKFLOW_RUN_REF, replayProcessingToInbox, WORKFLOW_RUN_GITIGNORE_PATH, WORKFLOW_RUN_RUNS_PREFIX, @@ -210,6 +238,7 @@ export { type WorkflowRunSupervisorPrincipal, } from "./workflow-run-kind"; export { + readWorkflowRunRefTips, restoreWorkflowRunToAllocation, WORKFLOW_RUN_RESTORE_REFS, } from "./workflow-run-restore"; diff --git a/vendor/intx/hub-sessions/src/parked-run.ts b/vendor/intx/hub-sessions/src/parked-run.ts new file mode 100644 index 0000000000..f3193de996 --- /dev/null +++ b/vendor/intx/hub-sessions/src/parked-run.ts @@ -0,0 +1,394 @@ +// Whether a lost worker's committed log is a park the runtime would +// re-attach. The answer is the resume guard (`resumeResidualOf`) plus the +// same loop and onTrigger planners the runtime drives. This module does +// not list park shapes of its own. + +import { + loopBodyRunId, + loopIterationCursor, + loopOccurrenceReattach, + onTriggerOccurrenceReattach, + resumeFromLog, + resumeResidualOf, + type BodyFailurePolicy, + type OccurrenceReattach, + type StepKindLookup, + type WorkflowEvent, +} from "@intx/workflow"; + +import type { + WorkflowRunEvent, + WorkflowRunReader, +} from "./workflow-run-reader"; +import { + WORKFLOW_RUN_REF, + workflowRunRepoIdForAddress, +} from "./workflow-run-kind"; + +export type ParkedRunVerdict = "parked" | "not-parked" | "unknown"; + +export type ParkedRunClassification = { + readonly verdict: ParkedRunVerdict; + readonly reason: string; +}; + +export type ParkedRunClassifier = (allocation: { + readonly anchorRunId: string; +}) => Promise; + +export function createParkedRunClassifier(deps: { + readonly runReader: WorkflowRunReader; + readonly addressForAnchor: (anchorRunId: string) => Promise; + readonly projectionForAnchor: (anchorRunId: string) => Promise; +}): ParkedRunClassifier { + return async (allocation) => { + try { + return await classifyAnchor(deps, allocation.anchorRunId); + } catch (cause) { + return { + verdict: "unknown", + reason: cause instanceof Error ? cause.message : String(cause), + }; + } + }; +} + +async function classifyAnchor( + deps: { + readonly runReader: WorkflowRunReader; + readonly addressForAnchor: (anchorRunId: string) => Promise; + readonly projectionForAnchor: (anchorRunId: string) => Promise; + }, + anchorRunId: string, +): Promise { + const address = await deps.addressForAnchor(anchorRunId); + if (address === null) { + return { verdict: "unknown", reason: "anchor run has no address" }; + } + const repoId = workflowRunRepoIdForAddress(address); + if (!(await deps.runReader.hasRepository(repoId))) { + return { verdict: "unknown", reason: "workflow run repository is missing" }; + } + const projection = await deps.projectionForAnchor(anchorRunId); + const steps = projectionSteps(projection); + if (steps === undefined) { + return { verdict: "unknown", reason: "anchor has no frozen projection" }; + } + const events = await deps.runReader.readRunEvents( + repoId, + WORKFLOW_RUN_REF, + anchorRunId, + ); + if (events.length === 0) { + return { verdict: "not-parked", reason: "committed log is empty" }; + } + const state = resumeFromLog(anchorRunId, events.map(toWorkflowEvent)); + if (state.phase !== "running") { + return { verdict: "not-parked", reason: `anchor phase is ${state.phase}` }; + } + const lookup = kindLookup(steps); + let hasPark = false; + for (const [stepId, step] of state.steps) { + const residual = resumeResidualOf(lookup, stepId, step.phase); + if (residual === "unsupported") { + return { + verdict: "not-parked", + reason: `step ${stepId} is ${step.phase}`, + }; + } + const kind = stepKind(steps[stepId]); + if ( + residual === "continue" && + step.phase === "awaiting-signal" && + kind !== "loop" && + kind !== "onTrigger" + ) { + hasPark = true; + } + } + for (const [stepId, step] of state.steps) { + if (step.phase !== "in-flight" && step.phase !== "awaiting-signal") { + continue; + } + const raw = steps[stepId]; + const kind = stepKind(raw); + if (kind === "loop") { + const judged = await judgeLoop({ + deps, + repoId, + anchorRunId, + stepId, + raw, + state, + log: events.map(toWorkflowEvent), + }); + if (judged.verdict !== undefined) return judged.classification; + if (judged.park) hasPark = true; + } else if (kind === "onTrigger") { + const judged = await judgeOnTrigger({ + deps, + repoId, + stepId, + raw, + state, + log: events.map(toWorkflowEvent), + }); + if (judged.verdict !== undefined) return judged.classification; + if (judged.park) hasPark = true; + } + } + return hasPark + ? { verdict: "parked", reason: "resume guard would re-attach a park" } + : { + verdict: "not-parked", + reason: "resume guard found no re-attachable park", + }; +} + +type Judge = + | { + readonly verdict: "stop"; + readonly classification: ParkedRunClassification; + readonly park?: false; + } + | { readonly verdict?: undefined; readonly park: boolean }; + +async function judgeLoop(args: { + deps: { + readonly runReader: WorkflowRunReader; + }; + repoId: ReturnType; + anchorRunId: string; + stepId: string; + raw: unknown; + state: ReturnType; + log: readonly WorkflowEvent[]; +}): Promise { + const maxIterations = maxIterationsOf(args.raw); + if (maxIterations === undefined) { + return stop("unknown", `loop ${args.stepId} has no maxIterations`); + } + const recorded = failurePolicyOf(args.raw, "onIterationFailure"); + if (recorded === "invalid") { + return stop( + "unknown", + `loop ${args.stepId} has an invalid onIterationFailure`, + ); + } + // Absent is the same default `loopIterationCursor` fills in at the read. + const policy = recorded === "tolerate" ? "tolerate" : "end"; + let frontier: number | undefined; + for (let iteration = 0; iteration < maxIterations; iteration += 1) { + const cursor = loopIterationCursor( + args.state, + args.anchorRunId, + args.stepId, + iteration, + policy, + ); + if (cursor.kind === "completed" || cursor.kind === "tolerated") continue; + if (cursor.kind === "frontier") frontier = iteration; + break; + } + if (frontier === undefined) return { park: false }; + const childRunId = loopBodyRunId(args.anchorRunId, args.stepId, frontier); + const childState = resumeFromLog( + childRunId, + await readChild(args.deps.runReader, args.repoId, childRunId), + ); + const reattach = await loopOccurrenceReattach({ + loopId: args.stepId, + runId: args.anchorRunId, + iteration: frontier, + state: args.state, + log: args.log, + childState, + }); + return finishReattach( + reattach, + childState, + loopBodyLookup(args.raw), + args.stepId, + ); +} + +async function judgeOnTrigger(args: { + deps: { readonly runReader: WorkflowRunReader }; + repoId: ReturnType; + stepId: string; + raw: unknown; + state: ReturnType; + log: readonly WorkflowEvent[]; +}): Promise { + const policy = failurePolicyOf(args.raw, "onBodyFailure"); + if (policy === "invalid") { + return stop( + "unknown", + `onTrigger ${args.stepId} has an invalid onBodyFailure`, + ); + } + const childState = resumeFromLog(args.stepId, []); + const reattach = await onTriggerOccurrenceReattach({ + sectionId: args.stepId, + ...(policy === "end" || policy === "tolerate" + ? { onBodyFailure: policy } + : {}), + state: args.state, + log: args.log, + childState, + }); + // The planner reads the child only when the container is not already + // parked. Load that child when the decision names one, and re-judge so + // the unrelayed-body throw sees the committed child log. + if (reattach.kind !== "parked" && reattach.kind !== "reenter") { + return { park: false }; + } + if (reattach.childRunId === undefined) + return { park: reattach.kind === "parked" }; + const loaded = resumeFromLog( + reattach.childRunId, + await readChild(args.deps.runReader, args.repoId, reattach.childRunId), + ); + const again = await onTriggerOccurrenceReattach({ + sectionId: args.stepId, + ...(policy === "end" || policy === "tolerate" + ? { onBodyFailure: policy } + : {}), + state: args.state, + log: args.log, + childState: loaded, + }); + return finishReattach( + again, + loaded, + onTriggerBodyLookup(args.raw), + args.stepId, + ); +} + +function finishReattach( + reattach: OccurrenceReattach, + childState: ReturnType, + body: StepKindLookup | "ref" | undefined, + stepId: string, +): Judge { + if (reattach.kind === "rejected" || reattach.kind === "open") { + return { park: false }; + } + if (reattach.kind === "parked" && reattach.childRunId === undefined) { + return { park: true }; + } + if (body === "ref" || body === undefined) { + return stop("unknown", `step ${stepId} body is not inline`); + } + for (const [id, step] of childState.steps) { + if (resumeResidualOf(body, id, step.phase) === "unsupported") { + return stop("not-parked", `child step ${id} is ${step.phase}`); + } + } + return { park: reattach.kind === "parked" }; +} + +function stop(verdict: ParkedRunVerdict, reason: string): Judge { + return { verdict: "stop", classification: { verdict, reason } }; +} + +async function readChild( + runReader: WorkflowRunReader, + repoId: ReturnType, + childRunId: string, +): Promise { + const events = await runReader.readRunEvents( + repoId, + WORKFLOW_RUN_REF, + childRunId, + ); + return events.map(toWorkflowEvent); +} + +function toWorkflowEvent(event: WorkflowRunEvent): WorkflowEvent { + const rest: Record = {}; + for (const [key, value] of Object.entries(event.body)) { + if (key === "type" || key === "seq") continue; + rest[key] = value; + } + const built: Record = { + ...rest, + kind: event.type, + seq: event.seq, + }; + // The reader has already required a string `type`. `resumeFromLog` + // narrows on `kind`; there is no second validator at this boundary. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- kind is the on-disk type; the state machine narrows the union + return built as unknown as WorkflowEvent; +} + +function projectionSteps( + projection: unknown, +): Readonly> | undefined { + if (!isRecord(projection)) return undefined; + const steps = projection["steps"]; + if (!isRecord(steps)) return undefined; + return steps; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stepKind(step: unknown): string | undefined { + if (!isRecord(step)) return undefined; + const kind = step["kind"]; + return typeof kind === "string" ? kind : undefined; +} + +function kindLookup(steps: Readonly>): StepKindLookup { + const out: Record = {}; + for (const [id, step] of Object.entries(steps)) { + const kind = stepKind(step); + if (kind !== undefined) out[id] = { kind }; + } + return { steps: out }; +} + +function maxIterationsOf(step: unknown): number | undefined { + if (!isRecord(step)) return undefined; + const value = step["maxIterations"]; + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) { + return undefined; + } + return value; +} + +function failurePolicyOf( + step: unknown, + field: "onIterationFailure" | "onBodyFailure", +): BodyFailurePolicy | "absent" | "invalid" { + if (!isRecord(step) || !Object.prototype.hasOwnProperty.call(step, field)) { + return "absent"; + } + const value = step[field]; + if (value === "end" || value === "tolerate") return value; + return "invalid"; +} + +function nestedSteps(value: unknown): StepKindLookup | undefined { + if (!isRecord(value)) return undefined; + const steps = value["steps"]; + if (!isRecord(steps)) return undefined; + return kindLookup(steps); +} + +function loopBodyLookup(step: unknown): StepKindLookup | "ref" | undefined { + if (!isRecord(step)) return undefined; + return nestedSteps(step["body"]); +} + +function onTriggerBodyLookup( + step: unknown, +): StepKindLookup | "ref" | undefined { + if (!isRecord(step) || !isRecord(step["body"])) return undefined; + const body = step["body"]; + if (typeof body["ref"] === "string") return "ref"; + return nestedSteps(body["inline"]); +} diff --git a/vendor/intx/hub-sessions/src/repo-store/index.ts b/vendor/intx/hub-sessions/src/repo-store/index.ts index de047b20e6..3b82e57242 100644 --- a/vendor/intx/hub-sessions/src/repo-store/index.ts +++ b/vendor/intx/hub-sessions/src/repo-store/index.ts @@ -23,7 +23,11 @@ export { authorizeUserPrincipal, type AuthorizeUserPrincipalArgs, } from "./user-principal-gate"; -export { createRepoStore, type CreateRepoStoreConfig } from "./store"; +export { + createRepoStore, + type CreateRepoStoreConfig, + type LocalRepoStore, +} from "./store"; export { subscribeKind, type SubscribeKindOpts, diff --git a/vendor/intx/hub-sessions/src/repo-store/store.ts b/vendor/intx/hub-sessions/src/repo-store/store.ts index 2c04610642..de29a45c43 100644 --- a/vendor/intx/hub-sessions/src/repo-store/store.ts +++ b/vendor/intx/hub-sessions/src/repo-store/store.ts @@ -8,10 +8,12 @@ import { receivePackObjects, collectReachableObjects, maybeGC, + syncDirectoryDurable, + syncRemovedPathDurable, type CommitSigner, type GCPolicy, } from "@intx/storage-isogit/node"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; import { getLogger } from "@intx/log"; import type { AuthorizeFn, @@ -126,7 +128,22 @@ export type CreateRepoStoreConfig = { gc?: GCPolicy & { kinds: readonly RepoKind[] }; }; -export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { +/** + * A `RepoStore` that owns its repositories on local disk and can remove one. + */ +export type LocalRepoStore = RepoStore & { + /** + * Delete a repository and forget everything the store keeps in memory about + * it, under its lock, so a repository created again at the same id starts + * from nothing. A missing repository is not an error. `last` names an entry + * of the repository's directory removed only after every other one, so a + * crash partway through leaves it in place. Removal is durable before + * returning; sync failures propagate. + */ + removeRepo(repoId: RepoId, opts?: { readonly last?: string }): Promise; +}; + +export function createRepoStore(config: CreateRepoStoreConfig): LocalRepoStore { const { dataDir, signingKey, handlers, authorize, signingCallback, gc } = config; @@ -207,8 +224,9 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return `${repoId.kind}/${repoId.id}`; } - // Per-commit reachable-object cache. createPack for a workflow-run - // ref walks the first-parent chain and unions every commit's + // Per-commit reachable-object cache, keyed by repository and commit so + // removing a repository can drop its entries. createPack for a + // workflow-run ref walks the first-parent chain and unions every commit's // reachable objects so the receiver gets the full history needed to // validate per-commit transitions. Without this cache the walk // recomputes the reachable set for every ancestor on every push, so @@ -1836,7 +1854,12 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { const commitSha = await git.resolveRef({ fs, dir, ref }); const tipKey = lastPackedTipKey(repoId, ref); const stopAt = lastPackedTip.get(tipKey) ?? null; - const oids = await collectChainReachableObjects(dir, commitSha, stopAt); + const oids = await collectChainReachableObjects( + repoId, + dir, + commitSha, + stopAt, + ); const result = await git.packObjects({ fs, dir, @@ -1889,6 +1912,7 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { // store — that commit is by definition not part of the local // supervisor's history and so cannot be a relevant ancestor. async function collectChainReachableObjects( + repoId: RepoId, dir: string, tipSha: string, stopAt: string | null, @@ -1897,10 +1921,11 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { let current: string | null = tipSha; while (current !== null) { if (current === stopAt) break; - let perCommit = chainReachabilityCache.get(current); + const cacheKey = `${indexCacheKey(repoId)}/${current}`; + let perCommit = chainReachabilityCache.get(cacheKey); if (perCommit === undefined) { perCommit = await collectReachableObjects(dir, current); - chainReachabilityCache.set(current, perCommit); + chainReachabilityCache.set(cacheKey, perCommit); } for (const o of perCommit) seen.add(o); let parsed: Awaited>; @@ -2167,6 +2192,59 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { return iterator; } + async function removeRepo( + repoId: RepoId, + { last }: { readonly last?: string } = {}, + ): Promise { + const dir = repoDir(repoId); + await withRepoLock(repoId, async () => { + try { + if (last !== undefined) { + let entries: string[]; + try { + entries = await fs.promises.readdir(dir); + } catch (cause) { + if ( + !(cause instanceof Error && "code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + entries = []; + } + for (const entry of entries) { + if (entry === last) continue; + await fs.promises.rm(path.join(dir, entry), { + recursive: true, + force: true, + }); + } + // Keep the recovery record until the other entries are durably gone. + if (entries.length > 0) await syncDirectoryDurable(dir); + await fs.promises.rm(path.join(dir, last), { + recursive: true, + force: true, + }); + } + await fs.promises.rm(dir, { recursive: true, force: true }); + await syncRemovedPathDurable(dir); + } finally { + // Failed deletion or sync may still have changed the repository on disk. + invalidateGitCache(dir); + existingCommitsCache.delete(indexCacheKey(repoId)); + const refPrefix = `${indexCacheKey(repoId)}/`; + for (const key of [...seqCache.keys()]) { + if (key.startsWith(refPrefix)) seqCache.delete(key); + } + for (const key of [...lastPackedTip.keys()]) { + if (key.startsWith(refPrefix)) lastPackedTip.delete(key); + } + for (const key of [...chainReachabilityCache.keys()]) { + if (key.startsWith(refPrefix)) chainReachabilityCache.delete(key); + } + } + }); + } + return { initRepo, writeTree, @@ -2182,5 +2260,6 @@ export function createRepoStore(config: CreateRepoStoreConfig): RepoStore { openCommittedReads, openCommittedReadsAtCommit, subscribe, + removeRepo, }; } diff --git a/vendor/intx/hub-sessions/src/repo-store/types.ts b/vendor/intx/hub-sessions/src/repo-store/types.ts index 96d2c06b42..9c28402c66 100644 --- a/vendor/intx/hub-sessions/src/repo-store/types.ts +++ b/vendor/intx/hub-sessions/src/repo-store/types.ts @@ -1,6 +1,6 @@ import { type } from "arktype"; -import { RepoAction as RepoActionSchema } from "@intx/types/sidecar"; -import type { RepoKind, RepoId, RepoAction } from "@intx/types/sidecar"; +import { RepoAction as RepoActionSchema } from "@intx/types/repo"; +import type { RepoKind, RepoId, RepoAction } from "@intx/types/repo"; export type { RepoKind, RepoId, RepoAction }; diff --git a/vendor/intx/hub-sessions/src/session-service.ts b/vendor/intx/hub-sessions/src/session-service.ts index ba8cf9b924..1d87063007 100644 --- a/vendor/intx/hub-sessions/src/session-service.ts +++ b/vendor/intx/hub-sessions/src/session-service.ts @@ -19,7 +19,8 @@ import { hexEncode } from "@intx/types"; import type { CredentialDelivery, CredentialMaterialEntry, -} from "@intx/types/sidecar"; +} from "@intx/types/credential-delivery"; +import { DeploymentRejectedError } from "@intx/types/sidecar"; import type { CredentialCipher } from "@intx/types"; import { sessionAsset as sessionAssetTable } from "@intx/db/schema"; import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; @@ -78,11 +79,13 @@ import { import { restoreWorkflowRunToAllocation } from "./workflow-run-restore"; import { committedReadsToSourceTree } from "./committed-source-tree"; import { + describeProbeGateRefusal, installAndApproveWorkflowDefinition, type InstallAndApproveArgs, type InstallAndApproveResult, } from "./workflow-probe-gate"; import { buildReferencedWorkflowSourcePins } from "./workflow-source-pins"; +import { workflowSourceRepoKind } from "./workflow-source-repo-kind"; import type { SidecarReconciliationContext } from "./sidecar-allocation/operation"; const logger = getLogger(["interchange", "hub", "session-service"]); @@ -154,7 +157,7 @@ export type InstallAndApproveWorkflowSourceParams = { * and asset-`tarball` variants; omitted for the asset-`source` variant. */ pin?: string; - /** The `workflow`-kind asset the frozen definition projects a definition over. */ + /** The definition asset: a workflow source tree or package-registry tarball. */ definitionAssetId: string; /** The exact provisioned generation that executes the probe. */ allocationTarget: AllocatedSidecarTarget; @@ -543,15 +546,15 @@ function isAssetDeployArgs( * freeze. * * Credential MATERIAL rides ONE `CredentialDelivery` delivered to the child on - * the frame, unioned from three rails and deduped by credentialId: tool bindings - * (grant-scoped, resolved here via `buildCredentialDelivery`); every top-level - * inference source; and every inline body step's inference source. Both inference - * rails are resolved HERE from the DB under the tenant-ownership authority - * (`resolveInferenceMaterials`), so the deploy is self-contained -- no caller - * pre-supplies material, and a spawned body child finds its secret in the cell. - * The merge is a post-authz union of already-cleared material (tool material is - * grant-scoped, inference material is tenant-ownership-scoped), never a shared - * authz check. + * the frame, unioned from three rails and deduped by credentialId: credential + * bindings (grant-scoped, resolved here via `buildCredentialDelivery`); every + * top-level inference source; and every inline body step's inference source. + * Both inference rails are resolved HERE from the DB under the tenant-ownership + * authority (`resolveInferenceMaterials`), so the deploy is self-contained -- + * no caller pre-supplies material, and a spawned body child finds its secret in + * the cell. The merge is a post-authz union of already-cleared material + * (binding material is grant-scoped, inference material is + * tenant-ownership-scoped), never a shared authz check. * Credential GRANT enforcement is a SEPARATE layer: the `credential:{id}` / * `use` grant the runtime gate checks is minted per-run by run-grant * materialization into `runs//grants.json`, not carried on this frame -- @@ -577,7 +580,7 @@ async function prepareSourceRefDeploy( const { approval, projection, closure } = args.approved; if (!approval.ok) { throw new Error( - `deployCodeSourcedWorkflow: refusing to deploy an unapproved workflow (gate reason: ${approval.reason})`, + `deployCodeSourcedWorkflow: refusing to deploy an unapproved workflow (${describeProbeGateRefusal(approval)})`, ); } @@ -677,7 +680,8 @@ async function prepareSourceRefDeploy( // Assemble the ONE credential delivery. Its `materials` cover three rails, each // authorized upstream on its own terms, deduped by credentialId into one cell: - // - tool bindings, grant-scoped through `buildCredentialDelivery` above; + // - credential bindings, grant-scoped through `buildCredentialDelivery` + // above; // - the inference source pinned to each top-level step that can actually // invoke inference, tenant-owned; // - the same for each inline body step (onTrigger/childWorkflow bodies @@ -692,10 +696,11 @@ async function prepareSourceRefDeploy( // authority, so this deploy is self-contained: a direct deploy (a test) that // seeds the credentials in the DB -- rather than pre-supplying material -- still // fills the cell, and a spawned body finds its secret rather than failing closed - // at resolve time. Precedence on a shared credentialId is tool material first - // (grant-scoped), then the inference material (tenant-ownership-scoped): the - // first material for an id wins. Inference sources carry NO binding descriptor - // -- they reference their credential by id directly. + // at resolve time. Precedence on a shared credentialId is binding material + // first (grant-scoped), then the inference material + // (tenant-ownership-scoped): the first material for an id wins. Inference + // sources carry NO binding descriptor -- they reference their credential by + // id directly. const materials = new Map(); for (const material of credentials?.materials ?? []) { materials.set(material.credentialId, material); @@ -749,10 +754,10 @@ async function prepareSourceRefDeploy( } } } - // A delivery with tool bindings always carries their material, so an empty map - // means no rail contributed anything -- send no delivery. (Tool bindings never - // produce a descriptor without a material, so bindings-without-materials cannot - // occur.) + // A delivery with credential bindings always carries their material, so an + // empty map means no rail contributed anything -- send no delivery. + // (Credential bindings never produce a descriptor without a material, so + // bindings-without-materials cannot occur.) const credentialDelivery: CredentialDelivery | undefined = materials.size > 0 ? { @@ -815,7 +820,7 @@ async function emitSourceRefDeployFrame( args: DeployCodeSourcedWorkflowArgs & { allocationTarget?: AllocatedSidecarTarget; sidecarAllocationRouter?: SidecarAllocationRouter; - onUnsentInitializationCleared(publicKey: string | null): void; + onInitializationCleared(): void; }, reconciliation: SidecarReconciliationContext, ): Promise<{ @@ -836,20 +841,18 @@ async function emitSourceRefDeployFrame( leaseId, signal, }; - let previousPublicKey: string | null | undefined; + let reserved = false; try { const result = await sendMultiStepDeployFrame( sendArgs, signal, async () => { - const reserved = - await allocationStore.beginInitialization(initialization); - if (reserved === null) { + if (!(await allocationStore.beginInitialization(initialization))) { throw new Error( "Allocation no longer permits this initialization attempt", ); } - previousPublicKey = reserved.previousPublicKey; + reserved = true; }, ); return { @@ -860,17 +863,37 @@ async function emitSourceRefDeployFrame( : {}), }; } catch (cause) { + if (cause instanceof DeploymentRejectedError && reserved) { + logger.error`Deployment ${args.anchorRunId} was rejected before starting: ${cause.code}: ${cause.message}`; + try { + const releasing = await allocationStore.rejectInitialization({ + ...initialization, + message: cause.message, + }); + if (releasing !== null) { + sendArgs.sidecarAllocationRouter.fenceAllocation( + releasing.id, + releasing.generation, + releasing.sidecarId === undefined + ? {} + : { cleanup: { sidecarId: releasing.sidecarId } }, + ); + args.onInitializationCleared(); + } + } catch (error) { + logger.error`Could not record deployment rejection for ${initialization.allocationId}: ${error instanceof Error ? error.message : String(error)}`; + } + throw cause; + } if (!isDeployFrameFailure(cause)) throw cause; - // Only a confirmed reservation gives us the key to restore. An ambiguous + // Only a confirmed reservation can be rolled back. An ambiguous // reservation response leaves its durable marker for conservative cleanup. // A confirmed unsent attempt can roll back even after lease cancellation. - if (!cause.frameSent && previousPublicKey !== undefined) { + if (!cause.frameSent && reserved) { try { - const cleared = await allocationStore.clearUnsentInitialization({ - ...initialization, - previousPublicKey, - }); - if (cleared) args.onUnsentInitializationCleared(previousPublicKey); + const cleared = + await allocationStore.clearUnsentInitialization(initialization); + if (cleared) args.onInitializationCleared(); } catch (error) { logger.warn`Could not clear unsent initialization for ${initialization.allocationId}: ${error instanceof Error ? error.message : String(error)}`; } @@ -909,7 +932,8 @@ async function emitSourceRefDeployFrame( * the wire (`isDeployFrameFailure && frameSent === false`); every other failure * -- a sent-but-unacked frame OR any untagged error -- is treated as * possibly-live: the anchor is fenced `deployed` -> `failed` and the error is - * `leakedAgent: true`. + * `leakedAgent: true`. An explicit pre-start rejection also fails the anchor, + * but preserves the rejection instead of claiming the worker may be live. * * The prepared provisioned path does NOT use this composition: its anchor row * already exists from prepare time, so it drives `emitSourceRefDeployFrame` and @@ -971,8 +995,8 @@ export async function deployCodeSourcedWorkflow( } throw new SessionLaunchError("start", cause, false); } - // A sent-but-unacked frame, OR any untagged/unexpected error: no positive - // proof of a clean send, so treat the agent as possibly-live. Fence the + // Preserve a rejected deployment's anchor as a recorded failure. Otherwise + // a sent-but-unacked frame or unexpected error leaves it possibly live. Fence the // anchor `deployed` -> `failed` (guarded so a self-flip to "running" by a // trigger that already landed is left alone). Do NOT delete: a live child // needs the anchor to bootstrap. @@ -995,6 +1019,9 @@ export async function deployCodeSourcedWorkflow( // despite the ack failure. Leave it; the leaked-agent disposition still // holds because the frame was (or may have been) sent. logger.warn`anchor-before-frame: anchor ${args.anchorRunId} already advanced past deployed on an unacked/failed emit; the agent is live and the run is progressing despite the ack failure`; + } else if (cause instanceof DeploymentRejectedError) { + logger.error`Deployment ${args.anchorRunId} was rejected before starting: ${cause.code}: ${cause.message}`; + throw cause; } else { logger.warn`anchor-before-frame: fenced anchor ${args.anchorRunId} deployed->failed on an unacked/failed emit; the agent may be leaked but the run is dead`; } @@ -1079,10 +1106,10 @@ export async function recoverSenderDeploy(args: { }): Promise { const { allocation, reconciliation } = args; reconciliation.signal.throwIfAborted(); - // A proven-unsent clear may still restore the previous key after this claim. - // Do not fail its mail using the claim's stale marker. Cleanup rechecks under - // the allocation lock; an advanced fence settles failure, while a rolled-back - // attempt is resolved by its caller or the next claim's completed key. + // A proven-unsent clear may still roll the attempt back after this claim, and + // its caller then settles the mail. Do not settle it from the claim's stale + // marker. Cleanup rechecks under the allocation lock; an advanced fence + // settles failure, while a rolled-back attempt is resolved by its caller. if (allocation.initializationLeaseId !== undefined) return; // Claiming the lease prevents the previous attempt from publishing. Its // marker and key now distinguish a committed initialization from failure, @@ -1322,8 +1349,8 @@ export function createSessionService( // pack, reserve the manifest row, then send the pack. The manifest // reservation MUST happen before the pack send: if the sidecar acks // but the row is missing, the session has materialization without - // a recorded manifest. An allocated replacement may reuse the exact row - // its predecessor recorded. If reservation fails, no pack is sent. + // a recorded manifest. A retried first deploy may reuse the exact row an + // earlier attempt recorded. If reservation fails, no pack is sent. // // The fan-out materializes the package-registry assets the // tool-package resolver picked. They live behind tenant @@ -1487,7 +1514,7 @@ export function createSessionService( const committed = await agentRepoStore.repoStore.openCommittedReadsAtCommit( HUB_PRINCIPAL, - { kind: "workflow", id: source.assetId }, + { kind: workflowSourceRepoKind(source), id: source.assetId }, source.package.commitSha, ); if (committed === null) { @@ -1561,7 +1588,7 @@ export function createSessionService( return source.kind === "asset" ? bindAssetAttachmentResolver( source.assetId, - source.package.format === "source" ? "workflow" : "package-registry", + workflowSourceRepoKind(source), ) : null; } @@ -1594,7 +1621,7 @@ export function createSessionService( if (!approved.approval.ok) { throw new WorkflowDefinitionInvalidError( approved.projection.id, - `code-sourced workflow install did not approve (reason: ${approved.approval.reason})`, + `code-sourced workflow install did not approve (${describeProbeGateRefusal(approved.approval)})`, ); } return approved; @@ -1688,8 +1715,8 @@ export function createSessionService( operatorApprovals: approval.approvedSurface, }); - // Restore the Hub-authoritative run ref onto the exact allocation generation - // before its address is routed. + // Seed the deployment with the Hub's copy of its history on the exact + // allocation generation before its address is routed. await restoreWorkflowRunToAllocation({ agentRepoStore, allocationRouter, @@ -1729,10 +1756,10 @@ export function createSessionService( ); } - let restoredPublicKey: string | null | undefined; + let initializationCleared = false; const commonEmit = { - onUnsentInitializationCleared(publicKey: string | null) { - restoredPublicKey = publicKey; + onInitializationCleared() { + initializationCleared = true; }, approved: params.approved, sidecarAllocationRouter: allocationRouter, @@ -1812,19 +1839,14 @@ export function createSessionService( publicKey: result.publicKey, }; } catch (error) { - if (restoredPublicKey !== undefined) { - sidecarRouter.noteSenderDeploySettled( - senderAttempt, - restoredPublicKey === null - ? { failed: error instanceof Error ? error.message : String(error) } - : { recorded: restoredPublicKey }, - ); - } // A sent deploy or cancelled publication may have committed despite its // lost response, as may an unsent rollback. Without a confirmed rollback, - // recovery owns transport failures too. Only an uncancelled preparation - // failure can settle definitively here. - else if (!signal.aborted && !(error instanceof SessionLaunchError)) { + // recovery owns transport failures too. A recorded rejection, confirmed + // rollback, or uncancelled preparation failure can settle here. + if ( + initializationCleared || + (!signal.aborted && !(error instanceof SessionLaunchError)) + ) { sidecarRouter.noteSenderDeploySettled(senderAttempt, { failed: error instanceof Error ? error.message : String(error), }); @@ -1856,8 +1878,8 @@ export function createSessionService( sourceCommitSha, }; - // Reserve durable recovery intent before the pack send. A replacement - // generation may reuse the exact row its predecessor recorded. + // Reserve durable recovery intent before the pack send. A retried first + // deploy may reuse the exact row an earlier attempt recorded. const inserted = await db .insert(sessionAssetTable) .values({ ...record, materializedAt: new Date() }) diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/adapter-manifest-digest.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/adapter-manifest-digest.ts new file mode 100644 index 0000000000..afd4d9c9e7 --- /dev/null +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/adapter-manifest-digest.ts @@ -0,0 +1,21 @@ +import { sha256 } from "@intx/crypto"; +import { hexEncode, type AdapterManifest } from "@intx/types"; + +/** + * Identifies the adapter manifest a sidecar boots with, so the Hub can tell + * whether a sidecar it already runs loaded the manifest a new placement needs. + * The sidecar treats an absent manifest as an empty one, so both hash alike. + * Entry order is kept because a later entry overrides an earlier one for the + * same provider. Each entry is rebuilt with its keys in a fixed order, so the + * key order of the parsed input does not change the digest. + */ +export async function adapterManifestDigest( + manifest: AdapterManifest | undefined, +): Promise { + const entries = (manifest ?? []).map((entry) => ({ + provider: entry.provider, + specifier: entry.specifier, + export: entry.export, + })); + return hexEncode(await sha256(JSON.stringify(entries))); +} diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/contracts.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/contracts.ts index b052c596dc..c84479d848 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/contracts.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/contracts.ts @@ -1,6 +1,9 @@ import { type } from "arktype"; -import type { SidecarCapabilityDeclaration } from "@intx/types"; +import type { + AdapterManifest, + SidecarCapabilityDeclaration, +} from "@intx/types"; export type EnsureSidecarRequest = { /** Cancellation is best effort; a cancelled ensure has an uncertain outcome. */ @@ -9,9 +12,26 @@ export type EnsureSidecarRequest = { readonly generation: number; readonly tenantId: string; readonly anchorRunId: string; + /** + * A new sidecar identity minted for this generation. Capacity the + * provisioner starts authenticates with it; a provisioner that places the + * work on a sidecar it already runs returns that sidecar's id instead. + */ readonly sidecarId: string; readonly token: string; readonly hubWebSocketUrl: string; + /** + * Operator-declared custom adapter manifest the spawned sidecar must load. + * Passed through opaquely — the sidecar's adapter registry is the single + * authority that resolves entries and admits `model_provider.plugin` + * values at deploy time. Absent means no custom adapters are declared. + * A sidecar reads its manifest only when it starts, so a provisioner that + * answers with a sidecar it already runs must return one that booted with + * this manifest, also when it retries an ensure for the same generation. + * The Hub refuses reuse across a manifest change, so a changed manifest + * applies to new placements only. + */ + readonly adapterManifest?: AdapterManifest; }; export type DestroySidecarRequest = { @@ -19,6 +39,17 @@ export type DestroySidecarRequest = { readonly signal?: AbortSignal; readonly allocationId: string; readonly generation: number; + /** + * The sidecar the Hub last recorded for this generation. When the Hub never + * learned an ensure's outcome, this is the identity that ensure was offered + * even if the provisioner placed the work on an existing sidecar, so a + * provisioner that reuses sidecars must be able to release by allocationId. + * A destroy releases the allocation's hold at `generation` or any older one: + * release and replacement advance the generation before destroying. The + * Hub never re-announces holds, so such a provisioner stores them durably; + * held only in memory, they are lost on its restart, leaking sidecars or + * stopping one that other work still uses. + */ readonly sidecarId: string; /** * Optional provider handle recorded after ensure returns. A Hub crash can @@ -41,6 +72,19 @@ export type SidecarOperationFailure = typeof SidecarOperationFailure.infer; /** * Acceptance means the requested infrastructure exists, not that it is ready. + * `sidecarId` names a sidecar this provisioner started for earlier work that + * now also hosts this generation; omit it when the request's own identity is + * used. The Hub accepts only a sidecar minted for the same provisioner binding + * and adapter manifest that is not retired, whether or not it still hosts + * other work. A rejected ensure, or a destroy that confirms cleanup, retires + * the identity. A deployment also needs the sidecar to have fewer active + * reservations than the capacity it advertised in hello. Copies + * with confirmed removal or retained storage do not consume active capacity. + * This is checked when an allocation is placed and when a deployment adopts + * its probe's sidecar, not when the probe is placed. + * Unknown inventory delays placement until hello or the connection deadline. + * This is not a capacity rejection. The pending acceptance is in memory; + * after a Hub restart, cleanup precedes any new provisioning attempt. * Rejection means no infrastructure exists for this generation (ensure-only; * destroy rejections below carry no such guarantee); a provisioner must throw * when it cannot determine whether the request took effect. @@ -48,24 +92,75 @@ export type SidecarOperationFailure = typeof SidecarOperationFailure.infer; export const EnsureSidecarResult = type({ kind: "'accepted'", "externalRef?": "string", + "sidecarId?": "string", }).or(SidecarOperationFailure); export type EnsureSidecarResult = typeof EnsureSidecarResult.infer; /** - * Destruction confirms the capacity is gone and older ensure calls are fenced. - * A non-retryable rejection stops automatic cleanup; capacity may still exist. + * Destruction fences older ensure calls and releases the provisioner's hold. + * `cleanup: confirmed` additionally proves that the deployment's worker and + * local state are gone and cannot return. A worker that remains, shared or + * kept running idle, needs `cleanup: required`; the Hub reserves its slot + * until the sidecar acknowledges removal. Retries must preserve this + * distinction, including after restarts. Stopping a worker in answer to a + * destroy must also remove its restorable deployment state and confirm + * cleanup. A result requiring sidecar cleanup must leave that cleanup path + * available; absence from a provider's in-memory map is not proof. + * A non-retryable rejection of an outstanding hold release stops automatic cleanup + * and leaves the provider obligation and cleanup binding for operator recovery. + * A copy holds an active slot until removal or its transfer to retained storage + * is confirmed. */ export const DestroySidecarResult = type({ kind: "'destroyed'", + cleanup: "'confirmed' | 'required'", }).or(SidecarOperationFailure); export type DestroySidecarResult = typeof DestroySidecarResult.infer; +/** + * `not_owned` is final: the Hub minted no sidecar with that id for this + * provisioner's current binding. + */ +export type SidecarStoppedResult = + | { readonly kind: "retired" } + | { readonly kind: "not_owned" }; + +/** + * `endReconnectWindow` stops the Hub waiting for the sidecar's work to + * reconnect: its allocations get no reconnect window, now or after a later + * disconnect, and go to its connection loss handling at once. Without it, + * that work keeps its reconnect window, in case the provisioner brings the + * sidecar back elsewhere. Work still connected runs until it disconnects; + * probes are not touched. + */ +export const SidecarStoppedOptions = type({ + "endReconnectWindow?": "boolean", +}); +export type SidecarStoppedOptions = typeof SidecarStoppedOptions.infer; + +/** Calls the Hub offers a provisioner through `attach`. */ +export interface SidecarProvisionerHost { + /** + * Reports that a sidecar this provisioner started under its current binding + * is gone for good, such as an idle sidecar it stopped after its last + * destroy. The Hub retires the identity. Idempotent. A throw is transient: + * keep the report durably and retry it, also after a Hub restart. Invalid + * arguments also throw, and retrying them never succeeds. + */ + sidecarStopped( + sidecarId: string, + options?: SidecarStoppedOptions, + ): Promise; +} + export interface SidecarProvisioner { readonly id: string; - readonly apiVersion: 1; + readonly apiVersion: 2; /** Stable, non-secret identity for the backend configuration. */ readonly bindingFingerprint: string; readonly capabilities: readonly SidecarCapabilityDeclaration[]; + /** Called once when the Hub starts, with the calls it offers. */ + attach?(host: SidecarProvisionerHost): void; /** * Converges infrastructure for this generation. Implementations must be * idempotent and reject generations older than one they have observed. @@ -81,16 +176,19 @@ export interface SidecarProvisioner { destroy(request: DestroySidecarRequest): Promise; } +type AllocationCredentialIdentity = { + readonly sidecarId: string; + readonly allocationId: string; + readonly tenantId: string; + readonly anchorRunId: string; + readonly workflowRunAddress: string; + readonly generation: number; +}; + +/** One probe, deployment, or cleanup obligation authenticated on a sidecar. */ export type SidecarCredentialIdentity = - | { - readonly kind: "allocated"; - readonly sidecarId: string; - readonly allocationId: string; - readonly tenantId: string; - readonly anchorRunId: string; - readonly workflowRunAddress: string; - readonly generation: number; - } + | (AllocationCredentialIdentity & { readonly kind: "allocated" }) + | (AllocationCredentialIdentity & { readonly kind: "cleanup" }) | { readonly kind: "probe"; readonly sidecarId: string; @@ -99,10 +197,58 @@ export type SidecarCredentialIdentity = readonly generation: number; }; +/** The sidecar a bearer token verifies as. */ +export type SidecarCredentials = { + readonly sidecarId: string; + /** + * The identity is retired with neither current work nor cleanup left. Its + * handshake undeploys what the sidecar reports and closes. + */ + readonly retired?: true; +}; + +/** + * `reclaim` also requires the deployment's first deploy to have completed: its + * anchor's key is committed and no initialization is in flight. A copy whose + * deploy is still uncertain is undeployed instead, since releasing it on a + * sidecar that hosts other work does not stop the sidecar. It further requires + * the anchor run not to be terminal: a copy of a run that has ended is not + * routed, even when its own history never recorded the end. + * + * `retention` permits retaining an initialized terminal copy while its capacity + * retention period remains open. `retention-transition` validates terminal + * copies independently of that deadline, including stop acknowledgements and + * acknowledgements for retains issued before expiry. Copies already due for + * release are stopped without consuming kept capacity. `cleanup` accepts + * releasing or replacing allocations and every failed cleanup, whatever + * stopped it. Their connection grants no workflow routes and preserves + * inventory. A reconnect can resume a disconnect timeout's cleanup, preserving + * its attempts and terminal workflow outcome. Other failed cleanups do not + * reopen. Replacing a parked deployment must confirm removal of its previous + * copy before binding replacement capacity. + */ +export type SidecarIdentityUse = + | "registration" + | "cleanup" + | "readiness" + | "routing" + | "reclaim" + | "retention" + | "retention-transition"; + export interface SidecarCredentialResolver { - resolve(token: string): Promise; + /** + * Resolves a bearer token to its sidecar, or null for an unknown token. A + * retired one with neither current work nor cleanup left resolves marked + * `retired`. A sidecar hosting nothing current still resolves. + */ + resolve(token: string): Promise; + /** The current bindings of a sidecar, empty when it hosts nothing. */ + resolveBindings( + sidecarId: string, + ): Promise; isCurrent( identity: SidecarCredentialIdentity, - use: "registration" | "readiness" | "routing", + use: SidecarIdentityUse, ): Promise; } diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/index.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/index.ts index de74e78e76..5620192c0f 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/index.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/index.ts @@ -20,8 +20,18 @@ export { type EnsureSidecarRequest, type SidecarCredentialIdentity, type SidecarCredentialResolver, + type SidecarCredentials, + type SidecarIdentityUse, type SidecarProvisioner, + type SidecarProvisionerHost, + type SidecarStoppedOptions, + type SidecarStoppedResult, } from "./contracts"; +export { + attachSidecarProvisioners, + type AttachSidecarProvisionersDeps, +} from "./provisioner-host"; +export { adapterManifestDigest } from "./adapter-manifest-digest"; export { createSidecarAllocationReconciler, type SidecarAllocationReconciler, diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts index 9ee5a3c451..d645f283d3 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts @@ -6,12 +6,63 @@ export type SidecarReconciliationContext = { }; export class SidecarOperationTimeoutError extends Error { - constructor(operation: string, timeoutMs: number) { - super(`${operation} timed out after ${String(timeoutMs)}ms`); + constructor(operation: string, timeoutMs: number, options?: ErrorOptions) { + super(`${operation} timed out after ${String(timeoutMs)}ms`, options); this.name = "SidecarOperationTimeoutError"; } } +/** + * The current connection of a deployment's sidecar did not report the + * deployment, though its first deploy completed: the sidecar no longer holds + * it, and the Hub does not deploy an address twice. + */ +export class SidecarDeploymentMissingError extends Error { + constructor(allocationId: string, generation: number) { + super( + `The sidecar of allocation ${allocationId} generation ${String(generation)} no longer holds its deployment`, + ); + this.name = "SidecarDeploymentMissingError"; + } +} + +/** + * The current connection of a deployment's sidecar reported the deployment + * stopped though the Hub did not stop it: its workflow child ended itself, or + * the sidecar could not restore it. The message is the sidecar's error. + */ +export class SidecarDeploymentStoppedError extends Error { + constructor(error: string) { + super(error); + this.name = "SidecarDeploymentStoppedError"; + } +} + +/** + * A first deploy failed before its deploy frame was sent, so nothing ran on + * the sidecar and the deploy can be tried again. + */ +export class SidecarFirstDeployError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = "SidecarFirstDeployError"; + } +} + +/** + * A deployment its sidecar reported stopped committed history the Hub does + * not hold yet; the Hub looks again at `retryAt` while it still waits. + */ +export class SidecarDeploymentHistoryPendingError extends Error { + readonly retryAt: Date; + + constructor(unreceived: string, retryAt: Date) { + super(`Waiting for history a stopped deployment committed: ${unreceived}`); + this.name = "SidecarDeploymentHistoryPendingError"; + this.retryAt = retryAt; + } +} + /** Stops waiting on cancellation or an optional deadline, even if work ignores the signal. */ export async function runSidecarOperation( operation: string, diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/plugin-registry.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/plugin-registry.ts index 553b08efcc..69aba5a802 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/plugin-registry.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/plugin-registry.ts @@ -55,7 +55,7 @@ export function createSidecarPluginRegistry({ const provisionersById = new Map(); for (const provisioner of provisioners) { validateId(provisioner.id); - if (provisioner.apiVersion !== 1) { + if (provisioner.apiVersion !== 2) { throw new Error( `Unsupported sidecar provisioner API version for ${provisioner.id}: ${String(provisioner.apiVersion)}`, ); diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/provisioner-host.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/provisioner-host.ts new file mode 100644 index 0000000000..0efa70947a --- /dev/null +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/provisioner-host.ts @@ -0,0 +1,68 @@ +import { type } from "arktype"; + +import type { + AllocationConnectionCheck, + ProvisionerSidecarRetirement, + SidecarProvisionerBinding, +} from "@intx/db"; +import { getLogger } from "@intx/log"; + +import { SidecarStoppedOptions, type SidecarProvisioner } from "./contracts"; + +const logger = getLogger(["hub", "sidecar-allocation"]); + +const SidecarStoppedArgs = type({ + sidecarId: "string > 0", + options: SidecarStoppedOptions.or("undefined"), +}); + +export type AttachSidecarProvisionersDeps = { + readonly provisioners: readonly SidecarProvisioner[]; + readonly retireProvisionerSidecar: (args: { + readonly sidecarId: string; + readonly binding: SidecarProvisionerBinding; + readonly endReconnectWindow: boolean; + readonly isConnected: AllocationConnectionCheck; + }) => Promise; + readonly isConnected: AllocationConnectionCheck; +}; + +// A provisioner can sit in both the deployment and the probe registry, so +// each object is attached once. +export function attachSidecarProvisioners({ + provisioners, + retireProvisionerSidecar, + isConnected, +}: AttachSidecarProvisionersDeps): void { + for (const provisioner of new Set(provisioners)) { + const binding: SidecarProvisionerBinding = { + provisionerId: provisioner.id, + provisionerApiVersion: provisioner.apiVersion, + provisionerBindingFingerprint: provisioner.bindingFingerprint, + }; + provisioner.attach?.({ + async sidecarStopped(rawSidecarId, rawOptions) { + // A provisioner is outside the Hub's type checks; its report is input. + const args = SidecarStoppedArgs({ + sidecarId: rawSidecarId, + options: rawOptions, + }); + if (args instanceof type.errors) + throw new Error( + `Provisioner ${provisioner.id} sent an invalid sidecarStopped report: ${args.summary}`, + ); + const { sidecarId, options } = args; + const retirement = await retireProvisionerSidecar({ + sidecarId, + binding, + endReconnectWindow: options?.endReconnectWindow ?? false, + isConnected, + }); + if (retirement === null) return { kind: "not_owned" }; + if (retirement.connectedAllocationIds.length > 0) + logger.warn`Provisioner ${provisioner.id} reported sidecar ${sidecarId} stopped while allocations ${retirement.connectedAllocationIds.join(", ")} are still connected to it; leaving them running`; + return { kind: "retired" }; + }, + }); + } +} diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts index 852e86e9ef..fbe4de9e43 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts @@ -1,9 +1,21 @@ import { type } from "arktype"; import { sha256 } from "@intx/crypto"; -import type { SidecarAllocation, SidecarAllocationStore } from "@intx/db"; +import { DeploymentRejectedError } from "@intx/types/sidecar"; +import { + SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE, + SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + SIDECAR_PROVISIONER_RELEASE_FAILURE_CODE, + SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE, + SidecarReuseRejectedError, + SidecarInventoryUnavailableError, + type SidecarAllocation, + type SidecarAllocationStore, +} from "@intx/db"; import { getLogger } from "@intx/log"; -import { hexEncode } from "@intx/types"; +import { hexEncode, type AdapterManifest } from "@intx/types"; import type { AllocatedSidecarTarget, @@ -17,10 +29,15 @@ import { EnsureSidecarResult, type SidecarProvisioner, } from "./contracts"; +import { adapterManifestDigest } from "./adapter-manifest-digest"; import type { SidecarPluginRegistry } from "./plugin-registry"; import { DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, runSidecarOperation, + SidecarDeploymentHistoryPendingError, + SidecarDeploymentMissingError, + SidecarDeploymentStoppedError, + SidecarFirstDeployError, SidecarOperationTimeoutError, type SidecarReconciliationContext, } from "./operation"; @@ -30,20 +47,30 @@ const logger = getLogger(["hub", "sidecar-allocation"]); type AllocationStore = Pick< SidecarAllocationStore, | "beginReplacement" + | "beginRelease" | "beginUnrecoverableRelease" + | "failStoppedDeployment" | "bindInitialSidecar" | "bindReplacementSidecar" | "claimNextReconcilable" | "extendReconciliationLease" | "failWithoutInfrastructure" | "listActive" + | "listForConnectionRepair" | "isReconciliationLeaseCurrent" + | "hasRunnableAnchor" | "markAllocated" | "markConnectionLost" | "markConnectionReady" | "markDestroyFailed" | "markReleased" + | "confirmDeploymentCleanup" + | "confirmProviderRelease" + | "recordCleanupConnection" + | "resumeCleanupAfterDisconnect" + | "retireSidecar" | "parkReconciliation" + | "scheduleReconnectAfterHubStart" | "scheduleReconnectIfUnscheduled" | "scheduleRetry" | "wakeReconciliation" @@ -55,11 +82,23 @@ export type SidecarAllocationReconcilerDeps = { readonly router: Pick< SidecarAllocationRouter, | "fenceAllocation" + | "getCleanupConnection" + | "holdsAllocatedBinding" | "isAllocatedSidecarReady" + | "reportedDeploymentFailure" | "retireAllocation" + | "syncSidecar" + | "undeployAllocation" | "waitForAllocatedSidecar" + | "waitForSidecarInventory" >; readonly hubWebSocketUrl: string; + /** + * Operator-declared custom adapter manifest forwarded to every provisioned + * sidecar via the ensure request. Opaque here — the sidecar's adapter + * registry is the authority that admits `model_provider.plugin` values. + */ + readonly adapterManifest?: AdapterManifest; /** * Resolve the previous attempt's deferred mail under the newly claimed * lease, before connection waits or cleanup. @@ -68,21 +107,25 @@ export type SidecarAllocationReconcilerDeps = { allocation: SidecarAllocation, reconciliation: SidecarReconciliationContext, ) => Promise; - /** Idempotently restores and deploys one connected allocation generation. */ + /** Idempotently runs a connected allocation generation's first deploy. */ readonly onReady?: ( allocation: SidecarAllocation, reconciliation: SidecarReconciliationContext, ) => Promise; /** - * Replace an allocated worker after its reconnect grace expires. Disabled by - * default because Hub recovery does not restore arbitrary sidecar or - * isolation-container filesystem state, so automatic continuation could run - * without state the previous worker produced. + * Asked only after an allocated worker misses its connect deadline. + * A parked verdict replaces that worker. Anything else releases it. + * Initialization-lease and leaked-launch releases do not ask. */ - readonly enableAutomaticReplacementRecovery?: boolean; + readonly classifyParkedRun?: (allocation: SidecarAllocation) => Promise<{ + readonly verdict: "parked" | "not-parked" | "unknown"; + readonly reason: string; + }>; readonly leaseDurationMs?: number; readonly connectTimeoutMs?: number; readonly operationTimeoutMs?: number; + /** Failed cleanup attempts allowed for a release or replacement before polling stops. */ + readonly maxCleanupAttempts?: number; /** * Bounds admitted claims, active reconciliation, and retained allocation * queries or writes. @@ -98,11 +141,14 @@ export type SidecarAllocationReconcilerDeps = { export type SidecarAllocationReconciler = { /** Rebuild all trust fences before accepting allocated connections. */ initialize(): Promise; - /** Starts a durable reconnect grace period for the exact lost generation. */ + /** + * Gives the exact lost generation as long as its allocation allows to + * reconnect before it is failed. + */ handleDisconnect(target: AllocatedSidecarTarget): Promise; /** Wakes recovery as soon as the exact generation reconnects. */ handleConnected(target: AllocatedSidecarTarget): Promise; - /** Repairs allocated generations left unscheduled after a lost event write. */ + /** Repairs missed readiness and cleanup wakes, and discards obsolete waits. */ repairUnscheduledConnections(): Promise; /** Reconcile at most one due allocation. Returns false when none are due. */ reconcileNext(): Promise; @@ -119,9 +165,27 @@ class ReconciliationLeaseLostError extends Error { } } +class CleanupPersistenceError extends Error { + constructor(cause: unknown) { + super( + `Could not record the cleanup connection: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + this.name = "CleanupPersistenceError"; + } +} + const DEFAULT_LEASE_DURATION_MS = 60_000; const DEFAULT_CONNECT_TIMEOUT_MS = 120_000; const MAX_RETRY_BACKOFF_ATTEMPT = 5; +// How long a first deploy that keeps failing before its deploy frame is sent +// is retried, from its first failure. +const FIRST_DEPLOY_RETRY_LIMIT_MS = 60_000; +const MAX_ENSURE_ATTEMPTS = 10; +const PROVISIONING_RETRY_EXHAUSTED = "sidecar_provisioning_retry_exhausted"; +const PROVISIONER_AVAILABILITY_GRACE_MS = 5 * 60_000; +const PROVISIONER_AVAILABILITY_RETRY_MS = 30_000; +const CLEANUP_PREPARATION_GRACE_MS = 5 * 60_000; function randomHex(bytes: number): string { return hexEncode(crypto.getRandomValues(new Uint8Array(bytes))); @@ -159,12 +223,14 @@ export function createSidecarAllocationReconciler({ plugins, router, hubWebSocketUrl, + adapterManifest, onInitializationRecovery, onReady, - enableAutomaticReplacementRecovery = false, + classifyParkedRun, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS, operationTimeoutMs = DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, + maxCleanupAttempts = 10, maxConcurrentClaims = DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, retryDelayMs = defaultRetryDelay, now = () => new Date(), @@ -182,6 +248,23 @@ export function createSidecarAllocationReconciler({ if (!Number.isSafeInteger(maxConcurrentClaims) || maxConcurrentClaims <= 0) { throw new Error("maxConcurrentClaims must be a positive integer"); } + if (!Number.isSafeInteger(maxCleanupAttempts) || maxCleanupAttempts <= 0) { + throw new Error("maxCleanupAttempts must be a positive integer"); + } + + const currentAdapterManifestDigest = adapterManifestDigest(adapterManifest); + + function fence(allocation: SidecarAllocation): void { + discardChangedPlacement(allocation); + router.fenceAllocation(allocation.id, allocation.generation, { + ...(allocation.sidecarId !== undefined && + (allocation.status === "releasing" || + allocation.status === "replacing" || + allocation.status === "destroy_failed") + ? { cleanup: { sidecarId: allocation.sidecarId } } + : {}), + }); + } const activeAllocations = new Map< string, @@ -193,7 +276,85 @@ export function createSidecarAllocationReconciler({ } >(); + type PendingPlacement = { + readonly generation: number; + readonly result: Extract; + readonly deadline: Date; + waiter: { controller: AbortController; ready: boolean } | undefined; + }; + const pendingPlacements = new Map(); + function discardPlacement(allocationId: string): void { + const pending = pendingPlacements.get(allocationId); + pendingPlacements.delete(allocationId); + pending?.waiter?.controller.abort(); + } + + function discardChangedPlacement(allocation: SidecarAllocation): void { + const pending = pendingPlacements.get(allocation.id); + if ( + pending !== undefined && + (allocation.status !== "provisioning" || + pending.generation !== allocation.generation) + ) + discardPlacement(allocation.id); + } + + function wakePendingPlacement( + allocationId: string, + pending: PendingPlacement, + ): Promise { + const waiter = pending.waiter; + return queueConnectionEvent( + { allocationId, generation: pending.generation }, + async () => { + if ( + pendingPlacements.get(allocationId) !== pending || + pending.waiter !== waiter + ) + return; + if ( + !(await allocationStore.wakeReconciliation( + allocationId, + pending.generation, + )) && + pendingPlacements.get(allocationId) === pending + ) + discardPlacement(allocationId); + }, + ); + } + + function watchPlacementInventory( + allocationId: string, + pending: PendingPlacement, + sidecarId: string, + ): void { + pending.waiter?.controller.abort(); + const waiter = { controller: new AbortController(), ready: false }; + pending.waiter = waiter; + // This subscription owns no claim or lease. The durable schedule enforces + // its deadline; cancellation and the repair sweep discard obsolete waits. + void router + .waitForSidecarInventory(sidecarId, waiter.controller.signal) + .then(async () => { + if ( + pendingPlacements.get(allocationId) !== pending || + pending.waiter !== waiter + ) + return; + // Preserve timely inventory across a delayed claim, without letting a + // late hello extend the original connection deadline. + waiter.ready = now() <= pending.deadline; + await wakePendingPlacement(allocationId, pending); + }) + .catch((cause: unknown) => { + if (!waiter.controller.signal.aborted) + logger.warn`Could not wake inventory placement ${allocationId}: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } + function trackAllocation(allocation: SidecarAllocation): void { + discardChangedPlacement(allocation); const active = activeAllocations.get(allocation.id); if (active === undefined) return; if (active.allocation.generation !== allocation.generation) @@ -218,19 +379,88 @@ export function createSidecarAllocationReconciler({ }); } + const provisionerUnavailableDeadlines = new Map(); + const cleanupPreparationDeadlines = new Map(); + function provisionerFor( allocation: SidecarAllocation, - ): SidecarProvisioner | null { + ): { provisioner: SidecarProvisioner } | { error: string } { const provisioner = plugins.getProvisioner(allocation.provisionerId); + if (provisioner === null) + return { + error: `Provisioner ${allocation.provisionerId} is missing from the plugin registry`, + }; + if (provisioner.apiVersion !== allocation.provisionerApiVersion) + return { + error: `Provisioner ${allocation.provisionerId} has API version ${String(provisioner.apiVersion)}; this allocation requires version ${String(allocation.provisionerApiVersion)}`, + }; if ( - provisioner === null || - provisioner.apiVersion !== allocation.provisionerApiVersion || provisioner.bindingFingerprint !== - allocation.provisionerBindingFingerprint - ) { - return null; + allocation.provisionerBindingFingerprint + ) + return { + error: `Provisioner ${allocation.provisionerId} has binding ${provisioner.bindingFingerprint}; this allocation requires binding ${allocation.provisionerBindingFingerprint}`, + }; + provisionerUnavailableDeadlines.delete(allocation.id); + return { provisioner }; + } + + async function waitForProvisioner( + allocation: SidecarAllocation, + leaseId: string, + reason: string, + cleanup?: CleanupProgress, + ): Promise { + const status = allocation.status; + if ( + status === "failed" || + status === "released" || + status === "destroy_failed" + ) + throw new Error( + `Cannot wait for a provisioner while allocation ${allocation.id} is ${status}`, + ); + let deadline = provisionerUnavailableDeadlines.get(allocation.id); + if (deadline === undefined) { + deadline = new Date(now().getTime() + PROVISIONER_AVAILABILITY_GRACE_MS); + provisionerUnavailableDeadlines.set(allocation.id, deadline); + } + const observedAt = now(); + if (deadline <= observedAt) return false; + logger.warn`Waiting for allocation ${allocation.id}'s provisioner until ${deadline.toISOString()}: ${reason}`; + if (cleanup?.requestFailed) { + await retryDestroy( + allocation, + leaseId, + cleanup, + { + code: SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + message: cleanup.message, + }, + deadline, + ); + } else { + const delay = Math.min( + PROVISIONER_AVAILABILITY_RETRY_MS, + deadline.getTime() - observedAt.getTime(), + ); + if (cleanup !== undefined) { + await retryCleanupWithoutAttempt(allocation, leaseId, cleanup, delay); + } else { + await finishReconciliation(allocation.id, () => + allocationStore.scheduleRetry({ + allocationId: allocation.id, + expectedStatus: status, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + nextAttemptAt: new Date(observedAt.getTime() + delay), + minimumDelayMs: delay, + now: observedAt, + }), + ); + } } - return provisioner; + return true; } function retryAt(attempt: number): Date { @@ -315,16 +545,25 @@ export function createSidecarAllocationReconciler({ const active = activeAllocations.get(allocation.id); if (active === undefined) throw new ReconciliationLeaseLostError(allocation.id); + const sidecarId = allocation.sidecarId; return runSidecarOperation( "Sidecar readiness", operationTimeoutMs, () => - trackAllocationQuery(allocation.id, () => - router.isAllocatedSidecarReady({ + trackAllocationQuery(allocation.id, async () => { + const target = { allocationId: allocation.id, generation: allocation.generation, - }), - ), + }; + // The allocation may have been placed on a sidecar that stayed + // connected, which registered before this binding existed. The sync + // waits behind the sidecar's other frames, such as another + // deployment's pack ingest, so it runs only for a binding not yet + // attached rather than hold a claim slot for one that is. + if (sidecarId !== undefined && !router.holdsAllocatedBinding(target)) + await router.syncSidecar(sidecarId); + return router.isAllocatedSidecarReady(target); + }), active.controller.signal, ); } @@ -336,7 +575,11 @@ export function createSidecarAllocationReconciler({ message: string, { onlyIfInitializationIncomplete = false, - }: { onlyIfInitializationIncomplete?: boolean } = {}, + replaceParked = false, + }: { + onlyIfInitializationIncomplete?: boolean; + replaceParked?: boolean; + } = {}, ): Promise { const initializationCheck = onlyIfInitializationIncomplete ? { @@ -352,16 +595,25 @@ export function createSidecarAllocationReconciler({ await queueReconciliationStep( { allocationId: allocation.id, generation: allocation.generation }, async () => { + // An allocated worker is released unless its committed run is parked. + // It may still be running, and the Hub cannot restore sidecar-local + // files. A parked run continues from the committed log. Provisioning + // ran nothing yet, so a new generation replaces it. + const exhausted = + allocation.status === "provisioning" && + allocation.ensureAttempts >= MAX_ENSURE_ATTEMPTS; const updated = - allocation.status === "allocated" && - !enableAutomaticReplacementRecovery + (allocation.status === "allocated" && !replaceParked) || exhausted ? await allocationStore.beginUnrecoverableRelease({ ...initializationCheck, allocationId: allocation.id, + expectedStatus: exhausted ? "provisioning" : "allocated", expectedGeneration: allocation.generation, expectedLeaseId: leaseId, - failureCode: code, - failureMessage: `Automatic recovery is disabled: ${message}`, + failureCode: exhausted ? PROVISIONING_RETRY_EXHAUSTED : code, + failureMessage: exhausted + ? `Provisioning failed after ${String(allocation.ensureAttempts)} attempts: ${message}` + : message, now: now(), }) : await allocationStore.beginReplacement({ @@ -374,16 +626,19 @@ export function createSidecarAllocationReconciler({ expectedGeneration: allocation.generation, expectedLeaseId: leaseId, nextAttemptAt: retryAt( - allocation.ensureAttempts + allocation.destroyAttempts, + Math.max(0, allocation.ensureAttempts - 1), ), + firstAttemptAt: retryAt(0), failureCode: code, failureMessage: message, now: now(), }); if (updated !== null) { + if (exhausted) + logger.error`Provisioning retry budget exhausted for allocation ${allocation.id} after ${String(allocation.ensureAttempts)} attempts. Last failure: ${message}`; // A late commit still advances the fence before this queued work // settles and its allocation becomes eligible for another claim. - router.fenceAllocation(updated.id, updated.generation); + fence(updated); } else { shouldRetryInitialization = onlyIfInitializationIncomplete; } @@ -463,11 +718,38 @@ export function createSidecarAllocationReconciler({ error instanceof SidecarIdentityValidationError ) throw error; + const message = error instanceof Error ? error.message : String(error); + // A sidecar that reported the deployment stopped and then stayed away + // fails it for the reason it gave, unless a Hub restart has since + // dropped that report. A stopped copy is not replaced. + const stopped = router.reportedDeploymentFailure(target); + let replaceParked = false; + if (stopped === undefined && classifyParkedRun !== undefined) { + let classification: { + verdict: "parked" | "not-parked" | "unknown"; + reason: string; + }; + try { + classification = await classifyParkedRun(allocation); + } catch (cause) { + classification = { + verdict: "unknown", + reason: cause instanceof Error ? cause.message : String(cause), + }; + } + logger.warn`Allocation ${allocation.id} anchor ${allocation.anchorRunId} connect loss classified ${classification.verdict}: ${classification.reason}`; + replaceParked = classification.verdict === "parked"; + } await replaceAfterFailure( allocation, leaseId, - "sidecar_connect_failed", - error instanceof Error ? error.message : String(error), + stopped === undefined + ? "sidecar_connect_failed" + : SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE, + stopped === undefined + ? message + : `${stopped}; its sidecar disconnected, and history the Hub never received may be lost`, + { replaceParked }, ); return; } @@ -493,6 +775,54 @@ export function createSidecarAllocationReconciler({ ); } catch (error) { if (error instanceof ReconciliationLeaseLostError) throw error; + if (error instanceof DeploymentRejectedError) { + await replaceAfterFailure( + allocation, + leaseId, + "sidecar_deployment_rejected", + error.message, + { onlyIfInitializationIncomplete: true }, + ); + return; + } + if (error instanceof SidecarDeploymentMissingError) { + await replaceAfterFailure( + allocation, + leaseId, + "sidecar_deployment_missing", + error.message, + ); + return; + } + if (error instanceof SidecarDeploymentHistoryPendingError) { + // A report that landed meanwhile may carry the tips the Hub holds. + await finishReconciliation(allocation.id, (pendingConnect) => + allocationStore.scheduleRetry({ + allocationId: allocation.id, + expectedStatus: "allocated", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + nextAttemptAt: pendingConnect ? now() : error.retryAt, + now: now(), + }), + ); + return; + } + if (error instanceof SidecarDeploymentStoppedError) { + // The capacity stays: the stopped copy's files are kept for + // inspection until the retention for failed runs releases them. + await finishReconciliation(allocation.id, () => + allocationStore.failStoppedDeployment({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + failureCode: SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE, + failureMessage: error.message, + now: now(), + }), + ); + return; + } if (error instanceof SessionLaunchError && error.leakedAgent) { await replaceAfterFailure( allocation, @@ -503,6 +833,27 @@ export function createSidecarAllocationReconciler({ ); return; } + const message = error instanceof Error ? error.message : String(error); + // Only a first deploy is given up on: a failure around a deployment + // that already runs, such as an unreachable database, says nothing + // about the deployment. + const firstDeployFailedAt = + error instanceof SidecarFirstDeployError + ? (allocation.firstDeployFailedAt ?? now()) + : undefined; + if ( + firstDeployFailedAt !== undefined && + now().getTime() - firstDeployFailedAt.getTime() >= + FIRST_DEPLOY_RETRY_LIMIT_MS + ) { + await replaceAfterFailure( + allocation, + leaseId, + "sidecar_initialization_failed", + `The first deploy kept failing for ${String(FIRST_DEPLOY_RETRY_LIMIT_MS / 1000)} seconds: ${message}`, + ); + return; + } await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({ allocationId: allocation.id, @@ -512,10 +863,10 @@ export function createSidecarAllocationReconciler({ // capped delay so a persistent launch error cannot create a hot loop. nextAttemptAt: retryAt(MAX_RETRY_BACKOFF_ATTEMPT), expectedLeaseId: leaseId, - failure: { - code: "sidecar_initialization_failed", - message: error instanceof Error ? error.message : String(error), - }, + failure: { code: "sidecar_initialization_failed", message }, + ...(firstDeployFailedAt !== undefined + ? { firstDeployFailedAt } + : {}), now: now(), }), ); @@ -526,9 +877,9 @@ export function createSidecarAllocationReconciler({ // A connect that lands during initialization schedules an immediate // follow-up even on success: the new socket may be a restarted worker // with an empty inventory (takeover suppresses the disconnect event), in - // which case the follow-up redeploys and restores it. When the worker is - // unchanged the follow-up is a no-op: deployReadyAllocation returns early - // once the workflow is active and its key is recorded. + // which case the follow-up fails the deployment it no longer holds. When + // the worker is unchanged the follow-up is a no-op: deployReadyAllocation + // returns early once the workflow is active and its key is recorded. await finishReconciliation(allocation.id, (pendingConnect) => pendingConnect ? allocationStore.scheduleRetry({ @@ -575,6 +926,7 @@ export function createSidecarAllocationReconciler({ sidecarId, token, hubWebSocketUrl, + ...(adapterManifest !== undefined ? { adapterManifest } : {}), }), operationTimeoutMs, ), @@ -590,6 +942,8 @@ export function createSidecarAllocationReconciler({ return; } if (result.kind === "rejected") { + // A rejected ensure started nothing. + await allocationStore.retireSidecar({ sidecarId, now: now() }); if (!result.retryable) { const failed = await allocationStore.failWithoutInfrastructure({ allocationId: allocation.id, @@ -617,15 +971,110 @@ export function createSidecarAllocationReconciler({ return; } - const allocated = await allocationStore.markAllocated({ - allocationId: allocation.id, - generation: allocation.generation, - ...(result.externalRef !== undefined - ? { externalRef: result.externalRef } - : {}), - expectedLeaseId: leaseId, - now: now(), - }); + await acceptPlacement(allocation, leaseId, result); + } + + async function failInventoryPlacement( + allocation: SidecarAllocation, + leaseId: string, + ): Promise { + await queueReconciliationStep( + { allocationId: allocation.id, generation: allocation.generation }, + async () => { + const releasing = await allocationStore.beginUnrecoverableRelease({ + allocationId: allocation.id, + expectedStatus: "provisioning", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + failureCode: "sidecar_inventory_unavailable", + failureMessage: + "Sidecar inventory did not become available before the connection deadline", + now: now(), + }); + if (releasing !== null) { + trackAllocation(releasing); + fence(releasing); + } + }, + ); + } + + async function acceptPlacement( + allocation: SidecarAllocation, + leaseId: string, + result: Extract, + ): Promise { + const pending = pendingPlacements.get(allocation.id); + if ( + pending !== undefined && + pending.deadline <= now() && + pending.waiter?.ready !== true + ) { + await failInventoryPlacement(allocation, leaseId); + return; + } + + let allocated: SidecarAllocation | null; + try { + allocated = await allocationStore.markAllocated({ + allocationId: allocation.id, + generation: allocation.generation, + ...(result.sidecarId !== undefined + ? { sidecarId: result.sidecarId } + : {}), + ...(result.externalRef !== undefined + ? { externalRef: result.externalRef } + : {}), + expectedLeaseId: leaseId, + now: now(), + }); + } catch (error) { + if (error instanceof SidecarInventoryUnavailableError) { + if (allocation.connectDeadline === undefined) + throw new Error( + `Provisioning allocation ${allocation.id} has no connection deadline`, + ); + const placement = pending ?? { + generation: allocation.generation, + result, + deadline: allocation.connectDeadline, + waiter: undefined, + }; + pendingPlacements.set(allocation.id, placement); + if (placement.deadline <= now()) { + await failInventoryPlacement(allocation, leaseId); + return; + } + watchPlacementInventory(allocation.id, placement, error.sidecarId); + await finishReconciliation(allocation.id, () => + allocationStore.scheduleRetry({ + allocationId: allocation.id, + expectedStatus: "provisioning", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + // A hello before this write must not be overwritten by the deadline. + // A hello during it queues its wake behind this same event lane. + nextAttemptAt: + placement.waiter?.ready === true ? now() : placement.deadline, + // Postgres may consider the deadline due before the Hub does. + // Keep those early claims at least a second apart on its clock. + ...(placement.waiter?.ready === true + ? {} + : { minimumDelayMs: 1_000 }), + now: now(), + }), + ); + return; + } + if (!(error instanceof SidecarReuseRejectedError)) throw error; + await replaceAfterFailure( + allocation, + leaseId, + "sidecar_reuse_rejected", + error.message, + ); + return; + } if (allocated !== null) { trackAllocation(allocated); if (await isSidecarReady(allocated)) { @@ -633,7 +1082,7 @@ export function createSidecarAllocationReconciler({ return; } // Provisioning acceptance and websocket readiness are separate durable - // transitions. Do not hold the single reconciliation loop for the full + // transitions. Do not hold a reconciliation slot for the full // connection timeout: park this lease at its persisted deadline and let // sidecar.allocated.connected wake it immediately when the worker arrives. await finishReconciliation(allocated.id, (pendingConnect) => @@ -660,15 +1109,30 @@ export function createSidecarAllocationReconciler({ provisioner: SidecarProvisioner, replacement: boolean, ): Promise { + if ( + !(await allocationStore.hasRunnableAnchor(allocation.anchorRunId, now())) + ) { + const releasing = await allocationStore.beginRelease({ + allocationId: allocation.id, + expectedStatus: replacement ? "replacing" : "pending", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + now: now(), + }); + if (releasing !== null) fence(releasing); + return; + } const token = createToken(); const sidecarId = createSidecarId(); const connectDeadline = new Date(now().getTime() + connectTimeoutMs); + const digest = await currentAdapterManifestDigest; const bound = replacement ? await allocationStore.bindReplacementSidecar({ allocationId: allocation.id, generation: allocation.generation, sidecarId, tokenHashSha256: await sha256(token), + adapterManifestDigest: digest, connectDeadline, expectedLeaseId: leaseId, now: now(), @@ -678,6 +1142,7 @@ export function createSidecarAllocationReconciler({ expectedGeneration: allocation.generation, sidecarId, tokenHashSha256: await sha256(token), + adapterManifestDigest: digest, connectDeadline, expectedLeaseId: leaseId, now: now(), @@ -685,13 +1150,16 @@ export function createSidecarAllocationReconciler({ if (bound === null) return; trackAllocation(bound); - router.fenceAllocation(bound.id, bound.generation); + fence(bound); await acceptEnsure(bound, leaseId, provisioner, token); } async function retryDestroy( allocation: SidecarAllocation, leaseId: string, + cleanup: CleanupProgress, + failure: { code: string; message: string }, + retryDeadline?: Date, ): Promise { if ( allocation.status !== "replacing" && @@ -702,12 +1170,36 @@ export function createSidecarAllocationReconciler({ ); } const status = allocation.status; + const outcome = cleanup.confirmed + ? "Deployment removal is confirmed, but the provisioner hold remains unreleased" + : "Deployment removal remains unconfirmed"; + logger.warn`Cleanup attempt ${String(allocation.destroyAttempts + 1)} failed for allocation ${allocation.id} on sidecar ${allocation.sidecarId ?? "unknown"} generation ${String(allocation.generation)}: ${failure.code}: ${failure.message}`; + if (allocation.destroyAttempts + 1 >= maxCleanupAttempts) { + await finishReconciliation(allocation.id, async () => { + const failed = await allocationStore.markDestroyFailed({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + code: SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE, + message: `${outcome} after ${String(allocation.destroyAttempts + 1)} attempts: ${failure.message}`, + now: now(), + }); + if (failed !== null) + logger.error`Cleanup retry budget exhausted for allocation ${allocation.id} on sidecar ${allocation.sidecarId ?? "unknown"} generation ${String(allocation.generation)}; operator recovery is required. ${outcome}. Cleanup error: ${failure.message}. Prior allocation reason: ${allocation.failureCode ?? "none"} ${allocation.failureMessage ?? ""}`; + }); + return; + } await finishReconciliation(allocation.id, () => allocationStore.scheduleRetry({ allocationId: allocation.id, expectedStatus: status, expectedGeneration: allocation.generation, - nextAttemptAt: retryAt(allocation.destroyAttempts), + nextAttemptAt: new Date( + Math.min( + retryAt(allocation.destroyAttempts).getTime(), + retryDeadline?.getTime() ?? Infinity, + ), + ), expectedLeaseId: leaseId, attempt: "destroy", now: now(), @@ -715,12 +1207,299 @@ export function createSidecarAllocationReconciler({ ); } + type CleanupProgress = { + confirmed: boolean; + providerHoldReleased: boolean; + requestFailed: boolean; + preparationFailed: boolean; + attemptedConnections: Set; + message: string; + }; + + async function retryCleanupWithoutAttempt( + allocation: SidecarAllocation, + leaseId: string, + cleanup: CleanupProgress, + delayMs: number, + ): Promise { + await finishReconciliation(allocation.id, () => { + const reconnected = + !cleanup.confirmed && + activeAllocations.get(allocation.id)?.pendingConnect?.generation === + allocation.generation; + return allocationStore.scheduleRetry({ + allocationId: allocation.id, + expectedStatus: + allocation.status === "replacing" ? "replacing" : "releasing", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + nextAttemptAt: new Date(now().getTime() + (reconnected ? 0 : delayMs)), + ...(reconnected ? {} : { minimumDelayMs: delayMs }), + now: now(), + }); + }); + } + + async function confirmCleanup( + allocation: SidecarAllocation, + leaseId: string, + cleanup: CleanupProgress, + ): Promise { + if (cleanup.confirmed) return; + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + await queueReconciliationStep(target, async () => { + if ( + !(await allocationStore.confirmDeploymentCleanup({ + ...target, + expectedLeaseId: leaseId, + now: now(), + })) + ) + throw new ReconciliationLeaseLostError(allocation.id); + cleanup.confirmed = true; + cleanupPreparationDeadlines.delete(allocation.id); + }); + } + + async function cleanUpConnectedSidecar( + allocation: SidecarAllocation, + leaseId: string, + cleanup: CleanupProgress, + ): Promise { + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + if (cleanup.confirmed) return; + if (allocation.sidecarId === undefined) return; + const sidecarId = allocation.sidecarId; + const connectionBeforeSync = router.getCleanupConnection(target); + // Retry failed synchronization on the next pass; use a connection that + // attached during provider cleanup without repeating the failed sync. + if (connectionBeforeSync === undefined && cleanup.preparationFailed) return; + let connection: ReturnType; + let sent = false; + try { + const acknowledged = await withReconciliationLease( + allocation, + leaseId, + "Sidecar deployment cleanup", + ({ signal }) => + trackAllocationQuery(allocation.id, async () => { + if (router.getCleanupConnection(target) === undefined) + await router.syncSidecar(sidecarId, signal); + signal.throwIfAborted(); + // Select the connection after its wake finishes, so a wake already + // handled by this attempt cannot bypass the retry backoff. + await queueReconciliationStep(target, async () => { + signal.throwIfAborted(); + const current = router.getCleanupConnection(target); + if (current === undefined) return; + const active = activeAllocations.get(allocation.id); + if (active !== undefined) active.pendingConnect = null; + if (cleanup.attemptedConnections.has(current)) return; + connection = current; + cleanup.attemptedConnections.add(current); + }); + if (connection === undefined) return false; + if (allocation.connectDeadline !== undefined) { + const connected = await allocationStore + .recordCleanupConnection({ + allocationId: allocation.id, + generation: allocation.generation, + expectedLeaseId: leaseId, + connected: true, + now: now(), + }) + .catch((cause: unknown) => { + // The outer catch handles failed sidecar requests. A failed + // database write must instead use the uncounted retry path. + if (cause instanceof ReconciliationLeaseLostError) + throw cause; + throw new CleanupPersistenceError(cause); + }); + if (connected === null) + throw new ReconciliationLeaseLostError(allocation.id); + signal.throwIfAborted(); + } + await router.undeployAllocation( + target, + operationTimeoutMs, + signal, + connection, + () => { + sent = true; + cleanupPreparationDeadlines.delete(allocation.id); + }, + ); + return true; + }), + operationTimeoutMs, + ); + if (!acknowledged) return; + } catch (error) { + if ( + error instanceof ReconciliationLeaseLostError || + error instanceof CleanupPersistenceError + ) + throw error; + // Observe the send directly: the enclosing timeout can reject before + // the router's own promise settles, during validation or after sending. + const message = error instanceof Error ? error.message : String(error); + if (sent) { + cleanup.requestFailed = true; + cleanup.message = message; + } else { + cleanup.preparationFailed = true; + if (!cleanup.requestFailed) cleanup.message = message; + } + logger.warn`${sent ? "Sidecar cleanup request failed" : "Could not prepare sidecar cleanup"} for allocation ${allocation.id} on sidecar ${sidecarId} generation ${String(allocation.generation)}: ${message}`; + if (connection === undefined && connectionBeforeSync !== undefined) { + // A failed setup handled this connection's wake too. Preserve a + // replacement that arrived during the failure for the next attempt. + await queueReconciliationStep(target, async () => { + const active = activeAllocations.get(allocation.id); + if ( + router.getCleanupConnection(target) === connectionBeforeSync && + active?.allocation.generation === allocation.generation + ) { + active.pendingConnect = null; + } + }); + } + return; + } + // Persistence failures use the reconciliation retry; they are not failed + // undeploys and must not consume the cleanup attempt budget. + await confirmCleanup(allocation, leaseId, cleanup); + } + + async function waitForCleanupConnection( + allocation: SidecarAllocation, + leaseId: string, + ): Promise { + await finishReconciliation(allocation.id, async () => { + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + let deadline: Date | undefined; + // This check and the park share the connection-event lane: a reconnect + // observed during the write schedules an immediate claim after it. + if (router.getCleanupConnection(target) === undefined) { + const waiting = await allocationStore.recordCleanupConnection({ + ...target, + expectedLeaseId: leaseId, + connected: false, + now: now(), + }); + if (waiting === null) return; + // Registration can finish while the deadline write waits on the database. + // Let the next claim use that connection instead of failing a live peer. + if (router.getCleanupConnection(target) === undefined) { + deadline = waiting.connectDeadline; + if (deadline === undefined) + throw new Error("Disconnected cleanup has no connection deadline"); + } + } + if (deadline !== undefined && deadline <= now()) { + // Bound disconnected cleanup waits even when removal remains uncertain. + // Only a cleanup reconnect resumes this failure. Later provider destruction, + // including release of a shared sidecar's last hold, can leave this row + // failed after the copy is gone because we no longer poll the provider. + const failed = await allocationStore.markDestroyFailed({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + code: SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + message: `Sidecar did not reconnect before the cleanup deadline ${deadline.toISOString()}`, + countAttempt: false, + now: now(), + }); + if (failed !== null) + logger.error`Cleanup connection deadline ${deadline.toISOString()} expired for allocation ${allocation.id} on sidecar ${allocation.sidecarId ?? "unknown"} generation ${String(allocation.generation)}; capacity remains reserved. Prior allocation reason: ${allocation.failureCode ?? "none"} ${allocation.failureMessage ?? ""}`; + return; + } + await allocationStore.scheduleRetry({ + allocationId: allocation.id, + expectedStatus: + allocation.status === "replacing" ? "replacing" : "releasing", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + nextAttemptAt: deadline ?? now(), + ...(deadline === undefined ? {} : { minimumDelayMs: 1_000 }), + now: now(), + }); + }); + } + + async function finishDeploymentCleanup( + allocation: SidecarAllocation, + leaseId: string, + cleanup: CleanupProgress, + ): Promise { + if (cleanup.confirmed) { + cleanupPreparationDeadlines.delete(allocation.id); + return true; + } + if (!cleanup.requestFailed && cleanup.preparationFailed) { + const observedAt = now(); + const deadline = + cleanupPreparationDeadlines.get(allocation.id) ?? + new Date(observedAt.getTime() + CLEANUP_PREPARATION_GRACE_MS); + cleanupPreparationDeadlines.set(allocation.id, deadline); + if (deadline <= observedAt) { + await finishReconciliation(allocation.id, async () => { + const failed = await allocationStore.markDestroyFailed({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + code: SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE, + message: `Could not prepare deployment cleanup before ${deadline.toISOString()}: ${cleanup.message}`, + countAttempt: false, + now: observedAt, + }); + if (failed !== null) { + cleanupPreparationDeadlines.delete(allocation.id); + logger.error`Cleanup preparation grace expired for allocation ${allocation.id} on sidecar ${allocation.sidecarId ?? "unknown"} generation ${String(allocation.generation)}; operator recovery is required: ${cleanup.message}`; + } + }); + return false; + } + await retryCleanupWithoutAttempt( + allocation, + leaseId, + cleanup, + Math.min( + retryDelayMs(MAX_RETRY_BACKOFF_ATTEMPT), + deadline.getTime() - observedAt.getTime(), + ), + ); + return false; + } + if (!cleanup.requestFailed) { + // Disconnecting is not cleanup progress and must not restart the grace. + await waitForCleanupConnection(allocation, leaseId); + return false; + } + await retryDestroy(allocation, leaseId, cleanup, { + code: SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + message: cleanup.message, + }); + return false; + } + async function destroyCurrent( allocation: SidecarAllocation, leaseId: string, provisioner: SidecarProvisioner, + cleanup: CleanupProgress, ): Promise { if (allocation.sidecarId === undefined) return true; + if (cleanup.providerHoldReleased && cleanup.confirmed) return true; const sidecarId = allocation.sidecarId; let result: DestroySidecarResult; try { @@ -744,37 +1523,227 @@ export function createSidecarAllocationReconciler({ ); } catch (error) { if (error instanceof ReconciliationLeaseLostError) throw error; - logger.warn`Destroy failed for allocation ${allocation.id}: ${error instanceof Error ? error.message : String(error)}`; - await retryDestroy(allocation, leaseId); + await cleanUpConnectedSidecar(allocation, leaseId, cleanup); + if (cleanup.providerHoldReleased) { + logger.warn`Could not confirm remaining deployment cleanup for allocation ${allocation.id} after provider release: ${error instanceof Error ? error.message : String(error)}`; + return finishDeploymentCleanup(allocation, leaseId, cleanup); + } + await retryDestroy(allocation, leaseId, cleanup, { + code: cleanup.confirmed + ? SIDECAR_PROVISIONER_RELEASE_FAILURE_CODE + : SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + message: error instanceof Error ? error.message : String(error), + }); return false; } - if (result.kind === "destroyed") return true; + // Retire before recording the release: once both cleanup flags are set, + // no later pass reaches this point. + if (result.kind === "destroyed" && result.cleanup === "confirmed") + await allocationStore.retireSidecar({ sidecarId, now: now() }); + if (result.kind === "destroyed" && !cleanup.providerHoldReleased) { + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + await queueReconciliationStep(target, async () => { + if ( + !(await allocationStore.confirmProviderRelease({ + ...target, + expectedLeaseId: leaseId, + now: now(), + })) + ) + throw new ReconciliationLeaseLostError(allocation.id); + cleanup.providerHoldReleased = true; + }); + } + if (result.kind === "destroyed" && result.cleanup === "confirmed") + await confirmCleanup(allocation, leaseId, cleanup); + if ( + result.kind === "destroyed" && + (result.cleanup === "confirmed" || cleanup.confirmed) + ) + return true; + + // A sidecar may have connected during destruction, including a failed + // provider call. Try that connection without repeating an earlier request. + await cleanUpConnectedSidecar(allocation, leaseId, cleanup); + if (result.kind === "destroyed" || cleanup.providerHoldReleased) { + if (result.kind === "rejected") + logger.warn`Could not confirm remaining deployment cleanup for allocation ${allocation.id} after provider release: ${result.code}: ${result.message}`; + return finishDeploymentCleanup(allocation, leaseId, cleanup); + } if (!result.retryable) { + logger.error`Provisioner permanently rejected cleanup for allocation ${allocation.id} on sidecar ${sidecarId} generation ${String(allocation.generation)}: ${result.code} ${result.message}. Prior allocation reason: ${allocation.failureCode ?? "none"} ${allocation.failureMessage ?? ""}`; const failed = await allocationStore.markDestroyFailed({ allocationId: allocation.id, expectedGeneration: allocation.generation, expectedLeaseId: leaseId, - code: result.code, + // Keep provider rejections distinct from Hub cleanup failures; + // only Hub disconnect timeouts may resume on reconnect. + code: + result.code === SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE || + result.code === SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE || + result.code === SIDECAR_CLEANUP_PREPARATION_TIMEOUT_FAILURE_CODE + ? "sidecar_provisioner_cleanup_rejected" + : result.code, message: result.message, now: now(), }); + if (failed !== null) fence(failed); + return false; + } + await retryDestroy(allocation, leaseId, cleanup, { + code: cleanup.confirmed + ? SIDECAR_PROVISIONER_RELEASE_FAILURE_CODE + : SIDECAR_CLEANUP_UNCONFIRMED_FAILURE_CODE, + message: result.message, + }); + return false; + } + + async function completeRelease( + allocation: SidecarAllocation, + leaseId: string, + ): Promise { + const released = await allocationStore.markReleased({ + allocationId: allocation.id, + generation: allocation.generation, + expectedLeaseId: leaseId, + now: now(), + }); + if (released !== null) { + cleanupPreparationDeadlines.delete(allocation.id); + router.retireAllocation({ + allocationId: released.id, + generation: released.generation, + }); + } + } + + async function reconcileCleanup( + allocation: SidecarAllocation, + leaseId: string, + ): Promise { + if ( + allocation.status === "replacing" && + allocation.ensureAttempts >= MAX_ENSURE_ATTEMPTS && + (await allocationStore.hasRunnableAnchor(allocation.anchorRunId, now())) + ) { + // beginReplacement already reset the budget if the previous worker + // initialized. Record exhaustion before cleanup can fail for another cause. + const lastFailure = allocation.failureMessage ?? allocation.failureCode; + const message = `Provisioning failed after ${String(allocation.ensureAttempts)} attempts${lastFailure === undefined ? "" : `: ${lastFailure}`}`; + const target = { + allocationId: allocation.id, + expectedStatus: "replacing" as const, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + now: now(), + }; + const failed = + allocation.deploymentCleanupConfirmed && allocation.providerHoldReleased + ? await allocationStore.failWithoutInfrastructure({ + ...target, + code: PROVISIONING_RETRY_EXHAUSTED, + message, + }) + : await allocationStore.beginUnrecoverableRelease({ + ...target, + failureCode: PROVISIONING_RETRY_EXHAUSTED, + failureMessage: message, + }); if (failed !== null) { - router.retireAllocation({ - allocationId: failed.id, - generation: failed.generation, + logger.error`Provisioning retry budget exhausted for allocation ${allocation.id} after ${String(allocation.ensureAttempts)} attempts. Last failure: ${lastFailure ?? "No failure reason recorded"}`; + if (failed.status === "failed") + router.retireAllocation({ + allocationId: failed.id, + generation: failed.generation, + }); + else fence(failed); + } + return; + } + if ( + allocation.status === "releasing" && + allocation.sidecarId === undefined + ) { + await completeRelease(allocation, leaseId); + return; + } + const cleanup: CleanupProgress = { + confirmed: allocation.deploymentCleanupConfirmed, + providerHoldReleased: allocation.providerHoldReleased, + requestFailed: false, + preparationFailed: false, + attemptedConnections: new Set(), + message: "No sidecar connection is available to confirm cleanup", + }; + // Stopping a connected deployment must not depend on provider availability. + await cleanUpConnectedSidecar(allocation, leaseId, cleanup); + if ( + allocation.status === "releasing" && + cleanup.providerHoldReleased && + cleanup.confirmed + ) { + await completeRelease(allocation, leaseId); + return; + } + const selected = provisionerFor(allocation); + if ("error" in selected) { + if (allocation.status === "releasing" && cleanup.providerHoldReleased) { + await finishDeploymentCleanup(allocation, leaseId, cleanup); + return; + } + if ( + await waitForProvisioner(allocation, leaseId, selected.error, cleanup) + ) + return; + logger.error`Provisioner availability grace expired for ${allocation.status} allocation ${allocation.id} on sidecar ${allocation.sidecarId ?? "unknown"} generation ${String(allocation.generation)}: ${selected.error}. Prior allocation reason: ${allocation.failureCode ?? "none"} ${allocation.failureMessage ?? ""}`; + if (cleanup.providerHoldReleased && cleanup.confirmed) { + const failed = await allocationStore.failWithoutInfrastructure({ + allocationId: allocation.id, + expectedStatus: "replacing", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + code: "provisioner_unavailable", + message: selected.error, + now: now(), }); + if (failed !== null) + router.retireAllocation({ + allocationId: failed.id, + generation: failed.generation, + }); + } else { + const failed = await allocationStore.markDestroyFailed({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + code: "provisioner_unavailable", + message: selected.error, + countAttempt: cleanup.requestFailed, + now: now(), + }); + if (failed !== null) fence(failed); } - return false; + return; } - await retryDestroy(allocation, leaseId); - return false; + const { provisioner } = selected; + if (!(await destroyCurrent(allocation, leaseId, provisioner, cleanup))) + return; + if (allocation.status === "replacing") { + await bindAndEnsure(allocation, leaseId, provisioner, true); + return; + } + await completeRelease(allocation, leaseId); } async function reconcile( allocation: SidecarAllocation, leaseId: string, ): Promise { - router.fenceAllocation(allocation.id, allocation.generation); + fence(allocation); if ( allocation.status === "released" || allocation.status === "failed" || @@ -782,6 +1751,13 @@ export function createSidecarAllocationReconciler({ ) { return; } + if ( + allocation.status === "releasing" || + allocation.status === "replacing" + ) { + await reconcileCleanup(allocation, leaseId); + return; + } if ( allocation.status === "allocated" && onInitializationRecovery !== undefined @@ -797,49 +1773,43 @@ export function createSidecarAllocationReconciler({ operationTimeoutMs, ); } - const provisioner = provisionerFor(allocation); - if (provisioner === null) { - if (allocation.status === "pending") { - const failed = await allocationStore.failWithoutInfrastructure({ - allocationId: allocation.id, - expectedStatus: "pending", - expectedGeneration: allocation.generation, - code: "provisioner_unavailable", - message: `Provisioner ${allocation.provisionerId} is unavailable or its binding changed`, - expectedLeaseId: leaseId, - now: now(), - }); - if (failed !== null) { - router.retireAllocation({ - allocationId: failed.id, - generation: failed.generation, - }); - } - } else { - const status = allocation.status; - await finishReconciliation(allocation.id, () => - allocationStore.scheduleRetry({ + const pendingPlacement = pendingPlacements.get(allocation.id); + if ( + allocation.status === "provisioning" && + pendingPlacement !== undefined + ) { + await acceptPlacement(allocation, leaseId, pendingPlacement.result); + return; + } + switch (allocation.status) { + case "pending": { + const selected = provisionerFor(allocation); + if ("error" in selected) { + if (await waitForProvisioner(allocation, leaseId, selected.error)) + return; + const failed = await allocationStore.failWithoutInfrastructure({ allocationId: allocation.id, - expectedStatus: status, + expectedStatus: "pending", expectedGeneration: allocation.generation, - nextAttemptAt: retryAt( - allocation.ensureAttempts + allocation.destroyAttempts, - ), + code: "provisioner_unavailable", + message: selected.error, expectedLeaseId: leaseId, now: now(), - }), - ); - } - return; - } - - switch (allocation.status) { - case "pending": - await bindAndEnsure(allocation, leaseId, provisioner, false); + }); + if (failed !== null) { + router.retireAllocation({ + allocationId: failed.id, + generation: failed.generation, + }); + } + return; + } + await bindAndEnsure(allocation, leaseId, selected.provisioner, false); return; - case "provisioning": - // The raw bearer token is deliberately not durable. Re-entering this - // state means the process died before ensure acceptance was recorded. + } + case "provisioning": { + // The raw bearer token and unrecorded acceptance are not durable. A + // restart has no parked result to resume, so it fences uncertain work. await replaceAfterFailure( allocation, leaseId, @@ -847,6 +1817,7 @@ export function createSidecarAllocationReconciler({ "Hub restarted before sidecar provisioning acceptance was recorded", ); return; + } case "allocated": if (allocation.initializationLeaseId !== undefined) { await replaceAfterFailure( @@ -860,26 +1831,6 @@ export function createSidecarAllocationReconciler({ } await waitUntilReady(allocation, leaseId); return; - case "replacing": - if (!(await destroyCurrent(allocation, leaseId, provisioner))) return; - await bindAndEnsure(allocation, leaseId, provisioner, true); - return; - case "releasing": { - if (!(await destroyCurrent(allocation, leaseId, provisioner))) return; - const released = await allocationStore.markReleased({ - allocationId: allocation.id, - generation: allocation.generation, - expectedLeaseId: leaseId, - now: now(), - }); - if (released !== null) { - router.retireAllocation({ - allocationId: released.id, - generation: released.generation, - }); - } - return; - } default: { const exhaustive: never = allocation.status; throw new Error( @@ -890,16 +1841,18 @@ export function createSidecarAllocationReconciler({ } async function initialize(): Promise { + const startedAt = now(); for (const allocation of await allocationStore.listActive()) { - router.fenceAllocation(allocation.id, allocation.generation); + fence(allocation); + if (allocation.status === "destroy_failed") continue; if (allocation.status === "allocated") { - await allocationStore.markConnectionLost({ + await allocationStore.scheduleReconnectAfterHubStart({ allocationId: allocation.id, generation: allocation.generation, - connectDeadline: - allocation.connectDeadline ?? - new Date(now().getTime() + connectTimeoutMs), - now: now(), + now: startedAt, + firstConnectDeadline: new Date( + startedAt.getTime() + connectTimeoutMs, + ), }); } else if (allocation.nextAttemptAt === undefined) { // Scheduled retries are durable state. Only repair an unscheduled @@ -988,11 +1941,12 @@ export function createSidecarAllocationReconciler({ function handleDisconnect(target: AllocatedSidecarTarget): Promise { noteDisconnect(target); return queueConnectionEvent(target, async () => { + const lostAt = now(); const disconnected = await allocationStore.markConnectionLost({ allocationId: target.allocationId, generation: target.generation, - connectDeadline: new Date(now().getTime() + connectTimeoutMs), - now: now(), + now: lostAt, + firstConnectDeadline: new Date(lostAt.getTime() + connectTimeoutMs), }); noteDisconnect(target, disconnected !== null); }); @@ -1001,16 +1955,106 @@ export function createSidecarAllocationReconciler({ function handleConnected(target: AllocatedSidecarTarget): Promise { noteConnect(target); return queueConnectionEvent(target, async () => { + if ( + router.getCleanupConnection(target) !== undefined && + (await allocationStore.resumeCleanupAfterDisconnect( + target.allocationId, + target.generation, + )) + ) { + noteConnect(target); + return; + } await allocationStore.wakeReconciliation( target.allocationId, target.generation, + { + preserveRetryBackoff: true, + connected: + router.getCleanupConnection(target) !== undefined || + router.holdsAllocatedBinding(target), + }, ); noteConnect(target); }); } async function repairUnscheduledConnections(): Promise { - for (const allocation of await allocationStore.listActive()) { + const parked = [...pendingPlacements]; + const unavailable = [...provisionerUnavailableDeadlines.keys()]; + const preparing = [...cleanupPreparationDeadlines.keys()]; + const allocations = await allocationStore.listForConnectionRepair(); + const current = new Map( + allocations.map((allocation) => [allocation.id, allocation]), + ); + for (const id of unavailable) { + const allocation = current.get(id); + if (allocation === undefined || allocation.status === "destroy_failed") + provisionerUnavailableDeadlines.delete(id); + } + for (const id of preparing) { + const allocation = current.get(id); + if ( + allocation === undefined || + allocation.deploymentCleanupConfirmed || + (allocation.status !== "releasing" && allocation.status !== "replacing") + ) + cleanupPreparationDeadlines.delete(id); + } + for (const [allocationId, pending] of parked) { + if (pendingPlacements.get(allocationId) !== pending) continue; + const allocation = current.get(allocationId); + if (allocation === undefined) discardPlacement(allocationId); + else { + discardChangedPlacement(allocation); + if ( + pendingPlacements.get(allocationId) === pending && + pending.waiter?.ready === true + ) { + try { + await wakePendingPlacement(allocationId, pending); + } catch (error) { + logger.warn`Failed to repair inventory placement ${allocationId}: ${error instanceof Error ? error.message : String(error)}`; + } + } + } + } + for (const allocation of allocations) { + const parkedCleanup = + (allocation.status === "releasing" || + allocation.status === "replacing") && + allocation.connectDeadline !== undefined && + allocation.nextAttemptAt?.getTime() === + allocation.connectDeadline.getTime() + ? { + deadline: allocation.connectDeadline, + destroyAttempts: allocation.destroyAttempts, + } + : undefined; + if ( + parkedCleanup !== undefined || + (allocation.status === "destroy_failed" && + allocation.failureCode === + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE) + ) { + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + try { + await queueConnectionEvent(target, async () => { + if (router.getCleanupConnection(target) !== undefined) + await allocationStore.resumeCleanupAfterDisconnect( + target.allocationId, + target.generation, + parkedCleanup, + ); + }); + } catch (error) { + logger.warn`Failed to resume disconnected cleanup for allocation ${allocation.id}: ${error instanceof Error ? error.message : String(error)}`; + } + continue; + } if ( allocation.status !== "allocated" || allocation.nextAttemptAt !== undefined || @@ -1023,8 +2067,9 @@ export function createSidecarAllocationReconciler({ allocationId: allocation.id, generation: allocation.generation, }; + let ready: boolean; try { - if (await router.isAllocatedSidecarReady(target)) continue; + ready = await router.isAllocatedSidecarReady(target); } catch (error) { // Unknown readiness is not absence. Leave the allocation for the next // repair sweep instead of scheduling a reconnect the worker may hold. @@ -1032,15 +2077,34 @@ export function createSidecarAllocationReconciler({ throw error; } try { - await allocationStore.scheduleReconnectIfUnscheduled({ - ...target, - connectDeadline: - allocation.connectDeadline ?? - new Date(now().getTime() + connectTimeoutMs), - now: now(), - }); + if (!ready) { + const repairedAt = now(); + await allocationStore.scheduleReconnectIfUnscheduled({ + ...target, + now: repairedAt, + firstConnectDeadline: new Date( + repairedAt.getTime() + connectTimeoutMs, + ), + }); + } else if ( + router.reportedDeploymentFailure(target) !== undefined && + allocation.failureCode !== SIDECAR_DEPLOYMENT_STOPPED_FAILURE_CODE && + (await allocationStore.hasRunnableAnchor( + allocation.anchorRunId, + now(), + )) + ) { + // A stop the sidecar reported is acted on by one wake. If that wake + // was lost, the allocation rests on its ready connection with the + // stop not yet recorded. Once the anchor can no longer run, a wake + // has nothing left to fail, so none is made. + await allocationStore.wakeReconciliation( + allocation.id, + allocation.generation, + ); + } } catch (error) { - logger.warn`Failed to repair allocation ${allocation.id} reconnect schedule: ${error instanceof Error ? error.message : String(error)}`; + logger.warn`Failed to repair allocation ${allocation.id} schedule: ${error instanceof Error ? error.message : String(error)}`; } } } @@ -1169,28 +2233,40 @@ export function createSidecarAllocationReconciler({ } }; checkLeaseExpiry(); + // The durable schedule survives the claim. Stop renewing and let the + // lease expire; handling a lease failure must not require another write. + const stopForLeaseLoss = (error: ReconciliationLeaseLostError): void => { + const cause = error.cause; + if (cause === undefined) { + logger.info`Allocation ${allocation.id} reconciliation stopped: lease ${leaseId} is no longer current`; + } else { + logger.warn`Allocation ${allocation.id} reconciliation stopped because lease ${leaseId} could not be confirmed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + }; try { active.controller.signal.throwIfAborted(); await reconcile(allocation, leaseId); } catch (error) { if (error instanceof ReconciliationLeaseLostError) { - const cause = error.cause; - if (cause === undefined) { - logger.info`Allocation ${allocation.id} reconciliation stopped: lease ${leaseId} is no longer current`; - } else { - logger.warn`Allocation ${allocation.id} reconciliation stopped because lease ${leaseId} could not be confirmed: ${cause instanceof Error ? cause.message : String(cause)}`; - } - // The durable schedule survives the claim. Stop renewing and let the - // lease expire; handling a lease failure must not require another write. + stopForLeaseLoss(error); return true; } logger.error`Allocation ${allocation.id} reconciliation failed: ${error instanceof Error ? error.message : String(error)}`; - await finishReconciliation(allocation.id, () => - allocationStore.parkReconciliation(allocation.id, leaseId, { - kind: "retry-after-error", - notBefore: retryAt(MAX_RETRY_BACKOFF_ATTEMPT), - }), - ); + try { + await finishReconciliation(allocation.id, () => + allocationStore.parkReconciliation(allocation.id, leaseId, { + kind: "retry-after-error", + notBefore: retryAt(MAX_RETRY_BACKOFF_ATTEMPT), + }), + ); + } catch (parkError) { + // The park waits behind the same stalled query. Losing the lease + // here is the same stop as above: another write cannot clear it. + if (!(parkError instanceof ReconciliationLeaseLostError)) { + throw parkError; + } + stopForLeaseLoss(parkError); + } } finally { finished = true; clearInterval(heartbeat); diff --git a/vendor/intx/hub-sessions/src/substrate.ts b/vendor/intx/hub-sessions/src/substrate.ts index 7d29aa1d3c..db564aae8d 100644 --- a/vendor/intx/hub-sessions/src/substrate.ts +++ b/vendor/intx/hub-sessions/src/substrate.ts @@ -30,6 +30,7 @@ export { requireEventSeq, StaleInboxEnqueueError, WORKFLOW_RUN_AGENT_STATE_PREFIX, + WORKFLOW_RUN_GITIGNORE_PATH, WORKFLOW_RUN_PARTS_DIR, WORKFLOW_RUN_EVENTS_DIR, WORKFLOW_RUN_RUNS_PREFIX, @@ -56,8 +57,6 @@ export { WORKFLOW_RUN_EVENTS_FILE, } from "./workflow-run-event-log"; -export { workflowDefinitionEnvelopeSchema } from "./workflow-kind"; - export { subscribeKind } from "./repo-store/subscribe-kind"; export type { SubscribeKindEntry } from "./repo-store/subscribe-kind"; @@ -73,3 +72,5 @@ export type { InitRepoOpts, NewlyTerminalRun, } from "./repo-store/types"; + +export { WORKFLOW_RUN_RESTORE_REFS } from "./workflow-run-restore"; diff --git a/vendor/intx/hub-sessions/src/workflow-allocation-service.ts b/vendor/intx/hub-sessions/src/workflow-allocation-service.ts index b16c9be78c..768ff4e5a9 100644 --- a/vendor/intx/hub-sessions/src/workflow-allocation-service.ts +++ b/vendor/intx/hub-sessions/src/workflow-allocation-service.ts @@ -6,8 +6,12 @@ import { createWorkflowProbeStore, createWorkflowRunLaunchSpecStore, resolveTenantSidecarCapabilityPolicies, + resolveDeploymentLifecyclePolicy, + canExecuteWorkflowRun, resolveSourcesByOfferingIds, + SidecarInventoryUnavailableError, type DB, + type DBExecutor, type SidecarAllocation, type WorkflowProbe, } from "@intx/db"; @@ -16,10 +20,17 @@ import { eq } from "drizzle-orm"; import { generateId } from "@intx/hub-common"; import { hexEncode, + lifecycleDeadline, + lifecycleDurationMs, SidecarCapabilityRule, + type AdapterManifest, type CredentialCipher, + type ResolvedWorkflowLifecyclePolicy, } from "@intx/types"; -import type { FrozenApprovalBundle } from "@intx/types/sidecar"; +import { + DeploymentRejectedError, + type FrozenApprovalBundle, +} from "@intx/types/sidecar"; import type { HarnessConfig } from "@intx/types/runtime"; import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; @@ -33,6 +44,7 @@ import { import type { DeployContent } from "./agent-repo"; import { + adapterManifestDigest, DestroySidecarResult, EnsureSidecarResult, type SidecarCapabilityMismatch, @@ -53,10 +65,20 @@ import type { InstallAndApproveResult } from "./workflow-probe-gate"; import { buildReferencedWorkflowSourcePins } from "./workflow-source-pins"; import { DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, + SidecarDeploymentHistoryPendingError, + SidecarDeploymentMissingError, + SidecarDeploymentStoppedError, + SidecarFirstDeployError, runSidecarOperation, + SidecarOperationTimeoutError, type SidecarReconciliationContext, } from "./sidecar-allocation/operation"; +// How long the Hub waits for history a deployment that stopped on its own +// committed before it fails the deployment without it, and how often it looks. +const STOPPED_HISTORY_WAIT_MS = 60_000; +const STOPPED_HISTORY_POLL_MS = 1_000; + export class WorkflowProvisioningError extends Error { readonly code: string; @@ -118,17 +140,31 @@ export type WorkflowAllocationServiceDeps = { readonly probeCapabilityRules?: readonly SidecarCapabilityRule[]; readonly allocationRouter: Pick< SidecarAllocationRouter, - | "disconnectAllocation" + | "detachAllocation" | "fenceAllocation" + | "getSidecarInventory" | "isAllocatedWorkflowActive" + | "reportedDeploymentFailure" + | "stoppedDeploymentHistory" | "retireAllocation" | "sendProbeToAllocation" + | "syncSidecar" | "waitForAllocatedSidecar" + | "waitForSidecarInventory" >; readonly hubWebSocketUrl: string; + /** Fills lifecycle fields no tenant or installed workflow sets. */ + readonly defaultLifecyclePolicy: ResolvedWorkflowLifecyclePolicy; + /** + * Operator-declared custom adapter manifest forwarded to every provisioned + * sidecar via the ensure request. Opaque here — the sidecar's adapter + * registry is the authority that admits `model_provider.plugin` values. + */ + readonly adapterManifest?: AdapterManifest; readonly createAllocationId?: () => string; readonly createSidecarId?: () => string; readonly createToken?: () => string; + /** One deadline for binding synchronization and connection readiness. */ readonly connectTimeoutMs?: number; readonly operationTimeoutMs?: number; readonly now?: () => Date; @@ -230,6 +266,8 @@ export function createWorkflowAllocationService({ probeCapabilityRules = [], allocationRouter, hubWebSocketUrl, + defaultLifecyclePolicy, + adapterManifest, createAllocationId = randomAllocationId, createSidecarId = randomSidecarId, createToken = randomToken, @@ -237,9 +275,12 @@ export function createWorkflowAllocationService({ operationTimeoutMs = DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, now = () => new Date(), }: WorkflowAllocationServiceDeps): WorkflowAllocationService { - const allocationStore = createSidecarAllocationStore(db); + const allocationStore = createSidecarAllocationStore(db, { + getSidecarInventory: allocationRouter.getSidecarInventory, + }); const probeStore = createWorkflowProbeStore(db); const launchSpecStore = createWorkflowRunLaunchSpecStore(db); + const currentAdapterManifestDigest = adapterManifestDigest(adapterManifest); const validatedProbeCapabilityRules = SidecarCapabilityRule.array()(probeCapabilityRules); @@ -348,7 +389,9 @@ export function createWorkflowAllocationService({ `Provisioner ${provisioner.id} rejected probe cleanup: ${destroyed.message}`, ); } - allocationRouter.disconnectAllocation({ + if (destroyed.cleanup === "confirmed") + await probeStore.retireSidecar({ sidecarId, now: now() }); + allocationRouter.detachAllocation({ allocationId: releasing.id, generation: releasing.generation, }); @@ -388,6 +431,7 @@ export function createWorkflowAllocationService({ provisioner: SidecarProvisioner; probe: WorkflowProbe; adoptProbe: boolean; + adapterManifestDigest: string; }): Promise { const { request, approved, provisioner, probe, adoptProbe } = args; const probeSidecarId = probe.sidecarId; @@ -395,7 +439,6 @@ export function createWorkflowAllocationService({ throw new Error(`Workflow probe ${probe.id} has no sidecar to adopt`); } const allocationId = adoptProbe ? probe.id : createAllocationId(); - const createdAt = now(); const deploymentAddress = deriveRunAddress({ runId: request.anchorRunId, domain: request.deploymentDomain, @@ -411,7 +454,14 @@ export function createWorkflowAllocationService({ ], }; - await db.transaction(async (tx) => { + const persistDeployment = async (tx: DBExecutor) => { + const createdAt = now(); + const lifecycle = await resolveDeploymentLifecyclePolicy( + tx, + request.tenantId, + approved.approval.definitionId, + defaultLifecyclePolicy, + ); await tx.insert(workflowRun).values({ id: request.anchorRunId, tenantId: request.tenantId, @@ -419,8 +469,11 @@ export function createWorkflowAllocationService({ definitionId: approved.approval.definitionId, address: deploymentAddress, status: "deployed", + lifecyclePolicy: lifecycle, + expiresAt: lifecycleDeadline(createdAt, lifecycle.maxLifetime), createdAt, }); + const maxDisconnectedMs = lifecycleDurationMs(lifecycle.maxDisconnected); await tx.insert(grant).values({ id: generateId("grant"), tenantId: request.tenantId, @@ -462,11 +515,13 @@ export function createWorkflowAllocationService({ provisionerApiVersion: provisioner.apiVersion, provisionerBindingFingerprint: provisioner.bindingFingerprint, sidecarId: probeSidecarId, + adapterManifestDigest: args.adapterManifestDigest, generation: probe.generation, ...(probe.externalRef !== null ? { externalRef: probe.externalRef } : {}), connectDeadline: new Date(createdAt.getTime() + connectTimeoutMs), + maxDisconnectedMs, now: createdAt, }, tx, @@ -489,19 +544,46 @@ export function createWorkflowAllocationService({ provisionerId: provisioner.id, provisionerApiVersion: provisioner.apiVersion, provisionerBindingFingerprint: provisioner.bindingFingerprint, + adapterManifestDigest: args.adapterManifestDigest, + maxDisconnectedMs, now: createdAt, }, tx, ); } - }); + }; - if (adoptProbe) { - allocationRouter.disconnectAllocation({ - allocationId: probe.id, - generation: probe.generation, - }); + const inventoryDeadline = performance.now() + connectTimeoutMs; + for (;;) { + try { + await db.transaction(persistDeployment); + break; + } catch (error) { + if (!adoptProbe || !(error instanceof SidecarInventoryUnavailableError)) + throw error; + // The creation transaction has rolled back before we wait. A timeout + // must never release the probe while an adoption can still commit. + const remaining = Math.ceil(inventoryDeadline - performance.now()); + const timeout = new SidecarOperationTimeoutError( + "Probe adoption inventory", + connectTimeoutMs, + { cause: error }, + ); + if (remaining <= 0) throw timeout; + try { + await runSidecarOperation( + "Probe adoption inventory", + remaining, + (signal) => + allocationRouter.waitForSidecarInventory(error.sidecarId, signal), + ); + } catch (cause) { + if (cause instanceof SidecarOperationTimeoutError) throw timeout; + throw cause; + } + } } + return { anchorRunId: request.anchorRunId, deploymentAddress, @@ -559,6 +641,7 @@ export function createWorkflowAllocationService({ }), ); const probeId = createAllocationId(); + const digest = await currentAdapterManifestDigest; let probe = await probeStore.create({ id: probeId, tenantId: args.tenantId, @@ -569,6 +652,7 @@ export function createWorkflowAllocationService({ provisionerId: probeProvisioner.id, provisionerApiVersion: probeProvisioner.apiVersion, provisionerBindingFingerprint: probeProvisioner.bindingFingerprint, + adapterManifestDigest: digest, now: now(), }); @@ -608,14 +692,20 @@ export function createWorkflowAllocationService({ sidecarId, token, hubWebSocketUrl, + ...(adapterManifest !== undefined ? { adapterManifest } : {}), }), ), ); if (ensured.kind === "rejected") { + // A rejected ensure started nothing. + await probeStore.retireSidecar({ sidecarId, now: now() }); throw new WorkflowProvisioningError(ensured.code, ensured.message); } const probing = await probeStore.markProbing({ probeId, + ...(ensured.sidecarId !== undefined + ? { sidecarId: ensured.sidecarId } + : {}), ...(ensured.externalRef !== undefined ? { externalRef: ensured.externalRef } : {}), @@ -625,9 +715,23 @@ export function createWorkflowAllocationService({ throw new Error(`Workflow probe ${probeId} changed after provisioning`); } probe = probing; - await allocationRouter.waitForAllocatedSidecar( - allocationTarget, + await runSidecarOperation( + "Probe connection", connectTimeoutMs, + async (signal) => { + if (probing.sidecarId !== null) { + // Joining a reused connection may wait behind its existing work. + // That wait belongs to the same deadline as readiness. + await allocationRouter.syncSidecar(probing.sidecarId, signal); + } + signal.throwIfAborted(); + await allocationRouter.waitForAllocatedSidecar( + allocationTarget, + connectTimeoutMs, + undefined, + signal, + ); + }, ); let resultRecorded = false; @@ -702,6 +806,7 @@ export function createWorkflowAllocationService({ provisioner: deploymentProvisioner, probe, adoptProbe, + adapterManifestDigest: digest, }); } catch (error) { try { @@ -736,11 +841,18 @@ export function createWorkflowAllocationService({ }; const anchor = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, allocation.anchorRunId), - columns: { publicKey: true, definitionId: true }, + columns: { + publicKey: true, + definitionId: true, + status: true, + expiresAt: true, + cancellationRequestedAt: true, + }, }); if (anchor === undefined) { throw new Error(`Allocation ${allocation.id} has no workflow anchor run`); } + if (!canExecuteWorkflowRun(anchor, now())) return null; if (await allocationRouter.isAllocatedWorkflowActive(allocationTarget)) { if (anchor.publicKey !== null) return null; throw new SessionLaunchError( @@ -751,85 +863,128 @@ export function createWorkflowAllocationService({ true, ); } - if (anchor.definitionId === null) { - throw new Error( - `Allocation ${allocation.id} anchor run has no frozen workflow definition`, + const stopped = + allocationRouter.reportedDeploymentFailure(allocationTarget); + if (stopped !== undefined) { + // The stopped copy may still be pushing history it committed. Failing + // the runs first would overwrite an outcome it recorded, so the Hub + // waits for that history for a while, then fails without it. + const history = + await allocationRouter.stoppedDeploymentHistory(allocationTarget); + if (history === undefined || history.unreceived === null) + throw new SidecarDeploymentStoppedError(stopped); + const giveUpAt = history.reportedAt.getTime() + STOPPED_HISTORY_WAIT_MS; + if (now().getTime() < giveUpAt) { + throw new SidecarDeploymentHistoryPendingError( + history.unreceived, + new Date( + Math.min(giveUpAt, now().getTime() + STOPPED_HISTORY_POLL_MS), + ), + ); + } + throw new SidecarDeploymentStoppedError( + `${stopped}; history it committed that the Hub never received is lost: ${history.unreceived}`, ); } - const spec = await launchSpecStore.get(allocation.anchorRunId); - if (spec === null) { - throw new Error( - `Allocation ${allocation.id} has no workflow launch specification`, + // The key commits only once the first deploy completed, so a connected + // sidecar that did not report the deployment no longer holds it. + if (anchor.publicKey !== null) { + throw new SidecarDeploymentMissingError( + allocation.id, + allocation.generation, ); } - // The frozen bundle deploys verbatim -- no re-probe. Rehydrate the approval - // hand-off from it: the persisted flat item list is partitioned back into - // the gate's `ApprovalSet`, and the frozen definition id is the anchor's own - // (set at prepare time from this freeze). - const bundle = spec.frozenApprovalBundle; - const approved: InstallAndApproveResult = { - approval: { - ok: true, - definitionId: anchor.definitionId, - approvedWireHash: bundle.approvedWireHash, - approvedSurface: approvalSetFromItems(bundle.approvedGrants), + try { + if (anchor.definitionId === null) { + throw new Error( + `Allocation ${allocation.id} anchor run has no frozen workflow definition`, + ); + } + const spec = await launchSpecStore.get(allocation.anchorRunId); + if (spec === null) { + throw new Error( + `Allocation ${allocation.id} has no workflow launch specification`, + ); + } + // The frozen bundle deploys verbatim -- no re-probe. Rehydrate the + // approval hand-off from it: the persisted flat item list is partitioned + // back into the gate's `ApprovalSet`, and the frozen definition id is the + // anchor's own (set at prepare time from this freeze). + const bundle = spec.frozenApprovalBundle; + const approved: InstallAndApproveResult = { + approval: { + ok: true, + definitionId: anchor.definitionId, + approvedWireHash: bundle.approvedWireHash, + approvedSurface: approvalSetFromItems(bundle.approvedGrants), + projection: bundle.projection, + }, projection: bundle.projection, - }, - projection: bundle.projection, - closure: bundle.closure, - }; - // Re-resolve the inference chain from the catalog at launch time -- the - // launch spec stores offering ids, never resolved sources, so a rotated - // credential is picked up here and no secret was ever persisted. - const resolved = await resolveSourcesByOfferingIds( - db, - allocation.tenantId, - spec.sourceOfferingIds, - credentialCipher, - ); - if (!resolved.ok) { - throw new Error( - `Catalog offering ${resolved.offeringId} is unavailable while recovering allocation ${allocation.id}`, + closure: bundle.closure, + }; + // Re-resolve the inference chain from the catalog at launch time -- the + // launch spec stores offering ids, never resolved sources, so a rotated + // credential is picked up here and no secret was ever persisted. + const resolved = await resolveSourcesByOfferingIds( + db, + allocation.tenantId, + spec.sourceOfferingIds, + credentialCipher, ); - } - const defaultSource = resolved.sources.find( - (source) => source.id === spec.defaultSourceOfferingId, - ); - if (defaultSource === undefined) { - throw new Error( - `Default offering ${spec.defaultSourceOfferingId} was not resolved for allocation ${allocation.id}`, + if (!resolved.ok) { + throw new Error( + `Catalog offering ${resolved.offeringId} is unavailable for allocation ${allocation.id}`, + ); + } + const defaultSource = resolved.sources.find( + (source) => source.id === spec.defaultSourceOfferingId, ); - } - const config = createProvisionedHarnessConfig({ - tenantId: allocation.tenantId, - anchorRunId: allocation.anchorRunId, - deploymentDomain: spec.deploymentDomain, - sessionId: spec.sessionId, - sourceAuthorityPrincipalId: spec.sourceAuthorityPrincipalId, - sources: resolved.sources, - defaultSource: defaultSource.id, - }); + if (defaultSource === undefined) { + throw new Error( + `Default offering ${spec.defaultSourceOfferingId} was not resolved for allocation ${allocation.id}`, + ); + } + const config = createProvisionedHarnessConfig({ + tenantId: allocation.tenantId, + anchorRunId: allocation.anchorRunId, + deploymentDomain: spec.deploymentDomain, + sessionId: spec.sessionId, + sourceAuthorityPrincipalId: spec.sourceAuthorityPrincipalId, + sources: resolved.sources, + defaultSource: defaultSource.id, + }); - reconciliation.signal.throwIfAborted(); - return preparedDeployer.deployPreparedCodeSourcedWorkflow({ - reconciliation, - tenantId: allocation.tenantId, - anchorRunId: allocation.anchorRunId, - deploymentDomain: spec.deploymentDomain, - agentAddress: config.agentAddress, - source: bundle.source, - approved, - config, - deployContent: preparedDeployContent(spec.deployContent), - // Pins rehydrated from the frozen launch spec. Undefined (or empty) - // skips the tool-package resolver: no manifest is staged and the - // child materializes empty step tools rather than the pinned closure. - ...(spec.toolPackagePins !== null - ? { toolPackagePins: spec.toolPackagePins } - : {}), - allocationTarget, - credentialCipher, - }); + reconciliation.signal.throwIfAborted(); + return await preparedDeployer.deployPreparedCodeSourcedWorkflow({ + reconciliation, + tenantId: allocation.tenantId, + anchorRunId: allocation.anchorRunId, + deploymentDomain: spec.deploymentDomain, + agentAddress: config.agentAddress, + source: bundle.source, + approved, + config, + deployContent: preparedDeployContent(spec.deployContent), + // Pins rehydrated from the frozen launch spec. Undefined (or empty) + // skips the tool-package resolver: no manifest is staged and the + // child materializes empty step tools rather than the pinned closure. + ...(spec.toolPackagePins !== null + ? { toolPackagePins: spec.toolPackagePins } + : {}), + allocationTarget, + credentialCipher, + }); + } catch (error) { + // Keep capacity refusals terminal and leave uncertain deploys and lost + // leases to reconciliation. Only other pre-deploy failures may retry. + if ( + error instanceof DeploymentRejectedError || + (error instanceof SessionLaunchError && error.leakedAgent) || + reconciliation.signal.aborted + ) + throw error; + throw new SidecarFirstDeployError(error); + } } function cleanupFinalStatus(probe: WorkflowProbe): "succeeded" | "failed" { diff --git a/vendor/intx/hub-sessions/src/workflow-closure-resolution.ts b/vendor/intx/hub-sessions/src/workflow-closure-resolution.ts index 717f9e6d89..f424d9b03b 100644 --- a/vendor/intx/hub-sessions/src/workflow-closure-resolution.ts +++ b/vendor/intx/hub-sessions/src/workflow-closure-resolution.ts @@ -36,9 +36,10 @@ import { } from "@intx/tool-packaging"; import type { WorkflowSourceAssetMount } from "@intx/types/sidecar"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; -import type { - WorkflowDefinitionAssetSource, - WorkflowDefinitionRegistrySource, +import { + workflowSourceAssetMountPath, + type WorkflowDefinitionAssetSource, + type WorkflowDefinitionRegistrySource, } from "@intx/types/workflow-sources"; import { @@ -193,16 +194,6 @@ export async function resolveWorkflowClosure( return resolver.resolveClosure([parsePin(args.pin)]); } -/** - * The workspace-relative mount path a workflow definition's source asset is - * delivered under, keyed by asset id. Both the probe (inline delivery) and the - * deploy (streamed fan-out) build their `assetId -> mountPath` maps through - * this one helper, so the same asset lands at the same path on both paths. - */ -export function workflowSourceAssetMountPath(assetId: string): string { - return `source-assets/${assetId}/`; -} - /** * Resolves an asset (by id) to the git pack a source-ref frame delivers for it. * The caller binds the ref: an asset source binds the asset's default ref, so diff --git a/vendor/intx/hub-sessions/src/workflow-definition-ensure.ts b/vendor/intx/hub-sessions/src/workflow-definition-ensure.ts index e8fb647177..c8b830d305 100644 --- a/vendor/intx/hub-sessions/src/workflow-definition-ensure.ts +++ b/vendor/intx/hub-sessions/src/workflow-definition-ensure.ts @@ -1,4 +1,4 @@ -import { eq } from "drizzle-orm"; +import { desc, eq } from "drizzle-orm"; import { resolveDefinitionIdForAsset, @@ -36,6 +36,8 @@ export async function ensureWorkflowDefinitionForAsset( selector: WorkflowDefinitionSelector, ): Promise<{ definitionId: string; created: boolean }> { const { assetId, wireHash } = selector; + // The lifecycle override route writes every definition of the asset under + // this lock, so a revision created here cannot miss a concurrent override. const assetRow = await db .select({ tenantId: asset.tenantId, @@ -46,12 +48,20 @@ export async function ensureWorkflowDefinitionForAsset( .from(asset) .where(eq(asset.id, assetId)) .limit(1) + .for("update") .then((rows) => rows[0]); if (assetRow === undefined) { throw new Error( `ensureWorkflowDefinitionForAsset: no asset found for id "${assetId}"`, ); } + // A new revision keeps the lifecycle override of its installed workflow. + const [installed] = await db + .select({ lifecyclePolicy: workflowDefinition.lifecyclePolicy }) + .from(workflowDefinition) + .where(eq(workflowDefinition.assetId, assetId)) + .orderBy(desc(workflowDefinition.updatedAt)) + .limit(1); const inserted = await db .insert(workflowDefinition) @@ -63,6 +73,7 @@ export async function ensureWorkflowDefinitionForAsset( wireHash, name: assetRow.name, description: assetRow.displayName, + lifecyclePolicy: installed?.lifecyclePolicy ?? null, }) .onConflictDoNothing({ target: [workflowDefinition.assetId, workflowDefinition.wireHash], diff --git a/vendor/intx/hub-sessions/src/workflow-dispatch-projection.ts b/vendor/intx/hub-sessions/src/workflow-dispatch-projection.ts new file mode 100644 index 0000000000..84c0dff0fc --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-dispatch-projection.ts @@ -0,0 +1,328 @@ +import { + and, + asc, + desc, + eq, + exists, + gt, + lte, + notInArray, + sql, +} from "drizzle-orm"; + +import { + createWorkflowPendingProjectionStore, + createWorkflowRunDispatchStore, + type DB, +} from "@intx/db"; +import { + isLiveWorkflowRunStatus, + sidecarAllocation, + workflowRun, + workflowRunDispatch, +} from "@intx/db/schema"; +import { getLogger } from "@intx/log"; +import { isSidecarAllocationDispatchable } from "@intx/types"; +import { SignalDeliverFrame } from "@intx/types/sidecar"; + +import { createAnchorRunSweep } from "./anchor-run-sweep"; +import type { CommittedReads, RepoStore } from "./repo-store"; +import { + listAcceptedWorkflowDispatches, + listConsumedWorkflowDispatches, +} from "./workflow-dispatch-settlement"; +import { + readCommittedWorkflowRunLifecycle, + workflowRunRepoIdForAddress, + WORKFLOW_RUN_REF, +} from "./workflow-run-kind"; + +const logger = getLogger(["hub", "workflow-dispatch-projection"]); +export const DEFAULT_WORKFLOW_PROJECTION_CONCURRENCY = 8; + +/** Projects accepted Git evidence without depending on a live worker or allocation. */ +export function createWorkflowDispatchProjection({ + db, + repoStore, + maxConcurrentProjections = DEFAULT_WORKFLOW_PROJECTION_CONCURRENCY, +}: { + db: DB["db"]; + repoStore: Pick; + /** Bounds background scans and their unfinished work; ingestion owns its own queue. */ + maxConcurrentProjections?: number; +}) { + if ( + !Number.isSafeInteger(maxConcurrentProjections) || + maxConcurrentProjections <= 0 + ) { + throw new Error("Projection concurrency must be a positive integer"); + } + const dispatches = createWorkflowRunDispatchStore(db); + const pendingProjections = createWorkflowPendingProjectionStore(db); + let admitted = 0; + + async function project(anchorRunId: string): Promise { + const [anchor] = await db + .select({ address: workflowRun.address }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, anchorRunId), + eq(workflowRun.anchorRunId, anchorRunId), + ), + ); + if (anchor === undefined || anchor.address === null) return; + // Capture the candidate deliveries before pinning Git. A later enqueue is + // outside this projection, even if the run becomes terminal during the read. + const unsettled = await dispatches.listUnsettled(anchorRunId); + if (unsettled.length === 0) return; + const repoId = workflowRunRepoIdForAddress(anchor.address); + const reads = await repoStore.openCommittedReads( + { kind: "hub" }, + repoId, + WORKFLOW_RUN_REF, + ); + // Pin run history first, then consumed outcomes so an older consumed view + // cannot hide a rejection already present when terminal state was observed. + const consumedReads = await repoStore.openCommittedReads( + { kind: "hub" }, + repoId, + "refs/heads/events", + ); + const remaining = new Map( + unsettled.map((dispatch) => [dispatch.messageId, dispatch]), + ); + const outcomes = new Map< + string, + | { status: "settled" } + | { status: "failed"; code: string; message: string } + >(); + const consumedDispatches = + consumedReads === null + ? [] + : await listConsumedWorkflowDispatches(consumedReads); + for (const consumed of consumedDispatches) { + if (consumed.address !== anchor.address) continue; + if (remaining.get(consumed.messageId)?.kind !== "mail") continue; + outcomes.set( + consumed.messageId, + consumed.rejection === undefined + ? { status: "settled" } + : { + status: "failed", + code: consumed.rejection.code, + message: consumed.rejection.message, + }, + ); + remaining.delete(consumed.messageId); + } + const messageIdsByRun = new Map>(); + for (const dispatch of remaining.values()) { + const runId = + dispatch.kind === "mail" + ? anchorRunId + : SignalDeliverFrame.assert( + JSON.parse(new TextDecoder().decode(dispatch.rawMessage)), + ).runId; + const ids = messageIdsByRun.get(runId) ?? new Set(); + ids.add(dispatch.messageId); + messageIdsByRun.set(runId, ids); + } + for (const [runId, ids] of messageIdsByRun) { + const acceptedDispatches = + reads === null + ? [] + : await listAcceptedWorkflowDispatches(reads, runId, ids); + for (const accepted of acceptedDispatches) { + const dispatch = remaining.get(accepted.messageId); + if (dispatch === undefined) continue; + // Mail resuming an input gate is also recorded as SignalReceived. + // The Hub's message-id uniqueness spans both dispatch kinds within + // this anchor; RunStarted alone still cannot prove a signal delivery. + if (accepted.kind === "mail" && dispatch.kind !== "mail") continue; + outcomes.set(accepted.messageId, { status: "settled" }); + remaining.delete(accepted.messageId); + } + if ( + (await readCommittedWorkflowRunLifecycle(reads, runId)) === "terminal" + ) { + for (const messageId of ids) { + // A mail rejection is recorded on the events ref, which can trail the + // run log, so mail waits for final history below. + if (remaining.get(messageId)?.kind === "mail") continue; + if (!remaining.delete(messageId)) continue; + outcomes.set(messageId, { + status: "failed", + code: "workflow_run_terminal", + message: `Workflow run ${runId} is terminal and cannot accept this dispatch`, + }); + } + } + } + const anchorTerminal = + remaining.size > 0 && + reads !== null && + (await readCommittedWorkflowRunLifecycle(reads, anchorRunId)) === + "terminal"; + const unscannedAbandoned = [...remaining.values()].filter( + (dispatch) => + dispatch.status === "abandoned" && dispatch.nextAttemptAt !== null, + ); + const historyFinal = + (anchorTerminal || unscannedAbandoned.length > 0) && + (await isHistoryFinal(anchorRunId, repoId, [ + { ref: WORKFLOW_RUN_REF, reads }, + { ref: "refs/heads/events", reads: consumedReads }, + ])); + if (anchorTerminal && historyFinal) { + for (const messageId of remaining.keys()) { + outcomes.set(messageId, { + status: "failed", + code: "workflow_run_terminal", + message: `Workflow deployment ${anchor.address} is terminal and cannot accept this dispatch`, + }); + } + } + // Accepted history cannot be rewritten. Each conditional row update can + // commit independently: a failed pass leaves the remainder discoverable, + // and a large backlog never holds allocation or run locks during cleanup. + const now = new Date(); + for (const [messageId, outcome] of outcomes) { + if (outcome.status === "settled") { + await dispatches.settle(anchorRunId, messageId, now); + } else { + await dispatches.fail({ + anchorRunId, + messageId, + code: outcome.code, + message: outcome.message, + now, + }); + } + } + // Final history has been read once, so later scans cannot resolve what + // remains abandoned. + if (historyFinal) + await dispatches.concludeAbandonedScans( + anchorRunId, + unscannedAbandoned.map((dispatch) => dispatch.messageId), + now, + ); + } + + // Pack receipt opens a pending row before Git can move and requires a live + // anchor row and allocated worker under the allocation lock. After either + // gate closes and no receive is pending, pinned refs that are still current + // are the deployment's final history. + async function isHistoryFinal( + anchorRunId: string, + repoId: { kind: "workflow-run"; id: string }, + pinned: readonly { ref: string; reads: CommittedReads | null }[], + ): Promise { + const [current] = await db + .select({ + status: workflowRun.status, + allocationStatus: sidecarAllocation.status, + }) + .from(workflowRun) + .leftJoin( + sidecarAllocation, + eq(sidecarAllocation.anchorRunId, workflowRun.id), + ) + .where(eq(workflowRun.id, anchorRunId)) + .limit(1); + const allocationClosed = + current?.allocationStatus != null && + !isSidecarAllocationDispatchable(current.allocationStatus); + if ( + current === undefined || + (isLiveWorkflowRunStatus(current.status) && !allocationClosed) || + (await pendingProjections.hasAny(anchorRunId)) + ) + return false; + for (const { ref, reads } of pinned) { + const latest = await repoStore.openCommittedReads( + { kind: "hub" }, + repoId, + ref, + ); + // Only a receive creates the repository or writes a ref, so with none + // pending an absent ref stays absent. + if (reads === null || latest === null) { + if (reads !== latest) return false; + continue; + } + if ((await latest.treeOid("")) !== (await reads.treeOid(""))) + return false; + } + return true; + } + + function eligibleRuns(activeRunIds: string[]) { + return and( + eq(workflowRun.id, workflowRun.anchorRunId), + ...(activeRunIds.length === 0 + ? [] + : [notInArray(workflowRun.id, activeRunIds)]), + exists( + db + .select({ id: workflowRunDispatch.id }) + .from(workflowRunDispatch) + .where( + and( + eq(workflowRunDispatch.anchorRunId, workflowRun.id), + // A SQL literal, so a plan made without parameter values can + // still use the partial index on unresolved dispatches. + sql`(${workflowRunDispatch.status} in ('pending', 'acknowledged') or (${workflowRunDispatch.status} = 'abandoned' and ${workflowRunDispatch.nextAttemptAt} is not null))`, + ), + ), + ), + ); + } + + const sweep = createAnchorRunSweep({ + async findPassEnd(activeRunIds) { + const [last] = await db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where(eligibleRuns(activeRunIds)) + .orderBy(desc(workflowRun.id)) + .limit(1); + return last?.id; + }, + async findNext({ afterRunId, passEnd, activeRunIds }) { + const [candidate] = await db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + eligibleRuns(activeRunIds), + lte(workflowRun.id, passEnd), + ...(afterRunId === undefined + ? [] + : [gt(workflowRun.id, afterRunId)]), + ), + ) + .orderBy(asc(workflowRun.id)) + .limit(1); + return candidate; + }, + }); + + async function reconcileNext(): Promise { + if (admitted >= maxConcurrentProjections) return; + admitted += 1; + let run: { id: string } | null = null; + try { + run = await sweep.select(); + if (run !== null) await project(run.id); + } catch (cause) { + logger.error`Dispatch projection failed for ${run?.id ?? "candidate selection"}: ${cause instanceof Error ? cause.message : String(cause)}`; + } finally { + if (run !== null) sweep.release(run.id); + admitted -= 1; + } + } + + return { project, reconcileNext }; +} diff --git a/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts b/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts index 7bb461ad05..95a2045404 100644 --- a/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts +++ b/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts @@ -268,6 +268,9 @@ export function createWorkflowDispatchService({ signalName: frame.signalName, signalId: frame.signalId, payload: frame.payload, + ...(frame.correlationId !== undefined + ? { correlationId: frame.correlationId } + : {}), }, signal, ); diff --git a/vendor/intx/hub-sessions/src/workflow-dispatch-settlement.ts b/vendor/intx/hub-sessions/src/workflow-dispatch-settlement.ts index 1167cfe38b..7eb1d6682f 100644 --- a/vendor/intx/hub-sessions/src/workflow-dispatch-settlement.ts +++ b/vendor/intx/hub-sessions/src/workflow-dispatch-settlement.ts @@ -51,6 +51,7 @@ export type ReceivedWorkflowSignal = { export type AcceptedWorkflowDispatch = { readonly runId: string; readonly messageId: string; + /** Event source: SignalReceived also proves mail used to resume an input gate. */ readonly kind: "mail" | "signal"; }; diff --git a/vendor/intx/hub-sessions/src/workflow-history-receives.ts b/vendor/intx/hub-sessions/src/workflow-history-receives.ts new file mode 100644 index 0000000000..15260e3b2d --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-history-receives.ts @@ -0,0 +1,27 @@ +/** + * Pending-projection rows whose workflow-run pack receive is still running in + * this process. Such a receive may advance Git after recovery reads it, so + * recovery must not claim its row. A receive registers before its row is + * inserted and leaves only after its Git transaction ends, so a listed row that + * is absent here belongs to a receive whose effect on Git is final. The Hub is + * the single writer of its workflow-run repositories, so rows left by an + * earlier process are always final. + */ +export function createWorkflowHistoryReceiveTracker() { + const inFlight = new Set(); + return { + begin(id: string): void { + inFlight.add(id); + }, + end(id: string): void { + inFlight.delete(id); + }, + isInFlight(id: string): boolean { + return inFlight.has(id); + }, + }; +} + +export type WorkflowHistoryReceiveTracker = ReturnType< + typeof createWorkflowHistoryReceiveTracker +>; diff --git a/vendor/intx/hub-sessions/src/workflow-kind.ts b/vendor/intx/hub-sessions/src/workflow-kind.ts index 816d7f48b6..10e41e3436 100644 --- a/vendor/intx/hub-sessions/src/workflow-kind.ts +++ b/vendor/intx/hub-sessions/src/workflow-kind.ts @@ -24,12 +24,7 @@ import { type } from "arktype"; import { getLogger } from "@intx/log"; -import { - CredentialBinding, - GrantRequirement, - SidecarCapabilityPolicy, -} from "@intx/types"; -import { InboundMailPolicy } from "@intx/types/runtime"; +import { workflowDefinitionEnvelopeSchema } from "@intx/types/workflow-definition"; import { PackageJSON, isContainedEntryPath } from "@intx/types/package-json"; import { authorizeUserPrincipal, @@ -57,62 +52,6 @@ export const PACKAGE_JSON_PATH = "package.json"; export const NODE_MODULES_PATH = "node_modules"; export const PNPM_WORKSPACE_PATH = "pnpm-workspace.yaml"; -/** - * Structural arktype validator for the `workflow.json` envelope. The - * substrate checks the cross-cutting shape of `WorkflowDefinition` - * (presence and primitive type of `id`, `triggers`, `steps`, - * `stepOrder`) but does not re-derive `defineWorkflow`'s DAG-level - * validation here — primitive-level shape, default-input application, - * and `after`-ref resolution belong to the runtime layer that hydrates - * the definition. The codebase push uses this validator to detect an - * ambiguous tree that also carries an envelope-valid `workflow.json`, - * and the hydrate-time definition loaders reuse it to validate a - * materialized definition before instantiation. - */ -const StepsObject = type("Record").narrow((value, ctx) => { - if (Array.isArray(value)) { - return ctx.mustBe("a JSON object, not an array"); - } - return true; -}); - -const StateObject = type("Record").narrow((value, ctx) => { - if (Array.isArray(value)) { - return ctx.mustBe("a JSON object, not an array"); - } - return true; -}); - -export const workflowDefinitionEnvelopeSchema = type({ - id: "string > 0", - triggers: "unknown[]", - steps: StepsObject, - stepOrder: "string[]", - "state?": StateObject, - // `grantRequirements` passes through the envelope whether or not it is - // declared here: arktype's `.onUndeclaredKey("ignore")` below is - // passthrough, not stripping (only `"delete"` strips), so the hydrate read - // sees the field either way. Declaring it here VALIDATES declared - // requirements at the deploy boundary — a malformed `source` is rejected - // rather than passed through unchecked — as defense in depth alongside the - // trigger route's own `GrantRequirements` re-validation. Compose the - // exported `GrantRequirement` arktype rather than restating its shape so the - // envelope and the definition stay in lockstep. - "grantRequirements?": GrantRequirement.array(), - // `credentialBindings` is validated here too -- same defense-in-depth - // rationale as grantRequirements above: a malformed binding (bad locator, - // authority, or handle) is rejected at the deploy boundary rather than - // passed through to launch-time resolution unchecked. - "credentialBindings?": CredentialBinding.array(), - "sidecarPlacement?": SidecarCapabilityPolicy, - // `inboundMailPolicy` is validated here too -- same defense-in-depth - // rationale as credentialBindings above: a malformed policy (an unknown - // outcome key or a value that is not reject/admit) is rejected at the deploy - // boundary rather than passed through to later admission resolution - // unchecked. - "inboundMailPolicy?": InboundMailPolicy, -}).onUndeclaredKey("ignore"); - const SidecarPrincipal = type({ kind: "'sidecar'", agentId: "string", diff --git a/vendor/intx/hub-sessions/src/workflow-lifecycle-service.ts b/vendor/intx/hub-sessions/src/workflow-lifecycle-service.ts new file mode 100644 index 0000000000..e9f99a85c0 --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-lifecycle-service.ts @@ -0,0 +1,1277 @@ +import { + and, + asc, + desc, + eq, + gt, + inArray, + isNotNull, + isNull, + lte, + notInArray, + or, + sql, + type SQL, +} from "drizzle-orm"; +import type { AnyPgColumn } from "drizzle-orm/pg-core"; + +import { + createSidecarAllocationStore, + createWorkflowPendingProjectionStore, + createWorkflowRunStore, + createWorkflowRunDispatchStore, + parseWorkflowRunRow, + workflowCapacityReleaseAt, + SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE, + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE, + type DB, + type DBExecutor, + type BeginSidecarReleaseArgs, + type SidecarAllocation, +} from "@intx/db"; +import { + liveWorkflowRunStatuses, + isLiveWorkflowRunStatus, + principal, + sidecarAllocation, + workflowPendingProjection, + workflowRun, +} from "@intx/db/schema"; +import { getLogger } from "@intx/log"; +import { isSidecarAllocationDispatchable } from "@intx/types"; + +import type { WorkflowHistoryReceiveTracker } from "./workflow-history-receives"; +import type { WorkflowRunReader } from "./workflow-run-reader"; +import { + WorkflowControlRejectedError, + WorkflowControlTimeoutError, + type AllocatedSidecarTarget, + type SidecarAllocationRouter, +} from "./ws/sidecar-handler"; +import { + classifyTerminalEvent, + workflowRunRepoIdForAddress, + WORKFLOW_RUN_REF, +} from "./workflow-run-kind"; +import { projectTerminalRun } from "./workflow-run-terminal-projection"; +import { + createAnchorRunSweep, + type AnchorRunSweepQueries, +} from "./anchor-run-sweep"; + +const logger = getLogger(["hub", "workflow-lifecycle"]); +const SWEEP_INTERVAL_MS = 1_000; +// A receive clears its own pending projection within one pack transfer. The +// sweep leaves younger rows to it so healthy deployments cost no Git reads. +const PENDING_PROJECTION_GRACE_MS = 30_000; +// A moving tip is reread a few times; after that the next pass retries, so a +// deployment that pushes constantly cannot hold a caller here. +const MAX_RECOVERY_ATTEMPTS = 3; +const RECOVERY_BACKOFF_MIN_MS = 5_000; +const RECOVERY_BACKOFF_MAX_MS = 5 * 60_000; +const CANCEL_CONTROL_TIMEOUT_MS = 5_000; +const STOP_CONTROL_TIMEOUT_MS = 10_000; +const MAX_CONCURRENT_RETAINS_PER_SIDECAR = 8; + +type Run = ReturnType; +type Allocation = typeof sidecarAllocation.$inferSelect; +type ReleaseResult = + | "pending" + | "released" + | "live" + | "history_pending" + | "not_found" + | "cleanup_exhausted" + | "cleanup_disconnected" + | "cleanup_failed"; +type ReleaseStoppedCleanupResult = + | "released" + | "not_found" + | "not_stopped" + | "generation_mismatch"; +// `request` is an explicit caller waiting on the answer, so it reads Git even +// while the sweep is backing off; `stop` and `sweep` respect the backoff, and +// only `sweep` leaves young rows to the receive that wrote them. +type RecoveryTrigger = "request" | "stop" | "sweep"; + +export type WorkflowLifecycleServiceDeps = { + db: DB["db"]; + retentionRouter: Pick< + SidecarAllocationRouter, + | "retainAllocation" + | "getAllocationRetention" + | "getRetentionCandidates" + | "holdsAllocatedBinding" + | "fenceAllocation" + | "retireAllocation" + >; + runReader: WorkflowRunReader; + historyReceives: WorkflowHistoryReceiveTracker; + now?: () => Date; + cancelGraceMs?: number; + unreachableStopGraceMs?: number; + sendControl?: SidecarAllocationRouter["sendWorkflowControl"]; +}; + +export function createWorkflowLifecycleService({ + db, + retentionRouter, + runReader, + historyReceives, + now = () => new Date(), + cancelGraceMs = 30_000, + unreachableStopGraceMs = 60_000, + sendControl, +}: WorkflowLifecycleServiceDeps) { + if (cancelGraceMs < 0 || !Number.isFinite(cancelGraceMs)) + throw new Error("Cancellation grace must be finite and non-negative"); + if (unreachableStopGraceMs < 0 || !Number.isFinite(unreachableStopGraceMs)) + throw new Error("Unreachable stop grace must be finite and non-negative"); + // Connections are rebuilt after a restart, so an unreachable worker gets the + // full grace from this service's start before its capacity is reclaimed. + const startedAt = now().getTime(); + const allocations = createSidecarAllocationStore(db); + const dispatches = createWorkflowRunDispatchStore(db); + const runs = createWorkflowRunStore(db); + const pendingProjections = createWorkflowPendingProjectionStore(db); + const recoveryBackoff = new Map< + string, + { failures: number; nextAt: number; pendingIds: readonly string[] } + >(); + const retentionBackoff = new Map< + string, + { + generation: number; + failures: number; + nextAt: number; + } + >(); + const retentionsInFlight = new Set(); + const retentionsPerSidecar = new Map(); + + function hasRetentionRoom(sidecarId: string): boolean { + return ( + (retentionsPerSidecar.get(sidecarId) ?? 0) < + MAX_CONCURRENT_RETAINS_PER_SIDECAR + ); + } + + async function withRun( + tenantId: string, + runId: string, + action: ( + tx: DBExecutor, + run: Run, + allocation: Allocation | undefined, + release: ( + args: BeginSidecarReleaseArgs, + ) => Promise, + ) => Promise, + ) { + const releases: SidecarAllocation[] = []; + const result = await db.transaction(async (tx) => { + // Match pack ingestion's lock order: allocation row first. + const [allocation] = await tx + .select() + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.anchorRunId, runId), + eq(sidecarAllocation.tenantId, tenantId), + ), + ) + .for("update"); + const [row] = await tx + .select() + .from(workflowRun) + .where( + and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + eq(workflowRun.anchorRunId, runId), + ), + ) + .for("update"); + if (row === undefined) return null; + return action(tx, parseWorkflowRunRow(row), allocation, async (args) => { + const released = await allocations.beginRelease(args, tx); + if (released !== null) releases.push(released); + return released; + }); + }); + for (const released of releases) + retentionRouter.fenceAllocation(released.id, released.generation, { + ...(released.sidecarId === undefined + ? {} + : { cleanup: { sidecarId: released.sidecarId } }), + }); + return result; + } + + // Project accepted history for the deployment and claim the pending rows + // that history covers. Returns false when Git moved after the read, so the + // read may have missed a receive and nothing was claimed. + async function reconcileFromGit( + tenantId: string, + runId: string, + anchor: { address: string | null; definitionId: string; createdAt: Date }, + claimable: readonly { id: string; historyRequired: boolean }[], + ): Promise { + const settled = await db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + eq(workflowRun.anchorRunId, runId), + notInArray(workflowRun.status, [...liveWorkflowRunStatuses]), + ), + ); + const settledIds = new Set(settled.map((row) => row.id)); + const repoId = + anchor.address === null + ? null + : workflowRunRepoIdForAddress(anchor.address); + // Settled rows are final, so only live runs and runs without a row can + // still be missing an accepted outcome. + const history: Awaited< + ReturnType + > = + repoId === null + ? { tip: null, events: new Map() } + : await runReader.readLatestRunEvents( + repoId, + WORKFLOW_RUN_REF, + (id) => !settledIds.has(id), + ); + // A receive writes the ref last and nothing removes refs, so a repository + // without one never advanced, while a missing repository means accepted + // history is unavailable, not empty. + if ( + history.tip === null && + claimable.some((row) => row.historyRequired) && + (repoId === null || !(await runReader.hasRepository(repoId))) + ) + throw new Error("Accepted workflow history is missing from the Hub"); + // Only the allocation row is locked: pack ingestion holds it while + // advancing Git, so an unchanged tip under it proves the read covered every + // claimed receive. Locking the anchor row as well would deadlock against a + // receive minting a child row, whose foreign key needs it. + return db.transaction(async (tx) => { + await tx + .select({ id: sidecarAllocation.id }) + .from(sidecarAllocation) + .where(eq(sidecarAllocation.anchorRunId, runId)) + .for("update"); + if ( + repoId !== null && + (await runReader.resolveRefTip(repoId, WORKFLOW_RUN_REF)) !== + history.tip + ) + return false; + for (const [id, event] of history.events) { + if (event === null) continue; + const terminal = classifyTerminalEvent(event.type); + if (!terminal.terminal) continue; + const outcome = await projectTerminalRun(tx, runs, { + anchor: { + id: runId, + tenantId, + definitionId: anchor.definitionId, + createdAt: anchor.createdAt, + }, + runId: id, + status: terminal.status, + terminalEvent: event.body, + now: now(), + }); + if (outcome === "foreign") + logger.error`Ignoring terminal event for run ${id}: it does not belong to deployment ${runId}`; + } + await pendingProjections.claim( + runId, + claimable.map((row) => row.id), + tx, + ); + return true; + }); + } + + // Reconcile a deployment whose run rows may disagree with accepted Git + // history. Failures back off per deployment; callers act on whatever the + // database records, and a forced stop waits while rows remain. + async function recoverHistory( + tenantId: string, + runId: string, + trigger: RecoveryTrigger, + ): Promise { + // Keyed by tenant too, so a request naming another tenant's run cannot + // reset that run's backoff. + const backoffKey = JSON.stringify([tenantId, runId]); + const backoff = recoveryBackoff.get(backoffKey); + if ( + trigger !== "request" && + backoff !== undefined && + now().getTime() < backoff.nextAt + ) + return; + const grace = trigger === "sweep" ? PENDING_PROJECTION_GRACE_MS : 0; + let attempted: readonly string[] = []; + try { + for (let attempt = 0; attempt < MAX_RECOVERY_ATTEMPTS; attempt += 1) { + const listed = await pendingProjections.list(runId); + // Only receives whose Git transaction has ended can be claimed; one + // still running may advance Git after the read. + const claimable = listed.filter( + (row) => !historyReceives.isInFlight(row.id), + ); + const cutoff = now().getTime() - grace; + if (!claimable.some((row) => row.createdAt.getTime() <= cutoff)) { + recoveryBackoff.delete(backoffKey); + return; + } + attempted = claimable.map((row) => row.id); + const anchor = await db.query.workflowRun.findFirst({ + where: and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + eq(workflowRun.anchorRunId, runId), + ), + columns: { address: true, definitionId: true, createdAt: true }, + }); + if (anchor === undefined) { + recoveryBackoff.delete(backoffKey); + return; + } + const claimed = await reconcileFromGit( + tenantId, + runId, + anchor, + claimable, + ); + if (claimed) { + if (backoff !== undefined) + logger.info`Reconciled accepted history for ${runId}`; + recoveryBackoff.delete(backoffKey); + return; + } + } + } catch (error) { + // Failures are consecutive only while history that already failed is + // still pending; once a receive settles it, the next failure starts over. + const failures = + backoff !== undefined && + (attempted.length === 0 || + attempted.some((id) => backoff.pendingIds.includes(id))) + ? backoff.failures + 1 + : 1; + const delay = Math.min( + RECOVERY_BACKOFF_MAX_MS, + RECOVERY_BACKOFF_MIN_MS * 2 ** (failures - 1), + ); + recoveryBackoff.set(backoffKey, { + failures, + nextAt: now().getTime() + delay, + pendingIds: + attempted.length === 0 ? (backoff?.pendingIds ?? []) : attempted, + }); + const message = error instanceof Error ? error.message : String(error); + if (failures === 1) + logger.warn`Cannot reconcile accepted history for ${runId}; its outcomes are recorded once it can: ${message}`; + else if ( + delay === RECOVERY_BACKOFF_MAX_MS && + RECOVERY_BACKOFF_MIN_MS * 2 ** (failures - 2) < RECOVERY_BACKOFF_MAX_MS + ) + logger.error`Accepted history for ${runId} is still unreconciled after ${failures} attempts; run statuses may be stale and a forced stop stays unrecorded: ${message}`; + } + } + + async function getStatus(tenantId: string, runId: string) { + const row = await db.query.workflowRun.findFirst({ + where: and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + eq(workflowRun.anchorRunId, runId), + ), + }); + if (row === undefined) return null; + const run = parseWorkflowRunRow(row); + const allocation = await allocations.findByAnchorRunId(runId); + return { + runId, + status: run.status, + policy: run.lifecyclePolicy ?? {}, + expiresAt: run.expiresAt?.toISOString() ?? null, + cancellationRequestedAt: + run.cancellationRequestedAt?.toISOString() ?? null, + cancellationDeadline: run.cancellationDeadline?.toISOString() ?? null, + cancellationReason: run.cancellationReason ?? null, + capacityReleaseAt: run.capacityReleaseAt?.toISOString() ?? null, + allocation: + allocation === null + ? null + : { + id: allocation.id, + generation: allocation.generation, + status: allocation.status, + failureCode: allocation.failureCode ?? null, + failureMessage: allocation.failureMessage ?? null, + }, + }; + } + + async function releaseCapacity( + tenantId: string, + runId: string, + ): Promise { + await recoverHistory(tenantId, runId, "request"); + const result = await withRun( + tenantId, + runId, + async (tx, run, allocation): Promise => { + // Report the cleanup outcome before pending history can mask it. + // Disconnect timeouts resume on reconnect; other failures need an operator. + if (allocation?.status === "destroy_failed") { + if ( + allocation.failureCode === + SIDECAR_CLEANUP_DISCONNECT_TIMEOUT_FAILURE_CODE + ) + return "cleanup_disconnected"; + return allocation.failureCode === + SIDECAR_CLEANUP_RETRY_EXHAUSTED_FAILURE_CODE + ? "cleanup_exhausted" + : "cleanup_failed"; + } + // A pending projection may hold the anchor's or a child's accepted + // terminal outcome, even when the anchor row is already terminal. + if (await pendingProjections.hasAny(runId, tx)) + return "history_pending"; + if (isLiveWorkflowRunStatus(run.status)) return "live"; + if ( + allocation === undefined || + allocation.status === "released" || + allocation.status === "failed" + ) + return "released"; + const requestedAt = now(); + if ( + run.capacityReleaseAt === null || + run.capacityReleaseAt > requestedAt + ) { + await tx + .update(workflowRun) + .set({ capacityReleaseAt: requestedAt }) + .where(eq(workflowRun.id, runId)); + } + return "pending"; + }, + ); + return result ?? "not_found"; + } + + async function releaseStoppedCleanup( + tenantId: string, + runId: string, + generation: number, + releasedBy: string, + ): Promise { + const result = await allocations.releaseStoppedCleanup({ + tenantId, + anchorRunId: runId, + generation, + releasedBy, + now: now(), + }); + if (result.kind === "released") + retentionRouter.retireAllocation({ + allocationId: result.allocation.id, + generation: result.allocation.generation, + }); + return result.kind === "already_released" ? "released" : result.kind; + } + + async function beginCancellation( + tx: DBExecutor, + run: Run, + reason: string, + ): Promise { + if (run.cancellationRequestedAt !== null) return run; + const requestedAt = now(); + const deadline = new Date(requestedAt.getTime() + cancelGraceMs); + await tx + .update(workflowRun) + .set({ + cancellationRequestedAt: requestedAt, + cancellationDeadline: deadline, + cancellationReason: reason, + }) + .where(eq(workflowRun.id, run.id)); + return { + ...run, + cancellationRequestedAt: requestedAt, + cancellationDeadline: deadline, + cancellationReason: reason, + }; + } + + function getRequestedCancellation(run: Run): { + deadline: Date; + reason: string; + } { + // beginCancellation writes the request, deadline, and reason together. + if (run.cancellationDeadline === null || run.cancellationReason === null) + throw new Error( + `Workflow run ${run.id} has an incomplete cancellation request`, + ); + return { + deadline: run.cancellationDeadline, + reason: run.cancellationReason, + }; + } + + async function requestCancellation( + tenantId: string, + runId: string, + reason: string, + ): Promise<"pending" | "terminal" | "not_found"> { + await recoverHistory(tenantId, runId, "request"); + const result = await withRun(tenantId, runId, async (tx, run) => { + if (!isLiveWorkflowRunStatus(run.status)) return "terminal" as const; + await beginCancellation(tx, run, reason); + return "pending" as const; + }); + return result ?? "not_found"; + } + + async function markStopped(tx: DBExecutor, run: Run): Promise { + // Forced termination may leave only a partial event log. The Hub records + // the outcome after the worker confirms its stop or the provisioner confirms + // destruction; it never reports cancellation while an unfenced worker lives. + // A pending projection means accepted history may hold an outcome a live + // row does not show yet, so the stop is recorded only after recovery clears + // it. The allocation lock keeps a new receive from advancing Git meanwhile. + if (await pendingProjections.hasAny(run.id, tx)) return; + const endedAt = now(); + const stopped = await tx + .update(workflowRun) + .set({ status: "cancelled", endedAt }) + .where( + and( + eq(workflowRun.anchorRunId, run.id), + inArray(workflowRun.status, [...liveWorkflowRunStatuses]), + ), + ) + .returning({ principalId: workflowRun.principalId }); + const principalIds = stopped.flatMap((row) => + row.principalId === null ? [] : [row.principalId], + ); + if (principalIds.length > 0) + await tx + .update(principal) + .set({ status: "deactivated", updatedAt: endedAt }) + .where(inArray(principal.id, principalIds)); + await dispatches.abandonUnsettled( + run.id, + "workflow_cancelled", + getRequestedCancellation(run).reason, + endedAt, + tx, + ); + } + + type ControlRequest = { + target: AllocatedSidecarTarget; + tenantId: string; + runId: string; + agentAddress: string; + action: "cancel" | "stop"; + reason: string; + timeoutMs: number; + cancellationDeadline: Date; + }; + + async function forceRelease(command: ControlRequest): Promise { + await withRun( + command.tenantId, + command.runId, + async (_tx, run, allocation, release) => { + if ( + !isLiveWorkflowRunStatus(run.status) || + allocation === undefined || + allocation.generation !== command.target.generation || + !isSidecarAllocationDispatchable(allocation.status) + ) + return; + await release({ + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedStatus: allocation.status, + failureCode: "workflow_stop_failed", + failureMessage: + "Workflow stop could not be confirmed; reclaiming capacity", + now: now(), + }); + }, + ); + } + + async function recordConfirmedStop(command: ControlRequest): Promise { + // An outcome that cannot be recorded yet is retried by the next sweep, + // which repeats the stop. + await recoverHistory(command.tenantId, command.runId, "stop"); + await withRun( + command.tenantId, + command.runId, + async (tx, run, allocation) => { + if ( + allocation?.generation === command.target.generation && + isLiveWorkflowRunStatus(run.status) + ) + await markStopped(tx, run); + }, + ); + } + + // Pack ingestion holds a live run's allocation lock for a whole receive, so + // a live run with nothing due skips the locked pass. Retention confirmation + // also needs no write; terminal runs lock only to derive or begin release. + async function needsLockedPass( + tenantId: string, + runId: string, + ): Promise { + const run = await db.query.workflowRun.findFirst({ + where: and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + eq(workflowRun.anchorRunId, runId), + ), + columns: { + status: true, + expiresAt: true, + cancellationRequestedAt: true, + infrastructureFailedAt: true, + capacityReleaseAt: true, + endedAt: true, + lifecyclePolicy: true, + }, + }); + if (run === undefined) return false; + if (run.infrastructureFailedAt !== null) return true; + if (!isLiveWorkflowRunStatus(run.status)) { + return run.capacityReleaseAt === null + ? workflowCapacityReleaseAt(run) !== null + : run.capacityReleaseAt <= now(); + } + return ( + run.cancellationRequestedAt !== null || + (run.expiresAt !== null && run.expiresAt <= now()) + ); + } + + async function retainTerminalAllocation(tenantId: string, runId: string) { + const [allocation] = await db + .select({ + id: sidecarAllocation.id, + sidecarId: sidecarAllocation.sidecarId, + generation: sidecarAllocation.generation, + status: workflowRun.status, + endedAt: workflowRun.endedAt, + capacityReleaseAt: workflowRun.capacityReleaseAt, + lifecyclePolicy: workflowRun.lifecyclePolicy, + }) + .from(sidecarAllocation) + .innerJoin(workflowRun, eq(workflowRun.id, sidecarAllocation.anchorRunId)) + .where( + and( + eq(workflowRun.id, runId), + eq(workflowRun.tenantId, tenantId), + notInArray(workflowRun.status, [...liveWorkflowRunStatuses]), + isNotNull(workflowRun.publicKey), + eq(sidecarAllocation.status, "allocated"), + eq( + sidecarAllocation.ensureAcceptedGeneration, + sidecarAllocation.generation, + ), + isNull(sidecarAllocation.initializationLeaseId), + ), + ); + if (allocation === undefined || allocation.sidecarId === null) + return undefined; + const target = { + allocationId: allocation.id, + generation: allocation.generation, + }; + if (!retentionRouter.holdsAllocatedBinding(target)) return undefined; + const retention = retentionRouter.getAllocationRetention(target); + const releaseAt = workflowCapacityReleaseAt(allocation); + // Overflow and policy expiry retain the existing accepted-history release + // policy. A kept decision alone does not prove the Hub received final history. + if (retention === "refused" || (releaseAt !== null && releaseAt <= now())) + return retention === undefined ? undefined : { ...target, retention }; + const previous = retentionBackoff.get(target.allocationId); + const backoff = + previous?.generation === target.generation ? previous : undefined; + if (backoff !== undefined && now().getTime() < backoff.nextAt) + return undefined; + if ( + !retentionsInFlight.has(target.allocationId) && + hasRetentionRoom(allocation.sidecarId) + ) { + retentionsInFlight.add(target.allocationId); + retentionsPerSidecar.set( + allocation.sidecarId, + (retentionsPerSidecar.get(allocation.sidecarId) ?? 0) + 1, + ); + void requestRetention( + target, + allocation.sidecarId, + runId, + backoff?.failures ?? 0, + ); + } + return retention === undefined ? undefined : { ...target, retention }; + } + + async function requestRetention( + target: AllocatedSidecarTarget, + sidecarId: string, + runId: string, + previousFailures: number, + ): Promise { + try { + await retentionRouter.retainAllocation(target, STOP_CONTROL_TIMEOUT_MS); + retentionBackoff.delete(target.allocationId); + } catch (cause) { + const failures = previousFailures + 1; + retentionBackoff.set(target.allocationId, { + generation: target.generation, + failures, + nextAt: + now().getTime() + + Math.min( + RECOVERY_BACKOFF_MAX_MS, + RECOVERY_BACKOFF_MIN_MS * 2 ** Math.min(failures - 1, 6), + ), + }); + logger.warn`Retention remains unconfirmed for ${runId}: ${cause instanceof Error ? cause.message : String(cause)}`; + } finally { + retentionsInFlight.delete(target.allocationId); + const count = retentionsPerSidecar.get(sidecarId); + if (count === undefined || count < 1) { + logger.error`Missing in-flight retention count for sidecar ${sidecarId} after retaining ${runId}`; + } else if (count === 1) { + retentionsPerSidecar.delete(sidecarId); + } else { + retentionsPerSidecar.set(sidecarId, count - 1); + } + } + } + + async function reconcileRun(tenantId: string, runId: string): Promise { + // Request retention before recovering history, but keep network waits out + // of the lifecycle workers so a silent sidecar cannot delay other runs' + // cancellation or expiry. The next sweep observes the router's confirmed + // retention, including refusals requiring release. + let retained: Awaited>; + try { + retained = await retainTerminalAllocation(tenantId, runId); + } catch (cause) { + logger.warn`Retention remains unconfirmed for ${runId}: ${cause instanceof Error ? cause.message : String(cause)}`; + } + await recoverHistory(tenantId, runId, "sweep"); + if ( + retained?.retention !== "refused" && + !(await needsLockedPass(tenantId, runId)) + ) + return; + const command = await withRun( + tenantId, + runId, + async ( + tx, + original, + allocation, + release, + ): Promise => { + // Capacity was lost for good while accepted history was unreconciled; + // once recovery has recorded that history, the runs still live fail. + if (original.infrastructureFailedAt !== null) { + await allocations.applyDeferredInfrastructureFailure(runId, tx); + return null; + } + let run = original; + if (isLiveWorkflowRunStatus(run.status)) { + if (run.expiresAt !== null && run.expiresAt <= now()) { + run = await beginCancellation( + tx, + run, + "Maximum deployment lifetime exceeded", + ); + } + if (run.cancellationRequestedAt === null) return null; + if ( + allocation === undefined || + allocation.status === "released" || + allocation.status === "failed" || + ((allocation.status === "releasing" || + allocation.status === "destroy_failed") && + allocation.deploymentCleanupConfirmed) + ) { + await markStopped(tx, run); + return null; + } + if ( + allocation.status === "releasing" || + allocation.status === "destroy_failed" + ) + return null; + if ( + allocation.status !== "allocated" || + allocation.sidecarId === null || + run.address === null + ) { + await release({ + allocationId: allocation.id, + expectedStatus: allocation.status, + expectedGeneration: allocation.generation, + now: now(), + }); + return null; + } + const cancellation = getRequestedCancellation(run); + const remaining = cancellation.deadline.getTime() - now().getTime(); + return { + target: { + allocationId: allocation.id, + generation: allocation.generation, + }, + tenantId, + runId, + agentAddress: run.address, + action: remaining > 0 ? "cancel" : "stop", + reason: cancellation.reason, + timeoutMs: + remaining > 0 + ? Math.max(1, Math.min(CANCEL_CONTROL_TIMEOUT_MS, remaining)) + : STOP_CONTROL_TIMEOUT_MS, + cancellationDeadline: cancellation.deadline, + }; + } + if ( + allocation === undefined || + !isSidecarAllocationDispatchable(allocation.status) + ) + return null; + if (run.status === "deployed" || run.status === "running") return null; + const refusal = + retained?.retention === "refused" && + retained.allocationId === allocation.id && + retained.generation === allocation.generation + ? { + failureCode: "sidecar_retention_limit_exceeded", + failureMessage: + "The sidecar could not retain this copy because its kept-record limit was reached", + } + : undefined; + if ( + refusal !== undefined && + !(await pendingProjections.hasAny(runId, tx)) + ) { + await release({ + allocationId: allocation.id, + expectedStatus: allocation.status, + expectedGeneration: allocation.generation, + ...refusal, + now: now(), + }); + return null; + } + const releaseAt = workflowCapacityReleaseAt(run); + if (releaseAt === null) return null; + if (run.capacityReleaseAt === null) + await tx + .update(workflowRun) + .set({ capacityReleaseAt: releaseAt }) + .where(eq(workflowRun.id, runId)); + if (releaseAt > now()) return null; + await release({ + allocationId: allocation.id, + expectedStatus: allocation.status, + expectedGeneration: allocation.generation, + ...refusal, + now: now(), + }); + return null; + }, + ); + if (command === null) return; + if (sendControl === undefined) { + logger.warn`Workflow control is unavailable for ${runId}`; + if (command.action === "stop") await forceRelease(command); + return; + } + try { + await sendControl( + command.target, + { + runId, + agentAddress: command.agentAddress, + action: command.action, + reason: command.reason, + }, + command.timeoutMs, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + // A live worker that stayed silent or refused has failed to stop. Any + // other failure is retried briefly, since this Hub may not hold the + // worker's connection yet, the worker may still be deploying, its + // acknowledgement may not have been processed in time, or its history + // may not have reached the Hub yet. Once the grace passes, no failure + // can postpone the stop further. + const failedDefinitively = + error instanceof WorkflowControlTimeoutError || + error instanceof WorkflowControlRejectedError; + const retriedTooLong = + now().getTime() >= + Math.max(command.cancellationDeadline.getTime(), startedAt) + + unreachableStopGraceMs; + if (command.action === "stop" && (failedDefinitively || retriedTooLong)) { + logger.warn`Workflow stop failed for ${runId}: ${message}`; + await forceRelease(command); + } else { + logger.info`Workflow ${command.action} for ${runId} deferred: ${message}`; + } + return; + } + if (command.action === "stop") await recordConfirmedStop(command); + } + + // A run whose work stays due, such as a cancellation waiting on its worker + // or history under backoff, remains eligible. Pause at the end of each pass + // so the scheduler's immediate refill cannot repeatedly scan it in a loop. + const sweep = createAnchorRunSweep( + createLifecycleSweepQueries(db, now, () => { + const candidates = retentionRouter.getRetentionCandidates(); + const current = new Map( + candidates.map((target) => [target.allocationId, target.generation]), + ); + for (const [id, backoff] of retentionBackoff) + if (current.get(id) !== backoff.generation) retentionBackoff.delete(id); + return candidates.filter((target) => { + // A confirmed refusal needs release, not a retention request. Other + // lifecycle branches still select expiry, cancellation and history work + // even when this retention branch has no request capacity. + if (retentionRouter.getAllocationRetention(target) === "refused") + return true; + if ( + retentionsInFlight.has(target.allocationId) || + !hasRetentionRoom(target.sidecarId) + ) + return false; + const backoff = retentionBackoff.get(target.allocationId); + return backoff === undefined || backoff.nextAt <= now().getTime(); + }); + }), + { + intervalMs: SWEEP_INTERVAL_MS, + now, + }, + ); + + async function reconcileNext(): Promise { + const run = await sweep.select(); + if (run === null) return false; + try { + await reconcileRun(run.tenantId, run.id); + } catch (error) { + logger.error`Lifecycle reconciliation failed for ${run.id}: ${error instanceof Error ? error.message : String(error)}`; + } finally { + sweep.release(run.id); + } + return true; + } + + return { + getStatus, + releaseCapacity, + releaseStoppedCleanup, + requestCancellation, + reconcileNext, + }; +} + +export type WorkflowLifecycleService = ReturnType< + typeof createWorkflowLifecycleService +>; + +/** + * Selects the deployments `reconcileRun` can act on: a live run that is + * cancelling or past its expiry, terminal copies needing retention or + * release, a deferred infrastructure failure, and + * accepted history left unprojected past its grace. A new action there needs a + * branch here. Deployments are never deleted, so each branch starts from an + * index over rows still active in its own sense and reaches an anchor only by + * key: a pass costs time in proportion to current capacity, not to every + * deployment ever created. Status lists are SQL literals rather than bound + * parameters: a plan made without parameter values could not otherwise use the + * partial indexes. + * + * Each branch keeps its next candidate for the rest of a pass and is read again + * only once the sweep takes that candidate, so a branch with nothing due is + * scanned once per pass rather than once per selection. Work that becomes due + * behind a branch's kept candidate waits for the next pass. + */ +export function createLifecycleSweepQueries( + db: DB["db"], + now: () => Date, + getRetentionCandidates: SidecarAllocationRouter["getRetentionCandidates"], +): AnchorRunSweepQueries<{ id: string; tenantId: string }> { + // Selection runs once per candidate of every pass, so it is built once. + function prepareBranches(afterCursor: boolean) { + const inPass = (column: AnyPgColumn) => + and( + ...(afterCursor ? [gt(column, sql.placeholder("afterRunId"))] : []), + lte(column, sql.placeholder("passEnd")), + sql`${column} <> all(${sql.placeholder("activeRunIds")})`, + ); + // An anchor is reached only by key from a candidate row. Postgres badly + // underestimates `id = anchor_run_id`, and a join would let it hash every + // deployment instead; a limited lateral lookup cannot be flattened into one. + const anchorOf = (anchorRunId: AnyPgColumn, condition?: SQL) => + db + .select({ present: sql`1`.as("present") }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, anchorRunId), + eq(workflowRun.anchorRunId, workflowRun.id), + condition, + ), + ) + .limit(1) + .as("anchor"); + // Retention work uses a connection snapshot; generation checks discard a + // candidate that changed while the pass ran. Release and history work do + // not depend on a connection or an outstanding retention decision. + const retentionDue = db + .select({ id: sidecarAllocation.anchorRunId }) + .from(sidecarAllocation) + .crossJoinLateral( + anchorOf( + sidecarAllocation.anchorRunId, + and( + notInArray(workflowRun.status, [...liveWorkflowRunStatuses]), + isNotNull(workflowRun.publicKey), + // Undated policies go through releaseDue, which derives the deadline. + or( + gt( + workflowRun.capacityReleaseAt, + sql.param( + sql.placeholder("now"), + workflowRun.capacityReleaseAt, + ), + ), + and( + isNull(workflowRun.capacityReleaseAt), + sql`${workflowRun.lifecyclePolicy} -> 'capacityRetention' ->> ${workflowRun.status} is null`, + ), + ), + ), + ), + ) + .where( + and( + eq(sidecarAllocation.status, "allocated"), + sql`${sidecarAllocation.id} = any(${sql.placeholder("retentionIds")}::text[])`, + sql`${sidecarAllocation.generation} = (${sql.placeholder("retentionGenerations")}::jsonb ->> ${sidecarAllocation.id})::integer`, + eq( + sidecarAllocation.ensureAcceptedGeneration, + sidecarAllocation.generation, + ), + isNull(sidecarAllocation.initializationLeaseId), + inPass(sidecarAllocation.anchorRunId), + ), + ) + .orderBy(asc(sidecarAllocation.anchorRunId)) + .limit(1); + const releaseDue = db + .select({ id: sidecarAllocation.anchorRunId }) + .from(sidecarAllocation) + .crossJoinLateral( + anchorOf( + sidecarAllocation.anchorRunId, + and( + notInArray(workflowRun.status, [...liveWorkflowRunStatuses]), + or( + lte( + workflowRun.capacityReleaseAt, + sql.param( + sql.placeholder("now"), + workflowRun.capacityReleaseAt, + ), + ), + and( + isNull(workflowRun.capacityReleaseAt), + sql`${workflowRun.lifecyclePolicy} -> 'capacityRetention' ->> ${workflowRun.status} is not null`, + ), + ), + ), + ), + ) + .where( + and( + sql`${sidecarAllocation.status} in ('pending', 'provisioning', 'allocated', 'replacing')`, + inPass(sidecarAllocation.anchorRunId), + ), + ) + .orderBy(asc(sidecarAllocation.anchorRunId)) + .limit(1); + const liveDue = db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + sql`${workflowRun.id} = ${workflowRun.anchorRunId} and ${workflowRun.status} in ('deployed', 'running')`, + or( + isNotNull(workflowRun.cancellationRequestedAt), + lte( + workflowRun.expiresAt, + sql.param(sql.placeholder("now"), workflowRun.expiresAt), + ), + ), + inPass(workflowRun.id), + ), + ) + .orderBy(asc(workflowRun.id)) + .limit(1); + const infrastructureFailed = db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where( + and( + isNotNull(workflowRun.infrastructureFailedAt), + eq(workflowRun.id, workflowRun.anchorRunId), + inPass(workflowRun.id), + ), + ) + .orderBy(asc(workflowRun.id)) + .limit(1); + // Any deployment, live or not, whose accepted history may still be + // unprojected. + const unprojectedHistory = db + .select({ id: workflowPendingProjection.anchorRunId }) + .from(workflowPendingProjection) + .crossJoinLateral(anchorOf(workflowPendingProjection.anchorRunId)) + .where( + and( + lte( + workflowPendingProjection.createdAt, + sql.param( + sql.placeholder("pendingCutoff"), + workflowPendingProjection.createdAt, + ), + ), + inPass(workflowPendingProjection.anchorRunId), + ), + ) + .orderBy(asc(workflowPendingProjection.anchorRunId)) + .limit(1); + const position = afterCursor ? "next" : "first"; + return { + retention: retentionDue.prepare(`lifecycle_sweep_retention_${position}`), + release: releaseDue.prepare(`lifecycle_sweep_release_${position}`), + live: liveDue.prepare(`lifecycle_sweep_live_${position}`), + infrastructure: infrastructureFailed.prepare( + `lifecycle_sweep_infrastructure_${position}`, + ), + history: unprojectedHistory.prepare( + `lifecycle_sweep_history_${position}`, + ), + }; + } + const first = prepareBranches(false); + const next = prepareBranches(true); + const branches = ( + ["retention", "release", "live", "infrastructure", "history"] as const + ).map((name) => ({ name, first: first[name], next: next[name] })); + // Postgres picks the lowest candidate, so the choice follows the same + // collation as the cursor comparisons. + const lowest = db + .select({ id: workflowRun.id, tenantId: workflowRun.tenantId }) + .from(workflowRun) + .where( + eq( + workflowRun.id, + sql`least(${sql.join( + branches.map( + (_, index) => sql`${sql.placeholder(`candidate${index}`)}::text`, + ), + sql`, `, + )})`, + ), + ) + .prepare("lifecycle_sweep_lowest"); + // Undefined until the branch is read this pass, null once it has nothing left. + let candidates: (string | null | undefined)[] = []; + let retentionIds: string[] = []; + let retentionGenerations = "{}"; + return { + async findPassEnd() { + const [last] = await db + .select({ id: workflowRun.id }) + .from(workflowRun) + .where(eq(workflowRun.id, workflowRun.anchorRunId)) + .orderBy(desc(workflowRun.id)) + .limit(1); + return last?.id; + }, + async findNext({ afterRunId, passEnd, activeRunIds }) { + if (afterRunId === undefined) { + candidates = branches.map(() => undefined); + const retention = getRetentionCandidates(); + retentionIds = retention.map((target) => target.allocationId); + retentionGenerations = JSON.stringify( + Object.fromEntries( + retention.map((target) => [target.allocationId, target.generation]), + ), + ); + } + const at = now(); + const values = { + passEnd, + activeRunIds, + retentionIds, + retentionGenerations, + now: at, + pendingCutoff: new Date(at.getTime() - PENDING_PROJECTION_GRACE_MS), + }; + for (const [index, branch] of branches.entries()) { + if (branch.name === "retention" && retentionIds.length === 0) { + candidates[index] = null; + continue; + } + const candidate = candidates[index]; + if ( + candidate === null || + (candidate !== undefined && candidate !== afterRunId) + ) + continue; + const [row] = + afterRunId === undefined + ? await branch.first.execute(values) + : await branch.next.execute({ ...values, afterRunId }); + candidates[index] = row?.id ?? null; + } + if (candidates.every((candidate) => candidate === null)) return undefined; + const [selected] = await lowest.execute( + Object.fromEntries( + candidates.map((candidate, index) => [ + `candidate${index}`, + candidate, + ]), + ), + ); + return selected; + }, + }; +} diff --git a/vendor/intx/hub-sessions/src/workflow-probe-gate.ts b/vendor/intx/hub-sessions/src/workflow-probe-gate.ts index 52cb9c4cc2..f43b989d12 100644 --- a/vendor/intx/hub-sessions/src/workflow-probe-gate.ts +++ b/vendor/intx/hub-sessions/src/workflow-probe-gate.ts @@ -52,6 +52,7 @@ import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; import type { WorkflowDefinitionAssetSource, WorkflowDefinitionRegistrySource, + WorkflowDefinitionSource, } from "@intx/types/workflow-sources"; import { EXECUTABLE_STEP_DESCENT, @@ -59,9 +60,12 @@ import { } from "@intx/workflow/definition"; import { approvalSetFromItems, + checkCredentialBindings, + describeCredentialBindingViolation, inertNestedBodies, isApprovedGrantRequirement, type ApprovalSet, + type CredentialBindingViolation, } from "@intx/workflow-deploy"; import { @@ -118,11 +122,13 @@ export type PersistFrozenApprovalFn = ( /** * The outcome of gating and freezing a probe result. `ok: true` is the frozen - * approval the deploy hand-off consumes. The `ok: false` arms name the five - * fail-closed paths: a shipped hash that does not match the hub recompute + * approval the deploy hand-off consumes. The `ok: false` arms name the + * fail-closed paths: unsupported creator provenance from a package-registry + * tarball, a shipped hash that does not match the hub recompute * (tamper-evidence), advisory grants the operator did not approve, declared * grant requirements the operator did not approve, a trigger type the runtime - * does not implement, and an executable step the grant walk left no record for. + * does not implement, credential bindings that cannot be delivered, and an + * executable step the grant walk left no record for. */ export type ProbeGateResult = | { @@ -158,12 +164,26 @@ export type ProbeGateResult = */ readonly unapprovedGrantRequirements: readonly GrantRequirement[]; } + | { + readonly ok: false; + readonly reason: "creator_requirements_unsupported_for_package_registry_tarball"; + /** Creator-sourced requirements rejected because the package publisher is unknown. */ + readonly creatorGrantRequirements: readonly GrantRequirement[]; + } | { readonly ok: false; readonly reason: "unimplemented_trigger"; /** The distinct reserved-but-unimplemented trigger types the projection declared. */ readonly unimplementedTriggerTypes: readonly string[]; } + | { + readonly ok: false; + readonly reason: "credential_bindings_invalid"; + /** Every credential binding problem, in walk order. */ + readonly violations: readonly CredentialBindingViolation[]; + /** One sentence per violation, joined. */ + readonly message: string; + } | { readonly ok: false; readonly reason: "steps_without_grant_record"; @@ -201,6 +221,43 @@ export type StepWithoutGrantRecord = { readonly recordStepId: string; }; +/** A gate outcome that refused the definition. */ +export type ProbeGateRefusal = Extract; + +// A requirement is named by what it asks for and whose authority it draws on. +// Its conditions are left out: they are free-form and can be large, and the +// action, resource, and source are what tell the author which one to change. +function describeGrantRequirement(requirement: GrantRequirement): string { + return `${requirement.action} on ${requirement.resource} (${requirement.source})`; +} + +/** + * Name why the gate refused a definition and what it refused, for the error + * the deploy caller receives. Every refusal starts with its reason code. + */ +export function describeProbeGateRefusal(refusal: ProbeGateRefusal): string { + switch (refusal.reason) { + case "wire_hash_mismatch": + return `wire_hash_mismatch: the shipped hash ${refusal.shippedWireHash} does not match the recomputed hash ${refusal.recomputedWireHash}`; + case "grants_not_approved": + return `grants_not_approved: ${refusal.unapprovedGrants.join(", ")}`; + case "grant_requirements_not_approved": + return `grant_requirements_not_approved: ${refusal.unapprovedGrantRequirements.map(describeGrantRequirement).join("; ")}`; + case "creator_requirements_unsupported_for_package_registry_tarball": + return `creator_requirements_unsupported_for_package_registry_tarball: a workflow from a package-registry tarball cannot declare creator-sourced grant requirements, because the hub does not know who published the package; declared: ${refusal.creatorGrantRequirements.map(describeGrantRequirement).join("; ")}`; + case "unimplemented_trigger": + return `unimplemented_trigger: ${refusal.unimplementedTriggerTypes.join(", ")}`; + case "credential_bindings_invalid": + return `credential_bindings_invalid: ${refusal.message}`; + case "steps_without_grant_record": + return `steps_without_grant_record: ${refusal.message}`; + default: { + const _exhaustive: never = refusal; + return `unknown gate refusal: ${JSON.stringify(_exhaustive)}`; + } + } +} + /** * Build the production persistence step of the freeze. Records identity through * the selector-keyed ensure helper (a definition keyed by `(assetId, @@ -386,8 +443,10 @@ function describeStepsWithoutGrantRecord( } export type GateAndFreezeArgs = { - /** The `workflow`-kind asset the frozen definition projects over. */ + /** The definition asset: a workflow source tree or package-registry tarball. */ readonly assetId: string; + /** The definition source; the creator-provenance check keys on its format. */ + readonly source: WorkflowDefinitionSource; /** The sidecar's inert probe answer: projection, advisory grants, shipped hash. */ readonly probeResult: WorkflowProbeResult; /** @@ -405,13 +464,16 @@ export type GateAndFreezeArgs = { * inert projection and grant set -- no author code runs here and the capability * walk is never re-run. * - * Fails closed on the three security-load-bearing checks before it writes - * anything: the recomputed wire hash must match the hash the sidecar shipped - * (tamper-evidence), every advisory grant must be operator-approved, and every - * declared grant requirement must be operator-approved. It also refuses a - * projection whose triggers include a reserved-but-unimplemented type -- not a + * Fails closed on the security-load-bearing checks before it writes anything: + * a package-registry tarball cannot claim creator authority while package + * publisher provenance is unavailable, the recomputed wire hash must match the + * hash the sidecar shipped (tamper-evidence), every advisory grant must be + * operator-approved, and every declared grant requirement must be + * operator-approved. It also refuses a projection whose triggers include a + * reserved-but-unimplemented type -- not a * security check, but the layer a pinned closure cannot carry a stale copy of, - * so it is where a workflow that could only sit inert is caught -- and one + * so it is where a workflow that could only sit inert is caught -- one whose + * credential bindings cannot be delivered to a consumer it runs, and one * whose executable closure reaches a step the grant walk left no record for -- * the layer holding both halves of the probe answer at once, and a presence * check on those records rather than a check that any record's contents are @@ -422,7 +484,21 @@ export type GateAndFreezeArgs = { export async function gateAndFreezeProbeResult( args: GateAndFreezeArgs, ): Promise { - const { assetId, probeResult, approvals, persist } = args; + const { assetId, source, probeResult, approvals, persist } = args; + + const creatorGrantRequirements = + source.kind === "asset" && source.package.format === "tarball" + ? probeResult.grantWalkSnapshot.grantRequirements.filter( + (requirement) => requirement.source === "creator", + ) + : []; + if (creatorGrantRequirements.length > 0) { + return { + ok: false, + reason: "creator_requirements_unsupported_for_package_registry_tarball", + creatorGrantRequirements, + }; + } // Tamper-evidence: recompute over the RECEIVED projection and compare to the // shipped hash. A mismatch means the projection the hub is approving is not @@ -459,6 +535,29 @@ export async function gateAndFreezeProbeResult( }; } + // Refuse credential bindings that cannot be delivered: a binding on a nested + // body, a binding whose consumer the deployment never runs, a duplicate + // binding, and a declared action handle with no binding. Like the trigger + // check this is an author-definition error, so it runs before the operator + // checks below; an operator cannot approve it away. It runs here, at install + // and approve, over the projection the freeze hashes -- before any credential + // is resolved or decrypted. A deployment approved earlier is not re-checked + // when it respawns. Tool consumers are keyed on the source-ref factory id, + // the only lineage that delivers bindings today. + const credentialBindingViolations = checkCredentialBindings( + probeResult.projection, + ); + if (credentialBindingViolations.length > 0) { + return { + ok: false, + reason: "credential_bindings_invalid", + violations: credentialBindingViolations, + message: credentialBindingViolations + .map(describeCredentialBindingViolation) + .join("; "), + }; + } + // Totality: every step the deployment can execute must have an approved-grant // record behind it. The two halves of a probe answer are produced // independently -- the projection by the hub's live->inert projector, the @@ -558,7 +657,7 @@ export async function gateAndFreezeProbeResult( type InstallAndApproveCommonArgs = { /** The `interchange.workflow` entry-module path the sidecar evaluates to project the definition. */ readonly entry: string; - /** The `workflow`-kind asset the frozen definition projects over. */ + /** The definition asset: a workflow source tree or package-registry tarball. */ readonly assetId: string; /** * The approval policy threaded to the gate: an operator `ApprovalSet` to gate @@ -727,6 +826,7 @@ export async function installAndApproveWorkflowDefinition( const approval = await gateAndFreezeProbeResult({ assetId: args.assetId, + source: args.source, probeResult, approvals: args.approvals, persist: createDbFrozenApprovalWriter(args.db), diff --git a/vendor/intx/hub-sessions/src/workflow-run-ended-at.ts b/vendor/intx/hub-sessions/src/workflow-run-ended-at.ts new file mode 100644 index 0000000000..19a5eb945a --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-run-ended-at.ts @@ -0,0 +1,24 @@ +import { type } from "arktype"; + +const TerminalEventTime = type({ "at?": "string" }); + +/** Bound event time to the run's lifetime; absent or invalid times use observation. */ +export function getWorkflowRunEndedAt( + eventBody: unknown, + createdAt: Date, + observedAt: Date, +): Date { + const event = TerminalEventTime(eventBody); + const at = + event instanceof type.errors || event.at === undefined + ? NaN + : Date.parse(event.at); + return new Date( + Math.max( + createdAt.getTime(), + Number.isFinite(at) + ? Math.min(observedAt.getTime(), at) + : observedAt.getTime(), + ), + ); +} diff --git a/vendor/intx/hub-sessions/src/workflow-run-kind.ts b/vendor/intx/hub-sessions/src/workflow-run-kind.ts index a2ffc1e1f8..57359f692c 100644 --- a/vendor/intx/hub-sessions/src/workflow-run-kind.ts +++ b/vendor/intx/hub-sessions/src/workflow-run-kind.ts @@ -166,6 +166,7 @@ import fs from "node:fs"; import git from "isomorphic-git"; import { type } from "arktype"; import { getLogger } from "@intx/log"; +import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy/addresses"; import { authorizeUserPrincipal, type AuthorizeFn, @@ -186,6 +187,17 @@ import { const logger = getLogger(["hub-sessions", "workflow-run-kind"]); +// Workflow-run events commit on the substrate's default branch; the +// supervisor wires the workflow-process child against this ref. +export const WORKFLOW_RUN_REF = "refs/heads/main"; + +export function workflowRunRepoIdForAddress(agentAddress: string) { + return { + kind: "workflow-run", + id: deriveWorkflowRunRepoId(agentAddress), + } as const satisfies RepoId; +} + export type WorkflowRunHubPrincipal = { readonly kind: "hub" }; export type WorkflowRunSidecarPrincipal = { @@ -2636,11 +2648,8 @@ export const workflowRunKindHandler: KindHandler = { // through the tip from whatever seq the first entry uses. Without // this, a downstream consumer that iterates the log by seq would // skip past a gap silently. `entries` is sorted by filenameSeq - // above. The first seq is not pinned to 0 because the runtime - // body's emptyState carries `lastSeq = 0` and emits its first - // event at `seq = lastSeq + 1 = 1`, while the supervisor's - // self-signed CancelRequested path lands seq=0 against an empty - // events tree. + // above. Producers start new logs at seq 1; the substrate checks + // contiguity independently of the first entry's sequence number. const firstEntry = entries[0]; if (firstEntry === undefined) throw new Error("unreachable"); const baseSeq = firstEntry.filenameSeq; diff --git a/vendor/intx/hub-sessions/src/workflow-run-reader.ts b/vendor/intx/hub-sessions/src/workflow-run-reader.ts index 65bba947d8..74753e16de 100644 --- a/vendor/intx/hub-sessions/src/workflow-run-reader.ts +++ b/vendor/intx/hub-sessions/src/workflow-run-reader.ts @@ -1,4 +1,5 @@ import fs from "node:fs"; +import path from "node:path"; import git from "isomorphic-git"; import type { RepoId } from "./repo-store/types"; @@ -61,6 +62,28 @@ export interface WorkflowRunReader { ref: string, runId: string, ): Promise; + /** + * List the runs present under `runs/` at one pinned ref tip and read the + * latest event of each run `include` selects, without reading prior blobs. + * The returned map holds exactly the selected runs, with `null` for a run + * directory that has no events yet. A missing Git object propagates + * rather than reading as absent history. + */ + readLatestRunEvents( + repoId: RepoId, + ref: string, + include: (runId: string) => boolean, + ): Promise<{ + tip: string | null; + events: Map; + }>; + /** Recheck the ref after taking the allocation lock. */ + resolveRefTip(repoId: RepoId, ref: string): Promise; + /** + * Whether the repository exists on disk, separating a repository whose ref + * was never written from one that is missing entirely. + */ + hasRepository(repoId: RepoId): Promise; } export function createWorkflowRunReader( @@ -203,6 +226,120 @@ export function createWorkflowRunReader( return events; } + async function resolveRefTip( + repoId: RepoId, + ref: string, + ): Promise { + const dir = repoDirOrNull(repoId); + return dir === null ? null : resolveRefOrNull(dir, ref); + } + + async function hasRepository(repoId: RepoId): Promise { + const dir = repoDirOrNull(repoId); + if (dir === null) return false; + try { + await fs.promises.stat(path.join(dir, ".git")); + return true; + } catch (cause) { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") + return false; + throw cause; + } + } + + async function readLatestRunEvents( + repoId: RepoId, + ref: string, + include: (runId: string) => boolean, + ): Promise<{ + tip: string | null; + events: Map; + }> { + const events = new Map(); + const dir = repoDirOrNull(repoId); + if (dir === null) return { tip: null, events }; + const tip = await resolveRefOrNull(dir, ref); + if (tip === null) return { tip, events }; + // Walk trees by oid rather than by filepath: isomorphic-git reports an + // absent path and a missing object with the same NotFoundError, and + // callers act on an empty result, so only a path absent from a readable + // parent tree may read as "no history". + const root = await git.readTree({ fs, dir, oid: tip }); + const runsEntry = root.tree.find( + (entry) => entry.path === WORKFLOW_RUN_RUNS_PREFIX, + ); + if (runsEntry === undefined) return { tip, events }; + const runsTree = await git.readTree({ fs, dir, oid: runsEntry.oid }); + for (const entry of runsTree.tree) { + if (entry.type !== "tree" || !include(entry.path)) continue; + events.set( + entry.path, + await readLatestEventAt(dir, entry.path, entry.oid), + ); + } + return { tip, events }; + } + + async function readLatestEventAt( + dir: string, + runId: string, + runTreeOid: string, + ): Promise { + const runDir = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`; + const runTree = await git.readTree({ fs, dir, oid: runTreeOid }); + const combined = runTree.tree.find( + (entry) => + entry.type === "blob" && entry.path === WORKFLOW_RUN_EVENTS_FILE, + ); + const perEventDir = runTree.tree.find( + (entry) => + entry.type === "tree" && entry.path === WORKFLOW_RUN_EVENTS_DIR, + ); + if (combined !== undefined && perEventDir !== undefined) { + throw new Error( + `workflow-run reader: run ${runId} carries both a combined ${WORKFLOW_RUN_EVENTS_FILE} and a per-event ${WORKFLOW_RUN_EVENTS_DIR}/ directory`, + ); + } + if (combined !== undefined) { + const blob = await git.readBlob({ fs, dir, oid: combined.oid }); + const source = `${runDir}/${WORKFLOW_RUN_EVENTS_FILE}`; + const last = splitCombinedEventLog( + new TextDecoder().decode(blob.blob), + ).at(-1); + return last === undefined ? null : parseRunEventLine(last, source); + } + if (perEventDir === undefined) return null; + const eventsDir = `${runDir}/${WORKFLOW_RUN_EVENTS_DIR}`; + const tree = await git.readTree({ fs, dir, oid: perEventDir.oid }); + let latest: { seq: number; oid: string; path: string } | undefined; + for (const entry of tree.tree) { + if (entry.type !== "blob") continue; + const path = `${eventsDir}/${entry.path}`; + const seq = requireEventSeq(entry.path, path); + if (latest === undefined || seq > latest.seq) + latest = { seq, oid: entry.oid, path }; + } + if (latest === undefined) return null; + const blob = await git.readBlob({ fs, dir, oid: latest.oid }); + const parsed = parseEventObject( + new TextDecoder().decode(blob.blob), + latest.path, + ); + const eventType = parsed["type"]; + if (typeof eventType !== "string") { + throw new Error( + `workflow-run reader: event at ${latest.path} is missing a string \`type\` field`, + ); + } + const bodySeq = parsed["seq"]; + if (typeof bodySeq === "number" && bodySeq !== latest.seq) { + throw new Error( + `workflow-run reader: event at ${latest.path} body seq ${String(bodySeq)} does not match filename seq ${String(latest.seq)}`, + ); + } + return { seq: latest.seq, type: eventType, body: parsed }; + } + // Parse one combined-log line (the verbatim text of a former // `events/.json` blob): the seq is read from the body, since the // combined form drops the per-event filename that carried it. @@ -240,5 +377,11 @@ export function createWorkflowRunReader( return { ...parsed }; } - return { listRunIds, readRunEvents }; + return { + listRunIds, + readRunEvents, + readLatestRunEvents, + resolveRefTip, + hasRepository, + }; } diff --git a/vendor/intx/hub-sessions/src/workflow-run-restore.ts b/vendor/intx/hub-sessions/src/workflow-run-restore.ts index 316b6575cb..0e161e824b 100644 --- a/vendor/intx/hub-sessions/src/workflow-run-restore.ts +++ b/vendor/intx/hub-sessions/src/workflow-run-restore.ts @@ -1,21 +1,40 @@ -import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; +import type { WorkflowRunRefTips } from "@intx/types/sidecar"; import type { AgentRepoStore } from "./agent-repo"; +import type { RepoStore } from "./repo-store"; +import { + workflowRunRepoIdForAddress, + WORKFLOW_RUN_REF, +} from "./workflow-run-kind"; import type { AllocatedSidecarTarget, SidecarAllocationRouter, } from "./ws/sidecar-handler"; export const WORKFLOW_RUN_RESTORE_REFS = [ - "refs/heads/main", + WORKFLOW_RUN_REF, "refs/heads/events", ] as const; +/** Read the Hub's tip of each authoritative ref of a deployment's history. */ +export async function readWorkflowRunRefTips( + repoStore: Pick, + agentAddress: string, +): Promise { + const repoId = workflowRunRepoIdForAddress(agentAddress); + const tips: WorkflowRunRefTips = {}; + for (const ref of WORKFLOW_RUN_RESTORE_REFS) + tips[ref] = await repoStore.resolveRef({ kind: "hub" }, repoId, ref); + return tips; +} + /** - * Replay every authoritative workflow-run ref the runtime understands onto an - * exact replacement allocation. Refs are sent sequentially and the function - * resolves only after the worker acknowledges each one, making it a barrier - * the deploy path can place before supervisor spawn. + * Seed a first deploy on its exact allocation with every workflow-run ref the + * Hub holds for the deployment. The Hub holds none unless the deployment was + * seeded, since its own copy only grows from the worker's pushes. Refs are + * sent sequentially and the function resolves only after the worker + * acknowledges each one, making it a barrier the deploy path can place before + * supervisor spawn. */ export async function restoreWorkflowRunToAllocation(args: { agentRepoStore: AgentRepoStore; @@ -30,10 +49,7 @@ export async function restoreWorkflowRunToAllocation(args: { const { agentRepoStore, allocationRouter, allocationTarget, agentAddress } = args; const principal = { kind: "hub" } as const; - const repoId = { - kind: "workflow-run" as const, - id: deriveWorkflowRunRepoId(agentAddress), - }; + const repoId = workflowRunRepoIdForAddress(agentAddress); for (const ref of WORKFLOW_RUN_RESTORE_REFS) { args.signal?.throwIfAborted(); diff --git a/vendor/intx/hub-sessions/src/workflow-run-terminal-projection.ts b/vendor/intx/hub-sessions/src/workflow-run-terminal-projection.ts new file mode 100644 index 0000000000..668258389b --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-run-terminal-projection.ts @@ -0,0 +1,108 @@ +import { and, eq } from "drizzle-orm"; + +import type { DBExecutor, WorkflowRunStore } from "@intx/db"; +import { principal, workflowRun } from "@intx/db/schema"; + +import { getWorkflowRunEndedAt } from "./workflow-run-ended-at"; + +export type TerminalProjectionAnchor = { + id: string; + tenantId: string; + definitionId: string; + createdAt: Date; +}; + +/** + * `projected` and `settled` both leave the row reflecting accepted history; + * `foreign` means the run id belongs to another deployment and was ignored. + * Every outcome is a final decision for the event. + */ +export type TerminalProjectionOutcome = "projected" | "settled" | "foreign"; + +/** + * Project one accepted terminal event onto its `workflow_run` row. Pack + * ingestion and lifecycle recovery both settle runs through this, so a run's + * status, end time, and principal follow the same rules on either path. + */ +export async function projectTerminalRun( + tx: DBExecutor, + runs: WorkflowRunStore, + args: { + anchor: TerminalProjectionAnchor; + runId: string; + status: "completed" | "failed" | "cancelled"; + terminalEvent: unknown; + now: Date; + }, +): Promise { + const { anchor, runId, status, terminalEvent, now } = args; + // Lazily anchor the run before settling it. An internal run that parks only + // on a plain signal gate never reaches `registerSignalCorrelation`, the sole + // other path that mints an internal run row, so its terminal event can be the + // first the hub sees of the run. A never-minted row is ordinary bookkeeping, + // not a deployment-boundary violation, so mint it here against this + // deployment's anchor rather than letting the ownership guard below mistake + // absence for foreignness. The insert no-ops when any row already exists, + // which keeps that guard authoritative for a row that exists and anchors + // elsewhere. The principal is null: an internal run inherits its + // deployment's grants and has none of its own. + // + // The mint necessarily precedes the ownership guard, so an id the hub has + // never seen is claimed under THIS anchor before anything establishes it + // belongs here. That ordering is required -- the guard reads the row the mint + // may have to create -- and it is bounded rather than unbounded: internal run + // ids are supplied by the sidecar and accepted verbatim, so the value is + // caller-influenced, but it is a different population from the anchor ids + // the hub mints itself, and nothing resolves an internal id without also + // constraining the anchor or the tenant. The insert cannot take a row away + // from another deployment; the worst it does is create one for an id that + // deployment would otherwise have created later. + // + // A row minted here may be written long after the run ended, so it starts + // at the terminal event's time rather than now; otherwise the end time + // below would be clamped forward to the mint. + await runs.createIfAbsent( + { + id: runId, + anchorRunId: anchor.id, + definitionId: anchor.definitionId, + tenantId: anchor.tenantId, + principalId: null, + status: "running", + createdAt: getWorkflowRunEndedAt(terminalEvent, anchor.createdAt, now), + }, + tx, + ); + const [ownedRun] = await tx + .select({ + anchorRunId: workflowRun.anchorRunId, + createdAt: workflowRun.createdAt, + }) + .from(workflowRun) + .where(eq(workflowRun.id, runId)) + .limit(1); + if (ownedRun?.anchorRunId !== anchor.id) return "foreign"; + const endedAt = getWorkflowRunEndedAt(terminalEvent, ownedRun.createdAt, now); + const won = await runs.markTerminal(runId, status, endedAt, tx); + // The row exists (the mint above guarantees it) and belongs to this + // deployment (the guard above), so no live row matched only because the run + // is already terminal. Leave its settled status and `endedAt` alone. + if (won === null) return "settled"; + // Deactivate the run's own principal, if it has one. Externally-triggered + // runs carry a principal; internal, workflow-spawned runs have + // `principalId = null` and inherit the deployment's grants, so there is + // nothing to deactivate. Deactivation is gated on winning the flip -- the + // single claim point -- not on the principal's own status. + if (won.principalId !== null) { + await tx + .update(principal) + .set({ status: "deactivated", updatedAt: now }) + // `won.principalId` is already this run's own principal and + // `principal.id` is the primary key, so the `refId` match only confirms + // the id we won belongs to this run. + .where( + and(eq(principal.id, won.principalId), eq(principal.refId, runId)), + ); + } + return "projected"; +} diff --git a/vendor/intx/hub-sessions/src/workflow-source-repo-kind.ts b/vendor/intx/hub-sessions/src/workflow-source-repo-kind.ts new file mode 100644 index 0000000000..0d604a7021 --- /dev/null +++ b/vendor/intx/hub-sessions/src/workflow-source-repo-kind.ts @@ -0,0 +1,25 @@ +import type { WorkflowDefinitionAssetSource } from "@intx/types/workflow-sources"; + +import type { RepoKind } from "./repo-store"; + +/** + * The repo kind that holds each format of an asset-sourced workflow + * definition. A `source` codebase lives in a `workflow` asset; a packed + * `tarball` lives in a `package-registry` asset, the same kind that publishes + * tool packages. The table is keyed by the format union, so a new format does + * not compile until it names its kind, and the route that admits a deploy and + * the resolvers that pack the asset read the same answer. + */ +const REPO_KIND_BY_FORMAT: Record< + WorkflowDefinitionAssetSource["package"]["format"], + RepoKind +> = { + source: "workflow", + tarball: "package-registry", +}; + +export function workflowSourceRepoKind( + source: WorkflowDefinitionAssetSource, +): RepoKind { + return REPO_KIND_BY_FORMAT[source.package.format]; +} diff --git a/vendor/intx/hub-sessions/src/ws/index.ts b/vendor/intx/hub-sessions/src/ws/index.ts index 462690a3e0..5e67939b92 100644 --- a/vendor/intx/hub-sessions/src/ws/index.ts +++ b/vendor/intx/hub-sessions/src/ws/index.ts @@ -1,13 +1,15 @@ export { createSidecarRouter, SidecarIdentityValidationError, + WorkflowControlHistoryPendingError, + WorkflowControlInitializingError, + WorkflowControlRejectedError, + WorkflowControlTimeoutError, + WorkflowControlUnconfirmedError, + WorkflowControlUnreachableError, type SidecarRouter, type SidecarRouterConfig, type SenderDeploySettledOutcome, - type OAuthLoginFinalOutcome, - type OAuthLoginGateOutcome, - type OAuthLoginRequestOutcome, - DEFAULT_OAUTH_LOGIN_TIMEOUT_MS, type AllocatedSenderDeployAttempt, type SidecarConnection, type SidecarAuthIdentity, diff --git a/vendor/intx/hub-sessions/src/ws/pending-tracker.ts b/vendor/intx/hub-sessions/src/ws/pending-tracker.ts index ca7f422ab0..2c14f1057b 100644 --- a/vendor/intx/hub-sessions/src/ws/pending-tracker.ts +++ b/vendor/intx/hub-sessions/src/ws/pending-tracker.ts @@ -39,7 +39,7 @@ const scheduleOnGlobalTimer: ScheduleTimeout = (handler, ms) => { }; }; -export type PendingEntry = { +export type PendingEntry = { key: Key; ws: WsHandle; /** @@ -50,7 +50,7 @@ export type PendingEntry = { */ meta: Meta; resolve(value: Value): void; - reject(error: string): void; + reject(error: string | Failure): void; /** * Disarms this entry's timeout. * @@ -70,12 +70,25 @@ export type PendingEntry = { * `token` can, because it is allocated per `register` call. It is internal * to the tracker: `get` hands callers the `PendingEntry` view. */ -type TrackedEntry = PendingEntry & { +type TrackedEntry = PendingEntry< + Key, + Value, + Meta, + Failure +> & { readonly token: symbol; }; -export class PendingTracker { - private readonly entries = new Map>(); +export class PendingTracker< + Key, + Value = void, + Meta = undefined, + Failure = never, +> { + private readonly entries = new Map< + Key, + TrackedEntry + >(); /** * `schedule` defaults to the global timer, which is what every production @@ -103,7 +116,7 @@ export class PendingTracker { timeoutMs: number; timeoutMessage: string; resolve(value: Value): void; - reject(error: string): void; + reject(error: string | Failure): void; }, meta: Meta, ): void { @@ -142,7 +155,7 @@ export class PendingTracker { return this.entries.has(key); } - get(key: Key): PendingEntry | undefined { + get(key: Key): PendingEntry | undefined { return this.entries.get(key); } @@ -163,7 +176,7 @@ export class PendingTracker { * Settle a pending entry as rejected. No-op when the entry is already * gone (timed out or swept on disconnect). */ - reject(key: Key, error: string): boolean { + reject(key: Key, error: string | Failure): boolean { const entry = this.entries.get(key); if (entry === undefined) return false; entry.cancelTimeout(); @@ -192,8 +205,20 @@ export class PendingTracker { * bookkeeping), identical to a frame-error rejection. */ rejectAllForWs(ws: WsHandle, error: string): void { + this.rejectForWs(ws, () => true, error); + } + + /** + * Reject the entries owned by `ws` that `matches` selects -- the sweep for + * one binding leaving a connection that stays open for its other bindings. + */ + rejectForWs( + ws: WsHandle, + matches: (entry: PendingEntry) => boolean, + error: string, + ): void { for (const [key, entry] of this.entries) { - if (entry.ws !== ws) continue; + if (entry.ws !== ws || !matches(entry)) continue; entry.cancelTimeout(); this.entries.delete(key); entry.reject(error); diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-events.ts b/vendor/intx/hub-sessions/src/ws/sidecar-events.ts index 10d1d79b89..58e6edb450 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-events.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-events.ts @@ -21,9 +21,10 @@ import type { PackRejectReason, - RepoId, RunGrantsFrame, + WorkflowRunRefTips, } from "@intx/types/sidecar"; +import type { RepoId } from "@intx/types/repo"; import type { ApprovalSnapshot, ConnectorThreadState, @@ -58,11 +59,14 @@ export type WorkflowRunPackSource = { * Outcome of reserving a mail-triggered run's grants. `skip` means the * recipient names no workflow deployment; `rejected` means the deployment's * stable run is terminal or its requirements cannot be authorized; + * `notReady` means credential resolution is not recorded yet, so nothing + * was committed and the caller should retry after initialization; * `materialized` carries the canonical persisted wire rows. */ export type MailTriggeredRunGrantsResult = | { outcome: "skip" } | { outcome: "rejected"; status: 403 | 409; code: string; message: string } + | { outcome: "notReady" } | { outcome: "materialized"; stepGrants: RunGrantsFrame["stepGrants"]; @@ -80,17 +84,17 @@ export type SidecarEventMap = { childRunId?: string; }; - /** Notification. Emitted once when a sidecar's connection closes, - * carrying every address the connection owned -- session addresses - * and hub-minted workflow-substrate deployment addresses alike -- - * so lifecycle teardown covers both. */ + /** Notification. Emitted when a sidecar's connection closes, and when + * one allocation leaves a connection that stays open for others, + * carrying every workflow address that lost its route: deployment + * addresses and the step addresses staged for them. */ "sidecar.disconnect": { ownedAddresses: string[]; - /** Present only when the closing socket was the current allocated owner. */ - allocated?: { + /** Allocation generations whose current connection this was. */ + allocated: { allocationId: string; generation: number; - }; + }[]; }; /** Notification after the exact authenticated allocation generation registers. */ @@ -99,10 +103,20 @@ export type SidecarEventMap = { generation: number; }; - /** Notification. Emitted when a mail.outbound frame from a sidecar - * names recipients that the wire layer could not deliver locally and - * could not enqueue for a disconnected agent. The host is free to - * relay it onto an external transport or drop it. */ + /** + * Notification after the sidecar reports that the current generation's + * deployment stopped though the Hub did not stop it. + */ + "deployment.stopped": { + allocationId: string; + generation: number; + }; + + /** Notification. Emitted for mail the Hub gives up on: recipients no + * connection routes, un-acked mail it stops redelivering that no + * dispatch row stands behind, and mail held for a sender key that never + * arrived. The host is free to relay it onto an external transport or + * drop it. */ "mail.outbound.undelivered": { rawMessage: string; recipients: string[]; @@ -180,6 +194,7 @@ export function createSidecarEmitter(): SidecarEventEmitter { "agent.event": new Set(), "sidecar.disconnect": new Set(), "sidecar.allocated.connected": new Set(), + "deployment.stopped": new Set(), "mail.outbound.undelivered": new Set(), "mail.persisted": new Set(), "mail.inbound.acknowledged": new Set(), @@ -281,7 +296,7 @@ export type SidecarLookups = { resolveSenderKey?: (address: string) => Promise; /** Strict sibling of `resolveSenderKey` for the reconnect reconciliation. The - * register/reconnect handler re-resolves each reported cached sender and acts + * `hello` handler re-resolves each cached sender a hello reports and acts * on the three-way outcome the best-effort resolver collapses: * - returns the hex key when the sender resolves -> push `sender.key.refresh`; * - returns `null` for a CONFIRMED absence (no matching principal = a deleted @@ -361,4 +376,15 @@ export type SidecarLookups = { ) => Promise< { accepted: true } | { accepted: false; reason: PackRejectReason } >; + + /** Reads the Hub's tip of each authoritative ref of the deployment's + * workflow-run repo, `null` for a ref it has not received. A stop is + * confirmed only once the worker reports the same tips. */ + readWorkflowRunRefTips?: ( + agentAddress: string, + ) => Promise; + /** The terminal anchor's admission cutoff is closed and earlier receives settled. */ + isWorkflowRunHistoryFinal?: ( + source: WorkflowRunPackSource, + ) => Promise; }; diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts b/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts index e853e3838b..d807ef32e3 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts @@ -1,5 +1,9 @@ import type { HarnessConfig } from "@intx/types/runtime"; +import type { HostedIncarnation } from "@intx/types/sidecar"; +import type { SidecarCredentials } from "../sidecar-allocation/contracts"; + +import { DEFAULT_SIDECAR_CAPACITY } from "@intx/types/sidecar"; import { createSidecarRouter, type AllocatedSidecarTarget, @@ -21,6 +25,24 @@ export const TEST_IDENTITY: Extract< generation: 1, }; +export const TEST_CREDENTIALS: SidecarCredentials = { + sidecarId: TEST_IDENTITY.sidecarId, +}; + +/** + * Router authentication that verifies each claimed sidecar id as hosting the + * bindings `lookup` returns for it. Like the production resolver, it accepts + * a sidecar that hosts nothing, and the router keeps that sidecar connected. + */ +export function sidecarAuth( + lookup: (sidecarId: string) => readonly SidecarAuthIdentity[], +): Pick { + return { + authenticateSidecar: async ({ sidecarId }) => ({ sidecarId }), + resolveSidecarBindings: async (sidecarId) => lookup(sidecarId), + }; +} + export const TEST_TARGET: AllocatedSidecarTarget = { allocationId: TEST_IDENTITY.allocationId, generation: TEST_IDENTITY.generation, @@ -87,40 +109,153 @@ export function createMockWs(): WsHandle & { }; } +/** The ref tips a stopped test worker reports, which the test Hub holds. */ +export const TEST_REF_TIPS = { + "refs/heads/main": "c".repeat(40), + "refs/heads/events": null, +}; + export function createAllocatedRouter( config: Partial = {}, ) { const router = createSidecarRouter({ - authenticateSidecar: async () => TEST_IDENTITY, + withExecutableWorkflowRun: async (_target, send) => send(), + authenticateSidecar: async () => TEST_CREDENTIALS, validateSidecarIdentity: async () => true, + resolveSidecarBindings: async () => [TEST_IDENTITY], hubPublicKey: "a".repeat(64), requestTimeoutMs: 500, ...config, + lookups: { + readWorkflowRunRefTips: async () => TEST_REF_TIPS, + ...config.lookups, + }, }); router.fenceAllocation(TEST_TARGET.allocationId, TEST_TARGET.generation); return router; } +/** Live incarnations of `addresses` at `generation`, as a hello reports them. */ +export function liveIncarnations( + addresses: readonly string[], + generation = TEST_IDENTITY.generation, +): HostedIncarnation[] { + return addresses.map((address) => ({ address, generation, state: "live" })); +} + +/** The `hello` a sidecar sends first, reporting what it holds. */ +export function helloFrame( + sidecarId: string, + incarnations: readonly HostedIncarnation[] = [], + token = "token", +): string { + return JSON.stringify({ + type: "hello", + sidecarId, + token, + capacity: DEFAULT_SIDECAR_CAPACITY, + incarnations, + }); +} + +/** + * Connect the test sidecar, reporting a live incarnation of each address at + * the test allocation's generation. + */ export async function connectAllocated( router: ReturnType, agentAddresses: string[] = [], - frameType: "register" | "reconnect" = "register", ) { const ws = createMockWs(); router.handleOpen(ws); router.handleMessage( ws, - JSON.stringify({ - type: frameType, - sidecarId: TEST_IDENTITY.sidecarId, - token: "token", - agentAddresses, - }), + helloFrame(TEST_IDENTITY.sidecarId, liveIncarnations(agentAddresses)), ); await tick(); return ws; } +type SentLifecycleRequest = { + type: string; + requestId: string; + agentAddress: string; + generation: number; +}; + +/** The last `agent.deploy` or `agent.undeploy` the router sent on `ws`. */ +export function lastRequest( + ws: { sent: string[] }, + type: "agent.deploy" | "agent.undeploy", + agentAddress?: string, +): SentLifecycleRequest { + const found = ws.sent + .map((raw): SentLifecycleRequest => JSON.parse(raw)) + .filter( + (frame) => + frame.type === type && + (agentAddress === undefined || frame.agentAddress === agentAddress), + ) + .at(-1); + if (found === undefined) { + throw new Error( + `No ${type} was sent${agentAddress === undefined ? "" : ` for ${agentAddress}`}`, + ); + } + return found; +} + +/** + * The sidecar's answer to the last `agent.deploy` sent on `ws`, naming its + * request id and incarnation. + */ +export function deployReply( + ws: { sent: string[] }, + answer: { publicKey: string } | { error: string }, + agentAddress?: string, +): string { + const { + requestId, + agentAddress: address, + generation, + } = lastRequest(ws, "agent.deploy", agentAddress); + return JSON.stringify( + "publicKey" in answer + ? { + type: "agent.deploy.ack", + requestId, + agentAddress: address, + generation, + publicKey: answer.publicKey, + } + : { + type: "agent.deploy.error", + requestId, + agentAddress: address, + generation, + error: { code: "deployment_failed", message: answer.error }, + }, + ); +} + +/** The sidecar's acknowledgement of the last `agent.undeploy` sent on `ws`. */ +export function undeployAck( + ws: { sent: string[] }, + agentAddress?: string, +): string { + const { + requestId, + agentAddress: address, + generation, + } = lastRequest(ws, "agent.undeploy", agentAddress); + return JSON.stringify({ + type: "agent.undeploy.ack", + requestId, + agentAddress: address, + generation, + }); +} + export function parsedFrames(ws: { sent: string[] }): unknown[] { return ws.sent.map((raw) => { const parsed: unknown = JSON.parse(raw); @@ -131,3 +266,41 @@ export function parsedFrames(ws: { sent: string[] }): unknown[] { export function tick(): Promise { return new Promise((resolve) => setTimeout(resolve, 0)); } + +/** + * The redelivery retry timer, driven by the test. + * + * The retry interval was already injectable, but arming was not, so a test + * wanting N redeliveries had to shorten the interval and sleep long enough + * for N of them to fit -- making the assertion a bet on how much the machine + * got through. Firing the retries explicitly makes the count exact. + */ +export function createManualRetries(retryIntervalMs: number): { + scheduleTimeout: (handler: () => void, ms: number) => () => void; + fireNext: () => void; + armedCount: () => number; +} { + const armed: { ms: number; fire: () => void; cancelled: boolean }[] = []; + return { + // The router arms its connection-liveness deadline through this same + // seam, so the delay is what tells the two apart. Firing indiscriminately + // closes the socket instead of redelivering. + scheduleTimeout(handler, ms) { + const entry = { ms, fire: handler, cancelled: false }; + armed.push(entry); + return () => { + entry.cancelled = true; + }; + }, + fireNext() { + const next = armed.find((e) => !e.cancelled && e.ms === retryIntervalMs); + if (next === undefined) { + throw new Error("no armed redelivery retry to fire"); + } + next.cancelled = true; + next.fire(); + }, + armedCount: () => + armed.filter((e) => !e.cancelled && e.ms === retryIntervalMs).length, + }; +} diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts b/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts index 22396bfd3b..6fe6b7b917 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts @@ -1,10 +1,15 @@ // Hub-side websocket handler for sidecar connections. // -// Accepts websocket upgrades, processes register frames, maintains a routing +// Accepts websocket upgrades, processes hello frames, maintains a routing // table of agentAddress → sidecar connection, and dispatches frames between // sidecars and the hub's internal systems. import { getLogger } from "@intx/log"; +import { + WorkflowRunNotExecutableError, + type SidecarInventory, + type WorkflowRunExecutionTarget, +} from "@intx/db"; import { chunkPack, createPackReceiver } from "@intx/pack-transport"; import { base64Decode, @@ -16,30 +21,45 @@ import type { GrantWalkSnapshot } from "@intx/types"; import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; import { type } from "arktype"; import { + DeploymentRejectedError, MAX_MAIL_OUTBOUND_BODY_BYTES, SidecarFrame, + WORKFLOW_CONTROL_INITIALIZING_ERROR, type AgentDeployAckFrame, + type AgentDeployErrorFrame, type AgentDeployFrame, + type AgentUndeployAckFrame, + type AgentUndeployErrorFrame, + type DeploymentStoppedFrame, + type DeploymentRetention, + type HostedIncarnation, + type SidecarCapacity, type PackAckFrame, type HubFrame, + type MailInboundFrame, type PackPushFrame, type PackDoneFrame, type PackRejectFrame, - type RepoId, type RunGrantsFrame, + type WorkflowControlFrame, + type WorkflowControlAckFrame, + type WorkflowRunRefTips, type SignalCorrelationRegisterFrame, - type CredentialDelivery, type WorkflowSourceAssetMount, type WorkflowProjectionDefinition, - type OAuthLoginTokens, - type OAuthLoginResultFrame, } from "@intx/types/sidecar"; +import { RepoId } from "@intx/types/repo"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; import type { ConnectorThreadState, HarnessConfig, InferenceSource, } from "@intx/types/runtime"; -import type { SidecarCredentialIdentity } from "../sidecar-allocation/contracts"; +import type { + SidecarCredentialIdentity, + SidecarCredentials, + SidecarIdentityUse, +} from "../sidecar-allocation/contracts"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import type { WorkflowDefinitionSource } from "@intx/types/workflow-sources"; import { @@ -53,6 +73,7 @@ import { type PendingEntry, type WsHandle, } from "./pending-tracker"; +import { workflowRunRepoIdForAddress } from "../workflow-run-kind"; const logger = getLogger(["hub", "ws", "sidecar"]); @@ -99,56 +120,165 @@ export class SidecarIdentityValidationError extends Error { } } +/** + * This Hub holds no current connection for the allocation generation, or the + * connection closed before the worker answered. The worker may be healthy + * behind another connection, so callers must retry rather than escalate. + */ +export class WorkflowControlUnreachableError extends Error { + constructor(message: string, cause?: unknown) { + super(message, { cause }); + this.name = "WorkflowControlUnreachableError"; + } +} + +/** A connected worker is still deploying and cannot process control yet. */ +export class WorkflowControlInitializingError extends Error { + constructor() { + super("Workflow deployment is still being initialized"); + this.name = "WorkflowControlInitializingError"; + } +} + +/** + * The worker acknowledged the control command, but the Hub could not process + * the acknowledgement in time. The worker answered, so this never counts as + * the worker failing to comply. + */ +export class WorkflowControlUnconfirmedError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowControlUnconfirmedError"; + } +} + +/** A live worker connection did not answer the control command in time. */ +export class WorkflowControlTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowControlTimeoutError"; + } +} + +/** A live worker answered the control command with an error. */ +export class WorkflowControlRejectedError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowControlRejectedError"; + } +} + +/** + * The worker stopped, but the Hub does not hold the history it reported. The + * stop stays unconfirmed and unfenced, so that history can still land. + */ +export class WorkflowControlHistoryPendingError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Workflow history has not reached the Hub: ${message}`, options); + this.name = "WorkflowControlHistoryPendingError"; + } +} + export type SidecarConnection = { sidecarId: string; - identity: SidecarAuthIdentity; - agentAddresses: Set; - // Workflow-substrate deployment run addresses (`run_@domain`) this - // connection hosts. Kept separate from the legacy `agentAddresses` set so - // workflow route teardown and recovery remain explicit. `handleClose` cleans - // both sets out of `addressIndex`. - workflowAddresses: Set; + capacity: SidecarCapacity; + /** + * The probe and allocation generations this sidecar currently hosts, keyed + * by allocation id. A provisioner may place several on one sidecar; each + * keeps its own generation fence. + */ + bindings: Map; + // Allocated deployment routes (including first deploy and reconnect) and + // transient step routes, each mapped to the allocation that owns it, so one + // allocation's routes can leave a connection that stays open for the + // others. `handleClose` cleans them out of `addressIndex`. + workflowAddresses: Map; + // Deploy frames the worker has not answered, by request id to their + // address, kept past the Hub's own deploy timeout. The worker handles an + // address's frames in order, so a control frame sent behind one of these + // for its address cannot start until the worker answers it. + unansweredDeploys: Map; + /** + * Undeploys sent on this connection that the sidecar has not answered yet, + * by address. The copy one removes can still be sending under the address + * until then, so a deploy of the address waits for the answer. + */ + undeploying: Map; send(frame: HubFrame): void; }; +type UndeployInFlight = { + /** The latest undeploy of the address sent on the connection. */ + requestId: string; + answered: Promise; + settle(): void; +}; + +/** The allocation that owns a workflow address routed on this connection. */ +function owningAllocation( + conn: SidecarConnection, + address: string, +): Extract | undefined { + const allocationId = conn.workflowAddresses.get(address); + const binding = + allocationId === undefined ? undefined : conn.bindings.get(allocationId); + return binding?.kind === "allocated" ? binding : undefined; +} + +/** + * The allocation this connection hosts whose deployment address is + * `address`, whether or not the address is routed. + */ +function deploymentBinding( + conn: SidecarConnection, + address: string, +): Extract | undefined { + return allocationBindings(conn).find( + (binding) => binding.workflowRunAddress === address, + ); +} + /** - * Whether this connection owns `address` for routing/lifecycle purposes. - * The legacy and workflow sets remain physically distinct, but ownership - * readers -- pack-transfer authorization, - * in-flight cancellation, disconnect teardown -- must see the union, or a - * reconnected workflow deployment (which lives only in `workflowAddresses`) - * is silently treated as unowned even though its mail routes. + * Whether a sidecar frame names the incarnation this connection routes for + * its address. A sidecar can still be draining a generation the Hub has + * fenced, so a frame naming any other generation is never credited to the + * routed one. */ -function connOwnsAddress(conn: SidecarConnection, address: string): boolean { +function namesRoutedIncarnation( + conn: SidecarConnection, + frame: { agentAddress: string; generation: number }, +): boolean { return ( - conn.agentAddresses.has(address) || conn.workflowAddresses.has(address) + owningAllocation(conn, frame.agentAddress)?.generation === frame.generation ); } /** - * Bind pack writes to the repository implied by the authenticated address. - * An allocated credential is narrower still: it may only write its one - * deployment's workflow-run repository and never a standalone agent-state - * repository. + * Bind pack writes to the repository implied by the authenticated + * incarnation: an allocation may only write its own deployment's workflow-run + * repository, never a standalone agent-state repository, and only as the + * generation this connection hosts. It authorizes the write even when the + * address is not routed: a stopped worker is not routed, but its stop is + * confirmed only once its remaining history reaches the Hub. */ function connCanPushRepo( conn: SidecarConnection, - agentAddress: string, - repoId: RepoId, + frame: { agentAddress: string; generation: number; repoId: RepoId }, ): boolean { - if (conn.identity.kind !== "allocated") return false; - if (agentAddress !== conn.identity.workflowRunAddress) { - return false; - } + const binding = deploymentBinding(conn, frame.agentAddress); + if (binding?.generation !== frame.generation) return false; return ( - repoId.kind === "workflow-run" && - repoId.id === deriveWorkflowRunRepoId(agentAddress) + frame.repoId.kind === "workflow-run" && + frame.repoId.id === deriveWorkflowRunRepoId(frame.agentAddress) ); } -/** The deduped set of every address this connection owns (session + workflow). */ -function ownedAddresses(conn: SidecarConnection): Set { - return new Set([...conn.agentAddresses, ...conn.workflowAddresses]); +function allocationBindings( + conn: SidecarConnection, +): Extract[] { + return [...conn.bindings.values()].flatMap((binding) => + binding.kind === "allocated" ? [binding] : [], + ); } export type SendPackOptions = { @@ -211,29 +341,6 @@ export type SenderDeploySettledOutcome = | { recorded: string } | { failed: string }; -/** The terminal arms of a sidecar-hosted loopback login. */ -export type OAuthLoginFinalOutcome = - | { status: "completed"; tokens: OAuthLoginTokens } - | { status: "error"; message: string }; - -/** The typed gate outcome: no sidecar passed the locality gate. The caller - * must surface "connect requires a local sidecar" — the hub never hosts a - * loopback listener for these flows and never falls back. */ -export type OAuthLoginGateOutcome = { status: "gate"; message: string }; - -export type OAuthLoginRequestOutcome = - | OAuthLoginGateOutcome - | { status: "error"; message: string } - | { - status: "started"; - requestId: string; - authorizeUrl: string; - completed: Promise; - }; - -/** Default whole-login timeout: a human completes the consent page. */ -export const DEFAULT_OAUTH_LOGIN_TIMEOUT_MS = 5 * 60_000; - export type AllocatedSenderDeployAttempt = AllocatedSidecarTarget & { readonly leaseId: string; }; @@ -253,21 +360,14 @@ export type SidecarRouter = { stepGrants: RunGrantsFrame["stepGrants"]; senderIdentities?: RunGrantsFrame["senderIdentities"]; }, - ): boolean; + ): Promise; /** - * Deliver a run's authorization grants to the sidecar hosting the named - * deployment-level mail address, ahead of the trigger mail that starts the - * run. Routes through the same per-address channel as `routeMail`: over the - * live connection when the deployment is connected, and into the disconnect - * queue when the deployment dropped in the window before its first - * reconnect (while its address is still on `agentAddresses`) -- so grants - * are queued for a disconnected deployment exactly when the trigger mail is, - * and ride the same reconnect flush. After an authenticated reconnect the - * address moves to `workflowAddresses`, which carries no queue (that - * generation's in-flight state is reconstructed sidecar-locally); a - * `run.grants` then has no queue to ride and this returns `false`. Returns - * `false` whenever the address is unroutable; the caller keeps any stable-run - * grant reservation so a later first-delivery attempt reuses it. + * Update a run's authorization grants independently of mail. For a trigger, + * pass `runGrants` to `routeMail` so both frames share one admission decision. + * Sends on the live connection, stamped with the incarnation the Hub routes + * for the address, and returns `false` whenever the address is unroutable; + * the caller keeps any stable-run grant reservation so a later + * first-delivery attempt reuses it. * * `senderIdentities` co-delivers the run's authorized senders' resolved keys * on the same barrier as the grant, so a recipient that caches from this @@ -328,7 +428,6 @@ export type SidecarRouter = { delivery: CredentialDelivery, revoke?: string[], ): Promise; - sendSyncRequest(agentAddress: string): void; /** * Deliver a workflow-run signal to the sidecar that hosts the named * deployment-level mail address. The sidecar's hub-link routes the @@ -349,7 +448,8 @@ export type SidecarRouter = { signalName: string; signalId: string; payload: unknown; - }): void; + correlationId?: string; + }): Promise; /** * Deliver a workflow-host drain control payload to the sidecar that * hosts the named deployment-level mail address. The sidecar's @@ -375,34 +475,16 @@ export type SidecarRouter = { getConnectedSidecars(): string[]; getRoutableAddresses(): string[]; - /** - * Start a sidecar-hosted loopback OAuth login (CL-7508) on the first - * connected sidecar that passes the router's locality gate. Resolves the - * typed gate outcome when no connected sidecar is local — never a - * hub-hosted fallback. On a gated-through sidecar the returned promise - * resolves once the sidecar reports `started` (callback server bound, - * authorize URL ready for the web UI to navigate); `completed` settles - * with the terminal outcome (tokens or a message — including bind - * failures like a pinned port already in use, which arrive as the - * error arm) or with the whole-login timeout. - */ - requestOAuthLogin(args: { - connectorId: "codex" | "xai-oauth"; - }): Promise; - /** Typed event emitter for the receiver-dispatch surface. See * `sidecar-events.ts` for the event map and emission semantics. */ events: SidecarEventEmitter; }; /** - * A verified sidecar-connection identity resolved by an authenticator from - * the credentials a sidecar presents on the WebSocket handshake. The - * `sidecarId` is the connection's own trusted id; it is not the untrusted - * `sidecarId` claimed on the register/reconnect frame, and it carries no - * tenant scope. Modeled as a discriminated union so a future non-sidecar - * principal (e.g. an operator user) can be added as an additional arm - * without changing existing consumers. + * One probe or allocation generation a sidecar hosts, as + * `resolveSidecarBindings` reads it for the sidecar the WebSocket handshake + * authenticated. The `sidecarId` is that authenticated id, not the untrusted + * `sidecarId` claimed on the hello frame. */ export type SidecarAuthIdentity = SidecarCredentialIdentity; @@ -411,9 +493,60 @@ export type AllocatedSidecarTarget = { readonly generation: number; }; +type WorkflowControlCommand = Omit< + WorkflowControlFrame, + "type" | "requestId" | "generation" | "action" +> & { action: "cancel" | "stop" }; + export type SidecarAllocationRouter = { - /** Advance the in-memory trust boundary before provisioning a generation. */ - fenceAllocation(allocationId: string, generation: number): void; + /** Opaque identity of the currently attached cleanup connection. */ + getCleanupConnection(target: AllocatedSidecarTarget): object | undefined; + /** + * Confirm cleanup of a releasing allocation. An expected connection token + * prevents sending on a different socket after asynchronous validation. + * Notify the owner when sent so its own timeout can distinguish preparation + * failures from attempted undeploys. + */ + undeployAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + signal: AbortSignal, + expectedConnection: object, + onSent: () => void, + ): Promise; + sendWorkflowControl( + target: AllocatedSidecarTarget, + command: WorkflowControlCommand, + timeoutMs: number, + ): Promise; + /** Concurrent callers share the existing request and its deadline. */ + retainAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + ): Promise; + getAllocationRetention( + target: AllocatedSidecarTarget, + ): DeploymentRetention | undefined; + /** Copies awaiting retention or history finality; the lifecycle checks run status. */ + getRetentionCandidates(): readonly (AllocatedSidecarTarget & { + readonly sidecarId: string; + })[]; + /** Undefined until the current connection's complete inventory is known. */ + getSidecarInventory(sidecarId: string): SidecarInventory | undefined; + /** Wait outside placement transactions; the caller's signal bounds the wait. */ + waitForSidecarInventory( + sidecarId: string, + signal: AbortSignal, + ): Promise; + /** + * Advance the trust boundary after persisting a generation. `cleanup` keeps + * its control binding while the reconciler owns the acknowledged teardown. + */ + fenceAllocation( + allocationId: string, + generation: number, + options?: { cleanup?: { sidecarId: string } }, + ): void; /** * Remove an exact generation's fence after its durable owner becomes * terminal. Durable identity validation rejects later stale reconnects. @@ -424,11 +557,14 @@ export type SidecarAllocationRouter = { * Throws `SidecarIdentityValidationError` when readiness cannot be * determined; only confirmed absence surfaces as a connection timeout. * `onValidation` observes notification lookups that may outlive this wait. + * Cancellation removes a parked waiter and discards pending validation + * results. */ waitForAllocatedSidecar( target: AllocatedSidecarTarget, timeoutMs: number, onValidation?: (validation: Promise) => void, + signal?: AbortSignal, ): Promise; /** * Check exact allocated readiness without parking a reconciliation worker. @@ -436,15 +572,55 @@ export type SidecarAllocationRouter = { * `false` means the worker is confirmed absent or stale. */ isAllocatedSidecarReady(target: AllocatedSidecarTarget): Promise; - /** Check for an active supervisor, throwing when identity validation fails. */ + /** + * Whether a connection already holds the allocation's binding at this + * generation, without validating it. A probe binding the allocation adopted + * is not that binding: only a sync or a registration attaches the allocation + * over it. + */ + holdsAllocatedBinding(target: AllocatedSidecarTarget): boolean; + /** + * Whether the generation's current connection routes its deployment. Throws + * when the generation has no current connection or identity validation + * fails: a sidecar that is only cut off may still hold the deployment. + */ isAllocatedWorkflowActive(target: AllocatedSidecarTarget): Promise; + /** + * Why the generation's sidecar reported its deployment stopped though the + * Hub did not stop it, or undefined when it reported no such stop. The + * report outlives the connection that carried it. + */ + reportedDeploymentFailure(target: AllocatedSidecarTarget): string | undefined; + /** + * For a stop `reportedDeploymentFailure` reports, how much of the history + * the stopped copy reported the Hub holds: `unreceived` describes the first + * ref whose reported tip the Hub does not hold, or is null once it holds + * them all, and `reportedAt` is when the stop was first reported. + */ + stoppedDeploymentHistory( + target: AllocatedSidecarTarget, + ): Promise<{ unreceived: string | null; reportedAt: Date } | undefined>; /** Probe a workflow on the exact provisioned allocation generation. */ sendProbeToAllocation( target: AllocatedSidecarTarget, args: SendProbeArgs, ): Promise; - /** Close an exact provisioned connection before changing its durable owner. */ - disconnectAllocation(target: AllocatedSidecarTarget): void; + /** + * Remove an exact generation from its sidecar's connection before changing + * its durable owner. The sidecar is told to undeploy an allocation's + * deployment, and the socket stays open while the sidecar hosts other + * bindings. + */ + detachAllocation(target: AllocatedSidecarTarget): void; + /** + * Bring a connected sidecar's bindings up to date with the database after + * a probe or allocation was placed on it, or left it, while it stayed + * connected. A sidecar that is registering, or not connected, picks its + * bindings up when its registration completes. The promise can settle + * before a binding attaches, so callers wait for its readiness instead. + * Cancellation prevents queued work or a pending binding read from attaching. + */ + syncSidecar(sidecarId: string, signal?: AbortSignal): Promise; sendAgentDeployToAllocation( target: AllocatedSidecarTarget, agentAddress: string, @@ -512,62 +688,61 @@ export type SidecarAllocationRouter = { signalName: string; signalId: string; payload: unknown; + correlationId?: string; }, signal?: AbortSignal, ): Promise; }; /** - * Resolves the credentials a sidecar presents on the handshake to a - * verified identity, or `null` when the credentials are not recognized. - * The claimed `sidecarId` is an unauthenticated hint; the authenticator - * derives the trusted identity from the `token` and the returned - * `sidecarId` is what the router keys connection state off of. + * Resolves the credentials a sidecar presents on the handshake to the + * verified sidecar, or `null` when they are not recognized. A sidecar that + * hosts nothing current still resolves and stays connected, idle, after its + * handshake undeploys what it reports. A retired one resolves marked + * `retired`, and its handshake undeploys what it reports and closes. The + * claimed `sidecarId` is an unauthenticated hint; the authenticator derives + * the trusted identity from the `token` and the returned `sidecarId` is what + * the router keys connection state off of. What the sidecar hosts is read through + * `resolveSidecarBindings` once the handshake has claimed the sidecar. */ export type SidecarAuthenticator = (claim: { sidecarId: string; token: string; -}) => Promise; +}) => Promise; export type SidecarRouterConfig = { requestTimeoutMs?: number; - /** - * Sidecar-hosted OAuth loopback login policy (CL-7508). The locality gate - * decides whether a connected sidecar may host a loopback login: a remote - * sidecar's `localhost:1455`/`127.0.0.1:1456` callback opens on the wrong - * machine, so the hub host supplies the predicate that separates a local - * sidecar (same host as the user's browser) from a provisioned remote one. - * Absent means no sidecar is ever local and every login request resolves - * the typed gate outcome — the hub never falls back to hosting the - * listener itself. - */ - oauthLogin?: { - isLocalSidecar: ( - identity: SidecarAuthIdentity, - ) => boolean | Promise; - /** Whole-login timeout (bind → authorize → callback → exchange). - * Defaults to 5 minutes — a human completes the consent page. */ - timeoutMs?: number; - }; /** Hex-encoded 32-byte Ed25519 public key for signing deploy commits. * Included in agent.deploy frames so sidecars can verify pack signatures. */ hubPublicKey?: string; - /** Resolves each register/reconnect handshake to a verified sidecar + /** Resolves each hello handshake to a verified sidecar * identity. Required: without it a connection could route on an * unverified frame claim. Return null to reject the handshake. */ authenticateSidecar: SidecarAuthenticator; /** Revalidate durable identity at registration and routing boundaries. */ validateSidecarIdentity: ( identity: SidecarAuthIdentity, - use: "registration" | "readiness" | "routing", + use: SidecarIdentityUse, + ) => Promise; + /** Lock current allocation/lifecycle state while the synchronous send runs. */ + withExecutableWorkflowRun: ( + target: WorkflowRunExecutionTarget, + send: () => boolean, + signal?: AbortSignal, ) => Promise; + /** The bindings a sidecar currently hosts, for `syncSidecar`. */ + resolveSidecarBindings: ( + sidecarId: string, + ) => Promise; /** Timeout for a `sendProbe` round-trip. A probe materializes a workflow's * dependency closure and evaluates it on the sidecar, so it can run longer * than a routine `sendRequest`; it gets its own timeout rather than sharing * the request timeout. */ probeTimeoutMs?: number; - disconnectQueueMaxSize?: number; - disconnectQueueTTLMs?: number; + /** How long un-acknowledged mail, and mail awaiting its sender's key or its + * recipient's credential resolution, is held for a delivery before the Hub + * gives up on it. */ + mailHoldTTLMs?: number; pingTimeoutMs?: number; /** Interval between redelivery attempts of a connected-window `mail.inbound` * the sidecar has not yet acknowledged with `mail.inbound.ack`. */ @@ -592,6 +767,8 @@ export type SidecarRouterConfig = { * typically by flipping a flag. */ scheduleTimeout?: (handler: () => void, ms: number) => () => void; + /** When a sidecar's stop report is first heard. Defaults to the wall clock. */ + now?: () => Date; /** Maximum redelivery attempts before the hub stops retrying an un-acked * connected-window `mail.inbound`. Bounds the retry so a sidecar that never * acks does not accumulate an unbounded timer per delivery. */ @@ -610,11 +787,20 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; // evaluates it on the sidecar, so it runs longer than a routine request; its // default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS. export const DEFAULT_PROBE_TIMEOUT_MS = 60_000; -const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100; -const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000; +const DEFAULT_MAIL_HOLD_TTL_MS = 5 * 60 * 1000; const DEFAULT_PING_TIMEOUT_MS = 60_000; const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000; const DEFAULT_MAIL_ACK_MAX_RETRIES = 5; +const INCARNATION_VALIDATION_CONCURRENCY = 8; +// A longer `setTimeout` fires at once, and a disconnect limit can be longer. +const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; + +const MalformedPackDone = type({ + type: "'repo.pack.done'", + transferId: "string > 0", + agentAddress: "string > 0", + repoId: RepoId, +}); // The hub re-resolves and re-pushes a key for each rotatable sender a sidecar // reports on (re)connect. A legitimate sidecar caches keys for tens, maybe low @@ -638,8 +824,9 @@ export function createSidecarRouter( hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity, - disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, - disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, + withExecutableWorkflowRun, + resolveSidecarBindings, + mailHoldTTLMs = DEFAULT_MAIL_HOLD_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, scheduleTimeout = (handler: () => void, ms: number) => { @@ -649,6 +836,7 @@ export function createSidecarRouter( }; }, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, + now = () => new Date(), lookups = {}, } = config; @@ -656,8 +844,72 @@ export function createSidecarRouter( // host code subscribes via `router.events`. const events = createSidecarEmitter(); + async function withAllocationWorkAdmission( + ws: WsHandle, + binding: Extract, + send: () => boolean, + signal?: AbortSignal, + ): Promise { + return withExecutableWorkflowRun( + binding, + () => { + signal?.throwIfAborted(); + if ( + allocationFences.get(binding.allocationId)?.generation !== + binding.generation || + allocatedConnections.get(binding.allocationId)?.ws !== ws || + !connections.has(ws) + ) + throw new Error( + `Workflow connection changed for allocation ${binding.allocationId}`, + ); + return send(); + }, + signal, + ); + } + + async function withWorkflowWorkAdmission( + ws: WsHandle, + conn: SidecarConnection, + agentAddress: string, + send: () => boolean, + signal?: AbortSignal, + ): Promise { + const binding = owningAllocation(conn, agentAddress); + const changed = () => + new Error( + `Workflow connection changed before delivery to ${agentAddress}`, + ); + if (binding === undefined) throw changed(); + return withAllocationWorkAdmission( + ws, + binding, + () => { + if ( + addressIndex.get(agentAddress) !== ws || + conn.workflowAddresses.get(agentAddress) !== binding.allocationId + ) + throw changed(); + return send(); + }, + signal, + ); + } + // ws handle → registered connection const connections = new Map(); + // sidecar id → its current socket + const sidecarSockets = new Map(); + // sidecar id → the socket whose handshake is registering it. The previous + // socket keeps serving until that registration takes over, but syncs for + // the sidecar queue behind the registration instead. + const sidecarClaims = new Map(); + // A handshake still authenticating when its socket closes must not + // register the socket afterwards. + const closedSockets = new WeakSet(); + // Sockets whose sidecar was sent `welcome`, and so is registered. + const welcomedSockets = new WeakSet(); const allocatedConnections = new Map< string, { @@ -665,7 +917,34 @@ export function createSidecarRouter( identity: SidecarAuthIdentity; } >(); - const allocationFences = new Map(); + const allocationFences = new Map< + string, + { generation: number; cleanup: { sidecarId: string } | undefined } + >(); + // allocationId -> generation whose worker acknowledged a workflow stop. Its + // later workflow-run packs are refused, including after a reconnect. + const stoppedAllocations = new Map(); + const connectionRetention = new WeakMap< + WsHandle, + Map + >(); + const retainingAllocations = new Map< + string, + { ws: WsHandle; generation: number; result: Promise } + >(); + // allocationId -> the stop a sidecar reported for that generation's + // deployment though the Hub did not stop it. It outlives the connection that + // carried it, until the allocation's fence moves past that generation. + const reportedStops = new Map< + string, + { + generation: number; + address: string; + error: string; + reportedAt: Date; + refTips?: WorkflowRunRefTips; + } + >(); type AllocationWaiter = { generation: number; resolve(): void; @@ -676,15 +955,27 @@ export function createSidecarRouter( onValidation?: (validation: Promise) => void; }; const allocationWaiters = new Map>(); + const inventoryWaiters = new Map void>>(); // agentAddress → ws handle (routing table) const addressIndex = new Map(); // requestId → pending promise (resolved by session.ack, rejected by - // session.error). `PendingTracker` owns the register/timeout/settle/sweep - // lifecycle shared by all five pending round-trips below; each entry's - // resolve/reject closures carry the per-round-trip cleanup. - const pendingRequests = new PendingTracker(); - // agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error) - const pendingDeploys = new PendingTracker(); + // session.error), carrying the target address so an allocation leaving a + // shared connection rejects only its own requests. `PendingTracker` owns the + // register/timeout/settle/sweep lifecycle shared by all five pending + // round-trips below; each entry's resolve/reject closures carry the + // per-round-trip cleanup. + const pendingRequests = new PendingTracker(); + // agentAddress → the deploy in flight for it, answered by the + // agent.deploy.ack or agent.deploy.error carrying its request id and + // generation. One deploy per address is in flight at a time; a reply that + // names another request answers a deploy the Hub already gave up on. + type PendingDeploy = { requestId: string; generation: number }; + const pendingDeploys = new PendingTracker< + string, + string, + PendingDeploy, + DeploymentRejectedError + >(); // Run addresses whose ALLOCATED deploy is mid-flight -- key-record has been // started but not yet committed. pendingDeploys clears at the deploy ack, but an // allocated run's key is recorded LATER by session-service's anchor-key update, @@ -695,12 +986,6 @@ export function createSidecarRouter( string, AllocatedSenderDeployAttempt >(); - // agentAddress → queued frames for disconnected agents awaiting reconnect - type DisconnectedAgent = { - queue: HubFrame[]; - timer: ReturnType; - }; - const disconnectedAgents = new Map(); // agentAddress → messageId → connected-window mail awaiting a // `mail.inbound.ack`. A `mail.inbound` delivered over a LIVE connection with // a hub-minted messageId is tracked here and redelivered -- identical bytes, @@ -713,7 +998,7 @@ export function createSidecarRouter( type PendingMailEntry = { agentAddress: string; messageId: string; - frame: HubFrame; + frame: MailInboundFrame; attempts: number; /** * Disarms this entry's redelivery retry. Safe to call more than once: @@ -723,6 +1008,13 @@ export function createSidecarRouter( * its spent canceller is still on it. */ cancelRetry: () => void; + /** + * The retry loop that owns the entry's redelivery, replaced when a + * redelivery rearms the retry. Cancelling a timer cannot stop a retry + * already awaiting its replay, so a retry whose loop was replaced neither + * sends nor rearms. + */ + retryLoop: symbol; // When this mail triggers a workflow run, the run's already-materialized // grants ride alongside it. Redelivery replays this snapshot as a // `run.grants` frame AHEAD of the mail so the redelivered trigger lands on @@ -736,8 +1028,15 @@ export function createSidecarRouter( stepGrants: RunGrantsFrame["stepGrants"]; senderIdentities?: RunGrantsFrame["senderIdentities"]; }; - /** Present for a durable trigger pinned to a provisioned allocation. */ - allocatedTarget?: AllocatedSidecarTarget; + /** The incarnation the mail was delivered to, which alone may ack it. */ + allocatedTarget: AllocatedSidecarTarget; + /** + * Whether a durable dispatch row stands behind the mail. The row delivers + * it again to whichever generation replaces the one it went to, so the + * entry covers only that generation. Mail with no row behind it follows + * the deployment to the generation routed next instead. + */ + dispatchBacked: boolean; }; const pendingMail = new Map>(); // agentAddress → retention TTL timer for un-acked pending mail held across a @@ -754,8 +1053,7 @@ export function createSidecarRouter( // unknown sender. Each entry holds what a re-drive of `handleMailOutbound` // needs -- the raw message and its recipients -- plus a per-entry TTL timer. // This is a SIBLING of `pendingMail`, keyed by SENDER: it means "the SENDER's - // key is missing," the opposite axis from the recipient-keyed disconnect queue - // (`disconnectedAgents`), which means "the RECIPIENT socket is gone." Here the + // key is missing," not that the recipient is unreachable. Here the // recipient is live; only the sender's key is absent. `noteSenderDeploySettled` // drives an entry to delivery once the key lands or to // `mail.outbound.undelivered` when the deploy fails; the TTL backstops the case @@ -767,6 +1065,33 @@ export function createSidecarRouter( timer: ReturnType; }; const deferredSenderMail = new Map>(); + // Recipient address → mail held because that deployment has not recorded + // credential resolution. A sibling of `deferredSenderMail`: keyed by the + // RECIPIENT. A recorded settle re-drives with the sender key bound when + // the mail was accepted, not the recipient's key. `rejected` is the + // wrong outcome here -- it drops the mail. + type DeferredRecipientResolutionMailEntry = { + recipient: string; + authenticatedSender: string; + rawMessage: string; + recipients: string[]; + // The sender key bound for this message before the hold. Null means + // that resolve found no key. A redrive passes this value so a later + // key cannot replace it; omitting it would resolve again. + recordedSenderKey: string | null; + // Armed when the attempt finishes parked with no wake. Null while the + // attempt is in flight, and after the hold is claimed. + cancelTimer: (() => void) | null; + // True until the attempt that parked this entry finishes. A settle during + // that await records `wake` and does not re-drive yet: the attempt may + // still route, and a re-drive beside it would deliver the mail twice. + inflight: boolean; + wake: SenderDeploySettledOutcome | null; + }; + const deferredRecipientResolutionMail = new Map< + string, + Set + >(); // agentAddress → set of subscriber callbacks for agent events const agentSubscribers = new Map void>>(); // agentAddress → cached connector-thread state, populated by @@ -781,8 +1106,8 @@ export function createSidecarRouter( // Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue` // for the split and the invariant behind it). A frame that establishes or // reads routing waits for earlier queued frames on the same ws to complete, - // so it observes their finished effects -- most importantly an async - // register's routing write, which would otherwise land after a following + // so it observes their finished effects -- most importantly a hello's + // routing write, which would otherwise land after a following // connector.state.changed / mail / pack frame and silently drop it. It holds // a single in-flight promise per ws (replaced each queued frame), cleared on // close. @@ -791,41 +1116,43 @@ export function createSidecarRouter( // transferId → pending pack transfer (resolved by repo.pack.ack, rejected // by repo.pack.reject). The entry carries the send-site agentAddress and // repoId so an ack/reject is honored only when it comes from the - // connection that owns the transfer for the same repo. + // connection that owns the transfer for the same repo. Transfer ids are + // random, so an answer to a transfer from before a Hub restart never + // matches a new one. type PackTransferMeta = { agentAddress: string; repoId: RepoId }; const pendingPacks = new PendingTracker(); - let packCounter = 0; - // agentAddress → pending undeploy (resolved by agent.undeploy.ack) - const pendingUndeploys = new PendingTracker(); + // agentAddress → the undeploy a caller awaits for it, answered by the + // agent.undeploy.ack or agent.undeploy.error carrying its request id. + const pendingUndeploys = new PendingTracker< + string, + void, + { requestId: string; generation: number; target?: AllocatedSidecarTarget } + >(scheduleTimeout); + const pendingWorkflowControls = new PendingTracker< + string, + DeploymentRetention | undefined, + AllocatedSidecarTarget & + Pick & { + terminal: boolean; + fail(error: Error): void; + received(): void; + } + >(scheduleTimeout); // requestId → pending workflow probe (resolved by workflow.probe.result, // rejected by workflow.probe.error). Result-carrying, unlike the other // trackers (which resolve void): a probe returns the sidecar's inert - // projection + grant set + wire hash. Keyed on requestId alone -- the - // probe runs in the sidecar's pre-deploy state and enters no address map, - // so `handleClose`'s ws-keyed sweep is its ONLY disconnect cleanup. - const pendingProbes = new PendingTracker(); - - // Sidecar-hosted loopback logins (CL-7508). Keyed on requestId alone — the - // login is tenant-level, not address-scoped — so, like probes, the - // connection close sweep is the only disconnect cleanup. A login settles in - // two phases: the `started` arm resolves the request promise (the hub - // returns the authorize URL to the web UI), and exactly one terminal arm - // settles `completed`. An error arm that arrives before any `started` - // (e.g. the pinned callback port was already bound) resolves the request - // promise as an error and tears the entry down — the hub never waits on a - // login that failed to start. - type PendingOAuthLogin = { - ws: WsHandle; - conn: SidecarConnection; - requestSettled: boolean; - resolveRequest(outcome: OAuthLoginRequestOutcome): void; - resolveFinal(outcome: OAuthLoginFinalOutcome): void; - completed: Promise; - timer: ReturnType; - }; - const pendingOAuthLogins = new Map(); + // projection + grant set + wire hash. The probe runs in the sidecar's + // pre-deploy state and enters no address map, so each entry carries its + // probe id: the ws-keyed sweeps -- the whole connection closing, or that + // probe leaving a connection other bindings keep open -- are its only + // disconnect cleanup. + const pendingProbes = new PendingTracker< + string, + WorkflowProbeResult, + string + >(); // Receives agent-state packs pushed from sidecars. The wire frames // (`repo.pack.push` / `repo.pack.done`) are shared with the @@ -839,46 +1166,6 @@ export function createSidecarRouter( let requestCounter = 0; - // Surface disconnect-queue mail that is being dropped rather than delivered. - // Every dropped frame reaches the same channel routing failures already use - // (`mail.outbound.undelivered`, logged by the orchestrator), plus a warn that - // names the recipient and the drop count so a size-cap eviction or a TTL - // expiry of a still-full queue is visible instead of silent. A queued frame - // is always a `mail.inbound` carrying the sender's rawMessage; a frame of any - // other shape has no rawMessage to relay and is surfaced by the warn alone. - function surfaceDroppedFrames( - agentAddress: string, - frames: HubFrame[], - reason: string, - ): void { - if (frames.length === 0) return; - logger.warn`Dropping ${String(frames.length)} queued message(s) for ${agentAddress}: ${reason}`; - for (const frame of frames) { - if (frame.type !== "mail.inbound") continue; - events.emit("mail.outbound.undelivered", { - rawMessage: frame.rawMessage, - recipients: [agentAddress], - }); - } - } - - function enqueueForDisconnected( - agentAddress: string, - frame: HubFrame, - ): boolean { - const entry = disconnectedAgents.get(agentAddress); - if (entry === undefined) return false; - - if (entry.queue.length >= disconnectQueueMaxSize) { - const evicted = entry.queue.shift(); - if (evicted !== undefined) { - surfaceDroppedFrames(agentAddress, [evicted], "disconnect queue full"); - } - } - entry.queue.push(frame); - return true; - } - // Arm a redelivery-retry timer for a tracked pending mail. Wraps the async // `retryPendingMail` so a rejection -- a socket write that throws once the // sidecar is gone -- is logged rather than floating out of the timer as an @@ -886,11 +1173,14 @@ export function createSidecarRouter( function scheduleMailRetry( agentAddress: string, messageId: string, + retryLoop: symbol, ): () => void { return scheduleTimeout(() => { - void retryPendingMail(agentAddress, messageId).catch((err: unknown) => { - logger.warn`Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`; - }); + void retryPendingMail(agentAddress, messageId, retryLoop).catch( + (err: unknown) => { + logger.warn`Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`; + }, + ); }, mailAckRetryIntervalMs); } @@ -901,13 +1191,14 @@ export function createSidecarRouter( function trackPendingMail( agentAddress: string, messageId: string, - frame: HubFrame, + frame: MailInboundFrame, + allocatedTarget: AllocatedSidecarTarget, + dispatchBacked: boolean, runGrants?: { runId: string; stepGrants: RunGrantsFrame["stepGrants"]; senderIdentities?: RunGrantsFrame["senderIdentities"]; }, - allocatedTarget?: AllocatedSidecarTarget, ): void { let byId = pendingMail.get(agentAddress); if (byId === undefined) { @@ -916,17 +1207,69 @@ export function createSidecarRouter( } const existing = byId.get(messageId); if (existing !== undefined) existing.cancelRetry(); + const retryLoop = Symbol("mail-retry"); byId.set(messageId, { agentAddress, messageId, frame, attempts: 0, - cancelRetry: scheduleMailRetry(agentAddress, messageId), + cancelRetry: scheduleMailRetry(agentAddress, messageId, retryLoop), + retryLoop, + allocatedTarget, + dispatchBacked, ...(runGrants !== undefined ? { runGrants } : {}), - ...(allocatedTarget !== undefined ? { allocatedTarget } : {}), }); } + // A pending mail the Hub stops redelivering. Mail a dispatch row stands + // behind is left to the row: it delivers the mail again once the deployment + // is next ready, unless the deployment ended, which abandons the row. Any + // other mail is surfaced as undelivered. + function abandonPendingMail(entry: PendingMailEntry, reason: string): void { + if (entry.dispatchBacked) { + logger.warn`Stopped redelivering mail ${entry.messageId} to ${entry.agentAddress} and left it to its dispatch row: ${reason}`; + return; + } + events.emit("mail.outbound.undelivered", { + rawMessage: entry.frame.rawMessage, + recipients: [entry.agentAddress], + }); + logger.warn`Dropping un-acked mail ${entry.messageId} for ${entry.agentAddress}: ${reason}`; + } + + // A pending mail whose generation no longer holds its address. Mail a + // dispatch row stands behind is left to the row, which delivers it to the + // generation routed now. Any other mail follows the deployment there, + // restamped for that generation, or is surfaced as undelivered when no + // generation holds the address. Returns whether the entry is still pending. + function followRoutedGeneration( + byId: Map, + entry: PendingMailEntry, + routed: Extract | undefined, + ): boolean { + if (entry.dispatchBacked) { + entry.cancelRetry(); + deletePendingMail(byId, entry.agentAddress, entry.messageId); + logger.info`Leaving mail ${entry.messageId} for ${entry.agentAddress} to its dispatch row: generation ${String(entry.allocatedTarget.generation)} no longer holds the address`; + return false; + } + if (routed === undefined) { + entry.cancelRetry(); + deletePendingMail(byId, entry.agentAddress, entry.messageId); + abandonPendingMail( + entry, + "no generation of it is routed to redeliver to", + ); + return false; + } + entry.allocatedTarget = { + allocationId: routed.allocationId, + generation: routed.generation, + }; + entry.frame = { ...entry.frame, generation: routed.generation }; + return true; + } + // Resolve the frame that must precede a redelivery of a trigger mail on the // FIFO socket, re-resolving a keyless run sender's key so it still // co-delivers. Returns: @@ -953,15 +1296,11 @@ export function createSidecarRouter( async function resolveReplayLeadFrame( entry: PendingMailEntry, ): Promise { - const authenticatedSender = - entry.frame.type === "mail.inbound" - ? entry.frame.authenticatedSender - : undefined; - const senderIsRun = - authenticatedSender !== undefined && isRunAddress(authenticatedSender); + const authenticatedSender = entry.frame.authenticatedSender; + const senderIsRun = isRunAddress(authenticatedSender); if (entry.runGrants === undefined) { - if (authenticatedSender === undefined || !senderIsRun) return undefined; + if (!senderIsRun) return undefined; const key = await reresolveRunSenderKey(authenticatedSender); if (key === null) return undefined; return { @@ -972,17 +1311,14 @@ export function createSidecarRouter( } let senderIdentities = entry.runGrants.senderIdentities; - if ( - senderIdentities === undefined && - authenticatedSender !== undefined && - senderIsRun - ) { + if (senderIdentities === undefined && senderIsRun) { const key = await reresolveRunSenderKey(authenticatedSender); senderIdentities = senderIdentitiesFromKey(authenticatedSender, key); } return { type: "run.grants", agentAddress: entry.agentAddress, + generation: entry.frame.generation, runId: entry.runGrants.runId, stepGrants: entry.runGrants.stepGrants, ...(senderIdentities !== undefined ? { senderIdentities } : {}), @@ -1018,37 +1354,85 @@ export function createSidecarRouter( // key) and then the mail itself over `conn`. Awaits the resolve FIRST, then // sends the lead frame and the mail back-to-back with NO await between them, // so the co-delivered key always precedes the mail on the FIFO socket. - // Returns whether the mail was (re)sent, so the caller re-arms the retry timer - // only for an entry it actually redelivered. + // Returns whether the mail was sent and arms its retry inside admission, + // before an acknowledgement can remove the pending entry. A retry passes its + // loop, and sends nothing once a redelivery has replaced that loop; a + // redelivery passes none and starts the entry's one retry loop afresh. async function replaySendPendingMail( conn: SidecarConnection, entry: PendingMailEntry, + retryLoop: symbol | undefined, ): Promise { const lead = await resolveReplayLeadFrame(entry); // The resolve above may have awaited real I/O; during that gap a queued // `mail.inbound.ack` can advance and run `resolvePendingMail` (delete + - // clearTimeout) on this entry. The window is opened by the timer-macrotask - // retry path, NOT by any bypass: `mail.inbound.ack` is a QUEUED frame - // (frameBypassesQueue returns false for it). It can interleave because the - // retry runs as an independent setTimeout macrotask (retryPendingMail), so - // the owning ws's message chain is free to advance the ack during the - // resolve await. On the reconnect/redeliver path the ack cannot interleave - // at all -- it queues behind the still-running reconnect handler on the - // same ws -- so here this guard is pure defense-in-depth. Re-confirm it is - // still the tracked entry before sending, or a post-ack redelivery would - // arm a retry timer on a detached entry. + // clearTimeout) on this entry. `mail.inbound.ack` is a QUEUED frame + // (frameBypassesQueue returns false for it), so it interleaves only with a + // replay that runs off the owning ws's message chain: the retry, an + // independent setTimeout macrotask (retryPendingMail). On the hello's + // redelivery the ack cannot interleave -- it queues behind the + // still-running registration on the same ws. Re-confirm it is still the + // tracked entry before sending, or a post-ack redelivery would arm a retry + // timer on a detached entry. if (pendingMail.get(entry.agentAddress)?.get(entry.messageId) !== entry) { return false; } + if (retryLoop !== undefined && entry.retryLoop !== retryLoop) return false; // The same gap can span a disconnect or a takeover that moves the address // off `conn`. Sending on the stale conn would write to a dead socket and // re-arm a retry that later drops a still-retained entry. Skip so the entry // survives for the reconnect redelivery. const ws = addressIndex.get(entry.agentAddress); if (ws === undefined || connections.get(ws) !== conn) return false; - if (lead !== undefined) conn.send(lead); - conn.send(entry.frame); - return true; + const stillOwned = (): boolean => + pendingMail.get(entry.agentAddress)?.get(entry.messageId) === entry && + (retryLoop === undefined || entry.retryLoop === retryLoop); + // Mail tracked since the address was routed here still has its retry + // armed, and a retry of it may still be awaiting its replay; one retry + // loop per entry, so a redelivery replaces the loop. + const rearm = (attempts: number): void => { + entry.cancelRetry(); + entry.attempts = attempts; + if (retryLoop === undefined) entry.retryLoop = Symbol("mail-retry"); + entry.cancelRetry = scheduleMailRetry( + entry.agentAddress, + entry.messageId, + entry.retryLoop, + ); + }; + try { + return await withWorkflowWorkAdmission( + ws, + conn, + entry.agentAddress, + () => { + if (!stillOwned()) return false; + if (lead !== undefined) conn.send(lead); + conn.send(entry.frame); + // Arm before releasing the database locks: an ack may arrive while + // the transaction finishes and must be able to cancel this timer. + rearm(retryLoop === undefined ? 0 : entry.attempts + 1); + return true; + }, + ); + } catch (error) { + if (error instanceof WorkflowRunNotExecutableError) { + logger.warn`Dropping un-acked mail ${entry.messageId} for ${entry.agentAddress}: its workflow run can no longer execute`; + resolvePendingMail( + entry.agentAddress, + entry.messageId, + entry.allocatedTarget.generation, + ); + } else { + // A transient database or socket failure must leave replay retryable, + // including during the registration handler's reconnect replay. + if (stillOwned() && addressIndex.has(entry.agentAddress)) { + rearm(entry.attempts + 1); + } + logger.warn`Mail replay failed for ${entry.agentAddress}: ${error instanceof Error ? error.message : String(error)}`; + } + return false; + } } function deletePendingMail( @@ -1072,26 +1456,23 @@ export function createSidecarRouter( async function retryPendingMail( agentAddress: string, messageId: string, + retryLoop: symbol, ): Promise { const byId = pendingMail.get(agentAddress); if (byId === undefined) return; const entry = byId.get(messageId); - if (entry === undefined) return; + if (entry?.retryLoop !== retryLoop) return; if (entry.attempts >= mailAckMaxRetries) { // The sidecar never acked within the retry budget. The ack is withheld // precisely because the sidecar's durable inbox write failed, so the - // mail was NOT delivered: surface it as undelivered so the host can relay - // it onto an external transport, then drop the pending entry so its timer - // does not leak. - if (entry.frame.type === "mail.inbound") { - events.emit("mail.outbound.undelivered", { - rawMessage: entry.frame.rawMessage, - recipients: [agentAddress], - }); - } + // mail was NOT delivered. Drop the pending entry so its timer does not + // leak. deletePendingMail(byId, agentAddress, messageId); - logger.warn`Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`; + abandonPendingMail( + entry, + `no acknowledgement after ${String(entry.attempts)} redelivery attempt(s)`, + ); return; } @@ -1099,34 +1480,36 @@ export function createSidecarRouter( // moved the address to a new connection since the original delivery. const ws = addressIndex.get(agentAddress); const conn = ws !== undefined ? connections.get(ws) : undefined; - const allocated = - entry.allocatedTarget === undefined - ? undefined - : allocatedConnections.get(entry.allocatedTarget.allocationId); + if (conn === undefined) { + followRoutedGeneration(byId, entry, undefined); + return; + } + const allocated = allocatedConnections.get( + entry.allocatedTarget.allocationId, + ); const targetStillOwnsAddress = - entry.allocatedTarget === undefined || - (allocated !== undefined && - allocated.identity.generation === entry.allocatedTarget.generation && - allocated.ws === ws); - if (conn === undefined || !targetStillOwnsAddress) { - // No live connection to recover into. Connected-window redelivery only - // applies while the address is routable; a disconnected address is not - // retried here. - deletePendingMail(byId, agentAddress, messageId); - logger.warn`Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`; + allocated !== undefined && + allocated.identity.generation === entry.allocatedTarget.generation && + allocated.ws === ws; + if ( + !targetStillOwnsAddress && + !followRoutedGeneration(byId, entry, owningAllocation(conn, agentAddress)) + ) { return; } - if (!(await replaySendPendingMail(conn, entry))) return; - entry.attempts += 1; - entry.cancelRetry = scheduleMailRetry(agentAddress, messageId); + await replaySendPendingMail(conn, entry, retryLoop); } - function resolvePendingMail(agentAddress: string, messageId: string): void { + function resolvePendingMail( + agentAddress: string, + messageId: string, + generation: number, + ): void { const byId = pendingMail.get(agentAddress); if (byId === undefined) return; const entry = byId.get(messageId); - if (entry === undefined) return; + if (entry?.allocatedTarget.generation !== generation) return; entry.cancelRetry(); deletePendingMail(byId, agentAddress, messageId); } @@ -1134,10 +1517,9 @@ export function createSidecarRouter( // Hold an address's un-acked pending mail across a disconnect. The per-entry // retry timers are cleared -- retrying over the dead socket is pointless -- // but the entries are KEPT so a verified reconnect can redeliver them. A - // retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar - // that never reconnects does not leak; on expiry the still-un-acked entries - // are surfaced as `mail.outbound.undelivered` so the host can relay them, - // since a withheld ack means the sidecar's durable write never landed. + // retention TTL bounds the hold so a sidecar that never reconnects does not + // leak; on expiry the still-un-acked entries are given up, since a withheld + // ack means the sidecar's durable write never landed. function retainPendingMailForAddress(agentAddress: string): void { const byId = pendingMail.get(agentAddress); if (byId === undefined) return; @@ -1148,30 +1530,31 @@ export function createSidecarRouter( pendingMailRetention.delete(agentAddress); const expired = pendingMail.get(agentAddress); pendingMail.delete(agentAddress); - if (expired !== undefined && expired.size > 0) { - for (const entry of expired.values()) { - if (entry.frame.type !== "mail.inbound") continue; - events.emit("mail.outbound.undelivered", { - rawMessage: entry.frame.rawMessage, - recipients: [agentAddress], - }); - } - logger.warn`Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`; + for (const entry of expired?.values() ?? []) { + abandonPendingMail( + entry, + "no reconnect redelivered it within the retention time", + ); } - }, disconnectQueueTTLMs); + }, mailHoldTTLMs); pendingMailRetention.set(agentAddress, timer); } - // Redeliver an address's retained un-acked pending mail on a verified - // reconnect. Replays identical bytes (same messageId) over the new - // connection, so the sidecar's inbox dedups a message it already wrote - // (effectively-once) and processes one it had dropped (no loss). Re-arms the - // connected-window retry over the new connection with a fresh per-generation - // budget, so a redelivery that is itself dropped before its ack is retried. + // Redeliver an address's retained un-acked pending mail once a `hello` + // routes the address again. + // Replays identical bytes (same messageId) over that connection, so the + // sidecar's inbox dedups a message it already wrote (effectively-once) and + // processes one it had dropped (no loss). Re-arms the connected-window retry + // over that connection with a fresh per-generation budget, so a redelivery + // that is itself dropped before its ack is retried. async function redeliverPendingMail( agentAddress: string, conn: SidecarConnection, ): Promise { + // A handshake that took the sidecar over meanwhile redelivers what it + // routes, and mail it does not route keeps its retention. + const routedOn = addressIndex.get(agentAddress); + if (routedOn === undefined || connections.get(routedOn) !== conn) return; const retention = pendingMailRetention.get(agentAddress); if (retention !== undefined) { clearTimeout(retention); @@ -1179,26 +1562,30 @@ export function createSidecarRouter( } const byId = pendingMail.get(agentAddress); if (byId === undefined) return; + let redelivered = 0; for (const entry of [...byId.values()]) { + const routed = owningAllocation(conn, agentAddress); + if (routed === undefined) { + // The route went while an earlier entry awaited its replay. The rest + // waits for the generation routed next, as a detach leaves it, unless + // another connection already routes the address and delivers it there. + if (!addressIndex.has(agentAddress)) { + retainPendingMailForAddress(agentAddress); + } + return; + } if ( - entry.allocatedTarget !== undefined && - (conn.identity.kind !== "allocated" || - conn.identity.allocationId !== entry.allocatedTarget.allocationId || - conn.identity.generation !== entry.allocatedTarget.generation) + (routed.allocationId !== entry.allocatedTarget.allocationId || + routed.generation !== entry.allocatedTarget.generation) && + !followRoutedGeneration(byId, entry, routed) ) { - // The Hub-owned dispatch row survives generation replacement and will - // be requeued by the allocation-ready callback. Do not leak or replay - // this generation-local retry entry onto a different worker. - entry.cancelRetry(); - deletePendingMail(byId, agentAddress, entry.messageId); continue; } - if (!(await replaySendPendingMail(conn, entry))) continue; - entry.attempts = 0; - entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId); + if (!(await replaySendPendingMail(conn, entry, undefined))) continue; + redelivered += 1; } - if (byId.size > 0) { - logger.info`Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`; + if (redelivered > 0) { + logger.info`Redelivered ${String(redelivered)} un-acked message(s) to ${agentAddress}`; } } @@ -1216,14 +1603,14 @@ export function createSidecarRouter( function handlePing(ws: WsHandle): void { resetLivenessTimer(ws); - // Always respond with pong, even before register/reconnect completes. - // The sidecar's ping timer starts on open, which may fire before the - // async registration handshake finishes. + // Always respond with pong, even before the hello is answered. The + // sidecar's ping timer starts on open, which may fire before the async + // registration handshake finishes. ws.send(JSON.stringify({ type: "pong" })); } function handleOpen(ws: WsHandle): void { - // Connection is not usable until a register frame arrives. + // Connection is not usable until a hello frame arrives. // Start the liveness timer immediately — a sidecar that connects // but never sends a ping will be reaped. resetLivenessTimer(ws); @@ -1240,6 +1627,27 @@ export function createSidecarRouter( const validated = SidecarFrame(raw); if (validated instanceof type.errors) { logger.warn`Invalid sidecar frame: ${validated.summary}`; + // An invalid hello cannot be answered with `welcome`, so the socket + // closes rather than leave the sidecar waiting on it. + if ( + typeof raw === "object" && + raw !== null && + "type" in raw && + raw.type === "hello" + ) { + handleClose(ws); + ws.close(); + } + const done = MalformedPackDone(raw); + if (!(done instanceof type.errors)) { + // Behind the frames the sidecar sent before it, such as the + // transfer's own chunks, so the rejection cannot overtake them. + enqueueOnConnection(ws, async () => { + rejectMalformedPackDone(ws, done); + }).catch((err: unknown) => { + logger.warn`Rejecting a malformed repo.pack.done failed: ${err instanceof Error ? err.message : String(err)}`; + }); + } return; } const frame = validated; @@ -1247,6 +1655,12 @@ export function createSidecarRouter( // Bypass frames (liveness + terminal responses to outbound requests) // dispatch immediately: they resolve the very promises a queued handler // may be blocked on, so queuing them would deadlock the round-trip. + // A queued stop acknowledgement may wait behind slow pack ingestion. The + // worker has answered, so its silence timeout stops here. + if (frame.type === "workflow.control.ack") { + const entry = pendingWorkflowControls.get(frame.requestId); + if (entry?.ws === ws) entry.meta.received(); + } if (frameBypassesQueue(frame)) { // Guard so a bypass handler's failure -- a synchronous throw or an async // ack handler's rejection -- is logged rather than floating out of the @@ -1268,6 +1682,26 @@ export function createSidecarRouter( messageChains.set(ws, next); } + // Run Hub-initiated connection work on the same per-ws chain as inbound + // frames, so it observes a completed registration and a registration + // observes its effects. The caller sees the task's own outcome; the chain + // only needs it settled. + function enqueueOnConnection( + ws: WsHandle, + task: () => Promise, + ): Promise { + const prev = messageChains.get(ws) ?? Promise.resolve(); + const result = prev.then(task); + messageChains.set( + ws, + result.then( + () => undefined, + () => undefined, + ), + ); + return result; + } + function assertNever(x: never): never { throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`); } @@ -1280,30 +1714,34 @@ export function createSidecarRouter( // (a response cannot resolve "too early" for a request that already went // out), and it is exactly what in-flight queued handlers block on, so it MUST // run out of band. Every other frame establishes or reads routing, or carries - // an inbound payload whose order matters, so it queues. The exhaustive switch - // + assertNever makes adding a SidecarFrame variant without classifying it a - // compile error, not a latent deadlock or a silent bypass hole. + // an inbound payload whose order matters, so it queues. A workflow stop + // acknowledgement queues too: it ends the worker's history, so the packs the + // worker sent before it must land before the stop resolves and later packs + // are fenced. The exhaustive switch + assertNever makes adding a SidecarFrame + // variant without classifying it a compile error, not a latent deadlock or a + // silent bypass hole. function frameBypassesQueue(frame: SidecarFrame): boolean { switch (frame.type) { case "ping": case "session.ack": case "session.error": case "agent.deploy.ack": - case "agent.error": + case "agent.deploy.error": case "agent.undeploy.ack": + case "agent.undeploy.error": case "repo.pack.ack": case "repo.pack.reject": case "workflow.probe.result": case "workflow.probe.error": - case "oauth.login.result": return true; - case "register": - case "reconnect": + case "hello": case "mail.outbound": case "agent.event": case "connector.state.changed": case "mail.inbound.ack": case "signal.correlation.register": + case "workflow.control.ack": + case "deployment.stopped": case "repo.pack.push": case "repo.pack.done": return false; @@ -1319,43 +1757,52 @@ export function createSidecarRouter( ws: WsHandle, frame: SidecarFrame, ): void | Promise { - const registeredIdentity = connections.get(ws)?.identity; - if ( - registeredIdentity?.kind === "probe" && - frame.type !== "register" && - frame.type !== "reconnect" && - frame.type !== "ping" && - frame.type !== "workflow.probe.result" && - frame.type !== "workflow.probe.error" - ) { - logger.warn`Rejected ${frame.type} from probe sidecar ${registeredIdentity.sidecarId}`; - handleClose(ws); - ws.close(); - return; - } switch (frame.type) { - case "register": { - const agentAddresses = frame.agentAddresses; + case "hello": { + const { incarnations } = frame; const cachedSenderAddresses = frame.cachedSenderAddresses ?? []; - return authenticateHandshake(ws, frame, (identity) => - handleRegister(ws, identity, agentAddresses, cachedSenderAddresses), - ); - } - case "reconnect": { - const agentAddresses = frame.agentAddresses; - const cachedSenderAddresses = frame.cachedSenderAddresses ?? []; - return authenticateHandshake(ws, frame, (identity) => - handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses), - ); + // A hello is answered with `welcome` or with the socket closing, never + // left waiting: a registration that fails before its welcome closes + // the socket, and the sidecar registers again on a new one. One that + // fails after it is only reported, since registering again would meet + // the same failure and resend the sidecar's pending mail each time. + return authenticateHandshake( + ws, + frame, + (sidecarId) => + handleRegistration( + ws, + sidecarId, + incarnations, + cachedSenderAddresses, + frame.capacity, + ), + (sidecarId) => undeployRetired(ws, sidecarId, incarnations), + ).catch((err: unknown) => { + const message = err instanceof Error ? err.message : String(err); + if (welcomedSockets.has(ws)) { + logger.error`Registration of sidecar ${frame.sidecarId} failed after its welcome: ${message}`; + return; + } + logger.error`Registration of sidecar ${frame.sidecarId} failed; closing its connection: ${message}`; + handleClose(ws); + ws.close(); + }); } case "agent.deploy.ack": return handleDeployAck(ws, frame); - case "agent.error": - rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error); - rejectUndeployPending(ws, frame.agentAddress, frame.error); + case "workflow.control.ack": + return handleWorkflowControlAck(ws, frame); + case "agent.deploy.error": + rejectDeployPendingFromFrame(ws, frame); return; case "agent.undeploy.ack": - resolveUndeployPending(ws, frame.agentAddress); + settleUndeploy(ws, frame); + resolveUndeployPending(ws, frame); + return; + case "agent.undeploy.error": + settleUndeploy(ws, frame); + rejectUndeployPending(ws, frame); return; case "ping": handlePing(ws); @@ -1363,8 +1810,13 @@ export function createSidecarRouter( case "mail.outbound": { const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.senderAddress)) { - logger.warn`Dropping mail.outbound from ${frame.senderAddress}: not registered to this sidecar`; + if ( + !namesRoutedIncarnation(conn, { + agentAddress: frame.senderAddress, + generation: frame.generation, + }) + ) { + logger.warn`Dropping mail.outbound from ${frame.senderAddress} generation ${String(frame.generation)}: this sidecar does not route that incarnation`; return; } // The DoS backstop and the trust boundary for an untrusted sidecar's @@ -1380,8 +1832,8 @@ export function createSidecarRouter( } if (frame.delivered !== true) { // frame.senderAddress is the sender this connection was just gated - // on by connOwnsAddress above -- a hub-verified value. Thread it so - // the relayed inbound frame is stamped with it, not the MIME From. + // on above -- a hub-verified value. Thread it so the relayed + // inbound frame is stamped with it, not the MIME From. return handleMailOutbound( frame.rawMessage, frame.senderAddress, @@ -1402,8 +1854,8 @@ export function createSidecarRouter( case "agent.event": { const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.agentAddress)) { - logger.warn`Dropping agent.event for ${frame.agentAddress}: not registered to this sidecar`; + if (!namesRoutedIncarnation(conn, frame)) { + logger.debug`Dropping agent.event for ${frame.agentAddress} generation ${String(frame.generation)}: this sidecar does not route that incarnation`; return; } events.emit("agent.event", { @@ -1433,27 +1885,32 @@ export function createSidecarRouter( return; case "mail.inbound.ack": { // Terminal receipt for a connected-window `mail.inbound`: the sidecar - // has durably written the message to its inbox. Gate on ownership -- - // like connector.state.changed and signal.correlation.register -- so a - // sidecar cannot clear another sidecar's pending mail. The messageId is - // a hub-minted id only the owning sidecar ever received on the frame, - // so the ownership check is defense-in-depth, not the sole guard. + // has durably written the message to the inbox of the incarnation the + // ack names. Only the incarnation this connection routes may settle + // it: an ack from a generation the Hub fenced is for an inbox that may + // be gone, and crediting it would settle mail the routed generation + // never received. const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.agentAddress)) { - logger.warn`Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`; + if (!namesRoutedIncarnation(conn, frame)) { + logger.warn`Dropping mail.inbound.ack for ${frame.agentAddress} generation ${String(frame.generation)}: this sidecar does not route that incarnation`; return; } - resolvePendingMail(frame.agentAddress, frame.messageId); + resolvePendingMail( + frame.agentAddress, + frame.messageId, + frame.generation, + ); + const acknowledging = owningAllocation(conn, frame.agentAddress); events.emit("mail.inbound.acknowledged", { agentAddress: frame.agentAddress, messageId: frame.messageId, - ...(conn.identity.kind === "allocated" + ...(acknowledging !== undefined ? { allocated: { - allocationId: conn.identity.allocationId, - anchorRunId: conn.identity.anchorRunId, - generation: conn.identity.generation, + allocationId: acknowledging.allocationId, + anchorRunId: acknowledging.anchorRunId, + generation: acknowledging.generation, }, } : {}), @@ -1463,9 +1920,11 @@ export function createSidecarRouter( case "signal.correlation.register": return handleSignalCorrelationRegister(ws, frame); case "session.ack": + if (pendingRequests.get(frame.requestId)?.ws !== ws) return; pendingRequests.resolve(frame.requestId); return; case "session.error": + if (pendingRequests.get(frame.requestId)?.ws !== ws) return; pendingRequests.reject(frame.requestId, frame.error); return; case "repo.pack.ack": @@ -1490,16 +1949,16 @@ export function createSidecarRouter( case "workflow.probe.error": rejectProbe(ws, frame.requestId, frame.error); return; - case "oauth.login.result": - settleOAuthLogin(frame.requestId, frame.outcome); + case "deployment.stopped": + handleDeploymentStopped(ws, frame); return; default: return assertNever(frame); } } - // Authenticate a register/reconnect handshake exactly once, then run the - // frame's handler with the verified identity. The claimed `sidecarId` on + // Authenticate a hello handshake exactly once, then run the + // frame's handler with the verified sidecar id. The claimed `sidecarId` on // the frame is an unauthenticated hint: it is logged if it disagrees with // the verified id but never trusted -- routing keys off the verified id. // Fails closed by closing the connection when the authenticator rejects @@ -1508,11 +1967,12 @@ export function createSidecarRouter( async function authenticateHandshake( ws: WsHandle, frame: { type: string; sidecarId: string; token: string }, - run: (identity: SidecarAuthIdentity) => Promise, + run: (sidecarId: string) => Promise, + retired: (sidecarId: string) => void, ): Promise { - let identity: SidecarAuthIdentity | null; + let credentials: SidecarCredentials | null; try { - identity = await authenticateSidecar({ + credentials = await authenticateSidecar({ sidecarId: frame.sidecarId, token: frame.token, }); @@ -1521,20 +1981,22 @@ export function createSidecarRouter( ws.close(); return; } - if (identity === null) { + if (credentials === null) { logger.warn`Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: invalid token`; ws.close(); return; } - if (identity.sidecarId !== frame.sidecarId) { - logger.warn`Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`; - } - if (!(await validateSidecarIdentity(identity, "registration"))) { - logger.warn`Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`; + const sidecarId = credentials.sidecarId; + if (credentials.retired === true) { + logger.warn`Rejected ${frame.type} from sidecar ${sidecarId}: its identity is retired`; + retired(sidecarId); ws.close(); return; } - await run(identity); + if (sidecarId !== frame.sidecarId) { + logger.warn`Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${sidecarId}; keying off the verified id`; + } + await run(sidecarId); } async function notifyAllocationWaiters(allocationId: string): Promise { @@ -1556,7 +2018,7 @@ export function createSidecarRouter( try { identityCurrent = await validation; } catch (cause) { - // A failed revalidation leaves the waiters parked: a later register + // A failed revalidation leaves the waiters parked: a later hello // revalidates, and at expiry the wait reports the failure rather than a // missed deadline. Registration itself was already gated, so this must // not fail the connection that just registered. @@ -1591,89 +2053,762 @@ export function createSidecarRouter( allocationWaiters.delete(allocationId); } - async function handleAllocatedRegister( - ws: WsHandle, - identity: SidecarAuthIdentity, - agentAddresses: string[], - cachedSenderAddresses: string[], - ): Promise { - if (allocationFences.get(identity.allocationId) !== identity.generation) { - logger.warn`Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`; - ws.close(); - return; + function isFencedAsCurrent(binding: SidecarAuthIdentity): boolean { + return ( + allocationFences.get(binding.allocationId)?.generation === + binding.generation + ); + } + + function getCleanupFence( + target: AllocatedSidecarTarget & { sidecarId: string }, + ) { + const fence = allocationFences.get(target.allocationId); + if ( + fence === undefined || + fence.cleanup?.sidecarId !== target.sidecarId || + fence.generation < target.generation + ) + return undefined; + return fence; + } + + // A committed cleanup binding can be observed before its releasing caller + // publishes the fence. Transfer the binding without issuing an untracked undeploy. + function fenceCleanupBindings( + bindings: readonly SidecarAuthIdentity[], + ): void { + for (const binding of bindings) { + if (binding.kind !== "cleanup") continue; + const fence = allocationFences.get(binding.allocationId); + if ( + fence !== undefined && + (fence.generation < binding.generation || + (fence.generation === binding.generation && + fence.cleanup?.sidecarId !== binding.sidecarId)) + ) + fenceAllocation(binding.allocationId, binding.generation, { + cleanup: { sidecarId: binding.sidecarId }, + }); } - if (identity.kind === "probe" && agentAddresses.length > 0) { - logger.warn`Rejected probe sidecar ${identity.sidecarId}: probe ${identity.allocationId} claimed workflow addresses`; - ws.close(); - return; + } + + // Make `bindings` the connection's current set. A binding the in-memory + // fence no longer accepts is skipped; one this socket held that is absent + // or superseded is detached; one still attached to another sidecar's socket + // moves here. Returns the attached bindings and, of those, the ones this + // socket did not already hold at the same generation and kind. A binding it + // already holds that way keeps its entry: readiness and routing checks read + // the entry, await its validation, and treat a replaced entry as a changed + // connection, so a sync that changes nothing must not replace it. A newer + // generation attaches at once even while the sidecar is still tearing an + // older one of the same deployment down: every frame names its generation, + // so nothing the older one still sends is credited to the newer one. + function attachBindings( + ws: WsHandle, + conn: SidecarConnection, + bindings: readonly SidecarAuthIdentity[], + beforeRead?: ReadonlyMap, + ): { attached: SidecarAuthIdentity[]; added: SidecarAuthIdentity[] } { + const beforeTransfer = new Map(conn.bindings); + const unchangedDuringRead = (id: string) => + beforeRead === undefined || beforeRead.get(id) === conn.bindings.get(id); + const observed = bindings.filter((binding) => + unchangedDuringRead(binding.allocationId), + ); + fenceCleanupBindings(observed); + const current = observed.filter(isFencedAsCurrent); + for (const binding of observed) { + if (!current.includes(binding)) { + logger.warn`Sidecar ${conn.sidecarId} skipped ${binding.kind} ${binding.allocationId} generation ${String(binding.generation)}: it is not fenced as current`; + } } + for (const [allocationId, held] of [...conn.bindings]) { + if (!unchangedDuringRead(allocationId)) continue; + const observed = bindings.find( + (binding) => binding.allocationId === allocationId, + ); + if (observed !== undefined && observed.generation < held.generation) + continue; + const next = current.find( + (binding) => binding.allocationId === allocationId, + ); + if (next === undefined || next.generation !== held.generation) { + detachBinding(ws, allocationId, "Its binding is no longer current", { + undeploy: next?.kind !== "cleanup", + }); + } + } + const added: SidecarAuthIdentity[] = []; + for (const binding of current) { + const elsewhere = allocatedConnections.get(binding.allocationId); + if (elsewhere !== undefined && elsewhere.ws !== ws) { + detachBinding( + elsewhere.ws, + binding.allocationId, + `It moved to sidecar ${conn.sidecarId}`, + { undeploy: true }, + ); + } + const held = conn.bindings.get(binding.allocationId); + const unchanged = + held?.generation === binding.generation && held.kind === binding.kind; + const previous = beforeTransfer.get(binding.allocationId); + if ( + previous?.generation !== binding.generation || + previous.kind !== binding.kind + ) + added.push(binding); + if ( + unchanged && + allocatedConnections.get(binding.allocationId)?.ws === ws + ) + continue; + conn.bindings.set(binding.allocationId, binding); + allocatedConnections.set(binding.allocationId, { ws, identity: binding }); + } + return { attached: [...conn.bindings.values()], added }; + } + + // Remove one binding from its connection: drop its routes and in-flight + // work and report the lost routes. Unless cleanup owns removal, also tell the + // sidecar to undeploy it. The socket stays open when nothing is left: whether + // an idle sidecar keeps running is its provisioner's decision. + function detachBinding( + ws: WsHandle, + allocationId: string, + reason: string, + { undeploy }: { undeploy: boolean }, + ): void { + const conn = connections.get(ws); + const binding = conn?.bindings.get(allocationId); + if (conn === undefined || binding === undefined) return; + conn.bindings.delete(allocationId); + const allocated: { allocationId: string; generation: number }[] = []; + if (allocatedConnections.get(allocationId)?.ws === ws) { + allocatedConnections.delete(allocationId); + if (binding.kind === "allocated") { + allocated.push({ allocationId, generation: binding.generation }); + } + } + const addresses = [...conn.workflowAddresses].flatMap(([address, owner]) => + owner === allocationId ? [address] : [], + ); + for (const address of addresses) { + conn.workflowAddresses.delete(address); + if (addressIndex.get(address) === ws) { + addressIndex.delete(address); + connectorStates.delete(address); + retainPendingMailForAddress(address); + } + agentStatePackReceiver.cancelByAgent(address); + workflowRunPackReceiver.cancelByAgent(address); + } + const deploymentAddress = + binding.kind !== "probe" ? binding.workflowRunAddress : undefined; + // A transfer to a cleanup binding keeps the physical generation's proof + // until retirement; other detachments discard the obsolete entry. + if (undeploy && deploymentAddress !== undefined) { + const retained = connectionRetention.get(ws); + const proof = retained?.get(deploymentAddress); + if (proof !== undefined && proof.generation <= binding.generation) + retained?.delete(deploymentAddress); + } + // An own-repository transfer needs no route, so the loop above can miss + // it. + if (deploymentAddress !== undefined) + workflowRunPackReceiver.cancelByAgent(deploymentAddress); + const error = `${binding.kind === "probe" ? "Probe" : "Allocation"} ${allocationId} left sidecar ${conn.sidecarId}: ${reason}`; + const released = new Set(addresses); + if (deploymentAddress !== undefined) released.add(deploymentAddress); + pendingRequests.rejectForWs(ws, (entry) => released.has(entry.meta), error); + pendingDeploys.rejectForWs(ws, (entry) => released.has(entry.key), error); + pendingPacks.rejectForWs( + ws, + (entry) => released.has(entry.meta.agentAddress), + error, + ); + pendingUndeploys.rejectForWs(ws, (entry) => released.has(entry.key), error); + pendingProbes.rejectForWs( + ws, + (entry) => entry.meta === allocationId, + error, + ); + pendingWorkflowControls.rejectForWs( + ws, + (entry) => entry.meta.allocationId === allocationId, + error, + ); + // The Hub cannot tell whether the sidecar still holds the deployment: a + // deploy that timed out or failed has already dropped its route and + // pending entry, yet the sidecar may have installed it. The undeploy + // names this generation, so it never removes a newer one, and a sidecar + // that does not hold it acknowledges it anyway. if ( - agentAddresses.length > 0 && - !(await validateSidecarIdentity(identity, "routing")) + undeploy && + deploymentAddress !== undefined && + binding.kind === "allocated" ) { - logger.warn`Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`; - ws.close(); + sendUndeploy( + conn, + { address: deploymentAddress, generation: binding.generation }, + reason, + { waitable: true }, + ); + } + if (addresses.length > 0 || allocated.length > 0) { + events.emit("sidecar.disconnect", { + ownedAddresses: addresses, + allocated, + }); + } + } + + // Nothing a retired sidecar reports is the Hub's any more, yet it may still + // run some of it, and none of its handshakes gets further than this. + function undeployRetired( + ws: WsHandle, + sidecarId: string, + incarnations: readonly HostedIncarnation[], + ): void { + const retired = { + sidecarId, + undeploying: new Map(), + send(frame: HubFrame) { + ws.send(JSON.stringify(frame)); + }, + }; + for (const incarnation of incarnations) + sendUndeploy(retired, incarnation, "The sidecar's identity is retired", { + waitable: false, + }); + } + + // Only a later deploy of the address on this connection waits on the + // answer, bounded by the request timeout. What the sidecar still sends for + // the incarnation names its generation and is dropped, and an incarnation + // still held after an undeploy lost with its connection is in the sidecar's + // next `hello`. Orphans are undeployed again; cleanup bindings leave removal + // to the reconciler's retry policy. An undeploy no deploy can wait on, + // such as one of an address the sidecar reports but no binding of it + // deploys, is not kept, so a sidecar cannot make the Hub keep any number of + // them. + function sendUndeploy( + conn: Pick, + incarnation: { address: string; generation: number }, + reason: string, + { waitable }: { waitable: boolean }, + ): void { + const requestId = nextRequestId(); + try { + conn.send({ + type: "agent.undeploy", + requestId, + agentAddress: incarnation.address, + generation: incarnation.generation, + reason, + }); + } catch (err) { + logger.warn`Failed to ask sidecar ${conn.sidecarId} to undeploy ${incarnation.address} generation ${String(incarnation.generation)}: ${err instanceof Error ? err.message : String(err)}`; return; } + if (waitable) noteUndeploying(conn, incarnation.address, requestId); + } - const existingOnSocket = connections.get(ws); - const newlyRoutedAddresses = new Set(); - for (const address of agentAddresses) { - const alreadyOwned = - existingOnSocket?.identity.kind === "allocated" && - existingOnSocket.identity.allocationId === identity.allocationId && - addressIndex.get(address) === ws && - connOwnsAddress(existingOnSocket, address); + function noteUndeploying( + conn: Pick, + agentAddress: string, + requestId: string, + ): void { + const inFlight = conn.undeploying.get(agentAddress); + if (inFlight !== undefined) { + // The sidecar answers an address's undeploys in the order it got them, + // so whoever waits on the earlier one waits for this later one. + inFlight.requestId = requestId; + return; + } + let settle = (): void => undefined; + const answered = new Promise((resolve) => { + settle = resolve; + }); + conn.undeploying.set(agentAddress, { requestId, answered, settle }); + } + + // Settled behind the frames the sidecar sent before its answer, so all the + // removed copy still sent is handled, and dropped, before a deploy waiting + // on the answer routes the address again. + function settleUndeploy( + ws: WsHandle, + frame: { agentAddress: string; requestId: string }, + ): void { + void enqueueOnConnection(ws, async () => { + const conn = connections.get(ws); + const inFlight = conn?.undeploying.get(frame.agentAddress); + if (conn === undefined || inFlight?.requestId !== frame.requestId) { + return; + } + conn.undeploying.delete(frame.agentAddress); + inFlight.settle(); + }); + } + + // Resolves once no undeploy of `agentAddress` is unanswered on `conn`. Fails + // when the answer takes longer than a request may, or the caller aborts. + async function undeployAnswered( + conn: SidecarConnection, + agentAddress: string, + signal?: AbortSignal, + ): Promise { + const inFlight = conn.undeploying.get(agentAddress); + if (inFlight === undefined) return; + let timer: ReturnType | undefined; + let onAbort = (): void => undefined; + try { + await Promise.race([ + inFlight.answered, + new Promise((_, reject) => { + timer = setTimeout(() => { + reject( + new Error( + `${agentAddress} is still being undeployed on sidecar ${conn.sidecarId} after ${String(requestTimeoutMs)}ms`, + ), + ); + }, requestTimeoutMs); + onAbort = () => { + reject( + signal?.reason instanceof Error + ? signal.reason + : new Error("The deploy was aborted"), + ); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }), + ]); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + } + } + + async function announceAttached( + ws: WsHandle, + bindings: readonly SidecarAuthIdentity[], + ): Promise { + for (const binding of bindings) { + await notifyAllocationWaiters(binding.allocationId); + // The socket can close or the binding move on while the waiters are + // notified, and a connected event after its disconnect would wake the + // allocation as if its sidecar were back. + const attached = allocatedConnections.get(binding.allocationId); + if ( + attached?.ws !== ws || + attached.identity.generation !== binding.generation + ) + continue; + if (binding.kind !== "probe") { + events.emit("sidecar.allocated.connected", { + allocationId: binding.allocationId, + generation: binding.generation, + }); + } + } + } + + // What a registering sidecar hosts: the bindings current for it, the + // reported incarnations that take their route back, stopped copies kept + // for inspection, live copies kept under retention, and reported copies + // the Hub does not keep. Null once registration has been rejected and its + // socket closed. + async function readHostedWork( + ws: WsHandle, + sidecarId: string, + existing: SidecarConnection | undefined, + incarnations: readonly HostedIncarnation[], + ): Promise<{ + bindings: SidecarAuthIdentity[]; + reclaimed: Map>; + retained: Map>; + unkept: HostedIncarnation[]; + stopOnly: Set; + failed: { + binding: Extract; + error: string; + }[]; + } | null> { + let bindings: SidecarAuthIdentity[]; + try { + const resolved = await resolveSidecarBindings(sidecarId); + const current = await Promise.all( + resolved.map((binding) => + validateSidecarIdentity(binding, "registration"), + ), + ); + bindings = resolved.filter((_, index) => current[index]); + fenceCleanupBindings(bindings); + } catch (err) { + logger.error`Rejected registration from sidecar ${sidecarId}: cannot resolve what it hosts: ${err instanceof Error ? err.message : String(err)}`; + handleClose(ws); + ws.close(); + return null; + } + // A reported incarnation takes its route back only when it is live, it is + // the generation of a binding current for this sidecar, that deployment's + // first deploy has completed, and its run has not ended. A stopped one of + // a current binding, and a live one whose run ended, stay + // unrouted, their local state kept for inspection until the Hub releases + // the deployment; one that stopped on its own is noted, and the reconciler + // fails its deployment. A cleanup binding leaves removal to allocation + // reconciliation. Every other one is undeployed: an earlier + // generation, a live one of a still-live run whose sidecar already reported + // it stopped, a deployment + // that left this sidecar while it was disconnected, one still deploying or + // tearing down, or one whose deploy is still uncertain. The reconciler + // fails a deployment whose current generation the sidecar no longer + // reports. + const reclaimed = new Map< + string, + Extract + >(); + const retained = new Map< + string, + Extract + >(); + const stopOnly = new Set(); + const unkept: HostedIncarnation[] = []; + const failed: { + binding: Extract; + error: string; + }[] = []; + async function readIncarnation(incarnation: HostedIncarnation) { + const address = incarnation.address; + const binding = bindings.find( + (candidate) => + candidate.kind === "allocated" && + candidate.workflowRunAddress === address && + candidate.generation === incarnation.generation, + ); if ( - identity.kind !== "allocated" || - (!alreadyOwned && address !== identity.workflowRunAddress) + binding?.kind !== "allocated" || + (incarnation.state !== "live" && incarnation.state !== "stopped") ) { - logger.warn`Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`; - ws.close(); - return; + return { incarnation, kind: "unkept" } as const; + } + if (incarnation.state === "stopped") { + return { incarnation, binding, kind: "stopped" } as const; + } + const reportedStopped = + reportedStops.get(binding.allocationId)?.generation === + binding.generation; + const alreadyRouted = + existing?.workflowAddresses.get(address) === binding.allocationId && + existing.bindings.get(binding.allocationId)?.generation === + binding.generation && + addressIndex.get(address) === ws; + try { + if ( + !reportedStopped && + (alreadyRouted || (await validateSidecarIdentity(binding, "reclaim"))) + ) + return { incarnation, binding, kind: "reclaimed" } as const; + if (await validateSidecarIdentity(binding, "retention-transition")) { + const retain = await validateSidecarIdentity(binding, "retention"); + return { + incarnation, + binding, + kind: retain ? "retained" : "stop-only", + } as const; + } + return { incarnation, kind: "unkept" } as const; + } catch (err) { + logger.error`Rejected registration from sidecar ${sidecarId}: cannot validate reported ${address} generation ${String(incarnation.generation)}: ${err instanceof Error ? err.message : String(err)}`; + throw err; } - if (!alreadyOwned) newlyRoutedAddresses.add(address); } + // Each validation reads the database. Bound the fan-out while avoiding a + // serial round trip for every deployment before the welcome deadline. + // Apply results in inventory order; no routes are published until every + // batch succeeds and handleRegistration rechecks the connection/fences. + try { + for ( + let offset = 0; + offset < incarnations.length; + offset += INCARNATION_VALIDATION_CONCURRENCY + ) { + const results = await Promise.all( + incarnations + .slice(offset, offset + INCARNATION_VALIDATION_CONCURRENCY) + .map(readIncarnation), + ); + for (const result of results) { + const { incarnation } = result; + switch (result.kind) { + case "reclaimed": + reclaimed.set(incarnation.address, result.binding); + break; + case "retained": + case "stop-only": + retained.set(incarnation.address, result.binding); + if (result.kind === "stop-only") + stopOnly.add(incarnation.address); + break; + case "stopped": + retained.set(incarnation.address, result.binding); + if (incarnation.error !== undefined) + failed.push({ + binding: result.binding, + error: incarnation.error, + }); + break; + case "unkept": + unkept.push(incarnation); + break; + } + } + } + } catch { + handleClose(ws); + ws.close(); + return null; + } + return { bindings, reclaimed, retained, unkept, failed, stopOnly }; + } - if (allocationFences.get(identity.allocationId) !== identity.generation) { + async function handleRegistration( + ws: WsHandle, + sidecarId: string, + incarnations: readonly HostedIncarnation[], + cachedSenderAddresses: string[], + capacity: SidecarCapacity, + ): Promise { + const existing = connections.get(ws); + if (existing !== undefined && existing.sidecarId !== sidecarId) { + logger.warn`Rejected sidecar ${sidecarId}: socket is registered as sidecar ${existing.sidecarId}`; + handleClose(ws); ws.close(); return; } - const current = allocatedConnections.get(identity.allocationId); - if (current !== undefined && current.ws !== ws) { - // A current-generation takeover is a reconnect, not a capacity loss. - // Remove the old socket from the allocation index before closing it so - // handleClose does not emit a false allocated-disconnect event. - allocatedConnections.delete(identity.allocationId); - handleClose(current.ws); - current.ws.close(); - } - - const conn: SidecarConnection = existingOnSocket ?? { - sidecarId: identity.sidecarId, - identity, - agentAddresses: new Set(), - workflowAddresses: new Set(), + + if (closedSockets.has(ws)) return; + + // Claim the sidecar before reading what it hosts. A sync for it queues + // behind this registration from here on, and a sync that ran earlier + // followed a placement this read sees, so work placed on the sidecar + // while it connects is attached either way. The claim ends with the read: + // the takeover below runs without yielding, or the registration stops. + sidecarClaims.set(sidecarId, ws); + let hosted: Awaited>; + let claimHeld: boolean; + try { + hosted = await readHostedWork(ws, sidecarId, existing, incarnations); + } finally { + claimHeld = sidecarClaims.get(sidecarId) === ws; + if (claimHeld) sidecarClaims.delete(sidecarId); + } + if (hosted === null) return; + if (!claimHeld) { + logger.info`Dropping a registration of sidecar ${sidecarId}: its socket closed, or a newer handshake took the sidecar over`; + // A socket already serving the sidecar keeps serving until that + // handshake takes its bindings over. + if (existing === undefined) { + handleClose(ws); + ws.close(); + } + return; + } + const { reclaimed, retained, unkept, failed, stopOnly } = hosted; + // A release can commit while the hello reads other deployments. Carry its + // cleanup binding into welcome so later stop reports still have an owner. + const bindings = hosted.bindings.map((binding) => { + if (binding.kind === "probe") return binding; + const fence = getCleanupFence(binding); + if (fence !== undefined) { + return { + ...binding, + kind: "cleanup" as const, + generation: fence.generation, + }; + } + return binding; + }); + + // A sidecar reconnecting on a new socket takes its bindings along. Move + // them off the previous socket first so its close reports only the + // bindings that did not follow. + const previous = sidecarSockets.get(sidecarId); + if (previous !== undefined && previous !== ws) { + for (const binding of bindings.filter(isFencedAsCurrent)) { + if (allocatedConnections.get(binding.allocationId)?.ws === previous) + allocatedConnections.delete(binding.allocationId); + } + handleClose(previous); + previous.close(); + } + + const conn: SidecarConnection = existing ?? { + sidecarId, + capacity, + bindings: new Map(), + workflowAddresses: new Map(), + unansweredDeploys: new Map(), + undeploying: new Map(), send(frame: HubFrame) { ws.send(JSON.stringify(frame)); }, }; - if ( - conn.identity.kind !== identity.kind || - conn.identity.allocationId !== identity.allocationId || - conn.identity.generation !== identity.generation - ) { - logger.warn`Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`; - ws.close(); - return; - } - + conn.capacity = capacity; connections.set(ws, conn); - for (const address of agentAddresses) { - conn.workflowAddresses.add(address); + sidecarSockets.set(sidecarId, ws); + + const deployable = new Set( + bindings.flatMap((binding) => + binding.kind === "allocated" ? [binding.workflowRunAddress] : [], + ), + ); + for (const incarnation of unkept) { + // The reconciler owns the acknowledged teardown for this release fence. + if ( + bindings.some( + (binding) => + binding.kind !== "probe" && + binding.workflowRunAddress === incarnation.address && + getCleanupFence({ + allocationId: binding.allocationId, + generation: incarnation.generation, + sidecarId, + }) !== undefined, + ) + ) + continue; + logger.warn`Sidecar ${sidecarId} reported ${incarnation.address} generation ${String(incarnation.generation)} ${incarnation.state}, which the Hub does not keep there; asking it to undeploy`; + sendUndeploy( + conn, + incarnation, + "The Hub does not keep this incarnation on this sidecar", + { waitable: deployable.has(incarnation.address) }, + ); + } + const { attached, added } = attachBindings(ws, conn, bindings); + const retention = new Map< + string, + { generation: number; retention: DeploymentRetention } + >(); + for (const incarnation of incarnations) { + if (incarnation.retention === undefined) continue; + const binding = attached.find( + (candidate) => + candidate.kind !== "probe" && + candidate.workflowRunAddress === incarnation.address && + (candidate.kind === "cleanup" + ? candidate.generation >= incarnation.generation + : candidate.generation === incarnation.generation), + ); + if (binding !== undefined) + retention.set(incarnation.address, { + generation: incarnation.generation, + retention: incarnation.retention, + }); + } + connectionRetention.set(ws, retention); + for (const { binding, error } of failed) { + if ( + conn.bindings.get(binding.allocationId)?.generation === + binding.generation + ) + noteReportedStop(binding, error); + } + const newlyRoutedAddresses = new Set(); + const routed: { address: string; generation: number }[] = []; + for (const [address, binding] of reclaimed) { + if (getCleanupFence(binding) !== undefined) continue; + if ( + conn.bindings.get(binding.allocationId)?.generation !== + binding.generation + ) { + sendUndeploy( + conn, + { address, generation: binding.generation }, + "The deployment is not current on this sidecar", + { waitable: true }, + ); + continue; + } + if (addressIndex.get(address) !== ws) newlyRoutedAddresses.add(address); + conn.workflowAddresses.set(address, binding.allocationId); addressIndex.set(address, ws); + routed.push({ address, generation: binding.generation }); } - allocatedConnections.set(identity.allocationId, { ws, identity }); + // Remove stale retained copies unless the current cleanup fence owns them. + for (const [address, binding] of retained) { + if (getCleanupFence(binding) !== undefined) continue; + if ( + conn.bindings.get(binding.allocationId)?.generation !== + binding.generation + ) { + sendUndeploy( + conn, + { address, generation: binding.generation }, + "The deployment is not current on this sidecar", + { waitable: true }, + ); + } + } + // Routing is settled, so the sidecar can stop holding back what must be + // delivered and re-drive what it owes each routed incarnation. + conn.send({ type: "welcome", routed }); + welcomedSockets.add(ws); + notifySidecarInventory(sidecarId); + // Live retained copies were validated in readHostedWork's bounded batches. + // Stop them without repeating those queries or awaiting acknowledgements + // inside the hello handler that must finish before processing the replies. + for (const incarnation of incarnations) { + if (incarnation.state !== "live") continue; + const validated = retained.get(incarnation.address); + if (validated === undefined) continue; + const binding = conn.bindings.get(validated.allocationId); + if ( + binding?.kind !== "allocated" || + binding.generation !== validated.generation || + !isFencedAsCurrent(binding) || + connections.get(ws) !== conn + ) + continue; + const target = { + allocationId: binding.allocationId, + generation: binding.generation, + }; + const stopping = stopOnly.has(incarnation.address); + const request = stopping + ? sendWorkflowControlOnConnection( + { ws, conn, binding }, + target, + { + runId: binding.anchorRunId, + agentAddress: binding.workflowRunAddress, + action: "stop", + reason: "The deployment's retention period ended", + }, + requestTimeoutMs, + { terminal: true }, + ) + : retainOnConnection({ ws, conn, binding }, target, requestTimeoutMs); + void request.catch((cause: unknown) => { + logger.warn`${stopping ? "Stop" : "Retention"} remains unconfirmed after reconnect for ${binding.allocationId}: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + } + if (attached.length === 0) + logger.info`Provisioned sidecar ${sidecarId} registered hosting nothing`; + else + logger.info`Provisioned sidecar ${sidecarId} registered for ${attached.map((binding) => `${binding.kind} ${binding.allocationId} generation ${String(binding.generation)}`).join(", ")}`; + // Replay can wait on database admission without delaying connection readiness. + // Newly attached cleanup bindings can wake pending attempts. Repeated + // hellos on this socket must not bypass the normal retry backoff. + await announceAttached( + ws, + attached.filter( + (binding) => binding.kind !== "cleanup" || added.includes(binding), + ), + ); + if (connections.get(ws) !== conn) return; for (const address of newlyRoutedAddresses) { await redeliverPendingMail(address, conn); } @@ -1699,7 +2834,10 @@ export function createSidecarRouter( // large cache cannot fan out into one concurrent DB query per reported // sender on every reconnect. const resolveSenderKeyStrict = lookups.resolveSenderKeyStrict; - if (identity.kind === "allocated" && resolveSenderKeyStrict !== undefined) { + if ( + allocationBindings(conn).length > 0 && + resolveSenderKeyStrict !== undefined + ) { const rotatableSenders = new Set(cachedSenderAddresses); // Resolve-don't-trust applied to input SIZE: bound the reported set before // acting on it. Run addresses count toward the cap by design -- the @@ -1709,7 +2847,7 @@ export function createSidecarRouter( // log the overflow so a misbehaving sidecar is detectable. let sendersToResync = [...rotatableSenders]; if (sendersToResync.length > MAX_RESYNC_SENDER_ADDRESSES) { - logger.warn`Sidecar ${identity.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses on allocation ${identity.allocationId} generation ${String(identity.generation)}, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`; + logger.warn`Sidecar ${conn.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`; sendersToResync = sendersToResync.slice(0, MAX_RESYNC_SENDER_ADDRESSES); } void (async () => { @@ -1750,45 +2888,55 @@ export function createSidecarRouter( // is gone. That means the connection left, so stop -- the remaining // sends would fail the same way. const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`Sender-key resync for sidecar ${identity.sidecarId} stopped: ${message}`; - }); - } - logger.info`Provisioned sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`; - await notifyAllocationWaiters(identity.allocationId); - if (identity.kind === "allocated") { - events.emit("sidecar.allocated.connected", { - allocationId: identity.allocationId, - generation: identity.generation, + logger.warn`Sender-key resync for sidecar ${conn.sidecarId} stopped: ${message}`; }); } } - async function handleRegister( - ws: WsHandle, - identity: SidecarAuthIdentity, - agentAddresses: string[], - cachedSenderAddresses: string[], - ): Promise { - await handleAllocatedRegister( - ws, - identity, - agentAddresses, - cachedSenderAddresses, - ); + // The socket a sync for the sidecar runs on: the one registering it while + // a handshake holds its claim, otherwise the one it is connected on. + function syncTarget(sidecarId: string): WsHandle | undefined { + return sidecarClaims.get(sidecarId) ?? sidecarSockets.get(sidecarId); } - async function handleReconnect( - ws: WsHandle, - identity: SidecarAuthIdentity, - agentAddresses: string[], - cachedSenderAddresses: string[], + async function syncSidecar( + sidecarId: string, + signal?: AbortSignal, ): Promise { - await handleAllocatedRegister( - ws, - identity, - agentAddresses, - cachedSenderAddresses, - ); + signal?.throwIfAborted(); + const ws = syncTarget(sidecarId); + if (ws === undefined) return; + await enqueueOnConnection(ws, async () => { + signal?.throwIfAborted(); + // The registration this queued behind may have been rejected, leaving + // the sidecar on its previous socket, or superseded. + if (syncTarget(sidecarId) !== ws) { + redirectSync(sidecarId, signal); + return; + } + const conn = connections.get(ws); + if (conn === undefined) return; + const before = new Map(conn.bindings); + const bindings = await resolveSidecarBindings(sidecarId); + signal?.throwIfAborted(); + if (syncTarget(sidecarId) !== ws) { + redirectSync(sidecarId, signal); + return; + } + if (connections.get(ws) !== conn) return; + // A snapshot must not undo a fence or retirement published during its read. + const { added } = attachBindings(ws, conn, bindings, before); + await announceAttached(ws, added); + }); + } + + // Not awaited: the socket the sync moves to may have work queued behind the + // one that redirects it. + function redirectSync(sidecarId: string, signal?: AbortSignal): void { + syncSidecar(sidecarId, signal).catch((err: unknown) => { + if (signal?.aborted) return; + logger.warn`Failed to sync the bindings of sidecar ${sidecarId}: ${err instanceof Error ? err.message : String(err)}`; + }); } // Park a pre-ack sender's mail synchronously and return its entry. Registering @@ -1820,7 +2968,7 @@ export function createSidecarRouter( recipients: entry.recipients, }); logger.warn`Dropping mail from ${entry.authenticatedSender}: its sender key was not recorded before the deferred-mail TTL expired`; - }, disconnectQueueTTLMs), + }, mailHoldTTLMs), }; parked.add(entry); return entry; @@ -1902,10 +3050,173 @@ export function createSidecarRouter( settleSenderMail(sender, outcome); } + function parkDeferredRecipientResolutionMail( + recipient: string, + rawMessage: string, + authenticatedSender: string, + recordedSenderKey: string | null, + ): DeferredRecipientResolutionMailEntry { + let parked = deferredRecipientResolutionMail.get(recipient); + if (parked === undefined) { + parked = new Set(); + deferredRecipientResolutionMail.set(recipient, parked); + } + // The hold owns this recipient alone. Its settle, TTL, and drain report + // `recipients`, and a recorded settle re-drives only that list. + const entry: DeferredRecipientResolutionMailEntry = { + recipient, + authenticatedSender, + rawMessage, + recipients: [recipient], + recordedSenderKey, + inflight: true, + wake: null, + cancelTimer: null, + }; + parked.add(entry); + return entry; + } + + function claimDeferredRecipientResolutionEntry( + entry: DeferredRecipientResolutionMailEntry, + ): boolean { + const parked = deferredRecipientResolutionMail.get(entry.recipient); + if (parked === undefined) return false; + const claimed = parked.delete(entry); + if (!claimed) return false; + if (entry.cancelTimer !== null) { + entry.cancelTimer(); + entry.cancelTimer = null; + } + if (parked.size === 0) + deferredRecipientResolutionMail.delete(entry.recipient); + return true; + } + + function redriveRecipientResolutionMail( + entry: DeferredRecipientResolutionMailEntry, + ): void { + // Re-drive with the sender key bound when this mail was accepted. The + // recipient's recorded key authenticates the deployment, not the + // sender, so it is not the fourth argument. + void Promise.resolve() + .then(() => + handleMailOutbound( + entry.rawMessage, + entry.authenticatedSender, + entry.recipients, + entry.recordedSenderKey, + ), + ) + .catch((err: unknown) => { + logger.error`Re-driving deferred mail to ${entry.recipient} failed: ${err instanceof Error ? err.message : String(err)}`; + }); + } + + function undeliverRecipientResolutionEntry( + entry: DeferredRecipientResolutionMailEntry, + reason: string, + ): void { + events.emit("mail.outbound.undelivered", { + rawMessage: entry.rawMessage, + recipients: entry.recipients, + }); + logger.warn`Dropping mail to ${entry.recipient}: ${reason}`; + } + + // The in-flight attempt is about to route. Drop the hold, including a + // settle that arrived during the await, so that settle does not route too. + function takeInFlightRecipientDelivery( + entry: DeferredRecipientResolutionMailEntry, + ): boolean { + entry.inflight = false; + entry.wake = null; + return claimDeferredRecipientResolutionEntry(entry); + } + + // The TTL starts once the attempt has finished parked. Arming it at + // park time can claim the hold while materialize is still in flight, + // which drops a recorded wake and supersedes a return that is ready + // to route. A drain during that await detaches the entry; arming then + // would leak a timer whose callback no-ops. + function armRecipientResolutionTtl( + entry: DeferredRecipientResolutionMailEntry, + ): void { + if (entry.cancelTimer !== null) return; + const parked = deferredRecipientResolutionMail.get(entry.recipient); + if (parked === undefined || !parked.has(entry)) return; + entry.cancelTimer = scheduleTimeout(() => { + if (!claimDeferredRecipientResolutionEntry(entry)) return; + events.emit("mail.outbound.undelivered", { + rawMessage: entry.rawMessage, + recipients: entry.recipients, + }); + logger.warn`Dropping mail to ${entry.recipient}: credential resolution was not recorded before the deferred-mail TTL expired`; + }, mailHoldTTLMs); + } + + // The in-flight attempt did not route. A settle that arrived during the + // await owns the next step; otherwise the entry stays parked for a later + // settle or the TTL. + function finishParkedRecipientResolution( + entry: DeferredRecipientResolutionMailEntry, + ): void { + entry.inflight = false; + const wake = entry.wake; + if (wake === null) { + armRecipientResolutionTtl(entry); + return; + } + if (!claimDeferredRecipientResolutionEntry(entry)) return; + if ("failed" in wake) { + undeliverRecipientResolutionEntry( + entry, + `recipient deploy failed: ${wake.failed}`, + ); + return; + } + redriveRecipientResolutionMail(entry); + } + + // A thrown attempt fails closed. Claim the hold so the TTL does not later + // report the original recipient list as undelivered. + function abandonInFlightRecipientResolution( + entry: DeferredRecipientResolutionMailEntry, + ): void { + entry.inflight = false; + entry.wake = null; + claimDeferredRecipientResolutionEntry(entry); + } + + function settleRecipientResolutionMail( + address: string, + outcome: SenderDeploySettledOutcome, + ): void { + // A failed attempt has not recorded resolution and does not prove a + // later attempt will not. Claiming the hold drops mail the next + // recorded settle can still deliver. The TTL undelivers if none does. + if ("failed" in outcome) return; + const parked = deferredRecipientResolutionMail.get(address); + if (parked === undefined) return; + const idle: DeferredRecipientResolutionMailEntry[] = []; + for (const entry of parked) { + if (entry.inflight) { + entry.wake = outcome; + } else { + idle.push(entry); + } + } + for (const entry of idle) { + if (!claimDeferredRecipientResolutionEntry(entry)) continue; + redriveRecipientResolutionMail(entry); + } + } + function settleSenderMail( address: string, outcome: SenderDeploySettledOutcome, ): void { + settleRecipientResolutionMail(address, outcome); if ("failed" in outcome) { drainDeferredSenderMail( address, @@ -2017,7 +3328,11 @@ export function createSidecarRouter( rawMessage: string, authenticatedSender: string, recipients: string[], - recordedSenderKey?: string, + // undefined resolves now. null is a finished resolve that found no + // key, and a string is the key already bound for this message. null + // and undefined are different: treating null as missing resolves again + // and can co-deliver a key this message was not accepted under. + recordedSenderKey?: string | null, ): Promise { // A mail addressed to more than one workflow deployment would birth a // run per recipient from a single inbound mail. The stable runId @@ -2068,23 +3383,52 @@ export function createSidecarRouter( }; if (!resolution.deliver) return; const senderIdentities = resolution.senderIdentities; + const boundIdentity = senderIdentities?.[0]; + const boundSenderKey = + boundIdentity === undefined ? null : boundIdentity.publicKey; - // Route to locally connected sidecars first, then try disconnect queues. + // Route each recipient over the connection that routes its address. const unrouted: string[] = []; for (const recipient of recipients) { // Each recipient is isolated: a materialization failure or a // fail-closed rejection for one must not drop the mail for its // co-recipients. The catch fails THIS recipient closed (its run never // starts under-authorized) and continues to the rest. + // Register the hold before awaiting materialization. A settle that + // lands during that await finds the entry and records a wake; it does + // not re-drive until this attempt finishes, so a ready attempt routes + // once and a not-ready attempt hands the mail to that wake. + let parked: DeferredRecipientResolutionMailEntry | undefined; try { + parked = + lookups.materializeMailTriggeredRunGrants !== undefined && + isRunAddress(recipient) + ? parkDeferredRecipientResolutionMail( + recipient, + rawMessage, + authenticatedSender, + boundSenderKey, + ) + : undefined; const outcome = await deliverMailToRecipient( recipient, rawMessage, authenticatedSender, senderIdentities, + parked, ); + if ( + parked !== undefined && + (outcome === "parked" || outcome === "superseded") + ) { + // This hold is only this recipient. Its settle, TTL, and drain + // report that recipient, so the rest of this attempt still runs. + if (outcome === "parked") finishParkedRecipientResolution(parked); + continue; + } if (outcome === "unrouted") unrouted.push(recipient); } catch (err) { + if (parked !== undefined) abandonInFlightRecipientResolution(parked); logger.error`Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`; } } @@ -2103,12 +3447,17 @@ export function createSidecarRouter( // Deliver an inbound mail to one recipient, materializing a // mail-triggered run's grants first when the recipient is a workflow // deployment. Returns: - // - `routed`: the mail reached a live connection or disconnect queue. + // - `routed`: the mail reached the connection that routes the recipient, + // or pending redelivery after a transient admission failure. // - `unrouted`: the mail was locally undeliverable and should be // relayed externally by the host. // - `failed-closed`: the run's grants could not be materialized safely, // so the mail is deliberately DROPPED for this recipient (not relayed) // to keep its run from starting under-authorized. + // - `parked`: credential resolution is not recorded yet. The caller holds + // the mail and retries after initialization; it is not relayed. + // - `superseded`: something else claimed the hold during this attempt. + // The caller must not also route. // // A workflow deployment is the only recipient whose inbound mail can first // fire its stable run. Its grants are reserved, and the `run.grants` frame is @@ -2122,7 +3471,10 @@ export function createSidecarRouter( rawMessage: string, authenticatedSender: string, senderIdentities: RunGrantsFrame["senderIdentities"], - ): Promise<"routed" | "unrouted" | "failed-closed"> { + hold?: DeferredRecipientResolutionMailEntry, + ): Promise< + "routed" | "unrouted" | "failed-closed" | "parked" | "superseded" + > { if ( lookups.materializeMailTriggeredRunGrants !== undefined && isRunAddress(recipient) @@ -2141,6 +3493,12 @@ export function createSidecarRouter( agentAddress: recipient, runId, }); + if (result.outcome === "notReady") return "parked"; + // Claim before any send. A settle that arrived during the await must + // not route beside this attempt. + if (hold !== undefined && !takeInFlightRecipientDelivery(hold)) { + return "superseded"; + } if (result.outcome === "rejected") { // The run's grants could not be materialized with sufficient // authority or it is already terminal. Fail the mail closed for this @@ -2161,16 +3519,6 @@ export function createSidecarRouter( // Finish asynchronous preparation before sending the grants and mail // together, keeping another delivery's key out of the gap between them. const messageId = await deriveMessageId(base64Decode(rawMessage)); - // Send the run's grants ahead of the mail. A `false` here means the - // deployment is unroutable. Do not route the mail that would dispatch - // it; the grants-only reservation remains the canonical snapshot for a - // later first-delivery attempt. - if ( - !sendRunGrants(recipient, runId, result.stepGrants, senderIdentities) - ) { - logger.error`Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`; - return "unrouted"; - } // Route through the messageId handshake `routeMail` -- NOT a // fire-and-forget send. This branch COMMITS a run, so a mail dropped in // the connected window (a socket that half-dies before the sidecar's @@ -2181,7 +3529,7 @@ export function createSidecarRouter( // mail's own id (derived over the same bytes the sidecar derives), so a // redelivery replays identically and the downstream RunStarted / // stable-runId dedup makes it effectively-once. - const outcome: "routed" | "unrouted" = routeMail( + const outcome: "routed" | "unrouted" = (await routeMail( recipient, rawMessage, authenticatedSender, @@ -2191,7 +3539,7 @@ export function createSidecarRouter( stepGrants: result.stepGrants, ...(senderIdentities !== undefined ? { senderIdentities } : {}), }, - ) + )) ? "routed" : "unrouted"; return outcome; @@ -2201,7 +3549,7 @@ export function createSidecarRouter( // authorize. No run is committed here, so no ack handshake is needed. } - return routeMail(recipient, rawMessage, authenticatedSender) + return (await routeMail(recipient, rawMessage, authenticatedSender)) ? "routed" : "unrouted"; } @@ -2242,16 +3590,13 @@ export function createSidecarRouter( ws: WsHandle, frame: SignalCorrelationRegisterFrame, ): Promise { - // Gate the co-write on the sending sidecar actually owning the named - // deployment address, mirroring the connector.state.changed and pack - // handlers. A workflow deployment routes on the keyless workflow set, so - // ownership is the union check, not addressIndex identity alone. Without - // it a misbehaving sidecar that knows another deployment's address could - // register a spurious correlation against it. + // Only the incarnation this connection routes may register a correlation + // for its address; otherwise a misbehaving sidecar that knows another + // deployment's address could register a spurious one against it. const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.agentAddress)) { - logger.warn`Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`; + if (!namesRoutedIncarnation(conn, frame)) { + logger.warn`Dropping signal.correlation.register for ${frame.agentAddress} generation ${String(frame.generation)}: this sidecar does not route that incarnation`; return; } @@ -2287,75 +3632,14 @@ export function createSidecarRouter( } function handleClose(ws: WsHandle): void { - const conn = connections.get(ws); - if (conn === undefined) return; - let allocated: { allocationId: string; generation: number } | undefined; - - for (const addr of conn.agentAddresses) { - // Only remove routing and pending state if this connection still - // owns the address. A reconnected sidecar may have already claimed it. - if (addressIndex.get(addr) === ws) { - addressIndex.delete(addr); - // Drop cached connector state for the same reason: a takeover - // sidecar's state lives in connectorStates under the same key, - // and only this owner's close should evict it. The next - // reconnect re-bootstraps via the router's - // restore-fires-callback path. - connectorStates.delete(addr); - // Retain this address's un-acked pending mail across the disconnect: - // its in-flight retry timers target a dead socket (cleared), but the - // entries are held so a verified reconnect redelivers them, closing the - // connected-window drop rather than losing the mail. Bounded by a - // retention TTL. - retainPendingMailForAddress(addr); - // Create a queue entry so messages can accumulate while the - // sidecar is disconnected. Skip if the agent is being undeployed -- - // there is no point queuing messages for an agent being torn down. - if (!pendingUndeploys.has(addr)) { - const timer = setTimeout(() => { - const expired = disconnectedAgents.get(addr); - disconnectedAgents.delete(addr); - if (expired !== undefined) { - surfaceDroppedFrames( - addr, - expired.queue, - "disconnect queue TTL expired", - ); - } - }, disconnectQueueTTLMs); - disconnectedAgents.set(addr, { queue: [], timer }); - } + closedSockets.add(ws); + for (const [sidecarId, claimant] of sidecarClaims) { + if (claimant === ws) { + sidecarClaims.delete(sidecarId); + // A failed takeover can leave the previous connection usable. + notifySidecarInventory(sidecarId); } } - // Remove this connection's workflow-substrate routes. No disconnect queue - // is created: these addresses re-register (with the complete live set) - // when the sidecar reconnects, and their in-flight run state is - // reconstructed sidecar-locally, not from a hub-side queue. The ownership - // guard mirrors the session loop above so a takeover by a newer ws is not - // clobbered by the prior owner's close. - for (const addr of conn.workflowAddresses) { - if (addressIndex.get(addr) === ws) { - addressIndex.delete(addr); - connectorStates.delete(addr); - // Retain un-acked workflow trigger mail across the disconnect for the - // same reason as the session loop above -- an authenticated reconnect - // redelivers it. This is un-acked TRIGGER mail, distinct from the - // deployment's in-flight run state (reconstructed sidecar-locally); the - // "no disconnect queue" note above is about that run state, not this. - retainPendingMailForAddress(addr); - } - } - const current = allocatedConnections.get(conn.identity.allocationId); - if (current?.ws === ws) { - allocatedConnections.delete(conn.identity.allocationId); - if (conn.identity.kind === "allocated") { - allocated = { - allocationId: conn.identity.allocationId, - generation: conn.identity.generation, - }; - } - } - connections.delete(ws); // Cancel the liveness timer for this connection. const cancelLiveness = livenessTimers.get(ws); @@ -2367,6 +3651,42 @@ export function createSidecarRouter( // Drop the per-ws serialization chain; no more frames will queue on it. messageChains.delete(ws); + const conn = connections.get(ws); + if (conn === undefined) return; + const allocated: { allocationId: string; generation: number }[] = []; + // A deploy waiting on an answer this socket will not carry goes on and + // finds the connection gone. + for (const inFlight of conn.undeploying.values()) inFlight.settle(); + conn.undeploying.clear(); + + // Remove this connection's workflow-substrate routes. The sidecar reports + // what it still holds in its next `hello`, and its in-flight run state is + // reconstructed sidecar-locally, not from a hub-side queue. Only routes + // this connection still owns are removed, so a takeover by a newer ws is + // not clobbered by the prior owner's close. + for (const addr of conn.workflowAddresses.keys()) { + if (addressIndex.get(addr) === ws) { + addressIndex.delete(addr); + connectorStates.delete(addr); + // Retain un-acked trigger mail across the disconnect: its in-flight + // retry timers target a dead socket (cleared), but the entries are + // held so the next registration that routes the same incarnation + // redelivers them. Bounded by a retention TTL. + retainPendingMailForAddress(addr); + } + } + for (const [allocationId, binding] of conn.bindings) { + if (allocatedConnections.get(allocationId)?.ws !== ws) continue; + allocatedConnections.delete(allocationId); + if (binding.kind === "allocated") { + allocated.push({ allocationId, generation: binding.generation }); + } + } + if (sidecarSockets.get(conn.sidecarId) === ws) { + sidecarSockets.delete(conn.sidecarId); + } + connections.delete(ws); + // Reject any in-flight requests that were sent to this sidecar. Each // entry's reject closure runs its own per-site cleanup (the deploy and // undeploy closures roll routing back), exactly as a frame-error @@ -2375,9 +3695,7 @@ export function createSidecarRouter( ws, `Sidecar ${conn.sidecarId} disconnected`, ); - // Reject every deploy issued on this socket, including allocated - // workflow deployments stored in `workflowAddresses` rather than - // `agentAddresses`. + // Reject every deploy issued on this socket. pendingDeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`); // Reject any in-flight pack transfers for this sidecar. pendingPacks.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`); @@ -2391,37 +3709,31 @@ export function createSidecarRouter( // without it a probe whose sidecar drops mid-flight would hang until its // own timeout instead of failing fast on the disconnect. pendingProbes.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`); - - // Fail any in-flight loopback logins on this sidecar the same way: the - // login dies with its callback server, so a disconnect is terminal even - // though the browser may still be sitting on the consent page. - for (const [requestId, login] of pendingOAuthLogins) { - if (login.ws !== ws) continue; - clearTimeout(login.timer); - pendingOAuthLogins.delete(requestId); - settleOAuthLoginError(requestId, login, "the sidecar disconnected"); - } + pendingWorkflowControls.rejectAllForWs( + ws, + `Sidecar ${conn.sidecarId} disconnected`, + ); // Cancel any in-flight inbound pack transfers from this sidecar // across both receivers. The two receivers track their own in- // flight transferIds, so a pending workflow-run transfer for an // agent that just disconnected won't outlive the connection just - // because the agent-state receiver has nothing to cancel. Iterate the - // owned union so a reconnected workflow deployment's transfer is - // cancelled too; the deduped set avoids a double-cancel for an address - // that is in both sets. A reclaimed address is not present here -- the - // verified reconnect path that took it over evicts it from this - // (superseded) connection's owned set -- so a stale close does not cancel - // the new owner's work. - const owned = ownedAddresses(conn); + // because the agent-state receiver has nothing to cancel. + const owned = new Set(conn.workflowAddresses.keys()); for (const addr of owned) { agentStatePackReceiver.cancelByAgent(addr); workflowRunPackReceiver.cancelByAgent(addr); } + // An own-repository transfer needs no route, so the owned addresses above + // can miss it. A successor for these allocations closes this connection + // while registering, before handling any frame of its own, so this cannot + // cancel the successor's transfers. + for (const binding of allocationBindings(conn)) + workflowRunPackReceiver.cancelByAgent(binding.workflowRunAddress); events.emit("sidecar.disconnect", { ownedAddresses: [...owned], - ...(allocated !== undefined ? { allocated } : {}), + allocated, }); logger.info`Sidecar ${conn.sidecarId} disconnected`; @@ -2431,25 +3743,39 @@ export function createSidecarRouter( return `req-${++requestCounter}`; } + // The connection and incarnation the Hub routes `agentAddress` to, or + // undefined when it routes none. Frames sent through it name the generation + // of this incarnation. + function routedIncarnation(agentAddress: string): + | { + ws: WsHandle; + conn: SidecarConnection; + binding: Extract; + } + | undefined { + const ws = addressIndex.get(agentAddress); + const conn = ws === undefined ? undefined : connections.get(ws); + const binding = + conn === undefined ? undefined : owningAllocation(conn, agentAddress); + if (ws === undefined || conn === undefined || binding === undefined) + return undefined; + return { ws, conn, binding }; + } + function sendRequest( agentAddress: string, - buildFrame: (requestId: string) => HubFrame, + buildFrame: (requestId: string, generation: number) => HubFrame, ): Promise { - const ws = addressIndex.get(agentAddress); - if (ws === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), - ); - } - const conn = connections.get(ws); - if (conn === undefined) { + const routed = routedIncarnation(agentAddress); + if (routed === undefined) { return Promise.reject( new Error(`No sidecar connected for agent "${agentAddress}"`), ); } + const { ws, conn, binding } = routed; const requestId = nextRequestId(); - const frame = buildFrame(requestId); + const frame = buildFrame(requestId, binding.generation); return new Promise((resolve, reject) => { pendingRequests.register( @@ -2463,7 +3789,7 @@ export function createSidecarRouter( reject(new Error(error)); }, }, - undefined, + agentAddress, ); conn.send(frame); @@ -2514,25 +3840,51 @@ export function createSidecarRouter( ); } - function resolveUndeployPending(ws: WsHandle, agentAddress: string): void { - const req = pendingUndeploys.get(agentAddress); - if (req === undefined) { - logger.warn`Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`; + function resolveUndeployPending( + ws: WsHandle, + frame: AgentUndeployAckFrame, + ): void { + const req = pendingUndeploys.get(frame.agentAddress); + if ( + req?.ws !== ws || + req.meta.requestId !== frame.requestId || + req.meta.generation !== frame.generation + ) { + logger.debug`agent.undeploy.ack for ${frame.agentAddress} generation ${String(frame.generation)} answers an undeploy no request waits on`; return; } - if (req.ws !== ws) return; - pendingUndeploys.resolve(agentAddress); + const target = req.meta.target; + if (target !== undefined) { + const current = allocatedConnections.get(target.allocationId); + if ( + allocationFences.get(target.allocationId)?.generation !== + target.generation || + current?.ws !== ws || + current.identity.kind !== "cleanup" || + current.identity.generation !== target.generation + ) { + pendingUndeploys.reject(frame.agentAddress, "Cleanup identity changed"); + return; + } + } + pendingUndeploys.resolve(frame.agentAddress); } function rejectUndeployPending( ws: WsHandle, - agentAddress: string, - error: string, + frame: AgentUndeployErrorFrame, ): void { - const req = pendingUndeploys.get(agentAddress); - if (req === undefined) return; - if (req.ws !== ws) return; - pendingUndeploys.reject(agentAddress, error); + const req = pendingUndeploys.get(frame.agentAddress); + if ( + req?.ws === ws && + req.meta.requestId === frame.requestId && + req.meta.generation === frame.generation + ) { + pendingUndeploys.reject(frame.agentAddress, frame.error); + return; + } + // No caller waits on this undeploy, so the failure surfaces only here. + logger.error`Sidecar failed to undeploy ${frame.agentAddress} generation ${String(frame.generation)}: ${frame.error}`; } function resolveProbe( @@ -2586,25 +3938,48 @@ export function createSidecarRouter( } } + // The generation whose workflow stop the Hub confirmed, when the pack is + // for its workflow-run repository. That history is final. + function stoppedPackSender( + conn: SidecarConnection, + frame: PackPushFrame | PackDoneFrame, + ): SidecarAuthIdentity | undefined { + const binding = deploymentBinding(conn, frame.agentAddress); + return frame.repoId.kind === "workflow-run" && + binding !== undefined && + stoppedAllocations.get(binding.allocationId) === binding.generation + ? binding + : undefined; + } + + // The sender holds a transfer open until it is answered, so a done that + // fails validation is still rejected when it names its transfer. + function rejectMalformedPackDone( + ws: WsHandle, + done: typeof MalformedPackDone.infer, + ): void { + const conn = connections.get(ws); + if (conn === undefined) return; + pickPackReceiver(done.repoId)?.receiver.cancel(done.transferId); + conn.send({ + type: "repo.pack.reject", + agentAddress: done.agentAddress, + repoId: done.repoId, + transferId: done.transferId, + reason: "corrupt", + }); + } + function handlePackPush(ws: WsHandle, frame: PackPushFrame): void { const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.agentAddress)) { - logger.warn`Received repo.pack.push for unrouted agent ${frame.agentAddress}`; - return; - } - if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) { - logger.warn`Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`; - conn.send({ - type: "repo.pack.reject", - agentAddress: frame.agentAddress, - repoId: frame.repoId, - transferId: frame.transferId, - reason: "path_violation", - }); + // A chunk of a transfer the connection may not make is dropped: the + // transfer is answered once, when its `repo.pack.done` is rejected. + if ( + stoppedPackSender(conn, frame) !== undefined || + !connCanPushRepo(conn, frame) + ) return; - } - if (conn.identity.kind !== "allocated") return; const picked = pickPackReceiver(frame.repoId); if (picked === null) { @@ -2637,12 +4012,23 @@ export function createSidecarRouter( ): Promise { const conn = connections.get(ws); if (conn === undefined) return; - if (!connOwnsAddress(conn, frame.agentAddress)) { - logger.warn`Received repo.pack.done for unrouted agent ${frame.agentAddress}`; + const stopped = stoppedPackSender(conn, frame); + if (stopped !== undefined) { + logger.warn`Rejected repo.pack.done from allocation ${stopped.allocationId} after its workflow stop`; + conn.send({ + type: "repo.pack.reject", + agentAddress: frame.agentAddress, + repoId: frame.repoId, + transferId: frame.transferId, + reason: "path_violation", + }); return; } - if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) { - logger.warn`Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`; + const identity = deploymentBinding(conn, frame.agentAddress); + if (identity === undefined || !connCanPushRepo(conn, frame)) { + // Answer rather than drop: the sender holds the transfer open until it + // is answered, and later pushes to the same repository wait behind it. + logger.warn`Rejected repo.pack.done for ${frame.agentAddress} generation ${String(frame.generation)} outside sidecar ${conn.sidecarId}'s routed repositories`; conn.send({ type: "repo.pack.reject", agentAddress: frame.agentAddress, @@ -2652,8 +4038,6 @@ export function createSidecarRouter( }); return; } - if (conn.identity.kind !== "allocated") return; - const identity = conn.identity; const picked = pickPackReceiver(frame.repoId); if (picked === null) { @@ -2691,20 +4075,29 @@ export function createSidecarRouter( return; } - const verdict = await receivePackLookup( - frame.repoId, - result.pack, - result.ref, - result.commitSha, - { - kind: "allocated", - agentAddress: frame.agentAddress, - allocationId: identity.allocationId, - anchorRunId: identity.anchorRunId, - generation: identity.generation, - }, - ); - + let verdict: Awaited>; + try { + verdict = await receivePackLookup( + frame.repoId, + result.pack, + result.ref, + result.commitSha, + { + kind: "allocated", + agentAddress: frame.agentAddress, + allocationId: identity.allocationId, + anchorRunId: identity.anchorRunId, + generation: identity.generation, + }, + ); + } catch (err) { + // A lookup that throws still owes the sender its answer: the socket + // stays open for the sidecar's other deployments, so nothing else + // settles the transfer, and later pushes to the repository wait on it. + logger.error`Receiving the pack for ${frame.agentAddress} generation ${String(frame.generation)} failed: ${err instanceof Error ? err.message : String(err)}`; + verdict = { accepted: false, reason: "corrupt" }; + } + // Connection may have closed during async verification. const currentConn = connections.get(ws); if (currentConn === undefined) return; @@ -2733,22 +4126,31 @@ export function createSidecarRouter( * and asset packs before the deployment-level frame spawns the child. * * The address is Hub-minted and workflow-derived, so it enters the - * `workflowAddresses` set rather than the legacy `agentAddresses` set and is + * `workflowAddresses` set and is * torn down by `unbindStepRoute` once the * step's packs land. `handleClose` reclaims it if the sidecar drops * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and - * drains use the deployment address), so the binding is transient: it is - * never persisted into the reconnect set and never resurrected on - * reconnect. + * drains use the deployment address), so the binding is transient: no + * `hello` reports it and no reconnect restores it. */ - function fenceAllocation(allocationId: string, generation: number): void { + function fenceAllocation( + allocationId: string, + generation: number, + { cleanup }: { cleanup?: { sidecarId: string } } = {}, + ): void { const existing = allocationFences.get(allocationId); - if (existing !== undefined && generation < existing) { + if (existing !== undefined && generation < existing.generation) { throw new Error( - `Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`, + `Cannot move allocation ${allocationId} fence backward from ${String(existing.generation)} to ${String(generation)}`, ); } - allocationFences.set(allocationId, generation); + allocationFences.set(allocationId, { generation, cleanup }); + const stopped = stoppedAllocations.get(allocationId); + if (stopped !== undefined && stopped < generation) + stoppedAllocations.delete(allocationId); + const reportedStop = reportedStops.get(allocationId); + if (reportedStop !== undefined && reportedStop.generation < generation) + reportedStops.delete(allocationId); // A durable generation advance resolves unfinished initialization as failed. // This also covers a cleanup transaction whose response was lost: the next @@ -2766,8 +4168,27 @@ export function createSidecarRouter( const current = allocatedConnections.get(allocationId); if (current !== undefined && current.identity.generation !== generation) { - handleClose(current.ws); - current.ws.close(); + const cleanupCurrent = cleanup?.sidecarId === current.identity.sidecarId; + const conn = connections.get(current.ws); + detachBinding( + current.ws, + allocationId, + `Generation ${String(generation)} superseded it`, + { undeploy: !cleanupCurrent }, + ); + if ( + cleanupCurrent && + conn !== undefined && + current.identity.kind !== "probe" + ) { + // Keep the control binding while the reconciler releases the provider + // hold. It carries no workflow routes, and cleanup still revalidates + // the durable generation before sending. + attachBindings(current.ws, conn, [ + ...conn.bindings.values(), + { ...current.identity, kind: "cleanup", generation }, + ]); + } } const waiters = allocationWaiters.get(allocationId); @@ -2786,10 +4207,16 @@ export function createSidecarRouter( } function retireAllocation(target: AllocatedSidecarTarget): void { - if (allocationFences.get(target.allocationId) !== target.generation) return; + if ( + allocationFences.get(target.allocationId)?.generation !== + target.generation + ) + return; - disconnectAllocation(target); + detachAllocation(target); allocationFences.delete(target.allocationId); + stoppedAllocations.delete(target.allocationId); + reportedStops.delete(target.allocationId); // The fence is gone, so a lingering attempt can never settle normally. // Fail it here rather than leaving a marker that blocks the address. @@ -2819,9 +4246,21 @@ export function createSidecarRouter( async function getProvisionedConnection( target: AllocatedSidecarTarget, - use: "readiness" | "routing", - ): Promise<{ ws: WsHandle; conn: SidecarConnection }> { - if (allocationFences.get(target.allocationId) !== target.generation) { + use: + | "readiness" + | "routing" + | "cleanup" + | "retention" + | "retention-transition", + ): Promise<{ + ws: WsHandle; + conn: SidecarConnection; + binding: SidecarAuthIdentity; + }> { + if ( + allocationFences.get(target.allocationId)?.generation !== + target.generation + ) { throw new Error( `Allocation ${target.allocationId} generation ${String(target.generation)} is not current`, ); @@ -2835,6 +4274,14 @@ export function createSidecarRouter( `Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`, ); } + if ( + (current.identity.kind === "cleanup" && use !== "cleanup") || + (current.identity.kind !== "cleanup" && use === "cleanup") + ) { + throw new Error( + `Allocation ${target.allocationId} does not permit ${use}`, + ); + } let identityCurrent: boolean; try { identityCurrent = await validateSidecarIdentity(current.identity, use); @@ -2846,9 +4293,37 @@ export function createSidecarRouter( ); } if (!identityCurrent) { - if (allocatedConnections.get(target.allocationId) === current) { - handleClose(current.ws); - current.ws.close(); + if ( + use !== "retention-transition" && + use !== "retention" && + allocatedConnections.get(target.allocationId) === current + ) { + // The database may have committed release before its caller published + // the fence. Do not let an old acknowledgement undeploy that copy. + let bindings: readonly SidecarAuthIdentity[]; + try { + bindings = await resolveSidecarBindings(current.identity.sidecarId); + } catch (cause) { + throw new SidecarIdentityValidationError( + target.allocationId, + target.generation, + cause, + ); + } + if (allocatedConnections.get(target.allocationId) === current) { + fenceCleanupBindings( + bindings.filter( + (binding) => binding.allocationId === target.allocationId, + ), + ); + if (allocatedConnections.get(target.allocationId) === current) + detachBinding( + current.ws, + target.allocationId, + "Its identity is no longer current", + { undeploy: true }, + ); + } } throw new Error( `Allocated sidecar identity is no longer current for allocation ${target.allocationId}`, @@ -2860,37 +4335,34 @@ export function createSidecarRouter( ); } const conn = connections.get(current.ws); + const binding = conn?.bindings.get(target.allocationId); if ( conn === undefined || - conn.identity.allocationId !== target.allocationId || - conn.identity.generation !== target.generation + binding === undefined || + binding.generation !== target.generation ) { throw new Error( `Allocated sidecar is not connected for allocation ${target.allocationId}`, ); } - return { ws: current.ws, conn }; + return { ws: current.ws, conn, binding }; } async function getAllocatedConnection( target: AllocatedSidecarTarget, - use: "readiness" | "routing", + use: "readiness" | "routing" | "retention" | "retention-transition", ): Promise<{ ws: WsHandle; - conn: SidecarConnection & { - identity: Extract; - }; + conn: SidecarConnection; + binding: Extract; }> { - const current = await getProvisionedConnection(target, use); - if (current.conn.identity.kind !== "allocated") { + const { ws, conn, binding } = await getProvisionedConnection(target, use); + if (binding.kind !== "allocated") { throw new Error( `Allocation ${target.allocationId} is connected as probe capacity`, ); } - return { - ws: current.ws, - conn: { ...current.conn, identity: current.conn.identity }, - }; + return { ws, conn, binding }; } async function isAllocatedSidecarReady( @@ -2908,36 +4380,193 @@ export function createSidecarRouter( } } + function holdsAllocatedBinding(target: AllocatedSidecarTarget): boolean { + const held = allocatedConnections.get(target.allocationId)?.identity; + return held?.kind === "allocated" && held.generation === target.generation; + } + + function getAllocationRetention( + target: AllocatedSidecarTarget, + ): DeploymentRetention | undefined { + const current = allocatedConnections.get(target.allocationId); + if ( + current?.identity.kind !== "allocated" || + current.identity.generation !== target.generation || + !isFencedAsCurrent(current.identity) || + connections.get(current.ws)?.bindings.get(target.allocationId) !== + current.identity + ) + return undefined; + const retained = connectionRetention + .get(current.ws) + ?.get(current.identity.workflowRunAddress); + return retained?.generation === target.generation + ? retained.retention + : undefined; + } + + function sidecarInventoryConnection(sidecarId: string) { + const ws = sidecarSockets.get(sidecarId); + const conn = ws === undefined ? undefined : connections.get(ws); + if ( + ws === undefined || + conn === undefined || + closedSockets.has(ws) || + !welcomedSockets.has(ws) || + sidecarClaims.has(sidecarId) + ) + return undefined; + return { ws, conn }; + } + + function notifySidecarInventory(sidecarId: string): void { + if (sidecarInventoryConnection(sidecarId) === undefined) return; + for (const ready of [...(inventoryWaiters.get(sidecarId) ?? [])]) ready(); + } + + async function waitForSidecarInventory( + sidecarId: string, + signal: AbortSignal, + ): Promise { + signal.throwIfAborted(); + if (sidecarInventoryConnection(sidecarId) !== undefined) return; + await new Promise((resolve, reject) => { + const waiters = inventoryWaiters.get(sidecarId) ?? new Set<() => void>(); + const cleanUp = () => { + signal.removeEventListener("abort", abort); + waiters.delete(ready); + if (waiters.size === 0) inventoryWaiters.delete(sidecarId); + }; + const ready = () => { + cleanUp(); + resolve(); + }; + const abort = () => { + cleanUp(); + reject(signal.reason); + }; + waiters.add(ready); + inventoryWaiters.set(sidecarId, waiters); + signal.addEventListener("abort", abort, { once: true }); + }); + } + + function getSidecarInventory( + sidecarId: string, + ): SidecarInventory | undefined { + const current = sidecarInventoryConnection(sidecarId); + if (current === undefined) return undefined; + const { ws, conn } = current; + const retained = connectionRetention.get(ws); + const retainedIncarnations = [...conn.bindings.values()].flatMap( + (binding) => { + if ( + binding.kind === "probe" || + !isFencedAsCurrent(binding) || + allocatedConnections.get(binding.allocationId)?.ws !== ws + ) + return []; + const copy = retained?.get(binding.workflowRunAddress); + if ( + copy?.retention !== "kept" || + (binding.kind === "cleanup" + ? copy.generation > binding.generation + : copy.generation !== binding.generation) + ) + return []; + return [ + { allocationId: binding.allocationId, generation: copy.generation }, + ]; + }, + ); + return { capacity: conn.capacity, retainedIncarnations }; + } + + function getRetentionCandidates() { + const candidates: (AllocatedSidecarTarget & { sidecarId: string })[] = []; + for (const [allocationId, current] of allocatedConnections) { + if ( + current.identity.kind !== "allocated" || + !isFencedAsCurrent(current.identity) || + connections.get(current.ws)?.bindings.get(allocationId) !== + current.identity + ) + continue; + const target = { allocationId, generation: current.identity.generation }; + if ( + getAllocationRetention(target) !== "kept" || + stoppedAllocations.get(allocationId) !== target.generation + ) + candidates.push({ ...target, sidecarId: current.identity.sidecarId }); + } + return candidates; + } + async function isAllocatedWorkflowActive( target: AllocatedSidecarTarget, ): Promise { - try { - const { conn } = await getAllocatedConnection(target, "readiness"); - if (conn.identity.kind !== "allocated") return false; - return conn.workflowAddresses.has(conn.identity.workflowRunAddress); - } catch (error) { - if (error instanceof SidecarIdentityValidationError) throw error; - return false; - } + const { conn, binding } = await getAllocatedConnection(target, "readiness"); + return ( + conn.workflowAddresses.get(binding.workflowRunAddress) === + binding.allocationId + ); + } + + function reportedStop(target: AllocatedSidecarTarget) { + const reported = reportedStops.get(target.allocationId); + return reported?.generation === target.generation ? reported : undefined; + } + + function reportedDeploymentFailure( + target: AllocatedSidecarTarget, + ): string | undefined { + return reportedStop(target)?.error; + } + + async function stoppedDeploymentHistory( + target: AllocatedSidecarTarget, + ): Promise<{ unreceived: string | null; reportedAt: Date } | undefined> { + const reported = reportedStop(target); + if (reported === undefined) return undefined; + // A generation whose stop the Hub confirmed is fenced, so the history the + // Hub holds is final. + return { + unreceived: + stoppedAllocations.get(target.allocationId) === target.generation + ? null + : await findUnreceivedWorkflowHistory( + reported.address, + reported.refTips, + ), + reportedAt: reported.reportedAt, + }; } async function waitForAllocatedSidecar( target: AllocatedSidecarTarget, timeoutMs: number, onValidation?: (validation: Promise) => void, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); // An indeterminable worker waits out the unknown while time remains: only // confirmed absence may surface as a connection timeout. At expiry the // wait reports the validation failure rather than a missed deadline, so // the caller retries instead of releasing a worker that may be healthy. let validationFailure: SidecarIdentityValidationError | undefined; + let ready = false; try { - if (await isAllocatedSidecarReady(target)) return; + ready = await isAllocatedSidecarReady(target); } catch (error) { if (!(error instanceof SidecarIdentityValidationError)) throw error; validationFailure = error; } - if (allocationFences.get(target.allocationId) !== target.generation) { + signal?.throwIfAborted(); + if (ready) return; + if ( + allocationFences.get(target.allocationId)?.generation !== + target.generation + ) { throw new Error( `Allocation ${target.allocationId} generation ${String(target.generation)} is not current`, ); @@ -2949,20 +4578,36 @@ export function createSidecarRouter( ); } + const waitMs = Math.min(timeoutMs, MAX_TIMER_DELAY_MS); await new Promise((resolve, reject) => { + const cleanUp = () => { + clearTimeout(waiter.timer); + signal?.removeEventListener("abort", onAbort); + const current = allocationWaiters.get(target.allocationId); + current?.delete(waiter); + if (current?.size === 0) allocationWaiters.delete(target.allocationId); + }; + const onAbort = () => { + waiter.reject( + signal?.reason instanceof Error + ? signal.reason + : new Error("Sidecar connection wait cancelled"), + ); + }; const waiter: AllocationWaiter = { generation: target.generation, validations: new Set(), ...(onValidation !== undefined ? { onValidation } : {}), - resolve, - reject, + resolve() { + cleanUp(); + resolve(); + }, + reject(error) { + cleanUp(); + reject(error); + }, timer: setTimeout(() => { - const current = allocationWaiters.get(target.allocationId); - current?.delete(waiter); - if (current?.size === 0) { - allocationWaiters.delete(target.allocationId); - } - reject( + waiter.reject( waiter.validationFailure ?? (waiter.validations.size > 0 ? new SidecarIdentityValidationError( @@ -2973,7 +4618,7 @@ export function createSidecarRouter( `Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`, )), ); - }, timeoutMs), + }, waitMs), ...(validationFailure !== undefined ? { validationFailure } : {}), }; let waiters = allocationWaiters.get(target.allocationId); @@ -2982,6 +4627,7 @@ export function createSidecarRouter( allocationWaiters.set(target.allocationId, waiters); } waiters.add(waiter); + signal?.addEventListener("abort", onAbort, { once: true }); void notifyAllocationWaiters(target.allocationId); }); } @@ -2997,7 +4643,13 @@ export function createSidecarRouter( `Workflow step ${stepAddress} is already routed to another sidecar`, ); } - conn.workflowAddresses.add(stepAddress); + const owner = conn.workflowAddresses.get(stepAddress); + if (owner !== undefined && owner !== target.allocationId) { + throw new Error( + `Workflow step ${stepAddress} is already routed to allocation ${owner}`, + ); + } + conn.workflowAddresses.set(stepAddress, target.allocationId); addressIndex.set(stepAddress, ws); } @@ -3013,7 +4665,10 @@ export function createSidecarRouter( return; } if (addressIndex.get(stepAddress) !== current.ws) return; - connections.get(current.ws)?.workflowAddresses.delete(stepAddress); + const conn = connections.get(current.ws); + if (conn?.workflowAddresses.get(stepAddress) !== target.allocationId) + return; + conn.workflowAddresses.delete(stepAddress); addressIndex.delete(stepAddress); } @@ -3024,12 +4679,13 @@ export function createSidecarRouter( ws: WsHandle, conn: SidecarConnection, agentAddress: string, + generation: number, pack: Uint8Array, ref: string, commitSha: string, options?: SendPackOptions, ): Promise { - const transferId = `pack-${++packCounter}`; + const transferId = `pack-${crypto.randomUUID()}`; // For the agent-state flow the destination agent and the source repo // are the same entity, so `repoId.id === agentAddress`. Asset packs // override this with the SOURCE asset's id so audit can correlate @@ -3062,6 +4718,7 @@ export function createSidecarRouter( conn.send({ type: "repo.pack.push", agentAddress, + generation, repoId, transferId, seq: chunk.seq, @@ -3073,6 +4730,7 @@ export function createSidecarRouter( conn.send({ type: "repo.pack.done", agentAddress, + generation, repoId, transferId, ref, @@ -3091,20 +4749,32 @@ export function createSidecarRouter( options?: SendPackOptions, ): Promise { const { ws, conn } = await getAllocatedConnection(target, "routing"); - if (addressIndex.get(agentAddress) !== ws) { + if ( + addressIndex.get(agentAddress) !== ws || + conn.workflowAddresses.get(agentAddress) !== target.allocationId + ) { throw new Error( `Address ${agentAddress} is not routed on allocation ${target.allocationId}`, ); } - return sendPackOnConnection( - ws, - conn, - agentAddress, - pack, - ref, - commitSha, - options, - ); + let transfer: Promise | undefined; + await withWorkflowWorkAdmission(ws, conn, agentAddress, () => { + transfer = sendPackOnConnection( + ws, + conn, + agentAddress, + target.generation, + pack, + ref, + commitSha, + options, + ); + // The acknowledgement can reject before admission's transaction ends. + void transfer.catch(() => undefined); + return true; + }); + if (transfer === undefined) throw new Error("Workflow pack was not sent"); + return transfer; } async function sendWorkflowRunPackToAllocation( @@ -3116,27 +4786,48 @@ export function createSidecarRouter( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const { ws, conn } = await getAllocatedConnection(target, "routing"); + const { ws, conn, binding } = await getAllocatedConnection( + target, + "routing", + ); signal?.throwIfAborted(); - if (agentAddress !== conn.identity.workflowRunAddress) { + if (agentAddress !== binding.workflowRunAddress) { throw new Error( `Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`, ); } - if (conn.workflowAddresses.has(agentAddress)) { - throw new Error( - `Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`, - ); - } - return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, { - repoId: { - kind: "workflow-run", - id: deriveWorkflowRunRepoId(agentAddress), + let transfer: Promise | undefined; + await withAllocationWorkAdmission( + ws, + binding, + () => { + if (conn.workflowAddresses.has(agentAddress)) { + throw new Error( + `Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`, + ); + } + transfer = sendPackOnConnection( + ws, + conn, + agentAddress, + target.generation, + pack, + ref, + commitSha, + { repoId: workflowRunRepoIdForAddress(agentAddress) }, + ); + // The acknowledgement can reject before admission's transaction ends. + void transfer.catch(() => undefined); + return true; }, - }); + signal, + ); + if (transfer === undefined) + throw new Error("Workflow restore was not sent"); + return transfer; } - function routeMail( + async function routeMail( agentAddress: string, rawMessage: string, authenticatedSender: string, @@ -3146,7 +4837,7 @@ export function createSidecarRouter( stepGrants: RunGrantsFrame["stepGrants"]; senderIdentities?: RunGrantsFrame["senderIdentities"]; }, - ): boolean { + ): Promise { // `authenticatedSender` is hub-assigned by the caller from a hub-verified // value (the ownership-gated sender of a relayed mail, or the triggering // principal's address) -- never the message's own MIME `From`. It rides @@ -3162,17 +4853,37 @@ export function createSidecarRouter( // Optional: the workflow-trigger and session-conversation callers supply // it (they participate in the ack/retry handshake); a caller without a // hub-minted id omits it and the delivery is not tracked for redelivery. - const frame: HubFrame = { + const routed = routedIncarnation(agentAddress); + if (routed === undefined) return false; + const { ws, conn, binding } = routed; + const frame: MailInboundFrame = { type: "mail.inbound", agentAddress, + generation: binding.generation, rawMessage, authenticatedSender, ...(messageId !== undefined ? { messageId } : {}), }; - const ws = addressIndex.get(agentAddress); - if (ws !== undefined) { - const conn = connections.get(ws); - if (conn !== undefined) { + const grantsFrame: HubFrame | undefined = + runGrants === undefined + ? undefined + : { + type: "run.grants", + agentAddress, + generation: binding.generation, + runId: runGrants.runId, + stepGrants: runGrants.stepGrants, + ...(runGrants.senderIdentities !== undefined + ? { senderIdentities: runGrants.senderIdentities } + : {}), + }; + const target = { + allocationId: binding.allocationId, + generation: binding.generation, + }; + try { + return await withWorkflowWorkAdmission(ws, conn, agentAddress, () => { + if (grantsFrame !== undefined) conn.send(grantsFrame); conn.send(frame); // Track the delivery for redelivery until the sidecar acks its durable // inbox write. Only mail carrying a hub-minted messageId participates @@ -3181,14 +4892,37 @@ export function createSidecarRouter( // run carries the run's grants so redelivery can replay them ahead of // the mail. if (messageId !== undefined) { - trackPendingMail(agentAddress, messageId, frame, runGrants); + trackPendingMail( + agentAddress, + messageId, + frame, + target, + false, + runGrants, + ); } return true; - } + }); + } catch (error) { + if (error instanceof WorkflowRunNotExecutableError) throw error; + logger.warn`Mail admission failed for ${agentAddress}: ${error instanceof Error ? error.message : String(error)}`; + if (messageId === undefined) return false; + // Admission may have awaited a disconnect and missed handleClose's + // pending-mail retention. Record the mail now, then either retry on + // the current owner or retain it for the next verified reconnect. + trackPendingMail( + agentAddress, + messageId, + frame, + target, + false, + runGrants, + ); + const current = addressIndex.get(agentAddress); + if (current === undefined || !connections.has(current)) + retainPendingMailForAddress(agentAddress); + return true; } - - // If the agent recently disconnected, queue for delivery on reconnect. - return enqueueForDisconnected(agentAddress, frame); } function sendRunGrants( @@ -3197,33 +4931,17 @@ export function createSidecarRouter( stepGrants: RunGrantsFrame["stepGrants"], senderIdentities: RunGrantsFrame["senderIdentities"], ): boolean { - const frame: HubFrame = { + const routed = routedIncarnation(agentAddress); + if (routed === undefined) return false; + routed.conn.send({ type: "run.grants", agentAddress, + generation: routed.binding.generation, runId, stepGrants, ...(senderIdentities !== undefined ? { senderIdentities } : {}), - }; - - const ws = addressIndex.get(agentAddress); - if (ws !== undefined) { - const conn = connections.get(ws); - if (conn !== undefined) { - conn.send(frame); - return true; - } - } - - // Mirror routeMail: if the address has a live disconnect queue, ride it - // so a run.grants issued in the window between deploy and the first - // reconnect survives the same way the dispatching trigger mail does. - // A queue exists only while the deployment address is still on - // agentAddresses (pre-first-reconnect); after an authenticated reconnect it - // moves to workflowAddresses, which handleClose leaves unqueued because - // that generation's in-flight run state is reconstructed sidecar-locally. - // Returning without enqueueing there is correct; enqueueing is what keeps - // grants and mail from diverging in the pre-reconnect window. - return enqueueForDisconnected(agentAddress, frame); + }); + return true; } async function sendWorkflowRunDispatchToAllocation( @@ -3239,7 +4957,10 @@ export function createSidecarRouter( signal?.throwIfAborted(); const { ws, conn } = await getAllocatedConnection(target, "routing"); signal?.throwIfAborted(); - if (addressIndex.get(agentAddress) !== ws) { + if ( + addressIndex.get(agentAddress) !== ws || + conn.workflowAddresses.get(agentAddress) !== target.allocationId + ) { throw new Error( `Address ${agentAddress} is not routed on allocation ${target.allocationId}`, ); @@ -3279,42 +5000,86 @@ export function createSidecarRouter( stepGrants, ...(senderIdentities !== undefined ? { senderIdentities } : {}), }; - conn.send({ - type: "run.grants", - agentAddress, - runId, - stepGrants, - ...(senderIdentities !== undefined ? { senderIdentities } : {}), - }); - const frame: HubFrame = { + const frame: MailInboundFrame = { type: "mail.inbound", agentAddress, + generation: target.generation, rawMessage, authenticatedSender, messageId, }; - conn.send(frame); - trackPendingMail(agentAddress, messageId, frame, runGrants, target); + await withWorkflowWorkAdmission( + ws, + conn, + agentAddress, + () => { + conn.send({ + type: "run.grants", + agentAddress, + generation: target.generation, + ...runGrants, + }); + conn.send(frame); + trackPendingMail( + agentAddress, + messageId, + frame, + target, + true, + runGrants, + ); + return true; + }, + signal, + ); + } + + // The deploy in flight for a reply's address on this connection, when the + // reply answers it: the one with the reply's request id and generation. + function deployAnsweredBy( + ws: WsHandle, + frame: AgentDeployAckFrame | AgentDeployErrorFrame, + ): + | PendingEntry + | undefined { + const req = pendingDeploys.get(frame.agentAddress); + return req?.ws === ws && + req.meta.requestId === frame.requestId && + req.meta.generation === frame.generation + ? req + : undefined; } async function handleDeployAck( ws: WsHandle, frame: AgentDeployAckFrame, ): Promise { - const req = pendingDeploys.get(frame.agentAddress); + connections.get(ws)?.unansweredDeploys.delete(frame.requestId); + const req = deployAnsweredBy(ws, frame); if (req === undefined) { - logger.warn`Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`; + logger.warn`Ignoring agent.deploy.ack ${frame.requestId} for ${frame.agentAddress} generation ${String(frame.generation)}: it answers no deploy in flight on this connection`; return; } - if (req.ws !== ws) return; + // The listeners below are awaited, and the deploy this ack answers can be + // settled meanwhile (a timeout, the allocation leaving) and the address + // deployed again. Only the deploy the ack answers may be settled by it. + const stillAnswered = (): boolean => { + if (pendingDeploys.get(frame.agentAddress) === req) return true; + logger.warn`Dropping agent.deploy.ack ${frame.requestId} for ${frame.agentAddress} generation ${String(frame.generation)}: its deploy was settled while the ack's listeners ran`; + return false; + }; if (events.listenerCount("agent.deploy.ack") > 0) { try { - const identity = connections.get(ws)?.identity; + const conn = connections.get(ws); + const identity = + conn === undefined + ? undefined + : owningAllocation(conn, frame.agentAddress); await events.emitAndAwait("agent.deploy.ack", { agentAddress: frame.agentAddress, publicKey: frame.publicKey, - ...(identity?.kind === "allocated" + ...(identity !== undefined ? { allocated: { allocationId: identity.allocationId, @@ -3325,31 +5090,39 @@ export function createSidecarRouter( : {}), }); } catch (err) { + if (!stillAnswered()) return; pendingDeploys.reject( frame.agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`, ); return; } + if (!stillAnswered()) return; } pendingDeploys.resolve(frame.agentAddress, frame.publicKey); } function rejectDeployPendingFromFrame( ws: WsHandle, - agentAddress: string, - error: string, + frame: AgentDeployErrorFrame, ): void { - const req = pendingDeploys.get(agentAddress); - if (req === undefined || req.ws !== ws) return; - // Settle by key, not by the `req` object: a key lookup observes the - // CURRENT entry, so a stale handle cannot settle a replaced round-trip. - pendingDeploys.reject(agentAddress, error); + connections.get(ws)?.unansweredDeploys.delete(frame.requestId); + if (deployAnsweredBy(ws, frame) === undefined) { + logger.warn`Ignoring agent.deploy.error ${frame.requestId} for ${frame.agentAddress} generation ${String(frame.generation)}: it answers no deploy in flight on this connection: ${frame.error.code}: ${frame.error.message}`; + return; + } + pendingDeploys.reject( + frame.agentAddress, + frame.error.code === "capacity_full" + ? new DeploymentRejectedError(frame.error.code, frame.error.message) + : frame.error.message, + ); } function sendAgentDeployOnConnection( ws: WsHandle, conn: SidecarConnection, + target: AllocatedSidecarTarget, agentAddress: string, harnessConfig: HarnessConfig, workflow?: AgentDeployFrame["workflow"], @@ -3368,75 +5141,64 @@ export function createSidecarRouter( ); } - const addressSet = - conn.identity.kind === "allocated" - ? conn.workflowAddresses - : conn.agentAddresses; - addressSet.add(agentAddress); + conn.workflowAddresses.set(agentAddress, target.allocationId); addressIndex.set(agentAddress, ws); - return new Promise<{ publicKey: string }>((resolve, reject) => { - // Timeout and frame-error rejections share this closure, so the routing - // rollback and the `frameSent: true` tag live in one place. - pendingDeploys.register( - agentAddress, - ws, - { - timeoutMs: requestTimeoutMs, - timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, - resolve(publicKey) { - resolve({ publicKey }); - }, - reject(error: string) { - if (addressIndex.get(agentAddress) === ws) { - addressSet.delete(agentAddress); - addressIndex.delete(agentAddress); - } - // A non-allocated deployment's key is recorded by the deploy-ack - // projection, whose failure (reject/timeout/agent.error/disconnect) - // is observed only here. Drain any pre-ack sender mail parked on - // this address so it surfaces as undelivered rather than waiting out - // the TTL. An allocated deployment's failure is drained by its - // session-service owner instead, so skip it here to keep one owner - // per case. - if (conn.identity.kind !== "allocated") { - drainDeferredSenderMail(agentAddress, `deploy failed: ${error}`); - } - reject(deployFrameFailure(error, true)); - }, + const requestId = nextRequestId(); + const response = Promise.withResolvers<{ publicKey: string }>(); + // Every failure removes the provisional route. Only an explicit rejection + // proves the sidecar did not start; other replies and timeouts stay uncertain. + pendingDeploys.register( + agentAddress, + ws, + { + timeoutMs: requestTimeoutMs, + timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, + resolve(publicKey) { + response.resolve({ publicKey }); }, - undefined, - ); + reject(error) { + if (addressIndex.get(agentAddress) === ws) { + conn.workflowAddresses.delete(agentAddress); + addressIndex.delete(agentAddress); + } + // The deployment's owner settles any pre-ack sender mail parked on + // this address once it knows the outcome. + response.reject( + typeof error === "string" ? deployFrameFailure(error, true) : error, + ); + }, + }, + { requestId, generation: target.generation }, + ); - try { - conn.send({ - type: "agent.deploy", - agentAddress, - agentId: harnessConfig.agentId, - config: harnessConfig, - hubPublicKey: hubPublicKeyHex, - ...(workflow !== undefined ? { workflow } : {}), - }); - } catch (err) { - // A synchronous send failure means the frame never reached the wire. - // Drop the pending entry (and its armed timer) and reject as not-sent - // so a caller may safely roll back what it staged. The drop bypasses - // the entry's reject closure: this failure must report - // `frameSent: false`, and the timer must not fire later and - // double-reject. - pendingDeploys.delete(agentAddress); - if (addressIndex.get(agentAddress) === ws) { - addressSet.delete(agentAddress); - addressIndex.delete(agentAddress); - } - reject( - deployFrameFailure( - `Deploy of "${agentAddress}" failed to send: ${err instanceof Error ? err.message : String(err)}`, - false, - ), - ); + try { + conn.send({ + type: "agent.deploy", + requestId, + agentAddress, + generation: target.generation, + agentId: harnessConfig.agentId, + config: harnessConfig, + hubPublicKey: hubPublicKeyHex, + ...(workflow !== undefined ? { workflow } : {}), + }); + conn.unansweredDeploys.set(requestId, agentAddress); + } catch (cause) { + // Throw synchronously on a proven-unsent frame. Returning the response + // promise below is the caller's evidence that the send took place. + pendingDeploys.delete(agentAddress); + if (addressIndex.get(agentAddress) === ws) { + conn.workflowAddresses.delete(agentAddress); + addressIndex.delete(agentAddress); } - }); + throw deployFrameFailure( + `Deploy of "${agentAddress}" failed to send: ${cause instanceof Error ? cause.message : String(cause)}`, + false, + cause, + ); + } + return response.promise; } async function sendAgentDeployToAllocation( @@ -3447,11 +5209,15 @@ export function createSidecarRouter( signal?: AbortSignal, beforeSend?: () => Promise, ): Promise<{ publicKey: string }> { + let response: Promise<{ publicKey: string }> | undefined; try { signal?.throwIfAborted(); - const { ws, conn } = await getAllocatedConnection(target, "routing"); + const { ws, conn, binding } = await getAllocatedConnection( + target, + "routing", + ); signal?.throwIfAborted(); - if (agentAddress !== conn.identity.workflowRunAddress) { + if (agentAddress !== binding.workflowRunAddress) { throw new Error( `Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`, ); @@ -3464,35 +5230,46 @@ export function createSidecarRouter( } if (hubPublicKeyHex === undefined) throw new Error("Hub signing key is required for agent deployment"); - if (pendingDeploys.has(agentAddress)) - throw new Error( - `Deploy already in progress for agent "${agentAddress}"`, - ); - await beforeSend?.(); - signal?.throwIfAborted(); - if ( - allocatedConnections.get(target.allocationId)?.ws !== ws || - allocationFences.get(target.allocationId) !== target.generation - ) { + if (pendingDeploys.has(agentAddress)) throw new Error( - `Allocated sidecar connection changed for allocation ${target.allocationId}`, + `Deploy already in progress for agent "${agentAddress}"`, ); - } - // Return without awaiting: only pre-send failures belong to this catch. - return sendAgentDeployOnConnection( + // The sidecar may still be tearing down a copy of this address it was + // told to undeploy, and anything that copy sends names the same address. + // Routing the address back before the answer would credit it all here. + await undeployAnswered(conn, agentAddress, signal); + signal?.throwIfAborted(); + await beforeSend?.(); + await withAllocationWorkAdmission( ws, - conn, - agentAddress, - harnessConfig, - workflow, + binding, + () => { + response = sendAgentDeployOnConnection( + ws, + conn, + target, + agentAddress, + harnessConfig, + workflow, + ); + // Own an early ack rejection while the admission transaction ends. + void response.catch(() => undefined); + return true; + }, + signal, ); + if (response === undefined) + throw new Error("Workflow deploy was not sent"); } catch (cause) { + // A failed transaction response cannot roll back a frame already sent. throw deployFrameFailure( cause instanceof Error ? cause.message : String(cause), - false, + response !== undefined, cause, ); } + // Never hold the lifecycle locks while waiting for the worker's reply. + return response; } /** @@ -3504,14 +5281,15 @@ export function createSidecarRouter( * step is provisioned, spawns the child. * * The step address must already be bound via `bindStepRoute`, which - * resolves and records the sidecar; this reuses that route rather than - * touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack` + * resolves and records the sidecar; this reuses that route. Waits for the + * sidecar's `agent.deploy.ack` * so the caller can safely deliver the deploy pack afterward. On failure * the caller owns tearing the route down via `unbindStepRoute`. */ function sendProvisionStepOnConnection( ws: WsHandle, conn: SidecarConnection, + generation: number, agentAddress: string, harnessConfig: HarnessConfig, ): Promise { @@ -3523,6 +5301,7 @@ export function createSidecarRouter( } const hubKey = hubPublicKeyHex; + const requestId = nextRequestId(); return new Promise((resolve, reject) => { // The sidecar's `agent.deploy.ack` resolves this through // `pendingDeploys.resolve`. The per-step address is workflow-derived @@ -3537,21 +5316,24 @@ export function createSidecarRouter( resolve(_publicKey) { resolve(); }, - reject(error: string) { - reject(new Error(error)); + reject(error) { + reject(typeof error === "string" ? new Error(error) : error); }, }, - undefined, + { requestId, generation }, ); conn.send({ type: "agent.deploy", + requestId, agentAddress, + generation, agentId: harnessConfig.agentId, config: harnessConfig, hubPublicKey: hubKey, provisionStep: true, }); + conn.unansweredDeploys.set(requestId, agentAddress); }); } @@ -3561,17 +5343,36 @@ export function createSidecarRouter( harnessConfig: HarnessConfig, ): Promise { const { ws, conn } = await getAllocatedConnection(target, "routing"); - if (addressIndex.get(agentAddress) !== ws) { + if ( + addressIndex.get(agentAddress) !== ws || + conn.workflowAddresses.get(agentAddress) !== target.allocationId + ) { throw new Error( `Step route ${agentAddress} is not bound to allocation ${target.allocationId}`, ); } - return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig); + let provisioned: Promise | undefined; + await withWorkflowWorkAdmission(ws, conn, agentAddress, () => { + provisioned = sendProvisionStepOnConnection( + ws, + conn, + target.generation, + agentAddress, + harnessConfig, + ); + // The acknowledgement can reject before admission's transaction ends. + void provisioned.catch(() => undefined); + return true; + }); + if (provisioned === undefined) + throw new Error("Workflow step was not provisioned"); + return provisioned; } function sendProbeOnConnection( ws: WsHandle, conn: SidecarConnection, + probeId: string, args: SendProbeArgs, ): Promise { const requestId = nextRequestId(); @@ -3588,7 +5389,7 @@ export function createSidecarRouter( reject(new Error(error)); }, }, - undefined, + probeId, ); conn.send({ @@ -3607,184 +5408,555 @@ export function createSidecarRouter( args: SendProbeArgs, ): Promise { const { ws, conn } = await getProvisionedConnection(target, "routing"); - return sendProbeOnConnection(ws, conn, args); + return sendProbeOnConnection(ws, conn, target.allocationId, args); } - function settleOAuthLoginError( - requestId: string, - entry: PendingOAuthLogin, - message: string, - ): void { - entry.resolveFinal({ status: "error", message }); - if (!entry.requestSettled) { - entry.requestSettled = true; - entry.resolveRequest({ status: "error", message }); + function detachAllocation(target: AllocatedSidecarTarget): void { + const current = allocatedConnections.get(target.allocationId); + if ( + current === undefined || + current.identity.generation !== target.generation + ) { + return; } + detachBinding(current.ws, target.allocationId, "It was released", { + undeploy: true, + }); } - function settleOAuthLogin( - requestId: string, - outcome: OAuthLoginResultFrame["outcome"], - ): void { - const entry = pendingOAuthLogins.get(requestId); - if (entry === undefined) { - logger.debug`Received oauth.login.result for uncorrelated ${requestId}`; - return; - } - if (outcome.status === "started") { - if (entry.requestSettled) return; - entry.requestSettled = true; - entry.resolveRequest({ - status: "started", - requestId, - authorizeUrl: outcome.authorizeUrl, - completed: entry.completed, - }); + async function handleWorkflowControlAck( + ws: WsHandle, + frame: WorkflowControlAckFrame, + ): Promise { + const entry = pendingWorkflowControls.get(frame.requestId); + if (entry === undefined || entry.ws !== ws) return; + const fail = (error: Error) => { + pendingWorkflowControls.delete(frame.requestId); + entry.meta.fail(error); + }; + try { + const current = await getAllocatedConnection( + entry.meta, + entry.meta.terminal ? "retention-transition" : "routing", + ); + if (current.ws !== ws) return; + } catch (cause) { + // A failed lookup leaves the stop unknown, not refused. The caller + // retries, and the worker acknowledges a repeated stop. + fail( + cause instanceof SidecarIdentityValidationError + ? cause + : new WorkflowControlUnreachableError( + "Workflow control allocation changed", + cause, + ), + ); return; } - clearTimeout(entry.timer); - pendingOAuthLogins.delete(requestId); - if (outcome.status === "completed") { - if (!entry.requestSettled) { - // A terminal frame arrived before the started acknowledgement — - // settle the request promise so the caller never hangs. This - // ordering drops the exchanged tokens: nothing consumes the - // completed promise without the started arm, so the person has - // to run the login again. - entry.requestSettled = true; - entry.resolveRequest({ - status: "error", - message: "the login completed before it started", + if (pendingWorkflowControls.get(frame.requestId) !== entry) return; + if (frame.error !== undefined) + fail( + frame.error === WORKFLOW_CONTROL_INITIALIZING_ERROR + ? new WorkflowControlInitializingError() + : new WorkflowControlRejectedError(frame.error), + ); + else { + if (entry.meta.action === "retain") { + if (frame.retention === undefined) { + fail(new WorkflowControlRejectedError("Retention was not confirmed")); + return; + } + // A durable kept decision frees active capacity independently of the + // earlier history receives that may still be settling on the Hub. + const retained = connectionRetention.get(ws) ?? new Map(); + retained.set(entry.meta.agentAddress, { + generation: entry.meta.generation, + retention: frame.retention, }); + connectionRetention.set(ws, retained); + removeRoute(ws, entry.meta.agentAddress); + } + if (entry.meta.action === "stop" || entry.meta.action === "retain") { + // A live stop waits for the worker's tips. Terminal controls follow the + // admission cutoff instead: later history cannot change an ended run. + // Both wait for earlier accepted writes before fencing further packs. + let unreceived: string | null = null; + let unreceivedCause: unknown; + if ( + stoppedAllocations.get(entry.meta.allocationId) !== + entry.meta.generation + ) { + if (entry.meta.terminal) { + unreceived = "the Hub has not finalized the terminal run's history"; + if (lookups.isWorkflowRunHistoryFinal !== undefined) { + try { + if ( + await lookups.isWorkflowRunHistoryFinal({ + kind: "allocated", + allocationId: entry.meta.allocationId, + anchorRunId: entry.meta.runId, + agentAddress: entry.meta.agentAddress, + generation: entry.meta.generation, + }) + ) + unreceived = null; + } catch (cause) { + unreceived = `the Hub cannot confirm final history: ${cause instanceof Error ? cause.message : String(cause)}`; + unreceivedCause = cause; + } + } + } else { + unreceived = await findUnreceivedWorkflowHistory( + entry.meta.agentAddress, + frame.refTips, + ); + } + } + // History lookup can outlive the request, connection, or release fence. + if (pendingWorkflowControls.get(frame.requestId) !== entry) return; + const current = allocatedConnections.get(entry.meta.allocationId); + if ( + current?.ws !== ws || + current.identity.generation !== entry.meta.generation || + !isFencedAsCurrent(current.identity) + ) { + fail( + new WorkflowControlUnreachableError( + "Workflow control allocation changed", + ), + ); + return; + } + if (unreceived !== null) { + fail( + new WorkflowControlHistoryPendingError( + unreceived, + unreceivedCause === undefined + ? undefined + : { cause: unreceivedCause }, + ), + ); + return; + } + stoppedAllocations.set(entry.meta.allocationId, entry.meta.generation); + removeRoute(ws, entry.meta.agentAddress); } - entry.resolveFinal({ status: "completed", tokens: outcome.tokens }); + pendingWorkflowControls.resolve( + frame.requestId, + entry.meta.action === "retain" ? frame.retention : undefined, + ); + } + } + + // A stop keeps its first error and when it was first reported, whichever + // connection or frame carries it later, so a reconnect cannot restart the + // wait for its history. The tips are the latest the sidecar reported. + function noteReportedStop( + binding: Extract, + error: string, + refTips?: WorkflowRunRefTips, + ): void { + const previous = reportedStops.get(binding.allocationId); + const first = + previous?.generation === binding.generation ? previous : undefined; + const tips = refTips ?? first?.refTips; + reportedStops.set(binding.allocationId, { + generation: binding.generation, + address: binding.workflowRunAddress, + error: first?.error ?? error, + reportedAt: first?.reportedAt ?? now(), + ...(tips !== undefined ? { refTips: tips } : {}), + }); + } + + // A deployment stopped though the Hub did not stop it. A current + // generation's stays unrouted, as a stopped one a hello reports does, and + // the reconciler fails it once the Hub holds the history the stopped copy + // reported, or once it stops waiting for it. Cleanup bindings leave removal + // to the reconciler; any other stopped incarnation is undeployed. + function handleDeploymentStopped( + ws: WsHandle, + frame: DeploymentStoppedFrame, + ): void { + const conn = connections.get(ws); + if (conn === undefined) return; + // Cleanup bindings retain access for the reconciler's acknowledged removal. + if ( + [...conn.bindings.values()].some( + (binding) => + binding.kind === "cleanup" && + binding.workflowRunAddress === frame.agentAddress && + binding.generation >= frame.generation && + isFencedAsCurrent(binding), + ) + ) + return; + const binding = deploymentBinding(conn, frame.agentAddress); + if ( + binding?.generation !== frame.generation || + !isFencedAsCurrent(binding) + ) { + sendUndeploy( + conn, + { address: frame.agentAddress, generation: frame.generation }, + "The Hub does not keep this incarnation on this sidecar", + { waitable: binding !== undefined }, + ); return; } - settleOAuthLoginError(requestId, entry, outcome.message); + removeRoute(ws, frame.agentAddress); + noteReportedStop(binding, frame.error, frame.refTips); + events.emit("deployment.stopped", { + allocationId: binding.allocationId, + generation: binding.generation, + }); } - async function requestOAuthLogin(args: { - connectorId: "codex" | "xai-oauth"; - }): Promise { - const gate = config.oauthLogin; - let target: { ws: WsHandle; conn: SidecarConnection } | undefined; - for (const [ws, conn] of connections) { - if (gate !== undefined && (await gate.isLocalSidecar(conn.identity))) { - target = { ws, conn }; - break; - } + // Describes the first ref whose reported tip the Hub does not hold, or + // returns null when the Hub holds the worker's whole history. + async function findUnreceivedWorkflowHistory( + agentAddress: string, + reported: WorkflowRunRefTips | undefined, + ): Promise { + if (reported === undefined) return "the worker did not report its ref tips"; + if (lookups.readWorkflowRunRefTips === undefined) + return "the Hub cannot read workflow history"; + let received: WorkflowRunRefTips; + try { + received = await lookups.readWorkflowRunRefTips(agentAddress); + } catch (cause) { + return `the Hub cannot read workflow history: ${cause instanceof Error ? cause.message : String(cause)}`; } - if (target === undefined || gate === undefined) { - return { - status: "gate", - message: - "Connecting this provider requires a local sidecar — its login opens a pinned loopback port (codex: localhost:1455, xai-oauth: 127.0.0.1:1456) on the machine you are browsing from, and no connected sidecar is local to this hub.", - }; + for (const [ref, tip] of Object.entries(received)) { + const reportedTip = reported[ref]; + if (reportedTip !== tip) + return `${ref} is at ${tip ?? "no commit"} on the Hub and ${reportedTip ?? "no commit"} on the worker`; } - const targetWs = target.ws; - const targetConn = target.conn; + return null; + } - const timeoutMs = gate.timeoutMs ?? DEFAULT_OAUTH_LOGIN_TIMEOUT_MS; - const requestId = nextRequestId(); - return new Promise((resolveRequest) => { - let resolveCompleted!: (outcome: OAuthLoginFinalOutcome) => void; - const completed = new Promise((resolve) => { - resolveCompleted = resolve; - }); - const entry: PendingOAuthLogin = { - ws: targetWs, - conn: targetConn, - requestSettled: false, - resolveRequest, - resolveFinal: resolveCompleted, - completed, - timer: setTimeout(() => { - pendingOAuthLogins.delete(requestId); - // Tell the sidecar to tear its callback listener down: the hub - // gave up, and an abandoned pinned-port bind would make every - // retry of this connector fail to bind until the stale login - // completes or the sidecar restarts. - targetConn.send({ type: "oauth.login.cancel", requestId }); - settleOAuthLoginError( - requestId, - entry, - `the login timed out after ${String(timeoutMs)}ms`, - ); - }, timeoutMs), - }; - pendingOAuthLogins.set(requestId, entry); - targetConn.send({ - type: "oauth.login.start", - requestId, - connectorId: args.connectorId, - }); + async function sendWorkflowControl( + target: AllocatedSidecarTarget, + command: WorkflowControlCommand, + timeoutMs: number, + ): Promise { + await requestWorkflowControl(target, command, timeoutMs); + } + + async function retainAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + ): Promise { + const connection = await getAllocatedConnection(target, "retention"); + return retainOnConnection(connection, target, timeoutMs); + } + + async function retainOnConnection( + connection: Awaited>, + target: AllocatedSidecarTarget, + timeoutMs: number, + ): Promise { + const { ws, binding } = connection; + const confirmed = getAllocationRetention(target); + if ( + confirmed !== undefined && + stoppedAllocations.get(target.allocationId) === target.generation + ) + return confirmed; + const held = retainingAllocations.get(target.allocationId); + if (held?.ws === ws && held.generation === target.generation) + return held.result; + const result = sendWorkflowControlOnConnection( + connection, + target, + { + runId: binding.anchorRunId, + agentAddress: binding.workflowRunAddress, + action: "retain", + reason: "The deployment ended; retain its stopped copy", + }, + timeoutMs, + { terminal: true }, + ).then((retention) => { + if (retention === undefined) + throw new WorkflowControlRejectedError("Retention was not confirmed"); + return retention; }); + const request = { ws, generation: target.generation, result }; + retainingAllocations.set(target.allocationId, request); + try { + return await result; + } finally { + if (retainingAllocations.get(target.allocationId) === request) + retainingAllocations.delete(target.allocationId); + } } - function disconnectAllocation(target: AllocatedSidecarTarget): void { - const current = allocatedConnections.get(target.allocationId); + async function requestWorkflowControl( + target: AllocatedSidecarTarget, + command: WorkflowControlCommand, + timeoutMs: number, + ): Promise { + let connection: Awaited>; + try { + connection = await getAllocatedConnection(target, "routing"); + } catch (cause) { + if (cause instanceof SidecarIdentityValidationError) throw cause; + throw new WorkflowControlUnreachableError( + cause instanceof Error ? cause.message : String(cause), + cause, + ); + } + return sendWorkflowControlOnConnection( + connection, + target, + command, + timeoutMs, + { terminal: false }, + ); + } + + function sendWorkflowControlOnConnection( + connection: Awaited>, + target: AllocatedSidecarTarget, + command: Omit, + timeoutMs: number, + { terminal }: { terminal: boolean }, + ): Promise { + const { ws, conn, binding } = connection; if ( - current === undefined || - current.identity.generation !== target.generation + command.agentAddress !== binding.workflowRunAddress || + command.runId !== binding.anchorRunId ) { - return; + throw new Error( + "Workflow control does not target the allocation's anchor run", + ); } - handleClose(current.ws); - current.ws.close(); + const requestId = nextRequestId(); + const timeoutMessage = `Workflow control ${requestId} timed out`; + const unconfirmedMessage = `Workflow control ${requestId} was acknowledged but not processed in time`; + return new Promise((resolve, reject) => { + let cancelProcessing: (() => void) | undefined; + const settle = () => { + cancelProcessing?.(); + }; + pendingWorkflowControls.register( + requestId, + ws, + { + timeoutMs, + timeoutMessage, + resolve: (retention) => { + settle(); + resolve(retention); + }, + // The tracker reports timeouts, disconnect sweeps, send failures, + // and the processing limit as strings. Only a timeout shows the live + // connection stayed silent. + reject: (error) => { + settle(); + reject( + error === timeoutMessage + ? [...conn.unansweredDeploys.values()].includes( + command.agentAddress, + ) + ? new WorkflowControlInitializingError() + : new WorkflowControlTimeoutError(error) + : error === unconfirmedMessage + ? new WorkflowControlUnconfirmedError(error) + : new WorkflowControlUnreachableError(error), + ); + }, + }, + { + ...target, + agentAddress: command.agentAddress, + runId: command.runId, + action: command.action, + terminal, + fail: (error) => { + settle(); + reject(error); + }, + received: () => { + const entry = pendingWorkflowControls.get(requestId); + if (entry === undefined || cancelProcessing !== undefined) return; + entry.cancelTimeout(); + cancelProcessing = scheduleTimeout(() => { + pendingWorkflowControls.reject(requestId, unconfirmedMessage); + }, timeoutMs); + }, + }, + ); + try { + conn.send({ + type: "workflow.control", + requestId, + generation: target.generation, + ...command, + }); + } catch (error) { + pendingWorkflowControls.reject( + requestId, + error instanceof Error ? error.message : String(error), + ); + } + }); } function sendAgentUndeploy( agentAddress: string, reason: string, ): Promise { - const ws = addressIndex.get(agentAddress); - if (ws === undefined) { + const routed = routedIncarnation(agentAddress); + if (routed === undefined) { return Promise.reject( new Error(`No sidecar connected for agent "${agentAddress}"`), ); } - const conn = connections.get(ws); - if (conn === undefined) { - return Promise.reject( - new Error(`No sidecar connected for agent "${agentAddress}"`), + const { ws, conn, binding } = routed; + return requestUndeploy({ + ws, + conn, + agentAddress, + generation: binding.generation, + reason, + timeoutMs: requestTimeoutMs, + }); + } + + function getCleanupConnection( + target: AllocatedSidecarTarget, + ): object | undefined { + const current = allocatedConnections.get(target.allocationId); + return allocationFences.get(target.allocationId)?.generation === + target.generation && + current?.identity.kind === "cleanup" && + current.identity.generation === target.generation + ? current.ws + : undefined; + } + + async function undeployAllocation( + target: AllocatedSidecarTarget, + timeoutMs: number, + signal: AbortSignal, + expectedConnection: object, + onSent: () => void, + ): Promise { + signal.throwIfAborted(); + const { ws, conn, binding } = await getProvisionedConnection( + target, + "cleanup", + ); + if (binding.kind !== "cleanup") { + throw new Error( + `Allocation ${target.allocationId} is not being cleaned up`, ); } + if (ws !== expectedConnection) { + throw new Error("Cleanup connection changed before the request was sent"); + } + await requestUndeploy({ + ws, + conn, + agentAddress: binding.workflowRunAddress, + generation: binding.generation, + reason: "The allocation is being released", + timeoutMs, + target, + signal, + onSent, + }); + } + function requestUndeploy(args: { + ws: WsHandle; + conn: SidecarConnection; + agentAddress: string; + generation: number; + reason: string; + timeoutMs: number; + signal?: AbortSignal; + target?: AllocatedSidecarTarget; + onSent?: () => void; + }): Promise { + const { + ws, + conn, + agentAddress, + generation, + reason, + timeoutMs, + signal, + target, + } = args; + signal?.throwIfAborted(); + if (pendingUndeploys.has(agentAddress)) { + return Promise.reject( + new Error(`Undeploy of ${agentAddress} is already pending`), + ); + } + const requestId = nextRequestId(); return new Promise((resolve, reject) => { - // Timeout, ack, and error rejection share one closure so the routing - // teardown runs exactly once no matter how the round-trip settles. + const finish = () => { + signal?.removeEventListener("abort", onAbort); + if (target === undefined) removeRoute(ws, agentAddress); + }; + const onAbort = () => { + if (pendingUndeploys.get(agentAddress)?.meta.requestId !== requestId) + return; + pendingUndeploys.delete(agentAddress); + finish(); + reject( + signal?.reason instanceof Error + ? signal.reason + : new Error("Cleanup aborted"), + ); + }; pendingUndeploys.register( agentAddress, ws, { - timeoutMs: requestTimeoutMs, - timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`, + timeoutMs, + timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${timeoutMs}ms`, resolve() { - removeAgentAddress(ws, agentAddress); + finish(); resolve(); }, reject(error: string) { - removeAgentAddress(ws, agentAddress); + finish(); reject(new Error(error)); }, }, - undefined, + { requestId, generation, ...(target !== undefined ? { target } : {}) }, ); - - conn.send({ - type: "agent.undeploy", - agentAddress, - reason, - }); + signal?.addEventListener("abort", onAbort, { once: true }); + try { + conn.send({ + type: "agent.undeploy", + requestId, + agentAddress, + generation, + reason, + }); + args.onSent?.(); + noteUndeploying(conn, agentAddress, requestId); + } catch (error) { + pendingUndeploys.reject( + agentAddress, + error instanceof Error ? error.message : String(error), + ); + } }); } - function removeAgentAddress(ws: WsHandle, agentAddress: string): void { - addressIndex.delete(agentAddress); - const conn = connections.get(ws); - if (conn !== undefined) { - conn.agentAddresses.delete(agentAddress); + function removeRoute(ws: WsHandle, agentAddress: string): void { + if (addressIndex.get(agentAddress) === ws) { + addressIndex.delete(agentAddress); + retainPendingMailForAddress(agentAddress); } + connections.get(ws)?.workflowAddresses.delete(agentAddress); } function dispatchToSubscribers(agentAddress: string, event: unknown): void { @@ -3838,10 +6010,11 @@ export function createSidecarRouter( sources: InferenceSource[], defaultSource: string, ): Promise { - await sendRequest(agentAddress, (requestId) => ({ + await sendRequest(agentAddress, (requestId, generation) => ({ type: "sources.update", requestId, agentAddress, + generation, sources, defaultSource, })); @@ -3852,59 +6025,45 @@ export function createSidecarRouter( delivery: CredentialDelivery, revoke?: string[], ): Promise { - await sendRequest(agentAddress, (requestId) => ({ + await sendRequest(agentAddress, (requestId, generation) => ({ type: "credentials.update", requestId, agentAddress, + generation, delivery, ...(revoke !== undefined ? { revoke } : {}), })); } - function sendSyncRequest(agentAddress: string): void { - const ws = addressIndex.get(agentAddress); - if (ws === undefined) { - throw new Error(`No sidecar connected for agent "${agentAddress}"`); - } - const conn = connections.get(ws); - if (conn === undefined) { - throw new Error(`No sidecar connected for agent "${agentAddress}"`); - } - - const transferId = `sync-${++packCounter}`; - conn.send({ - type: "sync.request", - agentAddress, - transferId, - }); - } - - function sendSignalDeliver(opts: { + async function sendSignalDeliver(opts: { agentAddress: string; runId: string; signalName: string; signalId: string; payload: unknown; - }): void { - const ws = addressIndex.get(opts.agentAddress); - if (ws === undefined) { - throw new Error( - `No sidecar connected for deployment "${opts.agentAddress}"`, - ); - } - const conn = connections.get(ws); - if (conn === undefined) { + correlationId?: string; + }): Promise { + const routed = routedIncarnation(opts.agentAddress); + if (routed === undefined) { throw new Error( `No sidecar connected for deployment "${opts.agentAddress}"`, ); } - conn.send({ - type: "signal.deliver", - agentAddress: opts.agentAddress, - runId: opts.runId, - signalName: opts.signalName, - signalId: opts.signalId, - payload: opts.payload, + const { ws, conn, binding } = routed; + await withWorkflowWorkAdmission(ws, conn, opts.agentAddress, () => { + conn.send({ + type: "signal.deliver", + agentAddress: opts.agentAddress, + generation: binding.generation, + runId: opts.runId, + signalName: opts.signalName, + signalId: opts.signalId, + payload: opts.payload, + ...(opts.correlationId !== undefined + ? { correlationId: opts.correlationId } + : {}), + }); + return true; }); } @@ -3916,36 +6075,48 @@ export function createSidecarRouter( signalName: string; signalId: string; payload: unknown; + correlationId?: string; }, signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); const { ws, conn } = await getAllocatedConnection(target, "routing"); signal?.throwIfAborted(); - if (addressIndex.get(opts.agentAddress) !== ws) { + if ( + addressIndex.get(opts.agentAddress) !== ws || + conn.workflowAddresses.get(opts.agentAddress) !== target.allocationId + ) { throw new Error( `Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`, ); } - conn.send({ type: "signal.deliver", ...opts }); + await withWorkflowWorkAdmission( + ws, + conn, + opts.agentAddress, + () => { + conn.send({ + type: "signal.deliver", + ...opts, + generation: target.generation, + }); + return true; + }, + signal, + ); } function sendDrain(opts: { agentAddress: string; deadlineMs: number }): void { - const ws = addressIndex.get(opts.agentAddress); - if (ws === undefined) { - throw new Error( - `No sidecar connected for deployment "${opts.agentAddress}"`, - ); - } - const conn = connections.get(ws); - if (conn === undefined) { + const routed = routedIncarnation(opts.agentAddress); + if (routed === undefined) { throw new Error( `No sidecar connected for deployment "${opts.agentAddress}"`, ); } - conn.send({ + routed.conn.send({ type: "drain.deliver", agentAddress: opts.agentAddress, + generation: routed.binding.generation, deadlineMs: opts.deadlineMs, }); } @@ -3959,8 +6130,17 @@ export function createSidecarRouter( noteSenderDeployStarted, noteSenderDeploySettled, sendProbeToAllocation, - disconnectAllocation, + detachAllocation, + syncSidecar, sendAgentUndeploy, + undeployAllocation, + getCleanupConnection, + sendWorkflowControl, + retainAllocation, + getAllocationRetention, + getRetentionCandidates, + getSidecarInventory, + waitForSidecarInventory, sendSourcesUpdate, sendCredentialsUpdate, sendPackToAllocation, @@ -3969,13 +6149,15 @@ export function createSidecarRouter( retireAllocation, waitForAllocatedSidecar, isAllocatedSidecarReady, + holdsAllocatedBinding, isAllocatedWorkflowActive, + reportedDeploymentFailure, + stoppedDeploymentHistory, sendAgentDeployToAllocation, bindAllocatedStepRoute, unbindAllocatedStepRoute, sendProvisionStepToAllocation, sendWorkflowRunDispatchToAllocation, - sendSyncRequest, sendSignalDeliver, sendSignalDeliverToAllocation, sendDrain, @@ -3984,7 +6166,6 @@ export function createSidecarRouter( getConnectedSidecars, getRoutableAddresses, getConnectorState, - requestOAuthLogin, events, }; } diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-token-authenticator.ts b/vendor/intx/hub-sessions/src/ws/sidecar-token-authenticator.ts index 0687c4bf00..c26781ebec 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-token-authenticator.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-token-authenticator.ts @@ -1,6 +1,10 @@ -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, or } from "drizzle-orm"; import { sha256 } from "@intx/crypto"; -import type { DB } from "@intx/db"; +import { + workflowRunExecutability, + workflowCapacityReleaseAt, + type DB, +} from "@intx/db"; import { sidecar, sidecarAllocation, @@ -11,85 +15,140 @@ import { import type { SidecarCredentialIdentity, SidecarCredentialResolver, + SidecarIdentityUse, } from "../sidecar-allocation/contracts"; import type { SidecarAuthenticator } from "./sidecar-handler"; export type CreateSidecarTokenAuthenticatorDeps = { db: DB["db"]; + now?: () => Date; }; /** - * Builds an authenticator that verifies a sidecar's presented token - * against the per-sidecar hash stored on the `sidecar` table. The token - * is hashed with SHA-256 and looked up by its digest; a matching row - * yields that row's id as the verified identity, and an unknown token - * resolves to `null` so the handshake is rejected. The claimed - * `sidecarId` on the frame is ignored: identity is derived from the - * token alone. + * Builds a resolver that verifies a sidecar's presented token against the + * per-sidecar hash stored on the `sidecar` table. The token is hashed with + * SHA-256 and looked up by its digest; a matching row yields that row's id as + * the verified identity, and `resolveBindings` reads what that sidecar + * currently hosts. An unknown token resolves to `null`; a retired one with + * neither current work nor cleanup left resolves marked `retired`. A sidecar + * that hosts nothing current still resolves and stays connected, idle, after + * its handshake undeploys what it reports. The claimed `sidecarId` on the + * frame is ignored: identity is derived from the token alone. */ export function createSidecarCredentialResolver({ db, + now = () => new Date(), }: CreateSidecarTokenAuthenticatorDeps): SidecarCredentialResolver { - async function resolve( - token: string, - ): Promise { + const cleanupStatus = inArray(sidecarAllocation.status, [ + "releasing", + "replacing", + "destroy_failed", + ]); + async function resolveBindings( + sidecarId: string, + ): Promise { + const [allocations, probes] = await Promise.all([ + db.query.sidecarAllocation.findMany({ + columns: { + id: true, + tenantId: true, + anchorRunId: true, + generation: true, + status: true, + }, + where: and( + eq(sidecarAllocation.sidecarId, sidecarId), + or( + inArray(sidecarAllocation.status, ["provisioning", "allocated"]), + cleanupStatus, + ), + ), + }), + db.query.workflowProbe.findMany({ + columns: { id: true, tenantId: true, generation: true }, + where: and( + eq(workflowProbe.sidecarId, sidecarId), + inArray(workflowProbe.status, ["provisioning", "probing"]), + ), + }), + ]); + const anchors = + allocations.length === 0 + ? [] + : await db.query.workflowRun.findMany({ + columns: { id: true, address: true }, + where: inArray( + workflowRun.id, + allocations.map((allocation) => allocation.anchorRunId), + ), + }); + const addressByAnchor = new Map( + anchors.map((anchor) => [anchor.id, anchor.address]), + ); + return [ + ...allocations.flatMap((allocation): SidecarCredentialIdentity[] => { + const address = addressByAnchor.get(allocation.anchorRunId); + return address === null || address === undefined + ? [] + : [ + { + kind: + allocation.status === "releasing" || + allocation.status === "replacing" || + allocation.status === "destroy_failed" + ? "cleanup" + : "allocated", + sidecarId, + allocationId: allocation.id, + tenantId: allocation.tenantId, + anchorRunId: allocation.anchorRunId, + workflowRunAddress: address, + generation: allocation.generation, + }, + ]; + }), + ...probes.map( + (probe): SidecarCredentialIdentity => ({ + kind: "probe", + sidecarId, + allocationId: probe.id, + tenantId: probe.tenantId, + generation: probe.generation, + }), + ), + ]; + } + + async function resolve(token: string) { const tokenHash = await sha256(token); const row = await db.query.sidecar.findFirst({ + columns: { id: true, retiredAt: true }, where: eq(sidecar.tokenHashSha256, tokenHash), }); if (row === undefined) return null; - - const allocation = await db.query.sidecarAllocation.findFirst({ - where: eq(sidecarAllocation.sidecarId, row.id), - }); - if (allocation !== undefined) { - if ( - allocation.status !== "provisioning" && - allocation.status !== "allocated" - ) { - return null; - } - const anchor = await db.query.workflowRun.findFirst({ - columns: { address: true }, - where: eq(workflowRun.id, allocation.anchorRunId), - }); - if (anchor?.address === null || anchor?.address === undefined) - return null; - - return { - kind: "allocated", - sidecarId: row.id, - allocationId: allocation.id, - tenantId: allocation.tenantId, - anchorRunId: allocation.anchorRunId, - workflowRunAddress: anchor.address, - generation: allocation.generation, - }; - } - - const probe = await db.query.workflowProbe.findFirst({ - where: eq(workflowProbe.sidecarId, row.id), - }); - if ( - probe === undefined || - (probe.status !== "provisioning" && probe.status !== "probing") - ) { - return null; - } - return { - kind: "probe", - sidecarId: row.id, - allocationId: probe.id, - tenantId: probe.tenantId, - generation: probe.generation, - }; + // Work still current on a retired sidecar keeps its reconnects. + if (row.retiredAt !== null && (await resolveBindings(row.id)).length === 0) + return { sidecarId: row.id, retired: true as const }; + return { sidecarId: row.id }; } async function isCurrent( identity: SidecarCredentialIdentity, - use: "registration" | "readiness" | "routing", + use: SidecarIdentityUse, ): Promise { + const copyCheck = + use === "reclaim" || + use === "retention" || + use === "retention-transition"; + if ( + identity.kind === "cleanup" && + use !== "registration" && + use !== "cleanup" + ) + return false; + if (use === "cleanup" && identity.kind !== "cleanup") return false; if (identity.kind === "probe") { + if (copyCheck) return false; const statuses = use === "registration" ? (["provisioning", "probing"] as const) @@ -107,10 +166,15 @@ export function createSidecarCredentialResolver({ return probe !== undefined; } - const statuses = - use === "registration" - ? (["provisioning", "allocated"] as const) - : (["allocated"] as const); + const statusCondition = + identity.kind === "cleanup" + ? cleanupStatus + : inArray( + sidecarAllocation.status, + use === "registration" + ? ["provisioning", "allocated"] + : ["allocated"], + ); const allocation = await db.query.sidecarAllocation.findFirst({ where: and( eq(sidecarAllocation.id, identity.allocationId), @@ -118,22 +182,47 @@ export function createSidecarCredentialResolver({ eq(sidecarAllocation.tenantId, identity.tenantId), eq(sidecarAllocation.anchorRunId, identity.anchorRunId), eq(sidecarAllocation.generation, identity.generation), - inArray(sidecarAllocation.status, statuses), + statusCondition, ), }); if (allocation === undefined) return false; if (use === "registration") return true; - if (allocation.ensureAcceptedGeneration !== identity.generation) { + if ( + identity.kind !== "cleanup" && + allocation.ensureAcceptedGeneration !== identity.generation + ) { + return false; + } + if (copyCheck && allocation.initializationLeaseId !== null) { return false; } const anchor = await db.query.workflowRun.findFirst({ - columns: { address: true }, + columns: { + address: true, + publicKey: true, + status: true, + expiresAt: true, + cancellationRequestedAt: true, + failureCode: true, + endedAt: true, + capacityReleaseAt: true, + lifecyclePolicy: true, + }, where: eq(workflowRun.id, identity.anchorRunId), }); - return anchor?.address === identity.workflowRunAddress; + if (anchor?.address !== identity.workflowRunAddress) return false; + if (!copyCheck) return true; + if (anchor.publicKey === null) return false; + // The Hub may have ended the run without the worker recording it, so the + // run row, not the copy that reconnects, decides whether it is over. + const ended = workflowRunExecutability(anchor) === "terminal"; + if (use === "retention-transition") return ended; + if (use === "reclaim") return !ended && anchor.failureCode === null; + const releaseAt = workflowCapacityReleaseAt(anchor); + return ended && (releaseAt === null || releaseAt > now()); } - return { resolve, isCurrent }; + return { resolve, resolveBindings, isCurrent }; } export function createSidecarTokenAuthenticator( diff --git a/vendor/intx/hub-sessions/tsconfig.json b/vendor/intx/hub-sessions/tsconfig.json index 8ae418e92a..f0bbc81259 100644 --- a/vendor/intx/hub-sessions/tsconfig.json +++ b/vendor/intx/hub-sessions/tsconfig.json @@ -22,7 +22,7 @@ "path": "../db/tsconfig.src.json" }, { - "path": "../inference/tsconfig.src.json" + "path": "../hub-common/tsconfig.src.json" }, { "path": "../mime/tsconfig.src.json" diff --git a/vendor/intx/hub-sessions/tsconfig.src.json b/vendor/intx/hub-sessions/tsconfig.src.json index fdc91b3cf2..351d20038d 100644 --- a/vendor/intx/hub-sessions/tsconfig.src.json +++ b/vendor/intx/hub-sessions/tsconfig.src.json @@ -26,7 +26,7 @@ "path": "../db/tsconfig.src.json" }, { - "path": "../inference/tsconfig.src.json" + "path": "../hub-common/tsconfig.src.json" }, { "path": "../mime/tsconfig.src.json" diff --git a/vendor/intx/inference/README.md b/vendor/intx/inference/README.md index f7bc4f25c9..6cd0e8f8be 100644 --- a/vendor/intx/inference/README.md +++ b/vendor/intx/inference/README.md @@ -9,6 +9,10 @@ prompt to settled tool calls. Consumed by `@intx/harness`, which composes the reactor with tool runners, directors, and runtime capabilities. +Hosts that only load adapter configuration can import `loadAdapterFactories` +from `@intx/inference/manifest` without loading the reactor and inference +assembly. The `AdapterManifest` schema lives in `@intx/types/adapter-manifest`. + ```ts import { createReactorAssembly, createDefaultDirector } from "@intx/inference"; diff --git a/vendor/intx/inference/VENDORED-FROM b/vendor/intx/inference/VENDORED-FROM index 10c6a0fa3d..acf369b618 100644 --- a/vendor/intx/inference/VENDORED-FROM +++ b/vendor/intx/inference/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/inference) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). providers/google-genai-files.ts builds its upload body as `new Uint8Array(bytes)`: TS 6's lib.dom `BodyInit` rejects `Uint8Array` and workbench compiles this source under DOM-lib packages (upstream compiles ESNext-only under TS 5.9). CL-7443 retires CL-7190's message_response resume branch in reactor.ts's resumePendingOperation/timeoutMessageFor (and its answer_result resume mode), plus the reactor.test.ts/testing/fakes.ts regression harness CL-7190 added to cover it: `ask_user` posts its question and ends the turn instead of parking a gate, so the reactor is back to zero local delta and zero tests, matching upstream. diff --git a/vendor/intx/inference/package.json b/vendor/intx/inference/package.json index 28bf1ecbfa..e350a7148a 100644 --- a/vendor/intx/inference/package.json +++ b/vendor/intx/inference/package.json @@ -1,7 +1,7 @@ { "name": "@intx/inference", "description": "Provider-agnostic inference runtime with Anthropic, OpenAI, and Google GenAI adapters", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -12,13 +12,21 @@ "./providers": { "types": "./src/providers/index.ts", "default": "./src/providers/index.ts" + }, + "./default-director": { + "types": "./src/default-director.ts", + "default": "./src/default-director.ts" + }, + "./manifest": { + "types": "./src/manifest.ts", + "default": "./src/manifest.ts" } }, "scripts": { "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:" }, diff --git a/vendor/intx/inference/src/assembly.ts b/vendor/intx/inference/src/assembly.ts index 538483e90d..2eb8fd209d 100644 --- a/vendor/intx/inference/src/assembly.ts +++ b/vendor/intx/inference/src/assembly.ts @@ -23,7 +23,7 @@ import { type ToolDefinition, type ToolResultTransform, type ToolRunner, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; import { createAuditCollector, type AuditCollector } from "./audit-collector"; import { diff --git a/vendor/intx/inference/src/default-director.ts b/vendor/intx/inference/src/default-director.ts index e9c2dbd570..41aec3a347 100644 --- a/vendor/intx/inference/src/default-director.ts +++ b/vendor/intx/inference/src/default-director.ts @@ -21,17 +21,17 @@ // reply so the problem is visible, and the agent remains alive for retries. import { getLogger } from "@intx/log"; -import { - formatSafetyRatingText, - type ReactorDirector, - type ReactorInboundEvent, - type ReactorState, - type ReactorCapabilities, - type ReactorAction, - type AssistantTurn, - type ToolCall, - type ToolDefinition, +import type { + ReactorDirector, + ReactorInboundEvent, + ReactorState, + ReactorCapabilities, + ReactorAction, + AssistantTurn, + ToolCall, + ToolDefinition, } from "@intx/types/runtime"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; const logger = getLogger(["interchange", "inference", "default-director"]); diff --git a/vendor/intx/inference/src/effort.ts b/vendor/intx/inference/src/effort.ts new file mode 100644 index 0000000000..d7aecc2296 --- /dev/null +++ b/vendor/intx/inference/src/effort.ts @@ -0,0 +1,76 @@ +import { getLogger } from "@intx/log"; +import type { InferenceEffort } from "@intx/types/runtime-core"; + +const logger = getLogger(["interchange", "inference", "adapter"]); + +/** + * The thinking budget an effort maps to on a model that takes a token + * budget rather than an effort name (classic Anthropic, Gemini 2.5). The + * low, medium, and high values follow Google's published mapping for its + * OpenAI-compatible endpoint. `max` repeats `high`: 24576 is the largest + * budget every Gemini 2.5 model accepts. + */ +export const EFFORT_THINKING_BUDGET: Readonly> = + { + low: 1024, + medium: 8192, + high: 24576, + max: 24576, + }; + +/** Name the setting a thinking budget came from, for an error message. */ +export function budgetSource( + effort: InferenceEffort, + budgetTokens: number | undefined, +): string { + return budgetTokens === undefined + ? `effort "${effort}"` + : "thinking.budgetTokens"; +} + +/** + * Reject a thinking budget that leaves no room under the output cap. The + * cap bounds thinking and answer together, and the caller owns the cap, so + * the adapter fails the call with a fix rather than raise the cap itself or + * let the provider reject the request. `source` names the setting the + * budget came from (see `budgetSource`). + */ +export function rejectBudgetOverCap( + model: string, + source: string, + budget: number, + maxTokens: number, +): void { + if (maxTokens > budget) return; + throw new Error( + `${source} on model ${model} needs a thinking budget of ` + + `${budget} tokens, but maxTokens is ${maxTokens}; set maxTokens ` + + `above ${budget}.`, + ); +} + +// A step routing to a model that cannot carry an option re-issues the same +// drop on every turn and every retry, so each distinct drop is reported once +// for the life of the process. +const reported = new Set(); + +/** Forget every reported drop so tests can assert on the next one. */ +export function resetDroppedOptionReports(): void { + reported.clear(); +} + +/** + * A per-call inference option the adapter cannot put on the wire for this + * model. The selector accepts the option with no model awareness, so the + * drop is warn-logged rather than a silent no-op. + */ +export function warnDroppedOption( + model: string, + option: string, + reason: string, +): void { + const key = JSON.stringify([model, option, reason]); + if (reported.has(key)) return; + reported.add(key); + logger.warn`Dropping ${option} for model ${model}: ${reason}`; +} diff --git a/vendor/intx/inference/src/errors.ts b/vendor/intx/inference/src/errors.ts index 75207ce02d..1be73a11d4 100644 --- a/vendor/intx/inference/src/errors.ts +++ b/vendor/intx/inference/src/errors.ts @@ -43,6 +43,14 @@ export function classifyNetworkError(cause: unknown): InferenceError { return { category: "retryable", message, raw: cause }; } +// An adapter that refuses to build a request has rejected the call's own +// options before any network call, so the identical request fails the same +// way on every attempt. +export function classifyRequestBuildError(cause: unknown): InferenceError { + const message = cause instanceof Error ? cause.message : String(cause); + return { category: "fatal", message, raw: cause }; +} + export function classifyAbortError(): InferenceError { return { category: "aborted", message: "inference aborted" }; } diff --git a/vendor/intx/inference/src/harness.ts b/vendor/intx/inference/src/harness.ts index cdc3501d5d..e345a4c906 100644 --- a/vendor/intx/inference/src/harness.ts +++ b/vendor/intx/inference/src/harness.ts @@ -49,6 +49,7 @@ import { injectCredentials } from "./auth"; import { classifyHTTPError, classifyNetworkError, + classifyRequestBuildError, classifyAbortError, classifyStreamError, classifyTimeoutError, @@ -346,7 +347,7 @@ async function* runSingleAttempt( type: "inference.error", seq: nextSeq(), data: { - error: classifyNetworkError(cause), + error: classifyRequestBuildError(cause), partial: snapshotPartial(partial), }, }; diff --git a/vendor/intx/inference/src/index.ts b/vendor/intx/inference/src/index.ts index bd19dfd238..2f6deabcb7 100644 --- a/vendor/intx/inference/src/index.ts +++ b/vendor/intx/inference/src/index.ts @@ -20,7 +20,6 @@ export type { AdapterRegistry, AdapterFactory, } from "./adapter"; -export { AdapterManifest, AdapterManifestEntry } from "./manifest"; export { CREDENTIAL_SENTINEL, BEARER_CREDENTIAL_SENTINEL } from "./auth"; export { classifyHTTPError, diff --git a/vendor/intx/inference/src/manifest.ts b/vendor/intx/inference/src/manifest.ts index 00666fc061..4fab2bad32 100644 --- a/vendor/intx/inference/src/manifest.ts +++ b/vendor/intx/inference/src/manifest.ts @@ -1,21 +1,5 @@ -import { type } from "arktype"; import type { AdapterFactory } from "./adapter"; - -// Describes one custom adapter: the provider identifier it serves, the module -// specifier to import, and the named export within that module to use as its -// factory. Specifiers are operator-config-only and resolve to arbitrary code -// via `import()`, so they must originate solely from trusted operator -// configuration, never from tenant or deploy data. The shape is validated at -// every deserialization boundary; the value is trusted operator input. -export const AdapterManifestEntry = type({ - provider: "string", - specifier: "string", - export: "string", -}); -export type AdapterManifestEntry = typeof AdapterManifestEntry.infer; - -export const AdapterManifest = AdapterManifestEntry.array(); -export type AdapterManifest = typeof AdapterManifest.infer; +import type { AdapterManifest } from "@intx/types"; // Imports a module by specifier. The production importer is `import()`; tests // inject a synthetic importer so they can exercise the loader without fixture diff --git a/vendor/intx/inference/src/providers/anthropic.ts b/vendor/intx/inference/src/providers/anthropic.ts index fe9fa2ab7d..20e20fd673 100644 --- a/vendor/intx/inference/src/providers/anthropic.ts +++ b/vendor/intx/inference/src/providers/anthropic.ts @@ -1,21 +1,25 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { ConversationTurn, ContentBlock, - InferenceEvent, InferenceOptions, LastCycleSource, MediaSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { - CitationBlock as CitationBlockType, - formatSafetyRatingText, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { CitationBlock as CitationBlockType } from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { CREDENTIAL_SENTINEL } from "../auth"; +import { + EFFORT_THINKING_BUDGET, + budgetSource, + rejectBudgetOverCap, + warnDroppedOption, +} from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -45,13 +49,16 @@ export const ADAPTIVE_THINKING_MODELS: ReadonlySet = new Set([ "claude-sonnet-4-6", ]); -// The effort this adapter sends on the adaptive-thinking wire in production. -// "high" is the Anthropic API default. The discovery capture rig deliberately -// sends "max" instead: only "max" reliably elicits a thinking block to capture, -// so the production default and the capture value are an intentional pair, not -// drift. ADAPTIVE_THINKING_MODELS above must match across the two layers; the -// effort values, by contrast, are meant to differ. A guard test in the -// discovery package checks both effort values. +// The effort this adapter sends on the adaptive-thinking wire when thinking +// is on and the call names no effort. "high" is the Anthropic API default for +// every model in ADAPTIVE_THINKING_MODELS; a model whose API default differs +// (some default to "medium") must not join that set without revisiting this. +// The discovery capture rig deliberately sends "max" instead: only "max" +// reliably elicits a thinking block to capture, so the production default and +// the capture value are an intentional pair, not drift. The model set above +// (ADAPTIVE_THINKING_MODELS) must match across the two layers; the effort +// values, by contrast, are meant to differ. A guard test in the discovery +// package checks both effort values. export const ADAPTIVE_THINKING_EFFORT = "high"; // --------------------------------------------------------------------------- @@ -79,9 +86,10 @@ function buildRequest( ? options.systemPrompt : systemText || undefined; + const maxTokens = options.maxTokens ?? 4096; const body: Record = { model, - max_tokens: options.maxTokens ?? 4096, + max_tokens: maxTokens, messages: conversationMessages.map((msg, i) => { // Place a cache breakpoint on the last user message so all prior // turns are cached on the next request. @@ -103,20 +111,7 @@ function buildRequest( ]; } - if (options.thinking?.enabled) { - // Adaptive models reject the classic budget_tokens shape with - // invalid_request_error and require thinking:{type:"adaptive"} - // plus output_config.effort. - if (ADAPTIVE_THINKING_MODELS.has(model)) { - body["thinking"] = { type: "adaptive" }; - body["output_config"] = { effort: ADAPTIVE_THINKING_EFFORT }; - } else { - body["thinking"] = { - type: "enabled", - budget_tokens: options.thinking.budgetTokens ?? 1024, - }; - } - } + applyReasoning(body, model, options, maxTokens); if (options.tools !== undefined && options.tools.length > 0) { const tools: Record[] = options.tools.map((t) => ({ @@ -146,6 +141,70 @@ function buildRequest( }; } +// Effort and thinking.budgetTokens are two spellings of one intent, so an +// effort with no thinking setting turns thinking on. Adaptive models take the +// effort name on output_config.effort, with or without thinking; classic +// models take it as a thinking budget. An explicit thinking.enabled: false +// wins over an effort on a classic model, where effort has no other field. +function applyReasoning( + body: Record, + model: string, + options: InferenceOptions, + maxTokens: number, +): void { + const { effort, thinking } = options; + const effortSet = effort !== undefined; + const thinkingOn = thinking?.enabled ?? effortSet; + + // Adaptive models reject the classic budget_tokens shape with + // invalid_request_error and require thinking:{type:"adaptive"}. + if (ADAPTIVE_THINKING_MODELS.has(model)) { + if (thinkingOn) { + body["thinking"] = { type: "adaptive" }; + } + if (effort !== undefined) { + body["output_config"] = { effort }; + } else if (thinkingOn) { + body["output_config"] = { effort: ADAPTIVE_THINKING_EFFORT }; + } + if (thinking?.budgetTokens !== undefined) { + warnDroppedOption( + model, + `thinking.budgetTokens ${thinking.budgetTokens}`, + "adaptive Anthropic takes effort, not a thinking budget", + ); + } + return; + } + + if (!thinkingOn) { + if (effort !== undefined) { + warnDroppedOption( + model, + `effort "${effort}"`, + "classic Anthropic carries effort only as a thinking budget, and thinking is off", + ); + } + return; + } + + let budget = 1024; + if (thinking?.budgetTokens !== undefined) { + budget = thinking.budgetTokens; + } else if (effort !== undefined) { + budget = EFFORT_THINKING_BUDGET[effort]; + } + if (effort !== undefined) { + rejectBudgetOverCap( + model, + budgetSource(effort, thinking?.budgetTokens), + budget, + maxTokens, + ); + } + body["thinking"] = { type: "enabled", budget_tokens: budget }; +} + // Anthropic's Messages API has no native structured-outputs surface. // `text` is the default and a no-op. `json` and `json-schema` raise // here at the marshaling boundary rather than silently dropping the diff --git a/vendor/intx/inference/src/providers/google-genai.ts b/vendor/intx/inference/src/providers/google-genai.ts index 3e4069a168..e608c13110 100644 --- a/vendor/intx/inference/src/providers/google-genai.ts +++ b/vendor/intx/inference/src/providers/google-genai.ts @@ -1,20 +1,26 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { CodeExecutionRequestBlock, CodeExecutionResultBlock, ConversationTurn, ContentBlock, - InferenceEvent, InferenceOptions, LastCycleSource, MediaSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { formatSafetyRatingText } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { CREDENTIAL_SENTINEL } from "../auth"; +import { + EFFORT_THINKING_BUDGET, + budgetSource, + rejectBudgetOverCap, + warnDroppedOption, +} from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -37,6 +43,25 @@ const THINKING_MANDATORY_MODELS: ReadonlySet = new Set([ "gemini-3.6-flash", ]); +// Gemini 3 models that take thinkingConfig.thinkingLevel, whose low, medium, +// and high levels match the effort names. Google rejects a request that +// carries both thinkingLevel and thinkingBudget with HTTP 400, so a level +// model sends exactly one of them. +const THINKING_LEVEL_MODELS: ReadonlySet = new Set([ + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gemini-3.6-flash", +]); + +// Image models with no thinking setting an effort name can map to: +// gemini-3.1-flash-image accepts only the "minimal" and "high" levels, and +// Google documents no thinking support on gemini-2.5-flash-image. +const NO_EFFORT_MODELS: ReadonlySet = new Set([ + "gemini-2.5-flash-image", + "gemini-3.1-flash-image", +]); + // Dynamic thinking budget sentinel: the model decides how much to // think. Used when suppressing thought parts on thinking-mandatory // models that reject a zero budget. @@ -502,6 +527,74 @@ function tryParseJSONObject(text: string): Record | null { // generationConfig // --------------------------------------------------------------------------- +// thinking.enabled === true -> ask Gemini to surface thought parts, with a +// level on a level model that names an +// effort, or a budget (from budgetTokens, +// else the effort's budget, else 1024) +// thinking.enabled === false -> suppress thoughts: budget 0 when the +// model allows it, or dynamic (-1) for +// thinking-mandatory models that reject +// a zero budget with HTTP 400 +// thinking absent -> an effort turns thinking on as above; +// with no effort, omit thinkingConfig and +// Gemini uses the model's default +function buildThinkingConfig( + model: string, + options: InferenceOptions, +): Record | undefined { + const { thinking } = options; + const effort = NO_EFFORT_MODELS.has(model) ? undefined : options.effort; + if (options.effort !== undefined && effort === undefined) { + warnDroppedOption( + model, + `effort "${options.effort}"`, + "this model has no thinking setting an effort maps to", + ); + } + const effortSet = effort !== undefined; + if (thinking === undefined && !effortSet) return undefined; + + const thinkingOn = thinking?.enabled ?? effortSet; + if (!thinkingOn) { + if (effort !== undefined) { + warnDroppedOption( + model, + `effort "${effort}"`, + "Gemini carries effort only as a thinking setting, and thinking is off", + ); + } + return { thinkingBudget: minimalThinkingBudget(model) }; + } + + if ( + effort !== undefined && + thinking?.budgetTokens === undefined && + THINKING_LEVEL_MODELS.has(model) + ) { + // "high" is the top level, so "max" maps onto it. + const thinkingLevel = effort === "max" ? "high" : effort; + return { thinkingLevel, includeThoughts: true }; + } + + let budget = 1024; + if (thinking?.budgetTokens !== undefined) { + budget = thinking.budgetTokens; + } else if (effort !== undefined) { + budget = EFFORT_THINKING_BUDGET[effort]; + } + // maxOutputTokens counts thought tokens and cuts the response off, so a + // budget at or above it leaves no room for the answer. + if (effort !== undefined && options.maxTokens !== undefined) { + rejectBudgetOverCap( + model, + budgetSource(effort, thinking?.budgetTokens), + budget, + options.maxTokens, + ); + } + return { thinkingBudget: budget, includeThoughts: true }; +} + function buildGenerationConfig( model: string, options: InferenceOptions, @@ -515,26 +608,9 @@ function buildGenerationConfig( config["temperature"] = options.temperature; } - // thinking.enabled === true -> include a budget (default 1024) and - // ask Gemini to surface thought parts - // thinking.enabled === false -> suppress thoughts: budget 0 when the - // model allows it, or dynamic (-1) for - // thinking-mandatory models that reject - // a zero budget with HTTP 400 - // thinking absent -> omit thinkingConfig entirely; Gemini - // uses the model's default - if (options.thinking !== undefined) { - if (options.thinking.enabled) { - const thinkingBudget = options.thinking.budgetTokens ?? 1024; - config["thinkingConfig"] = { - thinkingBudget, - includeThoughts: true, - }; - } else { - config["thinkingConfig"] = { - thinkingBudget: minimalThinkingBudget(model), - }; - } + const thinkingConfig = buildThinkingConfig(model, options); + if (thinkingConfig !== undefined) { + config["thinkingConfig"] = thinkingConfig; } if ( diff --git a/vendor/intx/inference/src/providers/index.ts b/vendor/intx/inference/src/providers/index.ts index c615b1b29a..e2e5a0dbb0 100644 --- a/vendor/intx/inference/src/providers/index.ts +++ b/vendor/intx/inference/src/providers/index.ts @@ -2,7 +2,8 @@ import { createAdapterRegistry } from "../adapter"; import type { AdapterFactory, AdapterRegistry } from "../adapter"; import { createDependencies, type Dependencies } from "../harness"; import { loadAdapterFactories } from "../manifest"; -import type { AdapterManifest, ModuleImporter } from "../manifest"; +import type { ModuleImporter } from "../manifest"; +import type { AdapterManifest } from "@intx/types"; import { createAnthropicAdapter } from "./anthropic"; import { createGoogleGenAIAdapter } from "./google-genai"; import { createOpenAIAdapter } from "./openai"; diff --git a/vendor/intx/inference/src/providers/openai.ts b/vendor/intx/inference/src/providers/openai.ts index 428d2d4815..867befb591 100644 --- a/vendor/intx/inference/src/providers/openai.ts +++ b/vendor/intx/inference/src/providers/openai.ts @@ -1,17 +1,19 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { ConversationTurn, ContentBlock, - InferenceEvent, + InferenceEffort, InferenceOptions, LastCycleSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { formatSafetyRatingText } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { BEARER_CREDENTIAL_SENTINEL } from "../auth"; +import { warnDroppedOption } from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -46,6 +48,11 @@ export const OpenAIQuirks = type({ // through the same adapter (e.g. OpenCode Zen) still take `max_tokens`. // Defaults to `max_tokens` so every existing deployment is unchanged. "maxTokensField?": "'max_tokens' | 'max_completion_tokens'", + // The reasoning_effort value that an effort of "max" sends. Models differ: + // gpt-5.2 through gpt-5.5 top out at "xhigh", gpt-5.6-sol accepts "max", + // and older models accept neither. Absent, "max" sends "high", the highest + // value every reasoning model accepts, and the adapter warns. + "maxEffort?": "'xhigh' | 'max'", // Reject unknown keys so a mistyped quirk name fails loudly at construction // rather than being silently ignored and running with default behavior. "+": "reject", @@ -65,12 +72,31 @@ type ResolvedOpenAIQuirks = { forceAssistantReasoningContent: boolean; reasoningFieldNames: readonly ReasoningField[]; maxTokensField: "max_tokens" | "max_completion_tokens"; + maxEffort: "xhigh" | "max" | null; }; // --------------------------------------------------------------------------- // Request building // --------------------------------------------------------------------------- +// low, medium, and high are reasoning_effort values every OpenAI reasoning +// model accepts, so they pass as given. "max" names the highest level, which +// is a per-model value the deployment supplies through the maxEffort quirk. +function toReasoningEffort( + model: string, + effort: InferenceEffort, + maxEffort: "xhigh" | "max" | null, +): string { + if (effort !== "max") return effort; + if (maxEffort !== null) return maxEffort; + warnDroppedOption( + model, + `effort "max"`, + 'no maxEffort quirk on this source, so it sends "high"', + ); + return "high"; +} + function buildRequest( messages: ConversationTurn[], model: string, @@ -92,6 +118,25 @@ function buildRequest( body["temperature"] = options.temperature; } + const hasTools = options.tools !== undefined && options.tools.length > 0; + // gpt-5.6 Chat Completions rejects function tools unless + // reasoning_effort is explicitly "none" (reasoned tool use is on + // the Responses API). Keep this list aligned with the discovery + // protocol builder's TOOL_CALL_REASONING_NONE_MODELS set. + const forceNoReasoning = + hasTools && + (model === "gpt-5.6-sol" || + model === "gpt-5.6-terra" || + model === "gpt-5.6-luna"); + + if (options.effort !== undefined && !forceNoReasoning) { + body["reasoning_effort"] = toReasoningEffort( + model, + options.effort, + quirks.maxEffort, + ); + } + if (options.tools !== undefined && options.tools.length > 0) { body["tools"] = options.tools.map((t) => ({ type: "function", @@ -101,17 +146,19 @@ function buildRequest( parameters: t.inputSchema, }, })); - // gpt-5.6 Chat Completions rejects function tools unless - // reasoning_effort is explicitly "none" (reasoned tool use is on - // the Responses API). Keep this list aligned with the discovery - // protocol builder's TOOL_CALL_REASONING_NONE_MODELS set. - if ( - model === "gpt-5.6-sol" || - model === "gpt-5.6-terra" || - model === "gpt-5.6-luna" - ) { - body["reasoning_effort"] = "none"; + } + + // Set after the tools, in the key order of the discovery protocol + // builder's captured bodies. + if (forceNoReasoning) { + if (options.effort !== undefined) { + warnDroppedOption( + model, + `effort "${options.effort}"`, + "gpt-5.6 Chat Completions tool calls require reasoning_effort none", + ); } + body["reasoning_effort"] = "none"; } if (options.systemPrompt) { @@ -1075,6 +1122,7 @@ export function createOpenAIAdapter( reasoningFieldNames: parsedQuirks.reasoningFieldNames ?? DEFAULT_REASONING_FIELDS, maxTokensField: parsedQuirks.maxTokensField ?? "max_tokens", + maxEffort: parsedQuirks.maxEffort ?? null, }; // Per-request indexer state. Adapter instances are created per diff --git a/vendor/intx/inference/src/reactor.ts b/vendor/intx/inference/src/reactor.ts index bf5efda17c..ce4b2cfd29 100644 --- a/vendor/intx/inference/src/reactor.ts +++ b/vendor/intx/inference/src/reactor.ts @@ -41,7 +41,8 @@ import type { } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; -import { ApprovalDecision, signalKindToGateType } from "@intx/types"; +import { ApprovalDecision, signalKindToGateType } from "@intx/types/signals"; +import { PerCallInferenceOptions } from "@intx/types/runtime-core"; import type { CredentialMaterialResolver } from "@intx/types"; import { canonicalJsonStringify } from "@intx/types/wire-definition-hash"; import { type } from "arktype"; @@ -73,6 +74,42 @@ function assertNever(x: never): never { throw new Error(`Unhandled resume case: ${JSON.stringify(x)}`); } +/** + * Layer director-named options over per-send ones; a director key wins over + * the per-send value for that same key. The merge is shallow: a director that + * names `thinking` replaces the whole object, so nested per-send fields such + * as `budgetTokens` are dropped rather than merged in. + */ +export function mergeInferenceOptions( + perSend: InferenceOptions | undefined, + director: InferenceOptions | undefined, +): InferenceOptions | undefined { + if (perSend === undefined) return director; + return { ...perSend, ...director }; +} + +/** + * Copy only the keys a send may set. A structurally wider bag still + * type-checks as `PerCallInferenceOptions`; without this copy, + * `systemPrompt`, `tools`, and `providerOptions` would enter the + * message-run slot and displace the deployed agent definition. The key + * set is the arktype declaration itself, so a key added there is copied + * without a matching edit here. The declaration also checks the values, + * so an out-of-range option is rejected here rather than at the provider. + */ +const StripToPerCallInference = + PerCallInferenceOptions.onDeepUndeclaredKey("delete"); + +function perCallInference( + options: PerCallInferenceOptions, +): PerCallInferenceOptions { + const stripped = StripToPerCallInference(options); + if (stripped instanceof type.errors) { + throw new Error(`Invalid per-call inference options: ${stripped.summary}`); + } + return stripped; +} + function buildHarnessOpts( turns: ConversationTurn[], source: InferenceSource, @@ -149,11 +186,20 @@ export type ReactorConfig = { doomLoopThreshold?: number | false; }; +export type DeliverOptions = { + inference?: PerCallInferenceOptions; +}; + export type Reactor = { /** Begin processing. Emits reactor.start. Must be called exactly once. */ start(): void; - /** Inject an inbound message into the reactor. */ - deliver(message: InboundMessage): void; + /** + * Inject an inbound message into the reactor. `opts.inference` applies to + * every inference call of the message run the delivery opens, or of the + * parked run it resumes, beneath any option the director names; it is + * never retained past that run. + */ + deliver(message: InboundMessage, opts?: DeliverOptions): void; /** Initiate graceful shutdown with a reason. */ abort(reason: AbortReason): void; }; @@ -344,6 +390,26 @@ export function createReactor(config: ReactorConfig): Reactor { // produces unambiguous start/end pairs downstream. let currentMessageRunId: string | null = null; let currentMessageId: string | null = null; + // Per-send inference options, keyed by the delivered message until it is + // dequeued (or correlated), then held for the message run it drives. + const deliveredInference = new WeakMap< + InboundMessage, + PerCallInferenceOptions + >(); + let messageRunInference: PerCallInferenceOptions | undefined; + // Correlation ids the currently open message run suspended on, cleared + // whenever that run closes. This is what tells a late correlated response + // apart from one that correlates to an operation some earlier, already- + // closed run left registered: `gates` itself never expires an entry on run + // close (only on clear/timeout), so a stale gate can outlive the run that + // opened it and still answer `gates.has`. The key is the correlation id, + // not the gate id, because a director may suspend under its own gate id + // while the tool's pending marker registered the operation under + // `pending-`; the correlation id is what both share, and it + // is also how `tryCorrelate` finds the gate to resume. Rehydrated gates + // (`rehydrateGates`) are deliberately not added here -- they predate any + // run this process has opened, so they can never be "this run's own". + const openRunCorrelationIds = new Set(); // Doom-loop detection state, scoped to the current message run. Each executed // tool-call turn is reduced to a batch signature; consecutive identical @@ -382,6 +448,8 @@ export function createReactor(config: ReactorConfig): Reactor { status: "completed" | "failed", error?: { message: string; kind?: string }, ): void { + messageRunInference = undefined; + openRunCorrelationIds.clear(); // Gated on the run id alone: a message that named no id still has to // close its bracket, or the run stays open for the rest of the session. if (currentMessageRunId === null) return; @@ -728,7 +796,7 @@ export function createReactor(config: ReactorConfig): Reactor { const harnessOpts = buildHarnessOpts( prompt, config.source, - options, + mergeInferenceOptions(messageRunInference, options), signal, nextSeq, config.readMaterial, @@ -1211,6 +1279,12 @@ export function createReactor(config: ReactorConfig): Reactor { correlationId, onGateCleared, ); + // Mark this correlation as belonging to the currently open run so a late + // correlated response can tell it apart from one an earlier, already- + // closed run left registered (see `openRunCorrelationIds`). + if (currentMessageRunId !== null && correlationId !== undefined) { + openRunCorrelationIds.add(correlationId); + } if (stateManager !== null) { stateManager.setGatesSnapshot(gates.snapshot()); @@ -1268,7 +1342,10 @@ export function createReactor(config: ReactorConfig): Reactor { // effective timeout the director-suspend fallback uses — rather than a // silent zero. This does not run through `suspendOnGate`: rehydration must // not re-emit `reactor.gate.blocked` (the suspension already happened before - // the restart) and must not commit (nothing changed). + // the restart) and must not commit (nothing changed). It also must not add + // to `openRunCorrelationIds`: a rehydrated gate predates any run this + // process has opened, so it can never be mistaken for the currently open + // run's own suspension. function rehydrateGates(ops: PendingOperation[]): void { for (const op of ops) { const timeoutMs = @@ -1321,9 +1398,11 @@ export function createReactor(config: ReactorConfig): Reactor { // Append inbound messages to conversation history so the provider sees them. // Each dequeued message.received opens a fresh per-message run bracket. - // If a prior bracket is still open (defensive — should not occur given - // the dequeue priority that drains cycle events before new messages), - // close it as completed first so the new bracket starts cleanly. + // An uncorrelated message can arrive while an earlier run is still + // parked on a gate (that run left no way to reject or supersede it), so + // finding a prior bracket still open here is a routine path, not a + // defensive fallback: close it as completed first so the new bracket + // starts cleanly. if (event.type === "message.received") { if (stateManager !== null) { const msg = createInboundTurn(event.message); @@ -1335,6 +1414,8 @@ export function createReactor(config: ReactorConfig): Reactor { closeMessageRun("completed"); } openMessageRun(event.message.headers.messageId); + messageRunInference = deliveredInference.get(event.message); + deliveredInference.delete(event.message); } // A parked approval that ended without running its tool (rejected or @@ -1666,6 +1747,28 @@ export function createReactor(config: ReactorConfig): Reactor { function processDelivery(message: InboundMessage): void { void (async () => { + // A correlated resume continues the parked run rather than opening a + // new one, and the loop may re-infer before tryCorrelate settles, so + // the resume's options are adopted up front whenever this message's + // correlation ID is one the currently open run suspended on + // (`openRunCorrelationIds`). `gates.has` alone is not enough: a gate an + // earlier, already-closed run left registered is still live there, and + // a late correlated response for it must not overwrite the run open + // now. An uncorrelated message either finds no run open yet (harmless + // to set now — opening its own run in the message.received handler + // sets the same value) or arrives while an unrelated run is still + // open, in which case it must not touch that run's options and + // instead overwrites them when its own run opens. + const inference = deliveredInference.get(message); + if (inference !== undefined) { + const correlationId = message.headers.interchangeCorrelationId; + const correlatesToOpenRun = + correlationId !== undefined && + openRunCorrelationIds.has(correlationId); + if (correlatesToOpenRun || currentMessageRunId === null) { + messageRunInference = inference; + } + } let correlated: boolean; try { correlated = await tryCorrelate(message); @@ -1698,8 +1801,11 @@ export function createReactor(config: ReactorConfig): Reactor { })(); } - function deliver(message: InboundMessage): void { + function deliver(message: InboundMessage, opts?: DeliverOptions): void { if (done) return; + if (opts?.inference !== undefined) { + deliveredInference.set(message, perCallInference(opts.inference)); + } if (startupDeliveries !== null) { startupDeliveries.push(message); return; diff --git a/vendor/intx/inference/src/transform.ts b/vendor/intx/inference/src/transform.ts index 3b820413e8..b02d89674b 100644 --- a/vendor/intx/inference/src/transform.ts +++ b/vendor/intx/inference/src/transform.ts @@ -9,11 +9,8 @@ // buildRequest paths also apply provider-specific history fixes. The // originating model is tracked per-message, not per-conversation. -import { - formatSafetyRatingText, - type ConversationTurn, - type ContentBlock, -} from "@intx/types/runtime"; +import type { ConversationTurn, ContentBlock } from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; export type TransformOptions = { targetModel: string; diff --git a/vendor/intx/inference/src/turns.ts b/vendor/intx/inference/src/turns.ts index 0edb91415b..a35779dd7f 100644 --- a/vendor/intx/inference/src/turns.ts +++ b/vendor/intx/inference/src/turns.ts @@ -8,7 +8,8 @@ import type { ToolCall, ToolResult, } from "@intx/types/runtime"; -import { attachmentCategory, base64Encode } from "@intx/types"; +import { attachmentCategory } from "@intx/types/attachments"; +import { base64Encode } from "@intx/types/base64"; import { getLogger } from "@intx/log"; const logger = getLogger(["interchange", "inference", "turns"]); diff --git a/vendor/intx/mail-memory/VENDORED-FROM b/vendor/intx/mail-memory/VENDORED-FROM index ca7c5637de..681092b434 100644 --- a/vendor/intx/mail-memory/VENDORED-FROM +++ b/vendor/intx/mail-memory/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mail-memory) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mail-memory/package.json b/vendor/intx/mail-memory/package.json index a8d4e17087..dd1613220c 100644 --- a/vendor/intx/mail-memory/package.json +++ b/vendor/intx/mail-memory/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mail-memory", "description": "In-memory MessageTransport for single-process and test environments", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,8 +14,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.4.0", - "@intx/log": "0.4.0", + "@intx/crypto": "0.5.0", + "@intx/log": "0.5.0", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", diff --git a/vendor/intx/mail-memory/src/index.ts b/vendor/intx/mail-memory/src/index.ts index b20ec7b8bf..908e49edaf 100644 --- a/vendor/intx/mail-memory/src/index.ts +++ b/vendor/intx/mail-memory/src/index.ts @@ -1,4 +1,8 @@ -export { InMemoryTransport, type HubTransport } from "./transport"; +export { + InMemoryTransport, + inboundMessageToRaw, + type HubTransport, +} from "./transport"; export type { RemoteSendHandler, MessageSentHandler, @@ -17,9 +21,14 @@ export type { * * const alphaTransport = transport.getTransportFor("alpha@local.interchange"); * await alphaTransport.send({ to: "beta@local.interchange", ... }); + * + * With `relayOnly`, registered addresses only sign: every recipient goes + * through the remote send handler and nothing is delivered locally. */ import { InMemoryTransport } from "./transport"; -export function createInMemoryTransport(): InMemoryTransport { - return new InMemoryTransport(); +export function createInMemoryTransport( + opts: { relayOnly?: boolean } = {}, +): InMemoryTransport { + return new InMemoryTransport(opts.relayOnly ?? false); } diff --git a/vendor/intx/mail-memory/src/send.ts b/vendor/intx/mail-memory/src/send.ts index d406068844..b506b48049 100644 --- a/vendor/intx/mail-memory/src/send.ts +++ b/vendor/intx/mail-memory/src/send.ts @@ -3,20 +3,8 @@ import type { SendReceipt, MailboxEvent, } from "@intx/types/runtime"; -import type { StoredEnvelope } from "@intx/mailbox"; -import { isConversationType } from "@intx/types/runtime"; -import { - assembleSignedContent, - assembleMessage, - buildMessageHeaders, - generateMessageId, - isMessageId, - parseHeaderSection, - createDetachedSignatureFromProvider, - type MessageHeaders as MimeMessageHeaders, - type ConversationContent, - type StructuredContent, -} from "@intx/mime"; +import { composeOutbound } from "@intx/mailbox"; +import { buildMessageHeaders, parseHeaderSection } from "@intx/mime"; import type { AddressEntry } from "./mailbox"; /** @@ -60,14 +48,16 @@ export type MessageSentHandler = (ctx: MessageSentContext) => Promise; /** * Execute the send() flow: - * 1. Validate sender registration, split recipients into local/remote - * 2. Build signed content part (MIME bytes to sign) - * 3. Sign with sender's CryptoProvider - * 4. Assemble the complete RFC 2822 message - * 5. Append to each local recipient's INBOX and sender's Sent mailbox - * 6. Forward to remote recipients via onRemoteSend - * 7. Schedule watch callbacks asynchronously via queueMicrotask - * 8. Fire onMessageSent callback (fire-and-forget) + * 1. Validate sender registration + * 2. Compose the signed message + * 3. Split recipients into local and remote + * 4. Append to each local recipient's INBOX and the sender's Sent mailbox + * 5. Forward to remote recipients via onRemoteSend + * 6. Schedule watch callbacks asynchronously via queueMicrotask + * 7. Fire onMessageSent callback (fire-and-forget) + * + * With `relayOnly`, every recipient is remote, registered or not, and + * nothing is stored locally: steps 4 and 6 are skipped. * * If onRemoteSend is not provided and there are remote recipients, send() * throws. If onRemoteSend rejects, the error propagates — local delivery @@ -79,6 +69,7 @@ export async function executeSend( senderAddress: string, message: OutboundMessage, entries: Map, + relayOnly: boolean, onRemoteSend?: RemoteSendHandler, onMessageSent?: MessageSentHandler, ): Promise { @@ -90,144 +81,62 @@ export async function executeSend( } const senderCrypto = senderEntry.crypto; - const recipients = Array.isArray(message.to) ? message.to : [message.to]; - if (recipients.length === 0) { - throw new Error("OutboundMessage must have at least one recipient"); - } - - const ccAddressList = - message.cc !== undefined - ? Array.isArray(message.cc) - ? message.cc - : [message.cc] - : []; - - const allAddressees = [...new Set([...recipients, ...ccAddressList])]; - const remoteRecipients = allAddressees.filter((addr) => !entries.has(addr)); + // Local recipients are classified before signing yields, so one + // unregistered meanwhile fails the send instead of being routed as remote. + const registeredBeforeSigning = new Set(entries.keys()); + const composed = await composeOutbound(senderAddress, message, senderCrypto); + const { messageId, rawBytes, envelope } = composed; + const recipients = composed.to; + const ccAddressList = composed.cc; + const allAddressees = composed.recipients; + const localRecipients = relayOnly + ? [] + : allAddressees.filter((addr) => registeredBeforeSigning.has(addr)); + const remoteRecipients = allAddressees.filter( + (addr) => !localRecipients.includes(addr), + ); if (remoteRecipients.length > 0 && onRemoteSend === undefined) { throw new Error( - `Recipient "${remoteRecipients[0]}" is not registered with this transport`, + relayOnly + ? `Cannot send from "${senderAddress}": this transport relays every recipient and has no remote send handler` + : `Recipient "${remoteRecipients[0]}" is not registered with this transport`, ); } - const isConversation = isConversationType(message.type); - - if (isConversation && message.payload !== undefined) { - throw new Error( - "Conversation messages must not carry a structured payload", - ); - } - if (!isConversation && message.content !== undefined) { - throw new Error("Structured messages must not carry a text content field"); - } - - // RFC 5322 section 3.6.4 defines `In-Reply-To` as `1*msg-id`, so a blank - // value names nothing rather than naming a shorter parent. - if ( - message.inReplyTo !== undefined && - message.inReplyTo.trim().length === 0 - ) { - throw new Error( - "OutboundMessage inReplyTo, when provided, must name a message identifier", - ); - } - - const messageId = generateMessageId(senderAddress); - const now = new Date(); - - let content: ConversationContent | StructuredContent; - if (isConversation) { - content = { - kind: "conversation", - text: message.content ?? "", - }; - } else { - const payload = message.payload ?? {}; - const envelope = { - type: message.type, - version: "1", - body: payload, - }; - const structured: StructuredContent = { - kind: "structured", - json: envelope, - }; - if (message.summary !== undefined) structured.summary = message.summary; - content = structured; - } - - const signedContentBytes = assembleSignedContent(content); - const signatureBytes = await createDetachedSignatureFromProvider( - signedContentBytes, - senderCrypto, - ); - - const ccAddresses = ccAddressList.length > 0 ? ccAddressList : undefined; - - const refs = buildReferences(message.inReplyTo, message.references); - - const mimeHeaders: MimeMessageHeaders = { - from: senderAddress, - to: recipients, - cc: ccAddresses, - date: now, - messageId, - subject: message.subject, - inReplyTo: message.inReplyTo, - references: refs, - mimeVersion: "1.0", - interchangeType: message.type, - interchangeCorrelationId: message.correlationId, - interchangeTenantId: message.tenantId, - interchangeAgentId: undefined, - interchangeSessionId: message.sessionId, - interchangeOfferingId: undefined, - interchangeSchemaVersion: undefined, - traceparent: undefined, - tracestate: undefined, - }; - - const rawBytes = assembleMessage( - mimeHeaders, - signedContentBytes, - signatureBytes, - ); - const envelope: StoredEnvelope = { - messageId, - from: senderAddress, - to: recipients, - subject: message.subject ?? "", - date: now, - inReplyTo: message.inReplyTo, - references: refs ?? [], - interchangeType: message.type, - interchangeCorrelationId: message.correlationId, - }; - - // Deliver to each local recipient's INBOX. - const deliveredUids: { address: string; uid: number }[] = []; - for (const recipient of allAddressees) { + // Every recipient's INBOX is resolved before any is appended to, so a + // recipient unregistered while the message was being signed fails the send + // before the others receive it. + const inboxes = localRecipients.map((recipient) => { const entry = entries.get(recipient); - if (entry === undefined) continue; + if (entry === undefined) { + throw new Error( + `Recipient "${recipient}" was unregistered while the message was being signed`, + ); + } const inbox = entry.mailboxes.get("INBOX"); if (inbox === undefined) { throw new Error( `Mailbox "INBOX" does not exist for recipient "${recipient}"`, ); } - const uid = inbox.append(rawBytes, envelope, []); - deliveredUids.push({ address: recipient, uid }); - } + return { address: recipient, inbox }; + }); + const deliveredUids = inboxes.map(({ address, inbox }) => ({ + address, + uid: inbox.append(rawBytes, envelope, []), + })); // Append copy to sender's Sent mailbox. - const sentStore = senderEntry.mailboxes.get("Sent"); - if (sentStore === undefined) { - throw new Error( - `Mailbox "Sent" does not exist for sender "${senderAddress}"`, - ); + if (!relayOnly) { + const sentStore = senderEntry.mailboxes.get("Sent"); + if (sentStore === undefined) { + throw new Error( + `Mailbox "Sent" does not exist for sender "${senderAddress}"`, + ); + } + sentStore.append(rawBytes, envelope, ["\\Seen"]); } - sentStore.append(rawBytes, envelope, ["\\Seen"]); // Fire local recipient watch callbacks ASYNCHRONOUSLY (per MESSAGE.md // requirement). queueMicrotask ensures callbacks never run synchronously @@ -287,31 +196,3 @@ export async function executeSend( status: remoteRecipients.length > 0 ? "queued" : "delivered", }; } - -/** - * Assemble the wire `References` chain for an outbound message. - * - * When the caller supplies a full ancestry (`existingReferences` -- the - * parent's References plus the parent's Message-Id, built by the threaded - * reply path), that chain is used and `inReplyTo` is appended only when it is - * not already the tail. Otherwise the chain is derived from `inReplyTo` alone, - * preserving the pre-existing single-element behavior. - * - * The caller-supplied chain is filtered to RFC 2822 message identifiers: - * inbound mail can carry a headerless-derived (sha256) or otherwise malformed - * Message-Id that is a valid claim-check key but not a valid `` - * identifier, and such a value must not leak into a `References` header. The - * `inReplyTo` value is appended without filtering, matching the pre-existing - * `In-Reply-To`/`References` behavior for a bare reply. - */ -function buildReferences( - inReplyTo: string | undefined, - existingReferences: string[] | undefined, -): string[] | undefined { - const refs = (existingReferences ?? []).filter(isMessageId); - if (inReplyTo === undefined) return refs.length > 0 ? refs : undefined; - if (!refs.includes(inReplyTo)) { - return [...refs, inReplyTo]; - } - return refs; -} diff --git a/vendor/intx/mail-memory/src/transport.ts b/vendor/intx/mail-memory/src/transport.ts index d406ee520e..c88c722036 100644 --- a/vendor/intx/mail-memory/src/transport.ts +++ b/vendor/intx/mail-memory/src/transport.ts @@ -70,9 +70,20 @@ export interface HubTransport { */ export class InMemoryTransport implements MessageTransport, HubTransport { readonly #entries = new Map(); + readonly #relayOnly: boolean; #remoteSendHandler: RemoteSendHandler | undefined; readonly #messageSentHandlers = new Set(); + /** + * `relayOnly` sends every recipient through the remote send handler, + * including addresses registered here, and keeps no local copy. A host + * that registers addresses only to sign their mail uses it, so mail + * between two of them takes the same route as mail to anyone else. + */ + constructor(relayOnly: boolean) { + this.#relayOnly = relayOnly; + } + /** * Set a handler for delivering messages to recipients not registered on * this transport. The federation layer calls this to wire up the websocket @@ -246,10 +257,10 @@ export class InMemoryTransport implements MessageTransport, HubTransport { throw new Error("Use getTransportFor(address) for per-address operations"); } - watch( + async watch( _mailbox: string, _callback: (event: MailboxEvent) => void, - ): Unsubscribe { + ): Promise { throw new Error("Use getTransportFor(address) for per-address operations"); } @@ -321,6 +332,11 @@ export class InMemoryTransport implements MessageTransport, HubTransport { * Throws if the address is not registered. */ deliver(address: string, message: Uint8Array): void { + if (this.#relayOnly) { + throw new Error( + `Cannot deliver mail to "${address}": this transport relays all mail and keeps no mailboxes`, + ); + } const entry = this.#entries.get(address); if (entry === undefined) { throw new Error( @@ -399,6 +415,7 @@ export class InMemoryTransport implements MessageTransport, HubTransport { return new ScopedMessageTransport( address, this.#entries, + this.#relayOnly, () => this.#remoteSendHandler, () => this.#messageSentHandlers, ); @@ -412,17 +429,20 @@ export class InMemoryTransport implements MessageTransport, HubTransport { class ScopedMessageTransport implements MessageTransport { readonly #address: string; readonly #entries: Map; + readonly #relayOnly: boolean; readonly #getRemoteSendHandler: () => RemoteSendHandler | undefined; readonly #getMessageSentHandlers: () => Set; constructor( address: string, entries: Map, + relayOnly: boolean, getRemoteSendHandler: () => RemoteSendHandler | undefined, getMessageSentHandlers: () => Set, ) { this.#address = address; this.#entries = entries; + this.#relayOnly = relayOnly; this.#getRemoteSendHandler = getRemoteSendHandler; this.#getMessageSentHandlers = getMessageSentHandlers; } @@ -475,6 +495,7 @@ class ScopedMessageTransport implements MessageTransport { this.#address, message, this.#entries, + this.#relayOnly, this.#getRemoteSendHandler(), aggregatedHandler, ); @@ -487,36 +508,8 @@ class ScopedMessageTransport implements MessageTransport { _signal?: AbortSignal, ): Promise { const store = this.#requireMailbox(mailbox); - // For append, we need to convert InboundMessage back to raw bytes. - // Since InboundMessage may come from a prior fetchFull, we need the raw - // bytes. This is a design gap — append() takes InboundMessage but we - // need Uint8Array. We store a minimal representation. - // - // For now, serialize the InboundMessage as a minimal RFC 2822 message. - // The stored envelope keys the mailbox index on the id and serializes the - // date, so neither can be absent here. `deliver` refuses the same two on - // the same grounds. Checked before serializing so a message that cannot - // be stored is not encoded first and then discarded. - const { messageId, date } = message.headers; - if (messageId === undefined) { - throw new Error("Cannot append message: missing Message-ID header"); - } - if (date === undefined) { - throw new Error("Cannot append message: missing Date header"); - } - const raw = inboundMessageToRaw(message); - const envelope = { - messageId, - from: message.headers.from, - to: message.headers.to, - subject: message.headers.subject ?? "", - date: new Date(date), - inReplyTo: message.headers.inReplyTo, - references: message.headers.references ?? [], - interchangeType: message.headers.interchangeType, - interchangeCorrelationId: message.headers.interchangeCorrelationId, - }; - const uid = store.append(raw, envelope, flags ?? []); + const stored = inboundMessageToRaw(message); + const uid = store.append(stored.raw, stored.envelope, flags ?? []); return { uid, mailbox }; } @@ -716,7 +709,13 @@ class ScopedMessageTransport implements MessageTransport { return { expungedUids: toExpunge.map((m) => m.uid) }; } - watch(mailbox: string, callback: (event: MailboxEvent) => void): Unsubscribe { + async watch( + mailbox: string, + callback: (event: MailboxEvent) => void, + ): Promise { + // Refuse before registering, and register before this method awaits. + // A missing mailbox must not arm a callback, and an acceptance that + // later becomes async must not miss an event delivered in the gap. this.#requireMailbox(mailbox); let callbacks = this.#entry.watchCallbacks.get(mailbox); if (callbacks === undefined) { @@ -800,7 +799,41 @@ class ScopedMessageTransport implements MessageTransport { } } -function inboundMessageToRaw(message: InboundMessage): Uint8Array { +/** + * The bytes and envelope an `append` stores for an inbound message. + * + * `append` takes an `InboundMessage` and the mailbox stores RFC 2822 bytes, so + * this builds a minimal text/plain message from the headers plus `content` (or + * `JSON.stringify(payload)` when there is no content). Attachments are not + * copied into those bytes. The stored envelope keys the mailbox index on the + * id and serializes the date, so neither can be absent: both are refused + * before anything is encoded, on the same grounds `deliver` refuses them. + * A date string that does not parse is refused too. The index writes it with + * `toISOString`, and an Invalid Date would stay in the mirror after flush + * throws. + */ +export function inboundMessageToRaw(message: { + headers: MessageHeaders; + content?: string; + payload?: InboundMessage["payload"]; +}): { + raw: Uint8Array; + envelope: StoredEnvelope; +} { + const { messageId, date } = message.headers; + if (messageId === undefined) { + throw new Error("Cannot append message: missing Message-ID header"); + } + if (date === undefined) { + throw new Error("Cannot append message: missing Date header"); + } + const parsedDate = new Date(date); + if (Number.isNaN(parsedDate.getTime())) { + throw new Error( + `Cannot append message: invalid Date header ${JSON.stringify(date)}`, + ); + } + const enc = new TextEncoder(); const CRLF = "\r\n"; let headers = ""; @@ -842,5 +875,18 @@ function inboundMessageToRaw(message: InboundMessage): Uint8Array { (message.payload !== undefined ? JSON.stringify(message.payload) : ""); headers += `Content-Type: text/plain${CRLF}`; headers += `${CRLF}`; - return enc.encode(headers + body); + return { + raw: enc.encode(headers + body), + envelope: { + messageId, + from: message.headers.from, + to: message.headers.to, + subject: message.headers.subject ?? "", + date: parsedDate, + inReplyTo: message.headers.inReplyTo, + references: message.headers.references ?? [], + interchangeType: message.headers.interchangeType, + interchangeCorrelationId: message.headers.interchangeCorrelationId, + }, + }; } diff --git a/vendor/intx/mailbox/VENDORED-FROM b/vendor/intx/mailbox/VENDORED-FROM index 324e4d7395..6eb692adbf 100644 --- a/vendor/intx/mailbox/VENDORED-FROM +++ b/vendor/intx/mailbox/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mailbox) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mailbox/package.json b/vendor/intx/mailbox/package.json index a30f2a4e46..f72fe1edd1 100644 --- a/vendor/intx/mailbox/package.json +++ b/vendor/intx/mailbox/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mailbox", "description": "Storage-agnostic IMAP mailbox model with search, threading, and fetch projections", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,8 +14,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.4.0", - "@intx/log": "0.4.0", + "@intx/crypto": "0.5.0", + "@intx/log": "0.5.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx/mailbox/src/compose.ts b/vendor/intx/mailbox/src/compose.ts new file mode 100644 index 0000000000..11ea2a5103 --- /dev/null +++ b/vendor/intx/mailbox/src/compose.ts @@ -0,0 +1,193 @@ +// Composition of an `OutboundMessage` into signed RFC 2822 bytes. +// +// Signing and the stored envelope are properties of the message, not of the +// mailbox that delivers it. This module owns that half: the body checks, the +// signed content, the header set, and the References chain. The caller keeps +// delivery. +// +// It sits beside `verifyMimeSignature` deliberately: composing a signed +// message and verifying one are peers over the same byte contract. + +import { + assembleMessage, + assembleSignedContent, + createDetachedSignatureFromProvider, + generateMessageId, + isMessageId, + type ConversationContent, + type MessageHeaders as MimeMessageHeaders, + type StructuredContent, +} from "@intx/mime"; +import type { CryptoProvider, OutboundMessage } from "@intx/types/runtime"; +import { isConversationType } from "@intx/types/runtime"; + +import type { StoredEnvelope } from "./mailbox"; + +/** + * The composed message: the wire bytes, the id stamped into them, the + * pre-parsed envelope a mailbox stores beside them, and the address lists the + * caller routes on. `recipients` is the deduplicated union of `to` and `cc` -- + * the set a transport delivers to -- while `to` and `cc` are kept apart because + * they are separate header fields and separate audit facts. + */ +export type ComposedMessage = { + messageId: string; + rawBytes: Uint8Array; + envelope: StoredEnvelope; + to: string[]; + cc: string[]; + recipients: string[]; +}; + +/** + * Assemble the wire `References` chain for an outbound message. + * + * When the caller supplies a full ancestry (`existingReferences` -- the + * parent's References plus the parent's Message-Id, built by the threaded + * reply path), that chain is used and `inReplyTo` is appended only when it is + * not already the tail. Otherwise the chain is derived from `inReplyTo` alone. + * + * The caller-supplied chain is filtered to RFC 2822 message identifiers: + * inbound mail can carry a headerless-derived (sha256) or otherwise malformed + * Message-Id that is a valid claim-check key but not a valid `` + * identifier, and such a value must not leak into a `References` header. The + * `inReplyTo` value is appended without filtering, matching the behavior for a + * bare reply. + */ +function buildReferences( + inReplyTo: string | undefined, + existingReferences: string[] | undefined, +): string[] | undefined { + const refs = (existingReferences ?? []).filter(isMessageId); + if (inReplyTo === undefined) return refs.length > 0 ? refs : undefined; + if (!refs.includes(inReplyTo)) { + return [...refs, inReplyTo]; + } + return refs; +} + +/** + * Split a `to`/`cc` field that accepts either one address or several into a + * list. Returns a fresh array so a caller cannot alias the message's own. + */ +function addressList(field: string | string[] | undefined): string[] { + if (field === undefined) return []; + return Array.isArray(field) ? [...field] : [field]; +} + +/** + * Compose an `OutboundMessage` into signed RFC 2822 bytes. + * + * Throws when the message contradicts itself -- a conversation type carrying a + * structured payload, a structured type carrying text content or attachments, + * a blank `inReplyTo`, or no recipient at all. These are the conditions that + * make the message unsendable whatever the transport, so they are refused here + * rather than once per transport. + */ +export async function composeOutbound( + senderAddress: string, + message: OutboundMessage, + senderCrypto: CryptoProvider, +): Promise { + const to = addressList(message.to); + if (to.length === 0) { + throw new Error("OutboundMessage must have at least one recipient"); + } + const cc = addressList(message.cc); + const recipients = [...new Set([...to, ...cc])]; + + const isConversation = isConversationType(message.type); + if (isConversation && message.payload !== undefined) { + throw new Error( + "Conversation messages must not carry a structured payload", + ); + } + if (!isConversation && message.content !== undefined) { + throw new Error("Structured messages must not carry a text content field"); + } + if (!isConversation && message.attachments !== undefined) { + throw new Error("Structured messages must not carry attachments"); + } + + // RFC 5322 section 3.6.4 defines `In-Reply-To` as `1*msg-id`, so a blank + // value names nothing rather than naming a shorter parent. + if ( + message.inReplyTo !== undefined && + message.inReplyTo.trim().length === 0 + ) { + throw new Error( + "OutboundMessage inReplyTo, when provided, must name a message identifier", + ); + } + + const messageId = generateMessageId(senderAddress); + const now = new Date(); + + let content: ConversationContent | StructuredContent; + if (isConversation) { + const conversation: ConversationContent = { + kind: "conversation", + text: message.content ?? "", + }; + if (message.attachments !== undefined) { + conversation.attachments = message.attachments; + } + content = conversation; + } else { + const structured: StructuredContent = { + kind: "structured", + json: { type: message.type, version: "1", body: message.payload ?? {} }, + }; + if (message.summary !== undefined) structured.summary = message.summary; + content = structured; + } + + const signedContentBytes = assembleSignedContent(content); + const signatureBytes = await createDetachedSignatureFromProvider( + signedContentBytes, + senderCrypto, + ); + + const references = buildReferences(message.inReplyTo, message.references); + + const mimeHeaders: MimeMessageHeaders = { + from: senderAddress, + to, + cc: cc.length > 0 ? cc : undefined, + date: now, + messageId, + subject: message.subject, + inReplyTo: message.inReplyTo, + references, + mimeVersion: "1.0", + interchangeType: message.type, + interchangeCorrelationId: message.correlationId, + interchangeTenantId: message.tenantId, + interchangeAgentId: undefined, + interchangeSessionId: message.sessionId, + interchangeOfferingId: undefined, + interchangeSchemaVersion: undefined, + traceparent: undefined, + tracestate: undefined, + }; + + const rawBytes = assembleMessage( + mimeHeaders, + signedContentBytes, + signatureBytes, + ); + + const envelope: StoredEnvelope = { + messageId, + from: senderAddress, + to, + subject: message.subject ?? "", + date: now, + inReplyTo: message.inReplyTo, + references: references ?? [], + interchangeType: message.type, + interchangeCorrelationId: message.correlationId, + }; + + return { messageId, rawBytes, envelope, to, cc, recipients }; +} diff --git a/vendor/intx/mailbox/src/fetch.ts b/vendor/intx/mailbox/src/fetch.ts index b078549baa..645b30adbb 100644 --- a/vendor/intx/mailbox/src/fetch.ts +++ b/vendor/intx/mailbox/src/fetch.ts @@ -1,5 +1,6 @@ import { type } from "arktype"; import { getLogger } from "@intx/log"; +import { mimeTypeAndSubtype } from "@intx/types/attachments"; import type { MessageHeaders, BodyStructure, @@ -69,7 +70,10 @@ export async function fetchStructure( } /** - * Fetch a single MIME part by dot-separated path. `contentType` carries the + * Fetch a single MIME part by dot-separated path. `content` is the + * CTE-decoded bytes, matching `extractAttachments` / `decodeMail`. + * `contentType` is the type/subtype `decodeMail` reports — parameters such + * as charset are spent once the bytes are decoded — and it carries the * RFC 2045 §6.4 relabel, since an undecodable part arrives undecoded. */ export async function fetchPart( @@ -83,7 +87,9 @@ export async function fetchPart( const part = parseMimePart(partBytes); const result: MessagePart = { - contentType: reportedContentType(part.contentType, part.headers), + contentType: mimeTypeAndSubtype( + reportedContentType(part.contentType, part.headers), + ), content: decodePartBytes(part.body, part.headers), }; const mechanism = transferEncodingMechanism(part.headers); diff --git a/vendor/intx/mailbox/src/index.ts b/vendor/intx/mailbox/src/index.ts index 85c9b78865..06bc3c77f4 100644 --- a/vendor/intx/mailbox/src/index.ts +++ b/vendor/intx/mailbox/src/index.ts @@ -9,3 +9,5 @@ export { executeSearch } from "./search"; export { executeThread } from "./thread"; export { fetchHeaders, fetchStructure, fetchPart, fetchFull } from "./fetch"; export { verifyMimeSignature } from "./verify-signature"; +export { composeOutbound } from "./compose"; +export type { ComposedMessage } from "./compose"; diff --git a/vendor/intx/mime/VENDORED-FROM b/vendor/intx/mime/VENDORED-FROM index 1a208a9742..b2579bd96f 100644 --- a/vendor/intx/mime/VENDORED-FROM +++ b/vendor/intx/mime/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mime) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mime/package.json b/vendor/intx/mime/package.json index 94f172a824..affd58623a 100644 --- a/vendor/intx/mime/package.json +++ b/vendor/intx/mime/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mime", "description": "RFC 2822 message assembly and parsing with PGP detached signatures", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,7 +14,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.4.0", + "@intx/crypto": "0.5.0", "@intx/types": "workspace:*", "arktype": "catalog:" }, diff --git a/vendor/intx/mime/src/index.ts b/vendor/intx/mime/src/index.ts index b68ae8c2eb..64540e7700 100644 --- a/vendor/intx/mime/src/index.ts +++ b/vendor/intx/mime/src/index.ts @@ -13,6 +13,7 @@ export { parseMailToEmail, extractAttachments, buildMessageHeaders, + decodeHeaderText, decodeMail, decodePartBytes, isRecognizedTransferEncoding, diff --git a/vendor/intx/mime/src/mail-builder.ts b/vendor/intx/mime/src/mail-builder.ts index 6c6afad0c8..63f7402f2e 100644 --- a/vendor/intx/mime/src/mail-builder.ts +++ b/vendor/intx/mime/src/mail-builder.ts @@ -22,12 +22,12 @@ import type { MessageHeaders, MessageRef, OutboundMessage, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; import { InterchangeType, SignatureStatus, isConversationType, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; import { generateMessageId } from "./mime"; /** diff --git a/vendor/intx/mime/src/mime.ts b/vendor/intx/mime/src/mime.ts index 554be9ec18..043845a470 100644 --- a/vendor/intx/mime/src/mime.ts +++ b/vendor/intx/mime/src/mime.ts @@ -20,13 +20,13 @@ */ import { type } from "arktype"; -import { base64Decode, base64Encode } from "@intx/types"; +import { base64Decode, base64Encode } from "@intx/types/base64"; import type { MessageAttachment, MessageHeaders as ParsedMessageHeaders, MessagePart, -} from "@intx/types/runtime"; -import { InterchangeType } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { InterchangeType } from "@intx/types/runtime-core"; // --------------------------------------------------------------------------- // Types @@ -340,9 +340,311 @@ function generateBoundary(): string { const CRLF = "\r\n"; +/** + * RFC 5322 section 2.1.1 recommends no more than 78 characters per line and + * REQUIRES no more than 998. Folding targets the recommendation: a generator + * that aims at 998 produces lines no human and few mail clients display well, + * and leaves nothing in hand for a relay that prepends to a header. + */ +const FOLD_TARGET_OCTETS = 78; + +/** + * Insert folding whitespace so a line stays near `FOLD_TARGET_OCTETS`. + * + * Folds at a space that is already in the value, which is the only place RFC + * 5322 section 2.2.3 permits. The continuation carries exactly one leading + * space, and that space is one space from the value. `parseHeaders` replaces + * a fold plus every whitespace character after it with a single space, so a + * second leading space would not survive. + * + * Any earlier space in the same run stays on the line being folded, even when + * that carries the line past the target. A token with no space in it is + * emitted whole. There is no legal fold point inside a `msg-id` or an `atom`, + * so breaking one would corrupt it. + * + * A run that ends in a tab is not a fold point: the continuation can carry + * only one leading space, so the tab would be deleted or turned into a + * space. An earlier space that ends its own run is still a fold. Skipping + * it to keep the tab on this line can push the line past 998. + */ +function foldHeaderLine(name: string, value: string): string { + if (value.length === 0) return `${name}:${CRLF}`; + + const head = `${name}: `; + let out = head; + let column = head.length; + let index = 0; + + // Last index of the whitespace run that contains `wsIndex`. + const runLast = (wsIndex: number): number => { + let last = wsIndex; + while ( + last + 1 < value.length && + (value[last + 1] === " " || value[last + 1] === "\t") + ) { + last += 1; + } + return last; + }; + + // `wsIndex` is whitespace in `value`. The continuation can carry only one + // leading space, so the fold has to be a space that ends the run. A tab + // after that space would be swallowed with the leading whitespace, and a + // tab used as the fold itself would come back as a space. When the run + // ends in a tab and no earlier fold remains, the whole run stays here. + const foldOnRun = (wsIndex: number): void => { + const last = runLast(wsIndex); + if (value[last] !== " ") { + out += value.slice(index, last + 1); + column += last + 1 - index; + index = last + 1; + return; + } + out += value.slice(index, last); + column += last - index; + index = last + 1; + out += `${CRLF} `; + column = 1; + }; + + while (index < value.length) { + const room = FOLD_TARGET_OCTETS - column; + if (room > 0 && value.length - index <= room) { + out += value.slice(index); + break; + } + + // The last space in reach whose run ends in a space. A run that ends + // in a tab is skipped so an earlier space can still be the fold. + let breakAt = -1; + const limit = Math.max(0, Math.min(room, value.length - index)); + for (let offset = 0; offset < limit; offset += 1) { + const at = index + offset; + if (value[at] === " " && value[runLast(at)] === " ") breakAt = at; + } + + if (breakAt !== -1 || value[index] === " ") { + foldOnRun(breakAt === -1 ? index : breakAt); + continue; + } + + // No space between here and the target, and the next character is not a + // space. Emit the token whole rather than inventing a fold point. + const nextSpace = value.indexOf(" ", index); + const end = nextSpace === -1 ? value.length : nextSpace; + out += value.slice(index, end); + column += end - index; + index = end; + } + + return out + CRLF; +} + +/** + * Serialize a STRUCTURED header: an address list, a message id, a date, a + * content type. Folded, never encoded -- RFC 2047 encoded-words are not + * permitted in a structured field's tokens, so a non-ASCII address here needs + * SMTPUTF8 and IDNA rather than an encoding this function could apply. + */ function hdr(name: string, value: string): string { assertNoLineBreaks(value, `${name} header`); - return `${name}: ${value}${CRLF}`; + return foldHeaderLine(name, value); +} + +/** Bytes of UTF-8 per encoded-word. + * + * RFC 2047 section 2 caps an encoded-word at 75 characters. `=?UTF-8?B?` and + * `?=` spend 12 of them, leaving 63 for base64 -- so 45 input bytes, the + * largest multiple of 3 that fits, encode to 60 characters and keep every word + * inside the cap. A multiple of 3 also means no word carries base64 padding + * except the last. + */ +const ENCODED_WORD_PAYLOAD_BYTES = 45; + +function isAscii(value: string): boolean { + // eslint-disable-next-line no-control-regex + return !/[^\x00-\x7F]/.test(value); +} + +/** + * Encode `value` as RFC 2047 encoded-words when it is not already ASCII. + * + * Base64 (`B`) rather than quoted-printable (`Q`) for every value that needs + * encoding at all. `Q` produces a more readable wire form for a value that is + * mostly ASCII, but it needs a per-character escape table whose omissions are + * silent, and it offers nothing a receiver can do that `B` cannot. One code + * path that is always correct beats two where one is prettier. + * + * Words are split on a CHARACTER boundary, never inside a multi-byte sequence: + * RFC 2047 section 5 requires each encoded-word to decode independently, and a + * sequence straddling two of them decodes to a replacement character in both. + */ +function encodeHeaderText(value: string): string { + if (isAscii(value)) return value; + + const bytes = new TextEncoder().encode(value); + const words: string[] = []; + let start = 0; + + while (start < bytes.length) { + let end = Math.min(start + ENCODED_WORD_PAYLOAD_BYTES, bytes.length); + // Walk back off a continuation byte (10xxxxxx) so the cut lands between + // characters rather than inside one. + while (end > start && end < bytes.length) { + const byte = bytes[end]; + if (byte === undefined || (byte & 0xc0) !== 0x80) break; + end -= 1; + } + const chunk = bytes.subarray(start, end); + words.push(`=?UTF-8?B?${Buffer.from(chunk).toString("base64")}?=`); + start = end; + } + + // Adjacent encoded-words are joined by whitespace, which a receiver deletes + // (RFC 2047 section 6.2). `foldHeaderLine` then folds at those same spaces, + // which is why each word has to fit a line on its own. + return words.join(" "); +} + +/** + * Serialize an UNSTRUCTURED header: text a person reads, such as `Subject`. + * + * Encoded before folding, because encoding is what creates the fold points: a + * long non-ASCII subject has no spaces to fold at until it becomes a sequence + * of encoded-words. + */ +function hdrText(name: string, value: string): string { + assertNoLineBreaks(value, `${name} header`); + return foldHeaderLine(name, encodeHeaderText(value)); +} + +/** + * Decode RFC 2047 encoded-words in a header value that was received. + * + * Needed for interoperability rather than for our own mail: every sender that + * writes a non-ASCII subject encodes it, and a receiver that does not decode + * shows an agent `=?UTF-8?B?...?=` as the subject it is supposed to read. Both + * `B` and `Q` are decoded even though only `B` is produced here, because the + * choice belongs to whoever sent the message. + * + * Whitespace BETWEEN two encoded-words is deleted and whitespace elsewhere is + * kept, per RFC 2047 section 6.2. A word whose charset or encoding this cannot + * handle is left exactly as it arrived: showing the encoded form is a visible + * fault a reader can report, whereas substituting replacement characters looks + * like the sender's own text. + */ +export function decodeHeaderText(value: string): string { + const word = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g; + if (!word.test(value)) return value; + word.lastIndex = 0; + + let out = ""; + let cursor = 0; + let previousWasWord = false; + + for (let match = word.exec(value); match !== null; match = word.exec(value)) { + const whole = match[0]; + const charset = match[1]; + const encoding = match[2]; + const payload = match[3]; + if ( + charset === undefined || + encoding === undefined || + payload === undefined + ) { + // Unreachable: all three groups are mandatory in the pattern. Narrowed + // rather than asserted so a future pattern edit cannot make it a crash. + continue; + } + const gap = value.slice(cursor, match.index); + + // Whitespace separating two encoded-words is a separator, not content. + if (!(previousWasWord && gap.trim() === "")) out += gap; + + const decoded = decodeEncodedWord(charset, encoding, payload); + out += decoded ?? whole; + previousWasWord = decoded !== undefined; + cursor = match.index + whole.length; + } + + return out + value.slice(cursor); +} + +/** + * Decode a base64 payload only when every character is in the alphabet and + * the bytes re-encode to the same text. `Buffer` accepts illegal characters + * by ignoring them, which would turn a corrupt encoded-word into different + * text instead of leaving the word as it arrived. + */ +function decodeBase64Strict(payload: string): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(payload)) return undefined; + if (payload.length % 4 === 1) return undefined; + const bytes = Buffer.from(payload, "base64"); + const canonical = Buffer.from(bytes).toString("base64").replace(/=+$/, ""); + const given = payload.replace(/=+$/, ""); + if (canonical !== given) return undefined; + return new Uint8Array(bytes); +} + +function decodeEncodedWord( + charset: string, + encoding: string, + payload: string, +): string | undefined { + // RFC 2231 permits a language suffix (`utf-8*en`); the charset is the part + // before it. + const normalized = charset.toLowerCase().split("*")[0] ?? ""; + // Only the charsets decoded here. An unsupported one returns undefined so the + // caller keeps the encoded form rather than guessing at the bytes. + const utf8 = + normalized === "utf-8" || + normalized === "utf8" || + normalized === "us-ascii"; + const latin1 = normalized === "iso-8859-1" || normalized === "latin1"; + if (!utf8 && !latin1) return undefined; + + let bytes: Uint8Array; + if (encoding.toLowerCase() === "b") { + const decoded = decodeBase64Strict(payload); + if (decoded === undefined) return undefined; + bytes = decoded; + } else { + // Q: `_` is a space, `=XX` is a hex octet. Anything else is literal. + const octets: number[] = []; + for (let i = 0; i < payload.length; i += 1) { + const char = payload[i]; + if (char === undefined) break; + if (char === "_") { + octets.push(0x20); + continue; + } + if (char === "=" && i + 2 < payload.length) { + const hex = payload.slice(i + 1, i + 3); + if (/^[0-9A-Fa-f]{2}$/.test(hex)) { + octets.push(parseInt(hex, 16)); + i += 2; + continue; + } + } + octets.push(char.charCodeAt(0) & 0xff); + } + bytes = new Uint8Array(octets); + } + + // Latin-1 needs no decoder: its code points ARE its byte values, which is the + // one case where that identity holds. + if (latin1) { + return Array.from(bytes, (byte) => String.fromCharCode(byte)).join(""); + } + + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + // Bytes that are not valid UTF-8 despite the charset saying so. Keeping the + // encoded form reports the sender's fault; replacement characters would + // read as the sender's own text. + return undefined; + } } function serializeMessageHeaders( @@ -358,7 +660,9 @@ function serializeMessageHeaders( out += hdr("Date", formatRFC2822Date(h.date)); out += hdr("Message-ID", h.messageId); if (h.subject !== undefined) { - out += hdr("Subject", h.subject); + // The one unstructured header this assembler writes, and so the only one + // whose value is text a person chose rather than a token or an address. + out += hdrText("Subject", h.subject); } if (h.inReplyTo !== undefined) { out += hdr("In-Reply-To", h.inReplyTo); @@ -416,7 +720,9 @@ function assertNoLineBreaks(value: string, field: string): void { /** * A double quote would close the `filename="..."` form, and this serializer - * emits no escape for an inner quote. + * emits no escape for an inner quote. A non-ASCII octet is the same class of + * defect: these part headers sit inside the signed bytes and carry no + * transfer encoding, so an 8-bit header is one a relay may rewrite. */ function assertAttachmentHeaderSafe(value: string, field: string): void { assertNoLineBreaks(value, field); @@ -425,6 +731,164 @@ function assertAttachmentHeaderSafe(value: string, field: string): void { `${field} must not contain a double quote: ${JSON.stringify(value)}`, ); } + if (!isAscii(value)) { + throw new Error(`${field} must be US-ASCII: ${JSON.stringify(value)}`); + } +} + +// --------------------------------------------------------------------------- +// Text part transfer encoding (RFC 2045 §6.7) +// --------------------------------------------------------------------------- +// +// A text part is labelled `7bit` only when it IS 7-bit: every octet US-ASCII, +// and every line inside the RFC 5322 §2.1.1 limit. Anything else is +// quoted-printable. The distinction is load-bearing once the bytes reach a real +// relay: a part labelled `7bit` that carries 8-bit octets is rewritten by any +// hop that does not advertise 8BITMIME, and that rewrite changes the exact +// bytes the PGP/MIME signature covers, so the recipient reports `invalid`. +// Encoding here keeps the part 7-bit clean, so no hop has a reason to touch it. + +/** The longest line quoted-printable permits, including the soft-break `=`. */ +const QP_MAX_LINE = 76; + +/** RFC 5322 §2.1.1: a line MUST NOT exceed 998 characters, excluding CRLF. */ +const MAX_LINE_OCTETS = 998; + +function hexEscape(byte: number): string { + return `=${byte.toString(16).toUpperCase().padStart(2, "0")}`; +} + +/** + * Octets RFC 2045 §6.7 rule 2 permits literally: 33-60 and 62-126. The gap at + * 61 is `=`, which always escapes because it introduces an escape. + */ +function isQuotedPrintableLiteral(byte: number): boolean { + return (byte >= 33 && byte <= 60) || (byte >= 62 && byte <= 126); +} + +/** + * True when the bytes are already a legal `7bit` body: US-ASCII only, no bare + * CR or LF, no NUL, and no line over the RFC 5322 limit. The caller has already + * canonicalized line endings to CRLF, so a CR or LF outside a CRLF pair is a + * defect rather than a line break to measure. + */ +function isSevenBitClean(bytes: Uint8Array): boolean { + let lineOctets = 0; + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]!; + if (byte === 0x0d && bytes[i + 1] === 0x0a) { + i += 1; + lineOctets = 0; + continue; + } + if (byte === 0x00 || byte === 0x0d || byte === 0x0a || byte > 0x7f) { + return false; + } + lineOctets += 1; + if (lineOctets > MAX_LINE_OCTETS) return false; + } + return true; +} + +/** + * Quoted-printable encode one hard line. Space and tab ride literally except + * where they would end an encoded line. RFC 2045 §6.7 rule 3 requires an + * escape there: a relay that strips trailing whitespace would otherwise change + * the signed bytes. That includes the line a soft break closes, not only the + * last line of the input. + * + * Soft breaks land between whole tokens, never inside an `=XX` escape, so the + * escape cannot be split across a line. Escaping a run of spaces writes `=20` + * onto the line the break opened, which can fill that line; the token that + * caused the break is written only after it still fits. + */ +function encodeQuotedPrintableLine(bytes: Uint8Array): string { + let out = ""; + let column = 0; + + // Leave a column for the soft-break `=`. A token is one octet or a + // three-octet hex escape, so it always fits once `column` is 0. + const fits = (length: number): boolean => + column === 0 || column + length <= QP_MAX_LINE - 1; + + const softBreak = (): void => { + out += `=${CRLF}`; + column = 0; + }; + + const writeToken = (token: string): void => { + if (!fits(token.length)) { + const trailing: string[] = []; + while (out.endsWith(" ") || out.endsWith("\t")) { + const last = out.at(-1); + if (last === undefined) break; + trailing.push(last); + out = out.slice(0, -1); + column -= 1; + } + softBreak(); + for (const ws of trailing.reverse()) { + const escaped = ws === " " ? "=20" : "=09"; + if (!fits(escaped.length)) softBreak(); + out += escaped; + column += escaped.length; + } + } + // The escaped whitespace may have filled the line it was moved onto. + // That line ends in `0` or `9`, so one more break opens a fresh one. + if (!fits(token.length)) softBreak(); + out += token; + column += token.length; + }; + + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]!; + const atEnd = i === bytes.length - 1; + const token = + byte === 0x20 || byte === 0x09 + ? atEnd + ? hexEscape(byte) + : String.fromCharCode(byte) + : isQuotedPrintableLiteral(byte) + ? String.fromCharCode(byte) + : hexEscape(byte); + writeToken(token); + } + return out; +} + +function encodeQuotedPrintable(text: string): string { + const encoder = new TextEncoder(); + return text + .split(CRLF) + .map((line) => encodeQuotedPrintableLine(encoder.encode(line))) + .join(CRLF); +} + +/** + * Resolve a canonicalized text body to the transfer encoding that honestly + * describes it, and the body in that encoding. Pass `canonical` already + * CRLF-normalized with trailing whitespace stripped per line. + */ +function encodeTextBody(canonical: string): { + encoding: string; + body: string; +} { + if (isSevenBitClean(new TextEncoder().encode(canonical))) { + return { encoding: "7bit", body: canonical }; + } + return { + encoding: "quoted-printable", + body: encodeQuotedPrintable(canonical), + }; +} + +/** CRLF-normalize and strip trailing whitespace per line (MIME canonical form). */ +function canonicalizeText(text: string): string { + return text + .split(/\r\n|\r|\n/) + .map((line) => line.replace(/[ \t]+$/, "")) + .join(CRLF); } /** @@ -456,29 +920,39 @@ function assembleConversationSignedPart( ): Uint8Array { const boundary = generateBoundary(); - // Canonicalize the text part: CRLF line endings, strip trailing - // whitespace per line. - const lines = text.split(/\r\n|\r|\n/); - const canonLines = lines.map((l) => l.replace(/[ \t]+$/, "")); - const canonical = canonLines.join(CRLF); + const text7bit = encodeTextBody(canonicalizeText(text)); let body = `Content-Type: multipart/mixed; boundary="${boundary}"${CRLF}${CRLF}`; // Text part (BODY[1.1]) body += `--${boundary}${CRLF}`; body += `Content-Type: text/plain; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${text7bit.encoding}${CRLF}`; body += `${CRLF}`; - body += `${canonical}${CRLF}`; + body += `${text7bit.body}${CRLF}`; // Attachment parts (BODY[1.2..N]) for (const att of attachments) { assertAttachmentHeaderSafe(att.contentType, "attachment contentType"); assertAttachmentHeaderSafe(att.name, "attachment name"); + // A filename is one token, so folding cannot bring it back under the + // limit. Refuse it here rather than emit a line a relay must break. + const typeLine = `Content-Type: ${att.contentType}`; + const dispositionLine = `Content-Disposition: attachment; filename="${att.name}"`; + if (typeLine.length > MAX_LINE_OCTETS) { + throw new Error( + `attachment contentType header exceeds ${MAX_LINE_OCTETS} octets`, + ); + } + if (dispositionLine.length > MAX_LINE_OCTETS) { + throw new Error( + `attachment name header exceeds ${MAX_LINE_OCTETS} octets`, + ); + } body += `--${boundary}${CRLF}`; - body += `Content-Type: ${att.contentType}${CRLF}`; + body += `${typeLine}${CRLF}`; body += `Content-Transfer-Encoding: base64${CRLF}`; - body += `Content-Disposition: attachment; filename="${att.name}"${CRLF}`; + body += `${dispositionLine}${CRLF}`; body += `${CRLF}`; body += `${base64Lines(att.data)}${CRLF}`; } @@ -497,26 +971,28 @@ function assembleStructuredSignedPart( summary?: string, ): Uint8Array { const boundary = generateBoundary(); - const jsonStr = JSON.stringify(json); + // The payload is machine-read, so it is encoded rather than canonicalized: + // stripping trailing whitespace inside a JSON string literal would change the + // value, and JSON.stringify emits no bare CR or LF to normalize. + const jsonPart = encodeTextBody(JSON.stringify(json)); let body = `Content-Type: multipart/mixed; boundary="${boundary}"${CRLF}${CRLF}`; // JSON payload part body += `--${boundary}${CRLF}`; body += `Content-Type: application/vnd.interchange+json; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${jsonPart.encoding}${CRLF}`; body += `${CRLF}`; - body += `${jsonStr}${CRLF}`; + body += `${jsonPart.body}${CRLF}`; // Optional human-readable summary if (summary !== undefined) { + const summaryPart = encodeTextBody(canonicalizeText(summary)); body += `--${boundary}${CRLF}`; body += `Content-Type: text/plain; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${summaryPart.encoding}${CRLF}`; body += `${CRLF}`; - const lines = summary.split(/\r\n|\r|\n/); - const canonLines = lines.map((l) => l.replace(/[ \t]+$/, "")); - body += `${canonLines.join(CRLF)}${CRLF}`; + body += `${summaryPart.body}${CRLF}`; } body += `--${boundary}--${CRLF}`; @@ -1122,7 +1598,16 @@ function decodeBodyBytes( if (cte === "quoted-printable") { const raw = new TextDecoder("utf-8", { fatal: false }).decode(body); - return { value: decodeQuotedPrintable(raw), isEncodingProblem: false }; + // Decode to octets first, then read those octets as UTF-8. A + // quoted-printable escape names one OCTET, so a multi-byte character + // arrives as several escapes; reading each escape as a character instead + // yields one code point per octet, which renders as mojibake. + return { + value: new TextDecoder("utf-8", { fatal: false }).decode( + decodeQuotedPrintableBytes(raw), + ), + isEncodingProblem: false, + }; } if (cte === "7bit" || cte === "8bit" || cte === "binary") { @@ -1136,6 +1621,11 @@ function decodeBodyBytes( return { value: bytesToBinaryString(body), isEncodingProblem: true }; } +/** + * Undo quoted-printable into a string of one code unit per decoded OCTET. The + * result is an octet sequence widened into a string, not text: a caller that + * wants text decodes those octets under the part's charset. + */ function decodeQuotedPrintable(text: string): string { return text .replace(/=\r\n/g, "") @@ -1145,6 +1635,16 @@ function decodeQuotedPrintable(text: string): string { ); } +/** The octets a quoted-printable body carries, narrowed from the widened form. */ +function decodeQuotedPrintableBytes(text: string): Uint8Array { + const widened = decodeQuotedPrintable(text); + const out = new Uint8Array(widened.length); + for (let i = 0; i < widened.length; i++) { + out[i] = widened.charCodeAt(i); + } + return out; +} + /** * Determine whether a MIME part is an attachment based on Content-Disposition * and content type. @@ -1309,10 +1809,13 @@ export function parseMailToEmail(raw: Uint8Array, mailId: string): JMAPEmail { } } + // The agent reads this field as text. `decodeMail` keeps the wire form. + const rawSubject = msgHeaders.get("subject"); + return { from: parseAddressList(msgHeaders.get("from") ?? ""), to: parseAddressList(msgHeaders.get("to") ?? ""), - subject: msgHeaders.get("subject") ?? null, + subject: rawSubject === undefined ? null : decodeHeaderText(rawSubject), sentAt: parseDateHeader(msgHeaders.get("date")), bodyValues: ctx.bodyValues, textBody: ctx.textBody, @@ -1328,6 +1831,12 @@ export function parseMailToEmail(raw: Uint8Array, mailId: string): JMAPEmail { * Unlike `decodeBodyBytes` (which produces a JMAP string value), this returns * the actual bytes. A malformed base64 body throws; an unrecognized mechanism * does not — RFC 2045 §6.4 hands back the bytes as they came. + * + * Limitation: the quoted-printable branch reads the wire text as UTF-8 + * before undoing the escapes, so a raw 8-bit byte in a quoted-printable + * body (which RFC 2045 forbids) comes out as a replacement character + * truncated to one byte rather than the original octet. Escaped bytes + * (`=E9`, `=C3=A9`) round-trip exactly. */ export function decodePartBytes( body: Uint8Array, @@ -1342,12 +1851,7 @@ export function decodePartBytes( if (cte === "quoted-printable") { const raw = new TextDecoder("utf-8", { fatal: false }).decode(body); - const decoded = decodeQuotedPrintable(raw); - const out = new Uint8Array(decoded.length); - for (let i = 0; i < decoded.length; i++) { - out[i] = decoded.charCodeAt(i); - } - return out; + return decodeQuotedPrintableBytes(raw); } // Identity encoding, including anything unrecognized (RFC 2045 §6.4). @@ -1356,7 +1860,8 @@ export function decodePartBytes( /** * Extract conversation attachments from raw message bytes as - * `MessageAttachment[]` with decoded payloads. + * `MessageAttachment[]` with decoded payloads and the IMAP part path + * of each sibling (`part`), matching `extractPartByPath` numbering. * * The conversation signed content is a multipart/mixed whose first part is * the text body and whose remaining attachment parts (Content-Disposition: @@ -1388,7 +1893,10 @@ export function extractAttachments(raw: Uint8Array): MessageAttachment[] { if (innerBoundary === undefined) return []; const attachments: MessageAttachment[] = []; - for (const subPartBytes of parseMultipart(signed.body, innerBoundary)) { + for (const [index, subPartBytes] of parseMultipart( + signed.body, + innerBoundary, + ).entries()) { const subPart = parseMimePart(subPartBytes); if (!isAttachmentPart(subPart.contentType, subPart.headers)) continue; attachments.push({ @@ -1398,6 +1906,7 @@ export function extractAttachments(raw: Uint8Array): MessageAttachment[] { subPart.headers, ), data: decodePartBytes(subPart.body, subPart.headers), + part: `1.${String(index + 1)}`, }); } return attachments; @@ -1472,8 +1981,11 @@ export function buildMessageHeaders( result.inReplyTo = inReplyTo; } + // Decoded, because a subject is text to read: a sender that wrote a + // non-ASCII one encoded it, and the agent reading this field wants the + // characters rather than their RFC 2047 transport form. const subject = headers.get("subject"); - if (subject !== undefined) result.subject = subject; + if (subject !== undefined) result.subject = decodeHeaderText(subject); const listId = headers.get("list-id"); if (listId !== undefined) result.listId = listId; @@ -1545,7 +2057,13 @@ function parseRawHeaders( if (line === "") continue; // Nothing to continue means no field (RFC 2822 2.2), not a field of its own. if (line.startsWith(" ") || line.startsWith("\t")) { - if (current !== null) current.value += ` ${line.trim()}`; + if (current !== null) { + // The leading whitespace is the fold, and unfolding replaces that + // run with one space. Trailing whitespace is content: trimming it + // would drop a space the fold left at the end of the line. + const body = line.replace(/^[ \t]+/, ""); + current.value += ` ${body}`; + } continue; } const idx = line.indexOf(":"); diff --git a/vendor/intx/types/README.md b/vendor/intx/types/README.md index 1ffe2e90cd..07823c3e89 100644 --- a/vendor/intx/types/README.md +++ b/vendor/intx/types/README.md @@ -6,9 +6,9 @@ wire frames. Nearly every other package imports from here, which makes this the canonical home for any shape that crosses a package boundary. -Each entry point pairs an ArkType validator with its inferred -TypeScript type so consumers can validate at the boundary and -trust the resulting value internally. +Runtime validators expose their inferred TypeScript types so consumers can +validate at the boundary and trust the resulting value internally. Dedicated +helper entry points avoid loading unrelated schema collections. ## Surface @@ -23,6 +23,17 @@ pull in the shapes they need: discovery view), observability, sidecar status enums (distinct from the wire frames under `@intx/types/sidecar` below), run addresses, hex and base64 helpers, and the `hasCode` error guard. +- `@intx/types/hex` — hexadecimal byte encoding and decoding. +- `@intx/types/posix-path` — pure POSIX path helpers (`normalize`, + `resolve`, `join`, `relative`) for code that cannot import `node:path`. +- `@intx/types/message-id` — canonical Message-ID parsing and derivation. +- `@intx/types/credential-cipher` — credential cipher contracts and the + authenticated-data helpers for sealed credentials and signing keys. +- `@intx/types/grants` — grant creation, update, and evaluation schemas, + grant requirements, approval items, and effect/origin enums. +- `@intx/types/adapter-manifest` — the operator-declared custom inference + adapter manifest schema and `parseAdapterManifestEnv`, the reader for + `SIDECAR_ADAPTER_MANIFEST`. - `@intx/types/authz` — grant rules, condition contexts, and authorization result shapes shared between `@intx/authz` and the hub. @@ -30,14 +41,34 @@ pull in the shapes they need: `ErrorRecord` shapes for tool-authorization and error records. - `@intx/types/content-type` — `detectResponseKind`, which classifies a response's `Headers` as an SSE stream or a JSON body. -- `@intx/types/runtime` — inference and harness contracts: +- `@intx/types/runtime` — aggregate entry for runtime contracts and + inference-event wire schemas. +- `@intx/types/runtime-core` — inference and harness contracts: `ContextStore`, `ToolRunner`, `ToolDefinition`, `AuditStore`, `InferenceSource` (the resolved provider/model/credential a call - executes against), retry policy, director and reactor types. + executes against), retry policy, director and reactor types. Use this + lighter entry in child processes that need runtime contracts without + constructing the inference-event union. +- `@intx/types/inference-events` — the `InferenceEvent` wire validator, + its discriminated TypeScript type, and `parseInferenceEvent`. +- `@intx/types/inbound-mail-policy` — `InboundMailOutcome`, + `AuthorControllableOutcome`, and the authored `InboundMailPolicy` schema. +- `@intx/types/format-safety-rating` — `formatSafetyRatingText`, the plain-text + rendering helper, without runtime schema initialization. - `@intx/types/runtime-capabilities` — the capability-registry contract harness extensions resolve against (e.g. mail transport, blob reader). - `@intx/types/sidecar` — hub-sidecar WebSocket wire frames. +- `@intx/types/credential-delivery` — credential material and binding + descriptors delivered to running workflow children. +- `@intx/types/repo` — `RepoKind`, `RepoAction`, and `RepoId` without the + sidecar frame schemas. +- `@intx/types/workflow-definition` — `workflowDefinitionEnvelopeSchema`, + the structural workflow envelope validator used by definition loaders + and the repository substrate. +- `@intx/types/grant-snapshot` — `GrantWalkSnapshot`, the per-step grant + declarations and workflow grant requirements serialized for deployment, + and `serializeGrantWalk`, which builds it from a capability walk. - `@intx/types/grant-wire` — grant-update wire frames pushed from the hub to the sidecar. - `@intx/types/tool-packages` — schemas for the tool-package @@ -59,4 +90,16 @@ pull in the shapes they need: - `@intx/types/workflow-sources` — schemas for where a code-sourced workflow definition's bytes come from: `WorkflowDefinitionRegistrySource`, `WorkflowDefinitionAssetSource` (with its `tarball` and `source` - `package` arms), unioned as `WorkflowDefinitionSource`. + `package` arms), unioned as `WorkflowDefinitionSource`, and the shared + `workflowSourceAssetMountPath` layout helper. +- `@intx/types/attachments` — MIME classification, attachment categories, + and attachment size/count limits. +- `@intx/types/signals` — signal kinds, approval decisions, and correlation + signal-name helpers. +- `@intx/types/agent-address` — formatting and parsing run mail addresses. +- `@intx/types/workflow-run-id` — `deriveWorkflowRunId`, which extracts a + deployment's top-level run ID from its mail address and throws on an + invalid address. +- `@intx/types/base64` — byte encoding and decoding. +- `@intx/types/concat` — `concatBytes` for combining byte arrays. +- `@intx/types/has-code` — the `hasCode` error guard. diff --git a/vendor/intx/types/VENDORED-FROM b/vendor/intx/types/VENDORED-FROM index 7c751ad83b..a623dd3500 100644 --- a/vendor/intx/types/VENDORED-FROM +++ b/vendor/intx/types/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/types) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-7443 retires CL-7190's local delta ("message_response" added to signals.ts's signalKinds, alongside signalKindToGateType): `ask_user` posts its question and ends the turn instead of parking a signal, so signals.ts is back to upstream's approval-only shape. CL-6396: AgentEventFrame gains an optional childRunId so a sidecar can name the occurrence (turn__) the event belongs to. CL-7510: catalog.ts gains the "openai-responses" model-provider plugin id, which the workbench loopback-OAuth providers (Codex, xai-oauth) ride. CL-7508: sidecar.ts hosts the sidecar loopback-login token shape (OAuthLoginTokens: access/refresh/expiresAt/idToken/accountId). Absorbed upstream at this re-vendor: the message-id.ts blank-line handling, the runtime.ts CONVERSATION_TYPES/isConversationType enumeration, and the tenants.ts TenantSlug validation. +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: AgentEventFrame gains an optional childRunId so a sidecar can name the occurrence (turn__) the event belongs to. CL-7510: catalog.ts gains the "openai-responses" model-provider plugin id, which the workbench loopback-OAuth providers (Codex, xai-oauth) ride. CL-10976: runtime.ts re-exports formatSafetyRatingText from ./format-safety-rating until @corbits/openai-responses switches its import to @intx/types/format-safety-rating. Retired at this pin: the CL-7508 OAuthLogin* frames in sidecar.ts (unused; OAuth login runs in-process on the hub). diff --git a/vendor/intx/types/package.json b/vendor/intx/types/package.json index 8f5a0ae819..6202548885 100644 --- a/vendor/intx/types/package.json +++ b/vendor/intx/types/package.json @@ -1,7 +1,7 @@ { "name": "@intx/types", "description": "Runtime validators, API contract types, runtime interfaces, and sidecar wire frames for Interchange", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -52,6 +52,94 @@ "./workflow-sources": { "types": "./src/workflow-sources.ts", "default": "./src/workflow-sources.ts" + }, + "./testing": { + "types": "./src/testing/index.ts", + "default": "./src/testing/index.ts" + }, + "./inbound-mail-policy": { + "types": "./src/inbound-mail-policy.ts", + "default": "./src/inbound-mail-policy.ts" + }, + "./workflow-definition": { + "types": "./src/workflow-definition.ts", + "default": "./src/workflow-definition.ts" + }, + "./format-safety-rating": { + "types": "./src/format-safety-rating.ts", + "default": "./src/format-safety-rating.ts" + }, + "./hex": { + "types": "./src/hex.ts", + "default": "./src/hex.ts" + }, + "./posix-path": { + "types": "./src/posix-path.ts", + "default": "./src/posix-path.ts" + }, + "./grant-snapshot": { + "types": "./src/grant-snapshot.ts", + "default": "./src/grant-snapshot.ts" + }, + "./credential-delivery": { + "types": "./src/credential-delivery.ts", + "default": "./src/credential-delivery.ts" + }, + "./repo": { + "types": "./src/repo.ts", + "default": "./src/repo.ts" + }, + "./runtime-core": { + "types": "./src/runtime-core.ts", + "default": "./src/runtime-core.ts" + }, + "./inference-events": { + "types": "./src/inference-events.ts", + "default": "./src/inference-events.ts" + }, + "./attachments": { + "types": "./src/attachments.ts", + "default": "./src/attachments.ts" + }, + "./base64": { + "types": "./src/base64.ts", + "default": "./src/base64.ts" + }, + "./signals": { + "types": "./src/signals.ts", + "default": "./src/signals.ts" + }, + "./agent-address": { + "types": "./src/agent-address.ts", + "default": "./src/agent-address.ts" + }, + "./has-code": { + "types": "./src/has-code.ts", + "default": "./src/has-code.ts" + }, + "./concat": { + "types": "./src/concat.ts", + "default": "./src/concat.ts" + }, + "./workflow-run-id": { + "types": "./src/workflow-run-id.ts", + "default": "./src/workflow-run-id.ts" + }, + "./message-id": { + "types": "./src/message-id.ts", + "default": "./src/message-id.ts" + }, + "./credential-cipher": { + "types": "./src/credential-cipher.ts", + "default": "./src/credential-cipher.ts" + }, + "./grants": { + "types": "./src/grants.ts", + "default": "./src/grants.ts" + }, + "./adapter-manifest": { + "types": "./src/adapter-manifest.ts", + "default": "./src/adapter-manifest.ts" } }, "scripts": { diff --git a/vendor/intx/types/src/adapter-manifest.ts b/vendor/intx/types/src/adapter-manifest.ts new file mode 100644 index 0000000000..cd2ff1229c --- /dev/null +++ b/vendor/intx/types/src/adapter-manifest.ts @@ -0,0 +1,43 @@ +import { type } from "arktype"; + +import { ModelProviderPlugin } from "./catalog"; + +// Describes one custom adapter: the provider identifier it serves, the module +// specifier to import, and the named export within that module to use as its +// factory. Specifiers are operator-config-only and resolve to arbitrary code +// via `import()`, so they must originate solely from trusted operator +// configuration, never from tenant or deploy data. The shape is validated at +// every deserialization boundary; the value is trusted operator input. +export const AdapterManifestEntry = type({ + provider: ModelProviderPlugin, + specifier: "string", + export: "string", +}); +export type AdapterManifestEntry = typeof AdapterManifestEntry.infer; + +export const AdapterManifest = AdapterManifestEntry.array(); +export type AdapterManifest = typeof AdapterManifest.infer; + +// Parses the raw `SIDECAR_ADAPTER_MANIFEST` environment value. Absent or +// whitespace-only yields `undefined`; the caller decides what an omitted +// declaration means at its own boundary. A present value must be a JSON +// array of manifest entries, and anything else throws naming the variable +// so a typo fails the boot rather than silently dropping adapters. +export function parseAdapterManifestEnv( + raw: string | undefined, +): AdapterManifest | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + throw new Error("SIDECAR_ADAPTER_MANIFEST is not valid JSON", { cause }); + } + const validated = AdapterManifest(parsed); + if (validated instanceof type.errors) { + throw new Error( + `SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, + ); + } + return validated; +} diff --git a/vendor/intx/types/src/agent-address.ts b/vendor/intx/types/src/agent-address.ts index 56610d931e..b08b00a696 100644 --- a/vendor/intx/types/src/agent-address.ts +++ b/vendor/intx/types/src/agent-address.ts @@ -5,11 +5,6 @@ // The shape of the right-hand side of the "@" is not validated beyond the // requirement that it be non-empty: tightening the contract (DNS-ish // validation, normalisation, etc.) is a separate follow-up. -// -// `@intx/hub-sessions`'s `parseAgentId` is the canonical throwing wrapper -// over `parseRunAddress` — call it when a `null` return would -// propagate as a silent bug, and keep this parser's `null` return -// reserved for callers that already have a structured fallback. const RUN_PREFIX = "run_"; diff --git a/vendor/intx/types/src/attachment-validation.ts b/vendor/intx/types/src/attachment-validation.ts new file mode 100644 index 0000000000..d8db5a9bb3 --- /dev/null +++ b/vendor/intx/types/src/attachment-validation.ts @@ -0,0 +1,201 @@ +import { + isAllowedMimeType, + MAX_ATTACHMENTS_PER_MESSAGE, + mimeTypeAndSubtype, + PER_ATTACHMENT_LIMIT_BYTES, + PER_MESSAGE_TOTAL_LIMIT_BYTES, +} from "./attachments"; +import { base64Decode } from "./base64"; +import type { MessageAttachment } from "./runtime"; +import type { AttachmentError } from "./sessions"; + +/** + * A single attachment as it arrives at a boundary: a MIME type, the bytes, + * and an optional filename. A string `data` is base64 (the request-body + * form); a caller that already holds the bytes passes them as-is. + */ +export type AttachmentInput = { + mimeType: string; + data: string | Uint8Array; + name?: string; +}; + +/** + * The attachment policy the route validates against. Defaults to the + * system-level allowlist and limits; a future per-agent or per-workflow + * lookup substitutes a narrowed policy here (the injection seam — no + * validation logic changes). + */ +export type AttachmentPolicy = { + isAllowed: (mimeType: string) => boolean; + perAttachmentLimitBytes: number; + perMessageTotalLimitBytes: number; + maxAttachments: number; +}; + +export const DEFAULT_ATTACHMENT_POLICY: AttachmentPolicy = { + isAllowed: isAllowedMimeType, + perAttachmentLimitBytes: PER_ATTACHMENT_LIMIT_BYTES, + perMessageTotalLimitBytes: PER_MESSAGE_TOTAL_LIMIT_BYTES, + maxAttachments: MAX_ATTACHMENTS_PER_MESSAGE, +}; + +// The error shape is the wire contract `AttachmentError` from @intx/types: +// a machine-actionable `code`, the fields needed to locate the rejection, +// and a human-readable `message`. Defining it once in @intx/types lets the +// route document the structured 400 in its OpenAPI surface. +export type AttachmentValidationError = AttachmentError; + +export type AttachmentValidationResult = + | { ok: true; attachments: MessageAttachment[] } + | { ok: false; error: AttachmentError }; + +function compactDecodedSize(compact: string): number { + const pad = compact.endsWith("==") ? 2 : compact.endsWith("=") ? 1 : 0; + return Math.floor(compact.length / 4) * 3 - pad; +} + +function compactBase64(data: string): string { + return data.replace(/\s+/g, ""); +} + +function decode(data: string | Uint8Array): Uint8Array | null { + if (typeof data !== "string") return data; + try { + return base64Decode(data); + } catch { + return null; + } +} + +/** + * Validate and decode attachments against a policy at either boundary + * (mail tools or the request body). + * + * The attachment count is checked first, before anything is decoded. + * Encoded base64 is size-checked from its compact length (whitespace + * stripped, padding counted) before decode, and the first error returns + * without decoding later entries. Remaining checks, in encounter order: + * encoded oversize, malformed base64, decoded per-attachment oversize, + * disallowed MIME type, invalid name. After every attachment passes, the + * per-message total is checked. On success the decoded + * `MessageAttachment[]` is returned with names defaulted to + * `attachment-{index}` by input position. + */ +export function validateAttachments( + inputs: readonly AttachmentInput[], + policy: AttachmentPolicy = DEFAULT_ATTACHMENT_POLICY, +): AttachmentValidationResult { + if (inputs.length > policy.maxAttachments) { + return { + ok: false, + error: { + code: "too_many_attachments", + message: `${inputs.length} attachments, over the limit of ${policy.maxAttachments}`, + count: inputs.length, + limit: policy.maxAttachments, + }, + }; + } + + const decoded: MessageAttachment[] = []; + + for (const [index, input] of inputs.entries()) { + let payload: string | Uint8Array; + if (typeof input.data === "string") { + payload = compactBase64(input.data); + const encodedSize = compactDecodedSize(payload); + if (encodedSize > policy.perAttachmentLimitBytes) { + return { + ok: false, + error: { + code: "oversize_attachment", + message: `attachment ${index} is ${encodedSize} bytes, over the ${policy.perAttachmentLimitBytes}-byte limit`, + attachmentIndex: index, + byteLength: encodedSize, + limitBytes: policy.perAttachmentLimitBytes, + }, + }; + } + } else { + payload = input.data; + } + + const bytes = decode(payload); + if (bytes === null) { + return { + ok: false, + error: { + code: "malformed_base64", + message: `attachment ${index} is not valid base64`, + attachmentIndex: index, + }, + }; + } + if (bytes.length > policy.perAttachmentLimitBytes) { + return { + ok: false, + error: { + code: "oversize_attachment", + message: `attachment ${index} is ${bytes.length} bytes, over the ${policy.perAttachmentLimitBytes}-byte limit`, + attachmentIndex: index, + byteLength: bytes.length, + limitBytes: policy.perAttachmentLimitBytes, + }, + }; + } + const mimeType = mimeTypeAndSubtype(input.mimeType); + if (!policy.isAllowed(mimeType)) { + return { + ok: false, + error: { + code: "disallowed_mime_type", + message: `attachment ${index} has unsupported content type "${input.mimeType}"`, + attachmentIndex: index, + mimeType: input.mimeType, + }, + }; + } + // A user-supplied name becomes the MIME part's quoted filename, so it + // must not contain characters that would break out of the header + // (line breaks or a double quote). Empty and whitespace-only names + // cannot round-trip through `filename=""`. The default name is always + // safe. + if ( + input.name !== undefined && + (input.name.trim() === "" || /[\r\n"]/.test(input.name)) + ) { + const empty = input.name.trim() === ""; + return { + ok: false, + error: { + code: "invalid_attachment_name", + message: empty + ? `attachment ${index} has an empty name` + : `attachment ${index} has a name with invalid characters (no quotes or line breaks)`, + attachmentIndex: index, + }, + }; + } + decoded.push({ + name: input.name ?? `attachment-${index}`, + contentType: mimeType, + data: bytes, + }); + } + + const totalBytes = decoded.reduce((sum, a) => sum + a.data.length, 0); + if (totalBytes > policy.perMessageTotalLimitBytes) { + return { + ok: false, + error: { + code: "oversize_total", + message: `attachments total ${totalBytes} bytes, over the ${policy.perMessageTotalLimitBytes}-byte limit`, + totalBytes, + limitBytes: policy.perMessageTotalLimitBytes, + }, + }; + } + + return { ok: true, attachments: decoded }; +} diff --git a/vendor/intx/types/src/attachments.ts b/vendor/intx/types/src/attachments.ts index ffae817061..97f05952fe 100644 --- a/vendor/intx/types/src/attachments.ts +++ b/vendor/intx/types/src/attachments.ts @@ -38,10 +38,20 @@ export const ATTACHMENT_ALLOWLIST = { export type AllowedMimeType = keyof typeof ATTACHMENT_ALLOWLIST; -export function isAllowedMimeType( - mimeType: string, -): mimeType is AllowedMimeType { - return mimeType in ATTACHMENT_ALLOWLIST; +const ALLOWLIST_BY_TYPE: Record = + ATTACHMENT_ALLOWLIST; + +/** + * The type/subtype of a Content-Type, ignoring parameters and case. + * Allowlist identity is this token, not the raw header value. + */ +export function mimeTypeAndSubtype(contentType: string): string { + const [mimeType = ""] = contentType.split(";"); + return mimeType.trim().toLowerCase(); +} + +export function isAllowedMimeType(mimeType: string): boolean { + return Object.hasOwn(ALLOWLIST_BY_TYPE, mimeTypeAndSubtype(mimeType)); } /** @@ -53,10 +63,22 @@ export function isAllowedMimeType( export function attachmentCategory( mimeType: string, ): AttachmentCategory | undefined { - if (isAllowedMimeType(mimeType)) { - return ATTACHMENT_ALLOWLIST[mimeType]; - } - return undefined; + return ALLOWLIST_BY_TYPE[mimeTypeAndSubtype(mimeType)]; +} + +/** + * True when a Content-Type names readable text: `text/*`, JSON, and the + * `+json` structured suffix. Callers use this to decide whether an + * attachment's content is carried as text or as opaque bytes. Any + * Content-Type parameters (e.g. `; charset=utf-8`) are ignored. + */ +export function isTextLikeMimeType(contentType: string): boolean { + const mime = mimeTypeAndSubtype(contentType); + return ( + mime.startsWith("text/") || + mime === "application/json" || + mime.endsWith("+json") + ); } // Default size limits, on decoded bytes. These are the system-level @@ -64,3 +86,7 @@ export function attachmentCategory( // limit that defaults to these. export const PER_ATTACHMENT_LIMIT_BYTES = 10 * 1024 * 1024; export const PER_MESSAGE_TOTAL_LIMIT_BYTES = 30 * 1024 * 1024; +// Cap on attachment count. The byte limits measure decoded payload, so +// without this a request could carry an unbounded number of empty +// attachments, each of which becomes a MIME part. +export const MAX_ATTACHMENTS_PER_MESSAGE = 20; diff --git a/vendor/intx/types/src/credential-delivery.ts b/vendor/intx/types/src/credential-delivery.ts new file mode 100644 index 0000000000..de62831cc3 --- /dev/null +++ b/vendor/intx/types/src/credential-delivery.ts @@ -0,0 +1,38 @@ +import { type } from "arktype"; + +/** + * The decrypted credential material and per-handle binding descriptors + * delivered to a running agent so its consumers -- tools and action handlers -- + * can use provider-backed credentials. Secrets are decrypted hub-side and ride + * this payload on the live channel ONLY -- the deploy frame at launch, a + * `credentials.update` frame on rotation, and the child's in-memory cell. They + * are NEVER written to disk (they do not ride the git-committed grants file) + * and NEVER copied into any snapshot, event, or state -- redaction is by + * construction, mirroring how an `InferenceSource`'s `apiKey` stays off every + * egress type. + * + * `materials` is keyed by `credentialId` (a credential can back several handles, + * so its secret is stored once); `bindings` maps each declared handle to the + * credential that backs it and the consumer identity allowed to use it. + */ +export const CredentialMaterialEntry = type({ + credentialId: "string", + providerKey: "string", + origin: "string", + secret: "string", +}); +export type CredentialMaterialEntry = typeof CredentialMaterialEntry.infer; + +export const CredentialBindingDescriptor = type({ + handle: "string", + credentialId: "string", + consumer: "string", +}); +export type CredentialBindingDescriptor = + typeof CredentialBindingDescriptor.infer; + +export const CredentialDelivery = type({ + bindings: CredentialBindingDescriptor.array(), + materials: CredentialMaterialEntry.array(), +}); +export type CredentialDelivery = typeof CredentialDelivery.infer; diff --git a/vendor/intx/types/src/credentials.ts b/vendor/intx/types/src/credentials.ts index aad64b732d..be517c1bbd 100644 --- a/vendor/intx/types/src/credentials.ts +++ b/vendor/intx/types/src/credentials.ts @@ -30,9 +30,10 @@ const CredType = type.enumerated(...credentialTypes); const CredStatus = type.enumerated(...credentialStatuses); const CredentialSourceType = type.enumerated(...credentialRequirementSources); -// A credential binding on a workflow definition maps a tool package's declared -// credential handle -- keyed `(package, handle)` against the tool-package -// declaration -- to a concrete credential resolved fresh at launch. `locator` +// A credential binding on a workflow definition maps one consumer's declared +// credential handle to a concrete credential resolved fresh at launch. The +// consumer is a tool package, keyed `(package, handle)`, or an action handler, +// keyed `(action, handle)`; a binding names exactly one of the two. `locator` // is which credential namespace the name is resolved in; today only `tenant` // exists (a tenant-owned credential, authorized by ownership). A second locator // that resolves a principal-owned credential -- and the delegation authority @@ -43,12 +44,9 @@ export type CredentialBindingLocator = const BindingLocator = type.enumerated(...credentialBindingLocators); -export const CredentialBinding = type({ - package: type("string").describe( - "The tool package the declared handle belongs to; matches the resolved manifest's top-level package name.", - ), +const credentialBindingFields = { handle: ToolCredentialHandle.describe( - "The credential handle the tool package declared; unique within its package.", + "The credential handle the consumer declared; unique within its consumer.", ), provider: type("string").describe( "The provider the bound credential resolves against.", @@ -59,6 +57,26 @@ export const CredentialBinding = type({ locator: BindingLocator.describe( "Which credential namespace the binding resolves the credential in. `tenant` resolves a tenant-owned credential by provider/name through the tenant walk-up; its use is authorized by tenant ownership.", ), +} as const; + +// Each branch forbids the other branch's key, so an object that names both +// consumers fails validation instead of matching the first branch with the +// extra key ignored. The forbidden key is typed `never`, which reports only +// "never" for such an object. `undefined` reports a clearer message, but it +// makes `toJsonSchema()` throw, and the API docs generator converts every +// exported type. +export const CredentialBinding = type({ + package: type("string").describe( + "The tool package the declared handle belongs to: the tool factory's bundle id.", + ), + "action?": "never", + ...credentialBindingFields, +}).or({ + action: type("string > 0").describe( + "The action handler the declared handle belongs to: the handler's export name in the workflow's `interchange.actions` module.", + ), + "package?": "never", + ...credentialBindingFields, }); export type CredentialBinding = typeof CredentialBinding.infer; diff --git a/vendor/intx/types/src/format-safety-rating.ts b/vendor/intx/types/src/format-safety-rating.ts new file mode 100644 index 0000000000..5369f82000 --- /dev/null +++ b/vendor/intx/types/src/format-safety-rating.ts @@ -0,0 +1,11 @@ +import type { SafetyRatingBlock } from "./runtime-core"; + +/** + * Human-readable rendering of a SafetyRatingBlock for reply text, + * timeline summaries, and request-history rewrites when a provider + * has no input wire shape for safety_rating. Single owner of the + * display string so reply / history / transform stay in lockstep. + */ +export function formatSafetyRatingText(block: SafetyRatingBlock): string { + return `Request blocked: ${block.blockReason}`; +} diff --git a/vendor/intx/types/src/grant-snapshot.ts b/vendor/intx/types/src/grant-snapshot.ts index d8b304ff64..d1cf7c607f 100644 --- a/vendor/intx/types/src/grant-snapshot.ts +++ b/vendor/intx/types/src/grant-snapshot.ts @@ -17,7 +17,7 @@ import { type } from "arktype"; -import { grantEffects, GrantRequirement } from "./grants"; +import { grantEffects, GrantRequirement, type GrantEffect } from "./grants"; const Effect = type.enumerated(...grantEffects); @@ -35,3 +35,35 @@ export const GrantWalkSnapshot = type({ }); export type GrantWalkSnapshot = typeof GrantWalkSnapshot.infer; + +/** + * Serialize the un-flattened capability walk into a plain-data + * `GrantWalkSnapshot`: the per-step grant declarations (each step's grant + * strings plus its tool-grant `grantEffects` map, converted from the + * walk's `Map` to a plain object) and the definition's full, unfiltered + * `grantRequirements`. This preserves the per-step grouping and the effect + * data a flattened grant set discards. A definition that declares no + * requirements snapshots an empty list. + */ +export function serializeGrantWalk( + walk: { + readonly perStep: ReadonlyMap< + string, + { + readonly grants: readonly string[]; + readonly grantEffects: ReadonlyMap; + } + >; + }, + grantRequirements: readonly GrantRequirement[] | undefined, +): GrantWalkSnapshot { + const perStep = [...walk.perStep].map(([stepId, declarations]) => ({ + stepId, + grants: [...declarations.grants], + grantEffects: Object.fromEntries(declarations.grantEffects), + })); + return { + perStep, + grantRequirements: [...(grantRequirements ?? [])], + }; +} diff --git a/vendor/intx/types/src/inbound-mail-policy.ts b/vendor/intx/types/src/inbound-mail-policy.ts new file mode 100644 index 0000000000..f65467aef9 --- /dev/null +++ b/vendor/intx/types/src/inbound-mail-policy.ts @@ -0,0 +1,91 @@ +import { type } from "arktype"; + +/** + * The admission outcome of an inbound message. This is the single vocabulary a + * delivery decision keys on, distinct from the two-axis signature verdict that + * produces it. + * + * - `clean` — nothing suspect; always admitted + * - `untrustedFrom` — the visible `From` is present but cannot be reduced to a + * single addr-spec + * - `mismatchedFrom` — the visible `From` names a different identity from the + * sender the hub stamped + * - `absentFrom` — the message carries no usable `From` + * - `invalid` — the signature check failed (tampering or the wrong key) + * - `missing` — the message carried no signature + * - `unknown` — no key was available to verify against + * - `error` — a fault stopped the check from running at all; always rejected + */ +export const InboundMailOutcome = type.enumerated( + "clean", + "untrustedFrom", + "mismatchedFrom", + "absentFrom", + "invalid", + "missing", + "unknown", + "error", +); +export type InboundMailOutcome = typeof InboundMailOutcome.infer; + +/** + * The subset of {@link InboundMailOutcome} a workflow author may relax to admit + * a message that would otherwise be rejected. It omits `clean` (which always + * admits, so there is nothing to relax) and `error` (pinned to reject, since a + * fault we could not check through is never something an author should be able + * to wave past). A per-workflow policy keys on exactly these outcomes. + */ +export const AuthorControllableOutcome = type.enumerated( + "untrustedFrom", + "mismatchedFrom", + "absentFrom", + "invalid", + "missing", + "unknown", +); +export type AuthorControllableOutcome = typeof AuthorControllableOutcome.infer; + +/** + * A per-workflow inbound-mail admission policy: for each admission outcome the + * author may control, whether a message that raises that outcome as a finding is + * `reject`ed or `admit`ted. The key set is exactly the + * {@link AuthorControllableOutcome} values -- `clean` (always admitted) and + * `error` (pinned to reject) are deliberately not keys. + * + * The object is SPARSE: every key is optional, and an omitted key is NOT a + * default of any kind here. It is left for a later resolution step to interpret + * an absent outcome. Keeping it sparse means the content hash covers only what + * the author actually declared, so a definition that omits the policy hashes + * identically to one authored before the field existed. Undeclared keys are + * rejected so a typo such as `clean` or `errror` fails at the wire boundary + * rather than riding through as an inert unknown key. + */ +export const InboundMailPolicy = type({ + "untrustedFrom?": "'reject' | 'admit'", + "mismatchedFrom?": "'reject' | 'admit'", + "absentFrom?": "'reject' | 'admit'", + "invalid?": "'reject' | 'admit'", + "missing?": "'reject' | 'admit'", + "unknown?": "'reject' | 'admit'", +}).onUndeclaredKey("reject"); +export type InboundMailPolicy = typeof InboundMailPolicy.infer; + +type AssertEqual = [A] extends [B] + ? [B] extends [A] + ? true + : false + : false; + +/** + * The three outcome vocabularies must move together, but nothing else connects + * them: each spells the same names out again in its own notation. + */ +const _policyKeysMatchAuthorControllable: AssertEqual< + keyof InboundMailPolicy, + AuthorControllableOutcome +> = true; + +const _authorControllableAreOutcomes: AssertEqual< + Exclude, + never +> = true; diff --git a/vendor/intx/types/src/index.ts b/vendor/intx/types/src/index.ts index 755229df80..5ae3df72f9 100644 --- a/vendor/intx/types/src/index.ts +++ b/vendor/intx/types/src/index.ts @@ -9,6 +9,7 @@ export * from "./signals"; export * from "./instances"; export * from "./workflows"; export * from "./attachments"; +export * from "./attachment-validation"; export * from "./sessions"; export * from "./approvals"; export * from "./wallets"; @@ -23,6 +24,7 @@ export * from "./offerings"; export * from "./models"; export * from "./capabilities"; export * from "./catalog"; +export * from "./adapter-manifest"; export * from "./observability"; export * from "./agent-address"; export * from "./agent-data"; @@ -36,3 +38,4 @@ export * from "./has-code"; export * from "./audit"; export * from "./sidecar-allocation"; export * from "./sidecar-capabilities"; +export * from "./workflow-lifecycle"; diff --git a/vendor/intx/types/src/inference-events.ts b/vendor/intx/types/src/inference-events.ts new file mode 100644 index 0000000000..3edc1dd159 --- /dev/null +++ b/vendor/intx/types/src/inference-events.ts @@ -0,0 +1,617 @@ +import { type } from "arktype"; +import { + PartialMessage, + RedactedThinkingBlock, + TokenUsage, + LastCycleSource, + AssistantTurn, + InferenceError, + CitationBlock, + SafetyRatingBlock, + CodeExecutionRequestBlock, + CodeExecutionResultBlock, + ImageBlock, + ToolCall, + ToolResult, + GateType, + ApprovalSnapshot, + ForkMode, + type InboundMessage, +} from "./runtime-core"; + +// --------------------------------------------------------------------------- +// Inference Event Protocol (INFERENCE.md § Event Protocol) +// --------------------------------------------------------------------------- + +/** + * Wire-safe representation of InboundMessage for use in InferenceEvent + * variants. The runtime InboundMessage type contains Uint8Array fields + * (MessageAttachment.data) that cannot survive JSON serialization, so the + * wire validator uses `unknown` for attachment data and accepts whatever + * JSON.parse produces. + */ +const WireInboundMessage = type({ + ref: { uid: "number", mailbox: "string" }, + headers: "Record", + flags: "string[]", + "content?": "string", + "payload?": "object", + "attachments?": "unknown[]", + signatureStatus: type.enumerated("valid", "invalid", "unknown", "missing"), +}); + +/** + * A single event in the inference event protocol. Every event carries a + * monotonic session-scoped sequence number. + * + * Event types are namespaced: `inference.*`, `tool.*`, `reactor.*`, + * `fork.*`, `message.*`, `custom.*`. + * + * (INFERENCE.md § Event Protocol) + */ +export const InferenceEvent = type.or( + { + type: "'inference.start'", + seq: "number", + data: { model: "string" }, + }, + { + type: "'inference.thinking.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.block.signature'", + seq: "number", + data: { signature: "string", "index?": "number" }, + }, + { + type: "'inference.thinking.redacted'", + seq: "number", + data: { redactedThinking: RedactedThinkingBlock, "index?": "number" }, + }, + { + type: "'inference.text.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.refusal.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.start'", + seq: "number", + data: { + callId: "string", + name: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.delta'", + seq: "number", + data: { + callId: "string", + argumentFragment: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.end'", + seq: "number", + data: { + callId: "string", + name: "string", + arguments: "Record", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.usage'", + seq: "number", + data: { usage: TokenUsage, source: LastCycleSource }, + }, + { + type: "'inference.done'", + seq: "number", + data: { + turn: AssistantTurn, + usage: TokenUsage, + source: LastCycleSource, + "pacingDelayMs?": "number", + }, + }, + { + type: "'inference.error'", + seq: "number", + data: { error: InferenceError, partial: PartialMessage }, + }, + { + type: "'inference.retry'", + seq: "number", + data: { + attempt: "number", + delayMs: "number", + previousError: InferenceError, + }, + }, + { + type: "'inference.citation'", + seq: "number", + // `index`, when present, names the source content block (typically + // a TextBlock) the citation annotates. The harness uses it to + // interleave the citation into the finalized turn's `content[]` + // immediately after the matching block. Adapters whose wire + // protocol does not carry per-citation block indices omit the + // field; the harness then appends those citations at the end of + // `content[]` and consumers attribute them to the nearest + // preceding TextBlock per the CitationBlock docstring. + data: { citation: CitationBlock, "index?": "number" }, + }, + { + type: "'inference.safety_rating'", + seq: "number", + // Prompt-level structured safety signal (observed Gemini + // `promptFeedback.blockReason`). No candidate index: the first + // capture has zero candidates. Harness appends the block to the + // finalized turn's `content[]`. + data: { safetyRating: SafetyRatingBlock }, + }, + { + type: "'inference.code_execution.start'", + seq: "number", + data: { request: CodeExecutionRequestBlock, "index?": "number" }, + }, + { + type: "'inference.code_execution.delta'", + seq: "number", + // requestId correlates fragments back to the originating + // CodeExecutionRequestBlock; index is the positional hint into + // the response's content-block stream. They are independent: a + // single response may stream code execution for multiple + // requests interleaved, distinguished by requestId; index lets + // the harness's per-block accumulator route the fragment to + // the correct block when the array isn't yet finalized. + data: { + requestId: "string", + codeFragment: "string", + "index?": "number", + }, + }, + { + type: "'inference.code_execution.result'", + seq: "number", + data: { result: CodeExecutionResultBlock, "index?": "number" }, + }, + { + type: "'inference.image_output'", + seq: "number", + // Fires mid-stream when an adapter finalizes an image-output + // block, signaling that the image is ready for downstream + // handoff before the full inference.done lands. The wrapped + // ImageBlock typically carries a base64 MediaSource — the + // payload can be large (Gemini's image-output captures show + // ~1MB inline blobs); consumers that subscribe to this event + // should treat it as a non-trivial transport size. + data: { image: ImageBlock, "index?": "number" }, + }, + { + type: "'tool.start'", + seq: "number", + data: { call: ToolCall }, + }, + { + type: "'tool.update'", + seq: "number", + data: { callId: "string", partial: "string" }, + }, + { + type: "'tool.done'", + seq: "number", + data: { result: ToolResult }, + }, + { + type: "'message.queued'", + seq: "number", + data: { message: WireInboundMessage }, + }, + { + type: "'message.run.started'", + seq: "number", + data: { + "messageId?": "string", + messageRunId: "string", + receivedAt: "number", + }, + }, + { + type: "'message.run.ended'", + seq: "number", + data: { + messageRunId: "string", + "messageId?": "string", + status: type.enumerated("completed", "failed"), + "error?": { + message: "string", + "kind?": "string", + }, + }, + }, + { + type: "'message.correlated'", + seq: "number", + data: { message: WireInboundMessage, correlationId: "string" }, + }, + { + type: "'connector.reply'", + seq: "number", + data: { content: "string", "checkpointHash?": "string" }, + }, + { + type: "'reactor.start'", + seq: "number", + data: "object", + }, + { + type: "'reactor.gate.blocked'", + seq: "number", + data: { + reason: GateType, + gateId: "string", + "correlationId?": "string", + "approvalSnapshot?": ApprovalSnapshot, + }, + }, + { + type: "'reactor.gate.cleared'", + seq: "number", + data: { + gateId: "string", + reason: type.enumerated("resolved", "timeout", "shutdown"), + }, + }, + { + type: "'reactor.done'", + seq: "number", + data: "object", + }, + { + type: "'reactor.error'", + seq: "number", + data: { error: "string", fatal: "boolean" }, + }, + { + type: "'fork.created'", + seq: "number", + data: { forkId: "string", parentId: "string", mode: ForkMode }, + }, + { + type: "'fork.done'", + seq: "number", + data: { forkId: "string", "result?": "unknown" }, + }, + { + type: "'fork.error'", + seq: "number", + data: { forkId: "string", error: "string" }, + }, + { + type: "'fork.aborted'", + seq: "number", + data: { forkId: "string" }, + }, + { + type: /^custom\./, + seq: "number", + data: "Record", + }, +); +// The TypeScript type is defined manually rather than inferred from the +// validator because the `custom.*` variant uses a regex pattern which +// arktype infers as `string`. A bare `string` in the discriminant position +// prevents TypeScript from narrowing the union in switch statements. +// The manually defined type uses a `custom.${string}` template literal +// for that variant, preserving the narrowing behavior downstream code +// relies on. +export type InferenceEvent = + | { type: "inference.start"; seq: number; data: { model: string } } + | { + type: "inference.thinking.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.block.signature"; + seq: number; + data: { signature: string; index?: number }; + } + | { + type: "inference.thinking.redacted"; + seq: number; + data: { redactedThinking: RedactedThinkingBlock; index?: number }; + } + | { + type: "inference.text.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.refusal.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.tool_call.start"; + seq: number; + data: { + callId: string; + name: string; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.tool_call.delta"; + seq: number; + data: { + callId: string; + argumentFragment: string; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.tool_call.end"; + seq: number; + data: { + callId: string; + name: string; + arguments: Record; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.usage"; + seq: number; + data: { usage: TokenUsage; source: LastCycleSource }; + } + | { + type: "inference.done"; + seq: number; + data: { + turn: AssistantTurn; + usage: TokenUsage; + source: LastCycleSource; + pacingDelayMs?: number; + }; + } + | { + type: "inference.error"; + seq: number; + data: { error: InferenceError; partial: PartialMessage }; + } + | { + /** + * Emitted between attempts when the per-call retry policy decides + * to retry after an error. `attempt` is the 1-indexed number of + * the attempt that just **failed** — the same value the policy + * saw on its `RetrySituation.attempt` reading. `delayMs` is the + * delay the wrapper will apply before the next attempt starts; + * `previousError` carries the classified error that triggered + * the retry. The event is not emitted when the policy aborts. + */ + type: "inference.retry"; + seq: number; + data: { + attempt: number; + delayMs: number; + previousError: InferenceError; + }; + } + | { + type: "inference.citation"; + seq: number; + data: { citation: CitationBlock; index?: number }; + } + | { + type: "inference.safety_rating"; + seq: number; + data: { safetyRating: SafetyRatingBlock }; + } + | { + type: "inference.code_execution.start"; + seq: number; + data: { request: CodeExecutionRequestBlock; index?: number }; + } + | { + type: "inference.code_execution.delta"; + seq: number; + data: { requestId: string; codeFragment: string; index?: number }; + } + | { + type: "inference.code_execution.result"; + seq: number; + data: { result: CodeExecutionResultBlock; index?: number }; + } + | { + type: "inference.image_output"; + seq: number; + data: { image: ImageBlock; index?: number }; + } + | { type: "tool.start"; seq: number; data: { call: ToolCall } } + | { + type: "tool.update"; + seq: number; + data: { callId: string; partial: string }; + } + | { type: "tool.done"; seq: number; data: { result: ToolResult } } + | { + type: "message.queued"; + seq: number; + data: { message: InboundMessage }; + } + | { + /** + * Per-message run-bracket open. Emitted by the reactor when it + * dequeues an inbound mail message and begins per-message work. + * + * `messageRunId` is reactor-minted, unique per dequeue. It is + * non-negotiable for crash-replay correlation: the reactor can + * legitimately dequeue the same `messageId` more than once across + * a crash + replay cycle, so two bracket-open events with the + * same `messageId` and no run-id cannot be unambiguously paired + * with their `message.run.ended` counterparts. + */ + type: "message.run.started"; + seq: number; + data: { + messageId?: string; + messageRunId: string; + receivedAt: number; + }; + } + | { + /** + * Per-message run-bracket close. Pairs with `message.run.started` + * by `messageRunId`. `messageId` is carried redundantly so log + * readers can correlate without a join against the open event; it is + * absent for a message that named no id of its own. + * + * The `status` enum is `"completed" | "failed"` only. + * Cancellation lives in the workflow-runtime's + * `CancelRequested` -> `RunFailed` vocabulary, not on the + * reactor's bracket: the reactor does not run a state machine + * and what it observes when cancellation arrives is a harness + * abort, which is structurally `"failed"` with a specific + * `error.kind`. + * + * `error.kind` is documented as one of + * `"inference_error" | "tool_error" | "reactor_fatal" | + * "harness_aborted" | "doom_loop"` initially, extensible as new + * failure categories surface. `"doom_loop"` marks a protective + * break the reactor took on the agent's behalf when the agent + * repeated an identical tool batch past the configured threshold; + * unlike `"reactor_fatal"` it is not an internal fault. + */ + type: "message.run.ended"; + seq: number; + data: { + messageRunId: string; + messageId?: string; + status: "completed" | "failed"; + error?: { + message: string; + kind?: string; + }; + }; + } + | { + type: "message.correlated"; + seq: number; + data: { message: InboundMessage; correlationId: string }; + } + | { + type: "connector.reply"; + seq: number; + data: { content: string; checkpointHash?: string }; + } + | { type: "reactor.start"; seq: number; data: Record } + | { + type: "reactor.gate.blocked"; + seq: number; + data: { + reason: GateType; + gateId: string; + correlationId?: string; + approvalSnapshot?: ApprovalSnapshot; + }; + } + | { + type: "reactor.gate.cleared"; + seq: number; + data: { + gateId: string; + reason: "resolved" | "timeout" | "shutdown"; + }; + } + | { type: "reactor.done"; seq: number; data: Record } + | { + type: "reactor.error"; + seq: number; + data: { error: string; fatal: boolean }; + } + | { + type: "fork.created"; + seq: number; + data: { forkId: string; parentId: string; mode: ForkMode }; + } + | { + type: "fork.done"; + seq: number; + data: { forkId: string; result?: unknown }; + } + | { + type: "fork.error"; + seq: number; + data: { forkId: string; error: string }; + } + | { type: "fork.aborted"; seq: number; data: { forkId: string } } + | { + type: `custom.${string}`; + seq: number; + data: Record; + }; + +// Load-bearing drift guards for the dual-maintained `reactor.gate.blocked` +// event. The arktype `InferenceEvent` validator and the hand-written +// `InferenceEvent` type are kept in lockstep by hand (the `custom.*` regex +// variant forces the manual mirror). arktype passes undeclared keys through at +// runtime, so a schema that dropped `approvalSnapshot` would not fail at +// runtime. Projecting the field off each inferred shape makes it load-bearing: +// `tsc` errors if either mirror stops carrying it, mirroring the +// `_persistedSuspendedCall` guard in storage-isogit. +const _arkGateBlockedApprovalSnapshot = ( + data: Extract< + typeof InferenceEvent.infer, + { type: "reactor.gate.blocked" } + >["data"], +): ApprovalSnapshot | undefined => data.approvalSnapshot; +void _arkGateBlockedApprovalSnapshot; + +const _tsGateBlockedApprovalSnapshot = ( + data: Extract["data"], +): ApprovalSnapshot | undefined => data.approvalSnapshot; +void _tsGateBlockedApprovalSnapshot; + +/** + * Validate unknown data as an InferenceEvent. ArkType's regex-based validator + * infers `custom.*` event types as `string`, but the manual InferenceEvent type + * uses a `custom.${string}` template literal for switch narrowing. This function + * centralizes that single unavoidable cast. + */ +export function parseInferenceEvent( + data: unknown, +): InferenceEvent | type.errors { + const result = InferenceEvent(data); + if (result instanceof type.errors) return result; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- arktype regex infers as string; manual type uses template literal + return result as InferenceEvent; +} diff --git a/vendor/intx/types/src/instances.ts b/vendor/intx/types/src/instances.ts index 9f2786cebc..2b33e50597 100644 --- a/vendor/intx/types/src/instances.ts +++ b/vendor/intx/types/src/instances.ts @@ -48,6 +48,12 @@ export const WorkflowRunResponse = type({ createdAt: "string", updatedAt: "string", "endedAt?": "string | null", + "failureCode?": type("string | null").describe( + "Why the Hub failed this run's deployment, such as `sidecar_connect_failed`; null for a run the Hub did not fail.", + ), + "failureMessage?": type("string | null").describe( + "The detail of `failureCode`, such as the error the sidecar reported.", + ), }); export const WorkflowRunHealth = type({ diff --git a/vendor/intx/types/src/mediated-credential.ts b/vendor/intx/types/src/mediated-credential.ts index 63461665f3..8feea1a9d7 100644 --- a/vendor/intx/types/src/mediated-credential.ts +++ b/vendor/intx/types/src/mediated-credential.ts @@ -34,15 +34,16 @@ export interface CredentialMaterial { export type CredentialMaterialSource = () => CredentialMaterial; /** - * Resolves the current material for a credential BY id from the run's credential - * cell. Inference uses this to fill a request's credential from - * `InferenceSource.credentialId` at send time -- the same cell tool credentials - * resolve from, so neither rail holds an inline secret. Keyed by `credentialId` - * (not bound to one, unlike `CredentialMaterialSource`) because a source's - * forward-only failover chain carries a distinct credential per entry. Reads - * live, so a rotation of the cell is picked up on the next call; fails closed - * when the credential is absent (revoked or never delivered). This is the single - * seam a future mode swaps to keep the raw secret out of the child entirely. + * Resolves the current material for a credential BY id from the run's + * credential cell. Inference uses this to fill a request's credential from + * `InferenceSource.credentialId` at send time -- the same cell binding + * credentials resolve from, so neither rail holds an inline secret. Keyed by + * `credentialId` (not bound to one, unlike `CredentialMaterialSource`) because + * a source's forward-only failover chain carries a distinct credential per + * entry. Reads live, so a rotation of the cell is picked up on the next call; + * fails closed when the credential is absent (revoked or never delivered). This + * is the single seam a future mode swaps to keep the raw secret out of the + * child entirely. */ export type CredentialMaterialResolver = ( credentialId: string, @@ -115,3 +116,13 @@ export interface CredentialProvider { export interface CredentialCapability { resolve(handle: string): Promise; } + +/** + * A `CredentialCapability` plus a host-only `dispose`. The consumer sees only + * `resolve`; the host runs `dispose` on teardown to release every handle shaped + * through this capability (an http handle holds nothing; a future key-file / + * socket handle would). + */ +export interface HostCredentialCapability extends CredentialCapability { + dispose(): Promise; +} diff --git a/vendor/intx/types/src/package-json.ts b/vendor/intx/types/src/package-json.ts index 80fe2fb32d..ef4a666cea 100644 --- a/vendor/intx/types/src/package-json.ts +++ b/vendor/intx/types/src/package-json.ts @@ -8,10 +8,10 @@ // set the build path emits, otherwise a freshly-packed builtin would // be rejected for shape reasons the build did not anticipate. -import path from "node:path"; - import { type } from "arktype"; +import { normalize } from "./posix-path"; + /** * A tool package's static declaration of one provider-backed credential it * needs: an abstract handle plus optional scopes. Advisory only -- a request @@ -90,9 +90,9 @@ export type PackageJSON = typeof PackageJSON.infer; * result does not depend on the host's separator or cwd. */ export function isContainedEntryPath(entry: string): boolean { - if (path.posix.isAbsolute(entry)) { + if (entry.startsWith("/")) { return false; } - const normalized = path.posix.normalize(entry); - return normalized !== ".." && !normalized.startsWith(`..${path.posix.sep}`); + const normalized = normalize(entry); + return normalized !== ".." && !normalized.startsWith("../"); } diff --git a/vendor/intx/types/src/posix-path.ts b/vendor/intx/types/src/posix-path.ts new file mode 100644 index 0000000000..acbab3b1c4 --- /dev/null +++ b/vendor/intx/types/src/posix-path.ts @@ -0,0 +1,68 @@ +// Pure POSIX path helpers. +// +// Portable code cannot import `node:path`: a browser bundler stubs it or +// replaces it with a polyfill whose `resolve` calls `process.cwd()` +// unguarded. These helpers work on +// `/`-separated strings only and never consult a working directory: a relative +// path stays relative, and `resolve` roots it at `/`. + +/** + * Collapse `//`, drop `.` segments and resolve `..` segments. A path that + * starts with `/` stays absolute, and `..` above the root is dropped. A + * relative path keeps its leading `..` segments. A trailing `/` is dropped. + * An empty relative result is `.`. + */ +export function normalize(filepath: string): string { + const absolute = filepath.startsWith("/"); + const segments: string[] = []; + for (const segment of filepath.split("/")) { + if (segment === "" || segment === ".") { + continue; + } + if (segment === "..") { + if (segments.length > 0 && segments.at(-1) !== "..") { + segments.pop(); + } else if (!absolute) { + segments.push(segment); + } + continue; + } + segments.push(segment); + } + + if (absolute) { + return segments.length === 0 ? "/" : `/${segments.join("/")}`; + } + return segments.length === 0 ? "." : segments.join("/"); +} + +/** `filepath` as an absolute path, rooting a relative path at `/`. */ +export function resolve(filepath: string): string { + return normalize(filepath.startsWith("/") ? filepath : `/${filepath}`); +} + +/** Join `parts` with `/` and normalize the result. */ +export function join(...parts: string[]): string { + if (parts.length === 0) { + return "."; + } + return normalize(parts.join("/")); +} + +/** The relative path from `from` to `to`, both resolved first. */ +export function relative(from: string, to: string): string { + const fromSegments = resolve(from).split("/").filter(Boolean); + const toSegments = resolve(to).split("/").filter(Boolean); + let shared = 0; + while ( + shared < fromSegments.length && + shared < toSegments.length && + fromSegments[shared] === toSegments[shared] + ) { + shared += 1; + } + return [ + ...fromSegments.slice(shared).map(() => ".."), + ...toSegments.slice(shared), + ].join("/"); +} diff --git a/vendor/intx/types/src/repo.ts b/vendor/intx/types/src/repo.ts new file mode 100644 index 0000000000..cffd8e4b36 --- /dev/null +++ b/vendor/intx/types/src/repo.ts @@ -0,0 +1,41 @@ +import { type } from "arktype"; + +/** + * Tag identifying a kind of repository in the hub's kind-keyed RepoStore. + * Lives in `@intx/types` because the wire-level pack frames reference it; + * the substrate package re-exports it for handler authors. + */ +export const RepoKind = type.enumerated( + "agent-state", + "skill", + "package-registry", + "workflow", + "workflow-run", +); +export type RepoKind = typeof RepoKind.infer; + +/** + * Operations a principal may invoke against a repo in the RepoStore. + * Lives in `@intx/types` so storage layers (e.g. `@intx/db`) can validate + * persisted action vocabularies without depending on the substrate + * package. The substrate re-exports it for handler authors. + */ +export const RepoAction = type.enumerated( + "init", + "writeTree", + "receivePack", + "createPack", + "resolveRef", +); +export type RepoAction = typeof RepoAction.infer; + +/** + * Hub-side identity of a repository in the RepoStore. Pack frames carry + * this alongside `agentAddress` so the hub can map a pack back to the + * originating repo independently of which sidecar/agent it is destined for. + */ +export const RepoId = type({ + kind: RepoKind, + id: "string", +}); +export type RepoId = typeof RepoId.infer; diff --git a/vendor/intx/types/src/runtime-core.ts b/vendor/intx/types/src/runtime-core.ts new file mode 100644 index 0000000000..9dd79f2068 --- /dev/null +++ b/vendor/intx/types/src/runtime-core.ts @@ -0,0 +1,2383 @@ +// Runtime definitions for the Interchange agent harness. +// +// Wire-facing data types (AbortReason, InferenceSource, ToolDefinition, +// HarnessConfig) are arktype validators so they can be composed into +// WebSocket frame validators and used for runtime validation at parse +// boundaries. Behavioral interfaces (ContextStore, MessageTransport, +// ToolRunner, etc.) remain plain TypeScript. + +import { type } from "arktype"; +import type { AuditRecord, ErrorRecord } from "./audit"; +import { WireGrantRule } from "./grant-wire"; +import type { SignalKind } from "./signals"; + +// --------------------------------------------------------------------------- +// Cryptographic Identity (ARCHITECTURE.md § Cryptographic Identity, +// IMPLEMENTATION.md § Cryptographic Identity: Key Formats) +// --------------------------------------------------------------------------- + +/** + * An Ed25519 key pair as raw bytes. The private key is 32 bytes; the public + * key is the corresponding 32-byte compressed point. + * + * Key material is represented as Uint8Array throughout so it stays + * runtime-agnostic (Bun, Node, browser) and never accidentally leaks through + * JSON serialization. + */ +export type KeyPair = { + privateKey: Uint8Array; + publicKey: Uint8Array; +}; + +/** + * A key-bound cryptographic provider. Each instance is constructed with a + * specific agent's Ed25519 key pair and holds the private key internally. + * + * `sign` uses the instance's own private key — no key parameter is accepted. + * `verify` accepts a public key parameter so the holder can verify messages + * from arbitrary senders without constructing a new provider instance. + * + * The in-memory transport stores one CryptoProvider per registered agent and + * calls `crypto.sign(content)` during `send()` without passing keys around. + * + * Key formats (IMPLEMENTATION.md): + * - Ed25519 in SSH format — control plane interactions + * - Ed25519 in PGP format — message-level signatures over SMTP/IMAP + * - Ed25519 in X.509 format — TLS mutual auth certificates + * + * `getPublicKey` returns the raw public key bytes so callers can publish + * them to the control plane or embed them in discovery metadata. + */ +export interface CryptoProvider { + /** + * Sign `content` with the instance's private key. Returns the Ed25519 + * detached signature as raw bytes. + */ + sign(content: Uint8Array): Promise; + + /** + * Sign `payload` with the instance's private key using the SSH signature + * envelope (sshsig). Returns an ASCII-armored SSH SIGNATURE block suitable + * for the `gpgsig` header of a git commit or any other site that consumes + * `git verify-commit`-compatible signatures. The framing differs from + * `sign`'s raw output; callers that need either format should pick the + * matching method rather than reframing the result themselves. + */ + signSSH(payload: string): Promise; + + /** + * Verify that `signature` over `content` was produced by `publicKey`. + * Returns true if the signature is valid; false otherwise. + */ + verify( + content: Uint8Array, + signature: Uint8Array, + publicKey: Uint8Array, + ): Promise; + + /** The public key for this instance, as raw bytes. */ + getPublicKey(): Uint8Array; +} + +/** + * Generate a fresh Ed25519 key pair. The returned pair is used to construct + * a CryptoProvider instance. + */ +export type GenerateKeyPair = () => Promise; + +// --------------------------------------------------------------------------- +// Message Transport (MESSAGE.md § Transport Interface) +// --------------------------------------------------------------------------- + +/** + * Opaque reference to a message in a specific mailbox. Carries the IMAP UID + * and the mailbox name. Passed to fetch, flag, and move operations without + * requiring re-search. + */ +export type MessageRef = { + uid: number; + mailbox: string; +}; + +/** + * Interchange payload types as defined in MESSAGE.md § Payload Types. + * The type field in structured messages matches the Interchange-Type header. + * + * Exposed as both an arktype validator (for runtime validation at parse + * boundaries and tool-argument schemas) and a derived TypeScript union. + */ +export const InterchangeType = type.enumerated( + "conversation.message", + "conversation.join", + "conversation.leave", + "offering.request", + "offering.response", + "offering.error", + "offering.discover", + "offering.catalog", + "payment.required", + "payment.receipt", + "payment.verified", + "approval.request", + "approval.granted", + "approval.denied", + "system.health", + "system.register", + "system.deregister", + "system.credential.refresh", +); +export type InterchangeType = typeof InterchangeType.infer; + +const CONVERSATION_TYPES: ReadonlySet = new Set([ + "conversation.message", + "conversation.join", + "conversation.leave", +]); + +/** + * Membership is enumerated rather than derived from the `conversation.` name + * prefix, so a member added to InterchangeType is not a conversation type until + * it is listed in CONVERSATION_TYPES. + */ +export function isConversationType(value: InterchangeType): boolean { + return CONVERSATION_TYPES.has(value); +} + +/** Attachment for an outbound message. Content is raw bytes. */ +export type MessageAttachment = { + name: string; + contentType: string; + data: Uint8Array; + /** + * IMAP BODY section of this part, stamped by `extractAttachments` from + * the parsed sibling numbering. Absent on outbound attachments. + */ + part?: string; +}; + +/** + * A message the harness submits for delivery via SMTP. The transport + * assembles the PGP/MIME multipart structure, signs it with the agent's + * CryptoProvider, and submits it. + * + * Conversation types (conversation.*) carry `content` as text/plain. + * Structured types carry `payload` as application/vnd.interchange+json. + * Providing both is an error. + * + * (MESSAGE.md § Transport Interface › Outbound) + */ +export type OutboundMessage = { + to: string | string[]; + cc?: string | string[]; + subject?: string; + + type: InterchangeType; + + /** Plain text body — used when type is a conversation.* type. */ + content?: string; + + /** Structured JSON body — used when type is a non-conversation type. */ + payload?: Record; + + /** Human-readable summary for structured messages (the text/plain part). */ + summary?: string; + + attachments?: MessageAttachment[]; + + /** Message-ID of the message being replied to. */ + inReplyTo?: string; + + /** + * The RFC 5322 References chain for a threaded reply: the parent's own + * References plus the parent's Message-ID, in order. When present the + * transport ships it verbatim (after appending `inReplyTo` if it is not + * already the tail) rather than deriving a single-element `[inReplyTo]` + * chain, so a reply carries the full conversational ancestry. Absent for a + * non-reply or a reply whose parent could not be located. + */ + references?: string[]; + + /** Correlation ID linking this message to a pending async request. */ + correlationId?: string; + + /** Reactor session ID from the Interchange-Session-ID header. */ + sessionId?: string; + + /** Tenant ID for the Interchange-Tenant-ID header. */ + tenantId?: string; +}; + +/** + * Receipt returned by `send()`. Contains the assigned Message-ID and + * delivery status. + * + * (MESSAGE.md § Transport Interface › Outbound) + */ +export type SendReceipt = { + messageId: string; + status: "delivered" | "queued"; +}; + +/** + * Parsed headers from an inbound message. Field names follow RFC 5322 and + * the Interchange-specific header conventions from MESSAGE.md § Headers. + * + * `from` is optional because a message can arrive carrying no originator at + * all, and there is no string that honestly stands for one. + */ +export type MessageHeaders = { + from?: string; + to: string[]; + cc?: string[]; + date?: string; + messageId?: string; + inReplyTo?: string; + references?: string[]; + subject?: string; + listId?: string; + + interchangeType?: InterchangeType; + interchangeCorrelationId?: string; + interchangeTenantId?: string; + interchangeAgentId?: string; + interchangeSessionId?: string; + interchangeOfferingId?: string; + interchangeSchemaVersion?: string; + + traceparent?: string; + tracestate?: string; +}; + +/** + * Signature verification status of an inbound message. + * + * - `valid` — signature verified against the sender's public key + * - `invalid` — signature check failed (tampering or wrong key) + * - `unknown` — public key not available for verification + * - `missing` — message was not signed + * + * (MESSAGE.md § Transport Interface › fetchFull) + */ +export const SignatureStatus = type.enumerated( + "valid", + "invalid", + "unknown", + "missing", +); +export type SignatureStatus = typeof SignatureStatus.infer; + +/** + * A parsed MIME part. `content` is the DECODED bytes in memory (the + * transfer-encoding has already been undone). `filename` and `disposition` are + * surfaced from the part's `Content-Disposition` / `Content-Type` so a consumer + * can distinguish an inline part from a named attachment without re-parsing + * headers. + */ +export type MessagePart = { + contentType: string; + content: Uint8Array; + filename?: string; + disposition?: "inline" | "attachment"; + /** + * The Content-Transfer-Encoding the part declared. Not set for a decoded + * mail part -- `content` is already decoded, so the wire encoding is spent + * transport metadata. + */ + encoding?: string; +}; + +/** + * A single part of a persisted `Mail`. The bytes live in the durable store; + * this descriptor carries the part's metadata plus an opaque, relocation-stable + * `ref` a `MailPartReader` resolves to the part's bytes. Small UTF-8 text parts + * also carry their decoded `text` inline so a selector can read them without + * resolving. + */ +export type MailPart = { + contentType: string; + filename?: string; + disposition?: "inline" | "attachment"; + ref: string; + text?: string; +}; + +/** + * The single, environment-agnostic interface for reading a persisted mail + * part's bytes. Modeled on `BlobReader`: a consumer -- a workflow step, an + * agent tool, the agent's content-block projection -- resolves a + * `MailPart.ref` to its bytes without knowing or caring where the bytes live + * (a committed file on the sidecar, a blob in a browser runtime). The `ref` + * is opaque; the reader owns its scheme. Threaded to consumers through the + * runtime so a browser runtime can supply its own implementation. + */ +export interface MailPartReader { + /** Resolve a `MailPart.ref` to the part's decoded bytes. Throws if the ref + * is unrecognized or its bytes are missing. */ + read(ref: string): Promise; +} + +/** + * A fully decoded mail message: every header (both the typed, ergonomic subset + * and a raw catch-all with nothing dropped) plus the flat list of decoded leaf + * parts. This is the lossless representation a deployed workflow receives as + * its trigger input; a workflow programs against it directly (select headers, + * walk parts, route on content type), and an agent step projects the parts into + * model content blocks. It is JSON-safe: each part carries a `ref` (not raw + * bytes), so binary content never enters the event log; the runtime resolves a + * `ref` into a loadable `MessagePart` on demand. + */ +export type Mail = { + headers: MessageHeaders; + /** Every header, lowercased name to its ordered values; nothing dropped. */ + rawHeaders: Record; + parts: MailPart[]; +}; + +const MailShape = type({ + // Require the recipient list a consumer dereferences unconditionally; other + // header fields stay optional and are carried losslessly in `rawHeaders`. + // `from` is optional here deliberately: a mail with no usable originator is + // still mail, and requiring it makes `isMail` reject one. + headers: { + "from?": "string", + to: "string[]", + }, + rawHeaders: { + "[string]": "string[]", + }, + parts: type({ + contentType: "string", + ref: "string", + "filename?": "string", + "disposition?": "'inline' | 'attachment'", + "text?": "string", + }) + .onUndeclaredKey("reject") + .array(), +}).onUndeclaredKey("reject"); + +/** + * Narrow an opaque value (a workflow step input) to a `Mail`. Used at the + * `agent.send` boundary to decide whether the input is a mail-derived message + * whose parts must be projected into content blocks, or an arbitrary value + * delivered as synthesized text. The strict undeclared-key rejection keeps an + * arbitrary step value that merely carries a `parts` field from matching. + */ +export function isMail(value: unknown): value is Mail { + return !(MailShape(value) instanceof type.errors); +} + +/** + * MIME tree metadata returned by `fetchStructure()`. Describes content types, + * sizes, and dispositions without transferring content. + * + * (MESSAGE.md § Partial Fetch) + */ +export type BodyStructure = { + contentType: string; + size?: number; + disposition?: string; + parts?: BodyStructure[]; +}; + +/** + * A fully parsed inbound message including structured payload, headers, + * attachments, and signature verification status. + * + * (MESSAGE.md § Transport Interface › fetchFull) + */ +export type InboundMessage = { + ref: MessageRef; + headers: MessageHeaders; + flags: string[]; + + /** Plain text body for conversation.* types. */ + content?: string; + + /** Parsed JSON payload for structured types. */ + payload?: { + type: InterchangeType; + version: string; + body: Record; + }; + + attachments?: MessageAttachment[]; + signatureStatus: SignatureStatus; +}; + +/** + * IMAP mailbox descriptor. + * + * (MESSAGE.md § Inbox Management) + */ +export type Mailbox = { + name: string; + role?: string; + delimiter?: string; +}; + +/** + * Current status of an IMAP mailbox, including QRESYNC identifiers. + * + * (MESSAGE.md § Inbox Management) + */ +export type MailboxStatus = { + total: number; + unseen: number; + recent: number; + uidNext: number; + uidValidity: number; + highestModSeq: number; +}; + +/** + * Structured IMAP search query. Maps the IMAP SEARCH grammar to a typed + * object. Supports recursive boolean composition via `and`, `or`, `not`. + * + * (MESSAGE.md § Search) + */ +export type SearchQuery = { + from?: string; + to?: string; + cc?: string; + bcc?: string; + header?: { field: string; contains: string }; + before?: Date; + after?: Date; + on?: Date; + sentBefore?: Date; + sentAfter?: Date; + sentOn?: Date; + hasFlags?: string[]; + missingFlags?: string[]; + body?: string; + text?: string; + largerThan?: number; + smallerThan?: number; + and?: SearchQuery[]; + or?: SearchQuery[]; + not?: SearchQuery; +}; + +/** + * A thread node returned by `thread()`. Carries a message reference and + * child threads representing replies. Implements the RFC 5256 REFERENCES + * threading algorithm. + * + * (MESSAGE.md § Thread Retrieval) + */ +export type Thread = { + ref: MessageRef; + children: Thread[]; +}; + +/** + * QRESYNC state the harness provides when reconnecting to the transport. + * + * (MESSAGE.md § Synchronization) + */ +export type SyncState = { + uidValidity: number; + uidNext: number; + highestModSeq: number; + knownUids?: number[]; +}; + +/** + * Result of a QRESYNC-style sync operation. + * + * (MESSAGE.md § Synchronization) + */ +export type SyncResult = { + vanished: number[]; + changed: { uid: number; flags: string[] }[]; + newMessages: MessageRef[]; + fullResyncRequired: boolean; +}; + +/** + * Distribution list metadata returned by `createList()`. + * + * (MESSAGE.md § Message Topologies) + */ +export type ListInfo = { + address: string; + name: string; + memberCount: number; + createdAt: string; +}; + +/** + * Event emitted by the mailbox watcher callback. Corresponds to IMAP IDLE + * notifications. + * + * (MESSAGE.md § Real-Time Notification) + */ +export type MailboxEvent = + | { type: "exists"; uid: number; headers: MessageHeaders } + | { type: "flagsChanged"; uid: number; flags: string[] } + | { type: "expunged"; uid: number }; + +/** Unsubscribe function returned by `watch()`. */ +export type Unsubscribe = () => void; + +/** + * The message transport interface. Abstracts SMTP and IMAP behind a + * TypeScript API. Implementations range from real SMTP/IMAP servers to + * in-process stubs that route messages through memory. + * + * All long-running operations accept an AbortSignal for cooperative + * cancellation. + * + * (MESSAGE.md § Transport Interface) + */ +export interface MessageTransport { + // --- Outbound --- + + /** Compose, sign, and deliver a message via SMTP. */ + send(message: OutboundMessage, signal?: AbortSignal): Promise; + + /** Append a raw message to a mailbox (IMAP APPEND). */ + append( + mailbox: string, + message: InboundMessage, + flags?: string[], + signal?: AbortSignal, + ): Promise; + + // --- Mailbox management --- + + listMailboxes(signal?: AbortSignal): Promise; + createMailbox(name: string, signal?: AbortSignal): Promise; + deleteMailbox(name: string, signal?: AbortSignal): Promise; + getMailboxStatus(name: string, signal?: AbortSignal): Promise; + + // --- Message search and retrieval --- + + search( + mailbox: string, + query: SearchQuery, + signal?: AbortSignal, + ): Promise; + + thread( + mailbox: string, + algorithm: "references" | "orderedsubject", + query?: SearchQuery, + signal?: AbortSignal, + ): Promise; + + fetchHeaders(ref: MessageRef, signal?: AbortSignal): Promise; + fetchStructure(ref: MessageRef, signal?: AbortSignal): Promise; + fetchPart( + ref: MessageRef, + partPath: string, + signal?: AbortSignal, + ): Promise; + fetchFull(ref: MessageRef, signal?: AbortSignal): Promise; + + // --- Flag management --- + + setFlags( + ref: MessageRef, + flags: string[], + signal?: AbortSignal, + ): Promise; + + clearFlags( + ref: MessageRef, + flags: string[], + signal?: AbortSignal, + ): Promise; + + // --- Message organization --- + + move(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; + + copy(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; + + /** + * Permanently remove every `\Deleted` message from the mailbox. Returns the + * uids that were expunged, so a caller can report how many messages it + * consumed and which ones. + */ + expunge( + mailbox: string, + signal?: AbortSignal, + ): Promise<{ expungedUids: number[] }>; + + // --- Real-time notification --- + + /** + * Monitor a mailbox for new messages and flag changes (IMAP IDLE). + * The promise resolves once the watch is accepted. Callers await it + * before relying on the callback, and a refusal rejects with the + * transport's condition. The unsubscribe function stays synchronous. + */ + watch( + mailbox: string, + callback: (event: MailboxEvent) => void, + ): Promise; + + // --- Synchronization --- + + /** Efficient reconnection using QRESYNC semantics. */ + sync( + mailbox: string, + knownState: SyncState, + signal?: AbortSignal, + ): Promise; + + // --- Distribution lists --- + + createList( + address: string, + name: string, + signal?: AbortSignal, + ): Promise; + + listMembers(address: string, signal?: AbortSignal): Promise; + + subscribe( + listAddress: string, + subscriberAddress: string, + signal?: AbortSignal, + ): Promise; + + unsubscribe( + listAddress: string, + subscriberAddress: string, + signal?: AbortSignal, + ): Promise; +} + +/** + * The condition a `MessageTransport` operation failed under. A transport that + * grows a condition this union does not carry takes its name from RFC 5530 § 3 + * rather than coining one. + * + * https://www.rfc-editor.org/rfc/rfc5530.html + */ +export const MessageTransportCondition = type( + "'NONEXISTENT' | 'CANNOT' | 'SERVERBUG'", +); +export type MessageTransportCondition = typeof MessageTransportCondition.infer; + +export class MessageTransportError extends Error { + readonly condition: MessageTransportCondition; + + constructor(condition: MessageTransportCondition, message: string) { + super(message); + this.name = "MessageTransportError"; + this.condition = condition; + } +} + +/** + * `instanceof` is not usable for this: a tool package is published as a bundle + * whose workspace imports are inlined (`bin/build-builtins.ts`), so a consumer + * loaded from a bundle holds its own copy of the class above and answers false + * against an error the host's copy constructed. + */ +export function isMessageTransportError( + value: unknown, +): value is Error & { readonly condition: MessageTransportCondition } { + if (!(value instanceof Error)) return false; + if (!("condition" in value)) return false; + return MessageTransportCondition.allows(value.condition); +} + +// --------------------------------------------------------------------------- +// Tool Execution (ARCHITECTURE.md § Tools, INFERENCE.md § Tool Execution) +// --------------------------------------------------------------------------- + +/** + * A tool call as requested by the model. Carries the provider-assigned call + * ID, the tool name, and the parsed arguments. + * + * (INFERENCE.md § Message Format › Content Types) + */ +export const ToolCall = type({ + id: "string", + name: "string", + arguments: "Record", +}); +export type ToolCall = typeof ToolCall.infer; + +/** + * Approver-facing snapshot of the tool call awaiting approval. Built at the + * authz `ask` branch from the tool's definition and the live call, then + * threaded unchanged from the reactor's pending operation through every + * suspend hop to the hub co-write that records it on the approval row. + * + * `name`, `description`, and `inputSchema` mirror the {@link ToolDefinition}; + * `arguments` is the live call's arguments. Carried as a sibling of the pending + * operation's `suspendedCall`, never folded into {@link ToolCall}, so the + * re-dispatch artifact and the approval snapshot stay separate concerns. + */ +export const ApprovalSnapshot = type({ + name: "string", + description: "string", + inputSchema: "Record", + arguments: "Record", +}); +export type ApprovalSnapshot = typeof ApprovalSnapshot.infer; + +/** + * The kind of a control-plane park: a step suspended awaiting an external + * event. `"approval"` and `"input"` park on a reserved + * `signalName(correlationId)` channel; `"signal-relay"` parks on an + * author-chosen name. + * + * - `"approval"` -- the step parked on a tool/authz gate and REQUIRES an + * {@link ApprovalSnapshot}; the runtime notifies the host (`env.onPark`) so + * the sidecar co-writes the approval/correlation rows the hub registers. + * - `"input"` -- the step parked awaiting its next input (e.g. a long-lived + * agent run awaiting the next mail so it can take another turn). It carries + * NO snapshot and does NOT notify the host: the run's owner delivers the + * input on the same channel and the step re-arms. It is a runtime-local + * concept -- deliberately NOT a {@link SignalKind}, so it never touches the + * approval-routing machinery (IPC register frames, the hub co-write, the + * approval columns). + * - `"signal-relay"` -- an onTrigger section container parked on an + * author-named signal so a body child's `awaitSignal` on that name is + * serviced through the deployment run: the external signal is delivered to + * the parent run and the runtime relays it down into the live body child. + * The channel name is the author's free-form signal name, NOT a reserved + * `signalName(correlationId)`, so recovery must branch on this kind BEFORE + * assuming the awaited name is a reserved control-plane channel. Carries no + * snapshot and is not hub-registered. + * + * The kinds are distinguished by an EXPLICIT discriminant everywhere the kind + * flows -- never inferred from the presence or absence of a snapshot, which + * would silently reclassify a malformed snapshot-less approval as another + * park kind rather than failing loud. + */ +export const ControlParkKind = type.enumerated( + "approval", + "input", + "signal-relay", +); +export type ControlParkKind = typeof ControlParkKind.infer; + +/** + * Maximum serialized size, in UTF-8 bytes, of an {@link ApprovalSnapshot} that + * crosses a trust boundary. A tool `inputSchema` is normally single-digit KB; + * a snapshot approaching this bound is malformed or hostile and is rejected at + * the parse boundary rather than co-written onto an approval row. + */ +export const APPROVAL_SNAPSHOT_MAX_BYTES = 131072; + +/** + * {@link ApprovalSnapshot} bounded to {@link APPROVAL_SNAPSHOT_MAX_BYTES}. + * Parse the snapshot through this validator where it crosses a trust boundary + * (the `park.notify` IPC frame, the `parked-correlations.response` IPC frame, + * and the sidecar→hub register frame); internal hops use the unbounded + * {@link ApprovalSnapshot}. `.narrow` bounds the runtime check only — its + * inferred type is identical to {@link ApprovalSnapshot} — so the cap holds only + * where a frame is actually parsed, not merely typed. + */ +export const BoundedApprovalSnapshot = ApprovalSnapshot.narrow( + (snapshot, ctx) => { + const bytes = new TextEncoder().encode(JSON.stringify(snapshot)).length; + return ( + bytes <= APPROVAL_SNAPSHOT_MAX_BYTES || + ctx.mustBe(`at most ${APPROVAL_SNAPSHOT_MAX_BYTES} bytes when serialized`) + ); + }, +); +export type BoundedApprovalSnapshot = typeof BoundedApprovalSnapshot.infer; + +/** + * Result of a tool execution. `content` is text or structured data the model + * sees as the tool result. `detail` is additional data that the harness may + * use (e.g., for validation or audit) but that is not shown to the model. + * + * When `isError` is true the model sees the result as an error. When + * `pendingMarker` is present the tool is async — the reactor registers the + * correlation ID and waits for a matching inbound message. + * + * (INFERENCE.md § Tool Execution Semantics) + */ +export const ToolResult = type({ + callId: "string", + content: "string | Record", + "detail?": "unknown", + "isError?": "boolean", + "pendingMarker?": { + status: "'pending'", + correlationId: "string", + }, +}); +export type ToolResult = typeof ToolResult.infer; + +/** + * The tool runner interface. The harness implements this; the reactor calls + * it when the director requests tool execution. + * + * Parallel execution is modeled by calling `run` concurrently for each call + * in a batch — the interface is per-call, not per-batch. + * + * (ARCHITECTURE.md § Agent Harness › Tools) + */ +export interface ToolRunner { + /** + * Execute a single tool call. Resolves with the result. Must not throw — + * errors are returned as `ToolResult` with `isError: true`. + */ + run(call: ToolCall, signal: AbortSignal): Promise; +} + +// --------------------------------------------------------------------------- +// Inference Event Building Blocks (INFERENCE.md § Event Protocol) +// --------------------------------------------------------------------------- + +/** + * Partial assistant message accumulated during streaming. Carries all + * content blocks seen so far so late-joining subscribers receive current + * state without replaying deltas. + * + * `text` and `thinking` are cumulative across every emitted delta of + * that kind in the current turn — intentionally flat, even when the + * harness's per-index block tracking has split the stream into + * multiple ThinkingBlocks or TextBlocks. Consumers that need per-block + * structure walk the finalized inference.done turn's content[]; this + * snapshot is the live "what bytes has the assistant streamed so + * far" view. + * + * (INFERENCE.md § Event Protocol › Partial State) + */ +export const PartialMessage = type({ + text: "string", + "thinking?": "string", + "toolCalls?": type({ + id: "string", + name: "string", + partialArguments: "string", + }).array(), +}); +export type PartialMessage = typeof PartialMessage.infer; + +/** + * Token usage for a single inference call. Cache read/write counts are + * provider-specific and may be zero when the provider does not report them. + * + * (INFERENCE.md § Token Accounting) + */ +export const TokenUsage = type({ + input: "number", + output: "number", + cacheRead: "number", + cacheWrite: "number", + thinking: "number", +}); +export type TokenUsage = typeof TokenUsage.infer; + +/** + * Slim source descriptor stamped onto `inference.usage` / `inference.done` + * events and onto `ReactorState.lastCycleSource`. + * + * Carries enough identity for state-aware policies (cost gating, budget + * caps, governance triggers, audit) to attribute usage to a specific + * inference source without re-reading the live, mutable `InferenceSource` + * the harness owns. + * + * Deliberately a strict subset of `InferenceSource` — `apiKey` and + * `baseURL` are intentionally excluded. Credentials and endpoints must + * not leak to director-side policy code or to external event consumers. + * Any code path that needs the full source obtains it through the + * harness's source registry, not through this descriptor. + * + * `sourceId` aliases `InferenceSource.id` to disambiguate from message + * ids, turn ids, and session ids in director-side code where `id` alone + * would be ambiguous. + */ +export const LastCycleSource = type({ + sourceId: "string", + provider: "string", + model: "string", +}); +export type LastCycleSource = typeof LastCycleSource.infer; + +// --------------------------------------------------------------------------- +// Internal Turn Format (INFERENCE.md § Message Format) +// --------------------------------------------------------------------------- + +/** + * A single content block within a conversation turn. Provider-agnostic. + * + * (INFERENCE.md § Message Format › Content Types) + */ +const TextBlock = type({ + type: "'text'", + text: "string", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini attaches a `thoughtSignature` to + // output parts (including plain text); absent for providers that do not + // sign this block kind. + "signature?": "string", +}); + +/** + * How a media payload is carried by a content block. One of three + * variants: inline as a base64-encoded string, by reference to an + * opaque provider-native handle (e.g. a Gemini fileUri, an Anthropic + * file_id), or by public URL the provider fetches itself. The wire + * shape each provider expects is built by the provider adapter; + * MediaSource is the internal, provider-agnostic representation. + * + * (INFERENCE.md § Generalized Multimodal Taxonomy) + */ +const MediaSourceBase64 = type({ + kind: "'base64'", + mimeType: "string", + data: "string", +}); + +const MediaSourceFileReference = type({ + kind: "'file-reference'", + mimeType: "string", + reference: "string", +}); + +const MediaSourceUrl = type({ + kind: "'url'", + mimeType: "string", + url: "string", +}); + +export const MediaSource = type.or( + MediaSourceBase64, + MediaSourceFileReference, + MediaSourceUrl, +); +export type MediaSource = typeof MediaSource.infer; + +// Exported because `inference.image_output` events reference it by +// name, following the same pattern as `CitationBlock`, +// `CodeExecutionRequestBlock`, and `RedactedThinkingBlock`. +export const ImageBlock = type({ + type: "'image'", + source: MediaSource, + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // inlineData part; absent otherwise. + "signature?": "string", +}); +export type ImageBlock = typeof ImageBlock.infer; + +const AudioBlock = type({ + type: "'audio'", + source: MediaSource, +}); + +const VideoBlock = type({ + type: "'video'", + source: MediaSource, +}); + +const DocumentBlock = type({ + type: "'document'", + source: MediaSource, + "title?": "string", + "context?": "string", +}); + +const ThinkingBlock = type({ + type: "'thinking'", + thinking: "string", + "signature?": "string", +}); + +/** + * A thinking block whose content the provider has filtered. The + * opaque `data` blob must echo back verbatim on every follow-up turn + * — Anthropic 400s the request if it changes or goes missing. Treat + * the bytes as opaque: do not log them and do not render them to + * users. + * + * Exported because `inference.thinking.redacted` events reference it + * by name. + */ +export const RedactedThinkingBlock = type({ + type: "'redacted_thinking'", + data: "string", +}); +export type RedactedThinkingBlock = typeof RedactedThinkingBlock.infer; + +/** + * A model-emitted refusal. Produced when a provider's strict-mode + * structured-outputs path declines to satisfy the requested schema — + * OpenAI's `delta.refusal` / `message.refusal` field is the canonical + * wire shape. The `reason` is the accumulated human-readable text the + * model emitted in lieu of conformant output. + * + * Refusal is semantically distinct from `inference.error`: the HTTP + * call succeeded and the model produced a coherent response, but that + * response is "I will not satisfy this schema" rather than schema- + * conformant content. Callers that distinguish policy declines from + * transport/protocol failures should branch on the block type rather + * than treat the assistant turn as an error. + * + * Exported because `inference.refusal.delta` events reference it by + * name and adapters construct RefusalBlocks in the finalized + * AssistantTurn from accumulated delta fragments. + */ +export const RefusalBlock = type({ + type: "'refusal'", + // Refusals must carry text — a zero-length reason corrupts the + // "human-readable text the model emitted in lieu of conformant + // output" contract and would round-trip indistinguishably from a + // refusal block whose payload was lost. The arktype constraint is + // belt-and-braces alongside the adapter's wire-boundary filter on + // empty `delta.refusal` chunks: synthetic fixtures or future + // adapters without that filter still cannot construct a vacuous + // refusal. + reason: "string > 0", +}); +export type RefusalBlock = typeof RefusalBlock.infer; +const ToolCallBlock = type({ + type: "'tool_call'", + id: "string", + name: "string", + arguments: "Record", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // functionCall part; absent otherwise. + "signature?": "string", +}); +/** + * Location of a citation's cited span within its source document. + * The unit of `start` and `end` varies by `kind`: + * - "page": 1-indexed page numbers (Anthropic `page_location`). + * - "char": UTF-16 character offsets, matching JS string semantics + * (Anthropic `char_location`; Gemini `groundingSupports[].segment`). + * - "content-block": index into a structured source's content blocks + * (Anthropic `content_block_location`). + */ +const CitationLocation = type({ + kind: "'page' | 'char' | 'content-block'", + start: "number", + end: "number", +}); + +const CitationSource = type({ + "title?": "string", + // Self-contained dereferenceable URL — populated by providers whose + // citations carry URLs directly (Gemini `groundingChunks[].web.uri`). + "uri?": "string", + // Back-pointer into the request's `documents` array, populated by + // providers that cite uploaded documents by position (Anthropic + // `document_index`). + "documentRef?": type({ index: "number" }), +}); + +/** + * A citation that supports a span of assistant text. Consumers + * receiving a CitationBlock without a paired source-block index MUST + * attribute it by adjacency to the nearest preceding TextBlock in the + * same turn. + * + * Citations are deliberately excluded from ToolResultBlock.content + * — they annotate model output, not tool output. + * + * Exported because `inference.citation` events reference it by name, + * following the same pattern as `AssistantTurn`, `ToolCall`, and + * `ToolResult`. See the `inference.citation` event docstring for how + * a paired source-block index is carried on the wire and consumed by + * the harness. + */ +export const CitationBlock = type({ + type: "'citation'", + // The exact substring of the preceding TextBlock this citation + // supports. Both providers emit it; required for inspection and + // for fallback offset reconstruction. + citedText: "string", + source: CitationSource, + "location?": CitationLocation, + // UTF-16 character offsets into the preceding TextBlock's text. + // Providers that emit offsets natively populate these directly; + // adapters that derive offsets from a cited substring populate + // them only when the substring appears unambiguously in the + // preceding text. Omitted when the offset cannot be determined. + "textOffset?": type({ start: "number", end: "number" }), +}); +export type CitationBlock = typeof CitationBlock.infer; + +/** + * A structured safety signal on model output or request filtering. + * + * The name `SafetyRatingBlock` follows the issue vocabulary; the + * payload is derived from the first real Gemini capture that engaged + * the structured classifier (2026-07-28). That wire shape is + * prompt-level only: + * + * `promptFeedback: { blockReason: "PROHIBITED_CONTENT" }` + * + * with no candidates and no per-category `safetyRatings` arrays. So + * this block carries `blockReason` and does **not** invent category / + * probability / blocked fields. When a future capture surfaces + * candidate-level ratings, extend the type from those bytes rather + * than from the API reference. + * + * Deliberately excluded from ToolResultBlock.content — safety + * signals annotate model/request filtering, not tool output. + * + * Exported because `inference.safety_rating` events reference it by + * name. + */ +export const SafetyRatingBlock = type({ + type: "'safety_rating'", + // Provider-native block reason string (observed: "PROHIBITED_CONTENT"). + // Open string so a new reason token does not force a type bump. + blockReason: "string > 0", +}); +export type SafetyRatingBlock = typeof SafetyRatingBlock.infer; + +/** + * The model's request to execute code via a server-side execution tool. + * Paired with a CodeExecutionResultBlock carrying the same `id` as the + * result's `requestId`. Streaming order within a single execution is + * `inference.code_execution.start` → zero or more + * `inference.code_execution.delta` → `inference.code_execution.result`, + * uninterrupted by other events that share the same `requestId`; events + * with different `requestId`s or for other block kinds at distinct + * `index`es may interleave. + * + * Exported because `inference.code_execution.start` references it by + * name. + */ +export const CodeExecutionRequestBlock = type({ + type: "'code_execution_request'", + // Identifier for the execution request. Populated from the + // provider's call id where one exists (Anthropic + // `srvtoolu_...`); synthesized by the adapter for providers that + // don't emit one (Gemini), using a deterministic per-response + // position-based scheme so replays match. + id: "string", + // Source code the model is asking to execute. + code: "string", + // Language hint. Absent when the provider does not emit one; + // adapters MUST NOT default this — callers narrow on its + // presence rather than fall through to a guessed language. + "language?": "string", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // executableCode part; absent otherwise. + "signature?": "string", +}); +export type CodeExecutionRequestBlock = typeof CodeExecutionRequestBlock.infer; + +/** + * The result of executing a CodeExecutionRequestBlock. The `requestId` + * back-points to the request block's `id`. Status is normalized across + * providers; raw provider signals (return code, native outcome string, + * abort reason) are preserved on optional fields for callers that need + * them. + * + * File outputs from code execution (e.g. generated plots that + * Anthropic returns in `code_execution_tool_result.content`) are NOT + * modeled by this block today. The block carries no field for them; + * surfacing file outputs is a separate concern. + * + * Exported because `inference.code_execution.result` references it by + * name. + */ +export const CodeExecutionResultBlock = type({ + type: "'code_execution_result'", + // Back-pointer to the originating CodeExecutionRequestBlock.id. + requestId: "string", + // Normalized outcome. Translated from provider-specific signals: + // - Anthropic: derived from `return_code` (0 → "ok", non-zero → + // "error") and `abort_reason` (non-null → "aborted" or + // "timeout" per the reason). + // - Gemini: derived from the `outcome` enum + // (OUTCOME_OK → "ok", OUTCOME_FAILED → "error", + // OUTCOME_DEADLINE_EXCEEDED → "timeout", etc.). + status: "'ok' | 'error' | 'aborted' | 'timeout'", + // Standard output. Providers that don't split stdout from stderr + // (Gemini) map their combined `output` here and leave `stderr` empty. + "stdout?": "string", + // Standard error. Empty for providers that don't split. + "stderr?": "string", + // Provider-native numeric return code when available + // (Anthropic `return_code`). Absent for providers whose outcome + // is enum-only (Gemini). + "returnCode?": "number", + // Provider-native outcome string preserved verbatim for callers + // that need the raw signal (Gemini `OUTCOME_OK` / + // `OUTCOME_FAILED` / `OUTCOME_DEADLINE_EXCEEDED` / ...). Absent + // when the provider does not emit one (Anthropic). + "providerOutcome?": "string", + // Human-readable reason populated when status is "aborted" + // (Anthropic `abort_reason`). Absent otherwise. + "abortReason?": "string", +}); +export type CodeExecutionResultBlock = typeof CodeExecutionResultBlock.infer; + +const ToolResultBlock = type({ + type: "'tool_result'", + callId: "string", + // Deliberately narrow: tool results carry user-facing media, not + // CitationBlocks (citations annotate the model's text output), not + // SafetyRatingBlocks (safety signals annotate model/request + // filtering), and not CodeExecution blocks (server-side code + // execution is a distinct lifecycle from the user-tool round-trip). + content: type + .or(TextBlock, ImageBlock, AudioBlock, VideoBlock, DocumentBlock) + .array(), + "detail?": "unknown", + "isError?": "boolean", +}); + +export const ContentBlock = type.or( + TextBlock, + ThinkingBlock, + RedactedThinkingBlock, + RefusalBlock, + ImageBlock, + AudioBlock, + VideoBlock, + DocumentBlock, + CitationBlock, + SafetyRatingBlock, + CodeExecutionRequestBlock, + CodeExecutionResultBlock, + ToolCallBlock, + ToolResultBlock, +); +export type ContentBlock = typeof ContentBlock.infer; + +/** + * A turn in the internal conversation history. The `model` field records + * which provider model produced this turn (present only on assistant + * turns). Used by cross-provider transformation to strip or preserve + * thinking blocks. + * + * (INFERENCE.md § Message Format) + */ +export type ConversationTurn = { + role: "user" | "assistant" | "system"; + content: ContentBlock[]; + model?: string; + timestamp: number; +}; + +/** + * A completed assistant turn returned in `inference.done`. Narrower type + * than ConversationTurn to make the inference boundary explicit. + */ +export const AssistantTurn = type({ + role: "'assistant'", + content: ContentBlock.array(), + model: "string", + timestamp: "number", +}); +export type AssistantTurn = typeof AssistantTurn.infer; + +// --------------------------------------------------------------------------- +// Error Classification (INFERENCE.md § Error Classification) +// --------------------------------------------------------------------------- + +/** + * Classified inference error. The category determines the reactor's default + * response; the director can override per its policy. + * + * (INFERENCE.md § Error Classification) + */ +export const InferenceError = type({ + category: type.enumerated( + "retryable", + "context_overflow", + "credential_failure", + "quota_exhausted", + "fatal", + "aborted", + "timeout", + "protocol_mismatch", + ), + message: "string", + "statusCode?": "number", + "retryAfterMs?": "number", + "raw?": "unknown", +}); +export type InferenceError = typeof InferenceError.infer; + +// --------------------------------------------------------------------------- +// Agent Reactor (INFERENCE.md § Agent Reactor) +// --------------------------------------------------------------------------- + +/** + * Gate types that can block the reactor. + * + * (INFERENCE.md § Gates) + */ +export const GateType = type.enumerated( + "approval", + "payment", + "credential", + "budget", + "child_completion", + "message_response", +); +export type GateType = typeof GateType.infer; + +/** + * Fork mode. `independent` creates a divergent reactor with its own context. + * `child` creates a reactor that reports results back to the parent. + * + * (INFERENCE.md § Forking) + */ +export const ForkMode = type.enumerated("independent", "child"); +export type ForkMode = typeof ForkMode.infer; + +/** + * A pending async operation registered in the reactor's async state. + * Correlates an outbound message (or payment/approval request) to the + * expected inbound response. + * + * (INFERENCE.md § Correlation) + */ +export type PendingOperation = { + correlationId: string; + kind: SignalKind; + registeredAt: number; + gateId: string; + /** + * Absolute deadline (epoch ms) for the gate that parks this operation. + * Persisted so that rehydration after a restart re-arms the gate with the + * remaining time against the original deadline rather than restarting the + * countdown. Absent for operations parked with no deadline. + */ + timeoutAt?: number; + /** + * The tool call that was suspended when this operation parked. Captured for + * `kind: "approval"` operations minted from the ask flow so the approved + * call can be re-run on resume. Absent for operations parked by the + * director path (async-tool pending markers), which carry no tool call. + */ + suspendedCall?: ToolCall; + /** + * Approver-facing snapshot of `suspendedCall`, built at the authz `ask` + * branch from the tool definition and the live arguments. A sibling of + * `suspendedCall`, not a widening of it: `suspendedCall` is the re-dispatch + * artifact, this is what the approver decides on. Present only for ask-rail + * operations that carry a `suspendedCall`; absent for async-tool pending + * markers. Threaded through the suspend hops to the hub co-write. + */ + approvalSnapshot?: ApprovalSnapshot; +}; + +/** + * Complete reactor state visible to the director decision function. + * + * `tokenUsage` is the cumulative usage across the session. + * + * `lastCycleUsage` and `lastCycleSource` describe the most recent + * *successful* inference call. They move together: both null before the + * first completion; every `inference.done` sets both atomically. + * `inference.error` does not clear either — the pair always reflects the + * last cycle that produced a well-defined turn and usage. The director's + * `afterInferenceDone` hook fires only on `inference.done`, so policy + * code never observes a torn or stale-vs-fresh window. + * + * The per-cycle values support compaction triggers that key off recent + * input cost rather than session totals, and state-aware policies (cost + * gating, budget caps, governance triggers) that need to attribute + * usage to the source that produced it. + * + * (INFERENCE.md § Agent Reactor › Director Decision Function) + */ +export type ReactorState = { + turns: ConversationTurn[]; + activeForks: { forkId: string; mode: ForkMode }[]; + pendingOperations: PendingOperation[]; + activeGates: { gateId: string; type: GateType; timeoutAt: number }[]; + tokenUsage: TokenUsage; + lastCycleUsage: TokenUsage | null; + lastCycleSource: LastCycleSource | null; + sessionId: string; +}; + +/** + * Actions the director can direct the reactor to take. + * + * (INFERENCE.md § Agent Reactor › Actions) + */ +export type ReactorAction = + | { + type: "infer"; + options?: InferenceOptions; + } + | { + type: "execute_tools"; + calls: ToolCall[]; + parallel?: boolean; + addToHistory?: boolean; + } + | { + type: "suspend"; + gate: { + type: GateType; + gateId: string; + timeoutMs: number; + correlationId?: string; + }; + } + | { + type: "fork"; + mode: ForkMode; + forkId: string; + } + | { + type: "emit"; + eventType: `custom.${string}`; + data: Record; + } + | { + type: "reply"; + content: string; + } + | { type: "checkpoint"; message: string } + | { type: "compact"; compactor: string; reason: string } + | { type: "wait" } + | { type: "done" }; + +/** + * The capabilities object passed to the director. Mirrors the `ReactorAction` + * union — provides a type-safe way for the director to construct actions. + * + * (INFERENCE.md § Agent Reactor › Director Decision Function) + */ +export type ReactorCapabilities = { + infer(options?: InferenceOptions): ReactorAction; + executeTools( + calls: ToolCall[], + parallel?: boolean, + addToHistory?: boolean, + ): ReactorAction; + suspend(gate: { + type: GateType; + gateId: string; + timeoutMs: number; + correlationId?: string; + }): ReactorAction; + fork(mode: ForkMode, forkId: string): ReactorAction; + emit( + eventType: `custom.${string}`, + data: Record, + ): ReactorAction; + reply(content: string): ReactorAction; + checkpoint(message?: string): ReactorAction; + compact(compactor: string, reason: string): ReactorAction; + wait(): ReactorAction; + done(): ReactorAction; +}; + +/** + * The inbound events delivered to the director decision function. + * + * `resume.execute_tools` is raised by the reactor when an approval resolves + * and a parked tool call must be re-run on resume. It carries the calls the + * reactor is about to dispatch so the director can seed its outstanding + * tool-result count before those calls' `tool.done` events arrive — the + * reactor drives the execution, the director counts the results. Without this + * seed the count would sit at zero and the first `tool.done` would drive an + * accidental re-inference off a negative count. + * + * `resume.tool_result` is raised by the reactor when a parked approval ends + * without running its tool — a rejected decision or a gate timeout. It carries + * a synthetic error tool result that answers the parked call so history stays + * well-formed; the director appends it and re-infers exactly once. No tool + * runs, so it seeds no outstanding-result count. + * + * (INFERENCE.md § Agent Reactor › Reactor Structure) + */ +export type ReactorInboundEvent = + | { type: "message.received"; message: InboundMessage } + | { + type: "inference.done"; + turn: AssistantTurn; + usage: TokenUsage; + source: LastCycleSource; + } + | { type: "inference.error"; error: InferenceError; partial: PartialMessage } + | { type: "tool.done"; result: ToolResult } + | { + type: "reactor.gate.cleared"; + gateId: string; + reason: "resolved" | "timeout" | "shutdown"; + } + | { type: "resume.execute_tools"; calls: ToolCall[] } + | { type: "resume.tool_result"; result: ToolResult } + | { type: "abort"; reason: AbortReason }; + +/** + * The core director is a single decision function: given an event and the + * current reactor state, return one or more actions. + * + * If the director throws, the reactor catches the exception, emits + * `reactor.error`, and initiates graceful shutdown. + * + * (INFERENCE.md § Reactor Director › Core Director) + */ +export interface ReactorDirector { + decide( + event: ReactorInboundEvent, + state: ReactorState, + capabilities: ReactorCapabilities, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Director Extension Hooks (INFERENCE.md § Reactor Director › Extension Hooks) +// --------------------------------------------------------------------------- + +/** + * Decision returned by a `BeforeToolExtension`. + * + * - `allow` — the tool proceeds. + * - `block` — the tool is answered with an error result carrying `reason`; + * the call is done. + * - `suspend` — the call is parked awaiting an external decision. The reactor + * registers `gate`, persists `pendingOp`, and does not answer the call: it + * is neither run nor error-completed. `gate.timeoutAt` is the absolute + * deadline (epoch ms) so the reactor can compute the remaining time; the + * `correlationId` on both `gate` and `pendingOp` ties an inbound resolution + * back to the suspension. + */ +export type BeforeToolDecision = + | { type: "allow" } + | { type: "block"; reason: string } + | { + type: "suspend"; + gate: { + type: GateType; + gateId: string; + correlationId: string; + timeoutAt: number; + }; + pendingOp: PendingOperation; + }; + +/** + * Extension that runs before a tool call is executed. Returns a + * `BeforeToolDecision`: `allow` lets the call run, `block` answers it with an + * error result, `suspend` parks it awaiting an external decision. + * + * `grantOneShot` registers a within-cycle bypass token keyed on a + * `ToolCall.id`: the next `beforeTool` for that id skips a suspension it would + * otherwise raise, consuming the token as it does so. It is optional because + * only extensions that can suspend a call have anything to bypass; extensions + * that never suspend omit it. + */ +export interface BeforeToolExtension { + beforeTool( + call: ToolCall, + state: ReactorState, + signal: AbortSignal, + ): Promise; + grantOneShot?(id: string): void; +} + +/** + * Extension that runs after a tool result is produced. Can modify the result + * (redaction, enrichment, audit logging). Extensions run in order. + */ +export interface AfterToolExtension { + afterTool( + result: ToolResult, + call: ToolCall, + state: ReactorState, + signal: AbortSignal, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Context Strategies: Transforms and Compactors +// (INFERENCE.md § Context Management, § Tool Result Lifecycle) +// --------------------------------------------------------------------------- + +/** + * Durable description of a single strategy invocation. Written to the + * per-cycle manifest in the context store so that future operators can + * reconstruct exactly which strategy made which change, with what + * parameters, and why. + * + * - `strategy` is the implementation name (e.g. `"size-cap"`). + * - `version` is the implementation version. Changes to the strategy's + * behavior bump the version so old manifest entries remain unambiguous. + * - `parameters` records the configuration the strategy ran with. + * - `reason` is a short machine-readable cause label + * (e.g. `"exceeded-cap"`, `"overflow-recovery"`). + * - `decisions` records strategy-specific details about what was actually + * done (e.g. the keep count, the spill key, the original byte size). + */ +export const TransformRecord = type({ + strategy: "string", + version: "string", + parameters: "Record", + reason: "string", + decisions: "Record", +}); +export type TransformRecord = typeof TransformRecord.infer; + +/** + * Per-invocation context passed to every `ContextStrategy.apply` call. + * `state` is the reactor's snapshot at the moment the strategy runs; + * `trigger` is a short label describing why the strategy was invoked + * (e.g. `"tool-result-ingest"`, `"pre-inference"`, `"director-request"`). + */ +export interface StrategyContext { + readonly state: ReactorState; + readonly trigger: string; +} + +/** + * Optional blob attachment emitted by a strategy. The reactor writes each + * blob to the context store's working tree via `ContextStore.writeBlob` + * (Phase 2) so the data is durable and migrates with the conversation. + */ +export type StrategyBlob = { + key: string; + bytes: Uint8Array; + contentType?: string; +}; + +/** + * Result returned by `ContextStrategy.apply`. Carries the transformed + * output, a `TransformRecord` describing what happened, and any blobs + * that should be persisted in the context store. + */ +export interface StrategyResult { + output: O; + record: TransformRecord; + blobs?: StrategyBlob[]; +} + +/** + * Generic base interface for content-mutating strategies. The role-specific + * aliases below specialize `I` and `O` for tool-result ingestion, pre- + * inference context shaping, and explicit compaction. + * + * Strategies are pure with respect to the context store: they describe what + * should change via their return value. The reactor decides where to write + * the result (history, prompt, manifest) and which blobs to persist. + */ +export interface ContextStrategy { + readonly name: string; + readonly version: string; + apply(input: I, ctx: StrategyContext): Promise>; +} + +/** + * Runs on each tool result entering history. Output is appended to the + * conversation; any emitted blobs are written to the context store's + * `tool-output/` directory. + */ +export type ToolResultTransform = ContextStrategy< + { call: ToolCall; result: ToolResult }, + ToolResult +>; + +/** + * Runs in order before every inference call, producing the materialized + * prompt. Output is written to `prompt.jsonl` for that cycle; the durable + * history in `turns.jsonl` is left untouched. + * + * (INFERENCE.md § Async State Awareness › Pending Status Injection) + */ +export type ContextTransform = ContextStrategy< + ConversationTurn[], + ConversationTurn[] +>; + +/** + * Named compaction strategy. Registered in a registry on the reactor and + * invoked explicitly via the director's `compact` action. Output overwrites + * `turns.jsonl`; a `TransformRecord` is appended to the manifest. + */ +export type Compactor = ContextStrategy; + +// --------------------------------------------------------------------------- +// Blob Reader (INFERENCE.md § Tool Result Lifecycle) +// --------------------------------------------------------------------------- + +/** + * Read-only capability for resolving `tool-output:///{callId}` URIs to the + * underlying blob bytes. A `ToolResultTransform` that spills oversized tool + * output writes a blob via `ContextStore.writeBlob` and returns a pointer of + * the form `tool-output:///{callId}`; the agent's read tool reaches the spill + * by calling `BlobReader.read(uri)`. + * + * The URI scheme is deliberately rigid: + * + * - Scheme: `tool-output` + * - Authority: empty (the `///` makes pathname carry the callId) + * - Path: `/{callId}` — preserves case so provider-assigned callIds with + * uppercase letters survive parsing + * - Query and fragment: rejected + * + * Any deviation (different scheme, missing or non-empty hostname, extra path + * segments, search string, or fragment) throws. Missing blobs throw. + * `BlobReader` never accepts a filesystem path; the agent has no direct view + * of the context store's working tree. + */ +export interface BlobReader { + /** + * Resolve `uri` to the underlying blob bytes. Throws if the URI is not a + * well-formed `tool-output:///{callId}` reference or if no blob exists for + * the extracted callId. + */ + read(uri: string): Promise; +} + +/** Source for blob bytes used by `createBlobReader`. */ +export interface BlobSource { + readBlob(key: string, signal?: AbortSignal): Promise; +} + +/** + * Parse a `tool-output:///{callId}` URI and return the callId. Throws on any + * deviation from the documented shape: wrong scheme, non-empty authority, + * missing or extra path components, search string, or fragment. + * + * The two-slash form `tool-output://abc` is rejected because the URL parser + * lowercases the hostname, which silently corrupts provider-assigned callIds + * that contain uppercase letters. The three-slash form puts the callId in + * `pathname`, where case is preserved. + */ +export function parseToolOutputURI(uri: string): string { + let parsed: URL; + try { + parsed = new URL(uri); + } catch (cause) { + throw new Error(`invalid tool-output URI: ${uri}`, { cause }); + } + if (parsed.protocol !== "tool-output:") { + throw new Error( + `invalid tool-output URI scheme: expected "tool-output:", got "${parsed.protocol}"`, + ); + } + if (parsed.hostname !== "") { + throw new Error( + `invalid tool-output URI: authority must be empty (use the form tool-output:///{callId}), got "${parsed.hostname}"`, + ); + } + if (parsed.search !== "") { + throw new Error( + `invalid tool-output URI: query string is not allowed, got "${parsed.search}"`, + ); + } + if (parsed.hash !== "") { + throw new Error( + `invalid tool-output URI: fragment is not allowed, got "${parsed.hash}"`, + ); + } + const path = parsed.pathname; + if (!path.startsWith("/")) { + throw new Error(`invalid tool-output URI: empty path: ${uri}`); + } + const callId = path.slice(1); + if (callId === "") { + throw new Error(`invalid tool-output URI: missing callId: ${uri}`); + } + if (callId.includes("/")) { + throw new Error( + `invalid tool-output URI: path must contain a single callId segment, got "${callId}"`, + ); + } + return callId; +} + +/** + * Construct a `BlobReader` that resolves `tool-output:///{callId}` URIs by + * delegating to `source.readBlob(callId)`. The most common source is a + * `ContextStore` (Phase 2 added `readBlob` to that interface), but any object + * implementing `BlobSource` works — this keeps tests trivial. + * + * URI parsing is performed in this layer; the source only ever sees the + * extracted callId. Missing blobs surface as whatever error the source + * raises (`ContextStore.readBlob` already throws for unknown keys). + */ +export function createBlobReader(source: BlobSource): BlobReader { + return { + async read(uri: string): Promise { + const callId = parseToolOutputURI(uri); + return source.readBlob(callId); + }, + }; +} + +// --------------------------------------------------------------------------- +// Abort Reasons (INFERENCE.md § Abort Handling) +// --------------------------------------------------------------------------- + +/** + * Reason codes for the `abort` reactor event. The reason determines the + * appropriate cleanup action. + * + * (INFERENCE.md § Abort Handling › Abort Reasons) + */ +export const AbortReason = type.enumerated( + "user_disconnect", + "wallet_exhaustion", + "admin_kill", + "session_timeout", + "credential_revocation", +); +export type AbortReason = typeof AbortReason.infer; + +// --------------------------------------------------------------------------- +// Inference Source (INFERENCE.md § Providers) +// --------------------------------------------------------------------------- + +/** + * Model-bound default knobs for an inference source. Per-call + * `InferenceOptions.X` overrides `defaults.X`; the merge happens once at + * the top of `runInference` before the adapter sees anything. New fields + * land here as separately-scoped issues. + */ +export const InferenceSourceDefaults = type({ + "maxTokens?": "number", + // A bag of provider-native knobs the caller wants merged into the + // outbound request body (Anthropic's `metadata.user_id`, + // OpenAI's `user`, Gemini's `safetySettings`, etc.). Adapters that + // recognize keys translate; unrecognized keys are passed through or + // dropped per the adapter's documented behavior. The merge into + // per-call `InferenceOptions.providerOptions` is shallow — a per- + // call providerOptions object wholesale replaces the source-bound + // one, it does not deep-merge per key. + "providerOptions?": "Record", +}); +export type InferenceSourceDefaults = typeof InferenceSourceDefaults.infer; + +/** + * A specific (provider, model) bundle the agent runtime can route to. + * Carries wire reachability, credentials, the model identity at the + * provider, and the model-bound default knobs. + * + * `id` is the catalog offering's primary key, set by the resolver from the + * matched offering. It is the routing key used by `AgentConfig.defaultSource` + * and `Agent.setSource`. + * + * Multi-model providers become multiple sources — `model` is part of the + * identity, not an optional override. + * + * `capabilities` is carried for the selection-policy layer (the model + * selector consumes it). The runtime ignores it; populating the field + * later is not a wire-format change. + * + * `quirks` is the opaque per-deployment bag of provider-specific adapter + * accommodations. The harness reads it once, at adapter instantiation, and + * passes it to `AdapterRegistry.resolve` as a sibling of the slim + * `LastCycleSource` — quirks are deliberately kept off `LastCycleSource`, + * which rides on every usage event. The field is present-and-populated or + * absent; it is never `null`. A source row with no quirks stores SQL `NULL`, + * and the catalog resolver translates that absence into an omitted key here, + * so downstream code sees `undefined`, never `null`. + * + * (INFERENCE.md § Providers) + */ +export const InferenceSource = type({ + id: "string", + provider: "string", + baseURL: "string", + // Reference into the run's credential-material cell. The provider's secret + // (formerly an inline `apiKey`) is resolved from that cell by `credentialId` + // at call time, so the source config carries no secret and the child never + // holds the key inline. The same cell backs binding credentials. + credentialId: "string", + model: "string", + "defaults?": InferenceSourceDefaults, + "capabilities?": "string[]", + "quirks?": "Record", +}); +export type InferenceSource = typeof InferenceSource.infer; + +/** + * Replace every field on `active` with the corresponding field from + * `next`, in place. Optional fields (`defaults`, `capabilities`, + * `quirks`) are `delete`d from `active` when absent on `next` so the + * swap is exact — no stale value from a previous rotation can survive. + * + * Used by both the agent's source registry and the harness's source + * hot-swap path to mutate the single shared `InferenceSource` object the + * reactor reads lazily at the start of each inference call. Putting the + * field list in one place means the next field added to + * `InferenceSource` only has to be remembered here. + */ +export function applyInferenceSourceFields( + active: InferenceSource, + next: InferenceSource, +): void { + active.id = next.id; + active.provider = next.provider; + active.baseURL = next.baseURL; + active.credentialId = next.credentialId; + active.model = next.model; + if (next.defaults !== undefined) { + active.defaults = next.defaults; + } else { + delete active.defaults; + } + if (next.capabilities !== undefined) { + active.capabilities = next.capabilities; + } else { + delete active.capabilities; + } + if (next.quirks !== undefined) { + active.quirks = next.quirks; + } else { + delete active.quirks; + } + + // Compile-time exhaustiveness check. `Required<>` forces optional + // keys to also be required in the guard — so a future optional field + // (e.g. `region?: string`) added to `InferenceSource` without being + // handled above is flagged by TypeScript, not silently dropped. + const _handled: { readonly [K in keyof Required]: true } = { + id: true, + provider: true, + baseURL: true, + credentialId: true, + model: true, + defaults: true, + capabilities: true, + quirks: true, + }; + void _handled; +} + +/** + * Outcome of a `RetryPolicy` consultation. Either abort the call + * (surface the most recent `inference.error` to the caller), or retry + * after `delayMs` milliseconds, measured against the harness Scheduler. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetryDecision = + | { kind: "abort" } + | { kind: "retry"; delayMs: number }; + +/** + * Context supplied to a `RetryPolicy` each time an attempt produces an + * `inference.error`. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetrySituation = { + /** The classified error the most recent attempt produced. */ + readonly error: InferenceError; + /** + * 1-indexed attempt counter. The first failure has `attempt: 1`; + * the second failure (after one retry) has `attempt: 2`; and so on. + */ + readonly attempt: number; + /** + * Milliseconds since the *first* attempt of this call started, + * measured via the harness `Scheduler.now()`. The default Scheduler + * uses `performance.now()` (sub-millisecond resolution), so the + * value may be fractional; virtual-clock test schedulers report + * integer virtual time. Both are valid; policies that compare + * against integer thresholds should `Math.floor` if they need that. + */ + readonly elapsedMs: number; +}; + +/** + * Per-call retry policy. The harness invokes the policy once per + * `inference.error` an attempt produces, in 1-indexed attempt order. + * Returning `{ kind: "abort" }` ends the call by surfacing the most + * recent error to the caller; returning `{ kind: "retry", delayMs }` + * causes the harness to discard the failed attempt's events, sleep + * `delayMs` milliseconds against the Scheduler, and re-issue the + * underlying HTTP request with the identical body. The policy may be + * async; the harness awaits the returned `Promise` if + * it is a thenable. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetryPolicy = ( + situation: RetrySituation, +) => RetryDecision | Promise; + +/** + * Provider-neutral reasoning effort. No provider accepts every name as a + * wire value, so each adapter maps the name onto what its model accepts. + * There is no "off": `thinking.enabled: false` already says that. + * + * (INFERENCE.md § Thinking and Reasoning) + */ +export const InferenceEffort = type.enumerated("low", "medium", "high", "max"); +export type InferenceEffort = typeof InferenceEffort.infer; + +/** + * The subset of `InferenceOptions` a caller may set for a single send (a + * workflow step's resolved `inference` selector). Excludes anything that + * displaces the deployed agent definition (`systemPrompt`, `tools`, + * `providerOptions`). + */ +export const PerCallInferenceOptions = type({ + "+": "reject", + "maxTokens?": "number.integer > 0", + "temperature?": "number >= 0", + "thinking?": { + "+": "reject", + enabled: "boolean", + "budgetTokens?": "number.integer > 0", + }, + // Reasoning intent. Effort and `thinking.budgetTokens` are two spellings + // of one intent; each adapter resolves them to the field its model + // accepts. When omitted the provider default applies. + "effort?": InferenceEffort, +}); +export type PerCallInferenceOptions = typeof PerCallInferenceOptions.infer; + +/** + * Options for a single inference call. Override the defaults from the agent + * configuration on a per-call basis. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type InferenceOptions = PerCallInferenceOptions & { + systemPrompt?: string; + tools?: ToolDefinition[]; + /** + * Modalities the caller wants the model to emit. Adapters translate + * to the provider-native shape (Gemini's + * `generationConfig.responseModalities` accepts `"TEXT"` / `"IMAGE"` + * uppercase; see `packages/inference-discovery-google-genai/sessions/ + * google-genai/gemini-2.5-flash-image/image-output/exchanges/0/request.json` + * for the captured shape). Providers that do not expose a modality + * switch ignore the + * field. When omitted the provider's default modalities apply. + */ + responseModalities?: ("text" | "image" | "audio")[]; + /** + * Structured-output constraint. Asks the model to produce text, free- + * form JSON, or JSON conforming to a specific schema. Adapters + * translate to the provider-native wire shape: + * + * - **OpenAI** (`response_format`): + * - `text` → `{ type: "text" }` + * - `json` → `{ type: "json_object" }` + * - `json-schema` → `{ type: "json_schema", json_schema: { name, schema, strict } }` + * When the model declines in strict mode, the wire emits + * `delta.refusal` chunks; the adapter surfaces them as + * `inference.refusal.delta` events and a final `RefusalBlock` in + * the assistant turn's `content[]`. + * - **Google GenAI** (`generationConfig`): + * - `text` → no constraint (default). + * - `json` → `{ responseMimeType: "application/json" }`. + * - `json-schema` → `{ responseMimeType: "application/json", responseSchema: }`. + * `name` and `strict` are OpenAI-specific and have no Gemini + * counterpart; adapters ignore them. Gemini enforces a subset of + * JSON Schema (no `oneOf`, limited `pattern`, no `$ref`, etc.) — + * the adapter forwards the schema verbatim and surfaces Gemini's + * HTTP error if the subset is violated. + * - **Anthropic**: no native structured-output API. + * - `text` is a no-op (the default). + * - `json` and `json-schema` throw at the adapter boundary; there + * is no shim that synthesizes a tool to extract structured + * output. + * + * When omitted the provider's default applies (typically free-form + * text). + */ + responseFormat?: + | { kind: "text" } + | { kind: "json" } + | { + kind: "json-schema"; + name: string; + schema: unknown; + strict?: boolean; + }; + /** + * A bag of provider-native knobs the adapter merges into the outbound + * request body. Primary home is `InferenceSourceDefaults.providerOptions` + * (model-bound); this field exists for per-call overrides through the + * standard merge precedence at the top of `runInference`. The merge is + * shallow: a per-call providerOptions object wholesale replaces the + * source-bound one, it does not deep-merge per key. + */ + providerOptions?: Record; + /** + * Per-call inactivity timeout in milliseconds. If the harness yields no + * event (other than `inference.start`) for this many ms, the underlying + * fetch is aborted and the call ends with `inference.error` of category + * `"timeout"`. Default 120_000 (2 min). Tune higher for reasoning models + * that exhibit long silent-thinking stretches between token bursts; tune + * lower to fail fast. `0` arms the timer to fire on the next tick (a + * "fail-fast even if the fetch is instant" mode useful in tests). + */ + inactivityTimeoutMs?: number; + /** + * Per-call total wall-clock cap in milliseconds. Starts at fetch. + * Default 600_000 (10 min). Backstop for streams that keep emitting + * forever without terminating. Same error category as `inactivityTimeoutMs`. + * `0` arms the timer to fire on the next tick. + */ + totalTimeoutMs?: number; + /** + * Per-call mechanical retry policy. Consulted once per attempt that + * ends in `inference.error`; see `RetryPolicy` for the contract. If + * omitted, a built-in default policy is applied. + */ + retryPolicy?: RetryPolicy; +}; + +// --------------------------------------------------------------------------- +// Context Store (INFERENCE.md § Context Management › Context Store, +// ARCHITECTURE.md § Change History) +// --------------------------------------------------------------------------- + +/** + * A named commit point in the context store. Corresponds to a git commit. + * + * (ARCHITECTURE.md § Change History › Named Checkpoints) + */ +export type ContextCommit = { + hash: string; + message: string; + timestamp: number; + parentHash?: string; +}; + +/** + * The state of an active connector thread. The connector is one durable + * thread per agent; participants accumulate as they speak. Persisted + * alongside the conversation context so the thread survives sidecar + * restarts. + * + * `replyTo` is the most recent speaker — the primary recipient (`to`) + * on the next outbound reply. `cc` is every other participant who has + * spoken on the thread, deduplicated, in arrival order — they ride as + * `cc` on the next outbound reply so everyone stays in the loop. + * `subject` is set when the thread starts and preserved for its life. + * + * Defined as an arktype so the wire layer (sidecar↔hub frames) and + * other parsing boundaries can validate snapshots without + * re-declaring the shape. + */ +export const ConnectorThreadState = type({ + "threadRoot?": "string", + "lastMessageId?": "string", + replyTo: "string", + cc: "string[]", + "subject?": "string", +}); +export type ConnectorThreadState = typeof ConnectorThreadState.infer; + +/** + * The context store interface. Implementations back the store with git + * (filesystem, in-memory, or virtual) depending on the execution environment. + * The reactor accepts any implementation that satisfies this interface. + * + * The store holds the turn history and reactor metadata. Forking creates + * a git branch. Compaction commits the compacted history. + * + * (INFERENCE.md § Context Management › Context Store) + */ +export interface ContextStore { + /** + * Load the current turn history and reactor metadata from the store. + * Called during reactor initialization. + */ + load(signal?: AbortSignal): Promise<{ + turns: ConversationTurn[]; + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }>; + + /** + * Buffer connector thread state for the next commit. The harness calls + * this before each checkpoint so that connector state is persisted + * atomically with the conversation context. + */ + setConnectorState(state: ConnectorThreadState | null): void; + + /** + * Commit whatever currently lives in the working tree, using the supplied + * commit message. The reactor's per-cycle checkpoint routes through this + * overload after writing the per-cycle files via `writeTurns`, + * `writePrompt`, `writeResponse`, `writeManifest`, and any `writeBlob` + * calls produced by transforms. + */ + commit( + options: { message: string }, + signal?: AbortSignal, + ): Promise; + + /** + * Create a branch for a fork operation. The branch starts from the current + * HEAD commit. + */ + branch(name: string, signal?: AbortSignal): Promise; + + /** + * List recent commits. Used by the agent's history query tools. + */ + log(limit?: number, signal?: AbortSignal): Promise; + + /** + * Read the turn history at a specific commit hash. Used for history + * inspection and rollback. + */ + readAt(hash: string, signal?: AbortSignal): Promise; + + /** + * Write an opaque blob to the working tree under `tool-output/`. Used by + * `ToolResultTransform`s that spill oversized payloads out of the inline + * conversation. The file is staged at the next `commit({ message })`. + * + * `key` is sanitized for filesystem safety; callers should pass the tool + * call id. `contentType` selects a file extension when known. + */ + writeBlob( + key: string, + bytes: Uint8Array, + contentType?: string, + signal?: AbortSignal, + ): Promise; + + /** + * Read a blob previously written via `writeBlob`. Throws if no blob with + * that key exists. + */ + readBlob(key: string, signal?: AbortSignal): Promise; + + /** + * Overwrite `prompt.jsonl` with the materialized prompt for the current + * inference cycle. One `ConversationTurn` per line. Staged at the next + * `commit({ message })`. + */ + writePrompt(turns: ConversationTurn[], signal?: AbortSignal): Promise; + + /** + * Overwrite `response.jsonl` with the assistant turn returned for the + * current cycle. Single-line JSONL for consistency with the per-cycle file + * conventions. Staged at the next `commit({ message })`. + */ + writeResponse(turn: AssistantTurn, signal?: AbortSignal): Promise; + + /** + * Overwrite `manifest.jsonl` with the ordered transform records produced + * for the current cycle. One `TransformRecord` per line. Staged at the + * next `commit({ message })`. + */ + writeManifest( + records: TransformRecord[], + signal?: AbortSignal, + ): Promise; + + /** + * Overwrite `turns.jsonl` with the durable conversation history. One + * `ConversationTurn` per line. Staged at the next `commit({ message })`. + */ + writeTurns(turns: ConversationTurn[], signal?: AbortSignal): Promise; + + /** + * Overwrite `metadata.json` with non-turn-shaped reactor state needed for + * restart: pending async operations and cumulative token usage. The store + * combines this with the most recently buffered connector state (from + * `setConnectorState`) and writes the merged payload. Staged at the next + * `commit({ message })`. + */ + writeMetadata( + metadata: { + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + }, + signal?: AbortSignal, + ): Promise; + + /** + * Read manifest entries from the most recent `limit` commits that contain + * a `manifest.jsonl`. Newest commit first; records within a commit are + * returned in their natural in-file order (chronological per-cycle). + */ + readManifestHistory( + limit: number, + signal?: AbortSignal, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Audit Store (INTR-4 § Audit Trail) +// --------------------------------------------------------------------------- + +/** + * Persistent store for tool invocation audit records. Separated from + * ContextStore so the audit capability is opt-in at the composition + * layer. The isogit implementation writes audit records as individual + * JSON files in the same git repo used for context storage. + */ +export interface AuditStore { + /** + * Persist a batch of audit records. Called at checkpoint boundaries + * with all records accumulated since the last checkpoint. + */ + commitAudit(records: AuditRecord[], signal?: AbortSignal): Promise; + + /** + * Load audit records for a session. Returns all records matching + * the given sessionId, ordered by seq. + */ + loadAudit(sessionId: string, signal?: AbortSignal): Promise; + + /** + * Persist a batch of error records. Called at checkpoint boundaries + * and shutdown with all error records accumulated since the last flush. + */ + commitErrors(records: ErrorRecord[], signal?: AbortSignal): Promise; +} + +// --------------------------------------------------------------------------- +// Agent / Harness Configuration (ARCHITECTURE.md § Agent Harness) +// --------------------------------------------------------------------------- + +/** + * Configured tool definition exposed to the model. The harness registers + * available tools; the reactor passes this list to the inference provider as + * part of each request. + * + * (ARCHITECTURE.md § Agent Harness › Tools) + */ +export const ToolDefinition = type({ + name: "string", + description: "string", + inputSchema: "Record", +}); +export type ToolDefinition = typeof ToolDefinition.infer; + +/** + * Agent harness configuration. Assembled from the agent definition package + * and capability grants during harness initialization. + * + * `principalId` is the agent's principal in the hub's authorization model. + * The sidecar needs it to reconstruct the in-memory grant store on restart + * (the store's `collectGrants` filters by principal). + * + * `grants` uses `WireGrantRule` because this type arrives over JSON where + * `GrantRule.expiresAt` is serialized as a string. The wire validator + * coerces strings back to Date instances. + * + * (ARCHITECTURE.md § Agent Harness) + */ +export const HarnessConfig = type({ + sessionId: "string", + agentId: "string", + tenantId: "string", + principalId: "string", + agentAddress: "string", + systemPrompt: "string", + tools: ToolDefinition.array(), + grants: WireGrantRule.array(), + sources: InferenceSource.array(), + defaultSource: "string", + "sessionChannelEnabled?": "boolean", +}); +export type HarnessConfig = typeof HarnessConfig.infer; diff --git a/vendor/intx/types/src/runtime.ts b/vendor/intx/types/src/runtime.ts index 6bb000fcd2..18a17402c2 100644 --- a/vendor/intx/types/src/runtime.ts +++ b/vendor/intx/types/src/runtime.ts @@ -1,3038 +1,5 @@ -// Runtime definitions for the Interchange agent harness. -// -// Wire-facing data types (AbortReason, InferenceSource, ToolDefinition, -// HarnessConfig) are arktype validators so they can be composed into -// WebSocket frame validators and used for runtime validation at parse -// boundaries. Behavioral interfaces (ContextStore, MessageTransport, -// ToolRunner, etc.) remain plain TypeScript. - -import { type } from "arktype"; -import type { AuditRecord, ErrorRecord } from "./audit"; -import { WireGrantRule } from "./grant-wire"; -import type { SignalKind } from "./signals"; - -// --------------------------------------------------------------------------- -// Cryptographic Identity (ARCHITECTURE.md § Cryptographic Identity, -// IMPLEMENTATION.md § Cryptographic Identity: Key Formats) -// --------------------------------------------------------------------------- - -/** - * An Ed25519 key pair as raw bytes. The private key is 32 bytes; the public - * key is the corresponding 32-byte compressed point. - * - * Key material is represented as Uint8Array throughout so it stays - * runtime-agnostic (Bun, Node, browser) and never accidentally leaks through - * JSON serialization. - */ -export type KeyPair = { - privateKey: Uint8Array; - publicKey: Uint8Array; -}; - -/** - * A key-bound cryptographic provider. Each instance is constructed with a - * specific agent's Ed25519 key pair and holds the private key internally. - * - * `sign` uses the instance's own private key — no key parameter is accepted. - * `verify` accepts a public key parameter so the holder can verify messages - * from arbitrary senders without constructing a new provider instance. - * - * The in-memory transport stores one CryptoProvider per registered agent and - * calls `crypto.sign(content)` during `send()` without passing keys around. - * - * Key formats (IMPLEMENTATION.md): - * - Ed25519 in SSH format — control plane interactions - * - Ed25519 in PGP format — message-level signatures over SMTP/IMAP - * - Ed25519 in X.509 format — TLS mutual auth certificates - * - * `getPublicKey` returns the raw public key bytes so callers can publish - * them to the control plane or embed them in discovery metadata. - */ -export interface CryptoProvider { - /** - * Sign `content` with the instance's private key. Returns the Ed25519 - * detached signature as raw bytes. - */ - sign(content: Uint8Array): Promise; - - /** - * Sign `payload` with the instance's private key using the SSH signature - * envelope (sshsig). Returns an ASCII-armored SSH SIGNATURE block suitable - * for the `gpgsig` header of a git commit or any other site that consumes - * `git verify-commit`-compatible signatures. The framing differs from - * `sign`'s raw output; callers that need either format should pick the - * matching method rather than reframing the result themselves. - */ - signSSH(payload: string): Promise; - - /** - * Verify that `signature` over `content` was produced by `publicKey`. - * Returns true if the signature is valid; false otherwise. - */ - verify( - content: Uint8Array, - signature: Uint8Array, - publicKey: Uint8Array, - ): Promise; - - /** The public key for this instance, as raw bytes. */ - getPublicKey(): Uint8Array; -} - -/** - * Generate a fresh Ed25519 key pair. The returned pair is used to construct - * a CryptoProvider instance. - */ -export type GenerateKeyPair = () => Promise; - -// --------------------------------------------------------------------------- -// Message Transport (MESSAGE.md § Transport Interface) -// --------------------------------------------------------------------------- - -/** - * Opaque reference to a message in a specific mailbox. Carries the IMAP UID - * and the mailbox name. Passed to fetch, flag, and move operations without - * requiring re-search. - */ -export type MessageRef = { - uid: number; - mailbox: string; -}; - -/** - * Interchange payload types as defined in MESSAGE.md § Payload Types. - * The type field in structured messages matches the Interchange-Type header. - * - * Exposed as both an arktype validator (for runtime validation at parse - * boundaries and tool-argument schemas) and a derived TypeScript union. - */ -export const InterchangeType = type.enumerated( - "conversation.message", - "conversation.join", - "conversation.leave", - "offering.request", - "offering.response", - "offering.error", - "offering.discover", - "offering.catalog", - "payment.required", - "payment.receipt", - "payment.verified", - "approval.request", - "approval.granted", - "approval.denied", - "system.health", - "system.register", - "system.deregister", - "system.credential.refresh", -); -export type InterchangeType = typeof InterchangeType.infer; - -const CONVERSATION_TYPES: ReadonlySet = new Set([ - "conversation.message", - "conversation.join", - "conversation.leave", -]); - -/** - * Membership is enumerated rather than derived from the `conversation.` name - * prefix, so a member added to InterchangeType is not a conversation type until - * it is listed in CONVERSATION_TYPES. - */ -export function isConversationType(value: InterchangeType): boolean { - return CONVERSATION_TYPES.has(value); -} - -/** Attachment for an outbound message. Content is raw bytes. */ -export type MessageAttachment = { - name: string; - contentType: string; - data: Uint8Array; -}; - -/** - * A message the harness submits for delivery via SMTP. The transport - * assembles the PGP/MIME multipart structure, signs it with the agent's - * CryptoProvider, and submits it. - * - * Conversation types (conversation.*) carry `content` as text/plain. - * Structured types carry `payload` as application/vnd.interchange+json. - * Providing both is an error. - * - * (MESSAGE.md § Transport Interface › Outbound) - */ -export type OutboundMessage = { - to: string | string[]; - cc?: string | string[]; - subject?: string; - - type: InterchangeType; - - /** Plain text body — used when type is a conversation.* type. */ - content?: string; - - /** Structured JSON body — used when type is a non-conversation type. */ - payload?: Record; - - /** Human-readable summary for structured messages (the text/plain part). */ - summary?: string; - - attachments?: MessageAttachment[]; - - /** Message-ID of the message being replied to. */ - inReplyTo?: string; - - /** - * The RFC 5322 References chain for a threaded reply: the parent's own - * References plus the parent's Message-ID, in order. When present the - * transport ships it verbatim (after appending `inReplyTo` if it is not - * already the tail) rather than deriving a single-element `[inReplyTo]` - * chain, so a reply carries the full conversational ancestry. Absent for a - * non-reply or a reply whose parent could not be located. - */ - references?: string[]; - - /** Correlation ID linking this message to a pending async request. */ - correlationId?: string; - - /** Reactor session ID from the Interchange-Session-ID header. */ - sessionId?: string; - - /** Tenant ID for the Interchange-Tenant-ID header. */ - tenantId?: string; -}; - -/** - * Receipt returned by `send()`. Contains the assigned Message-ID and - * delivery status. - * - * (MESSAGE.md § Transport Interface › Outbound) - */ -export type SendReceipt = { - messageId: string; - status: "delivered" | "queued"; -}; - -/** - * Parsed headers from an inbound message. Field names follow RFC 5322 and - * the Interchange-specific header conventions from MESSAGE.md § Headers. - * - * `from` is optional because a message can arrive carrying no originator at - * all, and there is no string that honestly stands for one. - */ -export type MessageHeaders = { - from?: string; - to: string[]; - cc?: string[]; - date?: string; - messageId?: string; - inReplyTo?: string; - references?: string[]; - subject?: string; - listId?: string; - - interchangeType?: InterchangeType; - interchangeCorrelationId?: string; - interchangeTenantId?: string; - interchangeAgentId?: string; - interchangeSessionId?: string; - interchangeOfferingId?: string; - interchangeSchemaVersion?: string; - - traceparent?: string; - tracestate?: string; -}; - -/** - * Signature verification status of an inbound message. - * - * - `valid` — signature verified against the sender's public key - * - `invalid` — signature check failed (tampering or wrong key) - * - `unknown` — public key not available for verification - * - `missing` — message was not signed - * - * (MESSAGE.md § Transport Interface › fetchFull) - */ -export const SignatureStatus = type.enumerated( - "valid", - "invalid", - "unknown", - "missing", -); -export type SignatureStatus = typeof SignatureStatus.infer; - -/** - * The admission outcome of an inbound message. This is the single vocabulary a - * delivery decision keys on, distinct from the two-axis signature verdict that - * produces it. - * - * - `clean` — nothing suspect; always admitted - * - `untrustedFrom` — the visible `From` is present but cannot be reduced to a - * single addr-spec - * - `mismatchedFrom` — the visible `From` names a different identity from the - * sender the hub stamped - * - `absentFrom` — the message carries no usable `From` - * - `invalid` — the signature check failed (tampering or the wrong key) - * - `missing` — the message carried no signature - * - `unknown` — no key was available to verify against - * - `error` — a fault stopped the check from running at all; always rejected - */ -export const InboundMailOutcome = type.enumerated( - "clean", - "untrustedFrom", - "mismatchedFrom", - "absentFrom", - "invalid", - "missing", - "unknown", - "error", -); -export type InboundMailOutcome = typeof InboundMailOutcome.infer; - -/** - * The subset of {@link InboundMailOutcome} a workflow author may relax to admit - * a message that would otherwise be rejected. It omits `clean` (which always - * admits, so there is nothing to relax) and `error` (pinned to reject, since a - * fault we could not check through is never something an author should be able - * to wave past). A per-workflow policy keys on exactly these outcomes. - */ -export const AuthorControllableOutcome = type.enumerated( - "untrustedFrom", - "mismatchedFrom", - "absentFrom", - "invalid", - "missing", - "unknown", -); -export type AuthorControllableOutcome = typeof AuthorControllableOutcome.infer; - -/** - * A per-workflow inbound-mail admission policy: for each admission outcome the - * author may control, whether a message that raises that outcome as a finding is - * `reject`ed or `admit`ted. The key set is exactly the - * {@link AuthorControllableOutcome} values -- `clean` (always admitted) and - * `error` (pinned to reject) are deliberately not keys. - * - * The object is SPARSE: every key is optional, and an omitted key is NOT a - * default of any kind here. It is left for a later resolution step to interpret - * an absent outcome. Keeping it sparse means the content hash covers only what - * the author actually declared, so a definition that omits the policy hashes - * identically to one authored before the field existed. Undeclared keys are - * rejected so a typo such as `clean` or `errror` fails at the wire boundary - * rather than riding through as an inert unknown key. - */ -export const InboundMailPolicy = type({ - "untrustedFrom?": "'reject' | 'admit'", - "mismatchedFrom?": "'reject' | 'admit'", - "absentFrom?": "'reject' | 'admit'", - "invalid?": "'reject' | 'admit'", - "missing?": "'reject' | 'admit'", - "unknown?": "'reject' | 'admit'", -}).onUndeclaredKey("reject"); -export type InboundMailPolicy = typeof InboundMailPolicy.infer; - -type AssertEqual = [A] extends [B] - ? [B] extends [A] - ? true - : false - : false; - -/** - * The three outcome vocabularies must move together, but nothing else connects - * them: each spells the same names out again in its own notation. - */ -const _policyKeysMatchAuthorControllable: AssertEqual< - keyof InboundMailPolicy, - AuthorControllableOutcome -> = true; - -const _authorControllableAreOutcomes: AssertEqual< - Exclude, - never -> = true; - -/** - * A parsed MIME part. `content` is the DECODED bytes in memory (the - * transfer-encoding has already been undone). `filename` and `disposition` are - * surfaced from the part's `Content-Disposition` / `Content-Type` so a consumer - * can distinguish an inline part from a named attachment without re-parsing - * headers. - */ -export type MessagePart = { - contentType: string; - content: Uint8Array; - filename?: string; - disposition?: "inline" | "attachment"; - /** - * The Content-Transfer-Encoding the part declared. Not set for a decoded - * mail part -- `content` is already decoded, so the wire encoding is spent - * transport metadata. - */ - encoding?: string; -}; - -/** - * A single part of a persisted `Mail`. The bytes live in the durable store; - * this descriptor carries the part's metadata plus an opaque, relocation-stable - * `ref` a `MailPartReader` resolves to the part's bytes. Small UTF-8 text parts - * also carry their decoded `text` inline so a selector can read them without - * resolving. - */ -export type MailPart = { - contentType: string; - filename?: string; - disposition?: "inline" | "attachment"; - ref: string; - text?: string; -}; - -/** - * The single, environment-agnostic interface for reading a persisted mail - * part's bytes. Modeled on `BlobReader`: a consumer -- a workflow step, an - * agent tool, the agent's content-block projection -- resolves a - * `MailPart.ref` to its bytes without knowing or caring where the bytes live - * (a committed file on the sidecar, a blob in a browser runtime). The `ref` - * is opaque; the reader owns its scheme. Threaded to consumers through the - * runtime so a browser runtime can supply its own implementation. - */ -export interface MailPartReader { - /** Resolve a `MailPart.ref` to the part's decoded bytes. Throws if the ref - * is unrecognized or its bytes are missing. */ - read(ref: string): Promise; -} - -/** - * A fully decoded mail message: every header (both the typed, ergonomic subset - * and a raw catch-all with nothing dropped) plus the flat list of decoded leaf - * parts. This is the lossless representation a deployed workflow receives as - * its trigger input; a workflow programs against it directly (select headers, - * walk parts, route on content type), and an agent step projects the parts into - * model content blocks. It is JSON-safe: each part carries a `ref` (not raw - * bytes), so binary content never enters the event log; the runtime resolves a - * `ref` into a loadable `MessagePart` on demand. - */ -export type Mail = { - headers: MessageHeaders; - /** Every header, lowercased name to its ordered values; nothing dropped. */ - rawHeaders: Record; - parts: MailPart[]; -}; - -const MailShape = type({ - // Require the recipient list a consumer dereferences unconditionally; other - // header fields stay optional and are carried losslessly in `rawHeaders`. - // `from` is optional here deliberately: a mail with no usable originator is - // still mail, and requiring it makes `isMail` reject one. - headers: { - "from?": "string", - to: "string[]", - }, - rawHeaders: { - "[string]": "string[]", - }, - parts: type({ - contentType: "string", - ref: "string", - "filename?": "string", - "disposition?": "'inline' | 'attachment'", - "text?": "string", - }) - .onUndeclaredKey("reject") - .array(), -}).onUndeclaredKey("reject"); - -/** - * Narrow an opaque value (a workflow step input) to a `Mail`. Used at the - * `agent.send` boundary to decide whether the input is a mail-derived message - * whose parts must be projected into content blocks, or an arbitrary value - * delivered as synthesized text. The strict undeclared-key rejection keeps an - * arbitrary step value that merely carries a `parts` field from matching. - */ -export function isMail(value: unknown): value is Mail { - return !(MailShape(value) instanceof type.errors); -} - -/** - * MIME tree metadata returned by `fetchStructure()`. Describes content types, - * sizes, and dispositions without transferring content. - * - * (MESSAGE.md § Partial Fetch) - */ -export type BodyStructure = { - contentType: string; - size?: number; - disposition?: string; - parts?: BodyStructure[]; -}; - -/** - * A fully parsed inbound message including structured payload, headers, - * attachments, and signature verification status. - * - * (MESSAGE.md § Transport Interface › fetchFull) - */ -export type InboundMessage = { - ref: MessageRef; - headers: MessageHeaders; - flags: string[]; - - /** Plain text body for conversation.* types. */ - content?: string; - - /** Parsed JSON payload for structured types. */ - payload?: { - type: InterchangeType; - version: string; - body: Record; - }; - - attachments?: MessageAttachment[]; - signatureStatus: SignatureStatus; -}; - -/** - * IMAP mailbox descriptor. - * - * (MESSAGE.md § Inbox Management) - */ -export type Mailbox = { - name: string; - role?: string; - delimiter?: string; -}; - -/** - * Current status of an IMAP mailbox, including QRESYNC identifiers. - * - * (MESSAGE.md § Inbox Management) - */ -export type MailboxStatus = { - total: number; - unseen: number; - recent: number; - uidNext: number; - uidValidity: number; - highestModSeq: number; -}; - -/** - * Structured IMAP search query. Maps the IMAP SEARCH grammar to a typed - * object. Supports recursive boolean composition via `and`, `or`, `not`. - * - * (MESSAGE.md § Search) - */ -export type SearchQuery = { - from?: string; - to?: string; - cc?: string; - bcc?: string; - header?: { field: string; contains: string }; - before?: Date; - after?: Date; - on?: Date; - sentBefore?: Date; - sentAfter?: Date; - sentOn?: Date; - hasFlags?: string[]; - missingFlags?: string[]; - body?: string; - text?: string; - largerThan?: number; - smallerThan?: number; - and?: SearchQuery[]; - or?: SearchQuery[]; - not?: SearchQuery; -}; - -/** - * A thread node returned by `thread()`. Carries a message reference and - * child threads representing replies. Implements the RFC 5256 REFERENCES - * threading algorithm. - * - * (MESSAGE.md § Thread Retrieval) - */ -export type Thread = { - ref: MessageRef; - children: Thread[]; -}; - -/** - * QRESYNC state the harness provides when reconnecting to the transport. - * - * (MESSAGE.md § Synchronization) - */ -export type SyncState = { - uidValidity: number; - uidNext: number; - highestModSeq: number; - knownUids?: number[]; -}; - -/** - * Result of a QRESYNC-style sync operation. - * - * (MESSAGE.md § Synchronization) - */ -export type SyncResult = { - vanished: number[]; - changed: { uid: number; flags: string[] }[]; - newMessages: MessageRef[]; - fullResyncRequired: boolean; -}; - -/** - * Distribution list metadata returned by `createList()`. - * - * (MESSAGE.md § Message Topologies) - */ -export type ListInfo = { - address: string; - name: string; - memberCount: number; - createdAt: string; -}; - -/** - * Event emitted by the mailbox watcher callback. Corresponds to IMAP IDLE - * notifications. - * - * (MESSAGE.md § Real-Time Notification) - */ -export type MailboxEvent = - | { type: "exists"; uid: number; headers: MessageHeaders } - | { type: "flagsChanged"; uid: number; flags: string[] } - | { type: "expunged"; uid: number }; - -/** Unsubscribe function returned by `watch()`. */ -export type Unsubscribe = () => void; - -/** - * The message transport interface. Abstracts SMTP and IMAP behind a - * TypeScript API. Implementations range from real SMTP/IMAP servers to - * in-process stubs that route messages through memory. - * - * All long-running operations accept an AbortSignal for cooperative - * cancellation. - * - * (MESSAGE.md § Transport Interface) - */ -export interface MessageTransport { - // --- Outbound --- - - /** Compose, sign, and deliver a message via SMTP. */ - send(message: OutboundMessage, signal?: AbortSignal): Promise; - - /** Append a raw message to a mailbox (IMAP APPEND). */ - append( - mailbox: string, - message: InboundMessage, - flags?: string[], - signal?: AbortSignal, - ): Promise; - - // --- Mailbox management --- - - listMailboxes(signal?: AbortSignal): Promise; - createMailbox(name: string, signal?: AbortSignal): Promise; - deleteMailbox(name: string, signal?: AbortSignal): Promise; - getMailboxStatus(name: string, signal?: AbortSignal): Promise; - - // --- Message search and retrieval --- - - search( - mailbox: string, - query: SearchQuery, - signal?: AbortSignal, - ): Promise; - - thread( - mailbox: string, - algorithm: "references" | "orderedsubject", - query?: SearchQuery, - signal?: AbortSignal, - ): Promise; - - fetchHeaders(ref: MessageRef, signal?: AbortSignal): Promise; - fetchStructure(ref: MessageRef, signal?: AbortSignal): Promise; - fetchPart( - ref: MessageRef, - partPath: string, - signal?: AbortSignal, - ): Promise; - fetchFull(ref: MessageRef, signal?: AbortSignal): Promise; - - // --- Flag management --- - - setFlags( - ref: MessageRef, - flags: string[], - signal?: AbortSignal, - ): Promise; - - clearFlags( - ref: MessageRef, - flags: string[], - signal?: AbortSignal, - ): Promise; - - // --- Message organization --- - - move(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; - - copy(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; - - /** - * Permanently remove every `\Deleted` message from the mailbox. Returns the - * uids that were expunged, so a caller can report how many messages it - * consumed and which ones. - */ - expunge( - mailbox: string, - signal?: AbortSignal, - ): Promise<{ expungedUids: number[] }>; - - // --- Real-time notification --- - - /** Monitor a mailbox for new messages and flag changes (IMAP IDLE). */ - watch(mailbox: string, callback: (event: MailboxEvent) => void): Unsubscribe; - - // --- Synchronization --- - - /** Efficient reconnection using QRESYNC semantics. */ - sync( - mailbox: string, - knownState: SyncState, - signal?: AbortSignal, - ): Promise; - - // --- Distribution lists --- - - createList( - address: string, - name: string, - signal?: AbortSignal, - ): Promise; - - listMembers(address: string, signal?: AbortSignal): Promise; - - subscribe( - listAddress: string, - subscriberAddress: string, - signal?: AbortSignal, - ): Promise; - - unsubscribe( - listAddress: string, - subscriberAddress: string, - signal?: AbortSignal, - ): Promise; -} - -/** - * The condition a `MessageTransport` operation failed under. A transport that - * grows a condition this union does not carry takes its name from RFC 5530 § 3 - * rather than coining one. - * - * https://www.rfc-editor.org/rfc/rfc5530.html - */ -export const MessageTransportCondition = type( - "'NONEXISTENT' | 'CANNOT' | 'SERVERBUG'", -); -export type MessageTransportCondition = typeof MessageTransportCondition.infer; - -export class MessageTransportError extends Error { - readonly condition: MessageTransportCondition; - - constructor(condition: MessageTransportCondition, message: string) { - super(message); - this.name = "MessageTransportError"; - this.condition = condition; - } -} - -/** - * `instanceof` is not usable for this: a tool package is published as a bundle - * whose workspace imports are inlined (`bin/build-builtins.ts`), so a consumer - * loaded from a bundle holds its own copy of the class above and answers false - * against an error the host's copy constructed. - */ -export function isMessageTransportError( - value: unknown, -): value is Error & { readonly condition: MessageTransportCondition } { - if (!(value instanceof Error)) return false; - if (!("condition" in value)) return false; - return MessageTransportCondition.allows(value.condition); -} - -// --------------------------------------------------------------------------- -// Tool Execution (ARCHITECTURE.md § Tools, INFERENCE.md § Tool Execution) -// --------------------------------------------------------------------------- - -/** - * A tool call as requested by the model. Carries the provider-assigned call - * ID, the tool name, and the parsed arguments. - * - * (INFERENCE.md § Message Format › Content Types) - */ -export const ToolCall = type({ - id: "string", - name: "string", - arguments: "Record", -}); -export type ToolCall = typeof ToolCall.infer; - -/** - * Approver-facing snapshot of the tool call awaiting approval. Built at the - * authz `ask` branch from the tool's definition and the live call, then - * threaded unchanged from the reactor's pending operation through every - * suspend hop to the hub co-write that records it on the approval row. - * - * `name`, `description`, and `inputSchema` mirror the {@link ToolDefinition}; - * `arguments` is the live call's arguments. Carried as a sibling of the pending - * operation's `suspendedCall`, never folded into {@link ToolCall}, so the - * re-dispatch artifact and the approval snapshot stay separate concerns. - */ -export const ApprovalSnapshot = type({ - name: "string", - description: "string", - inputSchema: "Record", - arguments: "Record", -}); -export type ApprovalSnapshot = typeof ApprovalSnapshot.infer; - -/** - * The kind of a control-plane park: a step suspended awaiting an external - * event. `"approval"` and `"input"` park on a reserved - * `signalName(correlationId)` channel; `"signal-relay"` parks on an - * author-chosen name. - * - * - `"approval"` -- the step parked on a tool/authz gate and REQUIRES an - * {@link ApprovalSnapshot}; the runtime notifies the host (`env.onPark`) so - * the sidecar co-writes the approval/correlation rows the hub registers. - * - `"input"` -- the step parked awaiting its next input (e.g. a long-lived - * agent run awaiting the next mail so it can take another turn). It carries - * NO snapshot and does NOT notify the host: the run's owner delivers the - * input on the same channel and the step re-arms. It is a runtime-local - * concept -- deliberately NOT a {@link SignalKind}, so it never touches the - * approval-routing machinery (IPC register frames, the hub co-write, the - * approval columns). - * - `"signal-relay"` -- an onTrigger section container parked on an - * author-named signal so a body child's `awaitSignal` on that name is - * serviced through the deployment run: the external signal is delivered to - * the parent run and the runtime relays it down into the live body child. - * The channel name is the author's free-form signal name, NOT a reserved - * `signalName(correlationId)`, so recovery must branch on this kind BEFORE - * assuming the awaited name is a reserved control-plane channel. Carries no - * snapshot and is not hub-registered. - * - * The kinds are distinguished by an EXPLICIT discriminant everywhere the kind - * flows -- never inferred from the presence or absence of a snapshot, which - * would silently reclassify a malformed snapshot-less approval as another - * park kind rather than failing loud. - */ -export const ControlParkKind = type.enumerated( - "approval", - "input", - "signal-relay", -); -export type ControlParkKind = typeof ControlParkKind.infer; - -/** - * Maximum serialized size, in UTF-8 bytes, of an {@link ApprovalSnapshot} that - * crosses a trust boundary. A tool `inputSchema` is normally single-digit KB; - * a snapshot approaching this bound is malformed or hostile and is rejected at - * the parse boundary rather than co-written onto an approval row. - */ -export const APPROVAL_SNAPSHOT_MAX_BYTES = 131072; - -/** - * {@link ApprovalSnapshot} bounded to {@link APPROVAL_SNAPSHOT_MAX_BYTES}. - * Parse the snapshot through this validator where it crosses a trust boundary - * (the `park.notify` IPC frame, the `parked-correlations.response` IPC frame, - * and the sidecar→hub register frame); internal hops use the unbounded - * {@link ApprovalSnapshot}. `.narrow` bounds the runtime check only — its - * inferred type is identical to {@link ApprovalSnapshot} — so the cap holds only - * where a frame is actually parsed, not merely typed. - */ -export const BoundedApprovalSnapshot = ApprovalSnapshot.narrow( - (snapshot, ctx) => { - const bytes = Buffer.byteLength(JSON.stringify(snapshot), "utf8"); - return ( - bytes <= APPROVAL_SNAPSHOT_MAX_BYTES || - ctx.mustBe(`at most ${APPROVAL_SNAPSHOT_MAX_BYTES} bytes when serialized`) - ); - }, -); -export type BoundedApprovalSnapshot = typeof BoundedApprovalSnapshot.infer; - -/** - * Result of a tool execution. `content` is text or structured data the model - * sees as the tool result. `detail` is additional data that the harness may - * use (e.g., for validation or audit) but that is not shown to the model. - * - * When `isError` is true the model sees the result as an error. When - * `pendingMarker` is present the tool is async — the reactor registers the - * correlation ID and waits for a matching inbound message. - * - * (INFERENCE.md § Tool Execution Semantics) - */ -export const ToolResult = type({ - callId: "string", - content: "string | Record", - "detail?": "unknown", - "isError?": "boolean", - "pendingMarker?": { - status: "'pending'", - correlationId: "string", - }, -}); -export type ToolResult = typeof ToolResult.infer; - -/** - * The tool runner interface. The harness implements this; the reactor calls - * it when the director requests tool execution. - * - * Parallel execution is modeled by calling `run` concurrently for each call - * in a batch — the interface is per-call, not per-batch. - * - * (ARCHITECTURE.md § Agent Harness › Tools) - */ -export interface ToolRunner { - /** - * Execute a single tool call. Resolves with the result. Must not throw — - * errors are returned as `ToolResult` with `isError: true`. - */ - run(call: ToolCall, signal: AbortSignal): Promise; -} - -// --------------------------------------------------------------------------- -// Inference Event Building Blocks (INFERENCE.md § Event Protocol) -// --------------------------------------------------------------------------- - -/** - * Partial assistant message accumulated during streaming. Carries all - * content blocks seen so far so late-joining subscribers receive current - * state without replaying deltas. - * - * `text` and `thinking` are cumulative across every emitted delta of - * that kind in the current turn — intentionally flat, even when the - * harness's per-index block tracking has split the stream into - * multiple ThinkingBlocks or TextBlocks. Consumers that need per-block - * structure walk the finalized inference.done turn's content[]; this - * snapshot is the live "what bytes has the assistant streamed so - * far" view. - * - * (INFERENCE.md § Event Protocol › Partial State) - */ -export const PartialMessage = type({ - text: "string", - "thinking?": "string", - "toolCalls?": type({ - id: "string", - name: "string", - partialArguments: "string", - }).array(), -}); -export type PartialMessage = typeof PartialMessage.infer; - -/** - * Token usage for a single inference call. Cache read/write counts are - * provider-specific and may be zero when the provider does not report them. - * - * (INFERENCE.md § Token Accounting) - */ -export const TokenUsage = type({ - input: "number", - output: "number", - cacheRead: "number", - cacheWrite: "number", - thinking: "number", -}); -export type TokenUsage = typeof TokenUsage.infer; - -/** - * Slim source descriptor stamped onto `inference.usage` / `inference.done` - * events and onto `ReactorState.lastCycleSource`. - * - * Carries enough identity for state-aware policies (cost gating, budget - * caps, governance triggers, audit) to attribute usage to a specific - * inference source without re-reading the live, mutable `InferenceSource` - * the harness owns. - * - * Deliberately a strict subset of `InferenceSource` — `apiKey` and - * `baseURL` are intentionally excluded. Credentials and endpoints must - * not leak to director-side policy code or to external event consumers. - * Any code path that needs the full source obtains it through the - * harness's source registry, not through this descriptor. - * - * `sourceId` aliases `InferenceSource.id` to disambiguate from message - * ids, turn ids, and session ids in director-side code where `id` alone - * would be ambiguous. - */ -export const LastCycleSource = type({ - sourceId: "string", - provider: "string", - model: "string", -}); -export type LastCycleSource = typeof LastCycleSource.infer; - -// --------------------------------------------------------------------------- -// Internal Turn Format (INFERENCE.md § Message Format) -// --------------------------------------------------------------------------- - -/** - * A single content block within a conversation turn. Provider-agnostic. - * - * (INFERENCE.md § Message Format › Content Types) - */ -const TextBlock = type({ - type: "'text'", - text: "string", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini attaches a `thoughtSignature` to - // output parts (including plain text); absent for providers that do not - // sign this block kind. - "signature?": "string", -}); - -/** - * How a media payload is carried by a content block. One of three - * variants: inline as a base64-encoded string, by reference to an - * opaque provider-native handle (e.g. a Gemini fileUri, an Anthropic - * file_id), or by public URL the provider fetches itself. The wire - * shape each provider expects is built by the provider adapter; - * MediaSource is the internal, provider-agnostic representation. - * - * (INFERENCE.md § Generalized Multimodal Taxonomy) - */ -const MediaSourceBase64 = type({ - kind: "'base64'", - mimeType: "string", - data: "string", -}); - -const MediaSourceFileReference = type({ - kind: "'file-reference'", - mimeType: "string", - reference: "string", -}); - -const MediaSourceUrl = type({ - kind: "'url'", - mimeType: "string", - url: "string", -}); - -export const MediaSource = type.or( - MediaSourceBase64, - MediaSourceFileReference, - MediaSourceUrl, -); -export type MediaSource = typeof MediaSource.infer; - -// Exported because `inference.image_output` events reference it by -// name, following the same pattern as `CitationBlock`, -// `CodeExecutionRequestBlock`, and `RedactedThinkingBlock`. -export const ImageBlock = type({ - type: "'image'", - source: MediaSource, - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // inlineData part; absent otherwise. - "signature?": "string", -}); -export type ImageBlock = typeof ImageBlock.infer; - -const AudioBlock = type({ - type: "'audio'", - source: MediaSource, -}); - -const VideoBlock = type({ - type: "'video'", - source: MediaSource, -}); - -const DocumentBlock = type({ - type: "'document'", - source: MediaSource, - "title?": "string", - "context?": "string", -}); - -const ThinkingBlock = type({ - type: "'thinking'", - thinking: "string", - "signature?": "string", -}); - -/** - * A thinking block whose content the provider has filtered. The - * opaque `data` blob must echo back verbatim on every follow-up turn - * — Anthropic 400s the request if it changes or goes missing. Treat - * the bytes as opaque: do not log them and do not render them to - * users. - * - * Exported because `inference.thinking.redacted` events reference it - * by name. - */ -export const RedactedThinkingBlock = type({ - type: "'redacted_thinking'", - data: "string", -}); -export type RedactedThinkingBlock = typeof RedactedThinkingBlock.infer; - -/** - * A model-emitted refusal. Produced when a provider's strict-mode - * structured-outputs path declines to satisfy the requested schema — - * OpenAI's `delta.refusal` / `message.refusal` field is the canonical - * wire shape. The `reason` is the accumulated human-readable text the - * model emitted in lieu of conformant output. - * - * Refusal is semantically distinct from `inference.error`: the HTTP - * call succeeded and the model produced a coherent response, but that - * response is "I will not satisfy this schema" rather than schema- - * conformant content. Callers that distinguish policy declines from - * transport/protocol failures should branch on the block type rather - * than treat the assistant turn as an error. - * - * Exported because `inference.refusal.delta` events reference it by - * name and adapters construct RefusalBlocks in the finalized - * AssistantTurn from accumulated delta fragments. - */ -export const RefusalBlock = type({ - type: "'refusal'", - // Refusals must carry text — a zero-length reason corrupts the - // "human-readable text the model emitted in lieu of conformant - // output" contract and would round-trip indistinguishably from a - // refusal block whose payload was lost. The arktype constraint is - // belt-and-braces alongside the adapter's wire-boundary filter on - // empty `delta.refusal` chunks: synthetic fixtures or future - // adapters without that filter still cannot construct a vacuous - // refusal. - reason: "string > 0", -}); -export type RefusalBlock = typeof RefusalBlock.infer; -const ToolCallBlock = type({ - type: "'tool_call'", - id: "string", - name: "string", - arguments: "Record", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // functionCall part; absent otherwise. - "signature?": "string", -}); -/** - * Location of a citation's cited span within its source document. - * The unit of `start` and `end` varies by `kind`: - * - "page": 1-indexed page numbers (Anthropic `page_location`). - * - "char": UTF-16 character offsets, matching JS string semantics - * (Anthropic `char_location`; Gemini `groundingSupports[].segment`). - * - "content-block": index into a structured source's content blocks - * (Anthropic `content_block_location`). - */ -const CitationLocation = type({ - kind: "'page' | 'char' | 'content-block'", - start: "number", - end: "number", -}); - -const CitationSource = type({ - "title?": "string", - // Self-contained dereferenceable URL — populated by providers whose - // citations carry URLs directly (Gemini `groundingChunks[].web.uri`). - "uri?": "string", - // Back-pointer into the request's `documents` array, populated by - // providers that cite uploaded documents by position (Anthropic - // `document_index`). - "documentRef?": type({ index: "number" }), -}); - -/** - * A citation that supports a span of assistant text. Consumers - * receiving a CitationBlock without a paired source-block index MUST - * attribute it by adjacency to the nearest preceding TextBlock in the - * same turn. - * - * Citations are deliberately excluded from ToolResultBlock.content - * — they annotate model output, not tool output. - * - * Exported because `inference.citation` events reference it by name, - * following the same pattern as `AssistantTurn`, `ToolCall`, and - * `ToolResult`. See the `inference.citation` event docstring for how - * a paired source-block index is carried on the wire and consumed by - * the harness. - */ -export const CitationBlock = type({ - type: "'citation'", - // The exact substring of the preceding TextBlock this citation - // supports. Both providers emit it; required for inspection and - // for fallback offset reconstruction. - citedText: "string", - source: CitationSource, - "location?": CitationLocation, - // UTF-16 character offsets into the preceding TextBlock's text. - // Providers that emit offsets natively populate these directly; - // adapters that derive offsets from a cited substring populate - // them only when the substring appears unambiguously in the - // preceding text. Omitted when the offset cannot be determined. - "textOffset?": type({ start: "number", end: "number" }), -}); -export type CitationBlock = typeof CitationBlock.infer; - -/** - * A structured safety signal on model output or request filtering. - * - * The name `SafetyRatingBlock` follows the issue vocabulary; the - * payload is derived from the first real Gemini capture that engaged - * the structured classifier (2026-07-28). That wire shape is - * prompt-level only: - * - * `promptFeedback: { blockReason: "PROHIBITED_CONTENT" }` - * - * with no candidates and no per-category `safetyRatings` arrays. So - * this block carries `blockReason` and does **not** invent category / - * probability / blocked fields. When a future capture surfaces - * candidate-level ratings, extend the type from those bytes rather - * than from the API reference. - * - * Deliberately excluded from ToolResultBlock.content — safety - * signals annotate model/request filtering, not tool output. - * - * Exported because `inference.safety_rating` events reference it by - * name. - */ -export const SafetyRatingBlock = type({ - type: "'safety_rating'", - // Provider-native block reason string (observed: "PROHIBITED_CONTENT"). - // Open string so a new reason token does not force a type bump. - blockReason: "string > 0", -}); -export type SafetyRatingBlock = typeof SafetyRatingBlock.infer; - -/** - * Human-readable rendering of a SafetyRatingBlock for reply text, - * timeline summaries, and request-history rewrites when a provider - * has no input wire shape for safety_rating. Single owner of the - * display string so reply / history / transform stay in lockstep. - */ -export function formatSafetyRatingText(block: SafetyRatingBlock): string { - return `Request blocked: ${block.blockReason}`; -} - -/** - * The model's request to execute code via a server-side execution tool. - * Paired with a CodeExecutionResultBlock carrying the same `id` as the - * result's `requestId`. Streaming order within a single execution is - * `inference.code_execution.start` → zero or more - * `inference.code_execution.delta` → `inference.code_execution.result`, - * uninterrupted by other events that share the same `requestId`; events - * with different `requestId`s or for other block kinds at distinct - * `index`es may interleave. - * - * Exported because `inference.code_execution.start` references it by - * name. - */ -export const CodeExecutionRequestBlock = type({ - type: "'code_execution_request'", - // Identifier for the execution request. Populated from the - // provider's call id where one exists (Anthropic - // `srvtoolu_...`); synthesized by the adapter for providers that - // don't emit one (Gemini), using a deterministic per-response - // position-based scheme so replays match. - id: "string", - // Source code the model is asking to execute. - code: "string", - // Language hint. Absent when the provider does not emit one; - // adapters MUST NOT default this — callers narrow on its - // presence rather than fall through to a guessed language. - "language?": "string", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // executableCode part; absent otherwise. - "signature?": "string", -}); -export type CodeExecutionRequestBlock = typeof CodeExecutionRequestBlock.infer; - -/** - * The result of executing a CodeExecutionRequestBlock. The `requestId` - * back-points to the request block's `id`. Status is normalized across - * providers; raw provider signals (return code, native outcome string, - * abort reason) are preserved on optional fields for callers that need - * them. - * - * File outputs from code execution (e.g. generated plots that - * Anthropic returns in `code_execution_tool_result.content`) are NOT - * modeled by this block today. The block carries no field for them; - * surfacing file outputs is a separate concern. - * - * Exported because `inference.code_execution.result` references it by - * name. - */ -export const CodeExecutionResultBlock = type({ - type: "'code_execution_result'", - // Back-pointer to the originating CodeExecutionRequestBlock.id. - requestId: "string", - // Normalized outcome. Translated from provider-specific signals: - // - Anthropic: derived from `return_code` (0 → "ok", non-zero → - // "error") and `abort_reason` (non-null → "aborted" or - // "timeout" per the reason). - // - Gemini: derived from the `outcome` enum - // (OUTCOME_OK → "ok", OUTCOME_FAILED → "error", - // OUTCOME_DEADLINE_EXCEEDED → "timeout", etc.). - status: "'ok' | 'error' | 'aborted' | 'timeout'", - // Standard output. Providers that don't split stdout from stderr - // (Gemini) map their combined `output` here and leave `stderr` empty. - "stdout?": "string", - // Standard error. Empty for providers that don't split. - "stderr?": "string", - // Provider-native numeric return code when available - // (Anthropic `return_code`). Absent for providers whose outcome - // is enum-only (Gemini). - "returnCode?": "number", - // Provider-native outcome string preserved verbatim for callers - // that need the raw signal (Gemini `OUTCOME_OK` / - // `OUTCOME_FAILED` / `OUTCOME_DEADLINE_EXCEEDED` / ...). Absent - // when the provider does not emit one (Anthropic). - "providerOutcome?": "string", - // Human-readable reason populated when status is "aborted" - // (Anthropic `abort_reason`). Absent otherwise. - "abortReason?": "string", -}); -export type CodeExecutionResultBlock = typeof CodeExecutionResultBlock.infer; - -const ToolResultBlock = type({ - type: "'tool_result'", - callId: "string", - // Deliberately narrow: tool results carry user-facing media, not - // CitationBlocks (citations annotate the model's text output), not - // SafetyRatingBlocks (safety signals annotate model/request - // filtering), and not CodeExecution blocks (server-side code - // execution is a distinct lifecycle from the user-tool round-trip). - content: type - .or(TextBlock, ImageBlock, AudioBlock, VideoBlock, DocumentBlock) - .array(), - "detail?": "unknown", - "isError?": "boolean", -}); - -export const ContentBlock = type.or( - TextBlock, - ThinkingBlock, - RedactedThinkingBlock, - RefusalBlock, - ImageBlock, - AudioBlock, - VideoBlock, - DocumentBlock, - CitationBlock, - SafetyRatingBlock, - CodeExecutionRequestBlock, - CodeExecutionResultBlock, - ToolCallBlock, - ToolResultBlock, -); -export type ContentBlock = typeof ContentBlock.infer; - -/** - * A turn in the internal conversation history. The `model` field records - * which provider model produced this turn (present only on assistant - * turns). Used by cross-provider transformation to strip or preserve - * thinking blocks. - * - * (INFERENCE.md § Message Format) - */ -export type ConversationTurn = { - role: "user" | "assistant" | "system"; - content: ContentBlock[]; - model?: string; - timestamp: number; -}; - -/** - * A completed assistant turn returned in `inference.done`. Narrower type - * than ConversationTurn to make the inference boundary explicit. - */ -export const AssistantTurn = type({ - role: "'assistant'", - content: ContentBlock.array(), - model: "string", - timestamp: "number", -}); -export type AssistantTurn = typeof AssistantTurn.infer; - -// --------------------------------------------------------------------------- -// Error Classification (INFERENCE.md § Error Classification) -// --------------------------------------------------------------------------- - -/** - * Classified inference error. The category determines the reactor's default - * response; the director can override per its policy. - * - * (INFERENCE.md § Error Classification) - */ -export const InferenceError = type({ - category: type.enumerated( - "retryable", - "context_overflow", - "credential_failure", - "quota_exhausted", - "fatal", - "aborted", - "timeout", - "protocol_mismatch", - ), - message: "string", - "statusCode?": "number", - "retryAfterMs?": "number", - "raw?": "unknown", -}); -export type InferenceError = typeof InferenceError.infer; - -// --------------------------------------------------------------------------- -// Agent Reactor (INFERENCE.md § Agent Reactor) -// --------------------------------------------------------------------------- - -/** - * Gate types that can block the reactor. - * - * (INFERENCE.md § Gates) - */ -export const GateType = type.enumerated( - "approval", - "payment", - "credential", - "budget", - "child_completion", - "message_response", -); -export type GateType = typeof GateType.infer; - -/** - * Fork mode. `independent` creates a divergent reactor with its own context. - * `child` creates a reactor that reports results back to the parent. - * - * (INFERENCE.md § Forking) - */ -export const ForkMode = type.enumerated("independent", "child"); -export type ForkMode = typeof ForkMode.infer; - -// --------------------------------------------------------------------------- -// Inference Event Protocol (INFERENCE.md § Event Protocol) -// --------------------------------------------------------------------------- - -/** - * Wire-safe representation of InboundMessage for use in InferenceEvent - * variants. The runtime InboundMessage type contains Uint8Array fields - * (MessageAttachment.data) that cannot survive JSON serialization, so the - * wire validator uses `unknown` for attachment data and accepts whatever - * JSON.parse produces. - */ -const WireInboundMessage = type({ - ref: { uid: "number", mailbox: "string" }, - headers: "Record", - flags: "string[]", - "content?": "string", - "payload?": "object", - "attachments?": "unknown[]", - signatureStatus: type.enumerated("valid", "invalid", "unknown", "missing"), -}); - -/** - * A single event in the inference event protocol. Every event carries a - * monotonic session-scoped sequence number. - * - * Event types are namespaced: `inference.*`, `tool.*`, `reactor.*`, - * `fork.*`, `message.*`, `custom.*`. - * - * (INFERENCE.md § Event Protocol) - */ -export const InferenceEvent = type.or( - { - type: "'inference.start'", - seq: "number", - data: { model: "string" }, - }, - { - type: "'inference.thinking.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.block.signature'", - seq: "number", - data: { signature: "string", "index?": "number" }, - }, - { - type: "'inference.thinking.redacted'", - seq: "number", - data: { redactedThinking: RedactedThinkingBlock, "index?": "number" }, - }, - { - type: "'inference.text.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.refusal.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.start'", - seq: "number", - data: { - callId: "string", - name: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.delta'", - seq: "number", - data: { - callId: "string", - argumentFragment: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.end'", - seq: "number", - data: { - callId: "string", - name: "string", - arguments: "Record", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.usage'", - seq: "number", - data: { usage: TokenUsage, source: LastCycleSource }, - }, - { - type: "'inference.done'", - seq: "number", - data: { - turn: AssistantTurn, - usage: TokenUsage, - source: LastCycleSource, - "pacingDelayMs?": "number", - }, - }, - { - type: "'inference.error'", - seq: "number", - data: { error: InferenceError, partial: PartialMessage }, - }, - { - type: "'inference.retry'", - seq: "number", - data: { - attempt: "number", - delayMs: "number", - previousError: InferenceError, - }, - }, - { - type: "'inference.citation'", - seq: "number", - // `index`, when present, names the source content block (typically - // a TextBlock) the citation annotates. The harness uses it to - // interleave the citation into the finalized turn's `content[]` - // immediately after the matching block. Adapters whose wire - // protocol does not carry per-citation block indices omit the - // field; the harness then appends those citations at the end of - // `content[]` and consumers attribute them to the nearest - // preceding TextBlock per the CitationBlock docstring. - data: { citation: CitationBlock, "index?": "number" }, - }, - { - type: "'inference.safety_rating'", - seq: "number", - // Prompt-level structured safety signal (observed Gemini - // `promptFeedback.blockReason`). No candidate index: the first - // capture has zero candidates. Harness appends the block to the - // finalized turn's `content[]`. - data: { safetyRating: SafetyRatingBlock }, - }, - { - type: "'inference.code_execution.start'", - seq: "number", - data: { request: CodeExecutionRequestBlock, "index?": "number" }, - }, - { - type: "'inference.code_execution.delta'", - seq: "number", - // requestId correlates fragments back to the originating - // CodeExecutionRequestBlock; index is the positional hint into - // the response's content-block stream. They are independent: a - // single response may stream code execution for multiple - // requests interleaved, distinguished by requestId; index lets - // the harness's per-block accumulator route the fragment to - // the correct block when the array isn't yet finalized. - data: { - requestId: "string", - codeFragment: "string", - "index?": "number", - }, - }, - { - type: "'inference.code_execution.result'", - seq: "number", - data: { result: CodeExecutionResultBlock, "index?": "number" }, - }, - { - type: "'inference.image_output'", - seq: "number", - // Fires mid-stream when an adapter finalizes an image-output - // block, signaling that the image is ready for downstream - // handoff before the full inference.done lands. The wrapped - // ImageBlock typically carries a base64 MediaSource — the - // payload can be large (Gemini's image-output captures show - // ~1MB inline blobs); consumers that subscribe to this event - // should treat it as a non-trivial transport size. - data: { image: ImageBlock, "index?": "number" }, - }, - { - type: "'tool.start'", - seq: "number", - data: { call: ToolCall }, - }, - { - type: "'tool.update'", - seq: "number", - data: { callId: "string", partial: "string" }, - }, - { - type: "'tool.done'", - seq: "number", - data: { result: ToolResult }, - }, - { - type: "'message.queued'", - seq: "number", - data: { message: WireInboundMessage }, - }, - { - type: "'message.run.started'", - seq: "number", - data: { - "messageId?": "string", - messageRunId: "string", - receivedAt: "number", - }, - }, - { - type: "'message.run.ended'", - seq: "number", - data: { - messageRunId: "string", - "messageId?": "string", - status: type.enumerated("completed", "failed"), - "error?": { - message: "string", - "kind?": "string", - }, - }, - }, - { - type: "'message.correlated'", - seq: "number", - data: { message: WireInboundMessage, correlationId: "string" }, - }, - { - type: "'connector.reply'", - seq: "number", - data: { content: "string", "checkpointHash?": "string" }, - }, - { - type: "'reactor.start'", - seq: "number", - data: "object", - }, - { - type: "'reactor.gate.blocked'", - seq: "number", - data: { - reason: GateType, - gateId: "string", - "correlationId?": "string", - "approvalSnapshot?": ApprovalSnapshot, - }, - }, - { - type: "'reactor.gate.cleared'", - seq: "number", - data: { - gateId: "string", - reason: type.enumerated("resolved", "timeout", "shutdown"), - }, - }, - { - type: "'reactor.done'", - seq: "number", - data: "object", - }, - { - type: "'reactor.error'", - seq: "number", - data: { error: "string", fatal: "boolean" }, - }, - { - type: "'fork.created'", - seq: "number", - data: { forkId: "string", parentId: "string", mode: ForkMode }, - }, - { - type: "'fork.done'", - seq: "number", - data: { forkId: "string", "result?": "unknown" }, - }, - { - type: "'fork.error'", - seq: "number", - data: { forkId: "string", error: "string" }, - }, - { - type: "'fork.aborted'", - seq: "number", - data: { forkId: "string" }, - }, - { - type: /^custom\./, - seq: "number", - data: "Record", - }, -); -// The TypeScript type is defined manually rather than inferred from the -// validator because the `custom.*` variant uses a regex pattern which -// arktype infers as `string`. A bare `string` in the discriminant position -// prevents TypeScript from narrowing the union in switch statements. -// The manually defined type uses a `custom.${string}` template literal -// for that variant, preserving the narrowing behavior downstream code -// relies on. -export type InferenceEvent = - | { type: "inference.start"; seq: number; data: { model: string } } - | { - type: "inference.thinking.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.block.signature"; - seq: number; - data: { signature: string; index?: number }; - } - | { - type: "inference.thinking.redacted"; - seq: number; - data: { redactedThinking: RedactedThinkingBlock; index?: number }; - } - | { - type: "inference.text.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.refusal.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.tool_call.start"; - seq: number; - data: { - callId: string; - name: string; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.tool_call.delta"; - seq: number; - data: { - callId: string; - argumentFragment: string; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.tool_call.end"; - seq: number; - data: { - callId: string; - name: string; - arguments: Record; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.usage"; - seq: number; - data: { usage: TokenUsage; source: LastCycleSource }; - } - | { - type: "inference.done"; - seq: number; - data: { - turn: AssistantTurn; - usage: TokenUsage; - source: LastCycleSource; - pacingDelayMs?: number; - }; - } - | { - type: "inference.error"; - seq: number; - data: { error: InferenceError; partial: PartialMessage }; - } - | { - /** - * Emitted between attempts when the per-call retry policy decides - * to retry after an error. `attempt` is the 1-indexed number of - * the attempt that just **failed** — the same value the policy - * saw on its `RetrySituation.attempt` reading. `delayMs` is the - * delay the wrapper will apply before the next attempt starts; - * `previousError` carries the classified error that triggered - * the retry. The event is not emitted when the policy aborts. - */ - type: "inference.retry"; - seq: number; - data: { - attempt: number; - delayMs: number; - previousError: InferenceError; - }; - } - | { - type: "inference.citation"; - seq: number; - data: { citation: CitationBlock; index?: number }; - } - | { - type: "inference.safety_rating"; - seq: number; - data: { safetyRating: SafetyRatingBlock }; - } - | { - type: "inference.code_execution.start"; - seq: number; - data: { request: CodeExecutionRequestBlock; index?: number }; - } - | { - type: "inference.code_execution.delta"; - seq: number; - data: { requestId: string; codeFragment: string; index?: number }; - } - | { - type: "inference.code_execution.result"; - seq: number; - data: { result: CodeExecutionResultBlock; index?: number }; - } - | { - type: "inference.image_output"; - seq: number; - data: { image: ImageBlock; index?: number }; - } - | { type: "tool.start"; seq: number; data: { call: ToolCall } } - | { - type: "tool.update"; - seq: number; - data: { callId: string; partial: string }; - } - | { type: "tool.done"; seq: number; data: { result: ToolResult } } - | { - type: "message.queued"; - seq: number; - data: { message: InboundMessage }; - } - | { - /** - * Per-message run-bracket open. Emitted by the reactor when it - * dequeues an inbound mail message and begins per-message work. - * - * `messageRunId` is reactor-minted, unique per dequeue. It is - * non-negotiable for crash-replay correlation: the reactor can - * legitimately dequeue the same `messageId` more than once across - * a crash + replay cycle, so two bracket-open events with the - * same `messageId` and no run-id cannot be unambiguously paired - * with their `message.run.ended` counterparts. - */ - type: "message.run.started"; - seq: number; - data: { - messageId?: string; - messageRunId: string; - receivedAt: number; - }; - } - | { - /** - * Per-message run-bracket close. Pairs with `message.run.started` - * by `messageRunId`. `messageId` is carried redundantly so log - * readers can correlate without a join against the open event; it is - * absent for a message that named no id of its own. - * - * The `status` enum is `"completed" | "failed"` only. - * Cancellation lives in the workflow-runtime's - * `CancelRequested` -> `RunFailed` vocabulary, not on the - * reactor's bracket: the reactor does not run a state machine - * and what it observes when cancellation arrives is a harness - * abort, which is structurally `"failed"` with a specific - * `error.kind`. - * - * `error.kind` is documented as one of - * `"inference_error" | "tool_error" | "reactor_fatal" | - * "harness_aborted" | "doom_loop"` initially, extensible as new - * failure categories surface. `"doom_loop"` marks a protective - * break the reactor took on the agent's behalf when the agent - * repeated an identical tool batch past the configured threshold; - * unlike `"reactor_fatal"` it is not an internal fault. - */ - type: "message.run.ended"; - seq: number; - data: { - messageRunId: string; - messageId?: string; - status: "completed" | "failed"; - error?: { - message: string; - kind?: string; - }; - }; - } - | { - type: "message.correlated"; - seq: number; - data: { message: InboundMessage; correlationId: string }; - } - | { - type: "connector.reply"; - seq: number; - data: { content: string; checkpointHash?: string }; - } - | { type: "reactor.start"; seq: number; data: Record } - | { - type: "reactor.gate.blocked"; - seq: number; - data: { - reason: GateType; - gateId: string; - correlationId?: string; - approvalSnapshot?: ApprovalSnapshot; - }; - } - | { - type: "reactor.gate.cleared"; - seq: number; - data: { - gateId: string; - reason: "resolved" | "timeout" | "shutdown"; - }; - } - | { type: "reactor.done"; seq: number; data: Record } - | { - type: "reactor.error"; - seq: number; - data: { error: string; fatal: boolean }; - } - | { - type: "fork.created"; - seq: number; - data: { forkId: string; parentId: string; mode: ForkMode }; - } - | { - type: "fork.done"; - seq: number; - data: { forkId: string; result?: unknown }; - } - | { - type: "fork.error"; - seq: number; - data: { forkId: string; error: string }; - } - | { type: "fork.aborted"; seq: number; data: { forkId: string } } - | { - type: `custom.${string}`; - seq: number; - data: Record; - }; - -// Load-bearing drift guards for the dual-maintained `reactor.gate.blocked` -// event. The arktype `InferenceEvent` validator and the hand-written -// `InferenceEvent` type are kept in lockstep by hand (the `custom.*` regex -// variant forces the manual mirror). arktype passes undeclared keys through at -// runtime, so a schema that dropped `approvalSnapshot` would not fail at -// runtime. Projecting the field off each inferred shape makes it load-bearing: -// `tsc` errors if either mirror stops carrying it, mirroring the -// `_persistedSuspendedCall` guard in storage-isogit. -const _arkGateBlockedApprovalSnapshot = ( - data: Extract< - typeof InferenceEvent.infer, - { type: "reactor.gate.blocked" } - >["data"], -): ApprovalSnapshot | undefined => data.approvalSnapshot; -void _arkGateBlockedApprovalSnapshot; - -const _tsGateBlockedApprovalSnapshot = ( - data: Extract["data"], -): ApprovalSnapshot | undefined => data.approvalSnapshot; -void _tsGateBlockedApprovalSnapshot; - -/** - * Validate unknown data as an InferenceEvent. ArkType's regex-based validator - * infers `custom.*` event types as `string`, but the manual InferenceEvent type - * uses a `custom.${string}` template literal for switch narrowing. This function - * centralizes that single unavoidable cast. - */ -export function parseInferenceEvent( - data: unknown, -): InferenceEvent | type.errors { - const result = InferenceEvent(data); - if (result instanceof type.errors) return result; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- arktype regex infers as string; manual type uses template literal - return result as InferenceEvent; -} - -/** - * A pending async operation registered in the reactor's async state. - * Correlates an outbound message (or payment/approval request) to the - * expected inbound response. - * - * (INFERENCE.md § Correlation) - */ -export type PendingOperation = { - correlationId: string; - kind: SignalKind; - registeredAt: number; - gateId: string; - /** - * Absolute deadline (epoch ms) for the gate that parks this operation. - * Persisted so that rehydration after a restart re-arms the gate with the - * remaining time against the original deadline rather than restarting the - * countdown. Absent for operations parked with no deadline. - */ - timeoutAt?: number; - /** - * The tool call that was suspended when this operation parked. Captured for - * `kind: "approval"` operations minted from the ask flow so the approved - * call can be re-run on resume. Absent for operations parked by the - * director path (async-tool pending markers), which carry no tool call. - */ - suspendedCall?: ToolCall; - /** - * Approver-facing snapshot of `suspendedCall`, built at the authz `ask` - * branch from the tool definition and the live arguments. A sibling of - * `suspendedCall`, not a widening of it: `suspendedCall` is the re-dispatch - * artifact, this is what the approver decides on. Present only for ask-rail - * operations that carry a `suspendedCall`; absent for async-tool pending - * markers. Threaded through the suspend hops to the hub co-write. - */ - approvalSnapshot?: ApprovalSnapshot; -}; - -/** - * Complete reactor state visible to the director decision function. - * - * `tokenUsage` is the cumulative usage across the session. - * - * `lastCycleUsage` and `lastCycleSource` describe the most recent - * *successful* inference call. They move together: both null before the - * first completion; every `inference.done` sets both atomically. - * `inference.error` does not clear either — the pair always reflects the - * last cycle that produced a well-defined turn and usage. The director's - * `afterInferenceDone` hook fires only on `inference.done`, so policy - * code never observes a torn or stale-vs-fresh window. - * - * The per-cycle values support compaction triggers that key off recent - * input cost rather than session totals, and state-aware policies (cost - * gating, budget caps, governance triggers) that need to attribute - * usage to the source that produced it. - * - * (INFERENCE.md § Agent Reactor › Director Decision Function) - */ -export type ReactorState = { - turns: ConversationTurn[]; - activeForks: { forkId: string; mode: ForkMode }[]; - pendingOperations: PendingOperation[]; - activeGates: { gateId: string; type: GateType; timeoutAt: number }[]; - tokenUsage: TokenUsage; - lastCycleUsage: TokenUsage | null; - lastCycleSource: LastCycleSource | null; - sessionId: string; -}; - -/** - * Actions the director can direct the reactor to take. - * - * (INFERENCE.md § Agent Reactor › Actions) - */ -export type ReactorAction = - | { - type: "infer"; - options?: InferenceOptions; - } - | { - type: "execute_tools"; - calls: ToolCall[]; - parallel?: boolean; - addToHistory?: boolean; - } - | { - type: "suspend"; - gate: { - type: GateType; - gateId: string; - timeoutMs: number; - correlationId?: string; - }; - } - | { - type: "fork"; - mode: ForkMode; - forkId: string; - } - | { - type: "emit"; - eventType: `custom.${string}`; - data: Record; - } - | { - type: "reply"; - content: string; - } - | { type: "checkpoint"; message: string } - | { type: "compact"; compactor: string; reason: string } - | { type: "wait" } - | { type: "done" }; - -/** - * The capabilities object passed to the director. Mirrors the `ReactorAction` - * union — provides a type-safe way for the director to construct actions. - * - * (INFERENCE.md § Agent Reactor › Director Decision Function) - */ -export type ReactorCapabilities = { - infer(options?: InferenceOptions): ReactorAction; - executeTools( - calls: ToolCall[], - parallel?: boolean, - addToHistory?: boolean, - ): ReactorAction; - suspend(gate: { - type: GateType; - gateId: string; - timeoutMs: number; - correlationId?: string; - }): ReactorAction; - fork(mode: ForkMode, forkId: string): ReactorAction; - emit( - eventType: `custom.${string}`, - data: Record, - ): ReactorAction; - reply(content: string): ReactorAction; - checkpoint(message?: string): ReactorAction; - compact(compactor: string, reason: string): ReactorAction; - wait(): ReactorAction; - done(): ReactorAction; -}; - -/** - * The inbound events delivered to the director decision function. - * - * `resume.execute_tools` is raised by the reactor when an approval resolves - * and a parked tool call must be re-run on resume. It carries the calls the - * reactor is about to dispatch so the director can seed its outstanding - * tool-result count before those calls' `tool.done` events arrive — the - * reactor drives the execution, the director counts the results. Without this - * seed the count would sit at zero and the first `tool.done` would drive an - * accidental re-inference off a negative count. - * - * `resume.tool_result` is raised by the reactor when a parked approval ends - * without running its tool — a rejected decision or a gate timeout. It carries - * a synthetic error tool result that answers the parked call so history stays - * well-formed; the director appends it and re-infers exactly once. No tool - * runs, so it seeds no outstanding-result count. - * - * (INFERENCE.md § Agent Reactor › Reactor Structure) - */ -export type ReactorInboundEvent = - | { type: "message.received"; message: InboundMessage } - | { - type: "inference.done"; - turn: AssistantTurn; - usage: TokenUsage; - source: LastCycleSource; - } - | { type: "inference.error"; error: InferenceError; partial: PartialMessage } - | { type: "tool.done"; result: ToolResult } - | { - type: "reactor.gate.cleared"; - gateId: string; - reason: "resolved" | "timeout" | "shutdown"; - } - | { type: "resume.execute_tools"; calls: ToolCall[] } - | { type: "resume.tool_result"; result: ToolResult } - | { type: "abort"; reason: AbortReason }; - -/** - * The core director is a single decision function: given an event and the - * current reactor state, return one or more actions. - * - * If the director throws, the reactor catches the exception, emits - * `reactor.error`, and initiates graceful shutdown. - * - * (INFERENCE.md § Reactor Director › Core Director) - */ -export interface ReactorDirector { - decide( - event: ReactorInboundEvent, - state: ReactorState, - capabilities: ReactorCapabilities, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Director Extension Hooks (INFERENCE.md § Reactor Director › Extension Hooks) -// --------------------------------------------------------------------------- - -/** - * Decision returned by a `BeforeToolExtension`. - * - * - `allow` — the tool proceeds. - * - `block` — the tool is answered with an error result carrying `reason`; - * the call is done. - * - `suspend` — the call is parked awaiting an external decision. The reactor - * registers `gate`, persists `pendingOp`, and does not answer the call: it - * is neither run nor error-completed. `gate.timeoutAt` is the absolute - * deadline (epoch ms) so the reactor can compute the remaining time; the - * `correlationId` on both `gate` and `pendingOp` ties an inbound resolution - * back to the suspension. - */ -export type BeforeToolDecision = - | { type: "allow" } - | { type: "block"; reason: string } - | { - type: "suspend"; - gate: { - type: GateType; - gateId: string; - correlationId: string; - timeoutAt: number; - }; - pendingOp: PendingOperation; - }; - -/** - * Extension that runs before a tool call is executed. Returns a - * `BeforeToolDecision`: `allow` lets the call run, `block` answers it with an - * error result, `suspend` parks it awaiting an external decision. - * - * `grantOneShot` registers a within-cycle bypass token keyed on a - * `ToolCall.id`: the next `beforeTool` for that id skips a suspension it would - * otherwise raise, consuming the token as it does so. It is optional because - * only extensions that can suspend a call have anything to bypass; extensions - * that never suspend omit it. - */ -export interface BeforeToolExtension { - beforeTool( - call: ToolCall, - state: ReactorState, - signal: AbortSignal, - ): Promise; - grantOneShot?(id: string): void; -} - -/** - * Extension that runs after a tool result is produced. Can modify the result - * (redaction, enrichment, audit logging). Extensions run in order. - */ -export interface AfterToolExtension { - afterTool( - result: ToolResult, - call: ToolCall, - state: ReactorState, - signal: AbortSignal, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Context Strategies: Transforms and Compactors -// (INFERENCE.md § Context Management, § Tool Result Lifecycle) -// --------------------------------------------------------------------------- - -/** - * Durable description of a single strategy invocation. Written to the - * per-cycle manifest in the context store so that future operators can - * reconstruct exactly which strategy made which change, with what - * parameters, and why. - * - * - `strategy` is the implementation name (e.g. `"size-cap"`). - * - `version` is the implementation version. Changes to the strategy's - * behavior bump the version so old manifest entries remain unambiguous. - * - `parameters` records the configuration the strategy ran with. - * - `reason` is a short machine-readable cause label - * (e.g. `"exceeded-cap"`, `"overflow-recovery"`). - * - `decisions` records strategy-specific details about what was actually - * done (e.g. the keep count, the spill key, the original byte size). - */ -export const TransformRecord = type({ - strategy: "string", - version: "string", - parameters: "Record", - reason: "string", - decisions: "Record", -}); -export type TransformRecord = typeof TransformRecord.infer; - -/** - * Per-invocation context passed to every `ContextStrategy.apply` call. - * `state` is the reactor's snapshot at the moment the strategy runs; - * `trigger` is a short label describing why the strategy was invoked - * (e.g. `"tool-result-ingest"`, `"pre-inference"`, `"director-request"`). - */ -export interface StrategyContext { - readonly state: ReactorState; - readonly trigger: string; -} - -/** - * Optional blob attachment emitted by a strategy. The reactor writes each - * blob to the context store's working tree via `ContextStore.writeBlob` - * (Phase 2) so the data is durable and migrates with the conversation. - */ -export type StrategyBlob = { - key: string; - bytes: Uint8Array; - contentType?: string; -}; - -/** - * Result returned by `ContextStrategy.apply`. Carries the transformed - * output, a `TransformRecord` describing what happened, and any blobs - * that should be persisted in the context store. - */ -export interface StrategyResult { - output: O; - record: TransformRecord; - blobs?: StrategyBlob[]; -} - -/** - * Generic base interface for content-mutating strategies. The role-specific - * aliases below specialize `I` and `O` for tool-result ingestion, pre- - * inference context shaping, and explicit compaction. - * - * Strategies are pure with respect to the context store: they describe what - * should change via their return value. The reactor decides where to write - * the result (history, prompt, manifest) and which blobs to persist. - */ -export interface ContextStrategy { - readonly name: string; - readonly version: string; - apply(input: I, ctx: StrategyContext): Promise>; -} - -/** - * Runs on each tool result entering history. Output is appended to the - * conversation; any emitted blobs are written to the context store's - * `tool-output/` directory. - */ -export type ToolResultTransform = ContextStrategy< - { call: ToolCall; result: ToolResult }, - ToolResult ->; - -/** - * Runs in order before every inference call, producing the materialized - * prompt. Output is written to `prompt.jsonl` for that cycle; the durable - * history in `turns.jsonl` is left untouched. - * - * (INFERENCE.md § Async State Awareness › Pending Status Injection) - */ -export type ContextTransform = ContextStrategy< - ConversationTurn[], - ConversationTurn[] ->; - -/** - * Named compaction strategy. Registered in a registry on the reactor and - * invoked explicitly via the director's `compact` action. Output overwrites - * `turns.jsonl`; a `TransformRecord` is appended to the manifest. - */ -export type Compactor = ContextStrategy; - -// --------------------------------------------------------------------------- -// Blob Reader (INFERENCE.md § Tool Result Lifecycle) -// --------------------------------------------------------------------------- - -/** - * Read-only capability for resolving `tool-output:///{callId}` URIs to the - * underlying blob bytes. A `ToolResultTransform` that spills oversized tool - * output writes a blob via `ContextStore.writeBlob` and returns a pointer of - * the form `tool-output:///{callId}`; the agent's read tool reaches the spill - * by calling `BlobReader.read(uri)`. - * - * The URI scheme is deliberately rigid: - * - * - Scheme: `tool-output` - * - Authority: empty (the `///` makes pathname carry the callId) - * - Path: `/{callId}` — preserves case so provider-assigned callIds with - * uppercase letters survive parsing - * - Query and fragment: rejected - * - * Any deviation (different scheme, missing or non-empty hostname, extra path - * segments, search string, or fragment) throws. Missing blobs throw. - * `BlobReader` never accepts a filesystem path; the agent has no direct view - * of the context store's working tree. - */ -export interface BlobReader { - /** - * Resolve `uri` to the underlying blob bytes. Throws if the URI is not a - * well-formed `tool-output:///{callId}` reference or if no blob exists for - * the extracted callId. - */ - read(uri: string): Promise; -} - -/** Source for blob bytes used by `createBlobReader`. */ -export interface BlobSource { - readBlob(key: string, signal?: AbortSignal): Promise; -} - -/** - * Parse a `tool-output:///{callId}` URI and return the callId. Throws on any - * deviation from the documented shape: wrong scheme, non-empty authority, - * missing or extra path components, search string, or fragment. - * - * The two-slash form `tool-output://abc` is rejected because the URL parser - * lowercases the hostname, which silently corrupts provider-assigned callIds - * that contain uppercase letters. The three-slash form puts the callId in - * `pathname`, where case is preserved. - */ -export function parseToolOutputURI(uri: string): string { - let parsed: URL; - try { - parsed = new URL(uri); - } catch (cause) { - throw new Error(`invalid tool-output URI: ${uri}`, { cause }); - } - if (parsed.protocol !== "tool-output:") { - throw new Error( - `invalid tool-output URI scheme: expected "tool-output:", got "${parsed.protocol}"`, - ); - } - if (parsed.hostname !== "") { - throw new Error( - `invalid tool-output URI: authority must be empty (use the form tool-output:///{callId}), got "${parsed.hostname}"`, - ); - } - if (parsed.search !== "") { - throw new Error( - `invalid tool-output URI: query string is not allowed, got "${parsed.search}"`, - ); - } - if (parsed.hash !== "") { - throw new Error( - `invalid tool-output URI: fragment is not allowed, got "${parsed.hash}"`, - ); - } - const path = parsed.pathname; - if (!path.startsWith("/")) { - throw new Error(`invalid tool-output URI: empty path: ${uri}`); - } - const callId = path.slice(1); - if (callId === "") { - throw new Error(`invalid tool-output URI: missing callId: ${uri}`); - } - if (callId.includes("/")) { - throw new Error( - `invalid tool-output URI: path must contain a single callId segment, got "${callId}"`, - ); - } - return callId; -} - -/** - * Construct a `BlobReader` that resolves `tool-output:///{callId}` URIs by - * delegating to `source.readBlob(callId)`. The most common source is a - * `ContextStore` (Phase 2 added `readBlob` to that interface), but any object - * implementing `BlobSource` works — this keeps tests trivial. - * - * URI parsing is performed in this layer; the source only ever sees the - * extracted callId. Missing blobs surface as whatever error the source - * raises (`ContextStore.readBlob` already throws for unknown keys). - */ -export function createBlobReader(source: BlobSource): BlobReader { - return { - async read(uri: string): Promise { - const callId = parseToolOutputURI(uri); - return source.readBlob(callId); - }, - }; -} - -// --------------------------------------------------------------------------- -// Abort Reasons (INFERENCE.md § Abort Handling) -// --------------------------------------------------------------------------- - -/** - * Reason codes for the `abort` reactor event. The reason determines the - * appropriate cleanup action. - * - * (INFERENCE.md § Abort Handling › Abort Reasons) - */ -export const AbortReason = type.enumerated( - "user_disconnect", - "wallet_exhaustion", - "admin_kill", - "session_timeout", - "credential_revocation", -); -export type AbortReason = typeof AbortReason.infer; - -// --------------------------------------------------------------------------- -// Inference Source (INFERENCE.md § Providers) -// --------------------------------------------------------------------------- - -/** - * Model-bound default knobs for an inference source. Per-call - * `InferenceOptions.X` overrides `defaults.X`; the merge happens once at - * the top of `runInference` before the adapter sees anything. New fields - * land here as separately-scoped issues. - */ -export const InferenceSourceDefaults = type({ - "maxTokens?": "number", - // A bag of provider-native knobs the caller wants merged into the - // outbound request body (Anthropic's `metadata.user_id`, - // OpenAI's `user`, Gemini's `safetySettings`, etc.). Adapters that - // recognize keys translate; unrecognized keys are passed through or - // dropped per the adapter's documented behavior. The merge into - // per-call `InferenceOptions.providerOptions` is shallow — a per- - // call providerOptions object wholesale replaces the source-bound - // one, it does not deep-merge per key. - "providerOptions?": "Record", -}); -export type InferenceSourceDefaults = typeof InferenceSourceDefaults.infer; - -/** - * A specific (provider, model) bundle the agent runtime can route to. - * Carries wire reachability, credentials, the model identity at the - * provider, and the model-bound default knobs. - * - * `id` is the catalog offering's primary key, set by the resolver from the - * matched offering. It is the routing key used by `AgentConfig.defaultSource` - * and `Agent.setSource`. - * - * Multi-model providers become multiple sources — `model` is part of the - * identity, not an optional override. - * - * `capabilities` is carried for the selection-policy layer (the model - * selector consumes it). The runtime ignores it; populating the field - * later is not a wire-format change. - * - * `quirks` is the opaque per-deployment bag of provider-specific adapter - * accommodations. The harness reads it once, at adapter instantiation, and - * passes it to `AdapterRegistry.resolve` as a sibling of the slim - * `LastCycleSource` — quirks are deliberately kept off `LastCycleSource`, - * which rides on every usage event. The field is present-and-populated or - * absent; it is never `null`. A source row with no quirks stores SQL `NULL`, - * and the catalog resolver translates that absence into an omitted key here, - * so downstream code sees `undefined`, never `null`. - * - * (INFERENCE.md § Providers) - */ -export const InferenceSource = type({ - id: "string", - provider: "string", - baseURL: "string", - // Reference into the run's credential-material cell. The provider's secret - // (formerly an inline `apiKey`) is resolved from that cell by `credentialId` - // at call time, so the source config carries no secret and the child never - // holds the key inline. The same cell backs tool credentials. - credentialId: "string", - model: "string", - "defaults?": InferenceSourceDefaults, - "capabilities?": "string[]", - "quirks?": "Record", -}); -export type InferenceSource = typeof InferenceSource.infer; - -/** - * Replace every field on `active` with the corresponding field from - * `next`, in place. Optional fields (`defaults`, `capabilities`, - * `quirks`) are `delete`d from `active` when absent on `next` so the - * swap is exact — no stale value from a previous rotation can survive. - * - * Used by both the agent's source registry and the harness's source - * hot-swap path to mutate the single shared `InferenceSource` object the - * reactor reads lazily at the start of each inference call. Putting the - * field list in one place means the next field added to - * `InferenceSource` only has to be remembered here. - */ -export function applyInferenceSourceFields( - active: InferenceSource, - next: InferenceSource, -): void { - active.id = next.id; - active.provider = next.provider; - active.baseURL = next.baseURL; - active.credentialId = next.credentialId; - active.model = next.model; - if (next.defaults !== undefined) { - active.defaults = next.defaults; - } else { - delete active.defaults; - } - if (next.capabilities !== undefined) { - active.capabilities = next.capabilities; - } else { - delete active.capabilities; - } - if (next.quirks !== undefined) { - active.quirks = next.quirks; - } else { - delete active.quirks; - } - - // Compile-time exhaustiveness check. `Required<>` forces optional - // keys to also be required in the guard — so a future optional field - // (e.g. `region?: string`) added to `InferenceSource` without being - // handled above is flagged by TypeScript, not silently dropped. - const _handled: { readonly [K in keyof Required]: true } = { - id: true, - provider: true, - baseURL: true, - credentialId: true, - model: true, - defaults: true, - capabilities: true, - quirks: true, - }; - void _handled; -} - -/** - * Outcome of a `RetryPolicy` consultation. Either abort the call - * (surface the most recent `inference.error` to the caller), or retry - * after `delayMs` milliseconds, measured against the harness Scheduler. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetryDecision = - | { kind: "abort" } - | { kind: "retry"; delayMs: number }; - -/** - * Context supplied to a `RetryPolicy` each time an attempt produces an - * `inference.error`. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetrySituation = { - /** The classified error the most recent attempt produced. */ - readonly error: InferenceError; - /** - * 1-indexed attempt counter. The first failure has `attempt: 1`; - * the second failure (after one retry) has `attempt: 2`; and so on. - */ - readonly attempt: number; - /** - * Milliseconds since the *first* attempt of this call started, - * measured via the harness `Scheduler.now()`. The default Scheduler - * uses `performance.now()` (sub-millisecond resolution), so the - * value may be fractional; virtual-clock test schedulers report - * integer virtual time. Both are valid; policies that compare - * against integer thresholds should `Math.floor` if they need that. - */ - readonly elapsedMs: number; -}; - -/** - * Per-call retry policy. The harness invokes the policy once per - * `inference.error` an attempt produces, in 1-indexed attempt order. - * Returning `{ kind: "abort" }` ends the call by surfacing the most - * recent error to the caller; returning `{ kind: "retry", delayMs }` - * causes the harness to discard the failed attempt's events, sleep - * `delayMs` milliseconds against the Scheduler, and re-issue the - * underlying HTTP request with the identical body. The policy may be - * async; the harness awaits the returned `Promise` if - * it is a thenable. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetryPolicy = ( - situation: RetrySituation, -) => RetryDecision | Promise; - -/** - * Options for a single inference call. Override the defaults from the agent - * configuration on a per-call basis. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type InferenceOptions = { - maxTokens?: number; - temperature?: number; - thinking?: { enabled: boolean; budgetTokens?: number }; - systemPrompt?: string; - tools?: ToolDefinition[]; - /** - * Modalities the caller wants the model to emit. Adapters translate - * to the provider-native shape (Gemini's - * `generationConfig.responseModalities` accepts `"TEXT"` / `"IMAGE"` - * uppercase; see `packages/inference-discovery-google-genai/sessions/ - * google-genai/gemini-2.5-flash-image/image-output/exchanges/0/request.json` - * for the captured shape). Providers that do not expose a modality - * switch ignore the - * field. When omitted the provider's default modalities apply. - */ - responseModalities?: ("text" | "image" | "audio")[]; - /** - * Structured-output constraint. Asks the model to produce text, free- - * form JSON, or JSON conforming to a specific schema. Adapters - * translate to the provider-native wire shape: - * - * - **OpenAI** (`response_format`): - * - `text` → `{ type: "text" }` - * - `json` → `{ type: "json_object" }` - * - `json-schema` → `{ type: "json_schema", json_schema: { name, schema, strict } }` - * When the model declines in strict mode, the wire emits - * `delta.refusal` chunks; the adapter surfaces them as - * `inference.refusal.delta` events and a final `RefusalBlock` in - * the assistant turn's `content[]`. - * - **Google GenAI** (`generationConfig`): - * - `text` → no constraint (default). - * - `json` → `{ responseMimeType: "application/json" }`. - * - `json-schema` → `{ responseMimeType: "application/json", responseSchema: }`. - * `name` and `strict` are OpenAI-specific and have no Gemini - * counterpart; adapters ignore them. Gemini enforces a subset of - * JSON Schema (no `oneOf`, limited `pattern`, no `$ref`, etc.) — - * the adapter forwards the schema verbatim and surfaces Gemini's - * HTTP error if the subset is violated. - * - **Anthropic**: no native structured-output API. - * - `text` is a no-op (the default). - * - `json` and `json-schema` throw at the adapter boundary; there - * is no shim that synthesizes a tool to extract structured - * output. - * - * When omitted the provider's default applies (typically free-form - * text). - */ - responseFormat?: - | { kind: "text" } - | { kind: "json" } - | { - kind: "json-schema"; - name: string; - schema: unknown; - strict?: boolean; - }; - /** - * A bag of provider-native knobs the adapter merges into the outbound - * request body. Primary home is `InferenceSourceDefaults.providerOptions` - * (model-bound); this field exists for per-call overrides through the - * standard merge precedence at the top of `runInference`. The merge is - * shallow: a per-call providerOptions object wholesale replaces the - * source-bound one, it does not deep-merge per key. - */ - providerOptions?: Record; - /** - * Per-call inactivity timeout in milliseconds. If the harness yields no - * event (other than `inference.start`) for this many ms, the underlying - * fetch is aborted and the call ends with `inference.error` of category - * `"timeout"`. Default 120_000 (2 min). Tune higher for reasoning models - * that exhibit long silent-thinking stretches between token bursts; tune - * lower to fail fast. `0` arms the timer to fire on the next tick (a - * "fail-fast even if the fetch is instant" mode useful in tests). - */ - inactivityTimeoutMs?: number; - /** - * Per-call total wall-clock cap in milliseconds. Starts at fetch. - * Default 600_000 (10 min). Backstop for streams that keep emitting - * forever without terminating. Same error category as `inactivityTimeoutMs`. - * `0` arms the timer to fire on the next tick. - */ - totalTimeoutMs?: number; - /** - * Per-call mechanical retry policy. Consulted once per attempt that - * ends in `inference.error`; see `RetryPolicy` for the contract. If - * omitted, a built-in default policy is applied. - */ - retryPolicy?: RetryPolicy; -}; - -// --------------------------------------------------------------------------- -// Context Store (INFERENCE.md § Context Management › Context Store, -// ARCHITECTURE.md § Change History) -// --------------------------------------------------------------------------- - -/** - * A named commit point in the context store. Corresponds to a git commit. - * - * (ARCHITECTURE.md § Change History › Named Checkpoints) - */ -export type ContextCommit = { - hash: string; - message: string; - timestamp: number; - parentHash?: string; -}; - -/** - * The state of an active connector thread. The connector is one durable - * thread per agent; participants accumulate as they speak. Persisted - * alongside the conversation context so the thread survives sidecar - * restarts. - * - * `replyTo` is the most recent speaker — the primary recipient (`to`) - * on the next outbound reply. `cc` is every other participant who has - * spoken on the thread, deduplicated, in arrival order — they ride as - * `cc` on the next outbound reply so everyone stays in the loop. - * `subject` is set when the thread starts and preserved for its life. - * - * Defined as an arktype so the wire layer (sidecar↔hub frames) and - * other parsing boundaries can validate snapshots without - * re-declaring the shape. - */ -export const ConnectorThreadState = type({ - "threadRoot?": "string", - "lastMessageId?": "string", - replyTo: "string", - cc: "string[]", - "subject?": "string", -}); -export type ConnectorThreadState = typeof ConnectorThreadState.infer; - -/** - * The context store interface. Implementations back the store with git - * (filesystem, in-memory, or virtual) depending on the execution environment. - * The reactor accepts any implementation that satisfies this interface. - * - * The store holds the turn history and reactor metadata. Forking creates - * a git branch. Compaction commits the compacted history. - * - * (INFERENCE.md § Context Management › Context Store) - */ -export interface ContextStore { - /** - * Load the current turn history and reactor metadata from the store. - * Called during reactor initialization. - */ - load(signal?: AbortSignal): Promise<{ - turns: ConversationTurn[]; - pendingOperations: PendingOperation[]; - tokenUsage: TokenUsage; - connectorState: ConnectorThreadState | null; - }>; - - /** - * Buffer connector thread state for the next commit. The harness calls - * this before each checkpoint so that connector state is persisted - * atomically with the conversation context. - */ - setConnectorState(state: ConnectorThreadState | null): void; - - /** - * Commit whatever currently lives in the working tree, using the supplied - * commit message. The reactor's per-cycle checkpoint routes through this - * overload after writing the per-cycle files via `writeTurns`, - * `writePrompt`, `writeResponse`, `writeManifest`, and any `writeBlob` - * calls produced by transforms. - */ - commit( - options: { message: string }, - signal?: AbortSignal, - ): Promise; - - /** - * Create a branch for a fork operation. The branch starts from the current - * HEAD commit. - */ - branch(name: string, signal?: AbortSignal): Promise; - - /** - * List recent commits. Used by the agent's history query tools. - */ - log(limit?: number, signal?: AbortSignal): Promise; - - /** - * Read the turn history at a specific commit hash. Used for history - * inspection and rollback. - */ - readAt(hash: string, signal?: AbortSignal): Promise; - - /** - * Write an opaque blob to the working tree under `tool-output/`. Used by - * `ToolResultTransform`s that spill oversized payloads out of the inline - * conversation. The file is staged at the next `commit({ message })`. - * - * `key` is sanitized for filesystem safety; callers should pass the tool - * call id. `contentType` selects a file extension when known. - */ - writeBlob( - key: string, - bytes: Uint8Array, - contentType?: string, - signal?: AbortSignal, - ): Promise; - - /** - * Read a blob previously written via `writeBlob`. Throws if no blob with - * that key exists. - */ - readBlob(key: string, signal?: AbortSignal): Promise; - - /** - * Overwrite `prompt.jsonl` with the materialized prompt for the current - * inference cycle. One `ConversationTurn` per line. Staged at the next - * `commit({ message })`. - */ - writePrompt(turns: ConversationTurn[], signal?: AbortSignal): Promise; - - /** - * Overwrite `response.jsonl` with the assistant turn returned for the - * current cycle. Single-line JSONL for consistency with the per-cycle file - * conventions. Staged at the next `commit({ message })`. - */ - writeResponse(turn: AssistantTurn, signal?: AbortSignal): Promise; - - /** - * Overwrite `manifest.jsonl` with the ordered transform records produced - * for the current cycle. One `TransformRecord` per line. Staged at the - * next `commit({ message })`. - */ - writeManifest( - records: TransformRecord[], - signal?: AbortSignal, - ): Promise; - - /** - * Overwrite `turns.jsonl` with the durable conversation history. One - * `ConversationTurn` per line. Staged at the next `commit({ message })`. - */ - writeTurns(turns: ConversationTurn[], signal?: AbortSignal): Promise; - - /** - * Overwrite `metadata.json` with non-turn-shaped reactor state needed for - * restart: pending async operations and cumulative token usage. The store - * combines this with the most recently buffered connector state (from - * `setConnectorState`) and writes the merged payload. Staged at the next - * `commit({ message })`. - */ - writeMetadata( - metadata: { - pendingOperations: PendingOperation[]; - tokenUsage: TokenUsage; - }, - signal?: AbortSignal, - ): Promise; - - /** - * Read manifest entries from the most recent `limit` commits that contain - * a `manifest.jsonl`. Newest commit first; records within a commit are - * returned in their natural in-file order (chronological per-cycle). - */ - readManifestHistory( - limit: number, - signal?: AbortSignal, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Audit Store (INTR-4 § Audit Trail) -// --------------------------------------------------------------------------- - -/** - * Persistent store for tool invocation audit records. Separated from - * ContextStore so the audit capability is opt-in at the composition - * layer. The isogit implementation writes audit records as individual - * JSON files in the same git repo used for context storage. - */ -export interface AuditStore { - /** - * Persist a batch of audit records. Called at checkpoint boundaries - * with all records accumulated since the last checkpoint. - */ - commitAudit(records: AuditRecord[], signal?: AbortSignal): Promise; - - /** - * Load audit records for a session. Returns all records matching - * the given sessionId, ordered by seq. - */ - loadAudit(sessionId: string, signal?: AbortSignal): Promise; - - /** - * Persist a batch of error records. Called at checkpoint boundaries - * and shutdown with all error records accumulated since the last flush. - */ - commitErrors(records: ErrorRecord[], signal?: AbortSignal): Promise; -} - -// --------------------------------------------------------------------------- -// Agent / Harness Configuration (ARCHITECTURE.md § Agent Harness) -// --------------------------------------------------------------------------- - -/** - * Configured tool definition exposed to the model. The harness registers - * available tools; the reactor passes this list to the inference provider as - * part of each request. - * - * (ARCHITECTURE.md § Agent Harness › Tools) - */ -export const ToolDefinition = type({ - name: "string", - description: "string", - inputSchema: "Record", -}); -export type ToolDefinition = typeof ToolDefinition.infer; - -/** - * Agent harness configuration. Assembled from the agent definition package - * and capability grants during harness initialization. - * - * `principalId` is the agent's principal in the hub's authorization model. - * The sidecar needs it to reconstruct the in-memory grant store on restart - * (the store's `collectGrants` filters by principal). - * - * `grants` uses `WireGrantRule` because this type arrives over JSON where - * `GrantRule.expiresAt` is serialized as a string. The wire validator - * coerces strings back to Date instances. - * - * (ARCHITECTURE.md § Agent Harness) - */ -export const HarnessConfig = type({ - sessionId: "string", - agentId: "string", - tenantId: "string", - principalId: "string", - agentAddress: "string", - systemPrompt: "string", - tools: ToolDefinition.array(), - grants: WireGrantRule.array(), - sources: InferenceSource.array(), - defaultSource: "string", - "sessionChannelEnabled?": "boolean", -}); -export type HarnessConfig = typeof HarnessConfig.infer; +// Runtime contracts and the inference event wire protocol. Process-specific +// consumers can import runtime-core without constructing the event union. +export * from "./runtime-core"; +export * from "./inference-events"; +export * from "./format-safety-rating"; diff --git a/vendor/intx/types/src/sessions.ts b/vendor/intx/types/src/sessions.ts index 8296b76eaf..04be03c43f 100644 --- a/vendor/intx/types/src/sessions.ts +++ b/vendor/intx/types/src/sessions.ts @@ -91,11 +91,12 @@ export const MailResponse = type({ }); export type MailResponse = typeof MailResponse.infer; -// Structured attachment-rejection errors returned by POST /:runId/mail. -// Each variant carries a machine-actionable `code` plus the fields a client -// needs to locate and explain the rejection, alongside a human-readable -// `message`. This is the wire contract for the route's attachment 400s; the -// route handler is the single producer. +// Structured attachment-rejection errors. Each variant carries a +// machine-actionable `code` plus the fields a client needs to locate and +// explain the rejection, alongside a human-readable `message`. The shared +// validator emits every variant but `invalid_encoding`, which only the +// mail tools produce; the mail routes return the union as their +// attachment 400 body and the mail tools return its `code` and `message`. export const AttachmentError = type.or( { code: "'oversize_attachment'", @@ -120,12 +121,23 @@ export const AttachmentError = type.or( message: "string", attachmentIndex: "number", }, + { + code: "'invalid_encoding'", + message: "string", + attachmentIndex: "number", + }, { code: "'oversize_total'", message: "string", totalBytes: "number", limitBytes: "number", }, + { + code: "'too_many_attachments'", + message: "string", + count: "number", + limit: "number", + }, ); export type AttachmentError = typeof AttachmentError.infer; diff --git a/vendor/intx/types/src/sidecar-allocation.ts b/vendor/intx/types/src/sidecar-allocation.ts index 6a5a4cf7e8..f830617cfe 100644 --- a/vendor/intx/types/src/sidecar-allocation.ts +++ b/vendor/intx/types/src/sidecar-allocation.ts @@ -12,9 +12,14 @@ export const sidecarAllocationStatuses = [ export type SidecarAllocationStatus = (typeof sidecarAllocationStatuses)[number]; +export type DispatchableSidecarAllocationStatus = Extract< + SidecarAllocationStatus, + "pending" | "provisioning" | "allocated" | "replacing" +>; + export function isSidecarAllocationDispatchable( status: SidecarAllocationStatus, -): boolean { +): status is DispatchableSidecarAllocationStatus { switch (status) { case "pending": case "provisioning": diff --git a/vendor/intx/types/src/sidecar.ts b/vendor/intx/types/src/sidecar.ts index d4ca93ceca..5f215a59ad 100644 --- a/vendor/intx/types/src/sidecar.ts +++ b/vendor/intx/types/src/sidecar.ts @@ -8,6 +8,8 @@ // efficient but JSON is simpler to debug and inspect. import { type } from "arktype"; +import { CredentialDelivery } from "./credential-delivery"; +import { RepoId } from "./repo"; import { GrantWalkSnapshot } from "./grant-snapshot"; import { ApprovalItem } from "./grants"; import { WireGrantRule } from "./grant-wire"; @@ -39,17 +41,59 @@ import { WorkflowDefinitionSource } from "./workflow-sources"; // and routes through the existing invalid-frame drop+log path; no handler change // is needed. -// A sidecar's reported agent addresses. The register/reconnect handler already -// gates each reported address against the allocation's single minted workflow -// address, so the legitimate count is ~1; this is a generous absurdity backstop. -export const MAX_AGENT_ADDRESSES_FRAME = 512; +export const DEFAULT_SIDECAR_ACTIVE_DEPLOYMENTS = 128; +export const DEFAULT_SIDECAR_RETAINED_DEPLOYMENTS = 256; +// Bounds inventory processing independently of an operator's capacity settings. +export const MAX_SIDECAR_INCARNATIONS = 2_048; + +export const SidecarCapacity = type({ + maxActiveDeployments: "number.integer >= 1", + maxRetainedDeployments: "number.integer >= 0", +}).narrow( + (capacity, ctx) => + capacity.maxActiveDeployments + capacity.maxRetainedDeployments <= + MAX_SIDECAR_INCARNATIONS || + ctx.mustBe( + `at most ${String(MAX_SIDECAR_INCARNATIONS)} deployments combined`, + ), +); +export type SidecarCapacity = typeof SidecarCapacity.infer; + +export const DEFAULT_SIDECAR_CAPACITY: SidecarCapacity = { + maxActiveDeployments: DEFAULT_SIDECAR_ACTIVE_DEPLOYMENTS, + maxRetainedDeployments: DEFAULT_SIDECAR_RETAINED_DEPLOYMENTS, +}; + +// Bounds on what a sidecar reports about a deployment it holds. The Hub keeps +// a reported stop until it fails the deployment, and stores a deploy error on +// each of the deployment's runs, so these, not the frame-size limit, bound +// what one sidecar can make it hold. A deployment address is a mail address +// of at most 320 characters, and its history has a few refs. +export const MAX_DEPLOYMENT_ADDRESS_LENGTH = 320; +export const MAX_DEPLOYMENT_ERROR_LENGTH = 4096; +export const MAX_REF_TIPS_FRAME = 16; +export const MAX_REF_TIP_ENTRY_LENGTH = 255; + +/** + * Fits an error to the non-empty, bounded string accepted in sidecar reports. + */ +export function fitDeploymentError(error: string): string { + if (error.length === 0) return "Deployment failed without an error message"; + if (error.length <= MAX_DEPLOYMENT_ERROR_LENGTH) return error; + // A cut between the two halves of a surrogate pair would leave a code + // unit the Hub stores as a replacement character. + let end = MAX_DEPLOYMENT_ERROR_LENGTH - 3; + const last = error.charCodeAt(end - 1); + if (last >= 0xd800 && last <= 0xdbff) end -= 1; + return `${error.slice(0, end)}...`; +} // A sidecar's reported cached sender addresses. This MUST stay well above the // `MAX_RESYNC_SENDER_ADDRESSES` handler cap (currently 2048 in the hub-sessions // sidecar-handler): that cap drives a graceful "resync the first N, log the // overflow" degrade rather than dropping the frame, so a schema ceiling at or // below it would turn the degrade into a hard reconnect outage -- the whole -// register frame would fail this parse and drop, and the sidecar could not +// hello frame would fail this parse and drop, and the sidecar could not // reconnect. The `@intx/types` package must not import from `@intx/hub-sessions`, // so the coupling is a documented invariant guarded by a test in that package. export const MAX_CACHED_SENDER_ADDRESSES_FRAME = 65536; @@ -96,102 +140,183 @@ export const MAX_MAIL_OUTBOUND_BODY_BYTES = 44 * 1024 * 1024; const FRAME_OVERHEAD_BYTES = 20 * 1024 * 1024; // The ceiling wired as the hub sidecar websocket's `maxPayloadLength`. Bun -// closes the connection on a RECEIVED message larger than this, so it must clear -// the largest legit received frame -- the `mail.outbound` frame, whose -// rawMessage is bounded by `MAX_MAIL_OUTBOUND_BODY_BYTES`, plus framing -// overhead. maxPayloadLength gates incoming messages only; it does NOT limit +// closes the connection on a RECEIVED message larger than this. It clears a +// `mail.outbound` body at its byte limit plus framing overhead; all other +// inbound frames, including complete deployment inventories, must also fit. +// maxPayloadLength gates incoming messages only; it does NOT limit // what the hub sends, so the hub->sidecar inline-asset deploy does not factor // into this number. export const MAX_SIDECAR_FRAME_BYTES = MAX_MAIL_OUTBOUND_BODY_BYTES + FRAME_OVERHEAD_BYTES; // --------------------------------------------------------------------------- -// Sidecar → Hub +// Deployment incarnations // --------------------------------------------------------------------------- +// +// One connection can carry several deployments, and an allocation's +// generation advances while its sidecar may still hold or send for the earlier +// one, as when the Hub releases a deployment its sidecar keeps running. An +// incarnation is a deployment address plus the allocation generation that owns +// it. Every frame +// that concerns one deployment names its incarnation, except +// `connector.state.changed` and replies matched to their request by an id, so +// neither end infers it from the socket, the order of frames, or their timing, +// and each end drops or refuses a frame for an incarnation it does not hold. + +/** An allocation generation. An allocation's generations only increase. */ +export const Generation = type("number.integer >= 0"); +export type Generation = typeof Generation.infer; /** - * Sent on first connect when the sidecar has no existing agents in its data - * directory. Identifies the sidecar and declares it ready to receive - * agent.deploy frames. + * The incarnation a deployment frame belongs to. `run.grants` and + * `signal.deliver` payloads are also validated and persisted away from the + * wire, where no generation applies yet, so their schemas leave it out and the + * frame unions below add this stamp to them. */ -export const RegisterFrame = type({ - type: "'register'", - sidecarId: "string", - token: "string", - agentAddresses: type("string") - .array() - .atMostLength(MAX_AGENT_ADDRESSES_FRAME), - // The rotatable (non-run) sender addresses this sidecar holds cached keys - // for. The hub re-resolves each current key and re-pushes it on a - // `sender.key.refresh`, so a user-principal rotation that landed while the - // sidecar was disconnected reaches its cache. Additive-optional and omitted - // when empty: a sidecar with no cached senders (or a pre-upgrade one) sends - // no field, and the hub treats absence as "nothing to refresh". - "cachedSenderAddresses?": type("string") - .array() - .atMostLength(MAX_CACHED_SENDER_ADDRESSES_FRAME), -}); -export type RegisterFrame = typeof RegisterFrame.infer; +export const IncarnationStamp = type({ generation: Generation }); +export type IncarnationStamp = typeof IncarnationStamp.infer; /** - * Sent on connect after a provisioned sidecar restores its deployment. - * The bearer token binds the connection to one allocation generation, so the - * Hub accepts only that allocation's workflow address. + * What a sidecar is doing with an incarnation it holds. A `stopped` one no + * longer runs, and its local state is kept for inspection until the Hub + * undeploys it. */ -export const ReconnectFrame = type({ - type: "'reconnect'", +export const IncarnationState = type.enumerated( + "deploying", + "live", + "stopped", + "tearing-down", +); +export type IncarnationState = typeof IncarnationState.infer; + +export const DeploymentRetention = type.enumerated("kept", "refused"); +export type DeploymentRetention = typeof DeploymentRetention.infer; + +export const HostedIncarnation = type({ + address: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), + generation: Generation, + state: IncarnationState, + "retention?": DeploymentRetention, + // Why a stopped incarnation stopped when the Hub did not stop it: its + // workflow child ended itself, or the sidecar could not restore it. The Hub + // fails such a deployment. + "error?": type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), +}).narrow( + (incarnation, ctx) => + incarnation.retention === undefined || + incarnation.state === "stopped" || + incarnation.state === "tearing-down" || + ctx.mustBe("stopped before deciding retention"), +); +export type HostedIncarnation = typeof HostedIncarnation.infer; + +// --------------------------------------------------------------------------- +// Sidecar → Hub +// --------------------------------------------------------------------------- + +/** + * The first frame on every connection. Authenticates the sidecar and reports + * every incarnation it holds, whether still deploying, live, stopped, or + * tearing down, so the Hub reconciles against what the sidecar actually runs: + * it routes current live work, keeps current stopped and retained copies + * unrouted, and stops terminal live copies after `welcome`, retaining them + * only while their policy allows it. Release cleanup bindings authenticate + * without routes and leave removal to allocation reconciliation. The Hub + * undeploys other unwanted copies. The sidecar sends nothing that must be + * delivered until `welcome`. + */ +export const HelloFrame = type({ + type: "'hello'", sidecarId: "string", token: "string", - agentAddresses: type("string") - .array() - .atMostLength(MAX_AGENT_ADDRESSES_FRAME), + capacity: SidecarCapacity, + // A sidecar holds one incarnation of an address, so a list that names one + // twice does not say which of them it holds. + incarnations: HostedIncarnation.array() + .atMostLength(MAX_SIDECAR_INCARNATIONS) + .narrow( + (incarnations, ctx) => + new Set(incarnations.map((incarnation) => incarnation.address)).size === + incarnations.length || + ctx.mustBe("a list that names each address once"), + ), // The rotatable (non-run) sender addresses this sidecar holds cached keys - // for; see `RegisterFrame`. Carried on both frames because the register vs - // reconnect choice turns on workflow-address presence, not sender-cache - // presence -- a sidecar that restored no workflow substrate still reports its - // cached senders on a register frame. Additive-optional, omitted when empty. + // for. The hub re-resolves each current key and re-pushes it on a + // `sender.key.refresh`, so a user-principal rotation that landed while the + // sidecar was disconnected reaches its cache. Omitted when empty, and the + // hub treats absence as "nothing to refresh". "cachedSenderAddresses?": type("string") .array() .atMostLength(MAX_CACHED_SENDER_ADDRESSES_FRAME), +}).narrow(({ incarnations, capacity }, ctx) => { + const retained = incarnations.filter( + (incarnation) => incarnation.retention === "kept", + ).length; + return ( + (retained <= capacity.maxRetainedDeployments && + incarnations.length - retained <= capacity.maxActiveDeployments) || + ctx.mustBe("within the advertised active and retained deployment limits") + ); }); -export type ReconnectFrame = typeof ReconnectFrame.infer; +export type HelloFrame = typeof HelloFrame.infer; /** * Acknowledges a successful agent deployment. Includes the agent's Ed25519 * public key (hex-encoded) for published identity and content provenance. - * Reconnect authority comes from the allocation credential. + * Echoes the request id and incarnation of the `agent.deploy` it answers. */ export const AgentDeployAckFrame = type({ type: "'agent.deploy.ack'", + requestId: "string", agentAddress: "string", + generation: Generation, publicKey: "string", }); export type AgentDeployAckFrame = typeof AgentDeployAckFrame.infer; -/** - * Reports a failed agent deployment. - */ -export const AgentErrorFrame = type({ - type: "'agent.error'", +/** Capacity refusal starts nothing; a general failure may leave deployment state. */ +export const DeploymentError = type({ + code: "'capacity_full' | 'deployment_failed'", + message: type("string").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), +}); +export type DeploymentError = typeof DeploymentError.infer; + +export class DeploymentRejectedError extends Error { + constructor( + readonly code: "capacity_full", + message: string, + ) { + super(message); + this.name = "DeploymentRejectedError"; + } +} + +/** Reports that the `agent.deploy` with this request id failed. */ +export const AgentDeployErrorFrame = type({ + type: "'agent.deploy.error'", + requestId: "string", agentAddress: "string", - error: "string", + generation: Generation, + error: DeploymentError, }); -export type AgentErrorFrame = typeof AgentErrorFrame.infer; +export type AgentDeployErrorFrame = typeof AgentDeployErrorFrame.infer; /** * A message from a local agent. When `delivered` is absent or false the hub * should route the message to its recipients. When `delivered` is true the - * message was already delivered locally and is forwarded for audit/projection + * message already went out and this copy is forwarded for audit/projection * only — the hub must not re-route it. * * Structured metadata (senderAddress, messageId, to, cc) is available for * audit and projection purposes without parsing the raw MIME bytes. + * `generation` names the sender's incarnation. */ export const MailOutboundFrame = type({ type: "'mail.outbound'", rawMessage: "string", recipients: type("string").array().atMostLength(MAX_MAIL_ADDRESSES_FRAME), senderAddress: "string", + generation: Generation, "sessionId?": "string", "messageId?": "string", "to?": type("string").array().atMostLength(MAX_MAIL_ADDRESSES_FRAME), @@ -207,6 +332,7 @@ export type MailOutboundFrame = typeof MailOutboundFrame.infer; export const AgentEventFrame = type({ type: "'agent.event'", agentAddress: "string", + generation: Generation, sessionId: "string", event: InferenceEvent, "childRunId?": "string", @@ -259,16 +385,28 @@ export const SessionErrorFrame = type({ export type SessionErrorFrame = typeof SessionErrorFrame.infer; /** - * Acknowledges that an agent has been fully undeployed: the deployment's - * workflow child stopped, state pushed (best-effort), and directory deleted. + * Acknowledges that the sidecar no longer holds the incarnation an + * `agent.undeploy` named and that its teardown, if it ran one, succeeded. + * Echoes the request id and incarnation. */ export const AgentUndeployAckFrame = type({ type: "'agent.undeploy.ack'", + requestId: "string", agentAddress: "string", - statePushed: "boolean", + generation: Generation, }); export type AgentUndeployAckFrame = typeof AgentUndeployAckFrame.infer; +/** Reports that the `agent.undeploy` with this request id failed. */ +export const AgentUndeployErrorFrame = type({ + type: "'agent.undeploy.error'", + requestId: "string", + agentAddress: "string", + generation: Generation, + error: type("string").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), +}); +export type AgentUndeployErrorFrame = typeof AgentUndeployErrorFrame.infer; + /** * Registers a control-signal correlation as a workflow agent step suspends. * The fields on this frame all converge at the sidecar's suspend emit point; @@ -288,6 +426,7 @@ export const SignalCorrelationRegisterFrame = type({ runId: "string", anchorRunId: "string", agentAddress: "string", + generation: Generation, kind: SignalKind, // Approver-facing snapshot of the suspended tool call, size-capped at this // trust boundary. Required: the ask rail is the only producer of this frame @@ -349,6 +488,7 @@ export type SignalCorrelationRegisterAckFrame = export const MailInboundFrame = type({ type: "'mail.inbound'", agentAddress: "string", + generation: Generation, rawMessage: "string", authenticatedSender: "string", "messageId?": "string", @@ -368,6 +508,7 @@ export type MailInboundFrame = typeof MailInboundFrame.infer; export const MailInboundAckFrame = type({ type: "'mail.inbound.ack'", agentAddress: "string", + generation: Generation, messageId: "string", }); export type MailInboundAckFrame = typeof MailInboundAckFrame.infer; @@ -396,6 +537,9 @@ export const SignalDeliverFrame = type({ signalName: "string", signalId: "string", payload: "unknown", + // Addresses one onTrigger relay park. Absent, the delivery stays + // name-FIFO and does not complete a correlated park. + "correlationId?": "string > 0", }); export type SignalDeliverFrame = typeof SignalDeliverFrame.infer; @@ -511,6 +655,7 @@ export type SenderKeyEvictFrame = typeof SenderKeyEvictFrame.infer; export const DrainDeliverFrame = type({ type: "'drain.deliver'", agentAddress: "string", + generation: Generation, deadlineMs: "number", }); export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; @@ -526,42 +671,6 @@ import { export { WorkflowStep } from "./wire-workflow"; export { WorkflowProjectionDefinition, WorkflowProjectionWithSources }; -/** - * The decrypted credential material and per-handle binding descriptors - * delivered to a running agent so its tools can use provider-backed - * credentials. Secrets are decrypted hub-side and ride this payload on the - * live channel ONLY -- the deploy frame at launch, a `credentials.update` - * frame on rotation, and the child's in-memory cell. They are NEVER written to - * disk (they do not ride the git-committed grants file) and NEVER copied into - * any snapshot, event, or state -- redaction is by construction, mirroring how - * an `InferenceSource`'s `apiKey` stays off every egress type. - * - * `materials` is keyed by `credentialId` (a credential can back several handles, - * so its secret is stored once); `bindings` maps each declared tool handle to - * the credential that backs it and the consumer identity allowed to use it. - */ -export const CredentialMaterialEntry = type({ - credentialId: "string", - providerKey: "string", - origin: "string", - secret: "string", -}); -export type CredentialMaterialEntry = typeof CredentialMaterialEntry.infer; - -export const CredentialBindingDescriptor = type({ - handle: "string", - credentialId: "string", - consumer: "string", -}); -export type CredentialBindingDescriptor = - typeof CredentialBindingDescriptor.infer; - -export const CredentialDelivery = type({ - bindings: CredentialBindingDescriptor.array(), - materials: CredentialMaterialEntry.array(), -}); -export type CredentialDelivery = typeof CredentialDelivery.infer; - /** * The source-ref pin: where a code-sourced (npm) workflow definition's bytes * come from (`source`) plus the frozen dependency closure the hub resolved for @@ -666,11 +775,12 @@ export const AgentDeployWorkflow = type({ // re-verified closure. Optional: only a deploy that carries an inline onTrigger // section or childWorkflow child populates it. "referencedDefinitions?": WorkflowProjectionWithSources.array(), - // Initial credential material for the deployment's tools, decrypted hub-side - // and delivered on the deploy frame so it is resident before any step runs - // (closing the race where a tool resolves a credential before a push lands). - // Run-global: a credential's secret is stored once, keyed by credentialId. - // Optional -- a deploy whose definition binds no credentials omits it. + // Initial credential material for the deployment's binding consumers and + // inference sources, decrypted hub-side and delivered on the deploy frame so + // it is resident before any step runs (closing the race where a tool resolves + // a credential before a push lands). Run-global: a credential's secret is + // stored once, keyed by credentialId. Optional -- a deploy whose definition + // binds no credentials omits it. "credentials?": CredentialDelivery, // The source-ref pin (`source` + frozen `closure`) the sidecar re-materializes // and evaluates the pinned code from. Required: source-ref is the only deploy @@ -700,10 +810,16 @@ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; * after every step is provisioned) spawns the child. * A frame carrying neither is rejected -- there is no in-process * fall-through. `workflow` and `provisionStep` are mutually exclusive. + * + * The sidecar answers with `agent.deploy.ack` or `agent.deploy.error` + * carrying the same `requestId`. A deploy of an address the sidecar already + * holds an incarnation of is refused. */ export const AgentDeployFrame = type({ type: "'agent.deploy'", + requestId: "string", agentAddress: "string", + generation: Generation, agentId: "string", config: HarnessConfig, hubPublicKey: "string", @@ -713,17 +829,104 @@ export const AgentDeployFrame = type({ export type AgentDeployFrame = typeof AgentDeployFrame.infer; /** - * Remove an agent from this sidecar. The sidecar shuts the deployment's - * supervisor down, pushes state to the hub (best-effort), deletes the agent - * directory, and responds with agent.undeploy.ack. + * Remove a deployment incarnation from this sidecar. The sidecar shuts the + * deployment's supervisor down, deletes the local state its teardown covers, + * and responds with `agent.undeploy.ack` or `agent.undeploy.error` carrying + * the same `requestId`. Every incarnation of the address up to `generation` + * goes. For one the sidecar does not hold, it still runs that teardown, and + * answers with an error if that fails. */ export const AgentUndeployFrame = type({ type: "'agent.undeploy'", + requestId: "string", agentAddress: "string", + generation: Generation, reason: "string", }); export type AgentUndeployFrame = typeof AgentUndeployFrame.infer; +/** + * Cancel a workflow, stop its process, or transfer a terminal copy into the + * retained pool. Retention reports kept/refused only after its decision is + * durable. The sidecar refuses a command for another generation. + */ +export const WorkflowControlFrame = type({ + type: "'workflow.control'", + requestId: "string", + agentAddress: "string", + generation: Generation, + runId: "string", + action: "'cancel' | 'stop' | 'retain'", + reason: "string", +}); +export type WorkflowControlFrame = typeof WorkflowControlFrame.infer; + +export const WORKFLOW_CONTROL_INITIALIZING_ERROR = "workflow_initializing"; + +/** The tip of each authoritative workflow-run ref, `null` for an absent ref. */ +export const WorkflowRunRefTips = type({ + "[string]": type("string").atMostLength(MAX_REF_TIP_ENTRY_LENGTH).or("null"), +}).narrow( + (tips, ctx) => + (Object.keys(tips).length <= MAX_REF_TIPS_FRAME && + Object.keys(tips).every( + (ref) => ref.length <= MAX_REF_TIP_ENTRY_LENGTH, + )) || + ctx.mustBe( + `at most ${String(MAX_REF_TIPS_FRAME)} refs whose names have at most ${String(MAX_REF_TIP_ENTRY_LENGTH)} characters`, + ), +); +export type WorkflowRunRefTips = typeof WorkflowRunRefTips.infer; + +export const WorkflowControlAckFrame = type({ + type: "'workflow.control.ack'", + requestId: "string", + "error?": "string", + "retention?": DeploymentRetention, + /** + * A stopped worker's ref tips, read after its child exited. The Hub + * confirms a live stop only once it holds the same tips. Retain follows + * the existing terminal-history cutoff instead of importing later commits. + */ + "refTips?": WorkflowRunRefTips, +}); +export type WorkflowControlAckFrame = typeof WorkflowControlAckFrame.infer; + +/** + * Reports a deployment that stopped though the Hub did not stop it: its + * workflow child ended itself, or the sidecar could not restore it. Sent when + * it happens and again after every `welcome` while the sidecar holds it, with + * the tips of its history branches, so the Hub can wait for that history + * before it fails the deployment. The tips are left out when the sidecar + * could not read them. + */ +export const DeploymentStoppedFrame = type({ + type: "'deployment.stopped'", + agentAddress: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), + generation: Generation, + error: type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), + "refTips?": WorkflowRunRefTips, +}); +export type DeploymentStoppedFrame = typeof DeploymentStoppedFrame.infer; + +/** + * The Hub's answer to `hello`, sent once it has reconciled the reported + * incarnations. `routed` lists the incarnations it routes on this connection; + * it has already asked the sidecar to undeploy every other one reported, but + * for the ones it keeps unrouted: stopped ones, live ones whose run ended + * (including Hub cancellations or failures), and copies under a cleanup + * binding whose removal belongs to the allocation reconciler. From here on + * the sidecar may send what must be delivered, and it re-drives what it owes + * the Hub for each routed incarnation. + */ +export const WelcomeFrame = type({ + type: "'welcome'", + routed: type({ address: "string", generation: Generation }) + .array() + .atMostLength(MAX_SIDECAR_INCARNATIONS), +}); +export type WelcomeFrame = typeof WelcomeFrame.infer; + /** * Keepalive pong sent by the hub in response to a ping frame. * If the sidecar stops receiving pongs, it considers the hub dead. @@ -744,6 +947,7 @@ export const SourcesUpdateFrame = type({ type: "'sources.update'", requestId: "string", agentAddress: "string", + generation: Generation, sources: InferenceSource.array().atLeastLength(1), defaultSource: "string", }); @@ -764,6 +968,7 @@ export const CredentialsUpdateFrame = type({ type: "'credentials.update'", requestId: "string", agentAddress: "string", + generation: Generation, delivery: CredentialDelivery, "revoke?": type("string") .array() @@ -796,50 +1001,15 @@ export type CredentialsUpdateFrame = typeof CredentialsUpdateFrame.infer; // `repoId` to name a non-agent source while `agentAddress` continues // to address the destination agent. // +// `repo.pack.push` and `repo.pack.done` also carry the generation of the +// incarnation the transfer belongs to. The ack and reject answer one transfer +// on the connection that carried it, and transfer ids are never reused, so +// they need no generation. +// // Flow control: deferred. Agent deploy trees are small enough that the sender // can push all chunks without windowing. If this becomes a problem, a credit- // based mechanism can be added later. -/** - * Tag identifying a kind of repository in the hub's kind-keyed RepoStore. - * Lives in `@intx/types` because the wire-level pack frames reference it; - * the substrate package re-exports it for handler authors. - */ -export const RepoKind = type.enumerated( - "agent-state", - "skill", - "package-registry", - "workflow", - "workflow-run", -); -export type RepoKind = typeof RepoKind.infer; - -/** - * Operations a principal may invoke against a repo in the RepoStore. - * Lives in `@intx/types` so storage layers (e.g. `@intx/db`) can validate - * persisted action vocabularies without depending on the substrate - * package. The substrate re-exports it for handler authors. - */ -export const RepoAction = type.enumerated( - "init", - "writeTree", - "receivePack", - "createPack", - "resolveRef", -); -export type RepoAction = typeof RepoAction.infer; - -/** - * Hub-side identity of a repository in the RepoStore. Pack frames carry - * this alongside `agentAddress` so the hub can map a pack back to the - * originating repo independently of which sidecar/agent it is destined for. - */ -export const RepoId = type({ - kind: RepoKind, - id: "string", -}); -export type RepoId = typeof RepoId.infer; - /** * A chunk of git pack data. The sender splits the packfile into chunks of at * most 64 KiB (before base64 encoding) and sends them in order. @@ -850,6 +1020,7 @@ export type RepoId = typeof RepoId.infer; export const PackPushFrame = type({ type: "'repo.pack.push'", agentAddress: "string", + generation: Generation, repoId: RepoId, transferId: "string", seq: "number", @@ -870,6 +1041,7 @@ export type PackPushFrame = typeof PackPushFrame.infer; export const PackDoneFrame = type({ type: "'repo.pack.done'", agentAddress: "string", + generation: Generation, repoId: RepoId, transferId: "string", ref: "string", @@ -1032,18 +1204,6 @@ export const DeployApplyErrorCategory = type.enumerated( ); export type DeployApplyErrorCategory = typeof DeployApplyErrorCategory.infer; -/** - * Hub requests the sidecar to push its current agent state. The sidecar - * responds by sending pack.push frames followed by pack.done using the - * same transferId. - */ -export const SyncRequestFrame = type({ - type: "'sync.request'", - agentAddress: "string", - transferId: "string", -}); -export type SyncRequestFrame = typeof SyncRequestFrame.infer; - // --------------------------------------------------------------------------- // Workflow probe (bidirectional) // --------------------------------------------------------------------------- @@ -1132,80 +1292,15 @@ export const WorkflowProbeErrorFrame = type({ }); export type WorkflowProbeErrorFrame = typeof WorkflowProbeErrorFrame.infer; -// --------------------------------------------------------------------------- -// Sidecar-hosted OAuth loopback login (CL-7508) -// --------------------------------------------------------------------------- - -/** Tokens a sidecar-hosted loopback login staged. The PKCE verifier never - * crosses the wire — it lives only in the sidecar's login service and dies - * with the callback server. */ -export const OAuthLoginTokens = type({ - access: "string", - "refresh?": "string", - /** Epoch ms the access token expires; absent when the issuer stated no - * lifetime (stored non-due, never a short artificial timer). */ - "expiresAt?": "number", - /** The issuer's id_token when it issues one (xai-oauth retains it). */ - "idToken?": "string", - /** id_token-derived account label (codex: `chatgpt_account_id`), the - * value `accountIdFromIdToken` decodes; threaded into credential - * metadata hub-side. */ - "accountId?": "string", -}); -export type OAuthLoginTokens = typeof OAuthLoginTokens.infer; - -/** Outcome arms a `oauth.login.result` frame may carry. A login sends - * `started` once its callback server is bound and the authorize URL is - * ready for the web UI to navigate; exactly one terminal arm (`completed` - * or `error`) follows. */ -export const OAuthLoginOutcome = type({ - status: "'started'", - authorizeUrl: "string", -}) - .or({ status: "'completed'", tokens: OAuthLoginTokens }) - .or({ status: "'error'", message: "string" }); -export type OAuthLoginOutcome = typeof OAuthLoginOutcome.infer; - -/** Hub → sidecar: run the named connector's loopback PKCE login on the - * machine this sidecar runs on. The connector must pin a fixed loopback - * redirect (`codex` → localhost:1455, `xai-oauth` → 127.0.0.1:1456); the - * hub never hosts a listener for these flows. */ -export const OAuthLoginStartFrame = type({ - type: "'oauth.login.start'", - requestId: "string", - connectorId: "'codex' | 'xai-oauth'", -}); -export type OAuthLoginStartFrame = typeof OAuthLoginStartFrame.infer; - -/** Sidecar → hub: the staged progress / terminal outcome of the login the - * hub requested with the same `requestId`. */ -export const OAuthLoginResultFrame = type({ - type: "'oauth.login.result'", - requestId: "string", - outcome: OAuthLoginOutcome, -}); -export type OAuthLoginResultFrame = typeof OAuthLoginResultFrame.infer; - -/** Hub → sidecar: tear the staged login with this `requestId` down and - * close its pinned-port callback listener. The hub sends it when it gives - * up on a login (whole-login timeout) so an abandoned bind does not wedge - * every retry of the connector until the sidecar restarts. */ -export const OAuthLoginCancelFrame = type({ - type: "'oauth.login.cancel'", - requestId: "string", -}); -export type OAuthLoginCancelFrame = typeof OAuthLoginCancelFrame.infer; - // --------------------------------------------------------------------------- // Discriminated frame unions // --------------------------------------------------------------------------- /** All frame types the sidecar sends to the hub. */ export const SidecarFrame = type.or( - RegisterFrame, - ReconnectFrame, + HelloFrame, AgentDeployAckFrame, - AgentErrorFrame, + AgentDeployErrorFrame, MailOutboundFrame, AgentEventFrame, ConnectorStateChangedFrame, @@ -1213,6 +1308,7 @@ export const SidecarFrame = type.or( SessionAckFrame, SessionErrorFrame, AgentUndeployAckFrame, + AgentUndeployErrorFrame, SignalCorrelationRegisterFrame, PackPushFrame, PackDoneFrame, @@ -1221,7 +1317,8 @@ export const SidecarFrame = type.or( MailInboundAckFrame, WorkflowProbeResultFrame, WorkflowProbeErrorFrame, - OAuthLoginResultFrame, + WorkflowControlAckFrame, + DeploymentStoppedFrame, ); export type SidecarFrame = typeof SidecarFrame.infer; @@ -1230,6 +1327,7 @@ export const HubFrame = type.or( MailInboundFrame, AgentDeployFrame, AgentUndeployFrame, + WelcomeFrame, PongFrame, SourcesUpdateFrame, CredentialsUpdateFrame, @@ -1237,16 +1335,14 @@ export const HubFrame = type.or( PackDoneFrame, PackAckFrame, PackRejectFrame, - SyncRequestFrame, - SignalDeliverFrame, - RunGrantsFrame, + SignalDeliverFrame.and(IncarnationStamp), + RunGrantsFrame.and(IncarnationStamp), SenderKeyRefreshFrame, SenderKeyEvictFrame, SignalCorrelationRegisterAckFrame, DrainDeliverFrame, WorkflowProbeRequestFrame, - OAuthLoginStartFrame, - OAuthLoginCancelFrame, + WorkflowControlFrame, ); export type HubFrame = typeof HubFrame.infer; diff --git a/vendor/intx/types/src/tenants.ts b/vendor/intx/types/src/tenants.ts index 033e266c3f..da91f54873 100644 --- a/vendor/intx/types/src/tenants.ts +++ b/vendor/intx/types/src/tenants.ts @@ -1,8 +1,10 @@ import { type } from "arktype"; import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; +import { WorkflowLifecyclePolicy } from "./workflow-lifecycle"; export const TenantConfig = type({ + "lifecycle?": WorkflowLifecyclePolicy, "sidecarPlacement?": SidecarCapabilityPolicy, "[string]": "unknown", }); @@ -22,6 +24,14 @@ export const TenantSlug = type( ).and("string<=63"); export type TenantSlug = typeof TenantSlug.infer; +/** Top-level `config` keys to change; `null` removes a key. */ +export const TenantConfigPatch = type({ + "lifecycle?": WorkflowLifecyclePolicy.or("null"), + "sidecarPlacement?": SidecarCapabilityPolicy.or("null"), + "[string]": "unknown", +}); +export type TenantConfigPatch = typeof TenantConfigPatch.infer; + export const CreateTenant = type({ name: "string", slug: TenantSlug, @@ -30,7 +40,7 @@ export const CreateTenant = type({ export const UpdateTenant = type({ "name?": "string", - "config?": TenantConfig, + "config?": TenantConfigPatch, }); export const TenantResponse = type({ diff --git a/vendor/intx/types/src/testing/index.ts b/vendor/intx/types/src/testing/index.ts new file mode 100644 index 0000000000..d2508f5473 --- /dev/null +++ b/vendor/intx/types/src/testing/index.ts @@ -0,0 +1,7 @@ +// @intx/types/testing -- waiting helpers for tests. +// +// These exist so a test can wait for a condition without choosing an interval +// to wait for it in. Nothing here belongs in production: a production caller +// that needs to know when something happened should be given the event, not a +// loop that re-reads until it sees it. +export { waitUntil } from "./wait-until"; diff --git a/vendor/intx/types/src/testing/wait-until.ts b/vendor/intx/types/src/testing/wait-until.ts new file mode 100644 index 0000000000..514fc7480d --- /dev/null +++ b/vendor/intx/types/src/testing/wait-until.ts @@ -0,0 +1,30 @@ +// Waiting for a condition without betting on how long it takes. +// +// The shape this replaces is `await new Promise((r) => setTimeout(r, 50))` +// followed by an assertion: a duration picked to be longer than the work, +// which on a loaded machine is not, so the assertion reads a state that had +// not arrived yet. The interval decides whether the run passes. +// +// `waitUntil` removes the decision. It re-checks after yielding the event +// loop and carries no deadline, so a slow machine makes it take longer and +// never makes it fail; a condition that never holds is caught by the lane +// timeout, which is where CONVENTIONS.md puts the failsafe for a hang. +// +// It is second-best on purpose. CONVENTIONS.md asks for the SIGNAL -- the +// emitted event, the reported mutation -- and where a double can report what +// it did, that is better than this: it wakes exactly once, on the thing the +// test is actually waiting for, instead of spinning. Prefer a report. Reach +// for this where the state under test has no reporter and giving it one would +// mean reshaping production code for a test's convenience. + +/** + * Resolve once `predicate` returns true, re-checking after each event-loop + * turn. No deadline: see the module comment. + */ +export async function waitUntil(predicate: () => boolean): Promise { + while (!predicate()) { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + } +} diff --git a/vendor/intx/types/src/wire-workflow.ts b/vendor/intx/types/src/wire-workflow.ts index 015cca33fe..81bcbf2aaf 100644 --- a/vendor/intx/types/src/wire-workflow.ts +++ b/vendor/intx/types/src/wire-workflow.ts @@ -7,7 +7,8 @@ import { type } from "arktype"; import { CredentialBinding } from "./credentials"; -import { InboundMailPolicy, InferenceSource } from "./runtime"; +import { InferenceSource } from "./runtime"; +import { InboundMailPolicy } from "./inbound-mail-policy"; import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; /** @@ -78,7 +79,7 @@ const WorkflowSteps = type({ "[string]": "unknown" }).narrow((steps, ctx) => { * `definition` is the wire projection of `WorkflowDefinition` from * `@intx/workflow`. The arktype validator enforces the structural * envelope the workflow-process child re-parses on the sidecar after - * materialization (`packages/hub-sessions/src/workflow-kind.ts`'s + * materialization (`packages/types/src/workflow-definition.ts`'s * `workflowDefinitionEnvelopeSchema`): `id`, `triggers`, `steps`, * `stepOrder`, optional `state`. The wire validator MUST require every * field the envelope requires — this projection is the approved surface diff --git a/vendor/intx/types/src/workflow-definition.ts b/vendor/intx/types/src/workflow-definition.ts new file mode 100644 index 0000000000..a51fa8d6fd --- /dev/null +++ b/vendor/intx/types/src/workflow-definition.ts @@ -0,0 +1,62 @@ +import { type } from "arktype"; + +import { CredentialBinding } from "./credentials"; +import { GrantRequirement } from "./grants"; +import { InboundMailPolicy } from "./inbound-mail-policy"; +import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; + +/** + * Structural arktype validator for the `workflow.json` envelope. The + * substrate checks the cross-cutting shape of `WorkflowDefinition` + * (presence and primitive type of `id`, `triggers`, `steps`, + * `stepOrder`) but does not re-derive `defineWorkflow`'s DAG-level + * validation here — primitive-level shape, default-input application, + * and `after`-ref resolution belong to the runtime layer that hydrates + * the definition. The codebase push uses this validator to detect an + * ambiguous tree that also carries an envelope-valid `workflow.json`, + * and the hydrate-time definition loaders reuse it to validate a + * materialized definition before instantiation. + */ +const StepsObject = type("Record").narrow((value, ctx) => { + if (Array.isArray(value)) { + return ctx.mustBe("a JSON object, not an array"); + } + return true; +}); + +const StateObject = type("Record").narrow((value, ctx) => { + if (Array.isArray(value)) { + return ctx.mustBe("a JSON object, not an array"); + } + return true; +}); + +export const workflowDefinitionEnvelopeSchema = type({ + id: "string > 0", + triggers: "unknown[]", + steps: StepsObject, + stepOrder: "string[]", + "state?": StateObject, + // `grantRequirements` passes through the envelope whether or not it is + // declared here: arktype's `.onUndeclaredKey("ignore")` below is + // passthrough, not stripping (only `"delete"` strips), so the hydrate read + // sees the field either way. Declaring it here VALIDATES declared + // requirements at the deploy boundary — a malformed `source` is rejected + // rather than passed through unchecked — as defense in depth alongside the + // trigger route's own `GrantRequirements` re-validation. Compose the + // exported `GrantRequirement` arktype rather than restating its shape so the + // envelope and the definition stay in lockstep. + "grantRequirements?": GrantRequirement.array(), + // `credentialBindings` is validated here too -- same defense-in-depth + // rationale as grantRequirements above: a malformed binding (bad locator, + // authority, or handle) is rejected at the deploy boundary rather than + // passed through to launch-time resolution unchecked. + "credentialBindings?": CredentialBinding.array(), + "sidecarPlacement?": SidecarCapabilityPolicy, + // `inboundMailPolicy` is validated here too -- same defense-in-depth + // rationale as credentialBindings above: a malformed policy (an unknown + // outcome key or a value that is not reject/admit) is rejected at the deploy + // boundary rather than passed through to later admission resolution + // unchecked. + "inboundMailPolicy?": InboundMailPolicy, +}).onUndeclaredKey("ignore"); diff --git a/vendor/intx/types/src/workflow-lifecycle.ts b/vendor/intx/types/src/workflow-lifecycle.ts new file mode 100644 index 0000000000..49319f70a9 --- /dev/null +++ b/vendor/intx/types/src/workflow-lifecycle.ts @@ -0,0 +1,174 @@ +import { type } from "arktype"; +import { sidecarAllocationStatuses } from "./sidecar-allocation"; + +// Fixed policy range with ample headroom for deployment and terminal timestamps. +const MAX_LIFECYCLE_DURATION_MS = 36_500 * 86_400_000; + +function parseDuration(value: string): number | null { + const match = /^(0|[1-9][0-9]*)(s|m|h|d)$/.exec(value); + if (match === null) return null; + const amount = Number(match[1]); + const unit = match[2]; + const multiplier = + unit === "d" + ? 86_400_000 + : unit === "h" + ? 3_600_000 + : unit === "m" + ? 60_000 + : 1_000; + const milliseconds = amount * multiplier; + return Number.isSafeInteger(milliseconds) && + milliseconds <= MAX_LIFECYCLE_DURATION_MS + ? milliseconds + : null; +} + +export const LifecycleDuration = type("string").narrow( + (value, ctx) => + parseDuration(value) !== null || + ctx.mustBe( + "a non-negative whole duration ending in s, m, h, or d, at most 36500d", + ), +); +export type LifecycleDuration = typeof LifecycleDuration.infer; + +export function lifecycleDurationMs(value: LifecycleDuration): number { + const milliseconds = parseDuration(value); + if (milliseconds === null) + throw new Error(`Invalid lifecycle duration: ${value}`); + return milliseconds; +} + +export function lifecycleDeadline( + start: Date, + duration: LifecycleDuration, +): Date { + const deadline = new Date(start.getTime() + lifecycleDurationMs(duration)); + if (!Number.isFinite(deadline.getTime())) { + throw new Error( + "Lifecycle deadline is outside the supported timestamp range", + ); + } + return deadline; +} + +const PositiveLifecycleDuration = LifecycleDuration.narrow( + (value, ctx) => + lifecycleDurationMs(value) > 0 || ctx.mustBe("greater than zero"), +); + +export const WorkflowLifecyclePolicy = type({ + "maxLifetime?": PositiveLifecycleDuration, + "maxDisconnected?": PositiveLifecycleDuration, + "capacityRetention?": type({ + "completed?": LifecycleDuration, + "failed?": LifecycleDuration, + "cancelled?": LifecycleDuration, + }).onUndeclaredKey("reject"), +}).onUndeclaredKey("reject"); +export type WorkflowLifecyclePolicy = typeof WorkflowLifecyclePolicy.infer; + +/** A policy with every field set, as a deployment's effective policy is. */ +export type ResolvedWorkflowLifecyclePolicy = { + maxLifetime: LifecycleDuration; + maxDisconnected: LifecycleDuration; + capacityRetention: Required< + NonNullable + >; +}; + +export const WorkflowLifecycleResponse = type({ + runId: "string", + status: "'deployed' | 'running' | 'completed' | 'failed' | 'cancelled'", + policy: WorkflowLifecyclePolicy, + expiresAt: "string | null", + cancellationRequestedAt: "string | null", + cancellationDeadline: "string | null", + cancellationReason: "string | null", + capacityReleaseAt: "string | null", + allocation: type({ + id: "string", + generation: "number", + status: type.enumerated(...sidecarAllocationStatuses), + failureCode: "string | null", + failureMessage: "string | null", + }).or("null"), +}); + +// Names the allocation generation the caller inspected, so a request based on +// an outdated status cannot release a later binding. +export const RecoverWorkflowCapacityRequest = type({ + generation: "number.integer >= 0", +}); + +export type WorkflowLifecyclePolicyResolution = + | { ok: true; policy: WorkflowLifecyclePolicy } + | { ok: false; field: string; requested: string; limit: string }; + +type LifecycleLimitConflict = Extract< + WorkflowLifecyclePolicyResolution, + { ok: false } +>; + +function foldWorkflowLifecyclePolicies( + policies: readonly WorkflowLifecyclePolicy[], +): { policy: WorkflowLifecyclePolicy; conflict?: LifecycleLimitConflict } { + const effective: WorkflowLifecyclePolicy = {}; + let conflict: LifecycleLimitConflict | undefined; + for (const policy of policies) { + for (const field of ["maxLifetime", "maxDisconnected"] as const) { + const requested = policy[field]; + if (requested === undefined) continue; + const limit = effective[field]; + if ( + limit !== undefined && + lifecycleDurationMs(requested) > lifecycleDurationMs(limit) + ) { + conflict ??= { ok: false, field, requested, limit }; + continue; + } + effective[field] = requested; + } + for (const status of ["completed", "failed", "cancelled"] as const) { + const requested = policy.capacityRetention?.[status]; + if (requested === undefined) continue; + const limit = effective.capacityRetention?.[status]; + if ( + limit !== undefined && + lifecycleDurationMs(requested) > lifecycleDurationMs(limit) + ) { + conflict ??= { + ok: false, + field: `capacityRetention.${status}`, + requested, + limit, + }; + continue; + } + effective.capacityRetention ??= {}; + effective.capacityRetention[status] = requested; + } + } + return conflict === undefined + ? { policy: effective } + : { policy: effective, conflict }; +} + +/** Resolve root-to-leaf policies; a descendant can only shorten an inherited limit. */ +export function resolveWorkflowLifecyclePolicy( + policies: readonly WorkflowLifecyclePolicy[], +): WorkflowLifecyclePolicyResolution { + const { policy, conflict } = foldWorkflowLifecyclePolicies(policies); + return conflict ?? { ok: true, policy }; +} + +/** + * Resolve root-to-leaf policies, capping any value above an inherited limit. + * An ancestor tightened after a descendant was saved must not strand it. + */ +export function clampWorkflowLifecyclePolicy( + policies: readonly WorkflowLifecyclePolicy[], +): WorkflowLifecyclePolicy { + return foldWorkflowLifecyclePolicies(policies).policy; +} diff --git a/vendor/intx/types/src/workflow-sources.ts b/vendor/intx/types/src/workflow-sources.ts index 4a93137874..484648f880 100644 --- a/vendor/intx/types/src/workflow-sources.ts +++ b/vendor/intx/types/src/workflow-sources.ts @@ -72,3 +72,13 @@ export const WorkflowDefinitionSource = WorkflowDefinitionRegistrySource.or( WorkflowDefinitionAssetSource, ); export type WorkflowDefinitionSource = typeof WorkflowDefinitionSource.infer; + +/** + * The workspace-relative mount path a workflow definition's source asset is + * delivered under, keyed by asset id. Both the probe (inline delivery) and the + * deploy (streamed fan-out) build their `assetId -> mountPath` maps through + * this one helper, so the same asset lands at the same path on both paths. + */ +export function workflowSourceAssetMountPath(assetId: string): string { + return `source-assets/${assetId}/`; +} diff --git a/vendor/intx/types/src/workflows.ts b/vendor/intx/types/src/workflows.ts index 014ed7d668..b5614c94b7 100644 --- a/vendor/intx/types/src/workflows.ts +++ b/vendor/intx/types/src/workflows.ts @@ -13,6 +13,7 @@ export type WorkflowDefinitionVersionStatus = (typeof workflowDefinitionVersionStatuses)[number]; import { type } from "arktype"; +import { WorkflowLifecyclePolicy } from "./workflow-lifecycle"; const WorkflowDefinitionStatusType = type.enumerated( ...workflowDefinitionStatuses, @@ -35,6 +36,7 @@ export const WorkflowDefinitionResponse = type({ name: "string", "description?": "string | null", currentVersion: "string", + "lifecycle?": WorkflowLifecyclePolicy, status: WorkflowDefinitionStatusType.describe( "Lifecycle state of the definition: `deployed` (a launchable version is active) or `stopped` (deactivated).", ), @@ -47,6 +49,10 @@ export const WorkflowRollbackRequest = type({ version: "string", }); +export const UpdateWorkflowDefinitionLifecycle = type({ + lifecycle: WorkflowLifecyclePolicy, +}).onUndeclaredKey("reject"); + export const WorkflowDeploymentStatus = type.enumerated( "deployed", "pending", @@ -63,7 +69,7 @@ export const WorkflowDeploymentResponse = type({ tenantId: "string", definitionAssetId: "string", status: WorkflowDeploymentStatus.describe( - "Deployment lifecycle status. `failed` is a terminal failure with no infrastructure. `destroy_failed` is a permanent cleanup failure where infrastructure may remain and require operator cleanup.", + "Deployment lifecycle status. `recovering` means the Hub is replacing deployment capacity. `failed` is a terminal allocation failure with no infrastructure. `destroy_failed` means automatic cleanup stopped. Read `GET /api/tenants/:tenantId/workflows/runs/:runId/lifecycle` for the allocation's failure code and message.", ), createdAt: "string", }); diff --git a/vendor/intx/workflow-deploy/README.md b/vendor/intx/workflow-deploy/README.md index 081a11177f..871a29e24e 100644 --- a/vendor/intx/workflow-deploy/README.md +++ b/vendor/intx/workflow-deploy/README.md @@ -1,5 +1,13 @@ # @intx/workflow-deploy +Use `@intx/workflow-deploy/capabilities` to import `walkCapabilities` and +its grant projections without loading deployment orchestration or package +materialization. + +Use `@intx/workflow-deploy/addresses` for run and step address derivation, +and `@intx/workflow-deploy/child-grants` for collecting a child workflow's +declared resources and credential consumers and filtering its inherited grants. + Deploy-time validation, capability walk, operator-approval gating, address derivation, and per-step source pinning for the code-sourced deploy. @@ -25,6 +33,22 @@ Public surface: - `walkCapabilities(workflow, registry, pluginDefs)` — the pure capability walk; reused to populate per-step capability declarations and as the input to the approval gate. +- `runtimeGrantRules(snapshot)` / `credentialUseGrantRules(bindings)`, from + `@intx/workflow-deploy/capabilities` only — a run's runtime + `tool:`/`effect:` rules and its `credential:` use rules + (`RuntimeGrantRule`, without storage identity). The hub stores them as the + run principal's grants; a host with no hub builds the same rules. +- `collectDeclaredResources(definition, directors, pluginDefs)` / + `collectDeclaredCredentialConsumers(definition, directors, pluginDefs)` / + `filterGrantsToDeclaredResources(parentGrants, declared, credentialConsumers)` + — cap a spawned child's inherited grants at what the child body + declares. The resource collector walks declared grant strings. The + consumer collector returns `toolConsumer(factory.id)` for each factory + the pre-rewrite body instantiates and `actionConsumer(handler)` for each + action it runs that declares credentials. The filter removes a + non-credential `allow` only when its pattern covers none of the + declared set, removes a `credential:` `allow` unless `conditions.tool` + equals one of those consumers, and keeps every `deny` and `ask`. - `createApprovalSet(grants, requirements)` — the edge of the approval vocabulary. An `ApprovalSet` is one approved surface behind one operator decision, carrying the two kinds of approved item in two fields: @@ -84,6 +108,13 @@ Public surface: projection so the hub can stage it and pin its per-step sources. The enumeration is purely structural; each body step's `(provider, model)` preference is read at pin time by `buildInertBodyStepSources`. +- `checkCredentialBindings(projection)` / + `describeCredentialBindingViolation(violation)` — every credential binding + in a frozen inert projection that cannot be delivered: a declared action + handle with no root binding, `credentialBindings` on a nested body, a + binding whose tool factory or action handler the workflow does not run, a + binding for a handle the handler does not declare, and a duplicate binding. + The hub's probe gate refuses a definition with any of these. - `inertNestedBodies(step, descent)` — the nested body projections one step of a frozen inert projection carries, filtered by a `StepWalkDescent`. This is the inert counterpart of `@intx/workflow`'s live `nestedWorkflowBodies`, so diff --git a/vendor/intx/workflow-deploy/VENDORED-FROM b/vendor/intx/workflow-deploy/VENDORED-FROM index ca431d962a..0efd2c2078 100644 --- a/vendor/intx/workflow-deploy/VENDORED-FROM +++ b/vendor/intx/workflow-deploy/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow-deploy) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/workflow-deploy/package.json b/vendor/intx/workflow-deploy/package.json index bdf98db0fe..3bac932f17 100644 --- a/vendor/intx/workflow-deploy/package.json +++ b/vendor/intx/workflow-deploy/package.json @@ -1,7 +1,7 @@ { "name": "@intx/workflow-deploy", "description": "Deploy-time validation, capability walk, and operator-approval gating for workflows", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -12,6 +12,18 @@ "./testing": { "types": "./src/testing/index.ts", "default": "./src/testing/index.ts" + }, + "./capabilities": { + "types": "./src/capability-walk.ts", + "default": "./src/capability-walk.ts" + }, + "./addresses": { + "types": "./src/addresses.ts", + "default": "./src/addresses.ts" + }, + "./child-grants": { + "types": "./src/child-grant-filter.ts", + "default": "./src/child-grant-filter.ts" } }, "scripts": { @@ -19,7 +31,7 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/tool-packaging": "0.4.0", + "@intx/authz": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*" }, diff --git a/vendor/intx/workflow-deploy/src/addresses.ts b/vendor/intx/workflow-deploy/src/addresses.ts new file mode 100644 index 0000000000..cd4dc99cc2 --- /dev/null +++ b/vendor/intx/workflow-deploy/src/addresses.ts @@ -0,0 +1,122 @@ +// The deployment address model is a pure function of `(runId, stepId, +// domain)`: a one-step workflow has no distinct step address -- the lone +// step IS the deployment head (`deriveRunAddress`) -- while a workflow with +// more than one step derives per-step addresses of the form +// `-@`. Because the derivation is pure, the +// supervisor reconstructs the same addresses at spawn time without any +// per-deploy state, and `resolveStepAddress` is the single owner of the +// head/step collapse decision for a consumer that must choose an address +// from the host-sourced step count alone. + +import { formatRunAddress } from "@intx/types/agent-address"; + +/** + * Pure function: derive a step's run address from + * `(runId, stepId, domain)`. Exported so the supervisor can reconstruct + * the same addresses at spawn time without sharing storage with the + * deploy flow. + * + * The local part IS the run id with the step suffix appended; the runId is + * already a minted `run_` carrying the `run_` marker `parseRunAddress` + * requires at the substrate boundary. The per-step local-part is concat-only + * because `stepId` is already constrained to `[a-zA-Z0-9_-]+` by the workflow + * definition validator. + */ +export function deriveStepAddress(args: { + runId: string; + stepId: string; + domain: string; +}): string { + return formatRunAddress(`${args.runId}-${args.stepId}`, args.domain); +} + +/** + * Derive the per-step agent id (the `agent-state` repo's id and the + * `HarnessConfig.agentId`). Pure function of `(runId, stepId)`. + */ +export function deriveStepAgentId(args: { + runId: string; + stepId: string; +}): string { + return `${args.runId}-${args.stepId}`; +} + +/** + * Derive the deployment-level mail address the supervisor registers on + * the bus. It is the run id `@` the domain; pure function of `(runId, domain)`. + * + * The supervisor uses this address as the inbound mail address for the + * deployment as a whole; per-step bindings carry their own + * derived-step addresses. + */ +export function deriveRunAddress(args: { + runId: string; + domain: string; +}): string { + return formatRunAddress(args.runId, args.domain); +} + +/** + * Resolve where a step's deploy tree lives, given the deployment's step + * count. This is the single owner of the head/step collapse DECISION for + * a consumer that must choose the address without knowing the deploy + * shape: a one-step workflow has no distinct steps, so its lone step IS + * the head (`deriveRunAddress`); a multi-step deployment keeps the + * head distinct from its per-step addresses (`deriveStepAddress`). The + * sidecar child reads its deploy tree from the address this returns, + * keyed only off the deployment mailbox and the host-sourced `stepCount`. + * + * The producers do not route through here -- each handles one shape + * unconditionally: the single-step deploy stages the tree at the head, + * the multi-step deploy at each per-step address. Because `stepCount` is + * the deployed definition's `stepOrder.length`, sourced from the host, + * the consumer's collapse always agrees with whichever producer staged + * the tree; the two processes never derive divergent addresses. + */ +export function resolveStepAddress(args: { + runId: string; + stepId: string; + domain: string; + stepCount: number; +}): string { + return args.stepCount === 1 + ? deriveRunAddress({ + runId: args.runId, + domain: args.domain, + }) + : deriveStepAddress(args); +} + +/** + * Derive the deployment-level agent id used on the `agent.deploy` + * frame's `agentId` field. Pure function of `(runId)`. + */ +export function deriveRunAgentId(args: { runId: string }): string { + return args.runId; +} + +/** + * Project a workflow-deployment run address into the substrate-safe + * id of its workflow-run repo (`{ kind: "workflow-run", id }`). Pure + * function of the deployment's run address. + * + * The workflow-run repo's `repoId.id` must match `SAFE_REPO_ID` + * (`/^[a-zA-Z0-9_-]+$/`, the substrate's repo-path-safety contract in + * `packages/hub-sessions/src/repo-store/types.ts`), and the supervisor + * principal's `runId` must equal `workflowRunRepoId.id` for the + * workflow-run kind handler's authz check to pass. That regex rejects + * `@` and `.`, both of which appear in every run address, so the + * address is sanitized by substituting every disallowed character with + * `-`. + * + * The mapping is lossy (two distinct addresses can collapse to the same + * slug) but deterministic. The sidecar's deploy router keys the + * workflow-run repo by this slug at write time; the hub's read routes + * reconstruct the deployment address via `deriveRunAddress` and + * apply this same derivation so read and write address the same repo. + * A collision implies two deployments are claiming the same workflow-run + * surface, which the sidecar's deploy router rejects at deploy time. + */ +export function deriveWorkflowRunRepoId(agentAddress: string): string { + return agentAddress.replaceAll(/[^a-zA-Z0-9_-]/g, "-"); +} diff --git a/vendor/intx/workflow-deploy/src/capability-approval.ts b/vendor/intx/workflow-deploy/src/capability-approval.ts index 374a8d9c6e..6b7dc90a67 100644 --- a/vendor/intx/workflow-deploy/src/capability-approval.ts +++ b/vendor/intx/workflow-deploy/src/capability-approval.ts @@ -15,7 +15,7 @@ import { isDeepStrictEqual } from "node:util"; -import { type ApprovalItem, GrantRequirement } from "@intx/types"; +import { type ApprovalItem, GrantRequirement } from "@intx/types/grants"; import type { CapabilityWalkResult } from "./capability-walk"; diff --git a/vendor/intx/workflow-deploy/src/capability-walk.ts b/vendor/intx/workflow-deploy/src/capability-walk.ts index ecbf768852..aec8d5c9b1 100644 --- a/vendor/intx/workflow-deploy/src/capability-walk.ts +++ b/vendor/intx/workflow-deploy/src/capability-walk.ts @@ -60,8 +60,9 @@ import { effectiveDirectorRef, toolApprovalEffect, UnknownDirectorIdError, -} from "@intx/agent"; -import type { GrantEffect } from "@intx/types"; +} from "@intx/agent/authoring"; +import { actionConsumer, toolConsumer } from "@intx/authz"; +import type { GrantEffect, GrantOrigin, GrantWalkSnapshot } from "@intx/types"; import { EXECUTABLE_STEP_DESCENT, walkNestedWorkflowSteps, @@ -127,6 +128,7 @@ export type PluginToolDefinitions = ReadonlyMap< interface GrantSet { readonly grants: Set; readonly effects: Map; + readonly credentialConsumers: Set; } /** @@ -162,9 +164,150 @@ export function walkCapabilities( registry: DirectorRegistry, pluginDefs: PluginToolDefinitions = new Map(), ): CapabilityWalkResult { + const walked = walkDefinition(workflow, registry, pluginDefs); + return Object.freeze({ + perStep: walked.perStep, + unresolvedDirectors: Object.freeze([...walked.unresolved]), + }); +} + +/** + * The fields of a grant rule a run's runtime authority needs, without the + * storage identity (id, tenant, principal, timestamps) a host adds. + */ +export type RuntimeGrantRule = { + readonly resource: string; + readonly action: string; + readonly effect: GrantEffect; + readonly origin: GrantOrigin; + readonly conditions: Record | null; + readonly expiresAt: null; +}; + +const TOOL_GRANT_PREFIX = "tool:"; +const EFFECT_GRANT_PREFIX = "effect:"; + +/** + * Project a grant-walk snapshot into a run's runtime grant rules -- the + * `tool:` and `effect:` grants the runtime enforces fail-closed. + * Every distinct grant string across all steps becomes one creator-origin + * rule with `action: invoke`. Other walk grants (`capability:`, `director:`, + * `inference.source:`, `mail.*`) are not runtime grants and are dropped. + * + * Tool grants carry the effect the tool's static declaration requested (`ask` + * for approval-gated tools, `allow` otherwise) via each step's `grantEffects` + * record. A tool in more than one step is emitted once; when two steps disagree + * on its effect, `ask` wins over `allow` so an approval-gated declaration is + * never silently downgraded. A tool grant with no `grantEffects` entry throws: + * a defaulted `allow` would silently downgrade an `ask` tool. + * + * Effect grants are always `allow` -- the `effect.requires` set names the + * capability floor an action needs, with no per-effect ask/allow distinction, + * so they are NOT routed through the `grantEffects` record (which covers tool + * grants only). An `effect:` in more than one step is emitted once. + * + * The `tool:` rules carry the bare tool names the walk reads from inline + * `agent.toolFactories`. They authorize a source-ref tool, which runs under + * that name; a pinned tool runs as `:` and takes its + * authority from the sidecar tool-mark floor instead. + */ +export function runtimeGrantRules( + snapshot: Pick, +): RuntimeGrantRule[] { + const effectByResource = new Map(); + for (const step of snapshot.perStep) { + const grantEffects = new Map( + Object.entries(step.grantEffects), + ); + for (const grant of step.grants) { + if (grant.startsWith(TOOL_GRANT_PREFIX)) { + const effect = grantEffects.get(grant); + if (effect === undefined) { + throw new Error( + `runtimeGrantRules: tool grant ${JSON.stringify(grant)} has no grantEffects entry; the grant-walk snapshot must carry an effect for every tool grant`, + ); + } + const existing = effectByResource.get(grant); + if (existing === "ask" || effect === "ask") { + effectByResource.set(grant, "ask"); + } else if (existing === undefined) { + effectByResource.set(grant, effect); + } + } else if (grant.startsWith(EFFECT_GRANT_PREFIX)) { + if (!effectByResource.has(grant)) { + effectByResource.set(grant, "allow"); + } + } + } + } + return [...effectByResource].map(([resource, effect]) => ({ + resource, + action: "invoke", + effect, + origin: "creator", + conditions: null, + expiresAt: null, + })); +} + +/** + * One system-origin `credential:{id}` / `use` rule per distinct + * `(credentialId, consumer)` on a run's resolved credential bindings. The + * consumer string is copied onto `{ tool }`, the condition Gate 2 checks. + * Inference credentials are not in this list: they have no binding and no + * consumer. + */ +export function credentialUseGrantRules( + bindings: readonly { credentialId: string; consumer: string }[], +): RuntimeGrantRule[] { + const seen = new Set(); + const rules: RuntimeGrantRule[] = []; + for (const binding of bindings) { + const key = `${binding.credentialId}\0${binding.consumer}`; + if (seen.has(key)) continue; + seen.add(key); + rules.push({ + resource: `credential:${binding.credentialId}`, + action: "use", + effect: "allow", + origin: "system", + conditions: { tool: binding.consumer }, + expiresAt: null, + }); + } + return rules; +} + +/** + * Consumer identities the definition runs, including those that appear only + * inside an inline nested body: `toolConsumer(factory.id)` for every tool + * factory, and `actionConsumer(handler)` for every action that declares + * credentials. This is the credential-cap input. It is not part of + * `CapabilityWalkResult`: those strings are not operator-facing grants. + * Plugin names are absent. A factory with an empty `definitions` array is + * still present, because the source arm keys Gate 2 on the factory id. + */ +export function collectCredentialConsumers( + workflow: WorkflowDefinition, + registry: DirectorRegistry, + pluginDefs: PluginToolDefinitions = new Map(), +): ReadonlySet { + return walkDefinition(workflow, registry, pluginDefs).credentialConsumers; +} + +function walkDefinition( + workflow: WorkflowDefinition, + registry: DirectorRegistry, + pluginDefs: PluginToolDefinitions, +): { + readonly perStep: Map; + readonly unresolved: Set; + readonly credentialConsumers: ReadonlySet; +} { const triggerGrants = collectTriggerGrants(workflow); const unresolved = new Set(); const perStep = new Map(); + const credentialConsumers = new Set(); for (const stepId of workflow.stepOrder) { const primitive = workflow.steps[stepId]; @@ -176,10 +319,9 @@ export function walkCapabilities( // Every top-level step gets a fresh grant set; `collectPrimitiveGrants` // routes the step and any nested bodies it carries through one dispatch, // so an approval covers every agent, action, and effect the step can run. - const collected: GrantSet = { - grants: new Set(), - effects: new Map(), - }; + // The consumer set is the same object the nested visit writes into, then + // unioned here, so a consumer that exists only on an inline body survives. + const collected = emptyGrantSet(); collectPrimitiveGrants( primitive, registry, @@ -188,12 +330,20 @@ export function walkCapabilities( collected, ); perStep.set(stepId, freezeDeclarations(collected, triggerGrants)); + for (const consumer of collected.credentialConsumers) { + credentialConsumers.add(consumer); + } } - return Object.freeze({ - perStep, - unresolvedDirectors: Object.freeze([...unresolved]), - }); + return { perStep, unresolved, credentialConsumers }; +} + +function emptyGrantSet(): GrantSet { + return { + grants: new Set(), + effects: new Map(), + credentialConsumers: new Set(), + }; } /** @@ -268,8 +418,8 @@ function collectPrimitiveGrants( /** * Union the grants a single primitive declares in its own right -- its agent's - * grants and its action's `effect:` grants -- into `collected`, without - * descending into anything it nests. + * grants and its action's `effect:` grants -- and the credential consumers + * it runs into `collected`, without descending into anything it nests. */ function collectOwnGrants( primitive: WorkflowDefinition["steps"][string], @@ -285,6 +435,13 @@ function collectOwnGrants( for (const grant of collectActionGrants(primitive)) { collected.grants.add(grant); } + // Unlike a tool factory, which counts even with empty `definitions` + // because the source arm keys Gate 2 on its id, an action counts only when + // it carries a `credentials` field: that list is its declared credential + // surface, so an action without the field consumes nothing. + if (primitive.kind === "action" && primitive.credentials !== undefined) { + collected.credentialConsumers.add(actionConsumer(primitive.handler)); + } } function collectAgentGrants( @@ -300,6 +457,9 @@ function collectAgentGrants( // packages/agent/src/agent.ts) after the deploy has already gone out. const seenToolNames = new Set(); for (const factory of agent.toolFactories) { + // Gate 2 keys the consumer on the factory id, including a factory whose + // `definitions` array is empty and therefore emits no `tool:` grant. + collected.credentialConsumers.add(toolConsumer(factory.id)); // Intra-factory: a repeated name within one factory's declarations // is a declaration bug; the runtime would collapse the two into one // dispatch entry, leaving one tool unreachable. Surface it at walk diff --git a/vendor/intx/workflow-deploy/src/child-grant-filter.ts b/vendor/intx/workflow-deploy/src/child-grant-filter.ts new file mode 100644 index 0000000000..46a7247f24 --- /dev/null +++ b/vendor/intx/workflow-deploy/src/child-grant-filter.ts @@ -0,0 +1,164 @@ +// Cap a spawned child's inherited grants at the capabilities the child body +// itself declares. +// +// A spawned child runs under the authority of the run that spawned it, but it +// must never hold MORE than the child body's own definition declares. Without +// this cap a child authorizes against the full parent grant set, so a +// parent-held `effect:` grant or a bare-name `tool:` collision the child body +// never declared would authorize a capability the child was never written to +// use. A `credential:` allow is gated by its `{ tool }` condition: it survives +// when that condition equals a consumer the child body runs (a tool factory it +// instantiates, or an action it runs that declares credentials), and an +// unconditioned or unmatched credential allow is dropped. +// The parent stays the ceiling (a child can never exceed it); the child body's +// declared capability set is the shape it is narrowed to -- the same "a run's +// authority is bounded by its declared capabilities" model the top level +// enforces through its own walk. + +import type { DirectorRegistry } from "@intx/agent"; +import { matchPattern } from "@intx/authz"; +import type { WorkflowDefinition } from "@intx/workflow"; +import { + collectCredentialConsumers, + walkCapabilities, + type PluginToolDefinitions, +} from "./capability-walk"; + +/** + * The flat union of every grant-shape resource string the child body declares, + * across all its steps AND every inline nested body (loop, onTrigger, + * childWorkflow) the capability walk folds into its enclosing step. + * + * Pass the child's PRE-rewrite definition -- whose grandchildren are still + * inline. The walk skips a `{ ref }` body, so a rewritten definition (children + * lifted to refs) would omit a grandchild's declared resources; the union must + * include them, because the caller uses this set to cap the grants it persists + * as the ceiling a grandchild spawn filters against in turn. + */ +export function collectDeclaredResources( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: PluginToolDefinitions, +): ReadonlySet { + const walk = walkCapabilities(definition, directors, pluginDefs); + const declared = new Set(); + for (const step of walk.perStep.values()) { + for (const grant of step.grants) { + declared.add(grant); + } + } + return declared; +} + +/** + * Consumer identities the child body runs: its tool factories and its actions + * that declare credentials. Same pre-rewrite definition as + * {@link collectDeclaredResources}: a `{ ref }` body is skipped, and an inline + * nested body's consumers are included. + */ +export function collectDeclaredCredentialConsumers( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: PluginToolDefinitions, +): ReadonlySet { + return collectCredentialConsumers(definition, directors, pluginDefs); +} + +/** + * Filter a parent's grant rules down to what a spawned child body declares. + * + * The result caps the child at the parent: it only ever REMOVES parent rules, + * never adds or widens one. Dropping a `deny` or an `ask` weakens safety, so + * those are kept. An `allow` is kept only when the child declares the + * capability it authorizes. + * + * - Every `deny` and `ask` rule is kept unconditionally. Both only ever + * restrict, so dropping one WEAKENS safety. An `ask` floor in particular is + * load-bearing: `evaluateGrants` ranks `ask` above `allow` at equal + * specificity so a workflow cannot declare its way under an approval gate, + * and filtering the `ask` out would punch straight through that gate. + * - A `credential:` `allow` (`credential:{id}` and `credential:*`) is kept + * only when `conditions.tool` is a string equal to one of + * `credentialConsumers`. The comparison is exact. An unconditioned + * credential allow, a non-string `{ tool }` value, and a condition that + * names a consumer the child does not run are dropped. This branch + * does not fall through to the resource-pattern test. + * - Every other `allow` is kept only when its resource PATTERN covers at + * least one resource the child body declares (via `matchPattern`, so a + * wildcard grant like `tool:*` survives when the child declares any + * matching `tool:`). An `allow` for a resource the child never declares -- + * a parent-only `effect:` or bare-name `tool:` -- is dropped: that is the + * escalation this cap closes. A resource of `*` stays on this test. + * - Any entry that is not an `allow` rule with a string `resource` is kept. A + * non-rule entry is inert at evaluation (`evaluateGrants` reads `.resource`), + * so keeping it never widens authority, and keeping-on-doubt can never + * wrongly drop a restriction. + * + * Action is deliberately not part of the coverage test. The declared set is + * resource strings; a parent rule whose action never matches a child query is + * inert (`evaluateGrants` re-gates on action at decision time), so testing the + * resource alone can only OVER-keep an inert `allow`, never wrongly drop one + * the child legitimately needs. + */ +export function filterGrantsToDeclaredResources( + parentGrants: readonly unknown[], + declared: ReadonlySet, + credentialConsumers: ReadonlySet, +): readonly unknown[] { + return parentGrants.filter((grant) => + keepGrantForDeclared(grant, declared, credentialConsumers), + ); +} + +function keepGrantForDeclared( + grant: unknown, + declared: ReadonlySet, + credentialConsumers: ReadonlySet, +): boolean { + if (!isAllowRuleWithResource(grant)) { + return true; + } + if (isCredentialResource(grant.resource)) { + return credentialConsumerMatches(grant, credentialConsumers); + } + for (const resource of declared) { + if (matchPattern(grant.resource, resource)) { + return true; + } + } + return false; +} + +function isCredentialResource(resource: string): boolean { + return resource.startsWith("credential:"); +} + +function credentialConsumerMatches( + grant: object, + consumers: ReadonlySet, +): boolean { + if (!("conditions" in grant)) { + return false; + } + const conditions = grant.conditions; + if (typeof conditions !== "object" || conditions === null) { + return false; + } + if (!("tool" in conditions)) { + return false; + } + const tool = conditions.tool; + return typeof tool === "string" && consumers.has(tool); +} + +function isAllowRuleWithResource( + grant: unknown, +): grant is { effect: "allow"; resource: string } { + if (typeof grant !== "object" || grant === null) { + return false; + } + if (!("effect" in grant) || !("resource" in grant)) { + return false; + } + return grant.effect === "allow" && typeof grant.resource === "string"; +} diff --git a/vendor/intx/workflow-deploy/src/credential-binding-check.ts b/vendor/intx/workflow-deploy/src/credential-binding-check.ts new file mode 100644 index 0000000000..4f46305d9f --- /dev/null +++ b/vendor/intx/workflow-deploy/src/credential-binding-check.ts @@ -0,0 +1,224 @@ +// Check a frozen inert projection's credential bindings against the consumers +// the workflow actually runs, before any credential is resolved. +// +// Deploy resolves only the root definition's `credentialBindings`, and a +// binding is delivered to the consumer it names. So a binding on a nested body +// is never resolved, a binding whose consumer the workflow never runs is never +// used, two bindings for one consumer and handle leave the delivered one to +// chance, and an action that declares a handle with no binding fails before +// its handler runs. This check refuses all of these at approval, over the +// projection the gate freezes, so a typo in a binding fails the deploy instead +// of leaving a step without its credential. +// +// Scope of the rules: +// - The declared-handle rule covers actions only; tool packages carry no +// handle declarations in the inert projection. +// - Tool consumers are keyed on the tool factory id, the consumer identity +// on the source-ref lineage, the only lineage that delivers bindings. A +// `package` binding for a plugin package, or for a factory used only in a +// `{ ref }` body, names no consumer this deployment runs and is refused. +// +// The projection types its steps as `unknown`, so every field read here is +// validated first. A malformed frozen projection is a projector defect, not an +// author error, so it throws rather than becoming a violation. + +import { type } from "arktype"; +import type { CredentialBinding } from "@intx/types"; +import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; +import { + EXECUTABLE_STEP_DESCENT, + walkStepTree, +} from "@intx/workflow/definition"; + +import { + inertNestedBodies, + readInertStepToolFactoryIds, +} from "./inert-ontrigger-bodies"; + +const CONTEXT = "credential binding check: "; + +/** One credential binding problem the check found. */ +export type CredentialBindingViolation = + | { + /** An action declares a handle that no root binding names for it. */ + readonly kind: "unbound_declared_handle"; + readonly handler: string; + readonly handle: string; + } + | { + /** A nested body definition carries `credentialBindings`. */ + readonly kind: "nested_body_bindings"; + readonly bodyId: string; + /** The step ids the walk reached the body's carrying step through. */ + readonly reachedThrough: readonly string[]; + } + | { + /** A `package` binding names no tool factory the workflow runs. */ + readonly kind: "unknown_binding_package"; + readonly package: string; + readonly handle: string; + } + | { + /** An `action` binding names a handler no declaring action step runs. */ + readonly kind: "unknown_binding_action"; + readonly action: string; + readonly handle: string; + } + | { + /** An `action` binding names a handle the handler never declared. */ + readonly kind: "undeclared_binding_handle"; + readonly action: string; + readonly handle: string; + } + | { + /** Two root bindings name the same consumer and handle. */ + readonly kind: "duplicate_binding"; + readonly consumer: string; + readonly handle: string; + }; + +const ActionStep = type({ kind: "'action'", handler: "string" }); +const DeclaredHandles = type("string[]"); +const HasBindingsKey = type({ credentialBindings: "unknown" }); + +/** The consumer a binding names, as the refusal message reports it. */ +function bindingConsumerLabel(binding: CredentialBinding): string { + return binding.package !== undefined + ? `package ${binding.package}` + : `action ${binding.action}`; +} + +/** + * Every credential binding problem in `projection`, in walk order. An empty + * list means every root binding names a consumer the workflow runs, once, + * every declared action handle is bound, and no nested body carries bindings. + */ +export function checkCredentialBindings( + projection: WorkflowProjectionDefinition, +): readonly CredentialBindingViolation[] { + const violations: CredentialBindingViolation[] = []; + const toolFactoryIds = new Set(); + const declaredByHandler = new Map>(); + + walkStepTree({ + tree: projection, + context: CONTEXT, + nestedTrees: (step) => inertNestedBodies(step, EXECUTABLE_STEP_DESCENT), + visit: ({ stepId, step, path }) => { + // Read each body off the step that carries it, so a body is checked + // once and whatever its step count. + for (const body of inertNestedBodies(step, EXECUTABLE_STEP_DESCENT)) { + if (!(HasBindingsKey(body) instanceof type.errors)) { + violations.push({ + kind: "nested_body_bindings", + bodyId: body.id, + reachedThrough: path, + }); + } + } + + for (const id of readInertStepToolFactoryIds(step, CONTEXT, stepId)) { + toolFactoryIds.add(id); + } + + const asAction = ActionStep(step); + if (asAction instanceof type.errors) return; + if ( + typeof step !== "object" || + step === null || + !("credentials" in step) + ) { + return; + } + const declared = DeclaredHandles(step.credentials); + if (declared instanceof type.errors) { + throw new Error( + `${CONTEXT}action step ${path.join(" > ")} declares credentials that are not a list of strings: ${declared.summary}`, + ); + } + const handles = + declaredByHandler.get(asAction.handler) ?? new Set(); + for (const handle of declared) handles.add(handle); + declaredByHandler.set(asAction.handler, handles); + }, + }); + + const bindings: readonly CredentialBinding[] = + projection.credentialBindings ?? []; + + for (const [handler, handles] of declaredByHandler) { + for (const handle of handles) { + const bound = bindings.some( + (binding) => binding.action === handler && binding.handle === handle, + ); + if (!bound) { + violations.push({ kind: "unbound_declared_handle", handler, handle }); + } + } + } + + const seen = new Set(); + for (const binding of bindings) { + const consumer = bindingConsumerLabel(binding); + const key = `${consumer}\u0000${binding.handle}`; + if (seen.has(key)) { + violations.push({ + kind: "duplicate_binding", + consumer, + handle: binding.handle, + }); + } + seen.add(key); + + if (binding.package !== undefined) { + if (!toolFactoryIds.has(binding.package)) { + violations.push({ + kind: "unknown_binding_package", + package: binding.package, + handle: binding.handle, + }); + } + continue; + } + const declared = declaredByHandler.get(binding.action); + if (declared === undefined) { + violations.push({ + kind: "unknown_binding_action", + action: binding.action, + handle: binding.handle, + }); + } else if (!declared.has(binding.handle)) { + violations.push({ + kind: "undeclared_binding_handle", + action: binding.action, + handle: binding.handle, + }); + } + } + + return violations; +} + +/** One sentence for a violation, for a refusal message. */ +export function describeCredentialBindingViolation( + violation: CredentialBindingViolation, +): string { + switch (violation.kind) { + case "unbound_declared_handle": + return `action ${violation.handler} declares credential handle "${violation.handle}", but no root binding names it for that handler`; + case "nested_body_bindings": + return `body ${violation.bodyId} (carried by step ${violation.reachedThrough.join(" > ")}) declares credentialBindings; remove them, because only the root definition's bindings are resolved`; + case "unknown_binding_package": + return `the binding for handle "${violation.handle}" names package ${violation.package}, which no tool factory in the workflow has`; + case "unknown_binding_action": + return `the binding for handle "${violation.handle}" names action handler ${violation.action}, which no action step that declares credentials runs`; + case "undeclared_binding_handle": + return `the binding for handle "${violation.handle}" names action handler ${violation.action}, which does not declare that handle`; + case "duplicate_binding": + return `two bindings name ${violation.consumer} and handle "${violation.handle}"`; + default: { + const _exhaustive: never = violation; + return `unknown credential binding violation: ${JSON.stringify(_exhaustive)}`; + } + } +} diff --git a/vendor/intx/workflow-deploy/src/index.ts b/vendor/intx/workflow-deploy/src/index.ts index 17cc5280b4..815c204b09 100644 --- a/vendor/intx/workflow-deploy/src/index.ts +++ b/vendor/intx/workflow-deploy/src/index.ts @@ -4,7 +4,9 @@ // Surfaces today: // - capability walk: structural lift of `getRequiredEnvKeys` that // emits the grant-shape declarations the operator-approval gate -// consumes. +// consumes, plus the child-grant cap that narrows a parent's +// grants to the resources and credential consumers the child +// body declares. // - approval gate: consumes the walk's output plus an operator- // supplied `ApprovalSet` and yields a per-step pending delta. // - deploy derivation + source pinning: pure address derivation @@ -21,6 +23,11 @@ export { type GrantDeclarations, type PluginToolDefinitions, } from "./capability-walk"; +export { + collectDeclaredCredentialConsumers, + collectDeclaredResources, + filterGrantsToDeclaredResources, +} from "./child-grant-filter"; export { approvalItemsFromSet, approvalSetFromItems, @@ -33,6 +40,11 @@ export { type ApprovalSource, type CapabilityApprovalGate, } from "./capability-approval"; +export { + checkCredentialBindings, + describeCredentialBindingViolation, + type CredentialBindingViolation, +} from "./credential-binding-check"; export { extractFoldedBody, type FoldedBody } from "./fold-synthesis"; export { enumerateInertBodies, @@ -49,12 +61,14 @@ export { buildInertProjectionStepSources, buildInertBodyStepSources, buildSingleStepAgentDefinition, + WorkflowDefinitionInvalidError, + type DeployContent, +} from "./orchestrator"; +export { deriveRunAddress, deriveRunAgentId, deriveStepAddress, resolveStepAddress, deriveStepAgentId, deriveWorkflowRunRepoId, - WorkflowDefinitionInvalidError, - type DeployContent, -} from "./orchestrator"; +} from "./addresses"; diff --git a/vendor/intx/workflow-deploy/src/inert-ontrigger-bodies.ts b/vendor/intx/workflow-deploy/src/inert-ontrigger-bodies.ts index 3023254f4f..979b1ccc3f 100644 --- a/vendor/intx/workflow-deploy/src/inert-ontrigger-bodies.ts +++ b/vendor/intx/workflow-deploy/src/inert-ontrigger-bodies.ts @@ -60,7 +60,7 @@ import { WorkflowProjectionDefinition, WorkflowStep, } from "@intx/types/sidecar"; -import { inlineBodyRef } from "@intx/workflow"; +import { inlineBodyRef } from "@intx/workflow/projection"; import { LOOP_BODY_DESCENT, walkStepTree, @@ -440,6 +440,50 @@ export function readInertStepInference( return { isAgent: false, preference: null }; } +// The agent surface the tool-factory reader reads, typing only the factory ids. +const AgentWithToolFactories = type({ + toolFactories: type({ id: "string" }).array(), +}); +const StepWithToolFactories = type({ + kind: "'step'", + agent: AgentWithToolFactories, +}); +const MapWithToolFactories = type({ + kind: "'map'", + step: { agent: AgentWithToolFactories }, +}); + +/** + * The tool factory ids a frozen inert step's agent instantiates: a `step` + * carries the agent directly, a `map` carries it on its inner step, and any + * other primitive has none. A `step`/`map` that fails the agent shape is a + * malformed projection and throws, prefixed with `context`. + */ +export function readInertStepToolFactoryIds( + stepValue: unknown, + context: string, + stepId: string, +): readonly string[] { + const asStep = StepWithToolFactories(stepValue); + if (!(asStep instanceof type.errors)) { + return asStep.agent.toolFactories.map((factory) => factory.id); + } + const asMap = MapWithToolFactories(stepValue); + if (!(asMap instanceof type.errors)) { + return asMap.step.agent.toolFactories.map((factory) => factory.id); + } + const kind = StepKind(stepValue); + if ( + !(kind instanceof type.errors) && + (kind.kind === "step" || kind.kind === "map") + ) { + throw new Error( + `${context}step ${stepId} is a ${kind.kind} primitive but carries no valid agent.toolFactories`, + ); + } + return []; +} + /** * Lift one inline body out of an enclosing projection: validate it, override its * id to `inlineBodyRef(enclosingId, stepId)` (the ref the sidecar stages diff --git a/vendor/intx/workflow-deploy/src/orchestrator.ts b/vendor/intx/workflow-deploy/src/orchestrator.ts index f247c7dc24..732c9bae84 100644 --- a/vendor/intx/workflow-deploy/src/orchestrator.ts +++ b/vendor/intx/workflow-deploy/src/orchestrator.ts @@ -1,14 +1,4 @@ -// Workflow-deploy derivation and source-pinning utilities. -// -// The deployment address model is a pure function of `(runId, stepId, -// domain)`: a one-step workflow has no distinct step address -- the lone -// step IS the deployment head (`deriveRunAddress`) -- while a workflow with -// more than one step derives per-step addresses of the form -// `-@`. Because the derivation is pure, the -// supervisor reconstructs the same addresses at spawn time without any -// per-deploy state, and `resolveStepAddress` is the single owner of the -// head/step collapse decision for a consumer that must choose an address -// from the host-sourced step count alone. +// Workflow-deploy source-pinning utilities. // // The source-pinning utilities (`pickStepInferenceSource`, // `buildInertProjectionStepSources`, `buildInertBodyStepSources`, @@ -28,7 +18,6 @@ import type { import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; -import { formatRunAddress } from "@intx/types"; import { type ApprovalSet } from "./capability-approval"; import { @@ -368,117 +357,6 @@ function sameInferenceSource(a: InferenceSource, b: InferenceSource): boolean { return a.id === b.id && a.provider === b.provider && a.model === b.model; } -/** - * Pure function: derive a step's run address from - * `(runId, stepId, domain)`. Exported so the supervisor can reconstruct - * the same addresses at spawn time without sharing storage with the - * deploy flow. - * - * The local part IS the run id with the step suffix appended; the runId is - * already a minted `run_` carrying the `run_` marker `parseRunAddress` - * requires at the substrate boundary. The per-step local-part is concat-only - * because `stepId` is already constrained to `[a-zA-Z0-9_-]+` by the workflow - * definition validator. - */ -export function deriveStepAddress(args: { - runId: string; - stepId: string; - domain: string; -}): string { - return formatRunAddress(`${args.runId}-${args.stepId}`, args.domain); -} - -/** - * Derive the per-step agent id (the `agent-state` repo's id and the - * `HarnessConfig.agentId`). Pure function of `(runId, stepId)`. - */ -export function deriveStepAgentId(args: { - runId: string; - stepId: string; -}): string { - return `${args.runId}-${args.stepId}`; -} - -/** - * Derive the deployment-level mail address the supervisor registers on - * the bus. It is the run id `@` the domain; pure function of `(runId, domain)`. - * - * The supervisor uses this address as the inbound mail address for the - * deployment as a whole; per-step bindings carry their own - * derived-step addresses. - */ -export function deriveRunAddress(args: { - runId: string; - domain: string; -}): string { - return formatRunAddress(args.runId, args.domain); -} - -/** - * Resolve where a step's deploy tree lives, given the deployment's step - * count. This is the single owner of the head/step collapse DECISION for - * a consumer that must choose the address without knowing the deploy - * shape: a one-step workflow has no distinct steps, so its lone step IS - * the head (`deriveRunAddress`); a multi-step deployment keeps the - * head distinct from its per-step addresses (`deriveStepAddress`). The - * sidecar child reads its deploy tree from the address this returns, - * keyed only off the deployment mailbox and the host-sourced `stepCount`. - * - * The producers do not route through here -- each handles one shape - * unconditionally: the single-step deploy stages the tree at the head, - * the multi-step deploy at each per-step address. Because `stepCount` is - * the deployed definition's `stepOrder.length`, sourced from the host, - * the consumer's collapse always agrees with whichever producer staged - * the tree; the two processes never derive divergent addresses. - */ -export function resolveStepAddress(args: { - runId: string; - stepId: string; - domain: string; - stepCount: number; -}): string { - return args.stepCount === 1 - ? deriveRunAddress({ - runId: args.runId, - domain: args.domain, - }) - : deriveStepAddress(args); -} - -/** - * Derive the deployment-level agent id used on the `agent.deploy` - * frame's `agentId` field. Pure function of `(runId)`. - */ -export function deriveRunAgentId(args: { runId: string }): string { - return args.runId; -} - -/** - * Project a workflow-deployment run address into the substrate-safe - * id of its workflow-run repo (`{ kind: "workflow-run", id }`). Pure - * function of the deployment's run address. - * - * The workflow-run repo's `repoId.id` must match `SAFE_REPO_ID` - * (`/^[a-zA-Z0-9_-]+$/`, the substrate's repo-path-safety contract in - * `packages/hub-sessions/src/repo-store/types.ts`), and the supervisor - * principal's `runId` must equal `workflowRunRepoId.id` for the - * workflow-run kind handler's authz check to pass. That regex rejects - * `@` and `.`, both of which appear in every run address, so the - * address is sanitized by substituting every disallowed character with - * `-`. - * - * The mapping is lossy (two distinct addresses can collapse to the same - * slug) but deterministic. The sidecar's deploy router keys the - * workflow-run repo by this slug at write time; the hub's read routes - * reconstruct the deployment address via `deriveRunAddress` and - * apply this same derivation so read and write address the same repo. - * A collision implies two deployments are claiming the same workflow-run - * surface, which the sidecar's deploy router rejects at deploy time. - */ -export function deriveWorkflowRunRepoId(agentAddress: string): string { - return agentAddress.replaceAll(/[^a-zA-Z0-9_-]/g, "-"); -} - /** * Assemble a single-step `AgentDefinition` from already-resolved fields. This * is the single place the single-step agent shape is constructed, so the diff --git a/vendor/intx/workflow-deploy/tsconfig.json b/vendor/intx/workflow-deploy/tsconfig.json index a957391137..94517d244f 100644 --- a/vendor/intx/workflow-deploy/tsconfig.json +++ b/vendor/intx/workflow-deploy/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" }, diff --git a/vendor/intx/workflow-deploy/tsconfig.src.json b/vendor/intx/workflow-deploy/tsconfig.src.json index 219ab4d0f7..927e3a3d24 100644 --- a/vendor/intx/workflow-deploy/tsconfig.src.json +++ b/vendor/intx/workflow-deploy/tsconfig.src.json @@ -22,6 +22,9 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, { "path": "../types/tsconfig.src.json" }, diff --git a/vendor/intx/workflow-host/README.md b/vendor/intx/workflow-host/README.md index 5ad593d565..5aced10349 100644 --- a/vendor/intx/workflow-host/README.md +++ b/vendor/intx/workflow-host/README.md @@ -3,6 +3,16 @@ Production-host implementations for the abstract `WorkflowRuntimeEnv` from `@intx/workflow`. +The one-shot probe has separate entry points: the sidecar host imports +`createWorkflowProbeExecutor` from `@intx/workflow-host/probe`, while its +child imports `runWorkflowProbeChildFromProcessEnv` from +`@intx/workflow-host/probe/child`. The child takes `walkCapabilities` from +`@intx/workflow-deploy/capabilities`. These child entry points load the +workflow definition and projection helpers without importing the supervisor, +repository storage, or execution-frame schemas. Workflow packages can use +`@intx/workflow/definition` and `@intx/agent/authoring` to preserve that +separation when their own entry modules load. + The host is the part of the workflow runtime that knows how to talk to real infrastructure: a `workflow-run` repo, the substrate's per-repo lock, an inference harness, child-workflow @@ -26,12 +36,22 @@ The package is organized along the abstract pieces it implements: startup recovery walk and a live `subscribeKind` loop so a `TimerSet` committed by an active workflow process fires without waiting for a process restart. `signal-channel.ts` funnels live - `SignalReceived` commits into the matching awaiter, with resume - rehydration consulting `unconsumedSignals` so a signal that - arrived while offline replays before live subscription begins. + `SignalReceived` commits into the matching awaiter. The channel + reads `unconsumedSignals` through its injected `readState` + reader, but every production call site passes an `emptyState` + reader, so that queue is always empty in production: a pre-await + signal resolves through the live `subscribeKind` tail, and + resume rehydration of a signal that arrived while the run was + offline is not wired. Plumbing the runtime body's own `RunState` + reader into the child is what that capability waits on. - `ipc/` — control and event channel implementations the supervisor wraps. Threat model lives at the top of - `ipc/index.ts`; the supervisor uses these primitives directly. + `ipc/index.ts`; the supervisor imports the individual primitives without + constructing the combined control-payload union. Each message kind's top-level + validator is built on first receipt and cached; shared payload schemas and + the message header are still constructed at module load. The event receiver + authenticates and validates each payload into the canonical `InferenceEvent` + type, which the deploy router forwards without a second validation pass. - `supervisor/` — the per-deployment supervisor. See the next section. - `child/` — the workflow-process child entry function @@ -48,7 +68,7 @@ Public surface (the package barrel re-exports these): - `createWorkflowRunBlobSubstrate` — the production `BlobSubstrate` adapter with 1 MiB inline-vs-blob spill threshold. - `createWorkflowStepInvoker` — the production `StepInvoker` adapter. -- `createWorkflowSpawnChild` — the production `SpawnChildWorkflow` +- `createInMemorySpawnChild` — the production `SpawnChildWorkflow` adapter. - `createWorkflowSupervisor` — the per-deployment supervisor factory. See "Supervisor" below for the bindings shape. @@ -58,10 +78,34 @@ Public surface (the package barrel re-exports these): primitive the supervisor invokes for every cancellation origin. Plus the relevant options and callback types (`StepEnvBase`, -`RunChildWorkflow`, `ChildTerminalStatus`, +`RunChildWorkflow`, `WorkflowSupervisorBindings`, `SubprocessSpawner`, `MailBusBindings`, `PrincipalSigner`, `SignedPayload`, etc.). +### Local entry + +`@intx/workflow-host/local` exports what a caller needs to run a definition +with `runLocal` and the real step invoker, with no hub and no sidecar: + +- `createWorkflowStepInvoker`, `createToolBearingAgentFactory`, + `attachStepTools`, `attachStepCredentialWiring` and + `deriveToolMarkFloorGrants` — the step invoker and the step tool builders + the deployed host uses. +- `createCredentialsBackedAuthorize` and `createGrantEvaluator` — the + authorize closure and grant evaluator a run child authorizes with. +- `createLocalCredentialsSnapshot` and `snapshotStepGrants` — the grants a + deployed source-ref run holds, derived from the definition, and their + per-step reader for Gate 2. + +Nothing this entry imports is a Node or Bun module, so it bundles for a +browser; `tests/workflow/local-runtime-browser-bundle.test.ts` checks that. +Some code it reaches uses the `Buffer` global at run time (MIME header +encoding in `@intx/mime`), so a browser caller installs one, as +`@intx/storage-isogit/browser` does. The +package root and the other subpaths are Node-only by design: the +supervisor spawns processes, and the run child, the substrate adapters and +the deploy program read and write the filesystem. + ## Supervisor `createWorkflowSupervisor(bindings)` returns a per-deployment @@ -104,7 +148,7 @@ The constructor argument shape: The sidecar's deploy router is the single ingress for inbound `agent.deploy` frames; its production wiring lives at -`apps/sidecar/src/workflow-host-wiring.ts` in +`src/deploy/workflow-host-wiring.ts` in `createSidecarDeployRouter`. Every deploy stages through the workflow-run substrate, and the router decides between two frame shapes: @@ -140,9 +184,22 @@ through `signAsPrincipal("supervisor", ...)` for every origin in the Q3 map. The `self`-origin case carries the workflow-process's stated reason; the supervisor wraps it into the same supervisor- signed shape as the operator and drain origins. +When a child is active, it first flushes and pauses its runtime event +writer. The supervisor commits the signed cancellation while that writer +is paused, then releases the child to apply cancellation. A child that +does not respond still requires the caller's forced-stop deadline. `shutdown()` unregisters the mail address, kills the child, and -disposes subscriptions. +disposes subscriptions. Concurrent callers await the same teardown through +confirmed child exit, including a replacement still awaiting readiness. +A replacement cannot spawn once shutdown begins. +It releases pending cancellation waits and requests the kill before waiting for +the dispatch loops, so a child that stops reading its control pipe cannot block +its own forced termination. The kill escalates from SIGTERM to SIGKILL after +the kill timeout, so a child that traps SIGTERM cannot block it either. +Already-started cancellation commits finish before the supervisor releases its +bindings, and so does every dispatch loop still running, including the loop of +a cohort a recycle has just replaced. `drain(opts)` sends the drain control mail and waits for in-flight runs to drain per each step's `drainBehavior`; on the drain-timeout it @@ -150,10 +207,10 @@ escalates to a signed `CancelRequested{origin: "supervisor-drain"}`. `recycle(opts)` tears the current child down and stands a fresh one up against the SAME deploy tree (same materialized source closure, same -per-step credential repos). It is strictly orthogonal to redeploy, -which mints a new deploy tree. Operator, supervisor-policy (max-uptime -/ max-rss / grants-staleness), and workflow-process-self-initiated -origins all funnel through the same path. +per-step credential repos). A deployment keeps its deploy tree for its +life; a different tree means a new deployment. Operator, +supervisor-policy (max-uptime / max-rss / grants-staleness), and +workflow-process-self-initiated origins all funnel through the same path. ### Respawn policy @@ -198,16 +255,19 @@ status: on latch the supervisor (the sole writer of the workflow-run repo) commits a `RunFailed` for the deployment's stable run, flipping its `workflow_run.status` to `failed` through the same pack path every other terminal run uses. External automation that watches run status -sees the crash-loop as a failed run. +sees the crash-loop as a failed run. The commit lands before the latch's +teardown resolves and before the host hears of the self-termination, so a +host that reads the stopped deployment's history finds it. ### Host wiring A host that wants to instantiate a supervisor constructs the -bindings against its own infrastructure. The reference -implementation for the in-tree sidecar lives at -`apps/sidecar/src/workflow-host-wiring.ts` and is intentionally -thin — anything that would benefit a future alternative-sidecar -implementation belongs inside this package, not in the wiring. +bindings against its own infrastructure. The in-tree sidecar's +supervisor and deploy router live in this package +(`src/deploy/workflow-host-wiring.ts`). The host entry passes the Bun +spawners from `apps/sidecar/src/workflow-child-spawner.ts` and +`apps/sidecar/src/workflow-probe-spawner.ts`. The child binary +closes the factory in `apps/sidecar/src/workflow-child-bindings.ts`. ## Child Entry @@ -285,21 +345,25 @@ The contract is intentionally narrow: only the keys the host listed in `RunWorkflowChildFromProcessEnvOpts.substrateConfigKeys`. The factory returns `RunWorkflowChildBindings`: substrate `RepoStore`, - principal, per-deployment repo ids, scheduler, step invoker, child - spawner, grant evaluator. The factory consumes the typed struct, - never `NodeJS.ProcessEnv` directly. Each spawner declares + principal, per-deployment repo ids, scheduler, step invoker, action + credentials factory, child spawner, grant evaluator. The action + credentials binding turns the run's credential wiring into the + factory that builds the credentials each action handler's `perform` + scopes resolve through. The factory consumes the typed struct, never + `NodeJS.ProcessEnv` directly. Each spawner declares `hasUpstreamSignalResolver` on the runtime env it builds: `true` for a run an answer can reach -- the deployment's own addressable - run, or a suspendable body whose container relays a decision back - down -- and `false` for a terminal `childWorkflow` child, which - carries no address and is run to its terminal rather than driven - across parks. The field is required, so an omission is a compile - error; a wrong `true` is not. Declared on a terminal-child seam it - type-checks and reinstates the hang the flag exists to remove. The - in-tree seams show both answers: `buildRuntimeEnv` declares `true` - for the deployment's own run, while in the sidecar's substrate - factory `createSidecarRunChild` declares `false` and - `createSidecarSpawnSuspendableChild` declares `true`. + run, a suspendable body whose container relays a decision back + down, and a `childWorkflow` child whose parent relays each park + onto that run. The child stays unaddressable. The field is + required, so an omission is a compile error. A `childWorkflow` + spawner forces `true` rather than inheriting a parent `false`, + which would refuse a gate the relay can answer. A run that sets + the flag `false` still refuses an untimed park before any + suspension is written. The in-tree seams declare `true`: + `buildRuntimeEnv` for the deployment's own run, and in the + sidecar's substrate factory both `createSidecarRunChild` and + `createSidecarSpawnSuspendableChild`. 3. **The helper fails loudly.** A missing or malformed spawn-time env throws via `parseSpawnTimeEnv`; a substrate-config key the host listed but the supervisor did not populate throws before the @@ -308,12 +372,16 @@ The contract is intentionally narrow: host's binary decides the exit semantics (the convention is `process.exit(1)` with a stderr message on rejection). +Use the `/child` entry to keep supervisor orchestration and parent wire +validators out of the child. Its control receiver validates each message with +an ArkType schema cached on first receipt of that message kind. + Example host binary (`apps//bin/workflow-child`): ```ts #!/usr/bin/env bun -import { runWorkflowChildFromProcessEnv } from "@intx/workflow-host"; -import { createSubstrate } from "../src/workflow-substrate-factory"; +import { runWorkflowChildFromProcessEnv } from "@intx/workflow-host/child"; +import { createSubstrate } from "../src/workflow-child-bindings"; await runWorkflowChildFromProcessEnv(createSubstrate, { substrateConfigKeys: ["SIDECAR_DATA_DIR" /* ... */], @@ -325,11 +393,15 @@ await runWorkflowChildFromProcessEnv(createSubstrate, { }); ``` -The reference in-tree implementation lives in `apps/sidecar`. An -alternative-sidecar implementer follows the same pattern: write a -substrate factory against its own infrastructure, ship a ~5-line -entry script, and resolve the `binaryPath` binding to that script -in its supervisor-wiring module. +The reference in-tree binary imports `createSubstrate` from +`apps/sidecar/src/workflow-child-bindings.ts`, which closes this +package's `createSidecarSubstrateFactory` over the app's tool +materializer and the grant cap from `@intx/workflow-deploy`. The child +binary path is resolved in +`apps/sidecar/src/workflow-child-spawner.ts`. An alternative host +follows the same pattern: write a substrate factory against its own +infrastructure, ship a short entry script, and pass that script as +`binaryPath`. ### Scheduler adapter diff --git a/vendor/intx/workflow-host/VENDORED-FROM b/vendor/intx/workflow-host/VENDORED-FROM index d72193c16a..3a41a54044 100644 --- a/vendor/intx/workflow-host/VENDORED-FROM +++ b/vendor/intx/workflow-host/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow-host) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6448: the suspendable-child spawn seam threads the parent child's credentials-backed authorize and MailPartReader (adapters/spawn-child.ts opts credentialWiring/mailPartReader, hoisted above the body resolver in child/run-child.ts), so a tool-bearing body agent gates its tool calls through the same per-step grant snapshot a top-level step does and reads an attachments-only inbound mail's parts instead of throwing; the sidecar still passes both (workflow-substrate-factory.ts bridges CredentialWiring onto upstream's newer native credentialMaterial cell). child/run-child.ts resolves a step's grants entry through findStepGrantsEntry (head collapse: a body step whose own id is absent from a single-step deployment's snapshot resolves to the sole entry). CL-6396: the event-channel envelope, sender, and receiver carry an optional childRunId; run-child stamps spawnInput.childRunId onto body-child events so overlapping turns HMAC-bind the occurrence they belong to. Temporary delta: buildInboundMessageFromMail omits `subject` when the decoded header is empty (a present-but-textless Subject header), since createInboundMessage rejects an empty subject and fails the step; see VENDORED.md's kill condition. Absorbed upstream at this re-vendor: the step-invoker describeReplyFailure/usableAddr rewording, the substrate-mailbox-store nullable-header widening, the supervisor-backed-transport MessageTransportError codes, the control-channel optional from/date/messageId, and the supervisor stored-envelope messageId/date narrowing. Retired at earlier pins: the empty-mail drop (CL-6164) and the action/loop runtime bind (CL-6325). +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). CL-6396: the event-channel envelope, sender, receiver, supervisor, and deploy-router wiring carry an optional childRunId; run-child stamps spawnInput.childRunId onto body-child events so overlapping turns HMAC-bind the occurrence they belong to. child/substrate-factory.ts (moved from apps/sidecar upstream) registers @corbits/credential-header's and @corbits/credential-mcp's providers alongside builtinCredentialProviders(), and createSidecarStepBuildEnv loads the step's director registry from the closure package (interchange.directors) instead of the built-ins only. Temporary delta: buildInboundMessageFromMail (exported from adapters/step-invoker.ts) omits `subject` when the decoded header is empty (a present-but-textless Subject header), since createInboundMessage rejects an empty subject and fails the step; see VENDORED.md's kill condition. Retired at this pin: the CL-6448 spawn-child authorize/mailPartReader/findStepGrantsEntry deltas (upstream's buildChildRunEnv in child/substrate-factory.ts now builds a per-child credentialsSnapshot over the child's own step ids and wires createCredentialsBackedAuthorize natively). diff --git a/vendor/intx/workflow-host/package.json b/vendor/intx/workflow-host/package.json index e5058ae510..269467deef 100644 --- a/vendor/intx/workflow-host/package.json +++ b/vendor/intx/workflow-host/package.json @@ -1,30 +1,59 @@ { "name": "@intx/workflow-host", "description": "Production-host implementations of the abstract WorkflowRuntimeEnv from @intx/workflow", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { ".": { "types": "./src/index.ts", "default": "./src/index.ts" + }, + "./testing": { + "types": "./src/testing/index.ts", + "default": "./src/testing/index.ts" + }, + "./deploy": { + "types": "./src/deploy/index.ts", + "default": "./src/deploy/index.ts" + }, + "./child": { + "types": "./src/child/program.ts", + "default": "./src/child/program.ts" + }, + "./local": { + "types": "./src/local.ts", + "default": "./src/local.ts" + }, + "./probe": { + "types": "./src/probe/index.ts", + "default": "./src/probe/index.ts" + }, + "./probe/child": { + "types": "./src/probe/child.ts", + "default": "./src/probe/child.ts" } }, "scripts": { "typecheck": "tsc --noEmit" }, "dependencies": { + "@corbits/credential-header": "^0.1.0", + "@corbits/credential-mcp": "^0.1.0", "@intx/agent": "workspace:*", - "@intx/crypto": "0.4.0", + "@intx/authz": "workspace:*", + "@intx/crypto": "0.5.0", + "@intx/harness": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.4.0", + "@intx/log": "0.5.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.4.0", + "@intx/storage-isogit": "0.5.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", + "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:" }, "devDependencies": { diff --git a/vendor/intx/workflow-host/src/adapters/blob-substrate.ts b/vendor/intx/workflow-host/src/adapters/blob-substrate.ts index d2246bf044..4373de5cf7 100644 --- a/vendor/intx/workflow-host/src/adapters/blob-substrate.ts +++ b/vendor/intx/workflow-host/src/adapters/blob-substrate.ts @@ -26,7 +26,7 @@ // `ephemeral: false` -- blob refs survive instance turnover because // they resolve back to bytes on the workflow-run repo. -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; import type { Principal, RepoId, diff --git a/vendor/intx/workflow-host/src/adapters/spawn-child.ts b/vendor/intx/workflow-host/src/adapters/spawn-child.ts index 8a0d059aec..a24f9b4455 100644 --- a/vendor/intx/workflow-host/src/adapters/spawn-child.ts +++ b/vendor/intx/workflow-host/src/adapters/spawn-child.ts @@ -3,8 +3,9 @@ // The runtime body sees the spawn callback shape: given a `definitionRef` // (the internal ref the deploy step assigned when it lifted the authored // inline child), a parent-allocated `childRunId`, the materialized child -// input, and parent attribution, settle once the child run reaches a terminal -// phase. The adapter itself does not execute the child workflow -- it resolves +// input, and parent attribution, return a live handle. The parent drives that +// handle across the child's parks. The adapter itself does not execute the +// child workflow -- it resolves // the `definitionRef` into a concrete `WorkflowDefinition` and delegates the // spawn to a runtime-supplied `runChild` callback. The supervisor wires the // callback against a child `WorkflowRuntimeEnv` and `runtimeRun`. @@ -20,15 +21,20 @@ // covered by the parent's re-verify -- no separate on-disk read and no // separate per-body re-verify. // -// - childWorkflow (the terminal adapter): an owned import embedded inline in -// the parent's definition. It is lifted to an internal `{ ref }` at child -// boot and resolved in-memory from the parent's closure map +// - childWorkflow: an owned import embedded inline in the parent's +// definition. It is lifted to an internal `{ ref }` at child boot and +// resolved in-memory from the parent's closure map // (`createInMemorySpawnChild`) -- exactly like a source-ref onTrigger // body, with NO on-disk asset and NO separate per-child re-verify (the // parent's re-verify already covers it, since the inline child rides the -// parent's hashed projection). The terminal-only drive (await the child's -// terminal, no park) is the only thing that distinguishes it from the -// suspendable body adapter. +// parent's hashed projection). The callback returns a live handle. The +// parent drives it across parks and relays each one onto the parent run. +// The child stays unaddressable. A parked child survives a process +// restart: `resumeFromEvents` re-adopts its log and the parent +// reestablishes the relay. The spawn step defaults to `"wait"`, so the +// drain signal does not abort it, whether the child is parked or still +// mid-step. An explicit `"cancel"` still aborts the child through +// `localAbort`. // // Drain coordination is handled by the supervisor's drain primitive // (`packages/workflow-host/src/supervisor`), not by this adapter. The @@ -60,27 +66,16 @@ // childRunId, ... }`) is the seam that makes the scoping unambiguous // at the boundary. -import type { InferenceEvent, MailPartReader } from "@intx/types/runtime"; +import type { InferenceEvent } from "@intx/types/runtime"; import type { SpawnChildWorkflow, SpawnSuspendableChild, SuspendableChildHandle, - WorkflowAuthorizeFn, WorkflowDefinition, WorkflowEvent, } from "@intx/workflow"; -import type { - CredentialMaterialRef, - CredentialWiring, -} from "../child/run-child"; - -/** - * The terminal-status shape the runtime body expects back from a - * spawn. Mirrored from `SpawnChildWorkflow`'s return type so the - * `runChild` callback's signature is symmetric with the adapter's. - */ -export type ChildTerminalStatus = "completed" | "failed" | "cancelled"; +import type { CredentialMaterialRef } from "../child/run-child"; /** * Runtime-supplied child execution callback. The supervisor owns the @@ -116,24 +111,25 @@ export type RunChildWorkflow = ( signal: AbortSignal; depth: number; maxChildSpawnDepth: number; + resumeFromEvents?: readonly WorkflowEvent[]; }, onEvent: (event: InferenceEvent) => void, credentialMaterial?: CredentialMaterialRef, -) => Promise<{ terminalStatus: ChildTerminalStatus }>; +) => Promise; /** - * Construct the terminal `WorkflowRuntimeEnv.SpawnChildWorkflow` adapter for an - * owned childWorkflow import. The child re-evaluated the whole pinned closure + * Construct the `WorkflowRuntimeEnv.spawnChild` adapter for an owned + * childWorkflow import. The child re-evaluated the whole pinned closure * and lifted every inline child to an internal `{ ref }`, so the child * definitions are in hand and already covered by the parent's re-verify. - * Resolve each `definitionRef` - * from the in-memory `bodies` map and delegate to the runtime-supplied - * `runChild`, with NO on-disk round-trip and NO separate per-child re-verify: - * materializing the child back out and re-fingerprinting it would round-trip - * trusted-in-hand data for no gain, and the closure re-eval on restart - * re-derives the same bodies durably. Mirrors - * {@link createInMemorySpawnSuspendableChild} but drives the child terminal-only - * (await its terminal status) rather than across approval parks. + * Resolve each `definitionRef` from the in-memory `bodies` map and + * delegate to the runtime-supplied `runChild`, with NO on-disk round-trip + * and NO separate per-child re-verify: materializing the child back out + * and re-fingerprinting it would round-trip trusted-in-hand data for no + * gain, and the closure re-eval on restart re-derives the same bodies + * durably. Mirrors {@link createInMemorySpawnSuspendableChild}. `runChild` + * returns the live handle the parent drives across the child's parks; + * this adapter passes that handle through without unwrapping it. */ /** * Host-side widening of the runtime {@link SpawnChildWorkflow} contract: the @@ -165,6 +161,7 @@ export function createInMemorySpawnChild(opts: { signal, depth, maxChildSpawnDepth, + resumeFromEvents, }, onEvent, credentialMaterial, @@ -192,7 +189,7 @@ export function createInMemorySpawnChild(opts: { throw abortError(signal); } - const result = await opts.runChild( + return opts.runChild( { definition, definitionRef, @@ -203,11 +200,11 @@ export function createInMemorySpawnChild(opts: { signal, depth, maxChildSpawnDepth, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), }, onEvent, credentialMaterial, ); - return { terminalStatus: result.terminalStatus }; }; } @@ -233,29 +230,6 @@ export type RunSuspendableChild = ( depth: number; maxChildSpawnDepth: number; resumeFromEvents?: readonly WorkflowEvent[]; - /** - * The parent child's credentials-backed workflow authorize (CL-6448). - * A body agent's tool calls gate through the SAME per-step grant - * snapshot the top-level step invoker consults; without this the - * body env's throwing authorize stub blocks every tool invocation a - * body agent makes. Optional so a host that runs bodies toolless - * keeps its stub. - */ - authorize?: WorkflowAuthorizeFn; - /** - * The parent child's live credential wiring (CL-6448), so a body step's - * tool bundles can shape consumer-scoped `credentials` capabilities - * exactly as a top-level step's do. Upstream's native `credentialMaterial` - * carries only the material cell; this additionally carries the - * per-step `resolveStepGrants` a tool bundle needs. - */ - credentialWiring?: CredentialWiring; - /** - * The parent child's mail part reader (CL-6448), so a body step's - * attachments-only inbound mail resolves its parts the way a top-level - * step's does instead of throwing for want of a reader. - */ - mailPartReader?: MailPartReader; }, /** * Live inference-event sink for the child's agent steps. Threaded from the @@ -306,10 +280,6 @@ export type HostSpawnSuspendableChild = ( export function createInMemorySpawnSuspendableChild(opts: { bodies: ReadonlyMap; runSuspendableChild: RunSuspendableChild; - /** Threaded through verbatim to every spawn's input (CL-6448). */ - authorize?: WorkflowAuthorizeFn; - credentialWiring?: CredentialWiring; - mailPartReader?: MailPartReader; }): HostSpawnSuspendableChild { return async ( { @@ -355,13 +325,6 @@ export function createInMemorySpawnSuspendableChild(opts: { depth, maxChildSpawnDepth, ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), - ...(opts.authorize !== undefined ? { authorize: opts.authorize } : {}), - ...(opts.credentialWiring !== undefined - ? { credentialWiring: opts.credentialWiring } - : {}), - ...(opts.mailPartReader !== undefined - ? { mailPartReader: opts.mailPartReader } - : {}), }, onEvent, credentialMaterial, diff --git a/vendor/intx/workflow-host/src/adapters/step-invoker.ts b/vendor/intx/workflow-host/src/adapters/step-invoker.ts index 063f1605b3..06a09a3c3d 100644 --- a/vendor/intx/workflow-host/src/adapters/step-invoker.ts +++ b/vendor/intx/workflow-host/src/adapters/step-invoker.ts @@ -58,19 +58,20 @@ import { type AgentDefinition, type AuthorizeFn, type BaseEnv, + type SendOptions, type SendResult, } from "@intx/agent"; import { getLogger } from "@intx/log"; import { createInboundMessage, extractAddrSpec, isMessageId } from "@intx/mime"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { InboundMessage, - InferenceEvent, InferenceSource, Mail, MailPartReader, MessageAttachment, -} from "@intx/types/runtime"; -import { isMail } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { isMail } from "@intx/types/runtime-core"; import type { AuthorizeContext, StepInvokeRequest, @@ -342,8 +343,9 @@ async function invokeColdStep( } /** - * The connector router lives in `@intx/harness`, which this package does not - * depend on, so its error is recognized by `name` rather than `instanceof`. + * The reply drain reports this cause through the structural `WarmReplyDrive`, + * so the harness error arrives as a value whose class is not part of that + * contract. Match it by name. */ function describeReplyFailure(cause: unknown): string { if (cause instanceof Error && cause.name === "NoActiveConnectorThreadError") { @@ -593,7 +595,10 @@ async function sendWithAbort( }; abortListener = onAbort; req.signal.addEventListener("abort", onAbort, { once: true }); - const sendOpts = cfg.closeOnAbort ? undefined : { signal: req.signal }; + const sendOpts: SendOptions = { + ...(cfg.closeOnAbort ? {} : { signal: req.signal }), + ...(req.inference !== undefined ? { inference: req.inference } : {}), + }; agent.send(message, sendOpts).then(resolve, (cause: unknown) => { reject(cause instanceof Error ? cause : new Error(String(cause))); }); diff --git a/vendor/intx/workflow-host/src/child/action-credentials.ts b/vendor/intx/workflow-host/src/child/action-credentials.ts new file mode 100644 index 0000000000..ac4654a34f --- /dev/null +++ b/vendor/intx/workflow-host/src/child/action-credentials.ts @@ -0,0 +1,61 @@ +// The credentials capability a deployed action handler resolves its declared +// handles through, built by the harness action credentials factory over the +// run child's live material cell and the action step's grants. + +import { + createActionCredentialsFactory, + type CredentialMaterialCell, + type CredentialProviderRegistry, +} from "@intx/harness"; +import type { GrantRule } from "@intx/types/authz"; +import type { ActionCredentialsFactory } from "@intx/workflow"; + +/** + * The per-run credential inputs a build resolves a step's credentials from. + * The live material cell and the grants resolver ride in from the run child + * (per-run state); the provider registry is sidecar-static and combined in at + * the invoke boundary. Absent when no credential material was threaded, which + * leaves the step's inference reader and credential capabilities unwired. + */ +export interface SidecarStepCredentialContext { + readonly materialCell: CredentialMaterialCell; + /** + * The step's grants, resolved live by base step id. The credentials + * snapshot types them `unknown[]`; `stepGrantsResolver` casts them to + * `GrantRule[]` where Gate 2 reads them. + */ + readonly resolveStepGrants: (stepId: string) => readonly unknown[]; + readonly providers: CredentialProviderRegistry; +} + +/** + * A thunk over one step's grants in the `GrantRule` shape Gate 2 reads. The + * credentials snapshot carries grants as `unknown[]`; this is the one place + * the host casts them. + */ +export function stepGrantsResolver( + context: SidecarStepCredentialContext, + stepId: string, +): () => readonly GrantRule[] { + return () => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- resolveStepGrants returns unknown[]; Gate 2 reads GrantRule + context.resolveStepGrants(stepId) as readonly GrantRule[]; +} + +/** + * Build the action credentials factory over a run child's credential context, + * through the harness builder. The run id is not needed to find a step's + * grants: every step in a sidecar run child's credentials snapshot carries the + * same grants (the run's flat grant set, or a child run's capped set). A step + * id can repeat inside one context, for example a loop-body step and a body + * step, and the lookup still finds those grants. + */ +export function createSidecarActionCredentialsFactory( + context: SidecarStepCredentialContext, +): ActionCredentialsFactory { + return createActionCredentialsFactory({ + materialCell: context.materialCell, + resolveStepGrants: ({ stepId }) => stepGrantsResolver(context, stepId)(), + providers: context.providers, + }); +} diff --git a/vendor/intx/workflow-host/src/child/cancellation-barrier.ts b/vendor/intx/workflow-host/src/child/cancellation-barrier.ts new file mode 100644 index 0000000000..ef5912e595 --- /dev/null +++ b/vendor/intx/workflow-host/src/child/cancellation-barrier.ts @@ -0,0 +1,63 @@ +import { withRunCommitBarrier, type RepoStore } from "@intx/workflow/runtime"; + +import type { ControlChannelSender, ControlPayload } from "../ipc"; + +type PrepareCancellation = Extract< + ControlPayload, + { type: "cancel.prepare" } +>["data"]; +type CancellationCommitted = Extract< + ControlPayload, + { type: "cancel.committed" } +>["data"]; + +/** Hold the runtime's event buffer until the supervisor finishes its signed write. */ +export function createCancellationBarrier( + repoStore: RepoStore, + sender: ControlChannelSender, +) { + const pending = new Map void>(); + let closed: string | undefined; + + return { + async prepare(data: PrepareCancellation): Promise { + if (closed !== undefined) throw new Error(closed); + const committed = Promise.withResolvers(); + pending.set(data.requestId, committed.resolve); + let prepared = false; + try { + await withRunCommitBarrier({ repoStore }, data.runId, async () => { + if (closed !== undefined) throw new Error(closed); + await sender.send({ + type: "cancel.prepared", + data: { requestId: data.requestId }, + }); + prepared = true; + const error = await committed.promise; + if (error !== undefined) throw new Error(error); + }); + } catch (cause) { + if (!prepared && closed === undefined) { + await sender.send({ + type: "cancel.prepared", + data: { + requestId: data.requestId, + error: cause instanceof Error ? cause.message : String(cause), + }, + }); + } + throw cause; + } finally { + pending.delete(data.requestId); + } + }, + complete(data: CancellationCommitted): void { + pending.get(data.requestId)?.(data.error); + }, + close(reason: string): void { + closed = reason; + for (const resolve of pending.values()) resolve(reason); + pending.clear(); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/child/child-mailbox-reader.ts b/vendor/intx/workflow-host/src/child/child-mailbox-reader.ts deleted file mode 100644 index cffa670a2e..0000000000 --- a/vendor/intx/workflow-host/src/child/child-mailbox-reader.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Child-side substrate mailbox reader (INBOUND half of mailbox ownership, -// design §3b). -// -// The supervisor commits an arrived message to the deployment's workflow-run -// substrate mailbox (`mailbox/INBOX/`), then fires a `mailbox.notify` control -// frame. A step agent's supervisor-backed transport answers the IMAP read -// surface (`search`, `fetchHeaders`, `fetchFull`, ...) by opening a fresh -// committed snapshot of that mailbox through this reader. `open` re-reads the -// committed subtree on every call, so a snapshot taken after a `mailbox.notify` -// observes the message the supervisor just committed. -// -// Modeled on the sibling `createMailPartReader` wiring: the same per-deployment -// substrate handles (substrate, repoId, principal, workflow-run ref), bound -// once so the transport reads without re-threading them. `createSubstrateMailboxStore` -// loads committed state on open and is async, so `open` returns a promise. - -import { - createSubstrateMailboxStore, - type SubstrateMailboxStore, - type SubstrateMailboxStoreOpts, -} from "../adapters/substrate-mailbox-store"; - -export interface ChildMailboxReader { - /** - * Open a fresh committed snapshot of the deployment's substrate INBOX. Each - * call re-reads the committed `mailbox/INBOX/` subtree, so a snapshot opened - * after a `mailbox.notify` observes the newly committed message. - */ - open(): Promise; -} - -export function createChildMailboxReader( - opts: SubstrateMailboxStoreOpts, -): ChildMailboxReader { - return { - open() { - return createSubstrateMailboxStore(opts); - }, - }; -} diff --git a/vendor/intx/workflow-host/src/child/credential-cell.ts b/vendor/intx/workflow-host/src/child/credential-cell.ts index 9662f6286f..8506c2bd2b 100644 --- a/vendor/intx/workflow-host/src/child/credential-cell.ts +++ b/vendor/intx/workflow-host/src/child/credential-cell.ts @@ -1,14 +1,15 @@ // Merge semantics for the child's in-memory credential-material cell. // // The cell is fed by several INDEPENDENTLY-SCOPED producers: the deploy frame -// (tool bindings plus the run's inference materials, frozen at deploy time and -// re-asserted on every spawn and pre-trigger barrier), an inference rotation -// (materials only, no binding), and a tool-grant push (tool bindings plus their -// materials). Each producer carries only its own slice of the cell. A wholesale -// swap would therefore let one producer evict another's credentials -- an -// inference rotation would drop every tool binding, a barrier re-push would drop -// a live-pushed tool credential. So a `credentials-updated` frame MERGES: -// materials upsert by `credentialId`, bindings upsert by `(consumer, handle)`. +// (credential bindings plus the run's inference materials, frozen at deploy +// time and re-asserted on every spawn and pre-trigger barrier), an inference +// rotation (materials only, no binding), and a credential-grant push +// (credential bindings plus their materials). Each producer carries only its +// own slice of the cell. A wholesale swap would therefore let one producer +// evict another's credentials -- an inference rotation would drop every +// credential binding, a barrier re-push would drop a live-pushed binding +// credential. So a `credentials-updated` frame MERGES: materials upsert by +// `credentialId`, bindings upsert by `(consumer, handle)`. // // Because omission no longer means "gone", revocation is EXPLICIT: a frame's // `revoke` list names the credentialIds to drop, and dropping one also drops @@ -23,7 +24,7 @@ import type { CredentialBindingDescriptor, CredentialDelivery, CredentialMaterialEntry, -} from "@intx/types/sidecar"; +} from "@intx/types/credential-delivery"; // A binding's identity is the (consumer, handle) pair: a handle string is only // unique within a consumer, so two consumers can each bind their own handle of diff --git a/vendor/intx/workflow-host/src/child/credentials-backed-authorize.ts b/vendor/intx/workflow-host/src/child/credentials-backed-authorize.ts new file mode 100644 index 0000000000..300cac987b --- /dev/null +++ b/vendor/intx/workflow-host/src/child/credentials-backed-authorize.ts @@ -0,0 +1,142 @@ +// The workflow-level authorize closure a run child gives its steps, and the +// grant evaluator it delegates to. +// +// The closure resolves a step's grants from the run's credentials snapshot by +// base step id, and the evaluator ranks those grants (plus any pinned-tool +// floor) with `evaluateGrants`. Nothing here imports a Node API, so a host +// that cannot load the run child can still build the same pair. + +import { evaluateGrants, type GrantRule } from "@intx/authz"; +import type { AuthzCallResult } from "@intx/inference"; +import type { WorkflowAuthorizeFn } from "@intx/workflow"; +import { baseStepId } from "@intx/workflow"; + +import type { CredentialsSnapshot } from "../supervisor/credentials"; + +/** + * The active credentials snapshot, held through a mutable reference: the + * authorize closure reads it on every call, so a `grants-updated` control + * frame swaps the snapshot in place without the caller having to reconstruct + * the closure. + */ +export type CredentialsSnapshotRef = { + current: CredentialsSnapshot | null; +}; + +/** + * Evaluates a `(resource, action)` request against one step's grants. The + * slot exists so the host wires its own grant-rule semantics; tests inject a + * spy that records inputs. + */ +export type GrantEvaluator = (input: { + resource: string; + action: string; + stepId: string; + attempt: number | undefined; + runId: string | undefined; + grants: readonly unknown[]; +}) => Promise; + +/** + * The grants of `stepId` in `snapshot`. The snapshot is keyed per base step, + * so a map iteration's scoped id `[]` resolves to its base entry + * and every iteration shares the base step's grants; `baseStepId` is the + * identity on an unscoped id. Throws the caller's `noEntry` message when the + * snapshot has no entry for the step. + */ +export function resolveSnapshotStepGrants( + snapshot: CredentialsSnapshot, + stepId: string, + noEntry: (lookupStepId: string) => string, +): readonly unknown[] { + const lookupStepId = baseStepId(stepId); + const entry = snapshot.steps.find((s) => s.stepId === lookupStepId); + if (entry === undefined) { + throw new Error(noEntry(lookupStepId)); + } + return entry.grants; +} + +/** + * Construct the workflow-level authorize closure backed by a mutable + * credentials snapshot reference. The closure evaluates a + * `(resource, action)` request against the snapshot entry for the + * originating step, looked up by base step id, through the grant evaluator. + * + * The workflow runtime supplies `stepId` from the `AuthorizeContext` it + * threads through every step's invoker, so a call without a step id, or + * before any snapshot is active, is a programming error. The closure throws + * loudly rather than deciding. + */ +export function createCredentialsBackedAuthorize( + ref: CredentialsSnapshotRef, + evaluate: GrantEvaluator, +): WorkflowAuthorizeFn { + return async (resource, action, ctx) => { + const stepId = ctx?.stepId; + if (stepId === undefined) { + throw new Error( + "credentials-backed authorize: missing stepId in AuthorizeContext; the runtime body must thread it through every step invocation", + ); + } + const snapshot = ref.current; + if (snapshot === null) { + throw new Error( + "credentials-backed authorize: no credentials snapshot is active; the host must set one before any step runs", + ); + } + const grants = resolveSnapshotStepGrants( + snapshot, + stepId, + (lookupStepId) => + `credentials-backed authorize: the credentials snapshot has no entry for step ${lookupStepId}${ + lookupStepId === stepId + ? "" + : ` (normalized from scoped invocation id ${stepId})` + }`, + ); + return evaluate({ + resource, + action, + stepId, + attempt: ctx?.attempt, + runId: ctx?.runId, + grants, + }); + }; +} + +/** + * The grant evaluator a run child's authorize closure delegates to. It ranks + * the step's snapshot grants together with the step's pinned-tool floor + * (`floorFor`, keyed by base step id) through `evaluateGrants`. + * + * The floor supplies the `tool:` authority a pinned tool never got from + * the hub's capability walk. It is ADDITIVE: `evaluateGrants` ranks by + * specificity then effect, so a declared `deny` (priority 2) still beats the + * derived `ask`/`allow` and an explicit denial is honored -- the floor only + * raises the minimum authority to the tool's static mark. A step with no floor + * contributes no rows, which can only fail more closed. A source-ref step has + * no floor: its `tool:` grant is already in the snapshot. + */ +export function createGrantEvaluator(opts: { + readonly floorFor: (baseStepId: string) => readonly GrantRule[]; +}): GrantEvaluator { + return async ({ resource, action, stepId, grants }) => { + const floor = opts.floorFor(baseStepId(stepId)); + const result = await evaluateGrants( + // The credentials snapshot types `grants` as `readonly unknown[]`. + // This module casts at the boundary where `evaluateGrants` reads the + // rows. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- credentialsSnapshot.steps[*].grants is typed unknown[]; evaluateGrants reads GrantRule + [...(grants as readonly GrantRule[]), ...floor], + resource, + action, + ); + return { + effect: result.effect, + matchingGrants: [], + resolvedBy: null, + }; + }; +} diff --git a/vendor/intx/workflow-host/src/child/env-bootstrap.ts b/vendor/intx/workflow-host/src/child/env-bootstrap.ts index 5c13db279f..57b84283f1 100644 --- a/vendor/intx/workflow-host/src/child/env-bootstrap.ts +++ b/vendor/intx/workflow-host/src/child/env-bootstrap.ts @@ -15,8 +15,8 @@ import { type } from "arktype"; -import { hexDecode } from "@intx/types"; -import { IPC_CRYPTO } from "../ipc/index"; +import { hexDecode } from "@intx/types/hex"; +import { IPC_CRYPTO } from "../ipc/crypto"; /** * The required spawn-time env keys, named once so the supervisor-side diff --git a/vendor/intx/workflow-host/src/child/from-process-env.ts b/vendor/intx/workflow-host/src/child/from-process-env.ts index efb2d26a8c..03719a8e03 100644 --- a/vendor/intx/workflow-host/src/child/from-process-env.ts +++ b/vendor/intx/workflow-host/src/child/from-process-env.ts @@ -39,12 +39,16 @@ import { createChildMailboxMutationBridge, type ChildMailboxMutationBridge, } from "./mailbox-mutation-bridge"; +import { + createChildMailboxCallBridge, + type ChildMailboxCallBridge, +} from "./mailbox-call-bridge"; import { createControlChannelSender, - type FrameWriter, type NdjsonReader, type NdjsonWriter, -} from "../ipc/index"; +} from "../ipc/control-sender"; +import type { FrameWriter } from "../ipc/event-sender"; /** * File descriptor the supervisor's `Bun.spawn` wires the @@ -118,6 +122,13 @@ export interface SubstrateFactoryEnv { * owned store, so the child never flushes the run ref itself. */ readonly mailboxMutationBridge: ChildMailboxMutationBridge; + /** + * Child-side IPC bridge for every mailbox method that is not `send` and + * not a flag or expunge mutation. The factory attaches it to the warm + * agent's transport. A build without an inbound surface does not receive + * it, and those methods fail as unwired. + */ + readonly mailboxCallBridge: ChildMailboxCallBridge; } /** @@ -217,12 +228,16 @@ export async function runWorkflowChildFromProcessEnv( const mailboxMutationBridge = createChildMailboxMutationBridge({ upstreamSender, }); + const mailboxCallBridge = createChildMailboxCallBridge({ + upstreamSender, + }); const bindings = await factory({ spawn, substrateConfig, substrateWriteBridge, outboundMailBridge, mailboxMutationBridge, + mailboxCallBridge, }); return runWorkflowChild({ env: spawn, @@ -241,6 +256,7 @@ export async function runWorkflowChildFromProcessEnv( substrateWriteBridge, outboundMailBridge, mailboxMutationBridge, + mailboxCallBridge, }); } diff --git a/vendor/intx/workflow-host/src/child/index.ts b/vendor/intx/workflow-host/src/child/index.ts index 7c38ac0c13..032145f236 100644 --- a/vendor/intx/workflow-host/src/child/index.ts +++ b/vendor/intx/workflow-host/src/child/index.ts @@ -1,12 +1,16 @@ export { createCredentialsBackedAuthorize, + createGrantEvaluator, + type CredentialsSnapshotRef, + type GrantEvaluator, +} from "./credentials-backed-authorize"; + +export { hashGrants, runWorkflowChild, type ChildStepInvoker, - type CredentialsSnapshotRef, type CredentialWiring, type DrainController, - type GrantEvaluator, type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, @@ -35,6 +39,14 @@ export { type MailboxMutationResult, } from "./mailbox-mutation-bridge"; +export { + createChildMailboxCallBridge, + type ChildMailboxCallBridge, + type CreateChildMailboxCallBridgeOpts, + type MailboxCall, + type MailboxCallSuccess, +} from "./mailbox-call-bridge"; + export { createSupervisorBackedTransport, type SupervisorBackedTransportInbound, @@ -45,11 +57,6 @@ export { type MailboxWatchRegistry, } from "./mailbox-watch-registry"; -export { - createChildMailboxReader, - type ChildMailboxReader, -} from "./child-mailbox-reader"; - export { createProxyWorkflowRunRepoStore, type CreateProxyWorkflowRunRepoStoreOpts, diff --git a/vendor/intx/workflow-host/src/child/mailbox-call-bridge.ts b/vendor/intx/workflow-host/src/child/mailbox-call-bridge.ts new file mode 100644 index 0000000000..be1aa1becd --- /dev/null +++ b/vendor/intx/workflow-host/src/child/mailbox-call-bridge.ts @@ -0,0 +1,130 @@ +// Child-side mailbox-call bridge. +// +// The supervisor owns the deployment mailbox. Every read, listing, append, +// move, copy, watch acceptance, and distribution-list call goes up through +// this bridge and comes back as the supervisor's answer, including a refusal. +// The child does not open the mailbox to produce that answer. +// +// Lifecycle of one call: +// +// 1. A mail tool calls a method on the supervisor-backed transport. The +// transport calls `bridge.submit(call)`. +// 2. `submit` mints a `requestId`, registers a pending awaiter, and emits +// `mailbox.call.request` upstream. `op` discriminates the operands. +// 3. The supervisor answers with `mailbox.call.response`, echoing `op`. +// 4. The bridge resolves the success, or rejects. A response that carries +// `condition` rejects with that `MessageTransportError`. A failure with +// no condition stays a plain error: the supervisor had no IMAP condition +// to name (a missing uid, a bad date, a bad part ref). + +import { getLogger } from "@intx/log"; + +import { MessageTransportError } from "@intx/types/runtime-core"; + +import type { + ControlChannelSender, + ControlPayload, +} from "../ipc/control-channel"; + +import { createPendingRequestCore } from "./pending-request"; + +const logger = getLogger(["workflow-host", "child", "mailbox-call-bridge"]); + +type MailboxCallRequest = Extract< + ControlPayload, + { type: "mailbox.call.request" } +>["data"]; + +type MailboxCallResponse = Extract< + ControlPayload, + { type: "mailbox.call.response" } +>["data"]; + +export type MailboxCallSuccess = Extract; + +/** + * One mailbox call without the `requestId` the bridge mints. The operands + * are the request frame's, so a field the wire does not carry cannot be + * submitted. + */ +export type MailboxCall = MailboxCallRequest extends infer Request + ? Request extends { requestId: string } + ? Omit + : never + : never; + +/** + * Bridge surface the child's supervisor-backed transport reaches into. + * `submit` sends a `mailbox.call.request` and resolves with the supervisor's + * success. `handleResult` is what the control loop calls when + * `mailbox.call.response` arrives. `cancelAll` rejects every still-pending + * call when the control loop exits. + */ +export interface ChildMailboxCallBridge { + submit(call: MailboxCall): Promise; + handleResult(data: MailboxCallResponse): void; + cancelAll(reason: string): void; + readonly pendingCount: number; +} + +export interface CreateChildMailboxCallBridgeOpts { + upstreamSender: ControlChannelSender; + /** + * Optional `requestId` allocator. Production wires a per-instance + * monotonic counter plus a random suffix; tests inject a deterministic + * factory so the upstream frame's `requestId` is predictable. + */ + allocateRequestId?: () => string; +} + +/** + * Construct the child-side mailbox-call bridge. Pending calls live in the + * shared pending-request core keyed by `requestId`. + */ +export function createChildMailboxCallBridge( + opts: CreateChildMailboxCallBridgeOpts, +): ChildMailboxCallBridge { + const pending = createPendingRequestCore({ + label: "workflow-child mailbox call", + allocatorPrefix: "mc", + allocateRequestId: opts.allocateRequestId, + }); + + return { + get pendingCount() { + return pending.pendingCount; + }, + async submit(call: MailboxCall): Promise { + const { requestId, promise } = pending.register(undefined); + try { + await opts.upstreamSender.send({ + type: "mailbox.call.request", + data: { ...call, requestId }, + }); + } catch (cause) { + pending.discard(requestId); + throw pending.sendFailedError(requestId, cause); + } + return promise; + }, + handleResult(data) { + const entry = pending.settle(data.requestId); + if (entry === undefined) { + logger.warn`mailbox.call.response landed with no pending entry; requestId=${data.requestId} dropped`; + return; + } + if (!data.ok) { + if (data.condition !== undefined) { + entry.reject(new MessageTransportError(data.condition, data.reason)); + return; + } + entry.reject(pending.rejectedError(data.requestId, data.reason)); + return; + } + entry.resolve(data); + }, + cancelAll(reason: string) { + pending.cancelAll(reason); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/child/mailbox-mutation-bridge.ts b/vendor/intx/workflow-host/src/child/mailbox-mutation-bridge.ts index 0f83e62209..7658523b95 100644 --- a/vendor/intx/workflow-host/src/child/mailbox-mutation-bridge.ts +++ b/vendor/intx/workflow-host/src/child/mailbox-mutation-bridge.ts @@ -3,11 +3,9 @@ // // The supervisor is the sole mail owner: it holds the long-lived // substrate mailbox store and is the only writer to the workflow-run -// ref. A step agent reads its INBOX locally (the supervisor-backed -// transport's read surface opens fresh committed snapshots), but every -// MUTATION of the mailbox -- flag writes (`\Seen`, `\Deleted`, ...) and -// `expunge` -- routes up to the supervisor through this bridge rather -// than being flushed from the child. A second writer flushing the same +// ref. Flag writes (`\Seen`, `\Deleted`, ...) and `expunge` route up +// to the supervisor through this bridge rather than being flushed from +// the child. A second writer flushing the same // ref from the child would race the supervisor's in-memory mirror and // break uid / modseq monotonicity, so the child never writes the // mailbox directly. @@ -22,8 +20,8 @@ // operands. // 3. The supervisor applies the op to its owned mailbox store, // flushes, and replies with `mailbox.mutate.response`. The reply is -// sent only after the flush, so the child's next committed read -// observes the mutation (the same flush-before-signal ordering +// sent only after the flush, so a later read of the committed +// mailbox observes the mutation (the same flush-before-signal ordering // `mailbox.notify` relies on). // 4. The bridge resolves / rejects the pending awaiter; the // transport method returns to the mail tool. A supervisor-side @@ -33,6 +31,8 @@ import { getLogger } from "@intx/log"; +import { MessageTransportError } from "@intx/types/runtime-core"; + import type { ControlChannelSender, ControlPayload, @@ -163,6 +163,12 @@ export function createChildMailboxMutationBridge( entry.resolve(result); return; } + if (data.result.condition !== undefined) { + entry.reject( + new MessageTransportError(data.result.condition, data.result.reason), + ); + return; + } entry.reject(pending.rejectedError(data.requestId, data.result.reason)); }, cancelAll(reason: string) { diff --git a/vendor/intx/workflow-host/src/child/outbound-mail-bridge.ts b/vendor/intx/workflow-host/src/child/outbound-mail-bridge.ts index 8f6e916134..8f99787da5 100644 --- a/vendor/intx/workflow-host/src/child/outbound-mail-bridge.ts +++ b/vendor/intx/workflow-host/src/child/outbound-mail-bridge.ts @@ -33,7 +33,7 @@ import { getLogger } from "@intx/log"; -import { base64Encode } from "@intx/types"; +import { base64Encode } from "@intx/types/base64"; import type { OutboundMessage, SendReceipt } from "@intx/types/runtime"; import type { @@ -57,7 +57,11 @@ const logger = getLogger(["workflow-host", "child", "outbound-mail-bridge"]); * down. */ export interface ChildOutboundMailBridge { - submit(senderAddress: string, message: OutboundMessage): Promise; + submit( + senderAddress: string, + message: OutboundMessage, + opts?: { readonly completeReferences: boolean }, + ): Promise; handleResult( data: Extract["data"], ): void; @@ -98,6 +102,7 @@ export function createChildOutboundMailBridge( async submit( senderAddress: string, message: OutboundMessage, + submitOpts?: { readonly completeReferences: boolean }, ): Promise { const { requestId, promise } = pending.register(undefined); try { @@ -106,6 +111,11 @@ export function createChildOutboundMailBridge( data: { requestId, senderAddress, + // Only `true` is a request to complete References. A false + // or omitted flag is the same send the supervisor leaves alone. + ...(submitOpts?.completeReferences === true + ? { completeReferences: true } + : {}), message: projectOutboundMessage(message), }, }); diff --git a/vendor/intx/workflow-host/src/child/parked-correlations.ts b/vendor/intx/workflow-host/src/child/parked-correlations.ts index 3aca54ee85..cec521cbd6 100644 --- a/vendor/intx/workflow-host/src/child/parked-correlations.ts +++ b/vendor/intx/workflow-host/src/child/parked-correlations.ts @@ -11,6 +11,12 @@ // durable snapshot by construction; a snapshot-less enumerated step is a // disagreement between the log and the step store, which this module surfaces // loudly rather than dropping. +// +// A relay is the exception to "the snapshot is on this step". A loop or +// `childWorkflow` parks the container on the body's correlation, and the +// snapshot stays in the body's step store. The container's own store has +// none. Enumeration follows the spawned child that is still awaiting that +// correlation, including a child that relays it onward. import type { RepoId, @@ -18,10 +24,12 @@ import type { } from "@intx/hub-sessions/substrate"; import { controlParkKindOf, + resumeFromLog, type RepoStore as RuntimeRepoStore, + type RunState, } from "@intx/workflow"; import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime"; -import { correlationIdFromSignalName } from "@intx/types"; +import { correlationIdFromSignalName, signalName } from "@intx/types/signals"; import { discoverInFlightRuns } from "./self-discovery"; @@ -66,6 +74,53 @@ export interface CollectParkedApprovalCorrelationsOpts { * disagreements between the reduced state and the durable store that must not * silently drop a correlation the hub is waiting to register. */ +/** + * The snapshot for a relayed approval. The container's store has none; the + * body step that is still awaiting the same correlation has it. A nested + * relay is followed the same way. A run is read once. + */ +async function loadRelayedApprovalSnapshot( + opts: CollectParkedApprovalCorrelationsOpts, + load: LoadParkedApproval, + state: RunState, + stepId: string, + correlationId: string, + seen: Set, +): Promise { + const name = signalName(correlationId); + for (const child of state.children.values()) { + if (child.spawnedBy !== stepId || child.terminalStatus !== undefined) { + continue; + } + if (seen.has(child.childRunId)) continue; + seen.add(child.childRunId); + const events = [...(await opts.runtimeRepoStore.read(child.childRunId))]; + if (events.length === 0) continue; + const childState = resumeFromLog(child.childRunId, events); + for (const childStep of childState.steps.values()) { + if (childStep.phase !== "awaiting-signal") continue; + if (childStep.awaitingSignal?.name !== name) continue; + const direct = await load({ + runId: child.childRunId, + stepId: childStep.stepId, + attempt: childStep.currentAttempt, + correlationId, + }); + if (direct !== undefined) return direct; + const nested = await loadRelayedApprovalSnapshot( + opts, + load, + childState, + childStep.stepId, + correlationId, + seen, + ); + if (nested !== undefined) return nested; + } + } + return undefined; +} + export async function collectParkedApprovalCorrelations( opts: CollectParkedApprovalCorrelationsOpts, ): Promise { @@ -104,12 +159,22 @@ export async function collectParkedApprovalCorrelations( `workflow-child parked-correlations: run ${run.runId} step ${step.stepId} is parked on control-plane correlation ${correlationId}, but no loadParkedApproval binding is wired to recover its snapshot`, ); } - const snapshot = await opts.loadParkedApproval({ + const direct = await opts.loadParkedApproval({ runId: run.runId, stepId: step.stepId, attempt: step.currentAttempt, correlationId, }); + const snapshot = + direct ?? + (await loadRelayedApprovalSnapshot( + opts, + opts.loadParkedApproval, + run.resumedState, + step.stepId, + correlationId, + new Set(), + )); if (snapshot === undefined) { throw new Error( `workflow-child parked-correlations: reduced state shows run ${run.runId} step ${step.stepId} awaiting control-plane correlation ${correlationId} (attempt ${String(step.currentAttempt)}), but durable storage carries no approval snapshot for it; the run log and the step store disagree`, diff --git a/vendor/intx/workflow-host/src/child/pending-request.ts b/vendor/intx/workflow-host/src/child/pending-request.ts index 371390093a..4e56cbcbda 100644 --- a/vendor/intx/workflow-host/src/child/pending-request.ts +++ b/vendor/intx/workflow-host/src/child/pending-request.ts @@ -1,10 +1,10 @@ -// Generic pending-request core shared by the three child-side control-IPC -// bridges (substrate-write, outbound-mail, mailbox-mutation). +// Generic pending-request core shared by the child-side control-IPC +// bridges (substrate-write, outbound-mail, mailbox-mutation, mailbox-call). // // Every bridge runs the same request/response round-trip over the control // channel: `submit` mints a `requestId`, registers a pending awaiter keyed // by that id, emits a request frame upstream, and resolves/rejects the -// awaiter when the supervisor's matching response frame lands. The three +// awaiter when the supervisor's matching response frame lands. The // bridges differ only in the request payload they send, the value they // resolve, the label their error messages carry, and (for the // substrate-write bridge) an intermediate merge round-trip that must peek @@ -166,7 +166,7 @@ export function createPendingRequestCore( * Default `requestId` allocator: a per-instance monotonic counter plus a * random suffix, so ids are unique across bridge instances without any * cross-instance coordination. The `prefix` names the owning bridge - * (`sw-`, `om-`, `mm-`) for triage in supervisor logs. + * (`sw-`, `om-`, `mm-`, `mc-`) for triage in supervisor logs. */ export function defaultRequestIdAllocator(prefix: string): () => string { let counter = 0; diff --git a/vendor/intx/workflow-host/src/child/program.ts b/vendor/intx/workflow-host/src/child/program.ts new file mode 100644 index 0000000000..64145874c9 --- /dev/null +++ b/vendor/intx/workflow-host/src/child/program.ts @@ -0,0 +1,16 @@ +// @intx/workflow-host/child -- the child process entry and substrate factory. +// +// Boots the workflow child from its process environment and builds its substrate. +// The host supplies tool materialization and the child-grant cap. +// The package barrel does not load this entry. + +export { + SIDECAR_SUBSTRATE_CONFIG_KEYS, + createSidecarSubstrateFactory, +} from "./substrate-factory"; +export type { + StepToolCacheConfig, + StepToolMaterialization, +} from "./step-tools"; + +export { runWorkflowChildFromProcessEnv } from "./from-process-env"; diff --git a/vendor/intx/workflow-host/src/child/run-child.ts b/vendor/intx/workflow-host/src/child/run-child.ts index 92eb80358f..f51ad27862 100644 --- a/vendor/intx/workflow-host/src/child/run-child.ts +++ b/vendor/intx/workflow-host/src/child/run-child.ts @@ -1,6 +1,6 @@ // `runWorkflowChild` -- the workflow-process child's runtime body. // -// The package-owned binary at `packages/workflow-host/bin/workflow-child` +// The sidecar binary at `apps/sidecar/bin/workflow-child` // is a thin wrapper that parses `process.env`, opens stdin/stdout for // the control channel, accepts the inherited event-channel fd, builds // the substrate `RepoStore`, and invokes this function. Tests bypass @@ -51,7 +51,7 @@ import { getLogger } from "@intx/log"; import { generateKeyPair } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; import type { Principal, @@ -64,12 +64,15 @@ import { rewriteInlineChildWorkflowBodies, enumerateInlineLoopBodies, eagerlyResolveLoopFns, + walkWorkflowSteps, + LOOP_BODY_DESCENT, } from "@intx/workflow"; -import type { AuthzCallResult } from "@intx/inference"; import type { + ActionCredentialsFactory, ActionHandler, RunResult, + RuntimeWorkflowRun, Scheduler, ReadParkedApprovalOps, StepInvokeRequest, @@ -81,11 +84,9 @@ import type { WorkflowAuthorizeFn, WorkflowDefinition, WorkflowPark, - WorkflowRun, WorkflowRuntimeEnv, } from "@intx/workflow"; import { - baseStepId, createDefaultActionInvoker, createInMemoryEffectLedger, createLoopIterationHandle, @@ -99,11 +100,11 @@ import { } from "../drain-controller"; import type { InferenceSource, MailPartReader } from "@intx/types/runtime"; -import type { CredentialDelivery } from "@intx/types/sidecar"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; import { createWorkflowRunRepoStore } from "../adapters/repo-store"; +import { createCancellationBarrier } from "./cancellation-barrier"; import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate"; -import { createMailPartReader } from "../adapters/mail-part-store"; import type { HostSpawnSuspendableChild, HostSpawnChild, @@ -116,15 +117,17 @@ import { } from "../adapters/spawn-child"; import { createControlChannelSender, - createEventChannelSender, - receiveControlChannel, type ControlChannelSender, - type ControlPayload, - type EventPayload, - type FrameWriter, type NdjsonReader, type NdjsonWriter, -} from "../ipc/index"; +} from "../ipc/control-sender"; +import { receiveControlChannel } from "../ipc/control-receiver"; +import type { ControlPayload } from "../ipc/control-payloads"; +import { + createEventChannelSender, + type FrameWriter, +} from "../ipc/event-sender"; +import type { EventPayload } from "../ipc/event-channel"; import { runBodyThenCleanup } from "../run-body-then-cleanup"; import { createWorkflowHostSignalChannel } from "../seams/signal-channel"; import type { CredentialsSnapshot } from "../supervisor/credentials"; @@ -143,31 +146,21 @@ import { type LoadParkedApproval, } from "./parked-correlations"; import type { ChildOutboundMailBridge } from "./outbound-mail-bridge"; +import type { ChildMailboxCallBridge } from "./mailbox-call-bridge"; +import { createSupervisorBackedMailPartReader } from "./supervisor-backed-mail-part-reader"; import type { ChildMailboxMutationBridge } from "./mailbox-mutation-bridge"; import type { MailboxWatchRegistry } from "./mailbox-watch-registry"; import { createWarmAgentCache, type WarmAgentCache } from "./warm-agent-cache"; import { mergeCredentialDelivery } from "./credential-cell"; +import { + createCredentialsBackedAuthorize, + resolveSnapshotStepGrants, + type CredentialsSnapshotRef, + type GrantEvaluator, +} from "./credentials-backed-authorize"; const logger = getLogger(["workflow-host", "child"]); -/** - * `WorkflowAuthorize` closure factory shape. The child's authorize - * evaluates a `(resource, action)` request against the active - * credentialsSnapshot for the originating step. The closure used here - * is intentionally permissive on missing context: the workflow runtime - * supplies `stepId` from the `AuthorizeContext` it threads through - * every step's invoker, so a bare-call without a step id is a - * programming error rather than a security-sensitive path. The closure - * surfaces it loudly. - * - * Read-site: the closure consults a mutable reference so a - * `grants-updated` control frame swaps the snapshot in place without - * the caller having to reconstruct the closure. - */ -export type CredentialsSnapshotRef = { - current: CredentialsSnapshot | null; -}; - /** * The deployment's decrypted credential material and per-handle descriptors, * held through a mutable reference. A `credentials-updated` frame MERGES into @@ -194,89 +187,6 @@ export type SourcesSnapshotRef = { current: Record; }; -/** - * Construct the workflow-level authorize closure backed by a - * mutable credentialsSnapshot reference. - * - * The closure looks up the step's grants by `stepId`, then delegates - * to the caller-supplied grant evaluator. The evaluator slot exists so - * the host wires its own grant-rule semantics without leaking the - * grant-rule grammar into the workflow-host package; tests inject a - * spy that records inputs. - */ -export type GrantEvaluator = (input: { - resource: string; - action: string; - stepId: string; - attempt: number | undefined; - runId: string | undefined; - grants: readonly unknown[]; -}) => Promise; - -/** - * CL-6448: the credentials snapshot is keyed by the PARENT step's own - * stepOrder, so an `onTrigger` body's step id (e.g. a section's inline - * `reply` step) never appears in it verbatim. For a single-step - * deployment the sole entry IS the deployment's whole grant set, so an - * unmatched lookup collapses to it — mirroring the head/step collapse - * the body's tool materialization already uses. A multi-step snapshot - * stays strict: an unmatched id there is genuinely ambiguous (which of - * several sibling steps' grants would apply?), so it falls through to - * the loud miss below instead of guessing. - */ -function findStepGrantsEntry( - steps: CredentialsSnapshot["steps"], - stepId: string, -): CredentialsSnapshot["steps"][number] | undefined { - const direct = steps.find((s) => s.stepId === stepId); - if (direct !== undefined) return direct; - if (steps.length === 1) return steps[0]; - return undefined; -} - -export function createCredentialsBackedAuthorize( - ref: CredentialsSnapshotRef, - evaluate: GrantEvaluator, -): WorkflowAuthorizeFn { - return async (resource, action, ctx) => { - const stepId = ctx?.stepId; - if (stepId === undefined) { - throw new Error( - "workflow-child authorize: missing stepId in AuthorizeContext; the runtime body must thread it through every step invocation", - ); - } - const snapshot = ref.current; - if (snapshot === null) { - throw new Error( - "workflow-child authorize: no credentialsSnapshot active; the supervisor must push one before any step runs", - ); - } - // The credentials snapshot is keyed per base step; a map iteration's - // scoped id `[]` resolves to its base entry so every - // iteration shares the base step's grants. `baseStepId` is the identity - // on an unscoped id, so a plain step is unaffected. - const lookupStepId = baseStepId(stepId); - const entry = findStepGrantsEntry(snapshot.steps, lookupStepId); - if (entry === undefined) { - const scopedNote = - lookupStepId === stepId - ? "" - : ` (normalized from scoped invocation id ${stepId})`; - throw new Error( - `workflow-child authorize: credentialsSnapshot has no entry for stepId ${lookupStepId}${scopedNote}`, - ); - } - return evaluate({ - resource, - action, - stepId, - attempt: ctx?.attempt, - runId: ctx?.runId, - grants: entry.grants, - }); - }; -} - /** * The workflow-host child's drain controller is the production * implementation defined in `../drain-controller.ts`. The control-loop @@ -313,11 +223,12 @@ export type DrainController = WorkflowHostDrainController; * only reads it through to the adapter. */ /** - * Per-run credential inputs the top-level step invoker carries to the - * substrate: the live material cell the control channel writes each delivery - * into, and a resolver for a step's grants (which the substrate gates + * Per-run credential inputs the top-level step and action invokers carry to + * the substrate: the live material cell the control channel writes each + * delivery into, and a resolver for a step's grants (which the substrate gates * credential use against). The substrate combines these with its own static - * provider registry to assemble each tool bundle's `credentials` capability. + * provider registry to assemble each tool bundle's and each action handler's + * `credentials` capability. * * Grants are typed `readonly unknown[]` here: this package owns no grant * grammar (the credentials snapshot's grants are `unknown[]` throughout), so @@ -370,6 +281,13 @@ export interface RunWorkflowChildBindings { * with the host's per-step env builder; tests inject a stub. */ invokeStep: ChildStepInvoker; + /** + * Builds the action credentials factory for a run from the run's credential + * wiring. The host combines the wiring with its provider registry, so an + * action handler resolves its declared handles against the run's live + * material and the action step's grants. + */ + actionCredentials: (wiring: CredentialWiring) => ActionCredentialsFactory; /** * Terminal child-spawn callback the runtime body invokes for a * `childWorkflow` primitive when the deployment embeds NO inline child @@ -494,11 +412,12 @@ export interface RunWorkflowChildBindings { */ initialSources?: Record; /** - * Bootstrap credential material for the deployment's tools, decrypted - * hub-side and delivered on the deploy frame so it is resident before any - * step runs. Seeds the mutable `credentialMaterialRef` the gated capability - * reads. Absent when the deployment binds no credentials; a later - * `credentials-updated` control frame refreshes it on rotation. + * Bootstrap credential material for the deployment's binding consumers and + * inference sources, decrypted hub-side and delivered on the deploy frame so + * it is resident before any step runs. Seeds the mutable + * `credentialMaterialRef` the gated capability reads. Absent when the + * deployment binds no credentials; a later `credentials-updated` control + * frame refreshes it on rotation. */ initialCredentialMaterial?: CredentialDelivery; /** @@ -593,6 +512,14 @@ export interface RunWorkflowChildOpts { * side asked for a mutation. */ mailboxMutationBridge?: ChildMailboxMutationBridge; + /** + * Optional mailbox-call bridge. The step agent's transport forwards every + * mailbox method that is not `send` and not a flag or expunge mutation + * through this bridge. The control loop routes `mailbox.call.response` + * to `handleResult` and `cancelAll`s it on exit. When omitted, a response + * is logged and dropped. + */ + mailboxCallBridge?: ChildMailboxCallBridge; /** * Optional mailbox watch registry (INBOUND half of mailbox ownership, * design §3b). The supervisor -- the sole mail owner -- commits an arrived @@ -667,18 +594,15 @@ export async function runWorkflowChild( const snapshot = credentialsRef.current; if (snapshot === null) { throw new Error( - `workflow-child credential wiring: no credentials snapshot for step ${stepId}; a tool-bearing step cannot resolve its grants before the run carries any`, + `workflow-child credential wiring: no credentials snapshot for step ${stepId}; a step that uses credentials cannot resolve its grants before the run carries any`, ); } - const entry = snapshot.steps.find( - (step) => step.stepId === baseStepId(stepId), + return resolveSnapshotStepGrants( + snapshot, + stepId, + (lookupStepId) => + `workflow-child credential wiring: credentials snapshot has no entry for step ${lookupStepId}`, ); - if (entry === undefined) { - throw new Error( - `workflow-child credential wiring: credentials snapshot has no entry for step ${baseStepId(stepId)}`, - ); - } - return entry.grants; }, }; const clock = opts.bindings.clock ?? defaultClock; @@ -804,11 +728,13 @@ export async function runWorkflowChild( ); // Action handlers resolve from the pinned closure's `interchange.actions` - // module, on the same terms as loop fns. Resolve every action handler ref - // reachable from the definition eagerly here (recursing into loop bodies, - // where an action body is the common case), so a deployment that declares an - // action whose handler the closure does not export fails at establish rather - // than mid-run. + // module, on the same terms as loop fns. This call walks the rewritten + // top-level definition plus each lifted onTrigger and childWorkflow body; + // loop bodies share the top-level env and are walked with it. The walk is + // `LOOP_BODY_DESCENT`, so a grandchild action still inline inside one of + // those bodies is not resolved here. That grandchild's own env resolves it + // when the grandchild is built. A handler this call does walk, and that the + // closure does not export, fails at establish. const actionResolver = await loadWorkflowActionHandlersFromClosure({ packageDir: opts.env.closurePackageDir, }); @@ -934,7 +860,11 @@ export async function runWorkflowChild( // race to settle the same residual and the loser throws an uncaught // TransitionError into its fire-and-forget continuation. Each site // removes its entry when the run reaches terminal. - const runsInFlight = new Map(); + const runsInFlight = new Map(); + const cancellationBarrier = createCancellationBarrier( + runtimeRepoStore, + upstreamSender, + ); for (const run of discovered) { const env = buildRuntimeEnv({ runId: run.runId, @@ -954,6 +884,9 @@ export async function runWorkflowChild( warmCache, sourcesRef, credentialWiring, + ...(opts.mailboxCallBridge !== undefined + ? { mailboxCallBridge: opts.mailboxCallBridge } + : {}), onEvent: (event, childRunId) => { void eventSender.send(event, childRunId).catch((cause) => { logger.error`event-channel send failed during resume run ${run.runId}: ${String(cause)}`; @@ -1062,6 +995,7 @@ export async function runWorkflowChild( drainController, triggeredRunIds, runsInFlight, + cancellationBarrier, warmCache, sourcesRef, credentialMaterialRef, @@ -1075,6 +1009,9 @@ export async function runWorkflowChild( ...(opts.mailboxMutationBridge !== undefined ? { mailboxMutationBridge: opts.mailboxMutationBridge } : {}), + ...(opts.mailboxCallBridge !== undefined + ? { mailboxCallBridge: opts.mailboxCallBridge } + : {}), ...(mailboxWatchRegistry !== undefined ? { mailboxWatchRegistry } : {}), @@ -1088,6 +1025,7 @@ export async function runWorkflowChild( }; const cleanupControlLoop = async (): Promise => { + cancellationBarrier.close("workflow-child control loop exited"); // Any exit path -- clean (iterator end), dirty (thrown error), // shutdown (already cancelled, repeat is a no-op on an empty map) // -- cancels every still-pending substrate write so the runtime @@ -1113,6 +1051,9 @@ export async function runWorkflowChild( "workflow-child control loop exited", ); } + if (opts.mailboxCallBridge !== undefined) { + opts.mailboxCallBridge.cancelAll("workflow-child control loop exited"); + } // Evict the warm-agent cache (design §3b) on every exit path: // graceful (shutdown frame -> iterator end), dirty (thrown error), // or the control channel closing. Eviction runs the wrapped @@ -1171,7 +1112,8 @@ async function handleControlPayload( upstreamSender: ControlChannelSender; drainController: DrainController; triggeredRunIds: string[]; - runsInFlight: Map; + runsInFlight: Map; + cancellationBarrier: ReturnType; warmCache: WarmAgentCache | undefined; sourcesRef: SourcesSnapshotRef; credentialMaterialRef: CredentialMaterialRef; @@ -1179,6 +1121,7 @@ async function handleControlPayload( substrateWriteBridge?: SubstrateWriteResponseSink; outboundMailBridge?: ChildOutboundMailBridge; mailboxMutationBridge?: ChildMailboxMutationBridge; + mailboxCallBridge?: ChildMailboxCallBridge; mailboxWatchRegistry?: MailboxWatchRegistry; }, ): Promise { @@ -1230,6 +1173,9 @@ async function handleControlPayload( warmCache: ctx.warmCache, sourcesRef: ctx.sourcesRef, credentialWiring: ctx.credentialWiring, + ...(ctx.mailboxCallBridge !== undefined + ? { mailboxCallBridge: ctx.mailboxCallBridge } + : {}), onEvent: (event, childRunId) => { void ctx.eventSender.send(event, childRunId).catch((cause) => { logger.error`event-channel send failed during run ${payload.data.runId}: ${String(cause)}`; @@ -1237,7 +1183,7 @@ async function handleControlPayload( }, upstreamSender: ctx.upstreamSender, }); - const handle: WorkflowRun = runtimeRun(ctx.definition, env, { + const handle = runtimeRun(ctx.definition, env, { runId: payload.data.runId, consumedMessageId: payload.data.messageId, triggerPayload, @@ -1353,6 +1299,7 @@ async function handleControlPayload( // delivery. const transientSignalChannel = createWorkflowHostSignalChannel({ repoStore: ctx.bindings.substrate, + runtimeStore: ctx.runtimeRepoStore, principal: ctx.bindings.principal, repoId: ctx.bindings.workflowRunRepoId, ref: ctx.bindings.workflowRunRef, @@ -1367,6 +1314,9 @@ async function handleControlPayload( payload.data.signalName, payload.data.payload, payload.data.signalId, + ...(payload.data.correlationId !== undefined + ? [payload.data.correlationId] + : []), ); } catch (cause) { const reason = cause instanceof Error ? cause.message : String(cause); @@ -1377,6 +1327,27 @@ async function handleControlPayload( })(); return false; } + case "cancel.prepare": { + // Preparation flushes through the same IPC stream; keep consuming replies. + void ctx.cancellationBarrier + .prepare(payload.data) + .then(() => + ctx.runsInFlight + .get(payload.data.runId) + ?.applyCommittedCancellation(), + ) + .catch((error: unknown) => { + logger.error`Failed to apply cancellation for ${payload.data.runId}: ${error instanceof Error ? error.message : String(error)}`; + }); + return false; + } + case "cancel.committed": { + ctx.cancellationBarrier.complete(payload.data); + return false; + } + case "cancel.prepared": { + throw new Error("workflow-child received an upstream cancellation reply"); + } case "drain": { // The supervisor's `drain` control mail flips the controller's // signal. The runtime body's four observation points read the @@ -1550,6 +1521,26 @@ async function handleControlPayload( ctx.mailboxMutationBridge.handleResult(payload.data); return false; } + case "mailbox.call.request": { + // `mailbox.call.request` is the child->supervisor mailbox-call frame; + // receiving one on the child's downstream side is a protocol violation + // in the same shape as a downstream `mailbox.mutate.request`. + throw new Error( + "workflow-child received a `mailbox.call.request` frame on its inbound control channel; this is a child-only upstream payload", + ); + } + case "mailbox.call.response": { + // Route the supervisor's answer to the mailbox-call bridge if one is + // wired. A response that lands without an active bridge means a stale + // frame for which no awaiter exists; log and drop rather than throwing + // so the runtime keeps progressing. + if (ctx.mailboxCallBridge === undefined) { + logger.warn`workflow-child mailbox.call.response received without a bridge wired; requestId=${payload.data.requestId} dropped`; + return false; + } + ctx.mailboxCallBridge.handleResult(payload.data); + return false; + } case "substrate.merge.request": { // Route the request to the substrate-write bridge if one is // wired. A request that lands without an active bridge means a @@ -1625,27 +1616,43 @@ async function handleControlPayload( */ /** - * Force-resolve every `action` handler ref reachable from these definitions - * against the resolver, so a missing action handler surfaces at establish - * rather than when the action is first invoked mid-run. Recurses into loop - * bodies (an action body is the common loop shape). The caller passes the - * lifted onTrigger/childWorkflow bodies separately, as with loop fns. + * Force-resolve every `action` handler ref these definitions run against the + * resolver, so a missing action handler surfaces before the run starts rather + * than when the action is first invoked. The walk is `LOOP_BODY_DESCENT`: a + * loop body shares this env, and an inline onTrigger or childWorkflow body + * does not. The establish caller passes those lifted bodies as their own + * definitions; a spawned body passes only the definition it is about to run, + * and a grandchild action is resolved when that grandchild builds its own env. */ -function eagerlyResolveActionHandlers( +export function eagerlyResolveActionHandlers( definitions: readonly WorkflowDefinition[], actionResolver: (ref: string) => ActionHandler, ): void { - const visit = (def: WorkflowDefinition): void => { - for (const step of Object.values(def.steps)) { - if (step.kind === "action") { - // Throws (fail closed) if the handler names no export, or a non-function. - actionResolver(step.handler); - } else if (step.kind === "loop") { - visit(step.body); - } - } + for (const definition of definitions) { + walkWorkflowSteps({ + definition, + descent: LOOP_BODY_DESCENT, + context: "eagerlyResolveActionHandlers: ", + visit: ({ step }) => { + if (step.kind === "action") { + // Throws (fail closed) if the handler names no export, or a non-function. + actionResolver(step.handler); + } + }, + }); + } +} + +function unwiredMailPartReader(): MailPartReader { + return { + read(ref) { + return Promise.reject( + new Error( + `mail part reader: this child has no mailbox call bridge; cannot read ${ref}`, + ), + ); + }, }; - for (const def of definitions) visit(def); } function buildRuntimeEnv(args: { @@ -1666,11 +1673,13 @@ function buildRuntimeEnv(args: { warmCache: WarmAgentCache | undefined; sourcesRef: SourcesSnapshotRef; credentialWiring: CredentialWiring; + mailboxCallBridge?: ChildMailboxCallBridge; onEvent: (event: EventPayload, childRunId?: string) => void; upstreamSender: ControlChannelSender; }): WorkflowRuntimeEnv { const signalChannel = createWorkflowHostSignalChannel({ repoStore: args.bindings.substrate, + runtimeStore: args.runtimeRepoStore, principal: args.bindings.principal, repoId: args.bindings.workflowRunRepoId, ref: args.bindings.workflowRunRef, @@ -1686,17 +1695,16 @@ function buildRuntimeEnv(args: { runId: args.runId, ref: args.bindings.workflowRunRef, }); - // Reader for inbound-mail parts, a sibling of `blobs` over the same - // workflow-run repo. The step invoker resolves a `Mail` part's `ref` to its - // bytes through it at `agent.send` time; the supervisor committed the bytes - // before the trigger. Deployment-scoped (the ref encodes the owning run), so - // one reader resolves any run's parts. - const mailPartReader = createMailPartReader({ - substrate: args.bindings.substrate, - repoId: args.bindings.workflowRunRepoId, - principal: args.bindings.principal, - ref: args.bindings.workflowRunRef, - }); + // The supervisor committed each part before the trigger. A step asks it + // for the bytes. A child with no call bridge cannot read them; inlined + // text never asks. + const mailPartReader = + args.mailboxCallBridge === undefined + ? unwiredMailPartReader() + : createSupervisorBackedMailPartReader({ + callBridge: args.mailboxCallBridge, + runId: args.runId, + }); // Wrap the step invoker so every `InferenceEvent` the harness emits // funnels through the per-run `onEvent` closure, which forwards // the event up the HMAC-authenticated event channel. The wrap is @@ -1730,7 +1738,9 @@ function buildRuntimeEnv(args: { : (spawnInput) => hostSuspendable( spawnInput, - args.onEvent, + (event) => { + args.onEvent(event, spawnInput.childRunId); + }, args.credentialWiring.materialRef, ); // Same adaptation for the terminal childWorkflow seam: inject THIS run's event @@ -1827,6 +1837,7 @@ function buildRuntimeEnv(args: { }); const childSignalChannel = createWorkflowHostSignalChannel({ repoStore: args.bindings.substrate, + runtimeStore: args.runtimeRepoStore, principal: args.bindings.principal, repoId: args.bindings.workflowRunRepoId, ref: args.bindings.workflowRunRef, @@ -1870,6 +1881,7 @@ function buildRuntimeEnv(args: { args.authorize, effects, args.actionResolver, + args.bindings.actionCredentials(args.credentialWiring), ); return env; } diff --git a/vendor/intx/workflow-host/src/child/step-tools.ts b/vendor/intx/workflow-host/src/child/step-tools.ts new file mode 100644 index 0000000000..22a6edd77f --- /dev/null +++ b/vendor/intx/workflow-host/src/child/step-tools.ts @@ -0,0 +1,555 @@ +// Per-step tool attachment and agent construction for the workflow child. +// +// The process that boots the child materializes pinned tool packages and +// passes the result in. This module attaches that result to the step env, +// derives the tool-mark floor from each factory's static declarations, and +// builds the agent whose `close()` tears the plugin chain and tool bundles +// down. It does not read a deploy tree or load a tool package. + +import { + createAgent, + defineAgent, + defineTool, + toolApprovalEffect, + type Agent, + type AgentDefinition, + type AnnotatedPluginFactory, + type AnnotatedToolFactory, + type BaseEnv, + type ToolBundle, + type ToolDeclaration, +} from "@intx/agent"; +import type { GrantRule } from "@intx/authz"; +import { + buildCredentialCapabilities, + type StepCredentialWiring, +} from "@intx/harness"; +import { getLogger } from "@intx/log"; +import type { HostCredentialCapability } from "@intx/types"; +import type { ToolCredentialDeclaration } from "@intx/types/package-json"; +import { + layerRuntimeCapabilities, + type RuntimeCapabilities, +} from "@intx/types/runtime-capabilities"; + +const logger = getLogger(["sidecar", "workflow-child", "step-tools"]); + +/** + * Cache and registry caps the per-step tool loader needs. Resolved at + * the sidecar boot edge from the existing `SIDECAR_CACHE_*` / + * `SIDECAR_REGISTRY_*` config keys and threaded into the child through + * the substrate config, so the child's per-step materialization is + * bounded by those boot-edge-resolved caps. + */ +export interface StepToolCacheConfig { + readonly cacheMaxBytes: number; + readonly registryMaxTarballBytes: number; +} + +/** + * Materialized tool runtime for one step's agent. Carried from + * `buildEnv` (which knows the step's identity) to the `agentFactory` + * (which knows the agent definition + env) via a symbol-keyed slot on + * the per-step env so the two callbacks of `createWorkflowStepInvoker` + * can cooperate without widening the portable adapter's surface. + */ +export interface StepToolMaterialization { + readonly factories: readonly { + readonly packageName: string; + readonly declaredCredentials: readonly ToolCredentialDeclaration[]; + readonly factory: AnnotatedToolFactory; + }[]; + readonly pluginFactories: readonly AnnotatedPluginFactory[]; +} + +/** + * Derive the per-step tool-mark floor grants from a step's materialized + * tool factories. Each loaded factory carries its static + * `definitions` (name + optional `approval` mark) verbatim from the tool + * package; every declared tool contributes a `tool:` / `invoke` + * grant whose effect is the tool's floor (`ask` for an approval-gated + * tool, `allow` otherwise), computed through the same + * `toolApprovalEffect` mapping the deploy-time capability walk uses so a + * pinned tool floors identically hub-side and sidecar-side. + * + * The hub's capability walk reads only INLINE `agent.toolFactories`, so a + * tool that ships as a pinned package loads in the child and never + * produces a `tool:` grant on the run principal. These derived rows + * supply that missing floor: they join the per-step grants the authz path + * evaluates as ADDITIONAL rows, so `evaluateGrants` precedence still + * resolves an explicit `deny` (priority 2) over the derived `ask`/`allow` + * -- the floor only raises the minimum authority to the tool's mark, it + * never overrides a declared denial. + * + * The grant `id` is deterministic (`floor:tool:`) rather than + * random: `evaluateGrants` never dedupes or joins on `id` (it ranks by + * specificity then effect), so a stable id keeps the rows reproducible + * without a generator, and a floor row that coincides with a + * hub-supplied `tool:` row resolves by effect precedence regardless + * of the ids. + */ +export function deriveToolMarkFloorGrants( + factories: readonly { + readonly packageName: string; + readonly definitions: readonly ToolDeclaration[]; + }[], +): GrantRule[] { + const rows: GrantRule[] = []; + for (const factory of factories) { + for (const definition of factory.definitions) { + rows.push({ + id: `floor:tool:${definition.name}`, + resource: `tool:${definition.name}`, + action: "invoke", + effect: toolApprovalEffect(definition), + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }); + } + } + return rows; +} + +/** + * Symbol-keyed slot the `buildEnv` callback sets on the env it returns + * and the `agentFactory` reads. Object spread (`{ ...envBase, + * authorize }`) inside the step-invoker adapter copies own enumerable + * symbol-keyed properties, so the slot survives the spread that + * produces the env handed to `agentFactory`. + */ +const STEP_TOOLS = Symbol("intx.sidecar.step-tools"); + +// The per-step env carries one private symbol-keyed slot the +// buildEnv/agentFactory pair cooperate over. The slot is read/written +// through `Reflect.get`/`Reflect.set` so neither site needs a type +// assertion: `BaseEnv` is an interface without a symbol index +// signature, so a structural cast would otherwise be required. +function setStepToolSlot( + env: object, + materialization: StepToolMaterialization, +): void { + Reflect.set(env, STEP_TOOLS, materialization); +} + +function getStepToolSlot(env: object): StepToolMaterialization | undefined { + const value: unknown = Reflect.get(env, STEP_TOOLS); + if (value === undefined) return undefined; + if (!isStepToolMaterialization(value)) { + throw new Error( + "sidecar workflow-child step tools: the per-step env's tool slot is not a StepToolMaterialization; the slot is private to this module and must only be set by attachStepTools", + ); + } + return value; +} + +function isStepToolMaterialization( + value: unknown, +): value is StepToolMaterialization { + return ( + typeof value === "object" && + value !== null && + "factories" in value && + "pluginFactories" in value && + Array.isArray(value.factories) && + Array.isArray(value.pluginFactories) + ); +} + +/** + * Symbol-keyed slot carrying the per-step credential wiring the + * `agentFactory` assembles each bundle's consumer-scoped `credentials` + * capability from: the live material cell, the step's grants, and the + * provider registry. Set by `buildEnv` alongside the tool slot, read by + * `createToolBearingAgentFactory`. Absent for a build that threaded no + * credential context (a unit test using the bare factory) -- then no + * credentials capability is assembled and bundles keep the base capabilities + * bag. + */ +const STEP_CREDENTIAL_WIRING = Symbol("intx.sidecar.step-credential-wiring"); + +function setStepCredentialWiring( + env: object, + wiring: StepCredentialWiring, +): void { + Reflect.set(env, STEP_CREDENTIAL_WIRING, wiring); +} + +function getStepCredentialWiring( + env: object, +): StepCredentialWiring | undefined { + const value: unknown = Reflect.get(env, STEP_CREDENTIAL_WIRING); + if (value === undefined) return undefined; + if (!isStepCredentialWiring(value)) { + throw new Error( + "sidecar workflow-child step tools: the per-step env's credential-wiring slot is not a StepCredentialWiring; the slot is private to this module and must only be set by attachStepCredentialWiring", + ); + } + return value; +} + +function isStepCredentialWiring(value: unknown): value is StepCredentialWiring { + return ( + typeof value === "object" && + value !== null && + "materialCell" in value && + "resolveGrants" in value && + "providers" in value + ); +} + +/** + * Attach a step's credential wiring to the per-step env so the tool-bearing + * `agentFactory` can assemble each bundle's `credentials` capability. Called + * by `buildEnv` for a step whose build threaded a credential context; omitted + * otherwise, which leaves the slot unset and the credentials capability + * unassembled. + */ +export function attachStepCredentialWiring( + env: Omit, + wiring: StepCredentialWiring, +): void { + setStepCredentialWiring(env, wiring); +} + +/** + * Read the base `RuntimeCapabilities` bag `buildEnv` set on the per-step env + * (`env.capabilities`, currently `mail.transport`). `BaseEnv` does not type + * the key -- it is a runtime-only widening the step env carries -- so it is + * read reflectively and validated. Throws when a bundle has a credentials + * capability to layer but the env carries no base bag: that is a wiring + * inconsistency (the same `buildEnv` sets both), not a condition to paper over. + */ +function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { + const value: unknown = Reflect.get(env, "capabilities"); + if ( + typeof value !== "object" || + value === null || + !("resolve" in value) || + typeof (value as { resolve: unknown }).resolve !== "function" + ) { + throw new Error( + "sidecar workflow-child step tools: a credentials capability was assembled for this bundle but the per-step env carries no base capabilities bag to layer it onto; buildEnv must set env.capabilities before assembling credentials", + ); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated structurally above: an object whose `resolve` is callable is the RuntimeCapabilities resolver buildEnv set + return value as RuntimeCapabilities; +} + +/** + * Attach a step's materialized tool runtime to the per-step env so the + * tool-bearing `agentFactory` can consume it. Called by `buildEnv` + * after `materializeStepTools` resolves. + * + * The parameter is `Omit` because `buildEnv` + * yields exactly that shape (the step-invoker adapter adds `authorize` + * before the env reaches `agentFactory`); the symbol slot survives the + * adapter's `{ ...envBase, authorize }` spread. + */ +export function attachStepTools( + env: Omit, + materialization: StepToolMaterialization, +): void { + setStepToolSlot(env, materialization); +} + +/** + * Re-wrap a loaded tool factory so its bundle's `dispose` (when present) + * is captured via `onDispose`, forwarding the loader's `id`, `requires`, + * and `definitions` so the result is a real `AnnotatedToolFactory`, not a + * hand-shaped lookalike. The static `definitions` declaration is + * forwarded verbatim: this wrapper does not rename tools, so the names + * the deploy-time walk enumerates must survive the re-wrap unchanged. + * + * When `credentials` is supplied, the bundle's factory sees a per-bundle + * capabilities bag: the step's base bag layered with THIS package's + * consumer-scoped `credentials` capability, so the bundle resolves only the + * handles its own package is authorized for. When it is absent (the package + * declares and binds no credential), the base bag is passed through unchanged. + * + * Exported so the definitions-preservation contract is testable in + * isolation; the production path calls it from + * `createToolBearingAgentFactory`. + */ +export function rewrapStepToolFactory( + annotated: AnnotatedToolFactory, + onDispose: (dispose: () => unknown) => void, + credentials: HostCredentialCapability | undefined, +): AnnotatedToolFactory { + return defineTool({ + id: annotated.id, + requires: annotated.requires, + definitions: annotated.definitions, + factory: (factoryEnv: BaseEnv): ToolBundle => { + const env = + credentials === undefined + ? factoryEnv + : layerCredentialsOntoEnv(factoryEnv, credentials); + const bundle = annotated(env); + if (bundle.dispose !== undefined) { + onDispose(bundle.dispose); + } + return bundle; + }, + }); +} + +/** + * Produce a copy of `factoryEnv` whose `capabilities` bag is the base bag + * layered with this bundle's consumer-scoped `credentials` capability. Object + * spread copies own enumerable properties -- including the runtime-only + * `capabilities` key `BaseEnv` does not type and the private tool slot -- so + * every other env surface survives; only `capabilities` is replaced. + */ +function layerCredentialsOntoEnv( + factoryEnv: BaseEnv, + credentials: HostCredentialCapability, +): BaseEnv { + const base = requireCapabilitiesBag(factoryEnv); + const layered = layerRuntimeCapabilities(base, { credentials }); + // Copy every env surface (spread carries own enumerable string AND symbol + // keys, so `transport`, `address`, and the private tool/credential slots + // survive), then replace the runtime-only `capabilities` key via Reflect -- + // a `{ ...factoryEnv, capabilities }` literal trips the excess-property + // check because BaseEnv does not type the key. + const layeredEnv: BaseEnv = { ...factoryEnv }; + Reflect.set(layeredEnv, "capabilities", layered); + return layeredEnv; +} + +/** + * Build the `agentFactory` the workflow step-invoker uses. The returned + * factory reads the materialized tool runtime off the env (set by + * `buildEnv` via `attachStepTools`), augments the step's + * `AgentDefinition` with the loaded tool factories (wrapped to capture + * each bundle's disposer), constructs the plugin chain on `env.plugins`, + * builds the agent, and wraps + * `agent.close()` so every plugin instance and tool bundle is disposed + * when the step's agent closes. + * + * When the env carries no materialized tools (the `buildEnv` did not + * run materialization, e.g. a unit test using the bare factory), the + * factory falls back to `createAgent(def, env)` unchanged. + */ +export function createToolBearingAgentFactory(): ( + def: AgentDefinition, + env: EnvReq, +) => Promise { + return async ( + def: AgentDefinition, + env: EnvReq, + ): Promise => { + const materialization = getStepToolSlot(env); + if (materialization === undefined) { + return createAgent(def, env); + } + + // Assemble each package's consumer-scoped `credentials` capability once, + // when the step carries credential wiring (a bare test build carries none, + // yielding an empty map). Every factory in a package shares the one + // capability; a package that declares a handle no binding resolves fails + // the launch here, loudly, rather than at the tool's first resolve. This + // can throw (reconcile fail-closed, malformed delivery) before any handle + // is shaped, so nothing is orphaned on that path. + const credentialWiring = getStepCredentialWiring(env); + const credentialCapabilities = + credentialWiring === undefined + ? new Map() + : buildCredentialCapabilities( + materialization.factories, + credentialWiring, + ); + + // Wrap each loaded tool factory so its bundle's `dispose` (when + // present) is captured. Dedupe by closure identity: a factory whose + // bundle returns the same `dispose` on every invocation must not be + // torn down once per push. Each package's credentials capability joins + // the same teardown set so its shaped handles are released with the agent. + const capturedDisposers = new Set<() => unknown>(); + for (const capability of credentialCapabilities.values()) { + capturedDisposers.add(() => capability.dispose()); + } + const factoriesWithCapture = materialization.factories.map((stf) => + rewrapStepToolFactory( + stf.factory, + (dispose) => { + capturedDisposers.add(dispose); + }, + credentialCapabilities.get(stf.packageName), + ), + ); + + // Run every captured disposer -- each credentials capability and each tool + // bundle -- guarding each so one failure does not strand the rest. Used on + // the success teardown AND on the construction-failure rollbacks below: the + // credentials capabilities are built before the plugin chain, so a plugin + // or agent build that throws must still release them (an http handle holds + // nothing, but a future key-file/socket handle would leak otherwise). + // Bundle disposers are idempotent, so re-running one `createAgent` already + // disposed on its own failure path is safe. + const runCapturedDisposers = async (): Promise => { + const failures: unknown[] = []; + for (const dispose of capturedDisposers) { + try { + await dispose(); + } catch (cause) { + logger.error`step tool bundle dispose failed: ${cause instanceof Error ? cause.message : String(cause)}`; + failures.push(cause); + } + } + return failures; + }; + + // Rebuild the def with the materialized tool factories. The + // serialized `def.toolFactories` carry only `{ id, requires }` + // metadata (the workflow projection strips closures on the wire), + // so the runnable factories come from materialization, not the + // incoming def. `defineAgent` owns the contravariance escape for + // the `BaseEnv`-typed loader factories (see its `EnvRequiredByAll` + // machinery). + const toolDef = defineAgent({ + id: def.id, + systemPrompt: def.systemPrompt, + tools: factoriesWithCapture, + capabilities: [...def.capabilities], + inference: { sources: [...def.inference.sources] }, + ...(def.description !== undefined + ? { description: def.description } + : {}), + ...(def.director !== undefined ? { director: def.director } : {}), + ...(def.tags !== undefined ? { tags: def.tags } : {}), + }); + + // Instantiate plugin factories one at a time so each successive + // factory sees the prior plugins' instances on `env.plugins`: + // posix's bundle reads `env.plugins` and threads ToolPlugin-shaped + // values into `createPosixTools`; the LSP plugin factory is what + // populates them. + // + // On a midway factory throw, every plugin instance already + // constructed releases what it acquired (the LSP plugin starts a + // subprocess) before the construction error propagates, so a + // partial-success chain never leaks an LSP subprocess. + const pluginInstances: unknown[] = []; + let chainEnv: BaseEnv = env; + try { + for (const factory of materialization.pluginFactories) { + const instance = factory(chainEnv); + pluginInstances.push(instance); + chainEnv = { + ...env, + plugins: [...pluginInstances], + }; + } + } catch (err) { + // Release the credentials capabilities (built above, before the plugin + // chain) and any bundle disposers captured so far, then the plugin + // instances this module owns. + await runCapturedDisposers(); + await disposeAll(pluginInstances, "plugin construction rollback"); + throw err; + } + + let agent: Agent; + try { + agent = await createAgent(toolDef, chainEnv); + } catch (err) { + // `createAgent` disposes the tool bundles it constructed on its + // own failure path, but the credentials capabilities and the plugin + // instances are this module's to own -- tear them down so a failed + // agent build does not leak a shaped handle or the LSP subprocess. + await runCapturedDisposers(); + await disposeAll(pluginInstances, "agent construction failure"); + throw err; + } + + return wrapAgentClose(agent, async () => { + // Captured disposers first (each credentials capability, then the tool + // bundles -- posix's bundle dispose chains through to the LSP plugin's + // `dispose`), then the plugin instances directly. Disposing the LSP + // plugin twice is safe: `lsp.dispose()` clears its client set and the + // posix bundle's dispose is idempotent. Running both guarantees the LSP + // subprocess is torn down even for a plugin no tool bundle consumed. + // Both loops run every disposer and collect failures rather than + // throwing mid-loop, so one failing disposer never strands the rest. + // A leaked or failing LSP subprocess must surface, not be swallowed: + // any collected failure fails the close, so the caller sees it. + const failures = [ + ...(await runCapturedDisposers()), + ...(await disposeAll(pluginInstances, "step teardown")), + ]; + if (failures.length > 0) { + throw new AggregateError( + failures, + `step agent close: ${String(failures.length)} disposer(s) failed during teardown; an LSP subprocess may be leaked`, + ); + } + }); + }; +} + +/** + * Return an `Agent` whose `close()` runs the original close and then + * the supplied teardown. The teardown runs AFTER the agent's own close + * so the reactor has stopped issuing tool calls before the tool/plugin + * resources are released. `close()` is idempotent at the agent layer; + * this wrapper guards its own teardown so a double `close()` does not + * double-dispose. + */ +function wrapAgentClose(agent: Agent, teardown: () => Promise): Agent { + let tornDown = false; + return { + ...agent, + send: (content, opts) => agent.send(content, opts), + stream: () => agent.stream(), + deliver: (message) => agent.deliver(message), + setSource: (source) => agent.setSource(source), + history: () => agent.history(), + checkpoints: (limit) => agent.checkpoints(limit), + readAt: (hash) => agent.readAt(hash), + blobReader: agent.blobReader, + async close() { + await agent.close(); + if (tornDown) return; + tornDown = true; + await teardown(); + }, + }; +} + +async function disposeAll( + instances: readonly unknown[], + context: string, +): Promise { + const failures: unknown[] = []; + for (const instance of instances) { + const dispose = pluginDispose(instance); + if (dispose === undefined) continue; + try { + // `await` accepts non-promise values verbatim, so this works + // whether the disposer is sync or async. + await dispose(); + } catch (cause) { + logger.error`step plugin dispose failed during ${context}: ${cause instanceof Error ? cause.message : String(cause)}`; + failures.push(cause); + } + } + return failures; +} + +/** + * Extract a callable `dispose` from a plugin instance whose static type + * is `unknown` (plugin factories return host-defined shapes the agent + * runtime does not interpret). Returns a bound disposer or `undefined` + * when the instance carries no `dispose` function. + */ +function pluginDispose(value: unknown): (() => unknown) | undefined { + if (value === null || typeof value !== "object") return undefined; + if (!("dispose" in value)) return undefined; + const dispose: unknown = value.dispose; + if (typeof dispose !== "function") return undefined; + const fn = dispose; + return () => fn.call(value); +} diff --git a/vendor/intx/workflow-host/src/child/substrate-factory.ts b/vendor/intx/workflow-host/src/child/substrate-factory.ts new file mode 100644 index 0000000000..5fcbe753de --- /dev/null +++ b/vendor/intx/workflow-host/src/child/substrate-factory.ts @@ -0,0 +1,2797 @@ +// Substrate factory the sidecar's `bin/workflow-child` hands to +// `runWorkflowChildFromProcessEnv`. The factory closes over the +// production substrate (`createAgentRepoStore`-backed `RepoStore`), +// the host-process scheduler singleton (adapted to the runtime's +// `Scheduler` shape), and the sidecar's grant-rule evaluator. +// +// The factory consumes the workflow-host's typed `SubstrateFactoryEnv` +// -- the parsed `SpawnTimeEnv` plus a narrow `substrateConfig` +// record carrying only the keys the binary listed in +// `RunWorkflowChildFromProcessEnvOpts.substrateConfigKeys`. The +// factory does not read `process.env` itself; the binary owns the +// only crossing of that boundary. +// +// Single-writer architecture: the workflow-run repo's ref has exactly +// one writer at a time -- the supervisor. The child opens a bare +// `createAgentRepoStore` against the shared on-disk data dir for +// read-only operations (`getRepoDir`, `subscribe`, `resolveRef`, +// etc.) and exposes a proxy `RepoStore` whose +// `writeTreePreservingPrefix` forwards every write over the control +// IPC into the supervisor's substrate. The supervisor's substrate is +// wrapped with the boot-edge pack-push facade, so the hub push fires +// as part of the supervisor's normal write path -- the child does +// not open its own pack-push pipeline. + +import fs from "node:fs"; +import path from "node:path"; + +import { type } from "arktype"; + +import { InferenceSource } from "@intx/types/runtime-core"; +import type { InferenceEvent } from "@intx/types/inference-events"; +import type { + ApprovalSnapshot, + AuditStore, + ContextStore, + InboundMessage, + MessageTransport, + PendingOperation, +} from "@intx/types/runtime-core"; +import type { RuntimeCapabilities } from "@intx/types/runtime-capabilities"; +import { + type AdapterManifest, + parseAdapterManifestEnv, +} from "@intx/types/adapter-manifest"; +import type { GrantRule } from "@intx/authz"; +import { createDependencies, type AdapterRegistry } from "@intx/inference"; +import { loadAdapterRegistry } from "@intx/inference/providers"; +import type { + AnnotatedPluginFactory, + DirectorRegistry, + ToolDeclaration, +} from "@intx/agent"; +import { createDefaultDirectorRegistry } from "@intx/agent"; +import { + rawAuthorizationCredentialProvider, + xApiKeyCredentialProvider, +} from "@corbits/credential-header"; +import { createMcpStreamableHttpCredentialProvider } from "@corbits/credential-mcp"; +import { + builtinCredentialProviders, + createCredentialProviderRegistry, + createHarnessRuntimeCapabilities, + createInferenceCredentialResolver, + driveConnectorReplies, + type AgentEventStream, + type ConnectorReplyDrain, + type CredentialMaterialCell, + type CredentialProviderRegistry, +} from "@intx/harness"; +import { createSSHSignature } from "@intx/crypto"; +import { parseRunAddress } from "@intx/types/agent-address"; +import { + createAgentRepoStore, + WORKFLOW_RUN_AGENT_STATE_PREFIX, + type Principal, + type RepoId, + type RepoStore, + type WorkflowRunWorkflowProcessPrincipal, +} from "@intx/hub-sessions/substrate"; +import { createIsogitStore } from "@intx/storage-isogit/node"; +import { + baseStepId, + collectDeclaredPluginNames, + createDefaultActionInvoker, + createInMemoryEffectLedger, + createLoopIterationHandle, + createNoopDrainController, + createSuspendableChildHandle, + eagerlyResolveLoopFns, + emptyState, + enumerateInlineLoopBodies, + rewriteInlineChildWorkflowBodies, + unavailableActionCredentials, + walkWorkflowSteps, + LOOP_BODY_DESCENT, + type ActionHandler, + type LoopFnRegistry, + type ParkedApprovalOp, + type ReadParkedApprovalOps, + type Scheduler, + type StepInvokeRequest, + type StepInvokeResult, + type WorkflowAuthorizeFn, + type WorkflowDefinition, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { createWorkflowRunBlobSubstrate } from "../adapters/blob-substrate"; +import { createWorkflowRunRepoStore } from "../adapters/repo-store"; +import { + createInMemorySpawnChild, + createInMemorySpawnSuspendableChild, + type RunChildWorkflow, + type RunSuspendableChild, +} from "../adapters/spawn-child"; +import { + createWorkflowStepInvoker, + type StepEnvBase, +} from "../adapters/step-invoker"; +import { + createDurableConversationRegistry, + reconstructDurableConversation, + type DurableConversationRegistry, +} from "../conversation-state"; +import { readRunGrants, runGrantsPath } from "../run-grants"; +import { + adaptHostScheduler, + createWorkflowHostScheduler, + createWorkflowHostSignalChannel, +} from "../seams/index"; +import { + hashGrants, + isErrnoNotFound, + type CredentialsSnapshot, +} from "../supervisor/credentials"; +import { + loadWorkflowActionHandlersFromClosure, + loadWorkflowDirectorRegistryFromClosure, + loadWorkflowLoopFnsFromClosure, + loadWorkflowPluginFactoriesFromClosure, + loadWorkflowPluginToolDefinitionsFromClosure, +} from "../workflow-definition-loader"; +import { + createSidecarActionCredentialsFactory, + stepGrantsResolver, + type SidecarStepCredentialContext, +} from "./action-credentials"; +import type { SubstrateFactory, SubstrateFactoryEnv } from "./from-process-env"; +import { createMailboxWatchRegistry } from "./mailbox-watch-registry"; +import type { ChildOutboundMailBridge } from "./outbound-mail-bridge"; +import type { LoadParkedApproval } from "./parked-correlations"; +import { createProxyWorkflowRunRepoStore } from "./proxy-repo-store"; +import { + createCredentialsBackedAuthorize, + createGrantEvaluator, + resolveSnapshotStepGrants, + type CredentialsSnapshotRef, + type GrantEvaluator, +} from "./credentials-backed-authorize"; +import { + eagerlyResolveActionHandlers, + type CredentialWiring, + type RunWorkflowChildBindings, + type SourcesSnapshotRef, +} from "./run-child"; +import { + attachStepCredentialWiring, + attachStepTools, + createToolBearingAgentFactory, + deriveToolMarkFloorGrants, + type StepToolCacheConfig, + type StepToolMaterialization, +} from "./step-tools"; +import { + createSupervisorBackedTransport, + type SupervisorBackedTransportInbound, +} from "./supervisor-backed-transport"; + +// The child does not construct a workflow-run pack-push pipeline of +// its own. The supervisor owns the workflow-run repo's write +// contract; the supervisor's substrate is wrapped at the sidecar's +// boot edge with the pack-pushing facade so any successful workflow- +// run write fires the hub push automatically. The child's proxy +// `RepoStore` forwards `writeTreePreservingPrefix` over IPC into the +// supervisor's wrapped substrate. + +/** + * Required substrate-config keys the sidecar's binary forwards into + * the factory's `substrateConfig` slot. Listed here so the binary + * passes the same names to the helper; the helper enforces + * presence-and-non-empty against this allowlist before the factory + * runs. + */ +export const SIDECAR_SUBSTRATE_CONFIG_KEYS = [ + "SIDECAR_DATA_DIR", + "WORKFLOW_RUN_REPO_ID", + "WORKFLOW_RUN_REF", + "SIDECAR_SIGNING_PUBLIC_KEY", + "SIDECAR_SIGNING_PRIVATE_KEY", + "STEP_INFERENCE_SOURCES", + "WORKFLOW_BODY_SOURCES", + "SIDECAR_CACHE_MAX_BYTES", + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + "SIDECAR_ADAPTER_MANIFEST", +] as const; + +const SubstrateConfig = type({ + SIDECAR_DATA_DIR: "string > 0", + WORKFLOW_RUN_REPO_ID: "string > 0", + WORKFLOW_RUN_REF: "string > 0", + SIDECAR_SIGNING_PUBLIC_KEY: "string > 0", + SIDECAR_SIGNING_PRIVATE_KEY: "string > 0", + STEP_INFERENCE_SOURCES: "string > 0", + // JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` of every spawned + // body's plaintext inference sources, decrypted sidecar-side from the run + // record. Always serialized by the deploy router (at least "{}" when the + // deployment spawns no bodies), so a missing key child-side is a serialization + // bug and must fail loud here. + WORKFLOW_BODY_SOURCES: "string > 0", + // Per-step tool-loader caps. The supervisor threads the boot edge's + // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` + // through `substrateEnv` so the child's per-step tool materialization is + // bounded by the sidecar's boot-edge-resolved caps. Validated as + // positive-finite-number strings at this boundary. + SIDECAR_CACHE_MAX_BYTES: "string > 0", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", + // JSON-encoded custom inference adapter manifest. Required: the boot + // edge always serializes it into `substrateEnv` (defaulting to "[]" + // when no custom adapters are configured), so a missing key child-side + // is a serialization bug and must fail loud here, exactly like the + // byte-cap fields. Validated as a non-empty string at this boundary; + // its JSON shape is re-validated against `AdapterManifest` in + // `parseAdapterManifest` before any module is imported. + SIDECAR_ADAPTER_MANIFEST: "string > 0", +}).onUndeclaredKey("ignore"); + +/** + * Parse a substrate-config cap entry (`SIDECAR_CACHE_MAX_BYTES` / + * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) into a positive finite number. + * The boot edge already validated these via the `config.ts` readers + * before serializing them into `substrateEnv`; this re-parse at the + * child boundary keeps the typed-config contract honest rather than + * trusting the wire blindly. + */ +function parseByteCap(raw: string, name: string): number { + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + throw new Error( + `sidecar workflow-child substrate config: ${name} must be a positive finite number; got ${JSON.stringify(raw)}`, + ); + } + return n; +} + +/** + * Per-step inference-source table parsed from the spawn-time + * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes + * `frame.workflow.sources` (a `Record`) as + * JSON and threads it through the supervisor's `substrateEnv`; the + * factory parses and validates the table once at construction time and + * seeds it into the run loop's mutable sources reference, which each + * `buildEnv` reads. Each value is the step's ordered + * failover chain -- element 0 is the active source, the tail are the + * reactor's forward-only failover targets -- so the list is non-empty. + */ +const StepInferenceSourceTable = type({ + "[string]": InferenceSource.array().atLeastLength(1), +}); +type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; + +/** + * Parse and validate the JSON-encoded `STEP_INFERENCE_SOURCES` entry + * the supervisor threaded through `substrateEnv`. A malformed JSON + * payload, a non-object root, or a value that does not match + * `Record` is rejected at the boundary with + * a structured error rather than being deferred to a deep-stack + * `buildEnv` failure. + */ +function parseStepInferenceSources(raw: string): StepInferenceSourceTable { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES is not valid JSON: ${reason}`, + ); + } + const validated = StepInferenceSourceTable(parsed); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES failed validation: ${validated.summary}`, + ); + } + return validated; +} + +/** + * Every spawned body's per-step inference-source table, keyed by the body's + * definition id. Parsed from the `WORKFLOW_BODY_SOURCES` env entry the deploy + * router serializes from the run record's decrypted body sources. Empty when the + * deployment spawns no bodies. + */ +const BodyInferenceSources = type({ + "[string]": StepInferenceSourceTable, +}); +type BodyInferenceSources = typeof BodyInferenceSources.infer; + +/** + * Parse and validate the JSON-encoded `WORKFLOW_BODY_SOURCES` entry. Mirrors + * `parseStepInferenceSources`: a malformed payload is rejected at the boundary + * rather than deep in a body spawn. + */ +function parseBodyInferenceSources(raw: string): BodyInferenceSources { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES is not valid JSON: ${reason}`, + ); + } + const validated = BodyInferenceSources(parsed); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES failed validation: ${validated.summary}`, + ); + } + return validated; +} + +/** + * Parse and validate the JSON-encoded `SIDECAR_ADAPTER_MANIFEST` entry + * the supervisor threaded through `substrateEnv` from the boot edge's + * `readAdapterManifest`. + * + * Trust boundary: the child's substrate config is operator-supplied + * (the supervisor's `Bun.spawn` env), so this re-validation is + * defense-in-depth at the deserialization boundary, NOT a trust + * upgrade. The manifest was already trusted operator config on the + * parent side; the same channel already carries the sidecar's signing + * private key, so it is not a lower-trust surface. Re-asserting the + * shape here keeps the typed-config contract honest rather than + * importing modules off an unvalidated wire value. + * + * Host contract for custom adapters: a manifest `specifier` must + * resolve from BOTH the sidecar's and this child's module-resolution + * roots (the child is a separate `bun` process spawned by the + * supervisor), and an adapter module MUST be import-side-effect-free — + * it is imported once per process by `loadAdapterRegistry`, and any + * top-level side effect would run independently in the parent and in + * every child. + */ +export function parseAdapterManifest(raw: string): AdapterManifest { + let manifest: AdapterManifest | undefined; + try { + manifest = parseAdapterManifestEnv(raw); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + throw new Error(`sidecar workflow-child substrate config: ${message}`, { + cause, + }); + } + if (manifest === undefined) { + throw new Error( + "sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST must be a serialized manifest", + ); + } + return manifest; +} + +/** + * Resolve the per-step inference-source failover chain from the table a + * build reads. The supervisor's multi-step branch only invokes + * a step whose `stepId` appears in `frame.workflow.sources`; a lookup + * miss here is a programmer error in the supervisor, not a wire-side + * failure, and the resolver surfaces it with the missing base step id + * named (plus the scoped invocation id, for a map iteration). The + * returned list is the step's ordered chain (element 0 the active + * source, the tail the reactor's failover targets); the table's arktype + * guarantees it is non-empty. + * + * A `map` iteration runs under a scoped step id `[]`, but + * deploy pins one source per base step, so the scoped id is resolved to + * its base before the lookup -- every iteration shares the base step's + * pinned source. `baseStepId` is the identity on an unscoped id, so a + * plain step is unaffected. + */ +function createStepInferenceSourceResolver( + table: StepInferenceSourceTable, +): (stepId: string) => InferenceSource[] { + return (stepId: string): InferenceSource[] => { + const base = baseStepId(stepId); + const sources = table[base]; + if (sources === undefined) { + const scopedNote = + base === stepId + ? "" + : ` (normalized from scoped invocation id ${JSON.stringify(stepId)})`; + throw new Error( + `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(base)}${scopedNote}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, + ); + } + return sources; + }; +} + +function hexDecode(hex: string, name: string): Uint8Array { + if (hex.length % 2 !== 0) { + throw new Error( + `${name} must be even-length hex; got ${String(hex.length)} chars`, + ); + } + const out = new Uint8Array(hex.length / 2); + for (let i = 0; i < out.length; i += 1) { + const byte = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + if (Number.isNaN(byte)) { + throw new Error(`${name} contains non-hex characters`); + } + out[i] = byte; + } + return out; +} + +/** + * Pinned tool-package materialization. The sidecar process owns the + * deploy-tree read; the factory calls this only on the pinned arm. + */ +type MaterializeStepTools = (args: { + dataDir: string; + mailboxAddress: string; + stepId: string; + stepCount: number; + storeDir: string; + cache: StepToolCacheConfig; +}) => Promise; + +/** + * Dependencies `createSidecarSubstrateFactory` closes over. Tool + * materialization and the child-grant cap are required: this package does + * not import the modules that implement them. `createBareRepoStore` stays + * optional so tests can inject an in-memory store. + */ +interface SidecarSubstrateFactoryDeps { + materializeStepTools: MaterializeStepTools; + collectDeclaredResources: ( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: ReadonlyMap, + ) => ReadonlySet; + collectDeclaredCredentialConsumers: ( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: ReadonlyMap, + ) => ReadonlySet; + filterGrantsToDeclaredResources: ( + parentGrants: readonly unknown[], + declared: ReadonlySet, + credentialConsumers: ReadonlySet, + ) => readonly unknown[]; + /** + * Override the bare-store constructor. Production callers omit this + * to get the `createAgentRepoStore`-backed `RepoStore` against + * `SIDECAR_DATA_DIR`; tests inject an in-memory recording stub. + * + * The bare store backs the child's read-only operations + * (`getRepoDir`, `subscribe`, `resolveRef`, `listRefs`, + * `resolveHead`, `createPack`). The child's workflow-run writes do + * NOT flow through this store; the proxy `RepoStore` forwards them + * over IPC into the supervisor's substrate. + */ + createBareRepoStore?: (config: { + dataDir: string; + signingKey: { publicKey: Uint8Array; privateKey: Uint8Array }; + }) => RepoStore; +} + +/** + * `CommitSigner` the per-step isogit stores use to sign every commit. + * The factory's Ed25519 signing keypair (the same key the child's bare + * `RepoStore` carries) is bound into an `sshsig`-shaped signer so the + * per-step agent-state commits are attributable to the sidecar's + * substrate identity, matching the signing surface the production + * `RepoStore` uses for workflow-run writes. + */ +function createStepStorageSigner(signingKey: { + publicKey: Uint8Array; + privateKey: Uint8Array; +}): (payload: string) => Promise { + return (payload: string) => + Promise.resolve( + createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey), + ); +} + +/** + * Root directory for a single step invocation's agent-state storage and + * workspace, derived from the sidecar data dir and the run/step/attempt + * coordinates the workflow runtime owns. + * + * The per-step agent storage is a distinct isogit repo, deliberately + * rooted OUTSIDE the workflow-run repo's working tree. The workflow-run + * repo's single writer is the supervisor, and its working tree carries + * the run-event log under `runs//events/...`; nesting a second + * git repo inside that tree would collide with the event subtree and + * with the supervisor's write contract. Rooting the per-step store under + * a dedicated `workflow-step-state/` sibling subtree keyed by the + * workflow-run repo id keeps every step's storage isolated per run and + * per step while never touching the run-event tree. + * + * Resume-attempt invariant: a suspended agent step commits its pending-op + * + turns under the `attempt-N` directory where N is the step's attempt + * at suspend time. On crash-resume the runtime (`runStep` in + * `packages/workflow`) recovers that attempt from the reduced state's + * `currentAttempt` and re-invokes with it, so this function reopens the + * SAME `attempt-N` store and the reactor's `rehydrateGates` finds the + * pending-op the delivered decision correlates against. This is why the + * resume path recovers `currentAttempt` rather than assuming attempt 1: + * attempt 1 is correct only for a step that never retried before + * suspending; a retried-then-suspended step lives under `attempt-2`+, and + * reopening `attempt-1` would rehydrate an empty store and hang on a + * decision that correlates nothing. `createSidecarStepBuildEnv` asserts + * this invariant loudly on the cold path (a resume opening a store that + * lacks the resumed correlationId's pending-op throws). + */ +export function stepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "runs", + args.runId, + "steps", + args.stepId, + `attempt-${String(args.attempt)}`, + ); +} + +/** + * Root directory for a single workflow-run subtree's per-step scratch: + * `/workflow-step-state//runs//`. The cold + * (multi-step) path's per-step `stepStorageRoot` nests under this, so + * reclaiming this subtree on run completion drops every step/attempt the + * run produced in one `rm -rf`. Kept distinct from `stepStorageRoot` so + * the deletion granularity (a whole run, not a single step/attempt) is + * expressed at the call site that owns run-completion cleanup. + */ +function runStepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "runs", + args.runId, + ); +} + +/** + * Stable per-agent scratch root for the WARM single-step agent's + * workspace + tool materialization (tarball-cache + apply-state). Keyed + * by the step identity exactly like the durable conversation store's + * `agent-conversation-state///` (conversation-state.ts), + * NOT by the arbitrary first-message runId. Keying it stably is what + * bounds the warm case: the cached agent reuses ONE workspace across + * every message in the child's lifetime, and that same workspace is + * re-derived (and so survives) across a child respawn, instead of + * stranding a fresh per-runId subtree each time. The whole subtree is + * reclaimed on undeploy, when the deployment's supervisor + child are + * already torn down. Rooted under a `warm/` sibling of the cold `runs/` + * subtree so the undeploy sweep of `workflow-step-state//` + * reclaims both with one removal and the two keyings never collide. + */ +function warmStepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + stepId: string; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "warm", + encodeURIComponent(args.stepId), + ); +} + +async function directoryExists(dir: string): Promise { + try { + return (await fs.promises.stat(dir)).isDirectory(); + } catch (cause) { + if (isErrnoNotFound(cause)) return false; + throw cause; + } +} + +function findApprovalSnapshot( + pendingOperations: readonly PendingOperation[], + correlationId: string, +): ApprovalSnapshot | undefined { + return pendingOperations.find((op) => op.correlationId === correlationId) + ?.approvalSnapshot; +} + +/** + * Read a cold (multi-step) parked step's durable pending operations from its + * on-disk per-attempt isogit store. The store is written at suspend and + * survives while the run is non-terminal -- a parked step keeps the run + * in-flight, so the run-completion reclamation (`cleanupRunStorage`) never + * fires against it. + * + * Returns an empty list when the store directory is absent rather than + * manufacturing an empty repo on the read path: `createIsogitStore` calls + * `initAgentRepo`, which would `mkdir` and init a fresh repo for a + * non-existent dir. The `directoryExists` guard keeps the read a read -- on an + * existing store `initAgentRepo` finds a repo and commits nothing, so no + * signer is needed (`load()` never signs). + */ +export async function readColdParkedPendingOperations(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; +}): Promise { + const storeDir = stepStorageRoot({ + dataDir: args.dataDir, + workflowRunRepoId: args.workflowRunRepoId, + runId: args.runId, + stepId: args.stepId, + attempt: args.attempt, + }); + if (!(await directoryExists(storeDir))) return []; + const store = await createIsogitStore(storeDir); + const { pendingOperations } = await store.load(); + return pendingOperations; +} + +export async function readColdParkedApprovalSnapshot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; + correlationId: string; +}): Promise { + return findApprovalSnapshot( + await readColdParkedPendingOperations(args), + args.correlationId, + ); +} + +/** + * Read a warm (single-step) parked agent's durable pending operations from + * substrate state. A warm agent's pending operations live in its durable + * conversation store, mirrored to the workflow-run substrate under + * `agent-state//`. + * + * Reconstructs that state read-only -- deliberately NOT through + * `DurableConversationRegistry.acquire`, whose first acquire writes and + * commits a substrate restore into the live store and would front-run the warm + * agent's own restore ordering. A respawned child has not rebuilt the live + * store when re-registration runs (resume re-parks without re-invoking the + * step), so the substrate is the only place the pending operations live at that + * moment. Returns an empty list when no durable state exists for the agent. + */ +export async function readWarmParkedPendingOperations(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + stepId: string; +}): Promise { + const agentStateDir = path.join( + args.substrate.getRepoDir(args.workflowRunRepoId), + WORKFLOW_RUN_AGENT_STATE_PREFIX, + encodeURIComponent(args.stepId), + ); + const reconstructed = await reconstructDurableConversation( + agentStateDir, + args.stepId, + ); + if (reconstructed === null) return []; + return reconstructed.pendingOperations; +} + +export async function readWarmParkedApprovalSnapshot(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + stepId: string; + correlationId: string; +}): Promise { + return findApprovalSnapshot( + await readWarmParkedPendingOperations(args), + args.correlationId, + ); +} + +/** + * Project a parked step's durable pending operations down to the minimal + * approval records the resume classifier needs. Filters to `approval` (the only + * control-plane kind today) and keeps only the correlationId and the optional + * epoch-ms deadline; the runtime reconstructs the lost `SignalAwaited` from + * those alone, and must not see the reactor's pending-operation internals. + */ +export function toParkedApprovalOps( + pendingOperations: PendingOperation[], +): ParkedApprovalOp[] { + return pendingOperations + .filter((op) => op.kind === "approval") + .map((op) => ({ + correlationId: op.correlationId, + ...(op.timeoutAt !== undefined ? { timeoutAtMs: op.timeoutAt } : {}), + })); +} + +export interface SidecarStepBuildEnvDeps { + dataDir: string; + workflowRunRepoId: RepoId; + signer: (payload: string) => Promise; + /** + * Deployment mailbox address the supervisor threaded into the child + * (`MAILBOX_ADDRESS`). Used to locate each step's on-disk deploy tree + * for tool materialization (see `stepDeployTreeDir`) AND as the step + * agent's outbound mail `address`: the supervisor signs the agent's + * outbound mail as this address through the host transport (OUTBOUND + * half of mailbox ownership, §3a). For the single-agent deploy this is + * the run address the host registered the agent's `CryptoProvider` + * against. + */ + mailboxAddress: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + * threaded from the host through the spawn-time env. Selects the + * head/step collapse when locating a step's deploy tree + * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step + * deployment reads at the head, a multi-step deployment at the per-step + * address, matching the host's producer push. + */ + stepCount: number; + /** + * Child-side outbound-mail bridge over the upstream control channel + * (OUTBOUND half of mailbox ownership, §3a). The per-step env builder + * wraps it in a supervisor-backed `MessageTransport` it supplies as + * the step agent's `env.transport`; the agent's mail tools call + * `transport.send`, which routes through the bridge to the supervisor + * for the actual signed send. The step agent never holds the signing + * key. + */ + outboundMailBridge: ChildOutboundMailBridge; + /** + * Inbound surface for the step agent's supervisor-backed transport + * (INBOUND half of mailbox ownership, §3b). Bundles the shared mailbox + * watch registry (the same instance the child's control loop fires + * `mailbox.notify` into), the mutation bridge for flag writes and expunge, + * and the call bridge for every other mailbox method. When present, the + * warm agent's mail tools ask the supervisor. Absent for a build that owns + * no inbound mailbox (a spawned child or an onTrigger body), whose + * transport inbound stays inert. + */ + inbound?: SupervisorBackedTransportInbound; + /** Per-step tool-loader caps (cache + registry tarball size). */ + cache: StepToolCacheConfig; + /** + * Adapter registry the step agent resolves inference adapters through. + * The child builds this eagerly at boot from the validated + * `SIDECAR_ADAPTER_MANIFEST` (built-ins merged with operator custom + * adapters) and the env builder sets it on `env.deps`, so a step whose + * source names a custom provider resolves in the child exactly as it + * does on the sidecar main path. Without it the step agent would fall + * back to `createAgent`'s built-ins-only default and a custom-provider + * source would fail to resolve at run time. + */ + adapters: AdapterRegistry; + /** + * Durable-conversation registry for the warm single-step agent + * (design §3c). When present, the env builder swaps the per-run isogit + * `ContextStore` for a per-agent durable store whose conversation is + * mirrored to the workflow-run substrate, and restores the prior + * conversation from the substrate before returning the env (so the + * agent's reactor `load()` and the warm cache's lazy build see the + * restored turns -- including the respawn-rebuild path). Absent for a + * multi-step deploy, whose per-step agents are not warm/long-lived and + * need no cross-run conversation durability. + */ + durableConversation?: DurableConversationRegistry; + /** + * Record the tool-mark floor grants derived from the step's + * materialized tool factories, keyed by the step's base id. The env + * builder is the only place the child holds the loaded factories' + * static `definitions` (name + approval mark), so it derives the floor + * here; the grant evaluator (a sibling closure in the substrate + * factory) reads the recorded floor by base step id and merges it under + * the credentials snapshot's grants at authorization time. Keyed by + * base id (matching the credentials snapshot and inference-source + * table) so a `map` iteration's scoped id shares its base step's floor. + * The recording persists across a warm agent's messages: the env + * builder runs once (first build) but the floor it records must be + * available for every later tool call the warm agent makes. + */ + recordToolMarkFloor: (baseStepId: string, grants: GrantRule[]) => void; + /** + * Feed the step agent's OWN evaluated tool factories into the + * materialization slot, instead of reading a pinned tool-package manifest + * off the deploy tree. Set for the source-ref lineage, whose child holds + * the live `WorkflowDefinition` (re-verified against the approved wire hash + * at run start) and threads each step's live `agent.toolFactories` in as + * `req.agent.toolFactories`. Those are the same `AnnotatedToolFactory` + * callables `materializeStepTools` would produce, bare-named (no + * namespacing loader runs), so the existing tool-bearing agent factory + * consumes them unchanged. + * + * The source-ref lineage stages no `tool-packages-manifest.json`, so + * `materializeStepTools` would find nothing and return empty tools; this + * arm is what actually runs a source workflow's tools. No tool-mark floor + * is recorded on this lineage: a source tool's runtime name is the bare + * `definition.name` the probe's capability walk already emitted a + * `tool:` grant for into the frozen credentials snapshot, so the + * snapshot authorizes it directly (unlike a pinned tool, whose namespaced + * name the walk never saw and which the floor exists to compensate for). + */ + sourceTools: boolean; + /** + * Sidecar-local directory of the materialized workflow-definition closure + * (`env.spawn.closurePackageDir`), present only on the source-ref lineage. + * The source arm materializes each step agent's declared plugin packages + * (`req.agent.plugins`) from this already-laid-out closure -- resolving them + * through the workflow package's `node_modules/` graph, no re-download -- and + * feeds the resulting plugin factories into the per-step plugin chain. A + * plugin package contributes no agent-visible tool factory, so this closure + * load is the only channel that reaches an `env.plugins` a source workflow's + * posix bundle consumes. Undefined on a build that materializes pinned tool + * packages instead, which reads no closure. + */ + closurePackageDir?: string; + /** Pinned tool-package materialization. Unused when `sourceTools` is set. */ + materializeStepTools: MaterializeStepTools; +} + +/** + * Materialize the plugin factories a source-ref step agent declares + * (`req.agent.plugins`) from the workflow's frozen closure. Returns an + * empty list when the agent declares no plugins. Fails closed when plugins + * are declared but the source-ref closure dir is absent -- that pairing is a + * wiring bug (the source arm always carries `closurePackageDir`), and + * silently dropping the plugins would run the workflow without the tools it + * declared and leave the reactor to fail closed on the un-loaded tool. + * + * The closure is already laid out on disk from the deploy-side apply, so + * this is a load-only pass -- no re-download -- resolving each plugin + * package through the workflow package's `node_modules/` graph and + * collecting its `definePlugin` factories. + */ +async function materializeSourcePluginFactories( + deps: SidecarStepBuildEnvDeps, + req: StepInvokeRequest, +): Promise { + const plugins = req.agent.plugins ?? []; + if (plugins.length === 0) { + return []; + } + if (deps.closurePackageDir === undefined) { + throw new Error( + `sidecar workflow-child step invoker: step agent ${JSON.stringify(req.agent.id)} declares plugins ${JSON.stringify(plugins)} but the source-ref closure package dir is absent; a source workflow's plugin factories can only be materialized from its frozen closure`, + ); + } + return loadWorkflowPluginFactoriesFromClosure({ + packageDir: deps.closurePackageDir, + plugins, + }); +} + +/** + * Build the step-invoker `buildEnv` callback the workflow-host's + * adapter consumes. Pulled out of `createSidecarSubstrateFactory` so + * the per-step env construction is observable without standing up the + * full substrate. + * + * The closure reads the per-step source table from the mutable + * reference passed per build, derives the `stepId` / `runId` / `attempt` + * from the runtime's `AuthorizeContext`, resolves the per-step + * `InferenceSource` from that table, and stands up + * a per-step isogit `ContextStore` (also serving as the audit store) + * plus a per-step workspace directory rooted under the run. A + * construction failure (mkdir, isogit init) surfaces here rather than + * being papered over with a stub: the single-step path now always runs + * a real agent against real storage. + */ +export function createSidecarStepBuildEnv( + deps: SidecarStepBuildEnvDeps, +): ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + credentialContext?: SidecarStepCredentialContext, +) => Promise { + // Memoized per buildEnv instance so every step invocation in one child + // shares one director registry loaded from the pinned closure. + let closureDirectors: Promise | undefined; + function directors(): Promise { + closureDirectors ??= + deps.closurePackageDir === undefined + ? Promise.resolve(createDefaultDirectorRegistry()) + : loadWorkflowDirectorRegistryFromClosure({ + packageDir: deps.closurePackageDir, + }); + return closureDirectors; + } + return async ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + credentialContext?: SidecarStepCredentialContext, + ): Promise => { + // Resolve against the live table each build so a source rotation that + // wrote `sourcesRef.current` before this build is reflected in the + // agent this build constructs. A warm agent that is already built does + // not pass through here again, so a rotation does not reach it through + // this path -- this ref covers only a build that has not happened yet. + const resolveStepInferenceSource = createStepInferenceSourceResolver( + sourcesRef.current, + ); + const { stepId, runId, attempt } = req.authzContext; + if (stepId === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.stepId is required for per-step InferenceSource resolution; the workflow runtime must populate stepId on every step-originated invocation", + ); + } + if (runId === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.runId is required to root per-step storage under the run; the workflow runtime must populate runId on every step-originated invocation", + ); + } + if (attempt === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", + ); + } + const sources = resolveStepInferenceSource(stepId); + // The resolver's arktype guarantees a non-empty chain; assert it here so + // the reactor's initial-source pin (element 0) is a checked fact rather + // than an unchecked index. + const activeSource = sources[0]; + if (activeSource === undefined) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, + ); + } + + // Root the per-step scratch (workspace + tool tarball-cache + + // apply-state). The cold (multi-step) path keys it per + // run/step/attempt: each run rebuilds the agent and its scratch, and + // the run's whole `runs//` subtree is reclaimed on run + // completion. The warm single-step path (`durableConversation` + // present) keys it STABLY per agent so the cached agent reuses one + // workspace across every message -- bounding the warm case to one + // dir per agent and letting that workspace survive child respawn -- + // and the subtree is reclaimed on undeploy. The two keyings live + // under disjoint `runs/` and `warm/` sub-roots so neither sweep + // touches the other's tree, and the durable conversation under + // `agent-conversation-state/` is a different root that neither + // sweep touches. + const storeDir = + deps.durableConversation !== undefined + ? warmStepStorageRoot({ + dataDir: deps.dataDir, + workflowRunRepoId: deps.workflowRunRepoId, + stepId, + }) + : stepStorageRoot({ + dataDir: deps.dataDir, + workflowRunRepoId: deps.workflowRunRepoId, + runId, + stepId, + attempt, + }); + // Conversation storage. For the warm single-step agent the + // conversation must survive child respawn, so it is backed by a + // per-agent durable store whose content is mirrored to the + // workflow-run substrate (design §3c); building it here restores the + // prior conversation before the agent's reactor loads. A multi-step + // deploy (no durable registry) keeps the per-run isogit store: its + // per-step agents are not warm/long-lived and have no cross-run + // conversation to carry. The workdir + tools stay per-run in both + // cases -- only the conversation context is durable across runs. + const storage: ContextStore & AuditStore = + deps.durableConversation !== undefined + ? (await deps.durableConversation.acquire(stepId)).storage + : await createIsogitStore(storeDir, deps.signer); + + // Cold-path resume keying assertion (correct-by-construction guard for + // the resume-attempt invariant documented on `stepStorageRoot`). An + // APPROVAL resume re-invocation delivers the correlated decision to the + // reactor, which rehydrates its gate from THIS store's pending + // operations. A re-dispatchable ask-rail approval gate carries a + // `suspendedCall` (the call the approval re-runs); an async-tool pending + // marker shares `kind: "approval"` but has no `suspendedCall`, and on + // resume the reactor clears its gate WITHOUT re-running the approved + // call. So the resume must find a `suspendedCall`-bearing op for its + // correlationId, mirroring the reactor's own discriminator. Two failure + // modes make that false: the runtime reopened the wrong `attempt`'s + // store (`stepStorageRoot` above), so no pending op matches at all and + // the reactor comes up gateless -- a silent forever-hang; or the only + // match is an async marker, so the gate clears and the approved call is + // silently skipped. Make either loud here, at the single seam that both + // opened the store AND knows an approval resume must find its gate, + // rather than letting it surface as a hang or a skipped call. The + // warm path keys its durable store per agent (not per attempt) and + // rehydrates from a different lifecycle, so + // this assertion is cold-path only. An `"input"` resume is exempt by + // construction: its correlationId names the runtime-minted re-arm channel + // awaiting the step's next trigger, not a reactor gate, so no pending + // operation ever exists for it and the delivered decision is sent as a + // plain next turn (no correlation to rehydrate). + if ( + deps.durableConversation === undefined && + req.resume !== undefined && + req.resume.kind === "approval" + ) { + const resumeCorrelationId = req.resume.correlationId; + const loaded = await storage.load(); + const hasPendingGate = loaded.pendingOperations.some( + (op) => + op.correlationId === resumeCorrelationId && + op.suspendedCall !== undefined, + ); + if (!hasPendingGate) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: resume of step ${JSON.stringify(stepId)} (run ${JSON.stringify(runId)}, attempt ${String(attempt)}) reopened a ContextStore with no re-dispatchable approval gate for correlationId ${JSON.stringify(resumeCorrelationId)}. The cold-path store is keyed by attempt (${storeDir}); a resume that finds no suspendedCall-bearing pending operation here means it reopened the wrong attempt's store (the reactor would come up gateless and the decision would correlate against nothing) or matched only an async pending marker (the reactor would clear the gate without re-running the approved call). This is a keying violation, not a recoverable state.`, + ); + } + } + + const workdir = path.join(storeDir, "workspace"); + await fs.promises.mkdir(workdir, { recursive: true }); + + // Assemble the step's tool runtime. Two arms: + // + // - Source-ref lineage (`sourceTools`): run the step agent's OWN + // evaluated tool factories. The child holds the live re-verified + // `WorkflowDefinition`, and the runtime threads each step's live agent + // in as `req.agent`, so `req.agent.toolFactories` are the real callable + // `AnnotatedToolFactory`s -- the same shape `materializeStepTools` + // produces (`LoadedToolFactory === AnnotatedToolFactory`), + // bare-named because no namespacing loader runs. The source deploy + // stages no `tool-packages-manifest.json`, so `materializeStepTools` + // would find nothing here; feeding the evaluated factories straight + // into the slot is what runs a source workflow's tools. `packageName` + // is the factory's bundle `id` (the credential Gate-2 key; a mail/plain + // tool declares no credentials, so it stays inert). Plugin factories -- + // which have no agent slot, so Option A cannot carry them -- are + // materialized separately from the frozen closure the child already + // holds (`materializeSourcePluginFactories`) and fed into the SAME + // `pluginFactories` slot `materializeStepTools` fills for pinned + // packages, so the existing per-step plugin chain wires them onto + // `env.plugins`. + // + // - Pinned tool packages (`materializeStepTools`): materialize the pinned + // tool-package closure (posix, LSP, mail, ...) from its on-disk deploy + // tree, rooted per step under `storeDir` so concurrent steps in one + // child never collide on the tarball cache or the apply-state tree. A + // step with no manifest yields empty tools (the legitimate + // `rawManifestBytes === undefined` case); a present-but-broken manifest + // surfaces loudly through `materializeStepTools` rather than degrading to + // empty tools. + const materialization: StepToolMaterialization = + deps.sourceTools === true + ? { + factories: req.agent.toolFactories.map((factory) => ({ + packageName: factory.id, + declaredCredentials: [], + factory, + })), + pluginFactories: await materializeSourcePluginFactories(deps, req), + } + : await deps.materializeStepTools({ + dataDir: deps.dataDir, + mailboxAddress: deps.mailboxAddress, + stepId, + stepCount: deps.stepCount, + storeDir, + cache: deps.cache, + }); + + // Derive and record the step's tool-mark floor from the just-loaded + // factories' static definitions. A pinned tool loads here in the + // child and never reached the hub's capability walk, so its + // `tool:` grant is absent from the credentials snapshot; the + // recorded floor is what lets the grant evaluator authorize a pinned + // tool against its own static mark. Keyed by base step id so the + // evaluator's `baseStepId(stepId)` lookup resolves for both a plain + // step and a `map` iteration's scoped id. + // + // Skipped on the source-ref lineage: a source tool's runtime name is the + // bare `definition.name` the capability walk already emitted a `tool:` + // grant for, so the credentials snapshot authorizes it directly with the + // correct effect -- no floor is needed, and recording one would blur that + // invariant. + if (deps.sourceTools !== true) { + deps.recordToolMarkFloor( + baseStepId(stepId), + deriveToolMarkFloorGrants( + materialization.factories.map((f) => ({ + packageName: f.packageName, + definitions: f.factory.definitions, + })), + ), + ); + } + + // Supervisor-backed transport for the step agent's mail tools (both + // halves of mailbox ownership, §3a OUTBOUND and §3b INBOUND). Outbound + // (`send`) routes over the control IPC to the supervisor, which performs + // the actual signed send through the host transport as `address`. + // `address` is the deployment mailbox address: the same identity the host + // registered the agent's `CryptoProvider` against, so the outbound mail + // carries the agent's signature with parity to the in-process path. + // Inbound (`deps.inbound`, present for the warm single-step agent) forwards + // every mailbox method other than `send` to the supervisor. A build that + // owns no inbound mailbox (a spawned child) leaves it undefined and those + // methods fail as unwired. Both `transport` and `address` are the env + // keys `@intx/tools-mail`'s sidecar bundle declares in its `requires`. + const transport = createSupervisorBackedTransport( + deps.outboundMailBridge, + deps.mailboxAddress, + deps.inbound, + ); + + // The host owns capability assembly: it builds the RuntimeCapabilities + // bag here (currently `mail.transport`) and puts it on `env.capabilities`, + // so a bundle consumes the assembled bag rather than re-wrapping a raw + // env key of its own. `@intx/tools-mail`'s sidecar bundle (`requires: + // ["capabilities", "address"]`) resolves `mail.transport` from it. + const capabilities = createHarnessRuntimeCapabilities({ transport }); + + // The step env carries `toolCwd`, `transport`, `address`, and the + // assembled `capabilities` beyond `BaseEnv`. `toolCwd` is the working + // tree the posix tools operate on; `capabilities` is what the mail bundle + // reads; `transport` remains a raw env surface for tool packages that + // read it directly. `address` is observability-only. These widen the + // returned `StepEnvBase` structurally, which the buildEnv return type + // (`StepEnvBase`) accepts (a wider object is assignable to the narrower + // type). + const env: StepEnvBase & { + toolCwd: string; + transport: MessageTransport; + address: string; + capabilities: RuntimeCapabilities; + } = { + // Feed the reactor the step's full ordered failover chain and pin + // its initial source to element 0. The reactor resolves the initial + // source by id and fails over forward through `sources`, so this + // restores cross-source failover inside the workflow-child. + sources, + defaultSource: activeSource.id, + storage, + workdir, + // The per-step workspace is both the lock boundary and the tree the + // filesystem tools operate on; a deployed step gets a throwaway + // scratch dir, so the two coincide. + toolCwd: workdir, + audit: storage, + directors: await directors(), + // Resolve inference adapters through the child's boot-built + // registry (built-ins + operator custom adapters), so a + // custom-provider step source resolves in the child the same way + // it does on the sidecar main path rather than hitting + // `createAgent`'s built-ins-only default. + deps: createDependencies(deps.adapters), + transport, + address: deps.mailboxAddress, + capabilities, + }; + // Carry the materialized tool runtime to the tool-bearing + // `agentFactory` via the env's symbol-keyed slot. The step-invoker + // adapter spreads this env (`{ ...envBase, authorize }`) before + // handing it to `agentFactory`; object spread preserves own + // symbol-keyed properties, so the slot survives the spread. + attachStepTools(env, materialization); + // Attach the credential wiring so the `agentFactory` can assemble each + // bundle's consumer-scoped `credentials` capability. Grants are wired as a + // THUNK, resolved (and cast to the sidecar's `GrantRule` grammar, the same + // cast `evaluateGrantsAdapter` makes) only when a package actually needs a + // capability -- a step with no credential-consuming package never reads + // them, so a self-discovery resume that precedes the grants barrier does not + // fault on a missing snapshot. Omitted when no credential context was + // threaded, in which case a credential-consuming tool fails closed and loud + // at its own `resolve("credentials")`. + if (credentialContext !== undefined) { + // Inference resolves its source's secret from the SAME live cell binding + // credentials resolve from, by `credentialId`, so the step's sources + // carry no inline key and the child never holds the cipher key. + env.readCurrentMaterial = createInferenceCredentialResolver( + credentialContext.materialCell, + ); + attachStepCredentialWiring(env, { + materialCell: credentialContext.materialCell, + resolveGrants: stepGrantsResolver(credentialContext, stepId), + providers: credentialContext.providers, + }); + } + return env; + }; +} + +/** + * Real per-step invoker for a spawned child's steps -- a childWorkflow child + * and an onTrigger section body alike. It widens the workflow-runtime + * `StepInvoker` with the child's credentials-backed `authorize` (the seam that + * resolves each tool call against the run's grants), the child's own per-step + * inference `sourcesRef` (built fresh per spawn from the env-delivered body + * sources, disjoint from the top-level's mutable table), and the parent run's + * `onEvent` funnel (so the child's live inference events reach the hub + * timeline). + */ +export type SidecarChildStepInvoker = ( + req: StepInvokeRequest, + authorize: WorkflowAuthorizeFn, + sourcesRef: SourcesSnapshotRef, + onEvent: (event: InferenceEvent) => void, + credentialContext?: SidecarStepCredentialContext, +) => Promise; + +/** + * Inputs required to construct the sidecar's in-process child runtime. + * Lifted out of `createSidecarSubstrateFactory` so the implementation + * is exercisable in isolation (the co-located test wires a hand-built + * substrate/principal/scheduler/invokeStep against this surface). + * + * Sub-namespace scoping: the child runtime is invoked with + * `runId: childRunId`. The runtime body threads that id through every + * `repoStore.read/append/subscribe` call, every `blobs.recordOutput` + * call, and every `signalChannel.deliver/awaitNext` call. The host- + * adapter implementations (`createWorkflowRunRepoStore`, + * `createWorkflowRunBlobSubstrate`, `createWorkflowHostSignalChannel`) + * each compute their on-disk path as `runs//...` against the + * supplied workflow-run repo. The net effect is that the child's + * events land under `runs//events/.json` of the + * parent's workflow-run repo, sibling to the parent's own + * `runs//...` subtree. + * + * Substrate identity: the child reuses the parent's wrapped `RepoStore` + * (the workflow-run pack-pushing wrap installed by the factory) so a + * successful child write fires the same hub pack push the parent's + * writes do. The substrate's signing principal (a workflow-process + * principal scoped to the parent's anchorRunId) is reused verbatim + * because the child runs under the same supervisor authority. + */ +interface SidecarRunChildDeps { + /** Wrapped workflow-run substrate (the factory's `substrate`). */ + substrate: RepoStore; + /** Workflow-run repo identifying the parent's deployment. */ + workflowRunRepoId: RepoId; + /** Workflow-run ref the child reads/writes against. */ + workflowRunRef: string; + /** Principal the child presents on every substrate operation. */ + principal: Principal; + /** Host-process scheduler singleton; shared with the parent. */ + scheduler: Scheduler; + /** + * Step invoker the child runtime delegates per-step invocations to. + * + * The in-process child runs a WorkflowDefinition whose stepIds are + * disjoint from the parent's. The parent's + * `STEP_INFERENCE_SOURCES`-pinned `buildStepEnv` knows only the + * parent's stepIds and throws on any other id; routing the child's + * step invocations through that closure surfaces a misleading + * "no InferenceSource pinned" error for every child step. Callers + * therefore supply a SEPARATE invokeStep for the child. The substrate + * factory's default (`childInvokeStep` in + * `createSidecarSubstrateFactory`) runs a real, tool-bearing agent + * against the child's own staged sources; a test may inject its own. + * One invoker serves every spawned child's steps -- a childWorkflow + * child, an onTrigger section body, and a body's own grandchildren -- + * because each of them runs real inference and so must be able to call + * tools. + * + * The invoker receives the child's credentials-backed `authorize` + * alongside the request, mirroring the workflow-process child's + * `ChildStepInvoker`: the runtime body calls `env.invokeStep` with the + * request only, so the invoker -- not the runtime -- is the seam that + * gates each tool call against the run's grants. + */ + invokeStep: SidecarChildStepInvoker; + /** + * Every spawned body's plaintext inference-source table, keyed by definition + * id, decrypted sidecar-side from the run record and delivered through the + * spawn env. Each body path resolves its own table from here by definition id, + * so the child never holds the sidecar cipher key. A body whose id is absent -- + * a deployment restored from a legacy record written before body sources moved + * into the record -- falls back to the on-disk `dataDir` file. + */ + bodySources: BodyInferenceSources; + /** + * Sidecar data dir. The legacy fallback for a body absent from `bodySources` + * reads its `assets/workflow//sources.json`; the terminal + * childWorkflow / body paths resolve per-step storage under it. Required. + */ + dataDir?: string; + /** + * Grant evaluator the child's credentials-backed `authorize` delegates + * each `(resource, action)` decision to. The parent factory owns the + * sidecar's grant-rule grammar and supplies its adapter here so the + * child resolves authorization against the same evaluator the parent's + * steps use. + */ + evaluateGrants: GrantEvaluator; + /** + * Sidecar-static credential provider registry, shared with the top level. + * The child's build combines it with the run's live material cell and the + * child's capped grants to assemble each bundle's consumer `credentials` + * capability, and to resolve its inference source secret against the same + * material. + */ + credentialProviders: CredentialProviderRegistry; + /** Director registry the child runtime uses; defaults to the canonical built-ins. */ + directors?: DirectorRegistry; + /** + * Sidecar-local directory of the materialized workflow-definition closure + * (`env.spawn.closurePackageDir`), the source-ref lineage's only closure. + * `buildChildRunEnv` loads the child body's declared plugin tool definitions + * from it so a plugin-contributed `tool:` the child loads is declared + * when capping the child's inherited grants, and loads `interchange.actions` + * when this body can reach an action. A body that declares neither a plugin + * nor an action needs no closure read; absent on a lineage that stages no + * closure. A body that reaches an action and has no directory fails at spawn. + */ + closurePackageDir?: string; + /** Clock for timestamp generation; defaults to `() => new Date()`. */ + clock?: () => Date; + /** + * Random id generator for run ids, signal ids, timer ids; defaults to + * a monotonic counter combined with a random suffix. + */ + newId?: (prefix: string) => string; + /** + * Collect the resources the child body declares, the credential consumers + * it instantiates, and narrow the parent grant array to that pair. The + * process that boots the child supplies all three. This factory persists + * the filter's array and does not import the walk. + */ + collectDeclaredResources: ( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: ReadonlyMap, + ) => ReadonlySet; + collectDeclaredCredentialConsumers: ( + definition: WorkflowDefinition, + directors: DirectorRegistry, + pluginDefs: ReadonlyMap, + ) => ReadonlySet; + filterGrantsToDeclaredResources: ( + parentGrants: readonly unknown[], + declared: ReadonlySet, + credentialConsumers: ReadonlySet, + ) => readonly unknown[]; +} + +/** + * Write a spawned child's inherited grants to its own + * `runs//grants.json` in the deployment's workflow-run repo. + * + * The write goes through the child proxy substrate's + * `writeTreePreservingPrefix` -- the only write path the proxy forwards + * to the supervisor. It names the shallow `runs//` prefix, whose + * merge input is only that level's flat blobs (the prefix read does not + * recurse), so the write rebuilds the level with just `grants.json`. This is + * safe ONLY because the sole caller (`capAndPersistChildGrants`) is write-once: + * it invokes this only when no grants file exists yet, i.e. at the run's birth + * before the runtime appends any event, so the `runs//` subtree is + * empty and nothing is dropped. Invoking it over a populated run would delete + * that run's committed `events/`/`blobs/` subtrees. A later event append names + * the nested `runs//events/` prefix, so its own subtree-delete does + * not reach `grants.json` one level up -- the sibling survives. + */ +async function writeChildRunGrants(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + principal: Principal; + ref: string; + childRunId: string; + grants: readonly unknown[]; +}): Promise { + const prefix = `runs/${args.childRunId}/`; + const grantsFile = runGrantsPath(args.childRunId); + const serialized = JSON.stringify({ grants: args.grants }, null, 2); + await args.substrate.writeTreePreservingPrefix( + args.principal, + args.workflowRunRepoId, + args.ref, + { + preservePrefix: prefix, + merge: async (existing) => { + const files: Record = {}; + for (const [k, v] of existing) files[k] = v; + files[grantsFile] = serialized; + return files; + }, + message: `Write inherited run grants for ${args.childRunId}`, + }, + ); +} + +/** + * Construct the `RunChildWorkflow` callback the spawn-child adapter + * delegates to. The returned callback, when invoked with the parent + * runtime's attribution + the parent-allocated `childRunId` + the + * resolved `WorkflowDefinition`, builds a fresh `WorkflowRuntimeEnv` + * scoped to `childRunId` and returns a live handle. The parent drives + * the child across its parks. The child run stays unaddressable. + * + * Abort propagation: the parent-supplied `signal` is the handle's + * local-abort seam. If it aborts -- mid-flight or pre-aborted -- the + * child's own cancel controller aborts, an in-flight step fails, and the + * handle's terminal is `"failed"`. There is no durable `CancelRequested`: + * an in-process child writes through the proxy substrate and cannot sign + * the supervisor cancel one would require. + * + * Resource lifecycle: the child's per-run signal channel is stopped when + * the child settles, from the handle's cleanup, so `resume` and + * `deliverSignal` can still write while a park is open. The blob + * substrate, repo store, and scheduler entries are + * either per-call (no handle to dispose) or shared with the parent + * (the scheduler). + */ +export function createSidecarRunChild( + deps: SidecarRunChildDeps, +): RunChildWorkflow { + const directors = deps.directors ?? createDefaultDirectorRegistry(); + const clock = deps.clock ?? defaultClock; + const newId = deps.newId ?? defaultNewId; + // No `controlPlanePrincipal`: an in-process child tears down through the + // runtime's local-abort seam (the handle aborts the child controller), not a durable + // `CancelRequested`, so it never writes a control-plane cancel this repo store + // would have to sign. Run-body events use the workflow-process `principal`. + // Created once and shared across every child this factory spawns (the + // runtime scopes reads/subscribes by runId), so sibling and grandchild + // spawns route through one repo-store handle rather than a fresh one each. + const repoStore = createWorkflowRunRepoStore({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + }); + // Self-referential `RunChildWorkflow` so a child env's recursive + // `spawnChild` (built via `createInMemorySpawnChild` in `buildChildRunEnv`) + // can route grandchild spawns back through the same adapter. Each invocation + // builds a per-runId env that itself wires a `spawnChild` slot whose + // `runChild` is this same `runChild` constant -- the recursion bottoms + // out when a rung's `WorkflowDefinition` has no `childWorkflow` + // primitive. Sub-namespace scoping continues to hold at every depth + // because `childRunId` flows verbatim into the per-rung + // `blobs`/`signalChannel` and the handle's run, keeping every rung's + // events under `runs//...` of the parent's workflow-run repo. + const runChild: RunChildWorkflow = async ( + { + definition, + childRunId, + input, + parentRunId, + signal, + depth, + maxChildSpawnDepth, + resumeFromEvents, + }, + onEvent, + credentialMaterial, + ) => { + const { + env, + signalChannel, + definition: rewrittenDefinition, + } = await buildChildRunEnv({ + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + onEvent, + // The parent drives this child across its parks and relays each one + // down onto this channel. The child run itself stays unaddressable. + hasUpstreamSignalResolver: true, + ...(credentialMaterial !== undefined + ? { materialCell: credentialMaterial } + : {}), + }); + // Thread this rung's depth/ceiling into the child run so its own + // childWorkflow spawns keep counting against the tree-wide bound (the + // guard lives in the runtime's runChildWorkflow). Without this the + // in-process sidecar recursion would reset to depth 0 each rung. + // A parent abort tears the child down through the handle's local-abort + // seam, not a control-plane cancel: the in-process child writes through + // the workflow-run PROXY substrate, which cannot sign the supervisor + // `CancelRequested` a cancel needs (the kind handler refuses a + // workflow-process-signed cancel). Local teardown aborts the child's own + // controller directly, so an in-flight step fails and the child settles + // `failed` under its own principal, with no durable cancel. + // The channel stays up until the child settles. `resume` and + // `deliverSignal` still write to it while a park is open. + return createSuspendableChildHandle(env, { + definition: rewrittenDefinition, + childRunId, + input, + depth, + maxChildSpawnDepth, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), + signal, + surfaceInputPark: true, + cleanup: () => signalChannel.stop(), + }); + }; + return runChild; +} + +/** + * Construct the `RunSuspendableChild` callback the suspendable-spawn + * adapter delegates to. Like {@link createSidecarRunChild}, it returns a + * live {@link SuspendableChildHandle}. The caller (`runOnTrigger`) drives + * that handle across the body's approval parks. + * + * Park surfacing: the built env's `onPark` sink translates the child body's + * control-plane parks into the handle's `next()` stream. An `"approval"` + * park is queued for the caller to proxy up on the same correlation; the + * caller's granted decision returns through `resume`, which delivers it on + * the child's own signal channel so the parked step unblocks. A body that + * parks on a control-plane `"input"` channel is a nested onTrigger re-arm + * the suspendable seam does not service -- the caller proxies approvals + * only, so nothing would ever deliver that input and the child would park + * forever. Rather than drop the park and hang, `onPark` surfaces it as a + * hard error on `next()` and tears the child down locally so the section run + * ends loudly (failed). + * + * Signal-channel lifecycle: the channel stays up until the child settles, + * so `resume` can deliver on an open park. Teardown is the handle's + * cleanup, the same place {@link createSidecarRunChild} stops its channel. + * That runs when the run reaches a terminal: normal completion, a + * parent-abort local teardown, an illegal-input-park local teardown, or a + * runtime failure. Stopping the channel on each `next()` would drop it + * when a parent abort makes `runOnTrigger` stop calling `next()` mid-park. + * + * Abort propagation: the parent-supplied `signal` tears the body down through + * `createSuspendableChildHandle`'s local-teardown seam -- the in-process body + * runs under the `workflow-process` principal and cannot sign the control-plane + * `CancelRequested` a durable cancel needs, so on abort its own step fails and + * the run settles `failed` (not `cancelled`), and the terminal-tied teardown + * runs. + */ +export function createSidecarSpawnSuspendableChild( + deps: SidecarRunChildDeps, +): RunSuspendableChild { + const directors = deps.directors ?? createDefaultDirectorRegistry(); + const clock = deps.clock ?? defaultClock; + const newId = deps.newId ?? defaultNewId; + // No `controlPlanePrincipal`: an in-process body tears down through the shared + // handle's local-abort seam (`createSuspendableChildHandle`), not a durable + // `CancelRequested`, so it never writes a control-plane cancel this repo store + // would have to sign. Run-body events use the workflow-process `principal`. + const repoStore = createWorkflowRunRepoStore({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + }); + // A body's own `childWorkflow` grandchildren use this same live handle: + // `buildChildRunEnv` wires the body env's `spawnChild` to + // `createSidecarRunChild`, so a park inside the grandchild relays through + // the body. A nested onTrigger inside a body still fails loud. + // `spawnSuspendableChild` is not wired on a body env. + const runChild = createSidecarRunChild(deps); + + return async ( + { + definition, + childRunId, + input, + parentRunId, + signal, + depth, + maxChildSpawnDepth, + resumeFromEvents, + }, + onEvent, + credentialMaterial, + ) => { + const { + env: baseEnv, + signalChannel, + definition: rewrittenDefinition, + } = await buildChildRunEnv({ + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + // The live event sink is always threaded (both a body step and a + // grandchild childWorkflow step run a real agent). The body's own steps + // and its childWorkflow grandchildren, built via the internal + // `createSidecarRunChild(deps)` above, both run through the tool-bearing + // `deps.invokeStep`. + onEvent, + // The run's live credential-material cell so the body's inference resolves + // its source secret against the parent's current delivery; a grandchild + // spawned from the body inherits it through the recursive spawnChild. + // + // Park-aware: the container drives this body across its parks and relays + // a decision back down onto the body's own channel, so a park here is + // answerable even though the body run carries no address of its own. + hasUpstreamSignalResolver: true, + ...(credentialMaterial !== undefined + ? { materialCell: credentialMaterial } + : {}), + }); + + return createSuspendableChildHandle(baseEnv, { + definition: rewrittenDefinition, + childRunId, + input, + depth, + maxChildSpawnDepth, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), + signal, + cleanup: () => signalChannel.stop(), + surfaceInputPark: false, + }); + }; +} + +/** + * Read the parent run's grants, cap them to what `definition` declares, and + * persist the capped set as the child run's own `runs//grants.json` + * -- the file the next spawn hop (a childWorkflow grandchild) reads back as its + * ceiling. Returns the capped grants so a caller building a fresh child env can + * also key its credentials snapshot on them. + * + * `definition` MUST be the PRE-rewrite body, its childWorkflow grandchildren + * still INLINE: both collectors skip a `{ ref }` body, so a rewritten + * definition would drop the grandchild's declared resources and the factories + * only its inline body instantiates, and under-authorize it. Every birth path + * materializes a run's grants file, so a + * missing parent file is a defect, not a run that legitimately holds none -- + * fail closed. The cap only removes rules (the parent stays the ceiling), + * matching the top level's "authority bounded by declared capabilities" model. + * + * The grants file is WRITE-ONCE per run: a run's authorization ceiling is fixed + * at its birth, so this reads back an existing `runs//grants.json` + * rather than rewriting it. Re-writing on a resume re-drive is not just + * redundant, it is CORRUPTING: `writeChildRunGrants` commits at the shallow + * `runs//` prefix through a substrate writer separate from the + * runtime's event-log writer, so a re-write racing the runtime's replay + * re-appends on the shared repo regresses another run's event seq (a + * single-writer-invariant violation), and its wholesale subtree-delete drops the + * run's committed `events/`/`blobs/` (the prefix read is non-recursive, so the + * merge cannot carry them forward). The read-back returns the same capped set + * the birth-time write persisted, so a fresh child-env caller keys its + * credentials snapshot on identical grants either way. + */ +async function capAndPersistChildGrants(args: { + deps: SidecarRunChildDeps; + directors: ReturnType; + definition: WorkflowDefinition; + childRunId: string; + parentRunId: string; +}): Promise { + const { deps, directors, definition, childRunId, parentRunId } = args; + const existingChildGrants = await readRunGrants({ + repoStore: deps.substrate, + anchorRunId: deps.workflowRunRepoId.id, + runId: childRunId, + }); + if (existingChildGrants !== undefined) return existingChildGrants; + const parentGrants = await readRunGrants({ + repoStore: deps.substrate, + anchorRunId: deps.workflowRunRepoId.id, + runId: parentRunId, + }); + if (parentGrants === undefined) { + throw new Error( + `sidecar runChild: parent run ${parentRunId} has no grants file at ${runGrantsPath(parentRunId)}; refusing to spawn child ${childRunId} under-authorized`, + ); + } + const pluginDefs: ReadonlyMap = + deps.closurePackageDir === undefined + ? new Map() + : await loadWorkflowPluginToolDefinitionsFromClosure({ + packageDir: deps.closurePackageDir, + plugins: collectDeclaredPluginNames(definition), + }); + const declaredResources = deps.collectDeclaredResources( + definition, + directors, + pluginDefs, + ); + const credentialConsumers = deps.collectDeclaredCredentialConsumers( + definition, + directors, + pluginDefs, + ); + const childGrants = deps.filterGrantsToDeclaredResources( + parentGrants, + declaredResources, + credentialConsumers, + ); + await writeChildRunGrants({ + substrate: deps.substrate, + workflowRunRepoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + childRunId, + grants: childGrants, + }); + return childGrants; +} + +/** + * Build the per-childRunId `WorkflowRuntimeEnv` a spawned child runs + * against: inherit the parent's grants, assemble the child's credentials + * snapshot, and wire the per-run repo store / blob substrate / signal + * channel plus a recursive `spawnChild`. Returned alongside the child's + * signal channel so the caller can `stop()` it once the child settles. + * Shared by the child-drive callers so the env construction lives in one + * place. + */ +async function buildChildRunEnv(args: { + deps: SidecarRunChildDeps; + directors: ReturnType; + clock: () => Date; + newId: (prefix: string) => string; + repoStore: ReturnType; + runChild: RunChildWorkflow; + definition: WorkflowDefinition; + childRunId: string; + parentRunId: string; + /** + * Per-run live inference-event sink, threaded from the parent run's event + * channel. Required for both paths -- a childWorkflow step and an onTrigger + * body step both run a real agent whose inference the hub stream must see, so + * a missing sink is a wiring defect, not a silent drop. + */ + onEvent: (event: InferenceEvent) => void; + /** + * The parent run's live credential-material cell. Threaded so the child's + * inference resolves its source secret by `credentialId` against the run's + * current delivery -- reached live through the shared reference on a rotation. + * Absent when a non-sidecar executor carries no credential material, in which + * case no credential context is assembled and the child's inference reader + * stays unset. + */ + materialCell?: CredentialMaterialCell; + /** + * Whether a park in this child can be answered from outside it. The two + * seams that share this builder differ precisely here, so neither gets a + * default: a suspendable body is driven across its parks by a container + * that relays a decision back down, while a childWorkflow child is + * driven by its parent, which relays each park down onto this channel. + */ + hasUpstreamSignalResolver: boolean; +}): Promise<{ + env: WorkflowRuntimeEnv; + signalChannel: ReturnType; + definition: WorkflowDefinition; +}> { + const { + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + onEvent, + materialCell, + } = args; + // A rung may itself embed a child (grandchild recursion) as an inline + // `childWorkflow`. Lift each to an internal `{ ref }` and run the rewritten + // definition whose children are refs -- the shape the runtime dispatches -- + // and keep the lifted definitions in an in-memory map the rung's own + // terminal resolver serves from, so a grandchild spawns with no on-disk read + // at any depth. Returned to the caller so the handle drives the rewritten + // form. + const { workflow: rewrittenDefinition, bodies: grandchildBodies } = + rewriteInlineChildWorkflowBodies(definition); + const grandchildMap = new Map( + grandchildBodies.map((b) => [b.ref, b.definition]), + ); + // Enumerate any `loop` bodies nested in this spawned body so a loop can run + // here (the runtime needs `env.loopFns` + `env.spawnLoopIteration`, wired + // below). A loop body stays inline on its primitive, so mint a ref-keyed copy + // for the loop-iteration host, keep the pre-rewrite form for the per-iteration + // grant cap, and merge each loop body's own childWorkflow grandchildren into + // `grandchildMap` so a grandchild spawned from a loop body resolves in-memory. + // Mirrors the top-level loop-body registration in run-child.ts. + const loopBodies = enumerateInlineLoopBodies(definition); + const loopBodiesMap = new Map(); + const loopBodyPreRewrite = new Map(); + for (const loopBody of loopBodies) { + loopBodyPreRewrite.set(loopBody.ref, loopBody.definition); + const bodyRewrite = rewriteInlineChildWorkflowBodies(loopBody.definition); + loopBodiesMap.set(loopBody.ref, bodyRewrite.workflow); + for (const grandchild of bodyRewrite.bodies) { + grandchildMap.set(grandchild.ref, grandchild.definition); + } + } + // Resolve the loop while/carry fns from the closure so a loop in this body can + // run. Only load when the body contains a loop. If it declares a loop but no + // closure is wired, that is a wiring defect -- fail loud, mirroring the + // establish-time posture rather than deferring to a mid-run resolve failure. + let loopFns: LoopFnRegistry | undefined; + if (loopBodies.length > 0) { + if (deps.closurePackageDir === undefined) { + throw new Error( + "sidecar child: a loop is nested in this spawned body but deps.closurePackageDir is missing; the loop while/carry fns cannot be resolved", + ); + } + loopFns = await loadWorkflowLoopFnsFromClosure({ + packageDir: deps.closurePackageDir, + }); + eagerlyResolveLoopFns( + [ + rewrittenDefinition, + ...loopBodiesMap.values(), + ...grandchildMap.values(), + ], + loopFns, + ); + } + // Read the parent's grants, cap them to what this body declares, and persist + // the capped set as the child's own `runs//grants.json` (the file + // a grandchild reads as its ceiling). `definition` here is the childWorkflow- + // inline form, so the cap keeps a grandchild's declared resources. + const childGrants = await capAndPersistChildGrants({ + deps, + directors, + definition, + childRunId, + parentRunId, + }); + // The child's credentials snapshot applies the capped grant set + // uniformly across every step the child definition declares, keyed on + // each step's id (the same shape the deploy-time and per-run snapshot + // assemblies produce). The in-process child has no per-step mail + // address, so the snapshot's `address` mirrors the step id -- + // `createCredentialsBackedAuthorize` reads only `grants`. + // + // "Every step the child definition declares" reaches past `stepOrder` into + // this body's `loop` bodies: a loop iteration runs under the inherited env, + // so its steps authorize against THIS snapshot under their own plain ids. A + // snapshot built from `stepOrder` alone leaves a loop nested one rung down + // with no entry, and the authorize throws on the body's first tool call. The + // capped set applies to them unchanged -- the cap is taken once, against the + // spawned body, and a loop inherits its enclosing run's authority rather than + // narrowing it again. + const credentialStepIds = new Set(); + walkWorkflowSteps({ + definition: rewrittenDefinition, + descent: LOOP_BODY_DESCENT, + context: "sidecar child credentials snapshot: ", + visit: ({ stepId }) => { + credentialStepIds.add(stepId); + }, + }); + const contentHash = await hashGrants(childGrants); + const credentialsSnapshot: CredentialsSnapshot = { + steps: [...credentialStepIds].map((stepId) => ({ + stepId, + address: stepId, + grants: childGrants, + contentHash, + })), + }; + const blobs = createWorkflowRunBlobSubstrate({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + runId: childRunId, + ref: deps.workflowRunRef, + }); + const signalChannel = createWorkflowHostSignalChannel({ + repoStore: deps.substrate, + runtimeStore: args.repoStore, + principal: deps.principal, + repoId: deps.workflowRunRepoId, + ref: deps.workflowRunRef, + runId: childRunId, + readState: () => emptyState(childRunId), + newId: () => newId("sig"), + clock, + }); + // The child's `env.authorize` binds to the inherited credentials + // snapshot: each `(resource, action)` decision looks up the step's + // grants and delegates to the parent factory's grant evaluator. The + // runtime body stores this on the env; the child's `invokeStep` + // wrapper below is the seam that consults it per tool call, and an + // action step's `EffectContext` calls it directly for each effect. + const credentialsRef: CredentialsSnapshotRef = { + current: credentialsSnapshot, + }; + const authorize = createCredentialsBackedAuthorize( + credentialsRef, + deps.evaluateGrants, + ); + // An action this env can run needs its own invoker: the same + // `createDefaultActionInvoker` the top-level run uses, closed over THIS + // body's capped authorize and a fresh ledger. Load only when the pre-rewrite + // body reaches an action under `LOOP_BODY_DESCENT` (loop bodies share this + // env; an inline childWorkflow or onTrigger does not). A grandchild action + // is resolved when that grandchild builds its own env. A reachable action + // with no closure directory is a wiring defect -- fail at spawn, before the + // handler can run. + let actionReachable = false; + walkWorkflowSteps({ + definition, + descent: LOOP_BODY_DESCENT, + context: "sidecar child action handlers: ", + visit: ({ step }) => { + if (step.kind === "action") actionReachable = true; + }, + }); + let actionResolver: ((ref: string) => ActionHandler) | undefined; + if (actionReachable) { + if (deps.closurePackageDir === undefined) { + throw new Error( + "sidecar child: an action is reachable in this spawned body but deps.closurePackageDir is missing; the action handlers cannot be resolved", + ); + } + actionResolver = await loadWorkflowActionHandlersFromClosure({ + packageDir: deps.closurePackageDir, + }); + eagerlyResolveActionHandlers([definition], actionResolver); + } + const drain = createNoopDrainController(rewrittenDefinition); + // Both the terminal childWorkflow path and the onTrigger BODY path run a real + // agent that resolves inference against its OWN per-step source table, keyed by + // the rewritten ref, and funnels live events to the parent run's channel. So + // `onEvent` is required for both; a missing one is a wiring defect, not a + // silent drop. + if (onEvent === undefined) { + throw new Error( + "sidecar child: onEvent is missing; child inference events would be silently dropped from the hub stream", + ); + } + const childOnEvent = onEvent; + // Resolve fresh per spawn into a `sourcesRef` disjoint from the top-level's + // mutable table, so a top-level source rotation never leaks into a child. The + // sources are delivered plaintext through the spawn env (`deps.bodySources`), + // decrypted sidecar-side, so the child never holds the cipher key. + const sourcesRef: SourcesSnapshotRef = { + current: await resolveBodyStepSources(deps, rewrittenDefinition.id), + }; + // Recursive `spawnChild`: a grandchild embedded inline in this rung is + // resolved from the in-memory map lifted above and flows back into this same + // `runChild` callback. Inject THIS run's event funnel (mirroring the rung-0 + // wrap in run-child.ts) so the grandchild's agent steps ride this run's + // channel; the runtime env keeps the narrow `SpawnChildWorkflow` (no event + // slot). Sub-namespace scoping holds at every depth via `childRunId`. + const spawnHost = createInMemorySpawnChild({ + bodies: grandchildMap, + runChild, + }); + const spawnChild: WorkflowRuntimeEnv["spawnChild"] = (spawnInput) => + spawnHost(spawnInput, childOnEvent, materialCell); + // Assemble the per-step credential context from the run's live material cell + // (threaded in from the parent) when one is present, so the child's inference + // resolves its source secret by `credentialId` against the parent's current + // delivery. Every spawned child's steps are tool-bearing and may run + // actions, so the context resolves the child's capped grants (mirroring the + // top level's snapshot lookup) and carries the sidecar-static providers for + // the tool and action `credentials` capabilities. Absent when no material was + // threaded (a non-sidecar executor), which leaves the child's inference + // reader unset and gives action handlers no credential source. + const childCredentialContext: SidecarStepCredentialContext | undefined = + materialCell === undefined + ? undefined + : { + materialCell, + resolveStepGrants: (stepId) => + resolveSnapshotStepGrants( + credentialsSnapshot, + stepId, + (lookupStepId) => + `sidecar child credential wiring: credentials snapshot has no entry for step ${lookupStepId}`, + ), + providers: deps.credentialProviders, + }; + // Per-step invocation seam. The runtime body invokes `env.invokeStep` with + // the request alone; the wrapper forwards the child's credentials-backed + // authorize (so each tool call gates against the inherited grants), the run's + // `sourcesRef`, the event funnel, and the credential context (so the step's + // inference reads the run's live material). One invoker serves both the + // childWorkflow path and the onTrigger body path: each runs a real + // tool-bearing agent through the source-tools arm against the same + // `sourcesRef`. + const invokeStep: WorkflowRuntimeEnv["invokeStep"] = (req) => + deps.invokeStep( + req, + authorize, + sourcesRef, + childOnEvent, + childCredentialContext, + ); + let effects: ReturnType | undefined; + let invokeAction: ReturnType | undefined; + if (actionResolver !== undefined) { + effects = createInMemoryEffectLedger(); + // An action handler resolves its declared handles against the run's live + // material and this body's capped grants. With no material threaded (a + // non-sidecar executor) there is no credential source, so an action that + // declares credentials fails before its handler runs. + invokeAction = createDefaultActionInvoker( + authorize, + effects, + actionResolver, + childCredentialContext === undefined + ? unavailableActionCredentials + : createSidecarActionCredentialsFactory(childCredentialContext), + ); + } + const env: WorkflowRuntimeEnv = { + repoStore, + scheduler: deps.scheduler, + signalChannel, + blobs, + directors, + authorize, + invokeStep, + spawnChild, + clock, + newId, + drain, + hasUpstreamSignalResolver: args.hasUpstreamSignalResolver, + ...(loopFns !== undefined ? { loopFns } : {}), + ...(effects !== undefined && invokeAction !== undefined + ? { effects, invokeAction } + : {}), + }; + // Wire loop-iteration spawning for a `loop` nested in this body. Assigned + // AFTER the env literal because the iteration host closes over `env`: a loop + // iteration re-enters THIS body env, so a nested loop composes and inherits + // the body's step invoker, capped grants, invokeAction, effect ledger, and + // in-memory spawnChild. + // + // This deliberately REPLICATES the top-level loop host in run-child.ts rather + // than sharing a helper. The two differ in the grants seam (here + // `capAndPersistChildGrants` is in scope and called directly; the top level + // injects it as a binding), and the loop-in-body crash-resume path is not + // yet proven identical to the top level's. Extract a shared helper only once + // a deployed loop-in-body resume test shows the two paths match. + // + // Boundary: the suspendable-child seam services APPROVAL parks only. A loop + // iteration in a body that awaits an externally-delivered signal, or re-arms + // an onTrigger, inherits that approvals-only limit and cannot park on an + // external input channel. + if (loopFns !== undefined) { + const loopIterationHost = createInMemorySpawnSuspendableChild({ + bodies: loopBodiesMap, + runSuspendableChild: async (loopInput, _onEvent) => { + const preRewriteBody = loopBodyPreRewrite.get(loopInput.definitionRef); + if (preRewriteBody === undefined) { + throw new Error( + `sidecar child: no pre-rewrite loop body for ref ${loopInput.definitionRef}`, + ); + } + // Cap the iteration's grants against the body's own grants (the + // iteration's parent IS the body run) and persist them before the + // iteration appends its first event. + await capAndPersistChildGrants({ + deps, + directors, + definition: preRewriteBody, + childRunId: loopInput.childRunId, + parentRunId: loopInput.parentRunId, + }); + const iterationSignalChannel = createWorkflowHostSignalChannel({ + repoStore: deps.substrate, + runtimeStore: args.repoStore, + principal: deps.principal, + repoId: deps.workflowRunRepoId, + ref: deps.workflowRunRef, + runId: loopInput.childRunId, + readState: () => emptyState(loopInput.childRunId), + newId: () => newId("sig"), + clock, + }); + return createLoopIterationHandle(env, { + definition: loopInput.definition, + childRunId: loopInput.childRunId, + input: loopInput.input, + depth: loopInput.depth, + maxChildSpawnDepth: loopInput.maxChildSpawnDepth, + ...(loopInput.resumeFromEvents !== undefined + ? { resumeFromEvents: loopInput.resumeFromEvents } + : {}), + signal: loopInput.signal, + signalChannel: iterationSignalChannel, + cleanup: () => iterationSignalChannel.stop(), + }); + }, + }); + env.spawnLoopIteration = (spawnInput) => + loopIterationHost(spawnInput, childOnEvent); + } + return { env, signalChannel, definition: rewrittenDefinition }; +} + +/** + * Resolve a spawned body's per-step inference sources by definition id. The + * primary path is sidecar-mediated: the sources arrive plaintext in + * `deps.bodySources`, decrypted sidecar-side from the sealed run record, so the + * child holds no cipher key. + * + * LEGACY FALLBACK: a body whose id is absent from the delivered set -- a + * deployment restored from a record written before body sources moved into the + * record -- reads its on-disk plaintext `sources.json`. That file predates the + * change and is plaintext, so the fallback constructs no cipher and the child + * stays key-free. Removable once no restorable record predates the + * record-carried body sources. + */ +async function resolveBodyStepSources( + deps: SidecarRunChildDeps, + definitionId: string, +): Promise { + const delivered = deps.bodySources[definitionId]; + if (delivered !== undefined) { + return delivered; + } + if (deps.dataDir === undefined) { + throw new Error( + `sidecar child: body ${definitionId} is absent from the delivered sources and deps.dataDir is missing, so its legacy on-disk sources cannot be read`, + ); + } + return readChildStepInferenceSources(deps.dataDir, definitionId); +} + +/** + * Legacy fallback reader (see `resolveBodyStepSources`): a spawned body's + * plaintext per-step pins from `${dataDir}/assets/workflow//sources.json`, + * staged by a pre-record deploy. Parsed through the same + * `parseStepInferenceSources` boundary the top-level `STEP_INFERENCE_SOURCES` env + * entry uses. Only reached for a body the delivered set does not carry, so a + * missing or malformed file is a defect and surfaces loudly. + */ +async function readChildStepInferenceSources( + dataDir: string, + childRef: string, +): Promise { + const sourcesPath = path.join( + dataDir, + "assets", + "workflow", + childRef, + "sources.json", + ); + let raw: string; + try { + raw = await fs.promises.readFile(sourcesPath, "utf8"); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar child: failed to read child inference sources at ${sourcesPath}: ${reason}`, + { cause }, + ); + } + return parseStepInferenceSources(raw); +} + +function defaultClock(): Date { + return new Date(); +} + +let runChildIdCounter = 0; +function defaultNewId(prefix: string): string { + runChildIdCounter += 1; + return `${prefix}-${String(runChildIdCounter)}-${Math.random().toString(36).slice(2, 8)}`; +} + +/** + * Build a `SubstrateFactory` closed over the supplied dependencies. + * The sidecar binding passes tool materialization and the grant cap. + * + * Construction order: + * 1. Narrow the `substrateConfig` record against the typed schema. + * A missing or empty key already threw inside the helper; this + * pass enforces the exact shape the factory consumes. + * 2. Open a bare `RepoStore` via `createAgentRepoStore` against the + * sidecar's data dir and Ed25519 keypair. This store backs the + * child's read-only operations against the workflow-run repo; + * the on-disk repo is shared with the supervisor's substrate so + * reads see whatever the supervisor has committed. + * 3. Construct a proxy `RepoStore` whose + * `writeTreePreservingPrefix` forwards over the upstream control + * channel via the substrate-write bridge. The supervisor's + * handler runs its own substrate's `writeTreePreservingPrefix` + * (wrapped at the boot edge with the pack-push facade) under the + * per-repo lock and replies with the resulting `commitSha`. + * 4. Start the host-process scheduler singleton against the proxy + * substrate, then adapt it to the runtime's `Scheduler` shape. + * 5. Construct the production `invokeStep` and `spawnChild` + * adapters. + * 6. Return the `RunWorkflowChildBindings` the runtime body + * consumes, with the proxy store in the `substrate` slot. + */ +export function createSidecarSubstrateFactory( + deps: SidecarSubstrateFactoryDeps, +): SubstrateFactory { + const createBareRepoStore = + deps.createBareRepoStore ?? + (({ dataDir, signingKey }) => + createAgentRepoStore({ dataDir, signingKey }).repoStore); + + return async (env: SubstrateFactoryEnv) => { + const validated = SubstrateConfig(env.substrateConfig); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config failed validation: ${validated.summary}`, + ); + } + + const stepInferenceSources = parseStepInferenceSources( + validated.STEP_INFERENCE_SOURCES, + ); + + // Build the child's adapter registry eagerly at boot from the + // operator-supplied manifest. `loadAdapterRegistry` imports every + // custom module now, so a bad specifier crashes the child loudly at + // construction rather than silently degrading to built-ins-only at + // first resolve. The closure registry the sidecar built at its own + // boot edge cannot cross the fork; the child rebuilds an equivalent + // one from the serialized-and-revalidated manifest. + const childAdapterRegistry = await loadAdapterRegistry( + parseAdapterManifest(validated.SIDECAR_ADAPTER_MANIFEST), + ); + + const signingKey = { + publicKey: hexDecode( + validated.SIDECAR_SIGNING_PUBLIC_KEY, + "SIDECAR_SIGNING_PUBLIC_KEY", + ), + privateKey: hexDecode( + validated.SIDECAR_SIGNING_PRIVATE_KEY, + "SIDECAR_SIGNING_PRIVATE_KEY", + ), + }; + + const bareStore: RepoStore = createBareRepoStore({ + dataDir: validated.SIDECAR_DATA_DIR, + signingKey, + }); + + const workflowRunRepoId = { + kind: "workflow-run" as const, + id: validated.WORKFLOW_RUN_REPO_ID, + }; + const principal: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: env.spawn.anchorRunId, + }; + + // Proxy substrate: writes are forwarded over IPC into the + // supervisor's substrate; reads consult the bare on-disk store. + // The supervisor is the sole writer of the workflow-run ref so + // the child's writes never race against the supervisor's + // claim-check writes (inbox / processing / consumed). + const substrate: RepoStore = createProxyWorkflowRunRepoStore({ + bareStore, + bridge: env.substrateWriteBridge, + workflowRunRepoId, + }); + + // INBOUND half of mailbox ownership (§3b). One watch registry per child, + // created at boot and shared by the step agent's supervisor-backed + // transport (its `watch` registers callbacks here, backing `mail_wait`) and + // the child's control loop (which fires each `mailbox.notify` into it). The + // registry rides out on the returned bindings so `runWorkflowChild` routes + // notifications to this same instance. Mail methods other than `send` ask + // the supervisor; this child does not open the mailbox to answer them. + const mailboxWatchRegistry = createMailboxWatchRegistry(); + const transportInbound: SupervisorBackedTransportInbound = { + watchRegistry: mailboxWatchRegistry, + mutationBridge: env.mailboxMutationBridge, + callBridge: env.mailboxCallBridge, + }; + + const hostScheduler = createWorkflowHostScheduler({ + repoStore: substrate, + runtimeStore: createWorkflowRunRepoStore({ + substrate, + principal, + repoId: workflowRunRepoId, + ref: validated.WORKFLOW_RUN_REF, + }), + principal, + listActiveDeployments: () => [workflowRunRepoId], + ref: validated.WORKFLOW_RUN_REF, + clock: () => new Date(), + }); + await hostScheduler.start(); + const scheduler = adaptHostScheduler(hostScheduler); + + // The single-step / top-level path runs a real agent. The per-step + // env builder stands up real per-step storage/workdir/audit/directors + // rooted under the run (see `createSidecarStepBuildEnv`), resolving + // the per-step `InferenceSource` from the pinned table; the real + // step-invoker instantiates the step's agent via `createAgent`, + // delivers the resolved input as a synthesized inbound message, and + // captures the agent's reply as the step output. + const stepToolCache: StepToolCacheConfig = { + cacheMaxBytes: parseByteCap( + validated.SIDECAR_CACHE_MAX_BYTES, + "SIDECAR_CACHE_MAX_BYTES", + ), + registryMaxTarballBytes: parseByteCap( + validated.SIDECAR_REGISTRY_MAX_TARBALL_BYTES, + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + ), + }; + + // The single-step / top-level path runs a real agent with REAL + // tools materialized in-child. The per-step env builder stands up + // real per-step storage/workdir/audit/directors rooted under the + // run (see `createSidecarStepBuildEnv`), resolves the per-step + // `InferenceSource`, and materializes the step's pinned + // tool-package closure (posix, LSP, mail, ...) from its on-disk + // deploy tree -- rooted per step so concurrent steps in one child + // never collide on the tarball cache or apply-state. The + // tool-bearing `agentFactory` below attaches those factories to the + // step's `AgentDefinition` and builds the plugin chain. + // Durable-conversation registry for the warm single-step agent + // (design §3c). Built only when the deployment is warm-kept: the + // sole long-lived agent's conversation must survive child respawn, + // so it is mirrored to the workflow-run substrate at a per-agent + // path. A multi-step deploy leaves this `undefined` -- its per-step + // agents are not warm/long-lived (§3b), so they carry no cross-run + // conversation and keep the per-run isogit store. The registry lives + // for the child's lifetime; on respawn the child rebuilds it empty + // and each store restores its prior snapshot from the substrate on + // first acquire. + const conversationSigner = createStepStorageSigner(signingKey); + const durableConversation: DurableConversationRegistry | undefined = env + .spawn.warmKeep + ? createDurableConversationRegistry({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + substrate, + principal, + signer: conversationSigner, + }) + : undefined; + + // Per-step tool-mark floor grants, keyed by base step id. The step + // env builder derives and records each step's floor from its + // materialized (pinned) tool factories; the grant evaluator reads it + // by base step id and merges it under the credentials snapshot's + // grants so a pinned tool authorizes against its own static mark. The + // map lives for the factory's (child's) lifetime, so a warm agent's + // floor -- recorded on its single first build -- remains available + // for every later tool call it makes. + const toolMarkFloorByStep = new Map(); + + const buildStepEnv = createSidecarStepBuildEnv({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + signer: conversationSigner, + mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, + outboundMailBridge: env.outboundMailBridge, + cache: stepToolCache, + adapters: childAdapterRegistry, + recordToolMarkFloor: (stepId, grants) => { + toolMarkFloorByStep.set(stepId, grants); + }, + materializeStepTools: deps.materializeStepTools, + // Source-ref is the only deploy lineage: the child runs each step agent's + // own evaluated tool factories (fed from `req.agent.toolFactories`) from + // the materialized closure, never a pinned tool-package manifest off a + // deploy tree. + sourceTools: true, + // The materialized closure dir the source arm materializes each step + // agent's declared plugin packages from. Always present (source-ref only). + closurePackageDir: env.spawn.closurePackageDir, + // Activate the warm agent's inbound mail surface. Mail tools ask the + // supervisor through this bundle. The spawned-child build below omits + // it (a spawned child owns no warm inbound mailbox). + inbound: transportInbound, + ...(durableConversation !== undefined ? { durableConversation } : {}), + }); + + // The tool-bearing agent factory reads the materialized tool + // runtime off the per-step env (set by `buildStepEnv` via + // `attachStepTools`), attaches the loaded tool factories to the + // step's `AgentDefinition`, builds the plugin chain on + // `env.plugins`, and wraps `agent.close()` so every plugin (the LSP + // subprocess included) and tool bundle is torn down with the agent + // on every exit path. The factory is stateless across steps, so it + // is pinned once here and shared by every per-step invoker built + // below. + const stepAgentFactory = createToolBearingAgentFactory(); + + // The credential provider registry that shapes a delivered credential into + // a mediated handle. Built once here from the sidecar-static built-ins (the + // origin-pinned http provider) and shared by every per-step build; the + // per-run material and grants ride in separately at each invoke. + const credentialProviders = createCredentialProviderRegistry([ + ...builtinCredentialProviders(), + xApiKeyCredentialProvider(), + rawAuthorizationCredentialProvider(), + createMcpStreamableHttpCredentialProvider(), + ]); + // The run's credential wiring combined with the sidecar-static providers: + // the context both a step's tool capabilities and an action handler's + // capability are built from. + const runCredentialContext = ( + wiring: CredentialWiring, + ): SidecarStepCredentialContext => ({ + materialCell: wiring.materialRef, + resolveStepGrants: wiring.resolveStepGrants, + providers: credentialProviders, + }); + + // Spawned-child step build env (INTR-310). Every spawned child's steps -- + // a childWorkflow child's and an onTrigger section body's alike -- run real, + // TOOL-BEARING agents through the same source-tools arm the top level uses: + // the child holds the live re-verified definition, so each step agent + // carries live `toolFactories` fed from `req.agent.toolFactories`, and the + // source arm materializes declared plugins from the shared closure. Tools + // are available wherever inference runs, so a body agent gets the same + // build a top-level step agent gets (see the tool-availability invariant in + // `packages/workflow/README.md`). + // + // Built COLD per invocation -- no `durableConversation`/warm hooks (a + // fan-out branch and a section body are each a fresh run per spawn) and no + // `inbound` (a spawned child owns no warm inbound mailbox; inbound-reading + // tools stay inert). The source arm records no tool-mark floor (a source + // tool's bare `tool:` grant is already in the credentials snapshot), + // so the floor recorder throw-asserts that invariant. + // + // The source arm also keeps the body's tools scoped correctly by + // construction: a body child runs under the PARENT deployment's + // `mailboxAddress`/`stepCount`, so a body step whose id collides with a + // parent step id would read the PARENT step's tools if tools were resolved + // off the deploy tree. Feeding each agent its own evaluated + // `req.agent.toolFactories` never consults that tree. + const coldChildBuildStepEnv = createSidecarStepBuildEnv({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + signer: conversationSigner, + mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, + outboundMailBridge: env.outboundMailBridge, + cache: stepToolCache, + adapters: childAdapterRegistry, + recordToolMarkFloor: () => { + throw new Error( + "source-tools child build-env must not record a tool-mark floor", + ); + }, + materializeStepTools: deps.materializeStepTools, + sourceTools: true, + closurePackageDir: env.spawn.closurePackageDir, + }); + // Spawned-child step invoker (INTR-310). It runs a real agent through + // `createWorkflowStepInvoker`, resolving inference against the child's own + // per-step `sourcesRef` (staged at deploy, read fresh per spawn) and + // funnelling live events to the parent run's channel. `buildChildRunEnv` + // threads in the run's `credentialContext` (the live material cell, the + // child's capped grants, the sidecar-static providers), so the build + // attaches each bundle's `credentials` capability and the step's inference + // resolves its source secret against the run's live material. Absent when no + // material was threaded, in which case a tool that declares a credential + // consumer fails closed and loud at its own `resolve("credentials")`, never + // silently. Wired as `childRunDeps.invokeStep`, so it covers every spawned + // child's steps: a childWorkflow child, an onTrigger section body, and a + // body's own childWorkflow grandchildren. + const childInvokeStep: SidecarChildStepInvoker = ( + req, + authorize, + sourcesRef, + onEvent, + credentialContext, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + buildEnv: (buildReq) => + coldChildBuildStepEnv(buildReq, sourcesRef, credentialContext), + agentFactory: stepAgentFactory, + sourcesRef, + onEvent, + })(req); + + // Adapt the workflow-runtime `StepInvoker` shape onto the host's + // `ChildStepInvoker` shape. The host's `onEvent` is the child's + // per-run event-channel sink: the runtime body passes it per step, + // and the chain from here is `onEvent -> child event-channel sender + // -> supervisor -> publishWorkflowInferenceEvent -> hub timeline`. + // + // The `authorize` argument is the child's credentials-backed + // authorize closure (`createCredentialsBackedAuthorize`), threaded + // in from `run-child.ts`'s runtime env. The step agent's runtime + // gates EVERY tool call through `env.authorize` with + // `resource = tool:`, `action = "invoke"` (the inference + // layer's authz before-tool extension); using the credentials-backed + // authorize here means each tool call resolves against the per-step + // grant snapshot the supervisor assembled from the agent's + // `state/grants.json` and pushed over the control IPC. A tool the + // agent's grants do not allow is blocked; a granted tool runs. The + // operator gate at deploy time (the capability walk's `tool:` + // approval) and this runtime grant check are complementary: the walk + // bounds the toolset the deploy may carry, the grant snapshot decides + // which of those the agent may invoke at run time. + // + // A fresh `createWorkflowStepInvoker` is built per invocation so the + // adapter subscribes the step agent's event stream to THIS step's + // `onEvent`. The per-step env builder and the tool-bearing agent + // factory are pinned (closed over above); the event sink and the + // authorize closure vary per step. + // + // The `warmCache` (design §3b) is the run-loop's per-deployment + // warm-agent cache, present only for the single-step long-lived + // deployment the deploy projection marked a warm candidate. When + // supplied, the adapter builds the agent once and reuses it across + // messages; when absent, it keeps instantiate-send-teardown per + // step. Forwarding it here is the only warm-keep wiring this binding + // needs -- the adapter and the run-loop own the rest of the + // lifecycle. + // Run-boundary durability flush (design §3c). When the deployment is + // warm-kept, mirror the warm agent's conversation snapshot to the + // workflow-run substrate after each message's send settles. The key + // is the step identity, the same key the env builder filed the + // durable store under, so the hook resolves the right per-agent + // store. Absent for a multi-step deploy (no durable registry). + const onRunBoundary: ((key: string) => Promise) | undefined = + durableConversation !== undefined + ? async (key: string) => { + await durableConversation.get(key).mirrorToSubstrate(); + } + : undefined; + + // Connector-thread seed (design §3c). When the deployment is warm-kept, + // route each mail-derived inbound message onto the warm agent's + // connector thread before its send, so the reply path has thread state. + // The key is the step identity, the same key the durable store is filed + // under. Absent for a multi-step deploy (no durable registry). + const seedInbound: + | ((key: string, message: InboundMessage) => Promise) + | undefined = + durableConversation !== undefined + ? async (key: string, message: InboundMessage) => { + await durableConversation.get(key).seedInbound(message); + } + : undefined; + + // Connector reply drain (design §3c). When the deployment is + // warm-kept, drive the warm agent's outbound replies through the shared + // connector reply drain: on each `connector.reply` the agent emits, + // compose a threaded reply from the durable store's connector thread, + // send it through the outbound bridge (`bridge.submit`, the same signed- + // send path the agent's own supervisor-backed transport uses), then + // advance the thread from the receipt. Established once per warm agent + // over its lifetime stream; the returned drain handle carries the lifetime + // `done` promise the step-invoker folds into the warm entry so eviction + // drains it, plus the per-turn settle barrier the warm step gates each + // reply turn on. The key is the step identity, the same key the durable + // store is filed under. Absent for a multi-step deploy (no durable + // registry). + const driveReplies: + | ((key: string, stream: AgentEventStream) => ConnectorReplyDrain) + | undefined = + durableConversation !== undefined + ? (key: string, stream: AgentEventStream) => + driveConnectorReplies({ + stream, + composeReply: () => durableConversation.get(key).composeReply(), + send: (message) => + env.outboundMailBridge.submit( + env.spawn.mailboxAddress, + message, + { + // Connector replies do not carry References. A bare + // inReplyTo on any other send must not be completed + // from this mailbox. + completeReferences: true, + }, + ), + onReplySent: (receipt) => + durableConversation.get(key).onReplySent(receipt), + }) + : undefined; + + const invokeStep: RunWorkflowChildBindings["invokeStep"] = async ( + req, + onEvent, + authorize, + warmCache, + sourcesRef, + credentialWiring, + mailPartReader, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + // Combine the per-run credential wiring (the live material cell and + // the step-grants resolver, ridden in from the run child) with the + // sidecar-static provider registry, so `buildStepEnv` attaches a + // complete credential context and the agentFactory can assemble each + // bundle's consumer-scoped `credentials` capability. + buildEnv: (buildReq) => + buildStepEnv( + buildReq, + sourcesRef, + runCredentialContext(credentialWiring), + ), + agentFactory: stepAgentFactory, + onEvent, + sourcesRef, + mailPartReader, + ...(warmCache !== undefined ? { warmCache } : {}), + ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), + ...(seedInbound !== undefined ? { seedInbound } : {}), + ...(driveReplies !== undefined ? { driveReplies } : {}), + })(req); + + const evaluateGrantsAdapter: GrantEvaluator = createGrantEvaluator({ + floorFor: (stepId) => toolMarkFloorByStep.get(stepId) ?? [], + }); + + const childRunDeps: SidecarRunChildDeps = { + substrate, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + principal, + scheduler, + invokeStep: childInvokeStep, + // Plaintext body sources decrypted sidecar-side from the run record; each + // body path resolves its own table from here by definition id. + bodySources: parseBodyInferenceSources(validated.WORKFLOW_BODY_SOURCES), + dataDir: validated.SIDECAR_DATA_DIR, + evaluateGrants: evaluateGrantsAdapter, + // Shared with the top level's `buildStepEnv`: a spawned child's build + // combines it with the run's live material and capped grants. + credentialProviders, + // The shared closure the child re-walks to cap its inherited grants at + // its declared capabilities. Source-ref only, so always present here. + closurePackageDir: env.spawn.closurePackageDir, + collectDeclaredResources: deps.collectDeclaredResources, + collectDeclaredCredentialConsumers: + deps.collectDeclaredCredentialConsumers, + filterGrantsToDeclaredResources: deps.filterGrantsToDeclaredResources, + }; + // Terminal childWorkflow executor. `run-child` builds the in-memory + // resolver from this plus the lifted-body map it extracts after loading + // the parent's re-verified definition, so an owned inline child spawns + // with no on-disk asset read. + const runChild = createSidecarRunChild(childRunDeps); + + // An onTrigger section runs each event's body as a suspendable child. + // Source-ref is the only deploy lineage: `run-child` builds the in-memory + // body resolver from this raw executor plus the lifted-body map it extracts + // after re-evaluating the parent's closure, so a body resolves in-process + // with no on-disk read and no separate per-body re-verify (the parent's + // re-verify already covers every inline body). + const runSuspendableChild = + createSidecarSpawnSuspendableChild(childRunDeps); + + // The top-level run's scratch belongs to the deployment and survives + // termination for inspection until allocation cleanup. Internal cold runs + // can reclaim their own scratch once they finish. + let cleanupRunStorage: ((runId: string) => Promise) | undefined; + if (!env.spawn.warmKeep) { + const topLevelRun = parseRunAddress(env.spawn.mailboxAddress); + if (topLevelRun === null) + throw new Error( + `Workflow mailbox ${env.spawn.mailboxAddress} is not a run address`, + ); + cleanupRunStorage = async (runId) => { + if (runId === topLevelRun.runId) return; + await fs.promises.rm( + runStepStorageRoot({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + }), + { recursive: true, force: true }, + ); + }; + } + + // Recover a parked correlation's approval snapshot for the child's + // re-registration enumeration. Wired unconditionally (unlike + // `cleanupRunStorage`, which is cold-only): a warm agent parks on approval + // just as a cold one does, and the branch on `warmKeep` selects the durable + // read -- cold reads the per-attempt isogit store, warm reconstructs the + // agent's durable conversation state from the substrate. + const loadParkedApproval: LoadParkedApproval = ({ + runId, + stepId, + attempt, + correlationId, + }) => + env.spawn.warmKeep + ? readWarmParkedApprovalSnapshot({ + substrate, + workflowRunRepoId, + stepId, + correlationId, + }) + : readColdParkedApprovalSnapshot({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + stepId, + attempt, + correlationId, + }); + + // Enumerate a crashed step's durable pending approval operations for the + // resume classifier, off the same cold/warm durable read as + // `loadParkedApproval`. Where that binding is a lookup by a known + // correlationId (answering the supervisor's re-registration), this is the + // enumeration the classifier needs when the correlationId never reached the + // log -- the crash-across-park case: read the pending operations, project + // to the minimal approval records the runtime reconstructs `SignalAwaited` + // from. + const readParkedApprovalOps: ReadParkedApprovalOps = async ({ + runId, + stepId, + attempt, + }) => + toParkedApprovalOps( + env.spawn.warmKeep + ? await readWarmParkedPendingOperations({ + substrate, + workflowRunRepoId, + stepId, + }) + : await readColdParkedPendingOperations({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + stepId, + attempt, + }), + ); + + const bindings: RunWorkflowChildBindings = { + substrate, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + principal, + invokeStep, + actionCredentials: (wiring) => + createSidecarActionCredentialsFactory(runCredentialContext(wiring)), + initialSources: stepInferenceSources, + runChild, + runSuspendableChild, + // A loop iteration runs under the inherited env (not `buildChildRunEnv`), + // so it is the one body birth path that writes no grants file of its own. + // Materialize it here -- capping the container run's grants to the loop + // body's declared resources -- so the body's childWorkflow grandchild + // spawn is authorized. `definition` is the PRE-rewrite loop body. + materializeLoopIterationGrants: async ({ + parentRunId, + childRunId, + definition, + }) => { + await capAndPersistChildGrants({ + deps: childRunDeps, + directors: childRunDeps.directors ?? createDefaultDirectorRegistry(), + definition, + childRunId, + parentRunId, + }); + }, + scheduler, + evaluateGrants: evaluateGrantsAdapter, + loadParkedApproval, + readParkedApprovalOps, + // The same registry the warm agent's transport registers `watch` + // callbacks into; `runWorkflowChild` routes each `mailbox.notify` to it. + mailboxWatchRegistry, + ...(cleanupRunStorage !== undefined ? { cleanupRunStorage } : {}), + }; + return bindings; + }; +} diff --git a/vendor/intx/workflow-host/src/child/substrate-write-bridge.ts b/vendor/intx/workflow-host/src/child/substrate-write-bridge.ts index 64ccad49f6..494c231f68 100644 --- a/vendor/intx/workflow-host/src/child/substrate-write-bridge.ts +++ b/vendor/intx/workflow-host/src/child/substrate-write-bridge.ts @@ -35,7 +35,7 @@ // the closure produces, both base64-encoded. import { getLogger } from "@intx/log"; -import { base64Decode, base64Encode } from "@intx/types"; +import { base64Decode, base64Encode } from "@intx/types/base64"; import type { ControlChannelSender, diff --git a/vendor/intx/workflow-host/src/child/supervisor-backed-mail-part-reader.ts b/vendor/intx/workflow-host/src/child/supervisor-backed-mail-part-reader.ts new file mode 100644 index 0000000000..d61807c6b8 --- /dev/null +++ b/vendor/intx/workflow-host/src/child/supervisor-backed-mail-part-reader.ts @@ -0,0 +1,31 @@ +// The step invoker resolves a mail part's ref through this reader. The +// supervisor owns the committed blob and answers `readMailPart`. This +// reader does not open the workflow-run repo. + +import { base64Decode } from "@intx/types/base64"; +import type { MailPartReader } from "@intx/types/runtime"; + +import type { ChildMailboxCallBridge } from "./mailbox-call-bridge"; + +export function createSupervisorBackedMailPartReader(opts: { + readonly callBridge: ChildMailboxCallBridge; + readonly runId: string; +}): MailPartReader { + return { + async read(ref) { + const response = await opts.callBridge.submit({ + op: "readMailPart", + // The frame schema requires a run id. The part ref names the blob; + // the supervisor does not consult this field. + runId: opts.runId, + partRef: ref, + }); + if (response.op !== "readMailPart") { + throw new Error( + `supervisor-backed mail part reader: mailbox.call.response op ${JSON.stringify(response.op)} does not match "readMailPart"`, + ); + } + return base64Decode(response.value.contentBase64); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/child/supervisor-backed-transport.ts b/vendor/intx/workflow-host/src/child/supervisor-backed-transport.ts index 9ff89c3417..6798365443 100644 --- a/vendor/intx/workflow-host/src/child/supervisor-backed-transport.ts +++ b/vendor/intx/workflow-host/src/child/supervisor-backed-transport.ts @@ -1,56 +1,33 @@ -// Supervisor-backed `MessageTransport` for a unified-host step agent -// (both halves of mailbox ownership, §3a OUTBOUND and §3b INBOUND). +// Supervisor-backed `MessageTransport` for a unified-host step agent. // -// Under the unified host the supervisor is the sole mail owner: it holds -// the durable inbox and the host transport against which the agent's -// address is registered with its signing key. The step agent therefore -// does NOT hold a signing key to send outbound mail, and it does NOT own -// the host-side inbox directly. Its mail tools are backed by this -// transport: +// The supervisor owns the durable inbox and the signing key. This transport +// does not open the mailbox and does not decide which mailbox exists. Every +// method other than `send` asks the supervisor and returns that answer, +// including a refusal. `send` still goes through the outbound-mail bridge. +// Flag writes and `expunge` still go through the mailbox-mutation bridge, +// because those frames already exist. The rest go through the mailbox-call +// bridge. // -// - INBOUND is a functional local IMAP read surface once the sidecar -// wires it (the `inbound` constructor argument). The supervisor -// commits an arrived message to the deployment's workflow-run -// substrate mailbox (`mailbox/INBOX/`) and fires a `mailbox.notify` -// control frame. The read surface (`search`, `thread`, -// `fetchHeaders`, `fetchStructure`, `fetchPart`, `fetchFull`, `sync`, -// `getMailboxStatus`) answers by opening a fresh committed snapshot of -// that mailbox through the child mailbox reader and running the -// `@intx/mailbox` pure query functions over it -- local, no hub or -// IPC round-trip. `watch` registers into the child watch registry, so -// `mail_wait` unblocks when the routed `mailbox.notify` fires. The -// WRITE methods (`setFlags` / `clearFlags` / `expunge`) do NOT touch -// the local read surface: they route up to the supervisor through the -// mailbox-mutation bridge (a `mailbox.mutate.request` frame), which -// applies the mutation to the supervisor's owned store and replies. -// The child never flushes the run ref, so it never races the -// supervisor's mirror. The agent owns only the `INBOX`, so every -// inbound method rejects a request for any other mailbox rather than -// silently serving `INBOX` under the wrong name. When the sidecar -// constructs the transport without the `inbound` argument, the inbound -// methods throw a clear "not wired" error rather than answer against a -// missing surface. -// - OUTBOUND (`send`) routes through the supervisor over the control -// IPC via the outbound-mail bridge. The supervisor performs the -// actual signed send through the host transport, so the outbound mail -// carries the agent's signature with full parity to the in-process -// path. The agent never holds the key. +// `watch` registers its callback before the round trip. `mailbox.notify` is +// not queued: a callback installed only after the supervisor accepts would +// miss mail that lands while the call is in flight, and `mail_wait` would +// then wait until its timeout. A refusal unregisters the callback and +// rejects, so nothing stays armed for a mailbox the supervisor refused. // -// A handful of methods stay unsupported and throw: they act on a resource -// the unified-host agent does not own. `append` and the mailbox-management -// methods (`listMailboxes` / `createMailbox` / `deleteMailbox`) target a -// mailbox the agent does not own; `move` / `copy` need a second mailbox it -// does not own; and the distribution-list methods are unimplemented across -// every transport. +// When the sidecar constructs the transport without `inbound`, every inbound +// method fails with `SERVERBUG` "not wired". A spawned child has no inbox +// surface, and that failure is the wiring error, not a mailbox policy. +import { base64Decode } from "@intx/types/base64"; +import { deriveWorkflowRunId } from "@intx/types/workflow-run-id"; import type { BodyStructure, - CryptoProvider, InboundMessage, ListInfo, Mailbox, MailboxEvent, MailboxStatus, + MessageAttachment, MessageHeaders, MessagePart, MessageRef, @@ -62,112 +39,155 @@ import type { SyncState, Thread, Unsubscribe, -} from "@intx/types/runtime"; -import { MessageTransportError } from "@intx/types/runtime"; - -import { - executeSearch, - executeThread, - fetchFull as doFetchFull, - fetchHeaders as doFetchHeaders, - fetchPart as doFetchPart, - fetchStructure as doFetchStructure, -} from "@intx/mailbox"; - -import { deriveWorkflowRunId } from "@intx/types"; - -import { MAILBOX_INBOX_DIR } from "../adapters/substrate-mailbox-store"; -import type { ChildMailboxReader } from "./child-mailbox-reader"; +} from "@intx/types/runtime-core"; +import { MessageTransportError } from "@intx/types/runtime-core"; + +import type { ChildMailboxCallBridge } from "./mailbox-call-bridge"; +import type { MailboxCallSuccess } from "./mailbox-call-bridge"; import type { ChildMailboxMutationBridge } from "./mailbox-mutation-bridge"; import type { MailboxWatchRegistry } from "./mailbox-watch-registry"; import type { ChildOutboundMailBridge } from "./outbound-mail-bridge"; /** - * The dependencies backing the transport's whole inbox capability: the local - * IMAP READ surface (`reader` / `watchRegistry` / `getCrypto`) that resolves - * against the deployment's substrate mailbox, plus the routed-WRITE channel - * (`mutationBridge`) that carries flag writes and expunge up to the supervisor. - * The sidecar wires the bundle only for a build that owns an inbound mailbox - * (the warm agent); a build without it has no inbox and every inbound method - * throws a clear "not wired" error. Reads are local; writes route upstream -- - * the child never flushes the run ref, so it never races the supervisor. + * The inbox surface a warm agent owns. `watchRegistry` is the same registry + * the control loop fires `mailbox.notify` into. `mutationBridge` carries flag + * writes and expunge. `callBridge` carries every other mailbox method. A + * build without this surface has no inbox. */ export interface SupervisorBackedTransportInbound { - /** - * Opens a fresh committed snapshot of the deployment's substrate `INBOX`. - * Every inbound read opens a new snapshot, so a read taken after a - * `mailbox.notify` -- or after a routed write the supervisor flushed before - * replying -- observes the committed state. - */ - reader: ChildMailboxReader; - /** - * The registry the child's control loop fires `mailbox.notify` into. It must - * be the same instance `runWorkflowChild` routes the frame to, so a `watch` - * installed here observes the supervisor's notification. - */ watchRegistry: MailboxWatchRegistry; - /** - * Resolve a sender address to its `CryptoProvider` so `fetchFull` can verify - * the message signature. Returns `undefined` when no key is known for the - * sender, in which case the signature status is reported as `unknown`. - */ - getCrypto: (fromAddress: string) => CryptoProvider | undefined; - /** - * The upstream channel the write methods route through. `setFlags` / - * `clearFlags` / `expunge` call `mutationBridge.submit`, which emits a - * `mailbox.mutate.request` and resolves once the supervisor applies the - * mutation to its owned store and replies. Bundled with the read surface - * because the write methods and the reads share one presence condition: - * this agent owns an inbox, or it owns none. - */ mutationBridge: ChildMailboxMutationBridge; + callBridge: ChildMailboxCallBridge; } +type MailboxCall = Parameters[0]; + +type DistributiveOmit = T extends unknown + ? Omit + : never; + +type WireSearchQuery = Extract["query"]; + /** - * Construct a `MessageTransport` whose outbound side routes through the - * supervisor (via `bridge`) and whose inbound side is a local IMAP read - * surface over `inbound`. `address` is the agent's mail address; the - * supervisor signs the outbound mail as this address through the host - * transport, so it must be the address the host registered the agent's - * `CryptoProvider` against. When `inbound` is omitted, the inbound methods - * throw a clear "not wired" error; the sidecar supplies it once the child's - * mailbox reader and watch registry are threaded through. + * Search dates are `Date` on the transport and strings on the frame. Convert + * the tree here. `toISOString` throws on an invalid date, so a bad one fails + * the call instead of becoming JSON `null` and crashing the control channel. + */ +function searchQueryToWire(query: SearchQuery): WireSearchQuery { + const wire: WireSearchQuery = {}; + if (query.from !== undefined) wire.from = query.from; + if (query.to !== undefined) wire.to = query.to; + if (query.cc !== undefined) wire.cc = query.cc; + if (query.bcc !== undefined) wire.bcc = query.bcc; + if (query.header !== undefined) { + wire.header = { + field: query.header.field, + contains: query.header.contains, + }; + } + if (query.before !== undefined) wire.before = query.before.toISOString(); + if (query.after !== undefined) wire.after = query.after.toISOString(); + if (query.on !== undefined) wire.on = query.on.toISOString(); + if (query.sentBefore !== undefined) { + wire.sentBefore = query.sentBefore.toISOString(); + } + if (query.sentAfter !== undefined) { + wire.sentAfter = query.sentAfter.toISOString(); + } + if (query.sentOn !== undefined) wire.sentOn = query.sentOn.toISOString(); + if (query.hasFlags !== undefined) wire.hasFlags = query.hasFlags; + if (query.missingFlags !== undefined) wire.missingFlags = query.missingFlags; + if (query.body !== undefined) wire.body = query.body; + if (query.text !== undefined) wire.text = query.text; + if (query.largerThan !== undefined) wire.largerThan = query.largerThan; + if (query.smallerThan !== undefined) wire.smallerThan = query.smallerThan; + if (query.and !== undefined) wire.and = query.and.map(searchQueryToWire); + if (query.or !== undefined) wire.or = query.or.map(searchQueryToWire); + if (query.not !== undefined) wire.not = searchQueryToWire(query.not); + return wire; +} + +function projectPart( + value: Extract["value"], +): MessagePart { + const part: MessagePart = { + contentType: value.contentType, + content: base64Decode(value.contentBase64), + }; + if (value.encoding !== undefined) part.encoding = value.encoding; + if (value.filename !== undefined) part.filename = value.filename; + if (value.disposition !== undefined) part.disposition = value.disposition; + return part; +} + +function projectFetched( + value: Extract["value"], +): InboundMessage { + const message: InboundMessage = { + ref: value.ref, + headers: value.headers, + flags: [...value.flags], + signatureStatus: value.signatureStatus, + }; + if (value.content !== undefined) message.content = value.content; + if (value.payload !== undefined) message.payload = value.payload; + if (value.attachments !== undefined) { + message.attachments = value.attachments.map((attachment) => { + const projected: MessageAttachment = { + name: attachment.name, + contentType: attachment.contentType, + data: base64Decode(attachment.dataBase64), + }; + if (attachment.part !== undefined) projected.part = attachment.part; + return projected; + }); + } + return message; +} + +/** + * Construct a `MessageTransport` whose mail methods are answered by the + * supervisor. `address` is the agent's mail address; the supervisor signs + * outbound mail as this address. When `inbound` is omitted, inbound methods + * throw `SERVERBUG`. */ export function createSupervisorBackedTransport( bridge: ChildOutboundMailBridge, address: string, inbound?: SupervisorBackedTransportInbound, ): MessageTransport { - function unsupported(method: string): never { - throw new MessageTransportError( - "CANNOT", - `supervisor-backed transport: ${method} is not supported for unified-host step agent ${address}; the supervisor owns the mailbox and the agent owns only its own ${MAILBOX_INBOX_DIR}`, - ); - } - - // Return the wired inbound surface, or fail loud when the sidecar - // constructed the transport without it -- an inbound read against a missing - // surface is a wiring error, not a silently-empty result. function requireInbound(method: string): SupervisorBackedTransportInbound { if (inbound === undefined) { throw new MessageTransportError( "SERVERBUG", - `supervisor-backed transport: ${method} needs the inbound surface, but it is not wired for unified-host step agent ${address}; the sidecar must construct the transport with its mailbox reader, watch registry, and crypto`, + `supervisor-backed transport: ${method} needs the inbound surface, but it is not wired for unified-host step agent ${address}`, ); } return inbound; } - // The unified-host agent owns exactly one mailbox, the substrate `INBOX` - // the reader opens. Reject any other name rather than serve `INBOX` under - // it, which would return the wrong mailbox's messages mislabeled. - function requireInbox(mailbox: string): void { - if (mailbox !== MAILBOX_INBOX_DIR) { - throw new MessageTransportError( - "NONEXISTENT", - `supervisor-backed transport: unified-host step agent ${address} owns only the "${MAILBOX_INBOX_DIR}" mailbox; "${mailbox}" is not available`, + // The request itself is the type parameter. Inferring `op` and then + // extracting the arm collapses ops that share a shape (`move`/`copy`, + // the fetches) to `never`. + async function ask< + const Request extends DistributiveOmit, + >( + request: Request, + ): Promise> { + const { callBridge } = requireInbound(request.op); + const response = await callBridge.submit({ + ...request, + runId: deriveWorkflowRunId(address), + }); + if (response.op !== request.op) { + throw new Error( + `supervisor-backed transport: mailbox.call.response op ${JSON.stringify(response.op)} does not match ${JSON.stringify(request.op)}`, ); } + // The comparison above is the narrow. The success union does not + // distribute over a generic `op`, so the assertion follows the check. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- response.op === request.op is the narrow; Extract does not distribute over a generic op + return response as Extract; } return { @@ -179,47 +199,62 @@ export function createSupervisorBackedTransport( }, async append( - _mailbox: string, - _message: InboundMessage, - _flags?: string[], + mailbox: string, + message: InboundMessage, + flags?: string[], _signal?: AbortSignal, ): Promise { - // `append` writes into a mailbox the agent owns; in the unified - // host the agent owns none. The mail tools do not append (they - // `send`), so a reachable `append` is a programming error. - return unsupported("append"); + if (message.attachments !== undefined && message.attachments.length > 0) { + throw new Error( + "Cannot append message: attachments are not carried on a mailbox append", + ); + } + const answer = await ask({ + op: "append", + mailbox, + headers: message.headers, + ...(message.content !== undefined ? { content: message.content } : {}), + ...(message.payload !== undefined ? { payload: message.payload } : {}), + ...(flags !== undefined ? { flags } : {}), + }); + return answer.value; }, async listMailboxes(_signal?: AbortSignal): Promise { - return unsupported("listMailboxes"); + const answer = await ask({ op: "listMailboxes" }); + return answer.value.map((mailbox) => { + const projected: Mailbox = { name: mailbox.name }; + if (mailbox.role !== undefined) projected.role = mailbox.role; + if (mailbox.delimiter !== undefined) { + projected.delimiter = mailbox.delimiter; + } + return projected; + }); }, - async createMailbox( - _name: string, - _signal?: AbortSignal, - ): Promise { - return unsupported("createMailbox"); + + async createMailbox(name: string, _signal?: AbortSignal): Promise { + const answer = await ask({ op: "createMailbox", name }); + const mailbox: Mailbox = { name: answer.value.name }; + if (answer.value.role !== undefined) mailbox.role = answer.value.role; + if (answer.value.delimiter !== undefined) { + mailbox.delimiter = answer.value.delimiter; + } + return mailbox; }, - async deleteMailbox(_name: string, _signal?: AbortSignal): Promise { - return unsupported("deleteMailbox"); + + async deleteMailbox(name: string, _signal?: AbortSignal): Promise { + await ask({ op: "deleteMailbox", name }); }, + async getMailboxStatus( name: string, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("getMailboxStatus"); - requireInbox(name); - const store = await reader.open(); - const unseen = store.messages.filter( - (m) => !m.flags.has("\\Seen"), - ).length; - return { - total: store.messages.length, - unseen, - recent: 0, - uidNext: store.uidNext, - uidValidity: store.uidValidity, - highestModSeq: store.highestModSeq, - }; + const answer = await ask({ + op: "getMailboxStatus", + mailbox: name, + }); + return answer.value; }, async search( @@ -227,58 +262,60 @@ export function createSupervisorBackedTransport( query: SearchQuery, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("search"); - requireInbox(mailbox); - const store = await reader.open(); - return await executeSearch(mailbox, store, query); + const answer = await ask({ + op: "search", + mailbox, + query: searchQueryToWire(query), + }); + return answer.value; }, + async thread( mailbox: string, algorithm: "references" | "orderedsubject", query?: SearchQuery, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("thread"); - requireInbox(mailbox); - const store = await reader.open(); - return await executeThread(mailbox, store, algorithm, query); + const answer = await ask({ + op: "thread", + mailbox, + algorithm, + ...(query !== undefined ? { query: searchQueryToWire(query) } : {}), + }); + return answer.value; }, + async fetchHeaders( ref: MessageRef, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("fetchHeaders"); - requireInbox(ref.mailbox); - const store = await reader.open(); - return await doFetchHeaders(ref, store); + const answer = await ask({ op: "fetchHeaders", ref }); + return answer.value; }, + async fetchStructure( ref: MessageRef, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("fetchStructure"); - requireInbox(ref.mailbox); - const store = await reader.open(); - return await doFetchStructure(ref, store); + const answer = await ask({ op: "fetchStructure", ref }); + return answer.value; }, + async fetchPart( ref: MessageRef, partPath: string, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("fetchPart"); - requireInbox(ref.mailbox); - const store = await reader.open(); - return await doFetchPart(ref, partPath, store); + const answer = await ask({ op: "fetchPart", ref, partPath }); + return projectPart(answer.value); }, + async fetchFull( ref: MessageRef, _signal?: AbortSignal, ): Promise { - const { reader, getCrypto } = requireInbound("fetchFull"); - requireInbox(ref.mailbox); - const store = await reader.open(); - return await doFetchFull(ref, store, getCrypto); + const answer = await ask({ op: "fetchFull", ref }); + return projectFetched(answer.value); }, async setFlags( @@ -287,11 +324,6 @@ export function createSupervisorBackedTransport( _signal?: AbortSignal, ): Promise { const { mutationBridge } = requireInbound("setFlags"); - requireInbox(ref.mailbox); - // Route the flag write to the supervisor -- the sole mailbox writer -- - // rather than flushing a second store against the run ref. `submit` - // resolves only after the supervisor flushes, so a subsequent read - // observes the flag. await mutationBridge.submit({ runId: deriveWorkflowRunId(address), mailbox: ref.mailbox, @@ -300,13 +332,13 @@ export function createSupervisorBackedTransport( flags, }); }, + async clearFlags( ref: MessageRef, flags: string[], _signal?: AbortSignal, ): Promise { const { mutationBridge } = requireInbound("clearFlags"); - requireInbox(ref.mailbox); await mutationBridge.submit({ runId: deriveWorkflowRunId(address), mailbox: ref.mailbox, @@ -317,32 +349,29 @@ export function createSupervisorBackedTransport( }, async move( - _ref: MessageRef, - _toMailbox: string, + ref: MessageRef, + toMailbox: string, _signal?: AbortSignal, ): Promise { - return unsupported("move"); + await ask({ op: "move", ref, toMailbox }); }, + async copy( - _ref: MessageRef, - _toMailbox: string, + ref: MessageRef, + toMailbox: string, _signal?: AbortSignal, ): Promise { - return unsupported("copy"); + await ask({ op: "copy", ref, toMailbox }); }, + async expunge( mailbox: string, _signal?: AbortSignal, ): Promise<{ expungedUids: number[] }> { const { mutationBridge } = requireInbound("expunge"); - requireInbox(mailbox); - // Route to the supervisor, which sweeps every `\Deleted` message out of - // its owned INBOX and returns the swept uids. The expunged bytes survive - // in git history (a workflow-run repo's objects are never GC'd), so the - // replication check permits the deletion. A caller expunging after a - // `setFlags(\Deleted)` MUST await the two in sequence -- the supervisor - // applies mutations in arrival order, so an unawaited (concurrent) pair - // could let the sweep run before the flag is set and miss the message. + // The supervisor applies mutations in arrival order. A caller that + // flags `\Deleted` and then expunges must await the two in sequence, + // or the sweep can run before the flag and miss the message. const result = await mutationBridge.submit({ runId: deriveWorkflowRunId(address), mailbox, @@ -351,17 +380,26 @@ export function createSupervisorBackedTransport( return { expungedUids: result.expungedUids ?? [] }; }, - watch( + async watch( mailbox: string, callback: (event: MailboxEvent) => void, - ): Unsubscribe { - const { watchRegistry } = requireInbound("watch"); - requireInbox(mailbox); - // The supervisor -- the sole mail owner -- fires `mailbox.notify` into - // the registry when new mail lands; the registry delivers the typed - // event to this callback. `mail_wait` installs the watch and unblocks on - // the first delivery. - return watchRegistry.watch(mailbox, callback); + ): Promise { + const { watchRegistry, callBridge } = requireInbound("watch"); + // Register before the first await. A notify that arrives while the + // acceptance is in flight is delivered only to callbacks already in + // the registry; the supervisor does not replay it. + const unsubscribe = watchRegistry.watch(mailbox, callback); + try { + await callBridge.submit({ + runId: deriveWorkflowRunId(address), + op: "watch", + mailbox, + }); + } catch (cause) { + unsubscribe(); + throw cause; + } + return unsubscribe; }, async sync( @@ -369,68 +407,56 @@ export function createSupervisorBackedTransport( knownState: SyncState, _signal?: AbortSignal, ): Promise { - const { reader } = requireInbound("sync"); - requireInbox(mailbox); - const store = await reader.open(); - const result = store.sync({ + // `knownUids` is not on the wire. The supervisor splits new messages + // from flag changes using `uidNext` and returns the finished result. + const answer = await ask({ + op: "sync", + mailbox, + uidNext: knownState.uidNext, uidValidity: knownState.uidValidity, highestModSeq: knownState.highestModSeq, }); - if (result.resync) { - return { - vanished: [], - changed: [], - newMessages: result.messages.map((m) => ({ uid: m.uid, mailbox })), - fullResyncRequired: true, - }; - } - // The backing reports every message whose modseq advanced past the - // client's known state as `changed`. Split it against the client's known - // `uidNext`: a uid at or beyond it is a new arrival, one below it is a - // flag change on a message the client already held. - const newMessages: MessageRef[] = []; - const changed: { uid: number; flags: string[] }[] = []; - for (const m of result.changed) { - if (m.uid >= knownState.uidNext) { - newMessages.push({ uid: m.uid, mailbox }); - } else { - changed.push({ uid: m.uid, flags: Array.from(m.flags) }); - } - } - return { - vanished: [...result.vanished], - changed, - newMessages, - fullResyncRequired: false, - }; + return answer.value; }, async createList( - _address: string, - _name: string, + listAddress: string, + name: string, _signal?: AbortSignal, ): Promise { - return unsupported("createList"); + const answer = await ask({ + op: "createList", + address: listAddress, + name, + }); + return answer.value; }, + async listMembers( - _address: string, + listAddress: string, _signal?: AbortSignal, ): Promise { - return unsupported("listMembers"); + const answer = await ask({ + op: "listMembers", + address: listAddress, + }); + return answer.value; }, + async subscribe( - _listAddress: string, - _subscriberAddress: string, + listAddress: string, + subscriberAddress: string, _signal?: AbortSignal, ): Promise { - return unsupported("subscribe"); + await ask({ op: "subscribe", listAddress, subscriberAddress }); }, + async unsubscribe( - _listAddress: string, - _subscriberAddress: string, + listAddress: string, + subscriberAddress: string, _signal?: AbortSignal, ): Promise { - return unsupported("unsubscribe"); + await ask({ op: "unsubscribe", listAddress, subscriberAddress }); }, }; } diff --git a/vendor/intx/workflow-host/src/child/warm-agent-cache.ts b/vendor/intx/workflow-host/src/child/warm-agent-cache.ts index 345be336a5..4423975578 100644 --- a/vendor/intx/workflow-host/src/child/warm-agent-cache.ts +++ b/vendor/intx/workflow-host/src/child/warm-agent-cache.ts @@ -58,8 +58,7 @@ export interface WarmEventSinkRef { * `waitForReplyAfter(snapshot)` so the run parks -- and the supervisor * consumes the inbound mail -- only after the reply is durably sent. This is * the structural subset of the harness `ConnectorReplyDrain` the warm path - * needs; declaring it here keeps the workflow-host package independent of - * `@intx/harness`, while the drain the sidecar builds satisfies it. + * needs. The drain the sidecar builds satisfies it. */ export type WarmReplySettlement = | { readonly ok: true } diff --git a/vendor/intx/workflow-host/src/conversation-state-root.ts b/vendor/intx/workflow-host/src/conversation-state-root.ts new file mode 100644 index 0000000000..e227acf832 --- /dev/null +++ b/vendor/intx/workflow-host/src/conversation-state-root.ts @@ -0,0 +1,13 @@ +import path from "node:path"; + +/** + * The directory holding a deployment's local durable-conversation stores, one + * per agent key, under the sidecar data dir. The workflow child writes them and + * the deploy router clears them, so both derive the path here. + */ +export function conversationStateRoot( + dataDir: string, + workflowRunRepoId: string, +): string { + return path.join(dataDir, "agent-conversation-state", workflowRunRepoId); +} diff --git a/apps/sidecar/src/conversation-state.ts b/vendor/intx/workflow-host/src/conversation-state.ts similarity index 53% rename from apps/sidecar/src/conversation-state.ts rename to vendor/intx/workflow-host/src/conversation-state.ts index 3f89d3f41d..8a31be923c 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/vendor/intx/workflow-host/src/conversation-state.ts @@ -1,8 +1,127 @@ -// Makes a warm single-step agent's multi-turn conversation durable in the -// workflow-run substrate (not the agent's per-run isogit store, which is -// lost on child respawn), so continuity survives across runs and respawns. -// See docs/sidecar-conversation-durability.md for the on-disk layout, -// substrate-merge constraints, and timing/failure guarantees. +// Durable conversation state for the warm single-step agent (design §3c, +// §4 Phase D1). +// +// A long-lived single-step agent holds its multi-turn conversation in +// the reactor's in-memory turn buffer, backed by a per-step isogit +// `ContextStore`. That store is rooted per run/attempt, so it is lost +// the moment the warm agent's child is killed and respawned: the +// rebuilt agent loads a fresh, empty per-run store and the conversation +// continuity is gone. +// +// This module makes the warm agent's conversation DURABLE in the +// workflow-run substrate (the single-writer proxy `RepoStore`, written +// through the supervisor). The durable copy lives under the workflow-run +// repo at a stable per-agent path (`agent-state//...`), sibling +// to the per-run event log under `runs//...` and NOT confused with +// it. On a new run (and after a child respawn, once the warm agent is +// rebuilt lazily) the conversation is restored from the substrate into +// the agent's local store BEFORE the agent's reactor loads, so multi-turn +// continuity holds across runs and across respawn. +// +// Two-tier on-disk layout (design §4, Phase D1). The prior design wrote +// the WHOLE conversation as a single `conversation.json` blob on every +// message. That re-serialized and re-hashed every prior turn per message, +// so the per-message durable cost grew O(N) in the turn count -- O(N^2) +// over a conversation. D1 replaces it with an append-only, bucket-sharded +// write-ahead log plus a periodic compacted checkpoint: +// +// agent-state// +// checkpoint.json compacted full snapshot (turns + metadata) +// checkpoint.meta.json { checkpointSeq: , turnCount, +// tokenUsage, pendingOperations, +// connectorState } +// wal//.json one per-boundary delta blob keyed by mirror +// BOUNDARY seq, carrying that boundary's +// 0-or-more new turns plus the freshest metadata +// +// The WAL is keyed by mirror BOUNDARY, not by turn: each `mirrorToSubstrate` +// writes exactly one entry, even when the boundary added no new turns. That +// makes metadata (pendingOperations, tokenUsage, connectorState) persist on +// EVERY boundary -- a turnless-but-metadata-mutating boundary still commits +// a zero-turn entry, so restore never reconstructs stale metadata. (Keying +// the entry by turn dropped metadata on turnless boundaries, which regressed +// the byte-for-byte metadata-equivalence invariant; per-boundary keying is +// the fix.) +// +// `bucket = floor(boundarySeq / WAL_BUCKET_SIZE)` (B = 128) bounds any +// single directory's tree-object size so no commit re-hashes a tree that +// grows with N (a flat `wal/.json` directory would itself be O(N) per +// commit). Compaction every CHECKPOINT_INTERVAL boundaries (K = 64, i.e. +// once the live WAL reaches K entries) folds the WAL into a fresh +// `checkpoint.json` (capturing the freshest metadata in checkpoint.meta) and +// truncates the WAL, so between checkpoints the WAL holds at most K entries +// and per-boundary durable cost is ~O(1) amortized. K and B are constants +// here; the design flags them as measurement-tunable. +// +// Restore = load `checkpoint.json` (folded turns + its metadata) then replay +// the WAL tail in boundary-seq order, concatenating each boundary's turns +// and taking the LATEST entry's metadata (the last WAL entry wins; the +// checkpoint's metadata is the base when the WAL is empty). This is pure +// state reconstruction from recorded outputs -- never re-inference. It +// rebuilds the EXACT turn list + metadata the old whole-blob mirror would +// have restored. +// +// Substrate-merge constraint (load-bearing, design §4 "Substrate-merge +// note"). `writeTreePreservingPrefix`'s `merge` callback receives only +// the DIRECT CHILDREN of `preservePrefix`, and the substrate's +// `clearPrefix` step recursively removes the whole `preservePrefix` +// subtree before writing the merge's returned set (paths outside the +// prefix pass through untouched). A WAL blob is two levels below +// `agent-state//`, so: +// +// - WAL append uses `preservePrefix = agent-state//wal//`. +// The bucket's existing blobs ARE direct children, so the merge +// pre-image is exactly that bucket and the append adds one entry -- +// no isogit side-read, and the checkpoint / other buckets are +// untouched (outside the prefix). +// - Checkpoint write + WAL truncate uses `preservePrefix = +// agent-state//`. The top-level checkpoint files are direct +// children; the merge returns ONLY those files and NO `wal/...` +// paths, so the recursive `clearPrefix` at `agent-state//` drops +// the entire WAL subtree in the same atomic commit. The truncate +// needs no nested read: omitting the WAL paths from the returned set +// IS the truncate. +// +// Persistence sink (the riskiest part, design §6). The connector router's +// `snapshot()` / `restore()` surface and the harness's +// `createWrappedStorageOverrides` are reused, but the persistence sink is +// repointed from the agent's local isogit store to the workflow-run +// substrate. Both the WAL append and the checkpoint write route through +// the proxy `writeTreePreservingPrefix`; because the supervisor is the +// single writer and serializes every write to the workflow-run ref under +// a per-repo lock, and the `agent-state//...` prefix is disjoint from +// the run-event prefix (`runs//events/`), the conversation write +// never races nor clobbers the run-event log -- both pass through the same +// single writer, and the preserve-prefix merge leaves every other subtree +// byte-for-byte intact. +// +// Timing (design §4 Phase D1, invariant 1). This is a STRUCTURE-only +// change. The mirror is still `await`ed synchronously at the same run +// boundary (`onRunBoundary` -> `mirrorToSubstrate`), so every turn is +// still durably committed before the next message is processed. D1 +// changes WHAT the write does (O(1) append instead of O(N) whole-blob), +// not WHEN it happens. The run log is NOT yet a durable backstop for the +// turn (it carries a constant ref, design rev-2 FACT 1), so the +// conversation copy here remains the sole durable copy of the agent's +// per-turn output -- which is exactly why D1 keeps the write synchronous. +// The async flusher, run-log enrichment, and crash reconciliation are +// later, conditional phases (D2-D4), not done here. +// +// Commit granularity (greybeard's pick, design §3c open question). The +// design calls for connector-state-change-driven commits via the router's +// `onStateChanged` hook. The warm-agent path drives the connector router +// through `seedInbound`: each mail-derived inbound message routes and +// commits its thread state before the agent's send, so `onStateChanged` +// fires and enqueues a change-driven mirror. The run-boundary mirror (per +// message) still runs unconditionally, so the two triggers are +// complementary -- the seed persists the connector state promptly, the +// boundary persists the turn delta. +// +// Defensive: a restore that finds a checkpoint or WAL but cannot +// parse/replay it THROWS (a lost or corrupt conversation on respawn is a +// correctness failure, not a silently-fresh start). A mirror write failure +// surfaces so a dropped durability write is visible rather than leaving +// the next respawn to read a stale snapshot. import fs from "node:fs"; import path from "node:path"; @@ -13,7 +132,11 @@ import { getLogger } from "@intx/log"; import { createConnectorRouter } from "@intx/harness"; import type { ConnectorReplyParts, RouteDecision } from "@intx/harness"; import { createIsogitStore } from "@intx/storage-isogit/node"; -import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions/substrate"; +import type { + Principal, + RepoId, + RepoStore, +} from "@intx/hub-sessions/substrate"; import { WORKFLOW_RUN_AGENT_STATE_PREFIX } from "@intx/hub-sessions/substrate"; import { ConnectorThreadState, @@ -24,7 +147,10 @@ import { type InboundMessage, type PendingOperation, type SendReceipt, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; + +import { isErrnoNotFound } from "./supervisor/credentials"; +import { conversationStateRoot } from "./conversation-state-root"; const logger = getLogger(["sidecar", "workflow-child", "conversation-state"]); @@ -49,14 +175,30 @@ const CHECKPOINT_INTERVAL = 64; */ const WAL_BUCKET_SIZE = 128; -/** Metadata stamped onto every WAL entry so restore recovers reactor state without a separate log. */ +/** + * Metadata carried alongside the checkpoint and stamped onto every WAL + * entry. Small and bounded -- it is NOT the O(N) cost; the turn array is. + * Stamping the latest metadata on each WAL entry lets the restore replay + * recover the exact non-turn reactor state without a separate metadata + * log: the last replayed entry's metadata wins. Because every mirror + * boundary writes exactly one WAL entry (even a turnless one), the latest + * metadata is ALWAYS captured durably -- a turnless boundary still commits + * its advanced metadata as a zero-turn entry. + */ const SnapshotMetadata = type({ pendingOperations: "unknown[]", tokenUsage: TokenUsage, connectorState: ConnectorThreadState.or("null"), }); -/** Compacted checkpoint blob; validated on read so a corrupt or partial file surfaces rather than half-applies. */ +/** + * On-disk shape of the compacted checkpoint blob committed at + * `agent-state//checkpoint.json`. Carries the folded turn + * history (turns 0..checkpointSeq-1) plus the non-turn reactor metadata. + * Validated on read because it crosses back into the program from the + * substrate working tree -- a corrupt or partially-written checkpoint must + * surface at the boundary, never be half-applied into the agent. + */ const CheckpointSnapshot = type({ turns: "unknown[]", pendingOperations: "unknown[]", @@ -64,7 +206,17 @@ const CheckpointSnapshot = type({ connectorState: ConnectorThreadState.or("null"), }); -/** Checkpoint pointer restore reads first; checkpointSeq counts mirror boundaries, not turns, since a boundary may carry zero or many turns. */ +/** + * On-disk shape of `agent-state//checkpoint.meta.json`. The + * checkpoint pointer the restore path reads first to learn the boundary seq + * the checkpoint folded to (`checkpointSeq`) -- and therefore which WAL + * boundary seqs remain to replay -- plus the folded turn count + * (`turnCount`, = `checkpoint.json`'s turn array length) and the freshest + * metadata at fold time. `checkpointSeq` counts MIRROR BOUNDARIES, not + * turns: a boundary may carry zero or many turns, so the boundary count and + * the turn count diverge in general (they coincide only when every boundary + * adds exactly one turn). + */ const CheckpointMeta = type({ checkpointSeq: "number", turnCount: "number", @@ -73,7 +225,16 @@ const CheckpointMeta = type({ connectorState: ConnectorThreadState.or("null"), }); -/** One entry per mirror boundary (not per turn); a turnless boundary still writes `turns: []` so its metadata persists. */ +/** + * On-disk shape of one WAL entry blob at + * `agent-state//wal//.json`. One entry per MIRROR + * BOUNDARY (keyed by boundary `seq`, not turn index). Records the 0-or-more + * new turns that boundary added (the O(1) append payload -- it never + * carries prior turns) plus the latest non-turn metadata snapshot. The + * append is UNCONDITIONAL: a turnless boundary still writes one entry with + * `turns: []` so its advanced metadata is durably committed (the invariant + * the per-turn keying broke -- metadata must persist on EVERY boundary). + */ const WalEntry = type({ seq: "number", turns: "unknown[]", @@ -133,9 +294,23 @@ export interface DurableConversationStore { * isogit store the non-warm path uses. */ readonly storage: ContextStore & AuditStore; - /** Restore prior conversation from the substrate before the agent builds; throws on a corrupt copy rather than starting fresh. */ + /** + * Pull the prior conversation from the substrate (checkpoint + WAL-tail + * replay) into the local store so the agent's reactor `load()` sees it. + * Called before the warm agent is built (lazy first build and respawn + * rebuild). Returns `true` when prior state was found and applied, + * `false` when none exists yet (the genuine first-ever run). A read that + * finds a checkpoint or WAL but cannot parse/replay it throws -- a + * corrupt durable copy is a correctness failure that must not silently + * start the agent fresh. + */ restoreFromSubstrate(): Promise; - /** Commit new turns as an O(1) WAL append, compacting into a checkpoint at the compaction interval. */ + /** + * Commit the local store's new turn(s) to the substrate as O(1) WAL + * appends, folding into a fresh checkpoint when the WAL reaches the + * compaction interval. Called synchronously at the run boundary (after + * the agent's send settles). A write failure surfaces. + */ mirrorToSubstrate(): Promise; /** * Advance the connector router from a received inbound message so the @@ -148,9 +323,24 @@ export interface DurableConversationStore { * failure surfaces. */ seedInbound(message: InboundMessage): Promise; - /** Threading headers for a reply on the active connector thread; throws NoActiveConnectorThreadError with none seeded. */ + /** + * Produce the threading headers for a reply on the active connector + * thread (the router's `composeReply`). Throws + * `NoActiveConnectorThreadError` when no thread has been seeded. The warm + * mail loop's reply drain reads this to address its outbound reply. + */ composeReply(): ConnectorReplyParts; - /** Advances thread state after a reply is sent so the next inbound continuation matches lastMessageId. */ + /** + * Advance the connector thread after a reply was sent. Forwards to the + * router's `onReplySent` (which moves `lastMessageId` to the sent reply's + * Message-ID so the next inbound continuation matches) and flushes the + * advanced connector state into the local store's metadata the same way + * `seedInbound` does, so `lastMessageId` persists across turns and across a + * child respawn. Called by the warm mail loop's reply drain after its + * outbound send settles. Throws `NoActiveConnectorThreadError` when no + * thread is active -- advancing outbound state has no meaning without a + * seeded thread. A metadata write failure surfaces. + */ onReplySent(receipt: SendReceipt): Promise; } @@ -194,11 +384,22 @@ export async function createDurableConversationStore( // checkpointBoundarySeq) and when to compact. let checkpointBoundarySeq = 0; - // Serializes the shared-counter critical section: mirror and restore can - // re-enter each other (connectorRouter.restore() can fire onStateChanged - // synchronously), so a single tail prevents two overlapping runs from - // emitting two WAL entries at the same boundary seq. Does not address the - // separate reactor-vs-mirror peek-snapshot window documented on runMirror. + // Serialize the shared-counter critical section. Both `mirrorToSubstrate` + // and `restoreFromSubstrate` read and advance the three counts above, and + // either can re-enter the other: `connectorRouter.restore()` can fire + // `onStateChanged` synchronously, which enqueues a mirror. A single + // per-instance tail runs every such op one-at-a-time regardless of entry + // point, so two overlapping runs cannot read the same boundary seq and + // emit two WAL entries at it. Modeled on `runRepoOp` in the hub-agent + // session manager: the stored tail swallows rejections so one failed op + // does not poison the chain, while each caller still observes its own op's + // result (or rejection) through the returned promise. + // + // This serializes mirror-vs-mirror and mirror-vs-restore only. It does NOT + // address the reactor-vs-mirror peek-snapshot window documented on + // `runMirror` below (nothing must append to the reactor's turn array + // between its last writeTurns and the mirror's peekTurns) -- that is a + // different concurrency axis and is out of scope here. let stateOpTail: Promise = Promise.resolve(); function serializeStateOp(op: () => Promise): Promise { const result = stateOpTail.then(op, op); @@ -231,7 +432,11 @@ export async function createDurableConversationStore( function substrateAgentStateFsDir(): string { const repoDir = opts.substrate.getRepoDir(opts.workflowRunRepoId); - return path.join(repoDir, WORKFLOW_RUN_AGENT_STATE_PREFIX, encodeURIComponent(opts.agentKey)); + return path.join( + repoDir, + WORKFLOW_RUN_AGENT_STATE_PREFIX, + encodeURIComponent(opts.agentKey), + ); } function bucketOf(seq: number): number { @@ -275,9 +480,14 @@ export async function createDurableConversationStore( // so a future change-driven mirror carries the right base. await baseStorage.writeTurns(reconstructed.turns); baseStorage.setConnectorState(reconstructed.connectorState); - // Set counts before restoring connector state: restore() can fire - // onStateChanged synchronously and enqueue a mirror, so this keeps the - // counts correct even if the serialization ordering is ever weakened. + // Establish the committed counts BEFORE restoring the connector state. + // `connectorRouter.restore()` can fire `onStateChanged` synchronously + // (when the restored state differs from current), which enqueues a + // mirror. Serialization already chains that mirror behind this restore, + // but setting the counts first keeps them correct even if that ordering + // guarantee is ever weakened. The counts reflect the substrate state + // `reconstructed` was read from, which is durable independent of the + // local-store commit below. mirroredBoundaryCount = reconstructed.boundaryCount; mirroredTurnCount = reconstructed.totalTurns; checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; @@ -292,7 +502,15 @@ export async function createDurableConversationStore( return true; } - /** Appends one WAL entry to its bucket; O(bucket size), independent of total turn count N. */ + /** + * Append one WAL entry for a mirror boundary (its 0-or-more new turns + + * the current metadata) to its bucket. The merge pre-image is exactly + * that bucket's existing blobs (direct children of `wal//`), so + * the append is the bucket's blobs plus the one new entry -- O(bucket + * size), independent of N. The append is the SINGLE synchronous write per + * boundary; the entry is keyed by boundary `seq` so a turnless boundary + * still commits its metadata as a zero-turn entry. + */ async function appendWalEntry( boundarySeq: number, turns: unknown[], @@ -324,7 +542,14 @@ export async function createDurableConversationStore( ); } - /** Folds the conversation into a fresh checkpoint and truncates the WAL atomically by omitting wal/ paths from the merge return. */ + /** + * Fold the full conversation into a fresh checkpoint and truncate the + * WAL in one atomic commit at `preservePrefix = agent-state//`. The + * merge returns ONLY the two checkpoint files and NO `wal/...` paths; + * because the substrate's `clearPrefix` recursively removes the whole + * `agent-state//` subtree before writing the returned set, omitting + * the WAL paths IS the truncate. + */ async function writeCheckpoint( boundarySeq: number, turns: unknown[], @@ -340,7 +565,11 @@ export async function createDurableConversationStore( tokenUsage: metadata.tokenUsage, connectorState: metadata.connectorState, }; - // metadata is the freshest snapshot, so a post-fold restore sees the same metadata the pre-fold WAL tail would have yielded. + // `metadata` is the freshest snapshot (the current local-store + // metadata, identical to the last appended WAL entry's metadata), so + // the fold captures the latest metadata into checkpoint.meta -- a + // restore from the post-fold checkpoint sees the same metadata the + // pre-fold WAL tail would have yielded. const meta = { checkpointSeq: boundarySeq, turnCount: turns.length, @@ -364,10 +593,18 @@ export async function createDurableConversationStore( } async function runMirror(): Promise { - // Reads the reactor's in-memory turn array rather than re-parsing - // turns.jsonl every boundary; relies on the reactor never appending - // between its last writeTurns and this peek, which serialization here - // does not itself enforce. + // Slice the new turns from the reactor's in-memory array (retained + // by the local single-writer store at the last writeTurns) instead + // of re-reading and re-parsing the whole turns.jsonl every + // boundary; only the bounded metadata.json is read from disk. This + // rests on a sequencing invariant the store cannot enforce: nothing + // mutates the reactor's turn array between its last writeTurns and + // this read. The mirror runs at onRunBoundary after send() settles, + // and the reactor only appends inside a cycle (each ending in + // writeTurns), so peekTurns() equals the on-disk state here. + // Serializing the mirror entry points keeps two mirrors from + // overlapping this read, but the reactor must still not append + // between its writeTurns and this peek -- that axis is not serialized. const turns = baseStorage.peekTurns(); const metadata = await baseStorage.loadMetadata(); @@ -385,9 +622,15 @@ export async function createDurableConversationStore( mirroredTurnCount = reconstructed?.totalTurns ?? 0; } - // One WAL entry per boundary, unconditionally, so a turnless boundary - // (e.g. a throwing send that still advanced tokenUsage) still commits - // its metadata; the payload is the turn delta, never the whole conversation. + // ONE WAL entry per mirror boundary, UNCONDITIONALLY -- even when no new + // turns were added since the last mirror. The entry carries the + // 0-or-more new turns plus the freshest metadata snapshot, so a + // turnless-but-metadata-mutating boundary (e.g. a throwing send that + // still advanced tokenUsage/pendingOperations, since onRunBoundary runs + // in a finally) still durably commits its metadata. The payload is the + // turn DELTA plus bounded metadata -- never the whole conversation, so + // the O(N^2) growth stays gone. This is the single synchronous write + // per boundary on the reply path. const newTurns = turns.slice(mirroredTurnCount); const boundarySeq = mirroredBoundaryCount; await appendWalEntry(boundarySeq, newTurns, metadata); @@ -405,7 +648,11 @@ export async function createDurableConversationStore( // with the freshest metadata and truncate the WAL. Amortizes the // unavoidable O(N) full rewrite to O(N/K) per boundary. if (mirroredBoundaryCount - checkpointBoundarySeq >= CHECKPOINT_INTERVAL) { - await writeCheckpoint(mirroredBoundaryCount, turns.slice(0, mirroredTurnCount), metadata); + await writeCheckpoint( + mirroredBoundaryCount, + turns.slice(0, mirroredTurnCount), + metadata, + ); checkpointBoundarySeq = mirroredBoundaryCount; } } @@ -425,9 +672,15 @@ export async function createDurableConversationStore( } } - // commit() fires onStateChanged, enqueuing a change-driven mirror that reads - // connector state from local-store metadata, so the write below is what - // makes the seeded state reach the substrate. + // Advance the connector router from a received inbound message and flush + // the resulting connector state into the local store's metadata. `commit` + // fires the router's `onStateChanged`, which enqueues a change-driven + // mirror behind this op on the shared serialization tail; because that + // mirror reads the connector state from the local store's metadata (not + // from the router), the metadata write below is what makes the seeded + // state reach the substrate. The write preserves the reactor's staged + // pendingOperations / tokenUsage -- a seed advances only connectorState. + // A passthrough decision advances nothing and writes nothing. async function runSeed(message: InboundMessage): Promise { const decision = routeOrPassthrough(message); connectorRouter.commit(decision); @@ -444,8 +697,17 @@ export async function createDurableConversationStore( }); } - // Same metadata-write pattern as runSeed; onReplySent throws when no thread - // is active so a phantom advance is never persisted. + // Advance the connector thread after a reply was sent and flush the + // resulting connector state into the local store's metadata. `onReplySent` + // moves `lastMessageId` to the reply's Message-ID and fires the router's + // `onStateChanged`, which enqueues a change-driven mirror behind this op on + // the shared serialization tail; because that mirror reads the connector + // state from the local store's metadata (not from the router), the metadata + // write below is what makes the advanced state reach the substrate. The + // write preserves the reactor's staged pendingOperations / tokenUsage -- an + // outbound advance touches only connectorState. `onReplySent` throws when no + // thread is active, which surfaces to the reply drain's failure callback + // rather than persisting a phantom advance. async function runReplySent(receipt: SendReceipt): Promise { connectorRouter.onReplySent(receipt); @@ -485,7 +747,16 @@ export interface DurableConversationRegistryOpts { signer: (payload: string) => Promise; } -/** Per-agent store registry, built lazily and reused across runs; empty after a respawn since the substrate is what survives. */ +/** + * Per-agent durable-conversation store registry (design §3c). One store + * per warm agent key, built lazily and reused across runs in the same + * child. The first `acquire` for a key builds the store and restores its + * prior conversation snapshot from the substrate -- the path that runs on + * the lazy first build AND on the respawn rebuild, so the warm agent + * resumes its conversation across child respawn. The registry is empty + * after a respawn (it lives in the child's address space); the substrate + * is the durable mirror that survives. + */ export interface DurableConversationRegistry { acquire(key: string): Promise; get(key: string): DurableConversationStore; @@ -502,9 +773,7 @@ export function createDurableConversationRegistry( function localStoreDir(key: string): string { return path.join( - opts.dataDir, - "agent-conversation-state", - opts.workflowRunRepoId.id, + conversationStateRoot(opts.dataDir, opts.workflowRunRepoId.id), encodeURIComponent(key), ); } @@ -524,9 +793,13 @@ export function createDurableConversationRegistry( principal: opts.principal, agentKey: key, }); - // Restore before the store is observable; a no-op on first-ever run, - // otherwise pulls the pre-respawn conversation back. A restore failure - // surfaces rather than silently starting fresh. + // Restore the prior conversation BEFORE the store is observable (and + // before the warm agent's reactor `load()` reads it). On a genuine + // first-ever run this is a no-op (no checkpoint/WAL yet); on a + // respawn rebuild it pulls the pre-respawn conversation back from the + // substrate (checkpoint + WAL replay). A restore failure surfaces -- + // a lost conversation on respawn is a correctness failure, not a + // silently-fresh start. await store.restoreFromSubstrate(); stores.set(key, store); building.delete(key); @@ -558,7 +831,15 @@ interface SnapshotMetadataValue { connectorState: ConnectorThreadState | null; } -/** Reconstructed conversation plus the boundary/turn bookkeeping the mirror path needs to resume appending. */ +/** + * The reconstructed conversation plus the bookkeeping the mirror path needs + * to resume appending. `totalTurns` is the full turn count (checkpoint + + * WAL). `boundaryCount` is the number of mirror boundaries durably + * committed (checkpoint's folded boundaries + replayed WAL entries) -- the + * next WAL entry uses this as its boundary seq. `checkpointBoundarySeq` is + * the boundary seq the checkpoint folded to (the first WAL boundary seq to + * expect), so the mirror knows the live WAL length and when to compact. + */ export interface ReconstructedConversation extends LoadedSnapshot { totalTurns: number; boundaryCount: number; @@ -566,11 +847,20 @@ export interface ReconstructedConversation extends LoadedSnapshot { } /** - * Pure read: checkpoint turns + replayed WAL tail. Returns `null` on a - * genuine first-ever run; throws on any corrupt blob or WAL seq gap rather - * than silently starting fresh or dropping a turn. Exported so a reader - * (durability test, recovery audit) uses the same reconstruction path the - * warm agent's restore does. + * Reconstruct the warm agent's conversation from the two-tier on-disk + * layout under `agentStateDir` (`/agent-state//`): the + * compacted `checkpoint.json` turns followed by the replayed WAL tail. + * Pure read against the substrate working tree -- no inference, no commit. + * Returns `null` when neither a checkpoint nor any WAL exists (the genuine + * first-ever run). The latest metadata source wins (the last replayed WAL + * entry, or the checkpoint when the WAL is empty), mirroring how each + * mirror stamps the current metadata. Throws on any corrupt/unparseable + * blob or a WAL seq gap -- a damaged durable copy must surface, never + * silently start the agent fresh or drop a turn. + * + * Exported so a reader (durability test, recovery audit) reconstructs the + * conversation through the SAME code path the warm agent's restore uses, + * rather than re-deriving the WAL/checkpoint fold independently. */ export async function reconstructDurableConversation( agentStateDir: string, @@ -578,7 +868,11 @@ export async function reconstructDurableConversation( ): Promise { const checkpoint = await readCheckpointFromDir(agentStateDir, agentKey); const baseBoundarySeq = checkpoint?.checkpointSeq ?? 0; - const wal = await readWalTailFromDir(agentStateDir, agentKey, baseBoundarySeq); + const wal = await readWalTailFromDir( + agentStateDir, + agentKey, + baseBoundarySeq, + ); if (checkpoint === null && wal.length === 0) return null; const turns: unknown[] = [...(checkpoint?.turns ?? [])]; @@ -635,7 +929,10 @@ async function readCheckpointFromDir( } | null> { let metaRaw: string; try { - metaRaw = await fs.promises.readFile(path.join(agentStateDir, CHECKPOINT_META_FILE), "utf8"); + metaRaw = await fs.promises.readFile( + path.join(agentStateDir, CHECKPOINT_META_FILE), + "utf8", + ); } catch (cause) { if (isErrnoNotFound(cause)) return null; throw cause; @@ -647,8 +944,14 @@ async function readCheckpointFromDir( `sidecar conversation-state: ${CHECKPOINT_META_FILE} for ${agentKey} failed validation: ${validatedMeta.summary}; refusing to start the warm agent fresh on a corrupt checkpoint`, ); } - const snapshotRaw = await fs.promises.readFile(path.join(agentStateDir, CHECKPOINT_FILE), "utf8"); - const snapshot = parseJsonOrThrow(snapshotRaw, `${agentKey} ${CHECKPOINT_FILE}`); + const snapshotRaw = await fs.promises.readFile( + path.join(agentStateDir, CHECKPOINT_FILE), + "utf8", + ); + const snapshot = parseJsonOrThrow( + snapshotRaw, + `${agentKey} ${CHECKPOINT_FILE}`, + ); const validatedSnapshot = CheckpointSnapshot(snapshot); if (validatedSnapshot instanceof type.errors) { throw new Error( @@ -671,12 +974,21 @@ async function readCheckpointFromDir( }; } -/** Reads and seq-orders WAL entries >= fromSeq; throws on a corrupt blob or a seq gap rather than silently dropping a boundary. */ +/** + * Read and seq-order the per-boundary WAL entries for boundary seqs >= + * `fromSeq` from `/wal//`. Throws on any unparseable + * or out-of-shape WAL blob -- a corrupt WAL must surface, never be skipped. + * Throws on a gap in the boundary seq sequence: a missing seq means a lost + * append, which would silently drop a boundary's turns + metadata from the + * reconstruction. + */ async function readWalTailFromDir( agentStateDir: string, agentKey: string, fromSeq: number, -): Promise<{ seq: number; turns: unknown[]; metadata: SnapshotMetadataValue }[]> { +): Promise< + { seq: number; turns: unknown[]; metadata: SnapshotMetadataValue }[] +> { const walDir = path.join(agentStateDir, WAL_DIR); let buckets: string[]; try { @@ -699,8 +1011,14 @@ async function readWalTailFromDir( `sidecar conversation-state: unexpected non-JSON WAL entry ${WAL_DIR}/${bucket}/${file} for ${agentKey}`, ); } - const raw = await fs.promises.readFile(path.join(bucketDir, file), "utf8"); - const parsed = parseJsonOrThrow(raw, `${agentKey} ${WAL_DIR}/${bucket}/${file}`); + const raw = await fs.promises.readFile( + path.join(bucketDir, file), + "utf8", + ); + const parsed = parseJsonOrThrow( + raw, + `${agentKey} ${WAL_DIR}/${bucket}/${file}`, + ); const validated = WalEntry(parsed); if (validated instanceof type.errors) { throw new Error( @@ -742,9 +1060,3 @@ function parseJsonOrThrow(raw: string, label: string): unknown { ); } } - -export function isErrnoNotFound(cause: unknown): boolean { - if (cause === null || typeof cause !== "object") return false; - const code = (cause as { code?: unknown }).code; - return code === "ENOENT"; -} diff --git a/vendor/intx/workflow-host/src/deploy/atomic-write.ts b/vendor/intx/workflow-host/src/deploy/atomic-write.ts new file mode 100644 index 0000000000..4e77ebadd1 --- /dev/null +++ b/vendor/intx/workflow-host/src/deploy/atomic-write.ts @@ -0,0 +1,113 @@ +// Atomic, durable file replacement for the sidecar's non-rebuildable +// on-disk records. Distinct from the cache's rebuildable temp+rename +// (no fsync, a lost write just forces a re-fetch) and from +// `fsyncWriteFile`'s in-place fsync write (no atomicity, a torn write +// leaves a half-file): this is the tier for a sole restore source that +// must survive both a process kill and a power loss without ever +// exposing a torn record. + +import fs from "node:fs/promises"; +import { dirname } from "node:path"; + +import { getLogger } from "@intx/log"; +import { + syncDirectoryDurable, + syncRemovedPathDurable, +} from "@intx/storage-isogit/node"; +import { hasCode } from "@intx/types/has-code"; +import { hexEncode } from "@intx/types/hex"; + +const logger = getLogger(["interchange", "sidecar", "atomic-write"]); + +export interface AtomicWriteOptions { + /** Permission mode applied when the temp file is created. */ + mode: number; + /** Reject rather than degrade when the directory entry cannot be synced. */ + requireDirectorySync?: boolean; +} + +/** + * Replace `path` with `contents` atomically and durably. The bytes land + * in a fresh per-write temp file that is fsynced and then `rename`d over + * `path`; because rename is atomic within a directory, a reader only + * ever observes the prior complete file or the new complete file, never + * a torn one. The fsync before the rename is what extends that + * guarantee past process death to OS crash / power loss: without it, the + * ext4 delayed-allocation window can surface the renamed path as a + * zero-length file after a power loss. + * + * The parent directory is fsynced after the rename so the new link is + * itself durable. By default, a failed directory sync is logged. Set + * `requireDirectorySync` when acknowledging this write permits a caller to + * release an obligation that the previous file still records. + * + * `mode` is applied on the temp file's creation, so it takes effect on + * every write. A plain in-place overwrite of an existing file would + * silently keep the original file's mode instead. + * + * The temp file follows `createTarballCache`'s `.tmp..` + * naming convention for consistency across the sidecar's staged writes. + */ +export async function writeFileAtomicDurable( + path: string, + contents: string, + options: AtomicWriteOptions, +): Promise { + const tmp = `${path}.tmp.${String(process.pid)}.${hexEncode(crypto.getRandomValues(new Uint8Array(8)))}`; + try { + const handle = await fs.open(tmp, "w", options.mode); + try { + await handle.writeFile(contents); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(tmp, path); + } catch (cause) { + // The write failed and is about to rethrow; unlink the temp so a + // failed write leaves no orphan. Best-effort: the temp may never + // have been created, and a second failure here must not mask the + // original cause. + await fs.unlink(tmp).catch(() => undefined); + throw cause; + } + + try { + await syncDirectoryDurable(dirname(path)); + } catch (err) { + if (options.requireDirectorySync === true) throw err; + logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is renamed and fsynced — ${err instanceof Error ? err.message : String(err)}`; + } +} + +/** + * Remove `path` durably: unlink it, then fsync the parent directory so the + * removal survives a power loss. A raw `unlink` leaves the directory-entry + * removal in the OS's delayed-metadata window, so a power loss can resurrect + * the file -- for a cache whose on-disk entry is the restore source, a + * resurrected entry re-stales the cache. Idempotent: a file already absent is a + * completed removal, so ENOENT on the unlink is success. + * + * Directory sync runs even when the file is absent, committing a prior + * removal whose sync failed. By default, sync failures are logged. Strict + * mode propagates failures and, if the parent was also removed, syncs the + * nearest remaining ancestor so that directory removal is durable too. + */ +export async function removeFileAtomicDurable( + path: string, + { requireDirectorySync = false }: { requireDirectorySync?: boolean } = {}, +): Promise { + try { + await fs.unlink(path); + } catch (err) { + if (!(hasCode(err) && err.code === "ENOENT")) throw err; + } + + try { + if (requireDirectorySync) await syncRemovedPathDurable(path); + else await syncDirectoryDurable(dirname(path)); + } catch (err) { + if (requireDirectorySync) throw err; + logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is unlinked — ${err instanceof Error ? err.message : String(err)}`; + } +} diff --git a/vendor/intx/workflow-host/src/deploy/index.ts b/vendor/intx/workflow-host/src/deploy/index.ts new file mode 100644 index 0000000000..a379873ed9 --- /dev/null +++ b/vendor/intx/workflow-host/src/deploy/index.ts @@ -0,0 +1,29 @@ +// @intx/workflow-host/deploy -- the sidecar deploy program. +// +// Stages a workflow deployment onto a supervised workflow-process child. +// The process that boots this program supplies closure apply, asset +// delivery, registry and platform resolution, repo-id derivation, and the +// child spawner and binary. The workflow child does not load this entry. +export { + createSidecarDeployRouter, + createSidecarWorkflowSupervisor, + resolveDeploymentAssetMounts, + type SidecarDeployRouter, +} from "./workflow-host-wiring"; +export { + createDeploymentAddressRegistry, + createMultistepCredentialsRouter, + createMultistepDrainRouter, + createMultistepGrantsRouter, + createMultistepMailRouter, + createMultistepSignalRouter, + createMultistepSourcesRouter, + createWorkflowRunPackClient, + createWorkflowRunPackPushingRepoStore, +} from "./workflow-run-pack-client"; +export { createWorkflowRunPackRestorer } from "./workflow-run-pack-restore"; +export { + removeFileAtomicDurable, + writeFileAtomicDurable, +} from "./atomic-write"; +export { WORKFLOW_RUN_RECORD_FILENAME } from "./workflow-run-record"; diff --git a/vendor/intx/workflow-host/src/deploy/workflow-host-wiring.ts b/vendor/intx/workflow-host/src/deploy/workflow-host-wiring.ts new file mode 100644 index 0000000000..26c6aac499 --- /dev/null +++ b/vendor/intx/workflow-host/src/deploy/workflow-host-wiring.ts @@ -0,0 +1,3344 @@ +// Sidecar deploy program for the workflow host. +// +// Stages a workflow deployment onto a supervised workflow-process child: +// materializes the frozen closure, records the run, and registers the +// deployment's mail, signal, drain, and grants routes. The process that +// boots this program supplies the host bindings: closure apply, asset +// delivery, registry and platform resolution, repo-id derivation, and the +// child spawner and binary. + +import { chmod, lstat, readdir, rm, stat } from "node:fs/promises"; +import { join as pathJoin } from "node:path"; + +import { type } from "arktype"; + +import { derivePublicKeyBytes, signEd25519 } from "@intx/crypto"; +import { getLogger } from "@intx/log"; +import { syncRemovedPathDurable } from "@intx/storage-isogit/node"; +import type { HubTransport } from "@intx/mail-memory"; +import type { + Principal, + RepoId, + RepoStore, + WorkflowRunSupervisorPrincipal, +} from "@intx/hub-sessions/substrate"; +import { hexEncode } from "@intx/types/hex"; +import { parseRunAddress } from "@intx/types/agent-address"; +import { deriveWorkflowRunId } from "@intx/types/workflow-run-id"; +import { workflowSourceAssetMountPath } from "@intx/types/workflow-sources"; +import type { CredentialCipher } from "@intx/types/credential-cipher"; +import type { SignalKind } from "@intx/types/signals"; +import type { + ApprovalSnapshot, + CryptoProvider, + InferenceEvent, + InferenceSource, + KeyPair, + SendReceipt, +} from "@intx/types/runtime"; +import type { + InboundMailOutcome, + InboundMailPolicy, +} from "@intx/types/inbound-mail-policy"; +import { + DeploymentRejectedError, + fitDeploymentError, + WORKFLOW_CONTROL_INITIALIZING_ERROR, + WorkflowProjectionDefinition, + type AgentDeployFrame, + type AgentUndeployFrame, + type DeploymentStoppedFrame, + type DeploymentRetention, + type HostedIncarnation, + type IncarnationState, + type SourceRefPin, + type WorkflowControlFrame, + type WorkflowRunRefTips, + type WorkflowSourceAssetMount, +} from "@intx/types/sidecar"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import { + STEP_ID_PATTERN, + type WorkflowDefinition, +} from "@intx/workflow/definition"; +import { projectLiveToInert } from "@intx/workflow/projection"; + +import { loadWorkflowDefinitionFromClosure } from "../workflow-definition-loader"; +import { readRunGrants, runGrantsPath } from "../run-grants"; +import { + wrapHubTransportAsMailBus, + type HubTransportMailBusAdapter, +} from "../mail-bus/hub-transport-adapter"; +import { conversationStateRoot } from "../conversation-state-root"; +import { + hashGrants, + STEP_GRANTS_PATH, + STEP_GRANTS_REF, + type CredentialsSnapshot, + type CredentialsSnapshotStep, + type DeriveStepAddress, + type DeriveStepRepoId, +} from "../supervisor/credentials"; +import { + createWorkflowSupervisor, + type SpawnOpts, + type WorkflowSupervisor, +} from "../supervisor/supervisor"; +import type { + DispatchTimingMark, + SubprocessSpawner, + SuspensionRegistration, +} from "../supervisor/types"; +import type { + MultistepDrainRouter, + MultistepGrantsRouter, + MultistepMailRouter, + MultistepSignalRouter, + MultistepSourcesRouter, + MultistepCredentialsRouter, +} from "./workflow-run-pack-client"; +import { + deleteWorkflowRunRecord, + markWorkflowRunRecord, + scanWorkflowRunRecords, + updateWorkflowRunRecordSources, + withWorkflowRunRecord, + writeWorkflowRunRecord, + type WorkflowRunRecord, +} from "./workflow-run-record"; + +const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); + +// Keep concurrency low so children do not starve each other past their ready +// timeout on small hosts. Higher active capacity takes longer to restore; +// size maxDisconnected accordingly. +const RESTORE_CONCURRENCY = 8; + +/** + * The durable per-deployment store the sidecar checks a source-ref deployment's + * source assets out into. A SIBLING of the closure instance dir, not a child: + * `materializeDeploymentClosure` reclaims the closure dir on every apply and + * restore, but never this store, so the checked-out assets survive a restart + * and re-materialization needs no re-delivery. The store is reclaimed on + * redeploy (at the deploy call site) and on undeploy. + */ +function deploymentSourceAssetRoot( + dataDir: string, + deploymentId: string, +): string { + return pathJoin(dataDir, "workflow-definition-sources", deploymentId); +} + +/** + * The durable indexed-`.git` store root a pinned deployment's source-format + * asset entries are checked out from. Sibling of the plain-file source store; + * both survive restart so re-materialization needs no re-delivery. + */ +function deploymentSourceGitRoot( + dataDir: string, + deploymentId: string, +): string { + return pathJoin(dataDir, "workflow-definition-source-gits", deploymentId); +} + +/** + * The `assetId -> mountPath` map a pinned closure's TARBALL `kind:"asset"` + * entries resolve against, derived purely from the pin (via the shared + * mount-path helper) so deploy and restore agree without the frame's delivered + * assets. Source-format entries resolve through `deriveSourceGitDirs` instead. + */ +function deriveSourceAssetMounts(pin: SourceRefPin): Map { + const mounts = new Map(); + for (const entry of pin.closure.entries) { + if ( + entry.source.kind === "asset" && + entry.source.package.format === "tarball" + ) { + mounts.set( + entry.source.assetId, + workflowSourceAssetMountPath(entry.source.assetId), + ); + } + } + return mounts; +} + +/** + * The `assetId -> gitDir` map a pinned closure's SOURCE `kind:"asset"` entries + * check subtrees out of, derived purely from the pin so deploy and restore + * agree without re-delivery. + */ +function deriveSourceGitDirs( + pin: SourceRefPin, + gitRoot: string, + sourceAssetGitDir: (gitDirRoot: string, assetId: string) => string, +): Map { + const gitDirs = new Map(); + for (const entry of pin.closure.entries) { + if ( + entry.source.kind === "asset" && + entry.source.package.format === "source" + ) { + gitDirs.set( + entry.source.assetId, + sourceAssetGitDir(gitRoot, entry.source.assetId), + ); + } + } + return gitDirs; +} + +async function isExistingDir(dir: string): Promise { + try { + return (await stat(dir)).isDirectory(); + } catch (err) { + if (err instanceof Error && "code" in err && err.code === "ENOENT") { + return false; + } + throw err; + } +} + +// A tool can leave a directory without write permission, and removing a tree +// that holds one fails every time. Children run as the sidecar's OS user, so +// the sidecar can make such a directory writable again. +async function removeTree(dir: string): Promise { + try { + await rm(dir, { recursive: true, force: true }); + } catch (err) { + if ( + !( + err instanceof Error && + "code" in err && + (err.code === "EACCES" || err.code === "EPERM") + ) + ) { + throw err; + } + if ((await lstat(dir)).isDirectory()) await makeDirectoriesWritable(dir); + await rm(dir, { recursive: true, force: true }); + } + await syncRemovedPathDurable(dir); +} + +async function makeDirectoriesWritable(dir: string): Promise { + await chmod(dir, 0o700); + for (const entry of await readdir(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + await makeDirectoriesWritable(pathJoin(dir, entry.name)); + } + } +} + +/** + * Resolve the durable source-asset store root and `assetId -> mountPath` map a + * pinned deployment materializes its `kind:"asset"` closure entries from, + * asserting every referenced asset's mount directory is present on disk. This + * is a cheap early gate for a missing checkout; the loader still SRI-verifies + * each tarball's bytes at materialization. Derived purely from the pin, so the + * deploy path and the boot-time restore path (which has only the pin, no + * re-delivery) resolve the identical mounts. A missing mount is a broken + * deployment that cannot run without it, so it fails loud rather than + * materializing against an absent store. + */ +export async function resolveDeploymentAssetMounts( + dataDir: string, + deploymentId: string, + pin: SourceRefPin, + sourceAssetGitDir: (gitDirRoot: string, assetId: string) => string, +): Promise<{ + assetRoot: string; + assetMounts: ReadonlyMap; + gitDirs: ReadonlyMap; +}> { + const assetRoot = deploymentSourceAssetRoot(dataDir, deploymentId); + const assetMounts = deriveSourceAssetMounts(pin); + for (const [assetId, mountPath] of assetMounts) { + const mountDir = pathJoin(assetRoot, mountPath); + if (!(await isExistingDir(mountDir))) { + throw new Error( + `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} is not present in the durable store at ${mountDir}; the deployment cannot run without it`, + ); + } + } + const gitRoot = deploymentSourceGitRoot(dataDir, deploymentId); + const gitDirs = deriveSourceGitDirs(pin, gitRoot, sourceAssetGitDir); + for (const [assetId, gitDir] of gitDirs) { + if (!(await isExistingDir(gitDir))) { + throw new Error( + `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} has no indexed git store at ${gitDir}; the deployment cannot run without it`, + ); + } + } + return { assetRoot, assetMounts, gitDirs }; +} + +/** + * Read a substrate-config byte cap (`SIDECAR_CACHE_MAX_BYTES` / + * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) from the multi-step substrate env and + * parse it to a positive finite number. The boot edge resolves these once and + * threads them through the substrate env; a source-ref deploy needs them to + * size the tarball cache and per-fetch cap when it materializes the frozen + * workflow closure. A missing or non-numeric value is a boot-edge wiring bug, + * so it fails loud rather than defaulting. + */ +function requireSubstrateByteCap( + env: Record, + key: string, +): number { + const raw = env[key]; + if (raw === undefined) { + throw new Error( + `sidecar deploy router: ${key} must be present in the multi-step substrate env to materialize a frozen workflow closure`, + ); + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `sidecar deploy router: ${key} must be a positive finite number, got ${JSON.stringify(raw)}`, + ); + } + return parsed; +} + +/** + * Hub principal the deploy router presents when it writes a step's + * grants into the agent-state repo on the sidecar's substrate. The + * agent-state kind handler gates `writeTree` as hub-only; the deploy + * router is the local stand-in for the hub on the sidecar's disk, so it + * claims the hub principal for this single bookkeeping write. The child + * reads the same repo via the working-tree path (`getRepoDir`), which is + * not authorize-gated. + */ +const GRANTS_WRITE_PRINCIPAL: Principal = { kind: "hub" }; + +/** + * Per-deploy address/repo strategy. The single-step launched-agent + * deploy and the derived multi-step deploy disagree on how the per-step + * mail address and agent-state repo id are computed; both + * `deriveStepAddress` (consumed by the supervisor's credentialsSnapshot + * assembly for the step's mail address) and `deriveStepRepoId` (consumed + * by the same assembly to locate each step's grants) must agree on the + * choice, so they are minted together. + */ +type StepStrategy = { + deriveStepAddress: DeriveStepAddress; + deriveStepRepoId: DeriveStepRepoId; +}; + +/** + * Decide the per-step address/repo strategy from the projection's step + * count. + * + * `stepOrder.length === 1` is the single-agent deploy: the sole + * step keeps the deploy's own mail address, so its grants live in the + * agent-state repo keyed by `deriveWorkflowRunId(address)`. The spawned + * child reads the agent's grants from where the run's identity already + * lives, and the deploy frame's run address is preserved (the + * workflow-run repo stays keyed by `deriveWorkflowRunRepoId(address)`). + * + * Any other step count is a derived multi-step deploy: each step gets a + * derived `-` mail address (via the router's + * `multistepDeriveStepAddress`) and a derived agent-state repo under the + * default `-` convention. + * + * NOTE: the supervisor's `deriveStepAddress` feeds the credentials + * snapshot's per-step mail `address` and the grants-repo derivation. It + * does NOT feed the child's on-disk tool read (`stepDeployTreeDir` in + * `apps/sidecar/src/step-tool-materialization.ts`), which re-derives the + * step address from the deployment mailbox address independently. The + * deploy tree must therefore be staged at the address `stepDeployTreeDir` + * computes, regardless of this strategy's address choice. + */ +function createStepStrategy(args: { + legacyAddress: string; + stepOrder: readonly string[]; + multistepDeriveStepAddress: DeriveStepAddress; +}): StepStrategy { + if (args.stepOrder.length === 1) { + return { + deriveStepAddress: () => args.legacyAddress, + // `deriveWorkflowRunId` is deferred into the closure rather than + // computed eagerly: the supervisor only invokes `deriveStepRepoId` + // while assembling the credentialsSnapshot inside `spawn()`, so a + // malformed address surfaces at the same point the rest of the + // spawn path would fault rather than ahead of the deploy router's + // other boundary checks. + deriveStepRepoId: () => ({ + kind: "agent-state", + id: deriveWorkflowRunId(args.legacyAddress), + }), + }; + } + return { + deriveStepAddress: args.multistepDeriveStepAddress, + deriveStepRepoId: ({ runId, stepId }) => ({ + kind: "agent-state", + id: `${runId}-${stepId}`, + }), + }; +} + +/** + * Write a grant set into the sidecar's substrate as the canonical + * `{ grants: WireGrantRule[] }` envelope -- the shape + * `assembleCredentialsSnapshot` validates (`{ grants: unknown[] }`) and + * the child's `evaluateGrants` adapter narrows to `GrantRule[]`. + * + * Two destinations share this write machinery, selected by `runId`: + * + * - `runId` absent (deploy time): write every step's grants into its + * own `agent-state` repo at `STEP_GRANTS_PATH`, keyed by the same + * `deriveStepRepoId` the supervisor reads with, so read and write + * address the same repo. `stepOrder` here is the deployment's whole + * flat step-id namespace, not the bare top-level order, because the + * supervisor assembles a snapshot entry for every loop-body step too. + * The write is on the spawn critical path: a failure rejects the + * deploy (the caller's `finally` unwinds the partial state) rather + * than spawning a child that would fail every authorize closed + * against an empty grant set. + * + * - `runId` present (per-run delivery): write a single + * `runs//grants.json` into the deployment's `workflow-run` + * repo, sibling to that run's `runs//events/` subtree. The + * `stepOrder` / `deriveStepRepoId` fields are unused in this mode -- + * the destination is the one workflow-run repo keyed by + * `runId`, not a per-step fan-out. + * + * Both destinations use the same hub principal and `refs/heads/main` + * ref: the agent-state kind handler gates `writeTree` as hub-only, and + * the workflow-run repo pins its run subtree under the same moving ref. + */ +async function writeStepGrants(args: { + repoStore: RepoStore; + anchorRunId: string; + stepOrder: readonly string[]; + deriveStepRepoId: DeriveStepRepoId; + grants: readonly unknown[] | undefined; + runId?: string; +}): Promise { + // The deploy frame's validated HarnessConfig always carries a `grants` + // array (possibly empty); an absent array means "no grants", which + // serializes to the same fail-closed empty file the snapshot expects. + // Coerce here so the on-disk envelope is always a valid `{ grants: [] }` + // rather than `{}` (which the snapshot's validator rejects). + const grants = args.grants ?? []; + const serialized = JSON.stringify({ grants }, null, 2); + if (args.runId !== undefined) { + await args.repoStore.writeTree( + GRANTS_WRITE_PRINCIPAL, + { kind: "workflow-run", id: args.anchorRunId }, + STEP_GRANTS_REF, + { + files: { [runGrantsPath(args.runId)]: serialized }, + message: `Write run grants for ${args.runId}`, + }, + ); + return; + } + for (const stepId of args.stepOrder) { + const repoId = args.deriveStepRepoId({ + runId: args.anchorRunId, + stepId, + }); + await args.repoStore.writeTree( + GRANTS_WRITE_PRINCIPAL, + repoId, + STEP_GRANTS_REF, + { + files: { [STEP_GRANTS_PATH]: serialized }, + message: `Write step grants for ${stepId}`, + }, + ); + } +} + +export type AssembleRunCredentialsSnapshotOpts = { + /** Substrate handle the sink reads the per-run grants file from. */ + repoStore: RepoStore; + /** Anchor run id keying the workflow-run repo the grants file lives in. */ + anchorRunId: string; + /** Run whose per-run grants file is read. */ + runId: string; + /** + * Every step id the snapshot must cover -- the deployment's flat step-id + * namespace, loop-body step ids included. The per-run grants apply + * uniformly across them. + */ + stepOrder: readonly string[]; + /** Per-step mail-address derivation. */ + deriveStepAddress: DeriveStepAddress; +}; + +/** + * Resolve a run's credentials snapshot for the `onRunStart` grants + * barrier from its per-run grants file. + * + * Every legitimate run birth path writes `runs//grants.json` in + * the deployment's workflow-run repo before the run dispatches -- the + * external trigger route and the mail-triggered path both ship a + * `run.grants` frame the sidecar writes, and a spawned child inherits its + * parent's grants directly at spawn without reaching this barrier. The + * per-run file IS the run's snapshot: the run's single flat grant set is + * applied uniformly across every step, keyed on each step's address. + * + * A missing file is therefore not an internal run inheriting deploy-time + * grants -- it is a run that reached its barrier with no grants written, + * so it FAILS CLOSED here rather than running under-authorized. A file + * that exists but is malformed also throws (via `readRunGrants`), for the + * same reason: the file's presence implies a grants frame was delivered, + * so a structural failure is a boundary bug, not a default. + */ +export async function assembleRunCredentialsSnapshot( + opts: AssembleRunCredentialsSnapshotOpts, +): Promise { + const runGrants = await readRunGrants({ + repoStore: opts.repoStore, + anchorRunId: opts.anchorRunId, + runId: opts.runId, + }); + if (runGrants === undefined) { + throw new Error( + `sidecar onRunStart: run ${opts.runId} has no grants file at ${runGrantsPath(opts.runId)}; refusing to start the run under-authorized`, + ); + } + const contentHash = await hashGrants(runGrants); + const steps: CredentialsSnapshotStep[] = opts.stepOrder.map((stepId) => ({ + stepId, + address: opts.deriveStepAddress({ + runId: opts.anchorRunId, + stepId, + }), + grants: runGrants, + contentHash, + })); + return { steps }; +} + +export type CreateSidecarWorkflowSupervisorOpts = { + /** Sidecar's hub mail transport. */ + transport: HubTransport; + /** Substrate-shaped RepoStore the workflow-host's supervisor reads from. */ + repoStore: RepoStore; + /** Sidecar's 32-byte Ed25519 private key seed for principal signing. */ + signingKeySeed: Uint8Array; + /** Workflow-run repo identity for the deployment. */ + workflowRunRepoId: RepoId; + /** Workflow-run repo ref the supervisor commits events to. */ + workflowRunRef: string; + /** Deployment id baked into principal claims and address derivation. */ + runId: string; + /** + * Decrypted credential material for the deployment's binding consumers and + * inference sources (from the deploy frame). Delivered to the child on the + * pre-trigger barrier. Absent when the deployment binds no credentials. + */ + credentialDelivery?: CredentialDelivery; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). + * Threaded into the child's spawn-time env so its deploy-tree read + * collapses onto the head for a single-step deployment. + */ + stepCount: number; + /** + * Every step id in the deployment's flat step-id namespace: its + * `stepOrder` plus the step ids of every `loop` body it carries. The + * `onRunStart` grants sink walks these to assemble the per-run + * credentialsSnapshot, so the sink needs the ids rather than the bare + * count -- and it needs the loop-body ids too, because a loop iteration + * inherits the parent run's env and authorizes against this same snapshot. + */ + stepOrder: readonly string[]; + /** Deployment's mail address. */ + deploymentMailAddress: string; + /** Per-step mail-address derivation. */ + deriveStepAddress: DeriveStepAddress; + /** + * Optional override of the per-step `agent-state` repo identity the + * supervisor reads grants from while assembling the + * credentialsSnapshot. Defaults to the `-` + * convention; the single-step launched-agent deploy supplies a + * derivation that returns the legacy agent-state repo so the spawned + * child reads grants from the same repo the legacy agent identity + * keys. + */ + deriveStepRepoId?: DeriveStepRepoId; + /** Substrate-config keys propagated to the child via spawn-time env. */ + substrateEnv: Record; + /** + * Dynamic spawn-env fragment the supervisor recomputes on every spawn and + * recycle respawn (e.g. a live-rotated inference-source list). Its keys + * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. + */ + dynamicSpawnEnv: () => Record; + /** + * Subprocess spawner. The booting process passes the Bun-backed + * spawner; tests pass a deterministic mock. + */ + subprocessSpawner: SubprocessSpawner; + /** Path of the workflow-child binary the spawner launches. */ + binaryPath: string; + /** + * Optional per-message dispatch-timing observer, forwarded verbatim to + * the supervisor's `onDispatchTiming` binding. Absent in production; + * the deploy router wires it (off a benchmark env gate) only for the + * Phase 4.7 latency gate, which needs the supervisor to emit the + * per-message infra round-trip from inside the sidecar subprocess. + */ + onDispatchTiming?: (mark: DispatchTimingMark) => void; + /** + * D2 §10c forced-repack A/B toggle, forwarded verbatim to the + * supervisor's `repackEveryMessages` binding. Absent in production; + * the deploy router wires it (off the same benchmark env gate) only + * for the D2 attribution run. + */ + repackEveryMessages?: { everyMessages: number }; + /** + * Consumed-dedup retention horizon (ms), forwarded to the + * supervisor's `consumedRetentionMs` binding. The boot edge resolves + * the operator's `CONSUMED_RETENTION_MS` config; absent, the + * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). + */ + consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms), forwarded to the supervisor's + * `readyTimeoutMs` binding. The boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). + */ + readyTimeoutMs?: number; + /** + * Control-plane suspension sink forwarded to the supervisor's + * `onSuspensionRegister` binding. The supervisor stamps `runId` + + * `agentAddress` and invokes this when a workflow-process child reports a + * `park.notify`; production wiring routes it to the hub link so a + * `signal.correlation.register` frame reaches the hub. Absent means the + * deployment registers no suspensions. + */ + onSuspensionRegister?: (registration: SuspensionRegistration) => void; + /** + * Self-termination sink forwarded to the supervisor's `onSelfTerminate` + * binding. The supervisor invokes it when it drives itself to a terminal + * phase (crash-loop latch, channel crash, recycle failure); production wiring + * routes it to the deploy router, which holds the deployment as stopped. + * Absent means a self-termination is not surfaced to the host. + */ + onSelfTerminate?: (info: { + phase: "stopped" | "crash-looping"; + reason: string; + }) => void; + /** + * Predicate consulted at the `onRunStart` grants barrier: returns `true` + * for a runId whose `run.grants` write was attempted and FAILED, so the + * per-run grants file the run needs never landed. The barrier fails such a + * run loudly rather than starting it under-authorized. This is a + * fast-fail with a precise cause: an absent grants file already fails the + * barrier closed on its own (every run birth path writes the file before + * dispatch, so its absence is a defect), and this predicate lets a run + * whose write is KNOWN to have failed reject with that specific reason + * instead of a generic missing-file error. Absent means no run is + * poisoned. + */ + isRunPoisoned?: (runId: string) => boolean; +}; + +export type SidecarWorkflowSupervisor = { + supervisor: WorkflowSupervisor; + /** + * Hand a delivered inbound message off to the supervisor's mail + * subscription. The returned promise resolves once the message is durably + * accepted and rejects when it was not, so the hub-link can send a + * `mail.inbound.ack` only on resolution (resolve = ack, reject = withhold). + */ + routeInbound(message: Uint8Array): Promise; + /** Snapshot accessor that proxies the supervisor's credentials view. */ + getCredentialsSnapshot(): CredentialsSnapshot | null; + /** + * The per-run grants barrier the supervisor awaits before firing a run's + * trigger. Rejects for a poisoned run (its `run.grants` write failed) and + * otherwise resolves the run's credentials snapshot. Exposed so the barrier + * can be exercised without driving a full spawn. + */ + onRunStart(args: { + runId: string; + anchorRunId: string; + }): Promise; + /** + * Refuse further outbound mail and wait for the sends already started. The + * undeploy and the stop call it once the supervisor has shut down, so + * everything the deployment sent reaches the link while its incarnation is + * still held, and is stamped with its generation. + */ + closeOutbound(): Promise; +}; + +/** + * Env key the multi-step branch uses to carry each step's ordered + * inference-source failover chain from `frame.workflow.sources` down to + * the workflow-process child. The substrate factory's `buildEnv` reads + * this and resolves a step's chain at step invocation, feeding it to the + * reactor for forward-only failover; the supervisor itself is opaque to + * the value (it is plumbed through `bindings.substrateEnv` verbatim). + * + * Listed here so the router and the future substrate-factory consumer + * spell the key the same way without a magic-string trip hazard. + */ +export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; + +/** + * Spawn-env key carrying every spawned body's per-step inference-source pins as + * a JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` map. The sidecar + * decrypts the sealed body sources from the run record and serializes the + * plaintext here so the run child resolves a body's sources without holding the + * sidecar's cipher key. Mirrors `STEP_INFERENCE_SOURCES` for the top level. + */ +export const WORKFLOW_BODY_SOURCES_ENV_KEY = "WORKFLOW_BODY_SOURCES"; + +/** + * A single env value cannot exceed the OS argument-string ceiling + * (`MAX_ARG_STRLEN`, 128 KiB on Linux); an over-long value makes the child + * `execve` fail opaquely at spawn. `WORKFLOW_BODY_SOURCES` sums over every body + * at every depth, so the deploy path validates the serialized size against this + * bound and rejects an over-large deployment loudly at deploy time rather than + * letting a much-later spawn fail. Held below the ceiling with headroom for the + * rest of the env block. + */ +const WORKFLOW_BODY_SOURCES_MAX_BYTES = 96 * 1024; + +/** + * Build the record's `bodySources` map from the deploy frame's referenced body + * definitions (the flat set of onTrigger sections and childWorkflow children, + * each already source-pinned by the hub). `undefined` when the deployment + * references no bodies, so the record omits the field. + */ +function buildBodySourcesMap( + referenced: NonNullable< + AgentDeployFrame["workflow"] + >["referencedDefinitions"], +): WorkflowRunRecord["bodySources"] { + if (referenced === undefined || referenced.length === 0) { + return undefined; + } + const map: NonNullable = {}; + for (const ref of referenced) { + map[ref.definition.id] = ref.sources; + } + return map; +} + +/** + * Validate the wire-projected workflow definition at the deploy-router + * boundary. The arktype `AgentDeployFrame` validator enforces the + * wire shape (`id` is non-empty, `stepOrder` is `string[]`, `steps` + * is an object, `sources` covers every `stepOrder` entry); this + * function takes `unknown`-typed inputs so it can also gate callers + * that bypass the wire boundary, and it enforces the invariants the + * router and the downstream supervisor rely on: + * + * - `definition.id` is a non-empty string. The arktype shape + * already enforces this on the wire; the re-check here protects + * bypass callers and keeps the failure shape consistent with the + * other invariants this function owns. + * - `definition.stepOrder` is non-empty. The wire shape admits + * `[]`; a zero-step workflow has no semantics here. + * - Every `stepOrder` entry matches `STEP_ID_PATTERN` so per-step + * mail-address derivation never needs escaping at the substrate + * boundary. + * - Every `stepOrder` entry has a corresponding `steps[id]` entry. + * The wire shape lets `steps[id]` be `unknown` and lets the + * entry be absent; presence is required so the workflow-process + * child can resolve each step's primitive at run time. + * - Every `stepOrder` entry has a corresponding `sources[id]` + * entry, and that entry is a non-empty array (the step's ordered + * failover chain). The arktype narrow already enforces both; the + * re-check here surfaces a structured router-side error instead of + * an arktype validation failure at the wire boundary, which keeps + * the failure shape consistent with the rest of the validations + * this function owns. An empty chain would leave the reactor with + * no initial source, so it is rejected here rather than deferred to + * a deep-stack child failure. + * + * A rejection here surfaces as a thrown `Error` the link's deploy + * frame caller converts into a structured failure reply. + */ +export function validateWorkflowProjection(projection: { + definition: { id: unknown; stepOrder: unknown; steps: unknown }; + sources: unknown; +}): void { + const def = projection.definition; + if (typeof def.id !== "string" || def.id.length === 0) { + throw new Error( + "sidecar deploy router: workflow.definition.id must be a non-empty string", + ); + } + if (!Array.isArray(def.stepOrder) || def.stepOrder.length === 0) { + throw new Error( + "sidecar deploy router: workflow.definition.stepOrder must be a non-empty array", + ); + } + if (typeof def.steps !== "object" || def.steps === null) { + throw new Error( + "sidecar deploy router: workflow.definition.steps must be an object", + ); + } + if (typeof projection.sources !== "object" || projection.sources === null) { + throw new Error( + "sidecar deploy router: workflow.sources must be an object", + ); + } + const steps = def.steps; + const sources = projection.sources; + for (const stepId of def.stepOrder) { + if (typeof stepId !== "string" || stepId.length === 0) { + throw new Error( + "sidecar deploy router: workflow.definition.stepOrder entries must be non-empty strings", + ); + } + if (!STEP_ID_PATTERN.test(stepId)) { + throw new Error( + `sidecar deploy router: stepId ${JSON.stringify(stepId)} must match ${STEP_ID_PATTERN.source}`, + ); + } + if (!Object.prototype.hasOwnProperty.call(steps, stepId)) { + throw new Error( + `sidecar deploy router: workflow.definition.steps is missing entry for stepId ${JSON.stringify(stepId)}`, + ); + } + if (!Object.prototype.hasOwnProperty.call(sources, stepId)) { + throw new Error( + `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, + ); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- sources is checked to be a non-null object above; this reads a value to re-check its array shape + const stepSources = (sources as Record)[stepId]; + if (!Array.isArray(stepSources) || stepSources.length === 0) { + throw new Error( + `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, + ); + } + } +} + +/** + * Derive the supervisor's principal public key from the sidecar's + * Ed25519 signing seed. The supervisor signs every workflow-run event + * with this key; the multi-step branch surfaces it to the link so the + * hub records the verifying key for the deployment's signed events. + */ +async function derivePrincipalPublicKeyHex( + signingKeySeed: Uint8Array, +): Promise { + return hexEncode(await derivePublicKeyBytes(signingKeySeed)); +} + +/** Values the link folds into the outbound deploy ack. */ +type DeployRouterResult = { + publicKey: string; +}; + +/** What a handled workflow control reports back to the hub. */ +type WorkflowControlOutcome = { + refTips?: WorkflowRunRefTips; + retention?: DeploymentRetention; +}; + +type ResolvedInboundMailPolicy = Record; + +/** + * The link routes `agent.deploy`, `agent.undeploy`, and `workflow.control` + * through `deploy`, `undeploy`, and `control`. The boot edge also calls + * `restoreWorkflowRuns` once before connecting to the hub, reads + * `incarnations` on each reconnect, and asks `reEmitParkedCorrelations` + * to recover correlations dropped during an outage. + */ +export interface SidecarDeployRouter { + deploy(frame: AgentDeployFrame): Promise; + undeploy(frame: AgentUndeployFrame): Promise; + control(frame: WorkflowControlFrame): Promise; + /** + * Re-establish every persisted workflow deployment on this sidecar's local + * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step + * head's mailbox/transport registration is live before the hub routes to it. + * Aborts boot if the inventory cannot be read durably or exceeds either + * capacity limit. A deployment that cannot be restored (unbuildable + * provider, a source closure that cannot be re-materialized, spawn failure) + * is held as stopped with why, and its record is marked so, so a later boot + * reports it instead of trying again and the Hub fails it. Its record is + * never deleted here. + */ + restoreWorkflowRuns(): Promise; + /** + * Every incarnation this router holds, from the moment its deploy or + * restore starts until its teardown finishes, with what it is doing with + * each. The hub link reports them in every `hello` and delivers a Hub frame + * only to one held live. Reflects `deploy`/`undeploy` and boot-time restore + * live, so a caller re-reads it whenever it needs it. + */ + incarnations(): HostedIncarnation[]; + /** + * Report deployments that stopped on their own and have no durable retention + * decision, with the error that ended them. Called after every `welcome`, + * as a report sent while the link was down is lost. + */ + reportStoppedDeployments(): void; + /** + * Trigger B: re-register every correlation the deployment at `address` is + * currently parked on, by asking its live supervisor to re-emit them to the + * hub. The boot edge calls this per address the Hub routes in its `welcome` + * (`onWorkflowAddressesRoutable`), so a register frame the link dropped from + * its bounded report queue during an outage is recovered. + * The address-dispatch wrapper around the supervisor's own no-arg + * `reEmitParkedCorrelations`; fire-and-forget (the driver is best-effort and + * watchdog-bounded inside the supervisor). + */ + reEmitParkedCorrelations(address: string): void; +} + +export function createSidecarDeployRouter< + THost, + TRegistries, + TCache extends object, +>(deps: { + sessions: { + initRepo(address: string): Promise; + /** Remove the directory durably before resolving; reject sync failures. */ + deleteAgentDir(address: string): Promise; + }; + keyStore: { + loadOrGenerateKey( + address: string, + ): Promise<{ keyPair: KeyPair; isNew: boolean }>; + recordHubKey(address: string, hexHubPublicKey: string): void; + forgetAgent(address: string): void; + }; + transport: HubTransport; + repoStore: RepoStore; + signingKeySeed: Uint8Array; + /** + * The sidecar cipher that seals credential material at rest. Used to seal + * each run record's inference-source apiKeys on persist and unseal them on + * the boot scan, so the durable record carries ciphertext rather than + * plaintext secrets. + */ + credentialCipher: CredentialCipher; + /** + * Per-agent crypto factory. Receives the agent's raw key pair and + * returns a `CryptoProvider` bound to it (production wires + * `@intx/crypto`'s `createEd25519Crypto`). The multi-step branch + * uses this to register the spawned single-step agent's signing key on + * the host transport before `spawn()`, so the supervisor's outbound + * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies + * with the AGENT's identity -- the OUTBOUND half of mailbox ownership + * (§3a). Without this registration the spawned agent's address has no + * `CryptoProvider` on the transport (nothing else registers one for + * it), and an outbound send would throw "address is not registered" + * rather than emit unsigned mail. + */ + createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; + /** + * Source-admission gate: throws if a step's pinned inference source + * names a provider this sidecar cannot build. The buildable-provider + * set is sidecar config (the boot edge's adapter registry), so this + * admission control lives at the sidecar -- the hub is a different + * process and cannot know a given sidecar's providers. Production wires + * the default harness builder's `canBuildSource` verbatim, so a rejected + * provider carries the same `"... is not registered"` message. + * + * Distinct from the orchestrator's operator-approval check + * (`pickStepInferenceSource`): that gates on whether the operator + * approved a `provider:model`; this gates on whether the provider is + * buildable at all. A source can be approved yet unbuildable. + */ + assertSourceBuildable: (source: InferenceSource) => void; + /** + * Record a `(runId -> incarnation)` mapping the boot edge's + * workflow-run pack push facade consults when it must address and stamp + * an outbound pack frame. Fires once per inbound `agent.deploy` frame + * before the deployment's supervisor spawns, so the first pack push + * the child triggers sees the mapping. Tests that do not exercise + * the pack push path may pass a no-op. + */ + registerDeployment: (entry: { + runId: string; + agentAddress: string; + generation: number; + }) => void; + /** + * Symmetric removal hook for `registerDeployment`. Fires from the + * link's `agent.undeploy` path so the boot edge's + * `DeploymentAddressRegistry` drops the mapping when the deployment + * is torn down. A subsequent stale `writeTreePreservingPrefix` + * against the dead deployment's workflow-run ref surfaces + * structurally (`registry.resolve` returns `null`) rather than + * silently resolving to the prior address. Tests that do not + * exercise the pack push path may pass a no-op. + */ + unregisterDeployment: (entry: { + runId: string; + agentAddress: string; + }) => void; + /** + * Read the tips of a stopped deployment's workflow-run refs and schedule a + * push of any commit the hub has not acknowledged. The hub confirms the stop + * only once it holds these tips. Tests that do not exercise the pack push + * path may report no refs. + */ + reportDeploymentRefTips: (incarnation: { + agentAddress: string; + generation: number; + }) => Promise; + /** + * Substrate-config env keys the multi-step branch propagates into + * the workflow-process child's spawn-time env (see + * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `substrate-factory.ts`). + * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step + * frame. Defaults to an empty record so a router built without + * substrate config (e.g. a test) needs no boot-edge threading. + */ + multistepSubstrateEnv?: Record; + /** + * Subprocess spawner the multi-step branch hands to the supervisor. + * The booting process passes the Bun-backed spawner; tests pass a + * deterministic mock. + */ + multistepSubprocessSpawner: SubprocessSpawner; + /** + * Path of the workflow-child binary the spawner launches. Empty is + * rejected at construction. + */ + multistepBinaryPath: string; + /** + * Callback the supervisor invokes for every verified InferenceEvent + * the workflow-process child publishes. The router threads the + * deployment's run address, its incarnation's generation, and the deploy's + * session id through to the callback so a downstream fan-out can route + * each event to the hub timeline keyed to the right session. The + * `InferenceEvent` itself + * is sessionless; the session id rides alongside it, sourced from the + * deploy frame's `HarnessConfig.sessionId` per deployment. It is + * optional because a deploy frame need not carry a session id (a + * headless deployment with no hub-side session); the sink decides what + * an absent session id means. Defaults to a no-op; production wiring + * supplies the event publisher. + */ + publishWorkflowInferenceEvent?: ( + agentAddress: string, + generation: number, + event: InferenceEvent, + sessionId: string | undefined, + childRunId?: string, + ) => void; + /** + * Callback the supervisor invokes for every control-plane suspension a + * workflow-process child reports (`park.notify`). The multi-step branch + * threads it into the supervisor's `onSuspensionRegister` binding so a + * parked run's correlation is registered at the hub (routing + approval + * rows). The supervisor stamps `runId` + `agentAddress` and the router + * stamps the incarnation's generation before invoking it. Defaults to a + * no-op; production wiring supplies the hub-link-backed publisher. + */ + publishWorkflowSuspension?: (registration: { + correlationId: string; + runId: string; + anchorRunId: string; + agentAddress: string; + generation: number; + kind: SignalKind; + approvalSnapshot?: ApprovalSnapshot; + }) => void; + /** + * Optional override for the multi-step branch's per-step mail-address + * derivation. Defaults to `${runId}-${stepId}@` + * derived from the frame's run address. Tests inject a deterministic + * factory. + */ + multistepDeriveStepAddress?: DeriveStepAddress; + /** + * Per-deployment-address mail handler registry the hub-link's + * `mail.inbound` path consults before falling back to the legacy + * session-routed delivery. The multi-step branch registers + * `wired.routeInbound` against the deployment's mail address once + * `supervisor.spawn` succeeds so inbound mail aimed at the + * deployment address flows into the supervisor's mail-bus + * subscription. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end mail loop can omit the binding; an absent registry + * simply means multi-step inbound mail cannot route through the + * hub-link until the wiring is plumbed. + */ + multistepMailRouter?: MultistepMailRouter; + /** + * Per-recipient-address registry of resolved inbound-mail admission + * policies the sidecar hub-link's `mail.inbound` seam enforces. The + * multi-step branch resolves this deployment's authored + * `inboundMailPolicy` into the registry once `supervisor.spawn` + * succeeds -- beside the mail-router registration -- and removes the + * entry in the same teardown, so a reused address never inherits a + * stale policy. + * + * Optional so tests that exercise the multi-step branch without the + * hub-link seam can omit it; an absent registry means an inbound frame + * for the address resolves to the fully-closed default and is rejected + * until the wiring is plumbed. + */ + inboundMailPolicyRegistry?: { + register(address: string, policy: ResolvedInboundMailPolicy): void; + unregister(address: string): void; + }; + /** + * Per-deployment-address signal handler registry the sidecar + * hub-link's `signal.deliver` path consults. The multi-step branch + * registers `wired.supervisor.deliverSignal` against the deployment's + * mail address once `supervisor.spawn` succeeds so a hub-side + * `signal.deliver` frame flows into the workflow-process child via + * the IPC's `signal.deliver` payload. The child commits the + * resulting `SignalReceived` event through its own substrate, + * preserving the workflow-run repo's single-writer invariant on the + * sidecar side. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end signal loop can omit the binding; an absent registry + * means hub-side signals cannot route through the hub-link until the + * wiring is plumbed. + */ + multistepSignalRouter?: MultistepSignalRouter; + /** + * Per-deployment-address drain handler registry the sidecar + * hub-link's `drain.deliver` path consults. The multi-step branch + * registers `wired.supervisor.drain` against the deployment's mail + * address once `supervisor.spawn` succeeds so a hub-side + * `drain.deliver` frame flows into the workflow-process child via + * the IPC's `drain` payload and arms the supervisor's per-run + * `drainTimeout` accumulators. Cancel-mode in-flight steps abort on + * the child side; wait-mode steps continue. Accumulators commit a + * signed `CancelRequested{origin: "supervisor-drain"}` against the + * workflow-run repo when the deadline expires. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end drain loop can omit the binding; an absent registry + * means hub-side drain frames cannot route through the hub-link until + * the wiring is plumbed. + */ + multistepDrainRouter?: MultistepDrainRouter; + /** + * Per-deployment-address grants handler registry the sidecar + * hub-link's `run.grants` path consults. The deploy router registers a + * handler against the deployment's mail address once `supervisor.spawn` + * succeeds, for single-step and multi-step deployments alike, so a + * hub-side `run.grants` frame writes the run's grants to + * `runs//grants.json` inside the deployment's `workflow-run` + * repo. The write is awaited in `tryRoute` so the frame's FIFO + * completion means the grants are durable on disk. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end grants loop can omit the binding; an absent registry + * means hub-side grants frames cannot route through the hub-link until + * the wiring is plumbed. + */ + multistepGrantsRouter?: MultistepGrantsRouter; + /** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (against the + * deployment's mail address once `supervisor.spawn` succeeds) so a + * rotation resolved for its address flows into + * `wired.supervisor.deliverSources` and on to the child's warm agent. A + * multi-step deployment registers none -- it has no single warm agent to + * rotate -- so `tryRoute` reports its address as unrouted. + * + * Optional so tests that exercise deploys without a rotation loop can + * omit the binding; an absent registry means no rotation handler is + * installed for any deployment. + */ + multistepSourcesRouter?: MultistepSourcesRouter; + /** + * Optional per-deployment credential-delivery handler registry. Every + * deployment with a supervisor registers a handler after `spawn` (not only + * warm single-step ones -- the material cell is per-child and read by every + * step's tool capabilities), and an inbound `credentials.update` for an + * unregistered (torn-down) address is unrouted. Optional so tests without a + * credential-delivery loop can omit the binding. + */ + multistepCredentialsRouter?: MultistepCredentialsRouter; + /** + * Optional per-message dispatch-timing observer the multi-step branch + * forwards to each supervisor it constructs. Resolved at the sidecar + * boot edge from the Phase 4.7 latency-gate env gate; absent in + * ordinary production. The supervisor runs in this sidecar subprocess, + * so the observer sees both ends of the per-message IPC round-trip in + * one process and can emit a parseable timing line the benchmark + * harness reads off the subprocess's output stream. + */ + onDispatchTiming?: (mark: DispatchTimingMark) => void; + /** + * D2 §10c forced-repack A/B toggle the multi-step branch forwards to + * each supervisor it constructs. Resolved at the sidecar boot edge from + * the same benchmark env gate; absent in ordinary production. + */ + repackEveryMessages?: { everyMessages: number }; + /** + * Consumed-dedup retention horizon (ms) forwarded to every supervisor + * the router constructs. The sidecar boot edge resolves the operator's + * `CONSUMED_RETENTION_MS` config; absent, the supervisor applies + * `DEFAULT_CONSUMED_RETENTION_MS` (24h). See the workflow-run kind + * handler for the operator-owned horizon invariant. + */ + consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms) forwarded to every supervisor the + * router constructs. The sidecar boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never + * signals ready is killed and its spawn rejected rather than hanging + * the deploy or boot-time restore. + */ + readyTimeoutMs?: number; + /** + * Run-record source updater, injectable so a test can block or fail the + * persist at a controlled point -- the natural seam for exercising a + * recycle that interleaves the source-rotation persist window. Defaults + * to the real `updateWorkflowRunRecordSources`; production never overrides + * it. + */ + updateWorkflowRunRecordSources?: typeof updateWorkflowRunRecordSources; + /** Create the closure cache shared by all deployments on this sidecar. */ + createWorkflowCache: (args: { rootDir: string; maxBytes: number }) => TCache; + /** + * Materialize a source-ref deployment's frozen closure. The booting + * process passes the real apply; a test passes a stub so the + * deploy/restore path can run without a live registry. + */ + applyFrozenWorkflowClosure: (args: { + readonly source: SourceRefPin["source"]; + readonly closure: SourceRefPin["closure"]; + readonly instanceDir: string; + readonly cache: TCache; + readonly registryMaxTarballBytes: number; + readonly registries: TRegistries; + readonly host: THost; + readonly loadDefinition: (loadArgs: { + readonly packageDir: string; + readonly importCacheKey?: string; + }) => Promise; + readonly assetRoot: string; + readonly assetMounts: ReadonlyMap; + readonly gitDirs: ReadonlyMap; + }) => Promise<{ + readonly definition: WorkflowDefinition; + readonly packageDir: string; + readonly deployDir: string; + }>; + /** Registry table the closure apply selects optional dependencies against. */ + readRegistries: () => TRegistries; + /** Platform pair the closure apply selects optional dependencies with. */ + resolveHostPlatform: () => THost; + /** + * Resolve a deployment's authored inbound-mail policy into the total + * decision map stored beside the mail-router registration. + */ + resolveInboundMailPolicy: ( + authored: InboundMailPolicy | undefined, + ) => ResolvedInboundMailPolicy; + /** + * Check a frame's inline source assets out into the durable stores the + * closure materializes from. + */ + materializeWorkflowAssets: (args: { + assets: readonly WorkflowSourceAssetMount[]; + closure: SourceRefPin["closure"]; + assetRoot: string; + gitDirRoot: string; + maxAssetPayloadBytes: number; + }) => Promise; + /** Cap on the total inline source-asset payload of one frame. */ + maxInlineAssetPayloadBytes: number; + /** Lossy address-to-repo-id substitution the workflow-run repo is keyed by. */ + deriveWorkflowRunRepoId: (agentAddress: string) => string; + /** + * Every step id in a frozen projection's flat step-id namespace. The + * credentials snapshot and the grants bridge both walk this set. + */ + inertFlatNamespaceStepIds: (args: { + definition: WorkflowProjectionDefinition; + context: string; + }) => readonly string[]; + /** Absolute git directory a source-format asset is indexed into. */ + sourceAssetGitDir: (gitDirRoot: string, assetId: string) => string; + /** Capacity settings resolved by the host at startup. */ + maxActiveDeployments: number; + maxRetainedDeployments: number; + /** + * Remove a deployment's workflow-run repository from the substrate, and with + * it the run record the repository's directory holds. Undeploy calls it + * last. Sync removed entries before deleting the record and confirm durable + * tree removal before resolving. + */ + removeRunRepository: (runId: string) => Promise; + /** + * Remove an agent-state repository from the substrate. Undeploy removes the + * one a single-step deployment keeps its grants in. Resolve only after the + * directory removal is durable; propagate sync failures. + */ + removeAgentStateRepository: (id: string) => Promise; + /** + * Send the Hub a report of a deployment that stopped though the Hub did not + * stop it, with the tips of its history branches. Tests that do not + * exercise the report omit it. + */ + publishDeploymentStopped?: ( + report: Omit, + ) => void; +}): SidecarDeployRouter { + // Validate the signing seed at construction so a malformed key fails + // sidecar boot rather than the first multi-step deploy, where the + // public key is derived from it (`derivePrincipalPublicKeyHex`). The + // seed also signs every workflow-run event via the supervisor. + if (deps.signingKeySeed.length !== 32) { + throw new Error( + `sidecar deploy router: Ed25519 signing seed must be 32 bytes, got ${deps.signingKeySeed.length}`, + ); + } + const publishInferenceEvent = + deps.publishWorkflowInferenceEvent ?? + (( + _address: string, + _generation: number, + _event: InferenceEvent, + _sessionId: string | undefined, + _childRunId?: string, + ): void => { + /* no-op default: tests and production-without-a-publisher + deployments do not consume events. */ + }); + const publishSuspension = + deps.publishWorkflowSuspension ?? + ((_registration: { + correlationId: string; + runId: string; + anchorRunId: string; + agentAddress: string; + generation: number; + kind: SignalKind; + approvalSnapshot?: ApprovalSnapshot; + }): void => { + /* no-op default: tests and production-without-a-publisher + deployments do not register suspensions. */ + }); + const multistepSubstrateEnv = deps.multistepSubstrateEnv ?? {}; + // Sidecar data dir the deployment's per-step scratch is rooted under + // (`/workflow-step-state//...`). Resolved once + // from the boot-edge substrate env so the undeploy hook can reclaim + // the whole subtree. Absent only when the router is wired without + // substrate config (a test that never spawns a child), in which case + // no child ever rooted scratch and the undeploy reclaim is correctly + // skipped. + const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; + if (deps.multistepBinaryPath.length === 0) { + throw new Error( + "sidecar deploy router: multistepBinaryPath must name the workflow child binary", + ); + } + const persistWorkflowRunSources = + deps.updateWorkflowRunRecordSources ?? updateWorkflowRunRecordSources; + const applyClosure = deps.applyFrozenWorkflowClosure; + const multistepSpawner = deps.multistepSubprocessSpawner; + const { maxActiveDeployments, maxRetainedDeployments } = deps; + const multistepDeriveStepAddress: DeriveStepAddress = + deps.multistepDeriveStepAddress ?? + (({ runId, stepId }) => `${runId}-${stepId}`); + + // Every incarnation this sidecar holds, by deployment address: what the + // link reports to the Hub and checks every Hub frame against. An entry + // exists from the moment a deploy or restore starts until its teardown + // finishes, synchronously before the first await, so a second deploy of an + // address cannot start while one is running and destructively overwrite the + // durable state the first one owns. `wired` is the supervisor that owns the + // workflow-process child, its IPC pipes, and its event-channel fd, once its + // spawn succeeded; the undeploy hook shuts it down so the child's lifetime + // ends with the deployment. + type HostedDeployment = { + generation: number; + state: IncarnationState; + retention?: DeploymentRetention; + retentionPending?: DeploymentRetention; + wired?: SidecarWorkflowSupervisor; + // Why a stopped incarnation stopped when the Hub did not stop it. + error?: string; + }; + const deployments = new Map(); + const retaining = new Set(); + + function retainedCount(): number { + let count = 0; + for (const held of deployments.values()) { + if (held.retention === "kept") count += 1; + } + return count; + } + + async function retainStoppedDeployment( + agentAddress: string, + hosted: HostedDeployment, + ): Promise { + const assertRetainable = () => { + if ( + deployments.get(agentAddress) !== hosted || + hosted.state === "tearing-down" + ) + throw new Error("The deployment is not available for retention"); + }; + assertRetainable(); + if (hosted.retention !== undefined) return; + if (stepStateDataDir === undefined) + throw new Error("Retaining a deployment requires its durable record"); + const previouslyPending = hosted.retentionPending; + const retention = + hosted.retentionPending ?? + (retainedCount() + retaining.size < maxRetainedDeployments + ? "kept" + : "refused"); + // Reserve the kept slot before persistence can yield to another address. + // The active slot remains held until the durable write succeeds. A failed + // sync may follow a successful rename, so reserve both until retry or removal. + hosted.retentionPending = retention; + if (retention === "kept") retaining.add(hosted); + const marked = await markWorkflowRunRecord( + stepStateDataDir, + deps.deriveWorkflowRunRepoId(agentAddress), + { + state: "stopped", + retention, + ...(hosted.error !== undefined ? { error: hosted.error } : {}), + }, + ); + if (!marked) { + // False means this attempt made no write. An earlier uncertain write + // still owns its reservation until persistence or removal is confirmed. + if (previouslyPending === undefined) { + delete hosted.retentionPending; + retaining.delete(hosted); + } + throw new Error("The deployment's retention record is missing"); + } + // Undeploy may have marked the copy tearing-down while this write awaited + // disk. Account for the persisted retention before it continues; the + // control outcome still refuses to acknowledge a copy being removed. + hosted.retention = retention; + delete hosted.retentionPending; + retaining.delete(hosted); + if (retention === "refused") { + logger.error`Retention refused for ${agentAddress}: this sidecar already keeps ${String(maxRetainedDeployments)} deployment records; the copy requires cleanup`; + } + } + + // Reading the tips also schedules the push of any commit the Hub has not + // acknowledged. The report does not wait for those pushes: the Hub waits for + // the history it lacks, for a bounded time. Tips that cannot be read leave + // the report without them rather than unsent, so the Hub still learns of + // the stop and records the history as possibly lost. + function reportStopped( + stopped: { + agentAddress: string; + generation: number; + error: string; + }, + isCurrent: () => boolean, + ): void { + if (!isCurrent()) return; + void deps + .reportDeploymentRefTips({ + agentAddress: stopped.agentAddress, + generation: stopped.generation, + }) + .catch((cause: unknown): undefined => { + if (!isCurrent()) return undefined; + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`Could not read the history tips of ${stopped.agentAddress}; reporting its stop without them: ${message}`; + return undefined; + }) + .then((refTips) => { + // Teardown may finish while the tips are read, releasing this address + // for another copy. Only the original stopped copy may report. + if (!isCurrent()) return; + deps.publishDeploymentStopped?.({ + ...stopped, + error: fitDeploymentError(stopped.error), + ...(refTips !== undefined ? { refTips } : {}), + }); + }) + .catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`Could not report that ${stopped.agentAddress} stopped: ${message}`; + }); + } + const workflowCancellationTasks = new WeakMap< + SidecarWorkflowSupervisor, + Promise + >(); + const workflowStopTasks = new Map>(); + + // Read after the supervisor has shut down, so the tips cover every run + // event and consumption record the stopped worker committed. Inbox + // enqueues, grants writes, and mailbox replays can still land later; none + // of them records an outcome or settles a delivery. + async function reportControlOutcome( + frame: WorkflowControlFrame, + ): Promise { + if (frame.action === "retain") { + const hosted = deployments.get(frame.agentAddress); + const retention = hosted?.retention; + if ( + retention === undefined || + hosted?.generation !== frame.generation || + hosted.state === "tearing-down" + ) { + throw new Error("Deployment retention was not confirmed"); + } + // Retain applies to a terminal run: the Hub has closed history admission. + // The acknowledgement proves this copy's durable capacity decision. + return { retention }; + } + return frame.action === "stop" + ? { + refTips: await deps.reportDeploymentRefTips({ + agentAddress: frame.agentAddress, + generation: frame.generation, + }), + } + : {}; + } + + // Slug-collision tracking. `deriveWorkflowRunRepoId` substitutes + // disallowed characters with `-`, which is deterministic but lossy: + // two distinct run addresses can collapse to the same slug, and + // a collision would let the second deploy silently overwrite the + // first deploy's workflow-run repo state (the slug IS the repoId). + // This map records the first-claimer; a subsequent deploy that + // produces the same slug from a different address is rejected at + // the router before any supervisor or repo state is touched. + const slugClaims = new Map(); + + function claimSlug(runId: string, agentAddress: string): void { + const existing = slugClaims.get(runId); + if (existing !== undefined && existing !== agentAddress) { + throw new Error( + `workflow-run repo id collision: run addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to runId ${JSON.stringify(runId)}`, + ); + } + // A same-address re-claim is a defensive no-op: the `deployments` + // guard rejects a live re-deploy before claimSlug is re-invoked, and a + // failed or undeployed deploy releases the slug first, so in practice + // `existing` is only ever undefined or a different address here. + slugClaims.set(runId, agentAddress); + } + + function releaseSlug(runId: string, agentAddress: string): void { + const existing = slugClaims.get(runId); + if (existing === agentAddress) slugClaims.delete(runId); + } + + function unregisterWorkflowRoutes(agentAddress: string): void { + deps.multistepMailRouter?.unregister(agentAddress); + deps.inboundMailPolicyRegistry?.unregister(agentAddress); + deps.multistepSignalRouter?.unregister(agentAddress); + deps.multistepDrainRouter?.unregister(agentAddress); + deps.multistepGrantsRouter?.unregister(agentAddress); + deps.multistepSourcesRouter?.unregister(agentAddress); + deps.multistepCredentialsRouter?.unregister(agentAddress); + } + + // Hold a deployment whose supervisor drove ITSELF to a terminal phase + // (crash-loop latch, channel crash, recycle failure) as stopped, with why, + // until the Hub undeploys it: the hello reports it, the Hub fails the + // deployment, and its local state stays for inspection meanwhile. Its record + // is marked too, so a restart reports it instead of spawning it again. The + // routers and the transport registration are released so a frame racing the + // stop is rejected at the boundary rather than dispatched into the dead + // supervisor. + // + // Two deliberate invariants: + // - This NEVER calls back into `wired.supervisor.*`. It runs as the + // supervisor's own `onSelfTerminate` sink, i.e. re-entrantly during the + // supervisor's terminal teardown; a call back in (e.g. `shutdown()`) + // would re-enter that teardown. The supervisor has already terminated, so + // there is nothing to shut down. + // - It does NOT call `unregisterDeployment`, so the deployment address + // mapping (`deploymentAddressRegistry`) is RETAINED. That mapping is + // consulted only by the OUTBOUND workflow-run pack push + // (`registry.resolve`), which the supervisor itself drives; no inbound + // frame consults it (a frame racing the stop is rejected by the routers + // above, not routed through this mapping). The crash-loop latch commits + // its `RunFailed` tombstone, resolving this mapping, before this sink + // fires. The undeploy that releases the incarnation clears it. + // + // Fully synchronous with no `await` between the guard and the mutations, so it + // is idempotent and cannot interleave with a concurrent operator `undeploy` of + // the same address; the record write runs after them. + function stopSelfTerminatedSupervisor(args: { + runId: string; + agentAddress: string; + hosted: HostedDeployment; + error: string; + }): void { + // Only the copy that terminated, and only once: an undeploy can land + // before a late sink fires, and the sink can fire twice. A copy the Hub + // is undeploying did not stop on its own, even when its child ends during + // the teardown. + if ( + deployments.get(args.agentAddress) !== args.hosted || + args.hosted.wired === undefined || + args.hosted.state === "tearing-down" + ) + return; + // Drop racing frames at the router boundary first, then unwind the + // underlying registrations -- the same ordering the undeploy hook uses. + unregisterWorkflowRoutes(args.agentAddress); + delete args.hosted.wired; + deps.transport.unregister(args.agentAddress); + args.hosted.state = "stopped"; + const error = fitDeploymentError(args.error); + args.hosted.error = error; + const report = (): void => { + reportStopped( + { + agentAddress: args.agentAddress, + generation: args.hosted.generation, + error, + }, + () => + deployments.get(args.agentAddress) === args.hosted && + args.hosted.state === "stopped" && + args.hosted.retention === undefined, + ); + }; + if (stepStateDataDir === undefined) { + report(); + return; + } + void markWorkflowRunRecord(stepStateDataDir, args.runId, { + state: "stopped", + error, + }) + .catch((cause: unknown) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`Could not record that ${args.agentAddress} stopped on its own; a restart would spawn it again: ${message}`; + }) + .finally(report); + } + + /** + * Remove the legacy `${dataDir}/assets/workflow//` staging a + * pre-sealed-record deploy wrote for a body (a world-readable plaintext + * `sources.json`, INTR-310). Body sources now ride sealed in the run record and + * reach the child through the spawn env, so the staging is retired; a redeploy + * of an address that predates the change reaches here and reclaims its orphaned + * file. A never-staged body is a no-op (`force`). + */ + async function sweepLegacyBodySources( + sidecarDataDir: string | undefined, + definitionId: string, + ): Promise { + if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the legacy body-source staging is reclaimed against this data dir", + ); + } + const bodyAssetDir = pathJoin( + sidecarDataDir, + "assets", + "workflow", + definitionId, + ); + await rm(bodyAssetDir, { recursive: true, force: true }); + } + + /** + * The per-deployment inputs the shared spawn core needs to stand up a + * workflow deployment, independent of the live deploy frame. The live + * deploy path builds this from `frame`/`projection`; a boot-time restore + * path builds the same shape from the persisted run record. + */ + interface WorkflowDeploySpec { + agentAddress: string; + /** The allocation generation of the incarnation being stood up. */ + generation: number; + /** + * The runnable definition, projected to its inert wire shape. Source-ref is + * the only deploy lineage, so this is always the closure evaluation + * (`projectLiveToInert(applied.definition)`), NOT a frame-carried inline + * definition (the deploy frame carries none). Both the deploy path and the + * boot-time restore derive it the same way, from the materialized closure. + */ + definition: WorkflowProjectionDefinition; + sources: NonNullable["sources"]; + /** + * Per spawned-body inference-source pins, keyed by the body's definition id + * (the flat set of onTrigger sections and childWorkflow children). Persisted + * in the record and delivered to the run child as plaintext through the spawn + * env, so the child resolves a body's sources without holding the sidecar's + * cipher key. `undefined` when the deployment spawns no bodies. + */ + bodySources: WorkflowRunRecord["bodySources"]; + /** + * The run's unified credential-material cell (inference + binding secrets, + * plus credential bindings). Persisted sealed in the record and delivered + * to the child on the pre-trigger barrier; the child resolves both its + * inference sources and its binding consumers from it by `credentialId`. + * `undefined` when the deployment binds no credentials and its sources need + * none. + */ + credentials: CredentialDelivery | undefined; + /** + * The hub-approved wire hash the deploy frame carried + * (`AgentDeployWorkflow.approvedWireHash`). The child's `DEFINITION_HASH` + * is sourced from this hub authority, NOT a sidecar recompute. Undefined + * only for a frame that carried no approved hash on the wire; the shared + * spawn core fails closed rather than substitute a recompute. + */ + approvedWireHash: string | undefined; + /** Correlates the child's inference events to the deploy's session. */ + sessionId: string | undefined; + /** + * Hub public key recorded at the head for deploy-pack verification and + * inbound hub-frame verification. Required for a single-step + * deployment (whose head IS the agent identity); undefined for a + * genuine multi-step deployment, which derives per-step addresses and + * records no head key. + */ + hubPublicKey: string | undefined; + /** + * Sidecar-local directory of the materialized workflow-definition closure, + * set after the frozen closure is applied. The spawn core threads it into + * the child's spawn env so the run child evaluates the pinned code to a live + * definition. Sidecar-local: it never travels on the hub deploy frame and is + * not persisted to the deployment record. Always present -- source-ref is + * the only deploy lineage. + */ + closurePackageDir: string; + /** + * The source-ref pin, carried so `buildWorkflowRunRecord` can persist it and + * a boot-time restore can re-run `applyFrozenWorkflowClosure` to + * re-materialize the pinned code. Its `source` carries no secret (the + * registry token is resolved from env at apply time); its `closure` is + * frozen versions + SRIs. + */ + sourceRef: NonNullable["sourceRef"]; + } + + /** + * Build the initial durable run record from the deployment specification. + */ + function buildWorkflowRunRecord(spec: WorkflowDeploySpec): WorkflowRunRecord { + // The record schema requires both for a source-ref record -- the only + // lineage -- so a spec missing either is a wiring defect. Fail loudly here + // rather than persist a record the boot scan would then reject as corrupt. + if (spec.approvedWireHash === undefined) { + throw new Error( + `buildWorkflowRunRecord: a source-ref deployment (${spec.agentAddress}) must carry approvedWireHash`, + ); + } + return { + version: 2 as const, + agentAddress: spec.agentAddress, + generation: spec.generation, + definitionId: spec.definition.id, + sources: spec.sources, + ...(spec.bodySources !== undefined + ? { bodySources: spec.bodySources } + : {}), + ...(spec.credentials !== undefined + ? { credentials: spec.credentials } + : {}), + ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), + ...(spec.hubPublicKey !== undefined + ? { hubPublicKey: spec.hubPublicKey } + : {}), + lineage: "source-ref", + // Feeds the restored child's DEFINITION_HASH so it re-verifies the + // evaluated closure against the hub-approved pin, and the source-ref pin a + // restore re-runs applyFrozenWorkflowClosure with. + approvedWireHash: spec.approvedWireHash, + sourceRef: spec.sourceRef, + }; + } + + // Deploys for different addresses apply closures concurrently, and the + // cache's in-use count protects an extraction from eviction only among + // callers of one instance, so every apply shares this one. + let sharedClosureCache: TCache | undefined; + + function closureCache(dataDir: string): TCache { + sharedClosureCache ??= deps.createWorkflowCache({ + rootDir: pathJoin(dataDir, "workflow-definition-closure-cache"), + maxBytes: requireSubstrateByteCap( + multistepSubstrateEnv, + "SIDECAR_CACHE_MAX_BYTES", + ), + }); + return sharedClosureCache; + } + + /** + * Materialize a source-ref deployment's frozen closure to its per-deployment + * instance dir and return the applied result. Owns the plumbing both the + * deploy path and the boot-time restore path share: the deterministic + * instance dir under `/workflow-definition-closures/`, + * the content-addressed cache root, the two substrate byte caps, and the + * registry table. The caller consumes the result: deploy validates + * `applied.definition` and takes `packageDir`; restore takes `packageDir` and + * re-carries the pin on its spec. + * + * The instance dir is force-reclaimed before the apply. `deploymentId` is + * deterministic per agent address, so a redeploy or a boot-restore reuses the + * same dir; a prior soft-failed deploy or a dead prior process can leave it + * half-materialized. The rm makes the apply write into a clean dir either + * way (a no-op on a never-deployed address). It is safe to reclaim only + * because no live reader holds the dir when this runs -- a precondition each + * caller establishes and notes at its call site. + */ + async function materializeDeploymentClosure( + dataDir: string, + deploymentId: string, + pin: SourceRefPin, + ) { + const instanceDir = pathJoin( + dataDir, + "workflow-definition-closures", + deploymentId, + ); + await rm(instanceDir, { recursive: true, force: true }); + + // Tarball `kind:"asset"` entries read from the durable plain-file store; a + // source-format entry checks its subtree out of the durable indexed git + // store. Deriving and asserting both from the pin alone is what makes this + // symmetric on deploy and restore. + const { assetRoot, assetMounts, gitDirs } = + await resolveDeploymentAssetMounts( + dataDir, + deploymentId, + pin, + deps.sourceAssetGitDir, + ); + + return applyClosure({ + source: pin.source, + closure: pin.closure, + instanceDir, + cache: closureCache(dataDir), + registryMaxTarballBytes: requireSubstrateByteCap( + multistepSubstrateEnv, + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + ), + registries: deps.readRegistries(), + host: deps.resolveHostPlatform(), + loadDefinition: loadWorkflowDefinitionFromClosure, + assetRoot, + assetMounts, + gitDirs, + }); + } + + /** + * The single owner of the workflow-deployment spawn sequence: construct + * the supervisor, register the single-step agent's outbound key + head + * repo + hub key, spawn the workflow-process child, then register the + * live deployment (supervisor, mail/signal/drain routers, address + * mapping). Its `try/finally` unwinds every piece of partial state if any + * step throws, so a failed spawn leaks nothing. Both the live deploy path + * and the boot-time restore path route through here so the two can never + * diverge on how a deployment is stood up. Callers materialize the + * deploy-only durable state (the source closure, step grants) before calling. + */ + async function spawnWorkflowRun( + spec: WorkflowDeploySpec, + hosted: HostedDeployment, + ): Promise { + // The run's credential material rides on `spec.credentials`: the sidecar now + // holds a cipher and seals it into the record, so both the live deploy and + // the boot-restore path carry it here (restore unseals it from the record), + // and it is delivered to the child on the pre-trigger barrier. + const credentialDelivery = spec.credentials; + // Fail loud unless the caller holds the address for this spawn. The deploy + // and boot restore paths both refuse an address already held before they + // get here, so this is a backstop: it fails before the lower-level + // `transport.register` duplicate throw and before the assignment below + // could clobber the running deployment's handle. + if ( + deployments.get(spec.agentAddress) !== hosted || + hosted.wired !== undefined + ) { + throw new Error( + `sidecar deploy router: ${spec.agentAddress} is not held for this spawn; refusing to spawn a second supervisor`, + ); + } + const runId = deps.deriveWorkflowRunRepoId(spec.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. A + // one-step deployment keeps the deployment's own (legacy) mail address + // and its grants in the legacy agent-state repo keyed by the legacy + // instance id. A multi-step deployment derives `-` + // per step for both the mail address and the agent-state repo id. + const stepStrategy = createStepStrategy({ + legacyAddress: spec.agentAddress, + stepOrder: spec.definition.stepOrder, + multistepDeriveStepAddress, + }); + + // Every step id the deployment's credentials snapshot must cover. This is + // NOT `stepOrder`: a `loop` body runs in-process as a child run inheriting + // the parent's env, so a body step authorizes against the SAME snapshot the + // top-level steps do, keyed by its own plain step id. A snapshot built from + // `stepOrder` alone carries no entry for it and the child's authorize + // throws on the body's first tool call. The address/repo strategy above + // stays on `stepOrder`, because the head/step collapse is a property of the + // deployment's own step count, not of what a body can run. + const credentialStepIds = deps.inertFlatNamespaceStepIds({ + definition: spec.definition, + context: "sidecar deploy router credentials snapshot: ", + }); + + // Unwind every piece of spawn state if any step in this block throws, + // so a failed spawn leaks no freshly-spawned workflow-process child, + // supervisor handle, transport registration, or multistep + // router registration. (The deployment-address registration happens + // before spawn and is unwound by its own guard.) The ordering inside + // the finally is the reverse of the success-path registration order. + // The caller owns the deployment slug: it must + // claim the collision guard before any durable write and release it on + // failure, so the slug is not touched here. + let succeeded = false; + let wiredForUnwind: SidecarWorkflowSupervisor | undefined; + let supervisorRegistered = false; + let routersRegistered = false; + let agentTransportRegistered = false; + let hubKeyRecorded = false; + let deploymentRegistered = false; + try { + // The child's `DEFINITION_HASH` is the HUB-APPROVED wire hash the deploy + // frame carried (`spec.approvedWireHash`) -- the hub is the authority, so + // the child re-verifies its own recompute against it. Both feeds into this + // core carry it: the production hub deploy builder always stamps it, and + // the boot restore re-attaches it from the persisted record. A missing + // hash here is therefore a wiring bug, not a legacy case to paper over: + // substituting a sidecar recompute would make the child re-verify against + // the sidecar's own hash rather than the hub authority -- a circular check + // that gives false assurance. Fail loud instead. + if (spec.approvedWireHash === undefined) { + throw new Error( + `workflow deploy spawn (${spec.agentAddress}): the deploy spec carries no approvedWireHash. The hub deploy builder must stamp the hub-approved wire hash and a restore must re-attach it from the persisted record; the sidecar will not recompute it, which would collapse the child's re-verify to a self-check.`, + ); + } + const definitionHash = spec.approvedWireHash; + + // Per-deployment substrate-config keys the substrate factory + // validator requires. The boot edge's `multistepSubstrateEnv` carries + // the boot-edge constants; the two workflow-run identity keys are + // derived per-deploy here. + // Spawned bodies' plaintext inference sources, serialized once for the + // deployment's lifetime (bodies do not live-rotate, unlike the top-level + // sources below). The child looks each body up by definition id and never + // holds the sidecar's cipher key. Guarded against the OS argument-string + // ceiling here, at deploy, so an over-large deployment fails loudly rather + // than at a later child `execve`. + const bodySourcesEnv = JSON.stringify(spec.bodySources ?? {}); + const bodySourcesBytes = Buffer.byteLength(bodySourcesEnv, "utf8"); + if (bodySourcesBytes > WORKFLOW_BODY_SOURCES_MAX_BYTES) { + throw new Error( + `sidecar deploy router: serialized ${WORKFLOW_BODY_SOURCES_ENV_KEY} is ${String(bodySourcesBytes)} bytes, over the ${String(WORKFLOW_BODY_SOURCES_MAX_BYTES)}-byte spawn-env limit; the deployment has too many spawned-body sources to deliver through the child env`, + ); + } + + const substrateEnv: Record = { + ...multistepSubstrateEnv, + WORKFLOW_RUN_REPO_ID: runId, + WORKFLOW_RUN_REF: "refs/heads/main", + // Thread the materialized closure's sidecar-local package dir so the + // run child EVALUATES the pinned code to a live definition and + // re-verifies by project-then-hash against `DEFINITION_HASH`. Source-ref + // is the only deploy lineage, so this is always present. Sidecar-local; + // carried on the frozen substrate env because the value is fixed for the + // deployment's lifetime. + CLOSURE_PACKAGE_DIR: spec.closurePackageDir, + [WORKFLOW_BODY_SOURCES_ENV_KEY]: bodySourcesEnv, + }; + // Live-rotatable per-step inference sources. Seeded from the deploy + // spec, then revised in place by the single-step sources-rotation + // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen + // `substrateEnv`: it is recomputed on every spawn and recycle respawn + // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of + // reverting to the deploy-time list. + let currentSources = spec.sources; + + // RunIds whose `run.grants` write was attempted and failed. The grants + // handler below records a runId here on a write failure; the grants + // barrier reads it through `isRunPoisoned` and fails the run rather than + // starting it under the empty deploy-time grant set. Scoped to this + // deployment's supervisor; a run that never had a `run.grants` frame is + // never added, so internal runs inherit the deployment's grants normally. + const poisonedRunIds = new Set(); + + const wired = createSidecarWorkflowSupervisor({ + transport: deps.transport, + repoStore: deps.repoStore, + signingKeySeed: deps.signingKeySeed, + workflowRunRepoId: { + kind: "workflow-run", + id: runId, + }, + workflowRunRef: "refs/heads/main", + runId, + stepCount: spec.definition.stepOrder.length, + stepOrder: credentialStepIds, + deploymentMailAddress: spec.agentAddress, + ...(credentialDelivery !== undefined ? { credentialDelivery } : {}), + deriveStepAddress: stepStrategy.deriveStepAddress, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + isRunPoisoned: (runId) => poisonedRunIds.has(runId), + // The supervisor stamps `runId` + `agentAddress` before + // invoking this; forward the fully-stamped registration to the + // hub-link-backed publisher so a `signal.correlation.register` frame + // reaches the hub for the parked run. + onSuspensionRegister: (registration) => { + publishSuspension({ ...registration, generation: spec.generation }); + }, + // Hold the deployment as stopped when the supervisor drives itself to + // a terminal phase (crash-loop latch, channel crash, recycle failure), + // so the Hub learns why it ended. + onSelfTerminate: ({ phase, reason }) => { + stopSelfTerminatedSupervisor({ + runId, + agentAddress: spec.agentAddress, + hosted, + error: `Its workflow child ended itself (${phase}): ${reason}`, + }); + }, + substrateEnv, + // Recomputed on every spawn AND recycle respawn. The rotation + // handler below revises `currentSources` in place, so a respawn + // re-serializes the current (possibly rotated) list rather than the + // frozen deploy-time value. + dynamicSpawnEnv: () => ({ + [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), + }), + subprocessSpawner: multistepSpawner, + binaryPath: deps.multistepBinaryPath, + ...(deps.onDispatchTiming !== undefined + ? { onDispatchTiming: deps.onDispatchTiming } + : {}), + ...(deps.repackEveryMessages !== undefined + ? { repackEveryMessages: deps.repackEveryMessages } + : {}), + ...(deps.consumedRetentionMs !== undefined + ? { consumedRetentionMs: deps.consumedRetentionMs } + : {}), + ...(deps.readyTimeoutMs !== undefined + ? { readyTimeoutMs: deps.readyTimeoutMs } + : {}), + }); + + // OUTBOUND half of mailbox ownership (§3a): register a signing key for + // the deployment mail address on the host transport so the supervisor + // signs the deployment's outbound mail. Every step -- single- or + // multi-step -- signs its outbound sends as `spec.agentAddress` (the + // one deployment mail address; no per-step sender reaches the host + // transport), so the transport MUST hold a `CryptoProvider` for it or + // `getTransportFor(senderAddress).send` throws "not registered". + // Registration happens before `spawn()` so the address is live the + // instant the first reply routes outbound. + const { keyPair } = await deps.keyStore.loadOrGenerateKey( + spec.agentAddress, + ); + deps.transport.register( + spec.agentAddress, + deps.createAgentCrypto(keyPair), + ); + agentTransportRegistered = true; + + // The deploy ack surfaces the deployment address's Ed25519 public key -- + // the key `loadOrGenerateKey` minted above and the workflow uses for + // signed content provenance. Every deployment acks it, single- and + // multi-step alike, so the Hub can publish the same identity. Reconnect + // ownership is established separately by the sidecar credential. + const deploymentPublicKey = hexEncode(keyPair.publicKey); + if (spec.definition.stepOrder.length === 1) { + // A single-step workflow stages its deploy tree at the head (the + // lone step IS the head). Initialize the head's on-disk deploy-tree + // repo (idempotent) so the hub's deploy-pack push has a repo to + // apply into. The narrow `initRepo` (not `provisionAgent`) is + // deliberate: the supervised child mints its own keypair and + // persists no hub-agent config. + await deps.sessions.initRepo(spec.agentAddress); + + // Record the hub's public key at the head so the deploy-pack apply + // (and any inbound hub-signed frame) verifies against it. The + // verifier resolves the key from the in-memory key store's + // `recordHubKey` map, so a single-step deployment cannot stand up + // without it. + if (spec.hubPublicKey === undefined) { + throw new Error( + "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", + ); + } + deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); + hubKeyRecorded = true; + } + + // Warm-keep is the single-step launched-agent deploy: the sole step + // IS the long-lived agent, so the child warm-keeps it across + // messages. A multi-step deploy keeps instantiate-send-teardown per + // step. The signal is carried explicitly down through the spawn env. + const warmKeep = spec.definition.stepOrder.length === 1; + const spawnOpts: SpawnOpts = { + stepOrder: [...credentialStepIds], + definitionHash, + warmKeep, + onInferenceEvent: (event, childRunId) => { + publishInferenceEvent( + spec.agentAddress, + spec.generation, + event, + spec.sessionId, + childRunId, + ); + }, + }; + + // Record the deployment-address mapping BEFORE `spawn`, because + // `spawn` kicks off `replayProcessingToInbox`, whose workflow-run + // substrate write routes through the boot-edge pack-pushing facade and + // resolves this mapping to address the outbound pack frame. Recording + // it after `spawn` (as the other registrations below are) loses the + // race: the replay's write throws "no run address registered" (a + // real defect masked as a swallowed best-effort warning in the + // supervisor's replay catch). Constraint ownership: the registry owns + // "address is resolvable"; the spawn path must satisfy that contract + // before the replay writes. The finally unwinds it on any failure + // between here and the end of the try. + deps.registerDeployment({ + runId, + agentAddress: spec.agentAddress, + generation: spec.generation, + }); + deploymentRegistered = true; + + // Surface spawn-time errors structurally: a subprocess spawner that + // crashes immediately rejects here, and the caller converts the + // rejection into a structured failure frame. The supervisor is + // registered against the deployment address only after spawn succeeds, + // so a spawn-time rejection leaves the registry untouched. + await wired.supervisor.spawn(spawnOpts); + wiredForUnwind = wired; + hosted.wired = wired; + supervisorRegistered = true; + + // Bind the deployment's mail address to this supervisor's + // `routeInbound` so the hub-link dispatches inbound mail into the + // supervisor's mail-bus subscription. Registration happens after + // `spawn` succeeds so a spawn-time rejection leaves the registry + // untouched. + deps.multistepMailRouter?.register(spec.agentAddress, (message) => + wired.routeInbound(message), + ); + // Resolve this deployment's authored inbound-mail policy into a total + // decision map ONCE and store it beside the mail-router registration, so + // the hub-link seam has the recipient's policy before any inbound frame + // for this address can route. + deps.inboundMailPolicyRegistry?.register( + spec.agentAddress, + deps.resolveInboundMailPolicy(spec.definition.inboundMailPolicy), + ); + // Register the signal-delivery handler so a hub `signal.deliver` frame + // dispatches through the supervisor's `deliverSignal`. + deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { + await wired.supervisor.deliverSignal({ + runId: args.runId, + signalName: args.signalName, + signalId: args.signalId, + payload: args.payload, + ...(args.correlationId !== undefined + ? { correlationId: args.correlationId } + : {}), + }); + }); + // Register the drain handler so a hub `drain.deliver` frame dispatches + // through the supervisor's `drain`. + deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { + await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); + }); + // Register the grants handler so a hub `run.grants` frame writes the + // run's grants to `runs//grants.json` in the deployment's + // workflow-run repo. The `runId` selects the per-run destination; the + // step-fan-out fields are inert in that mode but the shared write + // machinery still takes them. + deps.multistepGrantsRouter?.register(spec.agentAddress, async (args) => { + try { + // "Grant durable" implies "key durable": a run whose sender keys did + // not land must not start with a grant whose sender its recipient + // cannot verify. + if (!args.senderKeysCached) { + throw new Error( + `a sender key the grants of run ${args.runId} carry did not land in the sender-key cache`, + ); + } + await writeStepGrants({ + repoStore: deps.repoStore, + anchorRunId: runId, + stepOrder: spec.definition.stepOrder, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: args.stepGrants, + runId: args.runId, + }); + } catch (cause) { + // A sender-key or per-run grants write did not land. Poison the runId + // so the grants barrier fails the run instead of starting it under + // the deploy-time grant set, then re-throw so the hub-link logs the + // durable-write failure loudly. + poisonedRunIds.add(args.runId); + throw cause; + } + // The per-run file now carries the (possibly overlaid) floor. Refresh a + // live child so a standing ("always") approval resolved while the run + // is running but not parked -- with no signal to piggyback the + // `deliverSignal` refresh on -- lowers its floor immediately. + // Best-effort and non-fatal: the same durable file governs the next + // barrier/respawn, so a skipped push never leaves the run under a stale + // floor for good. + await wired.supervisor.deliverGrants(args.runId); + }); + // Register the sources-rotation handler ONLY for a single-step warm + // deployment: it has one long-lived agent whose sources can be + // swapped in place. A multi-step deployment has no single warm agent, + // so it registers no handler and `tryRoute` reports its address as + // unrouted. + if (warmKeep) { + // A single-step deployment's source table has exactly one entry, + // keyed by the head step. Derive that key once here (the layer that + // owns the single-key invariant); `deliverSources` stays flat and + // stepId-agnostic. + const rotationStepId = spec.definition.stepOrder[0]; + if (rotationStepId === undefined) { + throw new Error( + "single-step deploy has no step id for sources rotation", + ); + } + deps.multistepSourcesRouter?.register( + spec.agentAddress, + async (args) => { + const rotated = { [rotationStepId]: args.sources }; + // Swap `currentSources` synchronously BEFORE the durable persist. + // `currentSources` is the process-local respawn hint the + // supervisor reads synchronously through `dynamicSpawnEnv`, so a + // recycle that interleaves the persist `await` must respawn the + // child on the SAME sources being persisted, not the stale prior + // table. The obvious inverse -- persist first, then swap -- is + // rejected: it leaves the child on the OLD sources during the + // persist window while the record has already moved to NEW, so a + // recycle there respawns stale and a restart would "correct" it, + // i.e. the running child contradicts durable intent. Swapping + // first makes the only residual disagreement child-ahead-of- + // durable on a failed persist, which the next recycle heals down + // to the rolled-back durable truth -- the benign direction. The + // wire boundary guarantees `args.sources[0]` is the default, + // which the recycle env form pins as the active source. + const prevSources = currentSources; + currentSources = rotated; + // The durable write still precedes the LIVE swap + // (`deliverSources`), preserving persist-before-externally-visible + // for state that outlives the process; only the process-local + // respawn hint moves ahead. On a failed persist, roll the hint + // back so `currentSources` and the record stay in agreement in the + // common (no interleaved recycle) failure case -- the invariant + // restart consistency depends on. Persistence lets the rotation + // survive a full sidecar restart, not just a recycle: the boot + // scan reseeds spec.sources from record.sources. Updates only the + // sources, preserving a concurrent stop. Skipped when no data dir + // was wired (a test router that never persists), matching the + // restore guard. + if (stepStateDataDir !== undefined) { + try { + await persistWorkflowRunSources( + stepStateDataDir, + runId, + spec.generation, + rotated, + ); + } catch (cause) { + // Restoring unconditionally is safe because rotations for one + // deployment are serialized by the link's per-address frame + // lane: each hub frame for the deployment's address, + // sources.update included, runs its handler to completion + // before the next one for that address starts, so no second + // rotation is in flight whose committed table this rollback + // could clobber. This does NOT rely on the hub pacing its + // sends -- the hub dispatches sources.update fire-and-forget; + // the frame lane is the sole serializer. Handling one + // address's frames concurrently would break this rollback. + currentSources = prevSources; + throw cause; + } + } + await wired.supervisor.deliverSources({ + sources: args.sources, + defaultSource: args.defaultSource, + }); + }, + ); + } + + // Register the credential-delivery handler for EVERY deployment (not only + // warm single-step ones): the material cell is per-child and read by + // every step's tool capabilities. The handler hands the delivery to the + // supervisor's `deliverCredentials`, which sends a `credentials-updated` + // control frame to the child where the material cell is swapped. No + // durable persist -- credential material never touches disk. + deps.multistepCredentialsRouter?.register( + spec.agentAddress, + async (args) => { + await wired.supervisor.deliverCredentials({ + delivery: args.delivery, + ...(args.revoke !== undefined ? { revoke: args.revoke } : {}), + }); + }, + ); + routersRegistered = true; + + succeeded = true; + return { publicKey: deploymentPublicKey }; + } finally { + if (!succeeded) { + // Unwind in reverse registration order so each step undoes state + // the success path confirmed; ordering matches the `undeploy` hook. + if (routersRegistered) { + deps.multistepMailRouter?.unregister(spec.agentAddress); + deps.inboundMailPolicyRegistry?.unregister(spec.agentAddress); + deps.multistepSignalRouter?.unregister(spec.agentAddress); + deps.multistepDrainRouter?.unregister(spec.agentAddress); + deps.multistepGrantsRouter?.unregister(spec.agentAddress); + // Unregister unconditionally: the sources handler was registered + // only for a single-step deploy, but `unregister` is a no-op for + // an address that never registered one, so a multi-step unwind + // safely calls it too. + deps.multistepSourcesRouter?.unregister(spec.agentAddress); + deps.multistepCredentialsRouter?.unregister(spec.agentAddress); + } + if (supervisorRegistered) { + delete hosted.wired; + } + if (wiredForUnwind !== undefined) { + await wiredForUnwind.supervisor.shutdown().catch((cause) => { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`multi-step deploy unwind: supervisor.shutdown failed: ${message}`; + }); + } + if (agentTransportRegistered) { + // Drop the agent's transport registration so a failed deploy does + // not leave the address live with a dangling `CryptoProvider`. + deps.transport.unregister(spec.agentAddress); + } + if (hubKeyRecorded) { + // Reverse the single-step head's `recordHubKey` so a failed deploy + // leaves no in-memory hub key behind. `forgetAgent` also drops the + // agent keypair cache `loadOrGenerateKey` populated, which is safe: + // the transport registration is already unwound above and nothing + // reads that cache after unwind. The on-disk deploy-tree repo + // `initRepo` created, with the keypair under it in `keys/`, is left + // in place: the Hub releases a deployment whose deploy failed, and + // the release's undeploy deletes the agent directory. + deps.keyStore.forgetAgent(spec.agentAddress); + } + if (deploymentRegistered) { + // Reverse the pre-spawn `registerDeployment`: drop the address + // mapping so a failed spawn leaves the boot-edge registry as it + // found it. Registered first (before spawn), unwound last. A + // subsequent stale workflow-run write for the dead deployment then + // surfaces structurally (`registry.resolve` returns null) rather + // than resolving to the address of a deployment that never came up. + deps.unregisterDeployment({ + runId, + agentAddress: spec.agentAddress, + }); + } + } + } + } + + /** + * Provision one step of a multi-step deploy WITHOUT spawning. The hub + * stages each step's deploy tree before firing the deployment-level + * workflow frame; a full-closure deploy pack still needs an initialized + * agent-state repo to apply into and the hub key recorded to verify the + * pack commit signature. This does exactly those two things -- the same + * harness-free `initRepo` + `recordHubKey` seam the single-step head uses + * -- and constructs no supervisor or child. The deployment-level workflow + * frame (fired once after every step is provisioned) spawns the child, + * which reads each step's staged deploy tree from disk. + * + * Returns the sidecar's principal public key so the link's + * `agent.deploy.ack` carries a key, matching the multi-step ack. A + * per-step address is workflow-derived and records no `agent_instance` + * key, so the hub discards this value. + */ + async function provisionStep( + frame: AgentDeployFrame, + ): Promise { + const held = deployments.get(frame.agentAddress); + if (held !== undefined) { + throw new Error( + `sidecar deploy router: ${frame.agentAddress} generation ${String(held.generation)} is ${held.state} here; a step cannot be provisioned over it`, + ); + } + await deps.sessions.initRepo(frame.agentAddress); + deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); + return { + publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), + }; + } + + async function deployMultiStep( + frame: AgentDeployFrame, + projection: NonNullable, + ): Promise { + // Refuse a deploy of an address already held BEFORE touching any durable + // state. The durable writes below (the run record, the materialized + // closure, step grants) are destructive overwrites of state owned by + // whatever deployment holds the address, and an address is deployed once. + const held = deployments.get(frame.agentAddress); + if (held !== undefined) { + throw new Error( + `sidecar deploy router: ${frame.agentAddress} generation ${String(held.generation)} is ${held.state} here; generation ${String(frame.generation)} cannot deploy over it`, + ); + } + if (deployments.size - retainedCount() >= maxActiveDeployments) { + throw new DeploymentRejectedError( + "capacity_full", + `sidecar deploy router: this sidecar has no active deployment slot for ${frame.agentAddress}`, + ); + } + + const runId = deps.deriveWorkflowRunRepoId(frame.agentAddress); + + // Resolve the sidecar data dir once: the run record, the materialized + // closure, and the per-step scratch all root under it. Required for any + // deployment that spawns a child. + const dataDir = stepStateDataDir; + if (typeof dataDir !== "string" || dataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the run record and workflow-process child root under it", + ); + } + + // Claim the deployment slug BEFORE any durable write so a colliding runId + // (two distinct addresses projecting to the same slug) is rejected before + // the closure, the step grants, or the supervisor touch disk -- the + // router's "no repo state touched before rejection" guarantee. The claim is + // released after failed-deploy record cleanup succeeds; otherwise it stays + // held until undeploy completes. The spawn core owns unwinding the + // supervisor and registrations it stands up; the slug is the caller's. + claimSlug(runId, frame.agentAddress); + // Hold the address from here until the deploy settles. Everything above is + // synchronous and throws before any durable write, so the entry is only + // needed from the first await here onward, and nothing above can yield + // control to a concurrent frame before it is set. + const hosted: HostedDeployment = { + generation: frame.generation, + state: "deploying", + }; + deployments.set(frame.agentAddress, hosted); + try { + // Source-ref apply -- the only deploy lineage. Materialize EXACTLY the + // hub's frozen dependency `closure` and evaluate the PINNED CODE to the + // workflow definition; the deploy frame carries no inline definition to + // trust. The closure is applied byte-for-byte (concrete versions + + // integrity SRIs); the sidecar never re-resolves the pin at apply time. + // The re-evaluated projection IS the runnable definition, and the child's + // load-boundary re-verify recomputes the wire hash over the closure and + // fails closed if it diverges from the hub-approved hash, so a closure + // that no longer projects to the approved content cannot deploy. + // + // Check the frame's inline source assets out into the durable + // per-deployment store the closure materializes from. Reclaim the store + // first so a redeploy drops assets no longer referenced. This runs only on + // the DEPLOY path -- restore re-reads the store the deploy persisted, with + // no re-delivery -- so the checkout lives here, not in + // `materializeDeploymentClosure` (which also runs on restore). A + // registry-sourced pin delivers no assets and only clears the store. + const assetStore = deploymentSourceAssetRoot(dataDir, runId); + const gitStore = deploymentSourceGitRoot(dataDir, runId); + await rm(assetStore, { recursive: true, force: true }); + await rm(gitStore, { recursive: true, force: true }); + if (projection.assets !== undefined && projection.assets.length > 0) { + await deps.materializeWorkflowAssets({ + assets: projection.assets, + closure: projection.sourceRef.closure, + assetRoot: assetStore, + gitDirRoot: gitStore, + maxAssetPayloadBytes: deps.maxInlineAssetPayloadBytes, + }); + } + // Safe to reclaim the instance dir inside the helper: the `deploying` + // entry set above refuses a concurrent deploy of the address, and the + // child is not yet spawned, so no live reader holds it. + const applied = await materializeDeploymentClosure( + dataDir, + runId, + projection.sourceRef, + ); + const validatedDefinition = WorkflowProjectionDefinition( + projectLiveToInert(applied.definition), + ); + if (validatedDefinition instanceof type.errors) { + throw new Error( + `sidecar deploy router: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`, + ); + } + const effectiveDefinition = validatedDefinition; + + // Structural invariants the wire arktype does not cover (non-empty + // stepOrder, every stepOrder entry backed by a `steps` entry AND a + // `sources` entry), checked against the closure-derived definition -- the + // frame carries none to cover. Mirrors the restore path. + validateWorkflowProjection({ + definition: effectiveDefinition, + sources: projection.sources, + }); + + // Source-admission gate: reject a deploy where any step pins an inference + // provider this sidecar cannot build. Every source in a step's failover + // chain must be buildable -- a chain with an unbuildable tail would fail + // only after the reactor failed over onto it -- so this iterates the whole + // list. The throw propagates back through the deploy frame so the hub's + // `deployWorkflow` rejects synchronously at deploy time. + for (const stepId of effectiveDefinition.stepOrder) { + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + // Single-agent deploy vs. derived multi-step deploy. A one-step + // definition keeps the deploy's own mail address and its grants in the + // agent-state repo keyed by the run id; a multi-step definition derives + // `-` per step for both the mail address and the + // agent-state repo id, isolating each step's grants in its own repo. + const stepStrategy = createStepStrategy({ + legacyAddress: frame.agentAddress, + stepOrder: effectiveDefinition.stepOrder, + multistepDeriveStepAddress, + }); + + // The spec the shared spawn core consumes, and the durable record that + // lets a boot-time restore rebuild the SAME spec (definition re-evaluated + // from the pinned closure, grants from the step repos, and the record's + // frame/in-memory-only inputs: sources, session id, single-step hub key). + const spec: WorkflowDeploySpec = { + agentAddress: frame.agentAddress, + generation: frame.generation, + definition: effectiveDefinition, + sources: projection.sources, + bodySources: buildBodySourcesMap(projection.referencedDefinitions), + // The hub's unified credential-material cell (inference + binding + // secrets); persisted sealed in the record and delivered on the + // barrier. + credentials: projection.credentials, + approvedWireHash: projection.approvedWireHash, + sessionId: frame.config.sessionId, + hubPublicKey: + effectiveDefinition.stepOrder.length === 1 + ? frame.hubPublicKey + : undefined, + // The sidecar-local dir of the just-materialized closure the spawn core + // threads into the child's env so it re-evaluates the pinned code. + closurePackageDir: applied.packageDir, + // The source-ref pin the record persists so a restore can + // re-materialize the closure. + sourceRef: projection.sourceRef, + }; + const record = buildWorkflowRunRecord(spec); + + // Persist the run record BEFORE the spawn so a crash mid-spawn leaves a + // record the boot scan re-drives (an idempotent re-spawn; the child's + // in-flight-run discovery resumes any run). A soft-failed deploy removes + // it below; failed removal keeps the record and its capacity reservation. + await writeWorkflowRunRecord( + dataDir, + runId, + record, + deps.credentialCipher, + ); + + // Sweep any legacy plaintext body-source file this deployment's bodies + // left behind. Body sources now ride sealed in the run record and reach + // the child through the spawn env, so the `assets/workflow//` + // staging is retired. A deployment first seen on this build never wrote + // one (the rm is a no-op); a redeploy across the upgrade reaches here with + // its old world-readable plaintext file still on disk and removes it now + // that the record carries the sealed copy. + for (const referenced of projection.referencedDefinitions ?? []) { + await sweepLegacyBodySources(dataDir, referenced.definition.id); + } + + // Grants bridge: the spawned child does not see the frame; it reads + // each step's grants out of `state/grants.json` in the step's + // agent-state repo while the supervisor assembles the + // credentialsSnapshot. Write the operator-approved + // `frame.config.grants` to the same repo the supervisor reads via + // `deriveStepRepoId`, before the spawn core, so the read sees them. + // + // The write covers the deployment's whole flat step-id namespace, loop + // body steps included: the supervisor assembles a snapshot entry for each + // of them, and an entry read out of a repo that was never written carries + // an empty grant set, which denies the body every resource the deploy + // approved. + await writeStepGrants({ + repoStore: deps.repoStore, + anchorRunId: runId, + stepOrder: deps.inertFlatNamespaceStepIds({ + definition: effectiveDefinition, + context: "sidecar deploy router grants bridge: ", + }), + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: frame.config.grants, + }); + + // Hand off to the shared spawn core. + const result = await spawnWorkflowRun(spec, hosted); + hosted.state = "live"; + return result; + } catch (cause) { + hosted.state = "stopped"; + hosted.error = fitDeploymentError( + cause instanceof Error ? cause.message : String(cause), + ); + try { + await markWorkflowRunRecord(dataDir, runId, { + state: "stopped", + error: hosted.error, + }); + } catch (markError) { + const message = + markError instanceof Error ? markError.message : String(markError); + logger.error`Could not mark failed deployment ${frame.agentAddress} stopped; attempting removal: ${message}`; + } + try { + await deleteWorkflowRunRecord(dataDir, runId); + if (deployments.get(frame.agentAddress) === hosted) + deployments.delete(frame.agentAddress); + releaseSlug(runId, frame.agentAddress); + } catch (cleanupError) { + const message = + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError); + logger.error`Failed deployment ${frame.agentAddress} still occupies capacity: ${message}`; + } + throw cause; + } + } + + async function control( + frame: WorkflowControlFrame, + ): Promise { + if (deriveWorkflowRunId(frame.agentAddress) !== frame.runId) { + throw new Error( + "Workflow control run does not match the deployment address", + ); + } + // A command is for the incarnation it names: another generation held + // here is not its to cancel or stop. + const hosted = deployments.get(frame.agentAddress); + if (hosted !== undefined && hosted.generation !== frame.generation) { + throw new Error( + `${frame.agentAddress} generation ${String(hosted.generation)} is hosted here, not generation ${String(frame.generation)}`, + ); + } + const stopping = workflowStopTasks.get(frame.agentAddress); + if (hosted?.state === "tearing-down") + throw new Error("The deployment is being removed"); + if (stopping !== undefined) { + await stopping; + if (frame.action === "retain" && hosted?.retention === undefined) + return control(frame); + return reportControlOutcome(frame); + } + if (hosted?.state === "deploying") { + throw new Error(WORKFLOW_CONTROL_INITIALIZING_ERROR); + } + if (frame.action === "retain") { + if (hosted === undefined) + throw new Error("The deployment is not available for retention"); + if (hosted.retention !== undefined) return reportControlOutcome(frame); + } + const wired = hosted?.wired; + if (frame.action === "cancel" && wired !== undefined) { + const cancelling = workflowCancellationTasks.get(wired); + if (cancelling !== undefined) { + await cancelling; + return {}; + } + const pending = wired.supervisor + .requestCancel({ + runId: frame.runId, + origin: "supervisor-operator", + reason: frame.reason, + at: new Date().toISOString(), + }) + .then(() => undefined); + // The Hub resends cancel on every sweep until the run is terminal. + // Keep a committed cancellation so a resend reuses it instead of + // signing another CancelRequested; only a failed one is retried. + workflowCancellationTasks.set(wired, pending); + try { + await pending; + } catch (error) { + workflowCancellationTasks.delete(wired); + throw error; + } + return {}; + } + // Cancellation without a supervisor must still mark its restart record + // stopped. + // Publish stop before its first await. It fences new commands and deploys + // while shutdown releases the cancellation handshake and kills the child. + const pending = Promise.resolve().then(async () => { + unregisterWorkflowRoutes(frame.agentAddress); + if (wired !== undefined) { + try { + await wired.supervisor.shutdown(); + } finally { + await wired.closeOutbound(); + } + } + // The incarnation stays held, stopped, and keeps its scratch and + // source material for the allocation's retention period, until the + // Hub undeploys it. + if ( + hosted !== undefined && + deployments.get(frame.agentAddress) === hosted + ) { + if (hosted.wired !== undefined) { + delete hosted.wired; + deps.transport.unregister(frame.agentAddress); + } + if (hosted.state === "live") hosted.state = "stopped"; + } + // A retained marker is also a durable stop mark. Its write must finish + // before the acknowledgement permits the Hub to reuse the active slot. + if (frame.action === "retain" && hosted !== undefined) { + await retainStoppedDeployment(frame.agentAddress, hosted); + } else if (stepStateDataDir !== undefined) { + await markWorkflowRunRecord( + stepStateDataDir, + deps.deriveWorkflowRunRepoId(frame.agentAddress), + { + state: "stopped", + ...(hosted?.error !== undefined ? { error: hosted.error } : {}), + }, + ); + } + }); + workflowStopTasks.set(frame.agentAddress, pending); + try { + await pending; + } finally { + if (workflowStopTasks.get(frame.agentAddress) === pending) + workflowStopTasks.delete(frame.agentAddress); + } + return reportControlOutcome(frame); + } + + return { + async deploy(frame): Promise { + if (workflowStopTasks.has(frame.agentAddress)) { + throw new Error("Workflow deployment is still stopping"); + } + if (frame.provisionStep === true) { + return await provisionStep(frame); + } + if (frame.workflow !== undefined) { + return await deployMultiStep(frame, frame.workflow); + } + // Every deploy stages through the workflow-run substrate: a + // provision-step frame primes the per-step repo, and a workflow + // frame spawns the supervised child. A frame carrying neither is + // an unsupported shape -- there is no in-process fall-through. + throw new Error( + `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, + ); + }, + control, + async undeploy(frame): Promise { + // A newer incarnation of the address is not this frame's to remove. + const hosted = deployments.get(frame.agentAddress); + if (hosted !== undefined && hosted.generation > frame.generation) return; + // Own teardown before waiting, so a control waiting on the same stop + // cannot start another stop task and overwrite the teardown record. + if (hosted !== undefined) hosted.state = "tearing-down"; + for ( + let stopping = workflowStopTasks.get(frame.agentAddress); + stopping !== undefined; + stopping = workflowStopTasks.get(frame.agentAddress) + ) { + try { + await stopping; + } catch (cause) { + logger.warn`Continuing undeploy of ${frame.agentAddress} after a failed stop: ${cause instanceof Error ? cause.message : String(cause)}`; + } + } + // Symmetric teardown for `deploy`: release the per-deployment + // routing state both branches install so a stale `signal.deliver` + // / `drain.deliver` / `mail.inbound` aimed at the dead deployment + // address is rejected by the router rather than dispatched into + // an orphan supervisor handler. The unregister calls are + // idempotent -- they are no-ops when no handler is registered. + // + // Routers come down BEFORE the supervisor's `shutdown()` so any + // hub-side frame racing the undeploy is dropped at the router + // boundary rather than dispatched into a supervisor that is in + // the middle of tearing its child down. The pattern is: drop + // racing frames first, then unwind the underlying resource. + const runId = deps.deriveWorkflowRunRepoId(frame.agentAddress); + unregisterWorkflowRoutes(frame.agentAddress); + // Shut the per-deployment supervisor down so the workflow-process + // child, its IPC pipes, and its event-channel fd are released. + // The supervisor's `shutdown()` is idempotent (returns early when + // the supervisor is already in `idle`/`stopped`) and handles the + // kill + `exited` await internally. The incarnation stays held, as + // tearing down, until the teardown settles: the link keeps reporting it + // and stamps what it still sends with its generation. + const wired = hosted?.wired; + const dataDir = stepStateDataDir; + // A restorable record must be durably marked tearing down before its + // state is deleted. If marking fails, stop the child but keep its state + // for retry. Once marked, attempt every cleanup step and remove the + // record with the run repository only after the other steps succeed, + // so an interrupted cleanup stays reportable across a restart. + const failures: Error[] = []; + const attempt = async ( + step: string, + run: () => Promise, + ): Promise => { + try { + await run(); + return true; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + failures.push(new Error(`${step} failed: ${msg}`, { cause: err })); + return false; + } + }; + // A step below that throws still leaves the deployment without its + // address, so nothing may keep sending as this one. + try { + const canDelete = + dataDir === undefined || + (await attempt("marking its record", async () => { + await markWorkflowRunRecord(dataDir, runId, { + state: "tearing-down", + }); + })); + if (wired !== undefined) { + await attempt("shutting its supervisor down", async () => { + try { + await wired.supervisor.shutdown(); + } finally { + await wired.closeOutbound(); + } + }); + } + if (canDelete) { + await attempt("deleting its agent directory", () => + deps.sessions.deleteAgentDir(frame.agentAddress), + ); + // Reclaim the deployment's local state: its per-step scratch (the warm + // agent's workspace and any cold run subtrees), its materialized + // closure, its source stores, its local conversation copy and a + // single-step deployment's grants, which only that deployment's + // respawns and restarts read. Cleanup also runs without a live + // supervisor, so state an interrupted deploy or a failed restore + // left is reclaimed too. No child holds any of it: a live one was + // shut down above. + if (dataDir !== undefined) { + const localState: [string, string][] = [ + [ + "removing its step scratch", + pathJoin(dataDir, "workflow-step-state", runId), + ], + [ + "removing its closure", + pathJoin(dataDir, "workflow-definition-closures", runId), + ], + [ + "removing its source assets", + deploymentSourceAssetRoot(dataDir, runId), + ], + [ + "removing its source repository", + deploymentSourceGitRoot(dataDir, runId), + ], + [ + "removing its conversation copy", + conversationStateRoot(dataDir, runId), + ], + ]; + for (const [step, dir] of localState) { + await attempt(step, () => removeTree(dir)); + } + // A single-step deployment's grants live in the agent-state + // repository of its own run address. + const run = parseRunAddress(frame.agentAddress); + if (run !== null) { + await attempt("removing its grants repository", () => + deps.removeAgentStateRepository(run.runId), + ); + } + // A teardown that did not finish keeps them, so a restart still + // reports the incarnation and the next undeploy finishes it. + if (failures.length === 0) { + await attempt("removing its run repository and record", () => + withWorkflowRunRecord(dataDir, runId, () => + deps.removeRunRepository(runId), + ), + ); + } + } + if (failures.length === 0) releaseSlug(runId, frame.agentAddress); + } + } finally { + // Drop the deployment address's transport registration installed at + // spawn (OUTBOUND half of mailbox ownership, §3a). Both single- and + // multi-step register the deployment address for outbound signing, so + // this tears down a real registration for either; `unregister` is a + // no-op only if the spawn failed before registering, so it is safe to + // call unconditionally for any spawned deployment. + if (wired !== undefined) deps.transport.unregister(frame.agentAddress); + deps.unregisterDeployment({ + runId, + agentAddress: frame.agentAddress, + }); + const held = deployments.get(frame.agentAddress); + if (failures.length > 0) { + // A held incarnation stays held without its supervisor, so the hello + // keeps reporting it as tearing down until cleanup succeeds. + if (hosted !== undefined && held === hosted) delete hosted.wired; + } else { + if (hosted !== undefined && held === hosted) { + deployments.delete(frame.agentAddress); + retaining.delete(hosted); + } + } + } + if (failures.length > 0) { + throw new AggregateError( + failures, + failures.map((failure) => failure.message).join("; "), + ); + } + }, + async restoreWorkflowRuns(): Promise { + const dataDir = stepStateDataDir; + if (dataDir === undefined) { + // No substrate config was wired (a test router that never spawns a + // child): nothing was ever persisted under this data dir, so there + // is nothing to restore. + return; + } + + const scanned = await scanWorkflowRunRecords( + dataDir, + deps.credentialCipher, + ); + // Validate directory ownership before counting, so skipped records do + // not consume capacity that the runtime never reserves. + const inventory = scanned.filter(({ runId, record }) => { + const derived = deps.deriveWorkflowRunRepoId(record.agentAddress); + if (derived === runId) return true; + logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${runId}`; + return false; + }); + const kept = inventory.filter( + ({ record }) => record.retention === "kept", + ).length; + const active = inventory.length - kept; + if (kept > maxRetainedDeployments || active > maxActiveDeployments) { + throw new Error( + `Stored deployments exceed the sidecar capacity limits: ${String(active)} active records (limit ${String(maxActiveDeployments)}), ${String(kept)} kept records (limit ${String(maxRetainedDeployments)})`, + ); + } + // Inventory includes both pools. Only active records without a stopped + // mark resume execution. The spawns run several at once: the sidecar + // connects only once every deployment is restored, and the Hub fails the + // deployments of a sidecar that stays away too long. Restore runs before + // `hubLink.connect()`, so no deploy contends with it. Once inventory + // validation succeeds, each spawn's failure is caught so one bad + // deployment cannot strand the rest. + const spawns: (() => Promise)[] = []; + for (const { runId, record } of inventory) { + // An address already held keeps its slug and its hold: freeing either + // here would strand the running deployment's guards. + if (deployments.has(record.agentAddress)) { + logger.warn`sidecar deploy router: ${record.agentAddress} is already held; refusing to restore it again`; + continue; + } + // The slug is the caller's, matching `deployMultiStep`. Restore does NOT + // re-materialize the step grants or the onTrigger body sources -- both + // are already on disk from the original deploy. + claimSlug(runId, record.agentAddress); + // A deployment that no longer ran is held as it was, unspawned, until + // the Hub undeploys it. + if (record.state !== undefined) { + deployments.set(record.agentAddress, { + generation: record.generation, + state: record.state, + ...(record.retention !== undefined + ? { retention: record.retention } + : {}), + ...(record.error !== undefined ? { error: record.error } : {}), + }); + continue; + } + const hosted: HostedDeployment = { + generation: record.generation, + state: "deploying", + }; + deployments.set(record.agentAddress, hosted); + spawns.push(async () => { + try { + // Reconstruct this deployment's runnable definition. Source-ref is + // the only lineage: re-materialize the pinned closure and evaluate + // the pinned code to the live definition, then project it to the + // inert wire shape -- the SAME computation the deploy path applies + // (`WorkflowProjectionDefinition(projectLiveToInert(...))`). The + // closure IS the source of truth; no on-disk definition is read. + // The helper reclaims the instance dir first, which is safe here + // because the prior process (the only reader) is dead, each + // deployment has its own dir, and restore runs before + // `hubLink.connect()`, so no deploy reads it. Registry-sourced + // entries come from the content-addressed closure cache, or from + // their registry when the cache, which every deployment on this + // sidecar shares, has evicted them; asset-sourced entries read + // from the durable source store the original deploy checked out + // (`materializeDeploymentClosure` derives the mounts from the pin, + // so no re-delivery is needed). Both are SRI-verified. A failed + // registry fetch or store read fails the restore like any check + // below. The schema guarantees a source-ref record carries a + // `sourceRef` pin, so no undefined-check is needed. + const applied = await materializeDeploymentClosure( + dataDir, + runId, + record.sourceRef, + ); + const validatedDefinition = WorkflowProjectionDefinition( + projectLiveToInert(applied.definition), + ); + if (validatedDefinition instanceof type.errors) { + throw new Error( + `workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`, + ); + } + const definition: WorkflowProjectionDefinition = + validatedDefinition; + const closurePackageDir = applied.packageDir; + + // Structural invariants the wire arktype does not cover (non-empty + // stepOrder, every stepOrder entry backed by a `steps` entry AND a + // `sources` entry). The closure eval skips the deploy frame's + // coverage narrow, so this is where its definition-vs-sources + // coverage is checked. + validateWorkflowProjection({ definition, sources: record.sources }); + + // Re-run the source-admission gate: refuse to restore a deployment + // whose pinned provider this sidecar can no longer build. Every + // source in a step's failover chain must be buildable, so this + // iterates the whole list. + for (const stepId of definition.stepOrder) { + const chain = record.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + const spec: WorkflowDeploySpec = { + agentAddress: record.agentAddress, + generation: record.generation, + definition, + sources: record.sources, + // The record's unsealed body sources, re-delivered to the run + // child through the spawn env. `undefined` for a legacy record + // written before body sources were persisted here; the child then + // falls back to that deployment's on-disk plaintext file for the + // missing bodies. + bodySources: record.bodySources, + // The record's credential-material cell, unsealed by the boot + // scan and re-delivered to the child on the pre-trigger barrier + // so an offline restart restores both inference and tool + // credentials without the hub. `undefined` when the deployment + // bound none. + credentials: record.credentials, + // The hub-approved wire hash the original deploy persisted, so + // the restore re-spawn carries the same `DEFINITION_HASH` rather + // than a recompute. Always present -- the record schema requires + // it. + approvedWireHash: record.approvedWireHash, + sessionId: record.sessionId, + hubPublicKey: record.hubPublicKey, + // The sidecar-local dir of the just-materialized closure the + // spawn core threads into the child's env so it re-evaluates the + // pinned code. + closurePackageDir, + // Carry the source-ref pin so a post-restore source rotation -- + // which rebuilds the record from the spec -- re-persists it; + // without this a rotation would silently drop it and wedge the + // NEXT restart. + sourceRef: record.sourceRef, + }; + + await spawnWorkflowRun(spec, hosted); + hosted.state = "live"; + logger.info`Restored workflow deployment for ${record.agentAddress} generation ${String(record.generation)}`; + } catch (cause) { + // Held as stopped, with why, until the Hub fails and undeploys it, + // and recorded so the next boot does not try again. + const reason = + cause instanceof Error ? cause.message : String(cause); + logger.warn`Failed to restore workflow deployment ${runId}: ${reason}`; + delete hosted.wired; + hosted.state = "stopped"; + hosted.error = `The sidecar could not restore it: ${reason}`; + try { + await markWorkflowRunRecord(dataDir, runId, { + state: "stopped", + error: hosted.error, + }); + } catch (markCause) { + const message = + markCause instanceof Error + ? markCause.message + : String(markCause); + logger.error`Could not record that ${record.agentAddress} failed to restore; the next boot tries it again: ${message}`; + } + } + }); + } + await Promise.all( + Array.from( + { length: Math.min(RESTORE_CONCURRENCY, spawns.length) }, + async () => { + for ( + let spawn = spawns.shift(); + spawn !== undefined; + spawn = spawns.shift() + ) { + await spawn(); + } + }, + ), + ); + }, + reportStoppedDeployments(): void { + for (const [agentAddress, hosted] of deployments) { + if ( + hosted.state === "stopped" && + hosted.error !== undefined && + hosted.retention === undefined + ) { + reportStopped( + { + agentAddress, + generation: hosted.generation, + error: hosted.error, + }, + () => + deployments.get(agentAddress) === hosted && + hosted.state === "stopped" && + hosted.retention === undefined, + ); + } + } + }, + incarnations(): HostedIncarnation[] { + return [...deployments].map(([address, hosted]) => ({ + address, + generation: hosted.generation, + state: hosted.state, + ...(hosted.retention !== undefined + ? { retention: hosted.retention } + : {}), + ...(hosted.error !== undefined + ? { error: fitDeploymentError(hosted.error) } + : {}), + })); + }, + reEmitParkedCorrelations(address: string): void { + const wired = deployments.get(address)?.wired; + if (wired === undefined) { + // Edge boundary: the hub reports this address routable but no live + // supervisor owns it -- a deployment torn down, or not yet respawned. + // Re-registering a torn-down deployment would be wrong, so skipping is + // correct. Logged at debug (not warn) so a supervisor unexpectedly + // missing for a live deployment is still traceable without crying wolf + // on the ordinary torn-down case. + logger.debug`re-emit on reconnect: no active supervisor for ${address}; skipping`; + return; + } + // Fire-and-forget: the driver is best-effort and watchdog-bounded inside + // the supervisor, so the reconnect fan-out never awaits it. The promise + // is dropped deliberately; the `.catch` is defense-in-depth since the + // driver already swallows its own query failures. + void wired.supervisor.reEmitParkedCorrelations().catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`re-emit of parked correlations on hub reconnect failed for ${address}: ${message}`; + }); + }, + }; +} + +/** + * Logical mail-audit reference the supervisor stamps onto every + * inbox/processing/consumed envelope for sidecar-hosted deployments. + * The substrate does not dereference the value; it is a host-side + * pointer the audit consumer joins on. The mail audit is keyed by the + * deployment id plus the parsed messageId, which is unique per inbound + * message and stable across the FIFO pipeline's + * enqueue/dequeue/markConsumed transitions. + */ +export function deriveSidecarMailAuditRef(runId: string): ( + messageId: string, + rawMessage: Uint8Array, +) => { + store: string; + path: string; +} { + return (messageId, _rawMessage) => ({ + store: "sidecar-mail-audit", + path: `${runId}/${messageId}`, + }); +} + +/** + * Construct a per-deployment supervisor with the sidecar's bindings + * pre-wired. The router calls this once per multi-step `agent.deploy` + * frame to stand up the workflow-process child that hosts the + * deployment. + */ +export function createSidecarWorkflowSupervisor( + opts: CreateSidecarWorkflowSupervisorOpts, +): SidecarWorkflowSupervisor { + // The link stamps outbound mail with the incarnation it holds for the + // sender address, so a send must not outlive its deployment's teardown: + // `closeOutbound` refuses new sends and waits for the ones in flight. + const transportBus = wrapHubTransportAsMailBus(opts.transport); + const sendsInFlight = new Set>(); + let outboundClosed = false; + const mailBus: HubTransportMailBusAdapter = { + ...transportBus, + sendOutbound(senderAddress, message) { + if (outboundClosed) { + return Promise.reject( + new Error( + `${opts.deploymentMailAddress} is being undeployed; refusing to send mail as it`, + ), + ); + } + const send = transportBus.sendOutbound(senderAddress, message); + sendsInFlight.add(send); + const settle = (): void => { + sendsInFlight.delete(send); + }; + send.then(settle, settle); + return send; + }, + }; + const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { + kind: "supervisor", + anchorRunId: opts.runId, + }; + // Per-run grants sink. The supervisor awaits this and pushes the + // resulting snapshot to the child before the run's `trigger.fire`; a + // throw propagates and the dispatch barrier fails the run rather than + // firing the trigger against absent grants. The snapshot is the run's + // own per-run grants file, which every legitimate birth path writes + // before dispatch (see `assembleRunCredentialsSnapshot`). + const onRunStart = (args: { + runId: string; + anchorRunId: string; + }): Promise => { + // Fail closed on a run whose `run.grants` write was recorded as failed: + // its per-run grants file never landed. This is the known-failed-write + // case, distinct from the general absent-file backstop in + // `assembleRunCredentialsSnapshot` -- both fail the run closed, but this + // one names the recorded write failure specifically so a poisoned run's + // diagnostics point at the failed push rather than a generic absence. + if (opts.isRunPoisoned?.(args.runId) === true) { + return Promise.reject( + new Error( + `sidecar onRunStart: run ${args.runId} grants write failed; refusing to start the run under-authorized`, + ), + ); + } + return assembleRunCredentialsSnapshot({ + repoStore: opts.repoStore, + anchorRunId: args.anchorRunId, + runId: args.runId, + stepOrder: opts.stepOrder, + deriveStepAddress: opts.deriveStepAddress, + }); + }; + const supervisor = createWorkflowSupervisor({ + repoStore: opts.repoStore, + signAsPrincipal: async (kind, payload) => { + const sig = await signEd25519(opts.signingKeySeed, payload); + return { sig, principalKind: kind }; + }, + mailBus, + subprocessSpawner: opts.subprocessSpawner, + binaryPath: opts.binaryPath, + substrateEnv: opts.substrateEnv, + dynamicSpawnEnv: opts.dynamicSpawnEnv, + workflowRunRepoId: opts.workflowRunRepoId, + workflowRunRef: opts.workflowRunRef, + anchorRunId: opts.runId, + stepCount: opts.stepCount, + deploymentMailAddress: opts.deploymentMailAddress, + readPrincipal: supervisorPrincipal, + deriveStepAddress: opts.deriveStepAddress, + onRunStart, + ...(opts.credentialDelivery !== undefined + ? { credentialDelivery: opts.credentialDelivery } + : {}), + ...(opts.onSuspensionRegister !== undefined + ? { onSuspensionRegister: opts.onSuspensionRegister } + : {}), + ...(opts.onSelfTerminate !== undefined + ? { onSelfTerminate: opts.onSelfTerminate } + : {}), + ...(opts.deriveStepRepoId !== undefined + ? { deriveStepRepoId: opts.deriveStepRepoId } + : {}), + deriveMailAuditRef: deriveSidecarMailAuditRef(opts.runId), + ...(opts.onDispatchTiming !== undefined + ? { onDispatchTiming: opts.onDispatchTiming } + : {}), + ...(opts.repackEveryMessages !== undefined + ? { repackEveryMessages: opts.repackEveryMessages } + : {}), + ...(opts.consumedRetentionMs !== undefined + ? { consumedRetentionMs: opts.consumedRetentionMs } + : {}), + ...(opts.readyTimeoutMs !== undefined + ? { readyTimeoutMs: opts.readyTimeoutMs } + : {}), + }); + return { + supervisor, + routeInbound(message) { + return mailBus.routeInbound(opts.deploymentMailAddress, message); + }, + getCredentialsSnapshot: () => supervisor.getCredentialsSnapshot(), + onRunStart, + async closeOutbound() { + outboundClosed = true; + await Promise.allSettled([...sendsInFlight]); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/deploy/workflow-run-pack-client.ts b/vendor/intx/workflow-host/src/deploy/workflow-run-pack-client.ts new file mode 100644 index 0000000000..a9e8780847 --- /dev/null +++ b/vendor/intx/workflow-host/src/deploy/workflow-run-pack-client.ts @@ -0,0 +1,1018 @@ +// Workflow-run pack push client. +// +// Sits between the boot-edge substrate facade and the hub-link's +// `pushWorkflowRunPack` wire surface. The facade fires this client +// after a successful supervisor-authored `writeTreePreservingPrefix` +// against a `workflow-run` repo; the client builds the new pack via +// `RepoStore.createPack` under the supervisor principal and ships it +// over the hub link. The hub routes the pack to its workflow-run +// receiver because `repoId.kind === "workflow-run"`. +// +// The client does NOT mint a fresh signing key, transferId, or +// principal kind. The principal is the same `WorkflowRunSupervisorPrincipal` +// shape the supervisor uses for `writeTreePreservingPrefix`; the +// transferId is minted inside `HubLink.pushWorkflowRunPack`. + +import { type } from "arktype"; + +import { getLogger } from "@intx/log"; +import { SourcesUpdatedData } from "../ipc/control-payloads"; +import type { InferenceSource } from "@intx/types/runtime"; +import { CredentialDelivery } from "@intx/types/credential-delivery"; +import type { WorkflowRunRefTips } from "@intx/types/sidecar"; +import { + WORKFLOW_RUN_RESTORE_REFS, + type RepoId, + type RepoStore, + type WorkflowRunSupervisorPrincipal, +} from "@intx/hub-sessions/substrate"; + +const logger = getLogger([ + "interchange", + "sidecar", + "workflow-run-pack-client", +]); + +export type WorkflowRunPackClient = { + /** + * Build a pack of the workflow-run repo at `ref` and ship it to the + * hub. Resolves on the hub's `repo.pack.ack`; rejects on + * `repo.pack.reject`, on a disconnect that cancels the in-flight + * transfer, or on a substrate-side `createPack` failure. The push + * failure shape is intentionally loud per the project's + * defensive-coding rule. + */ + push(opts: { + agentAddress: string; + /** The generation of the incarnation whose commits the push ships. */ + generation: number; + repoId: RepoId; + ref: string; + }): Promise; + /** + * Seed the acknowledged-tip state after the Hub restores a ref into a fresh + * worker. This prevents a reconnect from trying to re-send an empty delta at + * the restored tip; the next real local commit remains incremental from it. + */ + markRestored(repoId: RepoId, ref: string, commitSha: string): void; + /** Forget the acknowledged tips of a repository its deployment left. */ + forget(repoId: RepoId): void; +}; + +export type CreateWorkflowRunPackClientOpts = { + substrate: RepoStore; + hubLink: { + pushWorkflowRunPack(opts: { + agentAddress: string; + generation: number; + repoId: RepoId; + pack: Uint8Array; + ref: string; + commitSha: string; + }): Promise; + }; +}; + +export function createWorkflowRunPackClient( + opts: CreateWorkflowRunPackClientOpts, +): WorkflowRunPackClient { + const { substrate, hubLink } = opts; + + // Shadow of the substrate's shipped-tip cursor, keyed by `(repoId.id, ref)`. + // This client is the SOLE caller of `commitPackedTip` for the workflow-run + // kind, so its own record of "the last commitSha I acked" cannot drift from + // the substrate cursor. It exists to answer one question `createPack` cannot + // answer to its caller: is the current ref tip already shipped? When the tip + // equals the last acked sha there is nothing un-acked to ship, and building + // a pack would produce an empty delta whose declared tip is not in the pack + // -- which the hub rejects as `sha_mismatch`. Skipping the wire send in that + // case is what makes a re-drive of an already-shipped tip a clean no-op + // rather than a spurious rejection. The reconnect re-drive path + // (`notifyAddressRoutable`) can legitimately fire for a slot whose commits + // already landed on an earlier attempt, so this guard is load-bearing. + const lastAckedSha = new Map(); + function ackKey(repoId: RepoId, ref: string): string { + return `${repoId.id}/${ref}`; + } + // A repository's pushes share a token that `forget` retires, so a push whose + // ack lands after the repository was forgotten writes nothing back. + const pushTokens = new Map(); + function pushToken(repoId: RepoId): symbol { + let token = pushTokens.get(repoId.id); + if (token === undefined) { + token = Symbol(repoId.id); + pushTokens.set(repoId.id, token); + } + return token; + } + + return { + markRestored(repoId, ref, commitSha) { + if (repoId.kind !== "workflow-run") { + throw new Error( + `workflow-run pack client: restored repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, + ); + } + substrate.commitPackedTip(repoId, ref, commitSha); + lastAckedSha.set(ackKey(repoId, ref), commitSha); + }, + forget(repoId) { + const prefix = `${repoId.id}/`; + for (const key of [...lastAckedSha.keys()]) { + if (key.startsWith(prefix)) lastAckedSha.delete(key); + } + pushTokens.delete(repoId.id); + }, + async push({ agentAddress, generation, repoId, ref }) { + if (repoId.kind !== "workflow-run") { + throw new Error( + `workflow-run pack client: repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, + ); + } + const token = pushToken(repoId); + const principal: WorkflowRunSupervisorPrincipal = { + kind: "supervisor", + anchorRunId: repoId.id, + }; + // Nothing to ship when the local tip is already the last acked tip: + // the run's commits landed on a prior push. Return without a wire send + // so a re-drive of an already-shipped tip is a clean no-op instead of + // an empty-delta pack the hub rejects. + const tip = await substrate.resolveRef(principal, repoId, ref); + if (tip !== null && tip === lastAckedSha.get(ackKey(repoId, ref))) { + return; + } + const { pack, commitSha } = await substrate.createPack( + principal, + repoId, + ref, + ); + await hubLink.pushWorkflowRunPack({ + agentAddress, + generation, + repoId, + pack, + ref, + commitSha, + }); + // `pushWorkflowRunPack` resolves only on the hub's + // `repo.pack.ack` and rejects on a reject or a reconnect that + // cancels the transfer. Advancing the substrate's shipped-tip + // cursor here — after the ack, never at build time — is what + // lets a cancelled transfer be re-shipped: a rejected push throws + // before this line, so the cursor stays put and the next + // `createPack` re-includes the un-acked commits. + if (pushTokens.get(repoId.id) !== token) return; + substrate.commitPackedTip(repoId, ref, commitSha); + lastAckedSha.set(ackKey(repoId, ref), commitSha); + }, + }; +} + +/** The incarnation a workflow-run repository's commits belong to. */ +export type DeploymentIncarnation = { + agentAddress: string; + generation: number; +}; + +/** + * Mapping registry the boot-edge substrate facade consults to resolve + * `repoId.id` (the workflow-run runId, which the deploy router + * derives by slugging the agent's mail address) back into the + * incarnation every outbound pack frame names. Populated by the + * deploy router as each deployment is stood up. + */ +export type DeploymentAddressRegistry = { + record(runId: string, incarnation: DeploymentIncarnation): void; + resolve(runId: string): DeploymentIncarnation | null; + unregister(runId: string): void; +}; + +export function createDeploymentAddressRegistry(): DeploymentAddressRegistry { + const table = new Map(); + return { + record(runId, incarnation) { + table.set(runId, incarnation); + }, + resolve(runId) { + return table.get(runId) ?? null; + }, + unregister(runId) { + table.delete(runId); + }, + }; +} + +/** + * Handler the multi-step deploy router installs on the + * `MultistepMailRouter` after a supervisor's `spawn` succeeds. The + * handler hands a delivered inbound mail off to the per-deployment + * supervisor's `routeInbound`, which dispatches into the workflow-host + * mail-bus the multi-step child's `awaitSignal` subscribes against. + */ +export type MultistepMailHandler = (message: Uint8Array) => Promise; + +/** + * Per-deployment-address mail handler registry the sidecar hub-link + * consults before falling back to `transport.deliver`. The multi-step + * deploy router registers a handler against the deployment's mail + * address after `wired.supervisor.spawn` succeeds, so an inbound + * `mail.inbound` frame for that address dispatches into the + * supervisor's mail-bus subscription rather than the + * never-provisioned-for-this-address transport mailbox. + * + * The registry is owned at the sidecar's host layer (not inside the + * workflow-host library) because the routing decision is between + * "legacy single-agent path" and "supervisor mail-bus path" -- two + * concrete sidecar host concerns. The workflow-host package stays + * agnostic to which transport surface its mail-bus rides on. + */ +export type MultistepMailRouter = { + register(address: string, handler: MultistepMailHandler): void; + unregister(address: string): void; + /** + * Dispatch `message` to the handler registered for `address`. Returns + * `null` when no handler is registered (the caller logs and drops, and + * sends no ack). Otherwise returns the handler's durable settlement: a + * promise that resolves once the message is durably accepted and rejects + * when it was not, so the caller acks only on resolution. + */ + tryRoute(address: string, message: Uint8Array): Promise | null; +}; + +export function createMultistepMailRouter(): MultistepMailRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + tryRoute(address, message) { + const handler = handlers.get(address); + if (handler === undefined) return null; + return handler(message); + }, + }; +} + +/** + * Per-deployment signal-delivery handler the multi-step deploy router + * installs against the `MultistepSignalRouter` after a supervisor's + * `spawn` succeeds. The handler hands the signal off to the supervisor's + * `deliverSignal`, which sends a `signal.deliver` control IPC frame to + * the workflow-process child. Routing every workflow-run signal through + * the child keeps the workflow-run repo's single-writer invariant + * intact -- the child is the only writer of `runs//events/` on + * the sidecar side, so the pack-push pipeline that propagates the + * commit to the hub never races against a concurrent host-side write. + */ +export type MultistepSignalHandler = (args: { + runId: string; + signalName: string; + signalId: string; + payload: unknown; + correlationId?: string; +}) => Promise; + +/** + * Per-deployment-address signal handler registry the sidecar hub-link + * consults on every inbound `signal.deliver` frame. The deploy router + * registers a handler against the deployment's mail address after + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches the signal into the + * supervisor's `deliverSignal`. + * + * The registry lives at the sidecar's host layer (not inside the + * workflow-host library) for the same boundary reason as + * `MultistepMailRouter`: the routing decision is a concrete sidecar + * host concern, and the workflow-host package stays agnostic to which + * transport surface its supervisor handle rides on. + */ +export type MultistepSignalRouter = { + register(address: string, handler: MultistepSignalHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "signal.deliver"; + agentAddress: string; + runId: string; + signalName: string; + signalId: string; + payload: unknown; + correlationId?: string; + }): Promise; +}; + +export function createMultistepSignalRouter(): MultistepSignalRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + if (handler === undefined) return false; + await handler({ + runId: frame.runId, + signalName: frame.signalName, + signalId: frame.signalId, + payload: frame.payload, + ...(frame.correlationId !== undefined + ? { correlationId: frame.correlationId } + : {}), + }); + return true; + }, + }; +} + +/** + * Per-deployment grants-write handler the deploy router installs against + * the `MultistepGrantsRouter` after a supervisor's `spawn` succeeds. The + * handler writes the run's grants to `runs//grants.json` inside the + * deployment's `workflow-run` repo -- sibling to the run's + * `runs//events/` subtree. The write is awaited so the frame's FIFO + * completion means the grants are durable on disk before the next frame is + * processed. + * + * The hub-link caches the sender keys co-delivered on the `run.grants` frame + * before it hands the grants over. `senderKeysCached` is false when one of + * them did not land; the handler then fails the grants, so a run never starts + * with a sender its recipient cannot verify. + */ +export type MultistepGrantsHandler = (args: { + runId: string; + stepGrants: readonly unknown[]; + senderKeysCached: boolean; +}) => Promise; + +/** + * Per-deployment-address grants handler registry the sidecar hub-link + * consults on every inbound `run.grants` frame. The deploy router + * registers a handler against the deployment's mail address after + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler writes the run's grants into the + * deployment's workflow-run repo. + * + * The registry lives at the sidecar's host layer (not inside the + * workflow-host library) for the same boundary reason as + * `MultistepMailRouter` / `MultistepSignalRouter`: the routing decision + * is a concrete sidecar host concern, and the workflow-host package + * stays agnostic to which transport surface its supervisor handle rides + * on. + */ +export type MultistepGrantsRouter = { + register(address: string, handler: MultistepGrantsHandler): void; + unregister(address: string): void; + tryRoute( + frame: { + type: "run.grants"; + agentAddress: string; + runId: string; + stepGrants: readonly unknown[]; + }, + senderKeysCached: boolean, + ): Promise; +}; + +export function createMultistepGrantsRouter(): MultistepGrantsRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame, senderKeysCached) { + const handler = handlers.get(frame.agentAddress); + if (handler === undefined) return false; + await handler({ + runId: frame.runId, + stepGrants: frame.stepGrants, + senderKeysCached, + }); + return true; + }, + }; +} + +/** + * Per-deployment drain handler the multi-step deploy router installs + * against the `MultistepDrainRouter` after a supervisor's `spawn` + * succeeds. The handler hands the drain opts off to the supervisor's + * `drain`, which sends a `drain` control IPC frame to the + * workflow-process child and arms one `drainTimeout` accumulator per + * in-flight run. Cancel-mode in-flight steps abort as the child's + * controller signal flips; wait-mode steps continue. Each accumulator + * commits a signed `CancelRequested{origin: "supervisor-drain"}` + * against the workflow-run repo when the deadline expires. + */ +export type MultistepDrainHandler = (args: { + deadlineMs: number; +}) => Promise; + +/** + * Per-deployment-address drain handler registry the sidecar hub-link + * consults on every inbound `drain.deliver` frame. The deploy router + * registers a handler against the deployment's mail address after + * `wired.supervisor.spawn` succeeds, for single-step and multi-step + * deployments alike; the handler dispatches into the supervisor's + * `drain`. + * + * The registry lives at the sidecar's host layer (not inside the + * workflow-host library) for the same boundary reason as + * `MultistepMailRouter` / `MultistepSignalRouter`: the routing decision + * is a concrete sidecar host concern, and the workflow-host package + * stays agnostic to which transport surface its supervisor handle + * rides on. + */ +export type MultistepDrainRouter = { + register(address: string, handler: MultistepDrainHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "drain.deliver"; + agentAddress: string; + deadlineMs: number; + }): Promise; +}; + +export function createMultistepDrainRouter(): MultistepDrainRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + if (handler === undefined) return false; + await handler({ deadlineMs: frame.deadlineMs }); + return true; + }, + }; +} + +/** + * Per-deployment sources-rotation handler the deploy router installs + * against the `MultistepSourcesRouter` after a supervisor's `spawn` + * succeeds -- but ONLY for a single-step (warm launched-agent) + * deployment. The handler hands the rotated list off to the supervisor's + * `deliverSources`, which sends a `sources-updated` control IPC frame to + * the workflow-process child, where the warm agent's live sources are + * swapped in place. A multi-step deployment has no single warm agent to + * rotate, so the router registers no handler for it and an inbound + * `sources.update` for a multi-step address is unrouted. + */ +export type MultistepSourcesHandler = (args: { + sources: InferenceSource[]; + defaultSource: string; +}) => Promise; + +/** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (after + * `wired.supervisor.spawn` succeeds); a multi-step deployment never + * does, so `tryRoute` resolves a rotation only for a registered + * single-step address and returns `false` for any other. + * + * The registry lives at the sidecar's host layer for the same boundary + * reason as the mail/signal/drain routers: the routing decision is a + * concrete sidecar host concern, and the workflow-host package stays + * agnostic to which transport surface its supervisor handle rides on. + */ +export type MultistepSourcesRouter = { + register(address: string, handler: MultistepSourcesHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "sources.update"; + agentAddress: string; + sources: InferenceSource[]; + defaultSource: string; + }): Promise; +}; + +export function createMultistepSourcesRouter(): MultistepSourcesRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + // Registration check first: an unregistered (multi-step or torn-down) + // address is unrouted -- reported as `false`, its payload never + // inspected, because it would not be acted on regardless. + if (handler === undefined) return false; + // Validate the rotation BEFORE dispatch. This is the only inbound + // router that validates its frame, and deliberately so: a bad list + // (duplicate ids, or a default that is not the head element) would + // reach the child's control-channel receiver and crash it on + // `SourcesUpdatedData`'s narrow -- the sources-updated frame is the + // only inbound frame carrying a crash-on-invalid narrow downstream, + // and the only one that is request/ack. Rejecting here throws, and + // the hub-link turns the throw into a truthful `session.error` + // instead of acking and detonating the child. + const validated = SourcesUpdatedData({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + if (validated instanceof type.errors) { + throw new Error(validated.summary); + } + await handler({ + sources: frame.sources, + defaultSource: frame.defaultSource, + }); + return true; + }, + }; +} + +/** + * Per-deployment credential-delivery handler the deploy router installs + * against the `MultistepCredentialsRouter` after a supervisor's `spawn` + * succeeds. The handler hands the delivery to the supervisor's + * `deliverCredentials`, which sends a `credentials-updated` control IPC frame + * to the workflow-process child, where the material cell is swapped in place. + * + * Unlike sources rotation, this registers for ANY deployment (single- or + * multi-step): the material cell is per-child and read by every step's tool + * capabilities, so there is no single-warm-agent restriction. There is no + * durable persist -- credential material never touches disk (it is re-resolved + * by the hub on reconnect). + */ +export type MultistepCredentialsHandler = (args: { + delivery: CredentialDelivery; + revoke?: string[]; +}) => Promise; + +/** + * Per-deployment-address credential-delivery handler registry. Mirrors + * `MultistepSourcesRouter`: `credentials.update` is a REQUEST/ACK frame, so a + * registered address that throws surfaces as a `session.error` and an + * unregistered address returns `false` (unrouted). Lives at the sidecar host + * layer for the same boundary reason -- the workflow-host package stays + * agnostic to the transport surface its supervisor rides on. + */ +export type MultistepCredentialsRouter = { + register(address: string, handler: MultistepCredentialsHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "credentials.update"; + agentAddress: string; + delivery: CredentialDelivery; + revoke?: string[]; + }): Promise; +}; + +export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + // Registration check first: an unregistered (torn-down) address is + // unrouted -- reported as `false`, its payload never inspected. + if (handler === undefined) return false; + // Validate the delivery BEFORE dispatch: a malformed delivery would + // reach the child's control-channel receiver and crash it on + // `CredentialsUpdateFrame`'s narrow. Rejecting here throws, and the + // hub-link turns the throw into a truthful `session.error` instead of + // acking and detonating the child. + const validated = CredentialDelivery(frame.delivery); + if (validated instanceof type.errors) { + throw new Error(validated.summary); + } + await handler({ + delivery: frame.delivery, + ...(frame.revoke !== undefined ? { revoke: frame.revoke } : {}), + }); + return true; + }, + }; +} + +/** + * Boot-edge facade around the substrate-shaped `RepoStore`. Forwards + * every method to the underlying store; intercepts the + * `writeTreePreservingPrefix` return path so a successful write + * against a `workflow-run` repo schedules a workflow-run pack push. + * Writes against any other `repoId.kind` (today, only `agent-state` + * via the deploy-applier path) flow through unchanged. + * + * Pack-push coalescing: the facade returns from + * `writeTreePreservingPrefix` as soon as the LOCAL commit lands and + * schedules an asynchronous pack push for `(repoId.id, ref)`. At + * most one push per (repoId, ref) is in flight at a time. Writes + * that arrive while a push is in flight are NOT enqueued as + * additional pushes; instead they mark the slot as "dirty", and the + * loop runs one more push after the current one settles. This means + * a burst of N writes against the same ref produces at most 2 + * pushes (the one already running when the burst starts, plus one + * more for everything that arrived during it), rather than N + * serial round-trips' worth of hub-ack latency. The push body + * captures the current local ref tip at the moment it runs, so the + * single pack it builds covers every commit landed since the prior + * ACKED tip -- the substrate's incremental `createPack` walks the + * chain from the cursor `commitPackedTip` last committed on an ack + * forward, so the receiver still sees every commit transition. + * + * Single-writer + FIFO correctness: the underlying substrate + * serialises local writes via `withRepoLock`, so commits land on + * disk in submission order. The hub's `receivePack` validates each + * commit's parent against its existing-commits set; as long as the + * pack carries the full chain from prior acked tip to current tip, + * every intermediate commit is validated by the receiver. Coalescing + * multiple local commits into one network push therefore preserves + * the receive-time CAS invariant while collapsing N hub round-trips + * into 1. + * + * Reconnect-safe re-shipping: the substrate advances its shipped-tip + * cursor on the ack (`push` calls `commitPackedTip` only after + * `pushWorkflowRunPack` resolves), never at build time. A transfer a + * reconnect cancels before its ack therefore leaves the cursor where + * it was, so the retry loop's next `createPack` re-includes the + * un-acked commits and the receiver gets a self-consistent chain + * rather than a pack whose base commit it never received. + * + * Failure surfacing: a failed push latches its error on the + * per-(repoId, ref) slot's `lastError` field. The next call to + * `writeTreePreservingPrefix` on that (repoId, ref) re-throws the + * latched error before doing its own work, keeping failures loud + * rather than swallowed by the fire-and-forget pipeline. The + * defensive-coding rule says errors must surface; this is how they + * surface from a coalescing writer. + * + * Flush: callers that need a hub-visible barrier (shutdown, + * integration tests that read hub-side state) call + * `flushWorkflowRunPushes(repoId, ref)` to await the per-(repoId, + * ref) slot to drain (both the in-flight push and any follow-up + * triggered by writes that arrived during it). + */ +export type WorkflowRunPackPushingRepoStoreOpts = { + underlying: RepoStore; + packClient: Pick; + registry: DeploymentAddressRegistry; + isConnectionLost: (error: unknown) => boolean; + deriveWorkflowRunRepoId: (agentAddress: string) => string; +}; + +/** + * The wrapped store plus a side-channel API for waiting on the + * per-(repoId.id, ref) pack-push pipeline to drain. The `RepoStore` + * shape is unchanged so call sites that consume `RepoStore` keep + * working; `flushWorkflowRunPushes` is opt-in for code that + * genuinely needs hub-side visibility (shutdown, integration tests, + * end-to-end benchmarks). Call sites that don't need it pay zero + * cost. + */ +export type WorkflowRunPackPushingRepoStore = RepoStore & { + /** + * Await the pack-push pipeline for `(repoId.id, ref)` to drain. + * Resolves once no push is in flight and no follow-up push is + * pending; rejects if the most recent push failed (the same + * latched error the next `writeTreePreservingPrefix` call would + * surface), or if the pipeline stopped with a push still due, because + * no connection could carry it or the deployment was undeployed first. + */ + flushWorkflowRunPushes: (repoId: RepoId, ref: string) => Promise; + /** + * Re-drive any workflow-run push for `agentAddress` that a disconnect + * cancelled or a block held. Called when a `welcome` routes the deployment + * address again. For each slot bound to `agentAddress` with a push still + * due or whose last push attempt failed (its `lastError` is latched), it + * re-arms the coalescing loop so a fresh `createPack` re-ships the un-acked + * commits -- the liveness path a synchronous single-step run lacks, because + * it has no later local write to re-set `dirty`. + * + * A re-ship that fails again re-latches without self-retrying, so a + * genuinely unrecoverable failure still surfaces loudly on the next local + * write rather than spinning. It is gated on the address being routable + * again (the caller only fires post-reconnect) so the re-ship cannot race + * ahead of the hub re-routing the address. + */ + notifyAddressRoutable: (agentAddress: string) => void; + /** + * Block workflow-run pushes for `agentAddress` until the next + * `notifyAddressRoutable`, or until a stop reports the address's tips. + * Called when a hub-link connection opens or closes: the Hub does not route + * the address on a connection until its `welcome` does, and the link carries + * no push until then. Holding the push at the block -- rather than shipping + * and failing -- is what lets the reconnect re-ship wait for route + * restoration. + */ + markAddressUnroutable: (agentAddress: string) => void; + /** + * Read the local tip of each authoritative ref of the deployment's + * workflow-run repo, `null` for an absent ref, and schedule a push of every + * existing ref without waiting for it. The push ships commits the hub has not + * acknowledged, including ones written outside this facade's write hooks, + * such as run grants. It lifts a disconnect's block on the address: no + * `welcome` routes a stopped deployment, so no reconnect would lift it. + */ + reportWorkflowRunRefTips: ( + incarnation: DeploymentIncarnation, + ) => Promise; + /** + * Drop the push state of a deployment the sidecar no longer hosts. A + * sidecar shared by many deployments runs long, and nothing else removes + * it. A push still in flight settles against the dropped state. + */ + forgetDeployment: (deploymentId: string, agentAddress: string) => void; +}; + +export function createWorkflowRunPackPushingRepoStore( + opts: WorkflowRunPackPushingRepoStoreOpts, +): WorkflowRunPackPushingRepoStore { + const { underlying, packClient, registry, deriveWorkflowRunRepoId } = opts; + + type Slot = { + agentAddress: string; + generation: number; + repoId: RepoId; + ref: string; + inFlight: Promise | null; + dirty: boolean; + lastError: Error | null; + settled: (() => void)[]; + }; + const slots = new Map(); + function slotKey(repoId: RepoId, ref: string): string { + return `${repoId.kind}/${repoId.id}/${ref}`; + } + + // Addresses the Hub does not route on the current connection yet. Absent + // means routable -- the steady state, and the state of a deployment deployed + // on the current connection, which the Hub routes from its `agent.deploy`. + // An address the link held when a connection opened or closed is added on + // `markAddressUnroutable` and removed on `notifyAddressRoutable`, once a + // `welcome` routes it, or when a stop reports its tips: no `welcome` routes + // a stopped deployment. A push for a blocked address is held: the coalescing + // loop pauses with `dirty` still set rather than shipping before the link can + // carry it -- which is what makes the re-ship wait for route restoration. + const blockedAddresses = new Set(); + + function notifySettled(slot: Slot): void { + const callbacks = slot.settled; + slot.settled = []; + for (const cb of callbacks) cb(); + } + + function startLoop(slot: Slot, repoId: RepoId, ref: string): void { + if (slot.inFlight !== null) return; + // Hold the push while the address is not routable (no `welcome` has + // routed it on the current connection). Leave `dirty` set and start no loop: the loop + // resumes when `notifyAddressRoutable` clears the block and re-arms it. + if (blockedAddresses.has(slot.agentAddress)) return; + slot.inFlight = (async () => { + while (slot.dirty) { + // Re-check routability each iteration: a disconnect mid-drain must + // pause the loop rather than push into a severed link. Leave `dirty` + // set so the post-reconnect resume re-ships. + if (blockedAddresses.has(slot.agentAddress)) break; + // A forgotten deployment ships nothing more: the Hub no longer routes + // it. + if (slots.get(slotKey(repoId, ref)) !== slot) break; + slot.dirty = false; + try { + await packClient.push({ + agentAddress: slot.agentAddress, + generation: slot.generation, + repoId, + ref, + }); + slot.lastError = null; + } catch (cause) { + // Nothing carried the push, so nothing about it failed: it stays due + // until a `welcome` routes the address again or a stop reports its + // tips. Failing the deployment's next write with it would blame the + // deployment for the link. + if (opts.isConnectionLost(cause)) { + slot.dirty = true; + break; + } + const msg = cause instanceof Error ? cause.message : String(cause); + logger.warn`workflow-run pack push failed for deployment ${repoId.id} (${slot.agentAddress}): ${msg}`; + slot.lastError = + cause instanceof Error ? cause : new Error(String(cause)); + } + } + slot.inFlight = null; + notifySettled(slot); + })(); + } + + function schedulePush( + incarnation: DeploymentIncarnation, + repoId: RepoId, + ref: string, + ): void { + const key = slotKey(repoId, ref); + let slot = slots.get(key); + if (slot === undefined) { + slot = { + agentAddress: incarnation.agentAddress, + generation: incarnation.generation, + repoId, + ref, + inFlight: null, + dirty: false, + lastError: null, + settled: [], + }; + slots.set(key, slot); + } + slot.dirty = true; + startLoop(slot, repoId, ref); + } + + function takeLatchedError(repoId: RepoId, ref: string): Error | null { + const slot = slots.get(slotKey(repoId, ref)); + if (slot === undefined) return null; + const err = slot.lastError; + if (err !== null) slot.lastError = null; + return err; + } + + function markAddressUnroutable(agentAddress: string): void { + // The Hub does not route this address on the current connection (it just + // opened or closed). Block its pushes until a `welcome` routes it. A push + // already in flight when the link dropped is cancelled and stays due; the + // block stops the coalescing loop from re-shipping it before then. + blockedAddresses.add(agentAddress); + } + + function notifyAddressRoutable(agentAddress: string): void { + // A `welcome` routed this address on the Hub. Clear the block and re-drive + // so a push the disconnect cancelled -- or one held while the block was + // up -- ships now. This is the liveness path a + // synchronous single-step run lacks: with all its events in one batch it + // has no later local write to re-arm the coalescing loop, so the drop + // would otherwise strand it forever. + blockedAddresses.delete(agentAddress); + for (const slot of slots.values()) { + if (slot.agentAddress !== agentAddress) continue; + // Re-drive a slot that has pending work (`dirty`, e.g. a push held at + // the block or cancelled with its connection) OR whose last attempt + // failed (`lastError` latched). A slot that is clean and already acked + // (`!dirty && lastError === null`) has nothing un-shipped -- the + // `packClient.push` empty-delta guard would skip it anyway, but not + // re-arming it avoids a pointless loop spin. Re-arming `dirty` and + // restarting is safe against double-ship: `startLoop` no-ops when a + // push is already in flight, and the per-(repoId, ref) serialization in + // the hub-link's `pushWorkflowRunPack` prevents overlapping transfers. + if (!slot.dirty && slot.lastError === null) continue; + slot.dirty = true; + startLoop(slot, slot.repoId, slot.ref); + } + } + + function forgetDeployment(deploymentId: string, agentAddress: string): void { + for (const [key, slot] of slots) { + if (slot.repoId.id === deploymentId) slots.delete(key); + } + blockedAddresses.delete(agentAddress); + packClient.forget({ kind: "workflow-run", id: deploymentId }); + } + + async function flushWorkflowRunPushes( + repoId: RepoId, + ref: string, + ): Promise { + const key = slotKey(repoId, ref); + const slot = slots.get(key); + if (slot === undefined) return; + if (slot.inFlight !== null) { + await new Promise((resolve) => { + slot.settled.push(resolve); + }); + } + if (slot.lastError !== null) { + const err = slot.lastError; + slot.lastError = null; + throw err; + } + if (!slot.dirty) return; + if (slots.get(key) !== slot) { + throw new Error( + `workflow-run pushes for ${repoId.id} ${ref} did not drain: ${slot.agentAddress} generation ${String(slot.generation)} was undeployed first`, + ); + } + // The loop stopped, or never started, with a push still due, because the + // Hub does not route the address on the current connection or the + // connection carrying the push closed. A `welcome` that routes the address, + // or a stop report, ships it; until then nothing drains. + throw new Error( + `workflow-run pushes for ${repoId.id} ${ref} did not drain: the Hub does not route ${slot.agentAddress} on a connection yet`, + ); + } + + async function reportWorkflowRunRefTips( + incarnation: DeploymentIncarnation, + ): Promise { + const { agentAddress } = incarnation; + const repoId: RepoId = { + kind: "workflow-run", + id: deriveWorkflowRunRepoId(agentAddress), + }; + const principal: WorkflowRunSupervisorPrincipal = { + kind: "supervisor", + anchorRunId: repoId.id, + }; + blockedAddresses.delete(agentAddress); + const tips: WorkflowRunRefTips = {}; + for (const ref of WORKFLOW_RUN_RESTORE_REFS) { + const tip = await underlying.resolveRef(principal, repoId, ref); + tips[ref] = tip; + if (tip !== null) schedulePush(incarnation, repoId, ref); + } + return tips; + } + + const wrapped: WorkflowRunPackPushingRepoStore = { + initRepo: underlying.initRepo.bind(underlying), + writeTree: underlying.writeTree.bind(underlying), + receivePack: underlying.receivePack.bind(underlying), + createPack: underlying.createPack.bind(underlying), + commitPackedTip: underlying.commitPackedTip.bind(underlying), + resolveRef: underlying.resolveRef.bind(underlying), + listRefs: underlying.listRefs.bind(underlying), + resolveHead: underlying.resolveHead.bind(underlying), + getRepoDir: underlying.getRepoDir.bind(underlying), + openCommittedReads: underlying.openCommittedReads.bind(underlying), + openCommittedReadsAtCommit: + underlying.openCommittedReadsAtCommit.bind(underlying), + subscribe: underlying.subscribe.bind(underlying), + flushWorkflowRunPushes, + notifyAddressRoutable, + markAddressUnroutable, + reportWorkflowRunRefTips, + forgetDeployment, + async writeTreePreservingPrefix(principal, repoId, ref, args) { + if (repoId.kind === "workflow-run") { + const latched = takeLatchedError(repoId, ref); + if (latched !== null) { + throw latched; + } + } + const result = await underlying.writeTreePreservingPrefix( + principal, + repoId, + ref, + args, + ); + if (repoId.kind !== "workflow-run") { + return result; + } + const incarnation = registry.resolve(repoId.id); + if (incarnation === null) { + throw new Error( + `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + ); + } + schedulePush(incarnation, repoId, ref); + return result; + }, + async writeTreeDelta(principal, repoId, ref, args) { + if (repoId.kind === "workflow-run") { + const latched = takeLatchedError(repoId, ref); + if (latched !== null) { + throw latched; + } + } + const result = await underlying.writeTreeDelta( + principal, + repoId, + ref, + args, + ); + if (repoId.kind !== "workflow-run") { + return result; + } + const incarnation = registry.resolve(repoId.id); + if (incarnation === null) { + throw new Error( + `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + ); + } + schedulePush(incarnation, repoId, ref); + return result; + }, + }; + return wrapped; +} diff --git a/apps/sidecar/src/workflow-run-pack-restore.ts b/vendor/intx/workflow-host/src/deploy/workflow-run-pack-restore.ts similarity index 63% rename from apps/sidecar/src/workflow-run-pack-restore.ts rename to vendor/intx/workflow-host/src/deploy/workflow-run-pack-restore.ts index 53b4245e59..bb21d92a57 100644 --- a/apps/sidecar/src/workflow-run-pack-restore.ts +++ b/vendor/intx/workflow-host/src/deploy/workflow-run-pack-restore.ts @@ -1,8 +1,12 @@ import { promises as fs } from "node:fs"; import path from "node:path"; -import { WORKFLOW_RUN_RESTORE_REFS, type RepoId, type RepoStore } from "@intx/hub-sessions"; -import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; +import { + WORKFLOW_RUN_GITIGNORE_PATH, + WORKFLOW_RUN_RESTORE_REFS, + type RepoId, + type RepoStore, +} from "@intx/hub-sessions/substrate"; const RESTORABLE_REFS: readonly string[] = WORKFLOW_RUN_RESTORE_REFS; @@ -14,14 +18,34 @@ export type WorkflowRunPackRestoreArgs = { commitSha: string; }; -export type WorkflowRunPackRestorer = (args: WorkflowRunPackRestoreArgs) => Promise; +export type WorkflowRunPackRestorer = ( + args: WorkflowRunPackRestoreArgs, +) => Promise; type MaterializedFile = { oid: string; read(): Promise; }; -async function materializeRestoredRefs(substrate: RepoStore, repoId: RepoId): Promise { +async function holdsOnlyGenesis( + substrate: RepoStore, + repoId: RepoId, + ref: string, +): Promise { + const reads = await substrate.openCommittedReads( + { kind: "hub" }, + repoId, + ref, + ); + if (reads === null) return true; + const entries = await reads.listDir(""); + return entries.every((entry) => entry.name === WORKFLOW_RUN_GITIGNORE_PATH); +} + +async function materializeRestoredRefs( + substrate: RepoStore, + repoId: RepoId, +): Promise { const hubPrincipal = { kind: "hub" } as const; const files = new Map(); const directories = new Set(); @@ -55,7 +79,9 @@ async function materializeRestoredRefs(substrate: RepoStore, repoId: RepoId): Pr } const existing = files.get(relPath); if (existing !== undefined && existing.oid !== entry.oid) { - throw new Error(`workflow_run_restore_conflict: ${relPath} differs across restored refs`); + throw new Error( + `workflow_run_restore_conflict: ${relPath} differs across restored refs`, + ); } if (existing === undefined) { files.set(relPath, { @@ -87,7 +113,9 @@ async function materializeRestoredRefs(substrate: RepoStore, repoId: RepoId): Pr for (const [relPath, file] of files) { const destination = destinations.get(relPath); if (destination === undefined) { - throw new Error(`workflow_run_restore_invalid: missing validated destination for ${relPath}`); + throw new Error( + `workflow_run_restore_invalid: missing validated destination for ${relPath}`, + ); } await fs.mkdir(path.dirname(destination), { recursive: true }); await fs.writeFile(destination, await file.read()); @@ -95,16 +123,17 @@ async function materializeRestoredRefs(substrate: RepoStore, repoId: RepoId): Pr } /** - * Build the sidecar boundary that installs Hub-authoritative workflow-run - * history before a replacement supervisor starts. Packs land on the - * unwrapped substrate so applying restored history cannot echo it back to the - * Hub as a new sidecar-authored update. + * Build the sidecar boundary that installs the Hub's copy of a deployment's + * workflow-run history before its first supervisor starts. Packs land on the + * unwrapped substrate so applying that history cannot echo it back to the Hub + * as a new sidecar-authored update. */ export function createWorkflowRunPackRestorer(args: { substrate: RepoStore; markRestored(repoId: RepoId, ref: string, commitSha: string): void; + deriveWorkflowRunRepoId: (agentAddress: string) => string; }): WorkflowRunPackRestorer { - const { substrate, markRestored } = args; + const { substrate, markRestored, deriveWorkflowRunRepoId } = args; const hubPrincipal = { kind: "hub" } as const; return async ({ agentAddress, repoId, pack, ref, commitSha }) => { @@ -120,7 +149,27 @@ export function createWorkflowRunPackRestorer(args: { ); } if (!RESTORABLE_REFS.includes(ref)) { - throw new Error(`workflow_run_restore_invalid: unsupported workflow-run ref ${ref}`); + throw new Error( + `workflow_run_restore_invalid: unsupported workflow-run ref ${ref}`, + ); + } + + // History seeds only a first deploy. A branch this sidecar already has + // holds what an earlier copy of the deployment left here, and moving it + // would mix that with the Hub's history, so only a missing branch or one + // already at this commit, as a retried first deploy finds it, is taken. + // A branch holding nothing but the store's genesis counts as missing: an + // attempt that failed after creating the repository left it, and it has + // no history to lose. + const current = await substrate.resolveRef(hubPrincipal, repoId, ref); + if ( + current !== null && + current !== commitSha && + !(await holdsOnlyGenesis(substrate, repoId, ref)) + ) { + throw new Error( + `workflow_run_restore_conflict: ${ref} of ${agentAddress} is already at ${current} here`, + ); } // `RepoStore` initializes a signed `.gitignore` genesis on main. Do that @@ -128,9 +177,20 @@ export function createWorkflowRunPackRestorer(args: { // null and then letting `receivePack` initialize it would observe a new // genesis inside the receive lock and reject the stale null pre-image. await substrate.initRepo(repoId); - const expectedOldSha = await substrate.resolveRef(hubPrincipal, repoId, ref); + const expectedOldSha = await substrate.resolveRef( + hubPrincipal, + repoId, + ref, + ); if (expectedOldSha !== commitSha) { - await substrate.receivePack(hubPrincipal, repoId, ref, pack, commitSha, expectedOldSha); + await substrate.receivePack( + hubPrincipal, + repoId, + ref, + pack, + commitSha, + expectedOldSha, + ); } await materializeRestoredRefs(substrate, repoId); diff --git a/vendor/intx/workflow-host/src/deploy/workflow-run-record.ts b/vendor/intx/workflow-host/src/deploy/workflow-run-record.ts new file mode 100644 index 0000000000..46963e9874 --- /dev/null +++ b/vendor/intx/workflow-host/src/deploy/workflow-run-record.ts @@ -0,0 +1,480 @@ +// Sidecar-local persistence of the per-run record needed to re-establish a +// workflow run across a sidecar PROCESS restart. The record is co-located +// with the run's workflow-run substrate at +// `${dataDir}/workflow-runs//deployment.json`, so a single +// teardown reclaims both and a boot scan can enumerate the active runs +// beside the run state they resume. +// +// It carries the inputs that are otherwise frame/in-memory only: `generation` +// (the allocation generation of the incarnation the record restores, which +// the sidecar reports to the Hub and stamps on what the deployment sends), +// `sources` (each step's ordered inference-source failover chain, threaded to +// the child via the spawn env and durable nowhere else), `sessionId` (inference-event +// correlation), `hubPublicKey` (the head's deploy-pack / inbound verification +// key), `approvedWireHash` (the hub-approved wire hash the deploy frame +// carried, so a restore re-spawn feeds the child the same re-verify anchor +// without recomputing it), `lineage`, and -- for a source-ref deployment -- +// the `sourceRef` pin (source + frozen closure) a restore needs to +// re-materialize and re-evaluate the pinned code. The definition itself is +// re-materialized from that `sourceRef` closure on restore, and each step's +// grants live in its agent-state repo, so neither is duplicated here. + +import { mkdir, readdir, readFile } from "node:fs/promises"; +import { dirname, join as pathJoin, resolve } from "node:path"; + +import { type } from "arktype"; + +import { getLogger } from "@intx/log"; +import { syncDirectoryDurable } from "@intx/storage-isogit/node"; +import { + credentialAad, + type CredentialCipher, +} from "@intx/types/credential-cipher"; +import { InferenceSource } from "@intx/types/runtime"; +import { CredentialDelivery } from "@intx/types/credential-delivery"; +import { + DeploymentRetention, + Generation, + SourceRefPin, +} from "@intx/types/sidecar"; + +import { + removeFileAtomicDurable, + writeFileAtomicDurable, +} from "./atomic-write"; + +const logger = getLogger(["interchange", "sidecar", "workflow-run-record"]); + +export const WORKFLOW_RUN_RECORD_FILENAME = "deployment.json"; + +/** True for a `node:fs` rejection whose `code` is `ENOENT`. */ +function isENOENT(cause: unknown): boolean { + return ( + cause instanceof Error && + "code" in cause && + (cause as { code: unknown }).code === "ENOENT" + ); +} + +// The base fields every run record carries, spread into the full schema below. +// `version` is 2 for the unified credential format: `sources`/`bodySources` are +// non-secret config carrying a `credentialId` per source, and the run's secret +// material lives once in `credentials` (the delivery cell) with each material's +// secret sealed under the sidecar cipher. A pre-unification record (inline +// per-source secrets) does not satisfy this schema and is soft-skipped at the +// scan boundary, so the sidecar no longer reports its deployment and the Hub +// fails it. +const workflowRunRecordBase = { + version: "1 | 2", + agentAddress: "string > 0", + // A record without it cannot say which incarnation it restores, so it + // fails validation and is skipped rather than guessed. + generation: Generation, + definitionId: "string > 0", + // Top-level per-step inference sources. Non-secret config only: each source + // carries a `credentialId` referencing an entry in `credentials`, no inline + // secret, so this rides in the clear. + sources: { + "[string]": InferenceSource.array().atLeastLength(1), + }, + // Per spawned-body inference-source config, keyed by the body's definition id + // (an onTrigger section or a childWorkflow child; a nested loop body folds its + // step sources into its container's table, so it is not a separate entry). + // Non-secret, like `sources`: each carries a `credentialId` into `credentials`. + // Optional: a deployment with no bodies carries none. + "bodySources?": { + "[string]": { + "[string]": InferenceSource.array().atLeastLength(1), + }, + }, + // The run's credential-material cell: the ONE at-rest home for every secret, + // inference and binding alike. `bindings` and each material's `credentialId`/ + // `providerKey`/`origin` ride in the clear; only each material's `secret` is + // sealed under the sidecar cipher (see `transformDeliveryMaterials`). + // Optional: a deployment that binds no credentials and whose sources need + // none carries none. Restored + re-delivered to the child on the pre-trigger + // barrier. + "credentials?": CredentialDelivery, + "sessionId?": "string > 0", + "hubPublicKey?": "string > 0", + // A deployment that no longer runs keeps its record until the Hub undeploys + // it, marked so a restart reports it instead of spawning it: `stopped`, with + // the error that ended it unless the Hub stopped it, or `tearing-down` when + // its undeploy did not finish. + "state?": "'stopped' | 'tearing-down'", + "retention?": DeploymentRetention, + "error?": "string > 0", +} as const; + +/** + * The on-disk deployment record. `version` guards the schema shape so a future + * reader can reject or migrate a stale record rather than parse it blindly. + * Validated at read time (the boot scan) at the trust boundary. + * + * Source-ref is the only deploy lineage, so the record REQUIRES a `sourceRef` + * pin (its co-required source + frozen closure, so a restore can re-materialize + * the pinned closure) AND `approvedWireHash` (so the restored child re-verifies + * the evaluated closure against the hub-approved pin rather than against a + * sidecar recompute of the inert projection -- the latter would collapse the + * out-of-band-pin property the barrier exists for). A record missing either -- + * including a legacy live-authored record written before this collapse -- fails + * validation at the scan boundary and is soft-skipped as corruption, so the + * restore loop needs no bespoke source-ref guard. + */ +export const WorkflowRunRecord = type({ + ...workflowRunRecordBase, + lineage: "'source-ref'", + // The hub-approved wire hash the restored child re-verifies the evaluated + // closure against. + approvedWireHash: "string > 0", + // The source-ref pin a restore re-runs `applyFrozenWorkflowClosure` with: + // `source` names the registry the definition package is published to (its + // token is resolved from the sidecar's env at apply time, so no secret is + // persisted here) and `closure` is the frozen dependency tree (concrete + // versions + integrity SRIs) -- plain strings, no secrets. Both rode the + // signed frame and co-travel (see `SourceRefPin`). + sourceRef: SourceRefPin, +}).narrow( + (record, ctx) => + record.retention === undefined || + record.state !== undefined || + ctx.mustBe("stopped before deciding retention"), +); +export type WorkflowRunRecord = typeof WorkflowRunRecord.infer; + +function recordPath(dataDir: string, runId: string): string { + return pathJoin( + dataDir, + "workflow-runs", + runId, + WORKFLOW_RUN_RECORD_FILENAME, + ); +} + +// Source updates, lifecycle marks and deletion share the record's read/write +// boundary. Atomic rename alone cannot protect a read-modify-write from a +// concurrent writer. Different deployments keep independent queues. +const recordOperations = new Map>(); + +/** + * Serialize an operation with this process's mutations of a deployment record. + * The callback receives the record's absolute path and must not acquire the + * same record queue again. + */ +export async function withWorkflowRunRecord( + dataDir: string, + runId: string, + operation: (path: string) => Promise, +): Promise { + const path = resolve(recordPath(dataDir, runId)); + const previous = recordOperations.get(path) ?? Promise.resolve(); + const result = previous.then(() => operation(path)); + const tail = result.then( + () => undefined, + () => undefined, + ); + recordOperations.set(path, tail); + try { + return await result; + } finally { + if (recordOperations.get(path) === tail) recordOperations.delete(path); + } +} + +// The AAD column binding a credential's sealed secret to its id, so a ciphertext +// cannot be swapped between credentials (or between runs) and still decrypt. +function credentialSecretColumn(credentialId: string): string { + return `credential:${credentialId}:secret`; +} + +// Transform each credential material's `secret` in the delivery under +// `transform` (encrypt on write, decrypt on scan), binding each to its run and +// credential. The non-secret fields -- each material's `credentialId`/ +// `providerKey`/`origin` and the whole `bindings` array -- ride in the clear. +// Returns a fresh delivery so the caller's in-memory record is not mutated. A +// decrypt failure (a rotated/wrong key or a tampered blob) throws, which the +// scan caller treats as record corruption and soft-skips the whole run. +async function transformDeliveryMaterials( + delivery: CredentialDelivery, + runId: string, + transform: (secret: string, aad: string) => Promise, +): Promise { + return { + bindings: delivery.bindings, + materials: await Promise.all( + delivery.materials.map(async (material) => ({ + ...material, + secret: await transform( + material.secret, + credentialAad(runId, credentialSecretColumn(material.credentialId)), + ), + })), + ), + }; +} + +/** + * Persist the initial run record after its slug is claimed and before the + * child is spawned, so a crash mid-spawn leaves a record the boot scan + * re-drives. Idempotent: it overwrites any existing record for the same run. + * + * Every credential secret in the run's `credentials` cell -- inference and + * binding alike -- is sealed under the sidecar `cipher` before it touches disk, + * so the record carries ciphertext (version 2). The `sources`/`bodySources` + * config is non-secret (each references a credential by id) and rides in the + * clear. + */ +export async function writeWorkflowRunRecord( + dataDir: string, + runId: string, + record: WorkflowRunRecord, + cipher: CredentialCipher, +): Promise { + return withWorkflowRunRecord(dataDir, runId, async (path) => { + await mkdir(dirname(path), { recursive: true }); + const sealed: WorkflowRunRecord = { + ...record, + version: 2, + ...(record.credentials !== undefined + ? { + credentials: await transformDeliveryMaterials( + record.credentials, + runId, + (secret, aad) => cipher.encrypt(secret, aad), + ), + } + : {}), + }; + // Atomic + durable: this is the sole restore source for the run's credential + // material, so an interrupted write must never expose a torn record the + // boot scan would then skip. Owner-only (0o600): the sealed secrets are + // ciphertext, but the record still names each source's provider/baseURL and + // the deployment's identity, so it stays off a shared host's world-readable + // set, matching the private-key writes elsewhere on the sidecar. + await writeFileAtomicDurable(path, JSON.stringify(sealed, null, 2), { + mode: 0o600, + requireDirectorySync: true, + }); + // Sync the containing links too: syncing the run directory alone does + // not persist its creation inside workflow-runs or that directory's link. + await syncDirectoryDurable(pathJoin(dataDir, "workflow-runs")); + await syncDirectoryDurable(dataDir); + }); +} + +async function readWorkflowRunRecord( + path: string, + runId: string, +): Promise { + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (cause) { + if (isENOENT(cause)) return null; + throw cause; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`not updating workflow-runs/${runId}: ${WORKFLOW_RUN_RECORD_FILENAME} is not valid JSON: ${reason}`; + return null; + } + const checked = WorkflowRunRecord(parsed); + if (checked instanceof type.errors) { + logger.warn`not updating workflow-runs/${runId}: ${WORKFLOW_RUN_RECORD_FILENAME} failed validation: ${checked.summary}`; + return null; + } + return checked; +} + +/** Update sources without replacing lifecycle fields or resealing credentials. */ +export async function updateWorkflowRunRecordSources( + dataDir: string, + runId: string, + generation: number, + sources: WorkflowRunRecord["sources"], +): Promise { + return withWorkflowRunRecord(dataDir, runId, async (path) => { + const record = await readWorkflowRunRecord(path, runId); + if (record === null) { + throw new Error( + `Cannot update sources for ${runId}: its deployment record is missing or invalid`, + ); + } + if (record.generation !== generation) { + throw new Error( + `Cannot update sources for ${runId}: its deployment generation changed`, + ); + } + await writeFileAtomicDurable( + path, + JSON.stringify({ ...record, sources }, null, 2), + { mode: 0o600 }, + ); + }); +} + +/** + * Mark a run's record with what became of its deployment, keeping the rest of + * the record as it is, so a restart reports the deployment instead of spawning + * it. Returns false when the run has no record the boot scan would accept: a + * restart neither spawns nor reports such a record, so there is nothing to + * mark, and refusing it would fail the undeploy that removes it. + */ +export async function markWorkflowRunRecord( + dataDir: string, + runId: string, + mark: { + readonly state: "stopped" | "tearing-down"; + readonly error?: string; + readonly retention?: DeploymentRetention; + }, +): Promise { + return withWorkflowRunRecord(dataDir, runId, async (path) => { + const checked = await readWorkflowRunRecord(path, runId); + if (checked === null) return false; + const { error: _replaced, ...record } = checked; + await writeFileAtomicDurable( + path, + JSON.stringify( + { + ...record, + state: mark.state, + ...(mark.retention !== undefined + ? { retention: mark.retention } + : {}), + ...(mark.error !== undefined ? { error: mark.error } : {}), + }, + null, + 2, + ), + { mode: 0o600, requireDirectorySync: true }, + ); + return true; + }); +} + +/** + * Remove a run record. Called on a soft-failed deploy so a never-completed + * run is not restored on the next boot; undeploy removes the record together + * with the run repository. A missing record is not an error, but its removal + * must be durable before the caller releases the deployment's reservation. + */ +export async function deleteWorkflowRunRecord( + dataDir: string, + runId: string, +): Promise { + return withWorkflowRunRecord(dataDir, runId, async (path) => { + await removeFileAtomicDurable(path, { requireDirectorySync: true }); + }); +} + +/** A restorable run: its directory-derived id plus the validated record. */ +export interface ScannedWorkflowRun { + /** The `workflow-runs/` directory name the record was found under. */ + runId: string; + record: WorkflowRunRecord; +} + +/** + * Enumerate the persisted run records under `workflow-runs/` so a boot-time + * restore can re-establish each run. Soft-fails per record: a missing + * `deployment.json`, unparseable JSON, or a record that fails schema + * validation is logged and skipped rather than wedging the whole boot. I/O + * and directory-sync failures abort the inventory: skipping an unconfirmed + * record write or deletion would make its capacity unsafe to reuse. + * An absent `workflow-runs/` directory is legitimate on first boot and yields + * an empty list. Before returning, sync the observed inventory: a previous process + * may have changed a record or removed a directory without completing its + * directory sync, and the new process must not reuse that capacity yet. + * + * The returned `runId` is the directory name; the caller cross-checks it + * against the record's own address before trusting it. + */ +export async function scanWorkflowRunRecords( + dataDir: string, + cipher: CredentialCipher, +): Promise { + const runsDir = pathJoin(dataDir, "workflow-runs"); + let entries; + try { + entries = await readdir(runsDir, { withFileTypes: true }); + } catch (cause) { + if (isENOENT(cause)) { + await syncDirectoryDurable(dataDir); + return []; + } + throw cause; + } + + const scanned: ScannedWorkflowRun[] = []; + for (const entry of entries.sort((a, b) => + a.name.localeCompare(b.name, "en"), + )) { + if (!entry.isDirectory()) continue; + const runId = entry.name; + const path = recordPath(dataDir, runId); + // This also commits an absent record's unlink before the scan skips it. + await syncDirectoryDurable(dirname(path)); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (cause) { + // A run directory with no record: a crash between mkdir and the record + // write, or a run whose record was already reclaimed. Nothing to + // restore from -- skip. + if (isENOENT(cause)) { + logger.warn`skipping workflow-runs/${runId}: no ${WORKFLOW_RUN_RECORD_FILENAME} to restore from`; + continue; + } + throw cause; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`skipping workflow-runs/${runId}: ${WORKFLOW_RUN_RECORD_FILENAME} is not valid JSON: ${reason}`; + continue; + } + + const record = WorkflowRunRecord(parsed); + if (record instanceof type.errors) { + logger.warn`skipping workflow-runs/${runId}: ${WORKFLOW_RUN_RECORD_FILENAME} failed validation: ${record.summary}`; + continue; + } + // A version-2 record seals every credential secret in its `credentials` + // cell; unseal them under the sidecar cipher for the restored run. A decrypt + // failure -- a rotated/wrong key or a tampered blob -- is treated as + // corruption: log and soft-skip the WHOLE run so one undecryptable run does + // not wedge the boot scan (the deployment is left unrestored and + // unreported, so the Hub fails it, as it does for any other corrupt + // record). A record with no `credentials` cell (a deployment that binds + // none) needs no unseal. + let restored = record; + if (record.version === 2 && record.credentials !== undefined) { + try { + restored = { + ...record, + credentials: await transformDeliveryMaterials( + record.credentials, + runId, + (secret, aad) => cipher.decrypt(secret, aad), + ), + }; + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`skipping workflow-runs/${runId}: sealed credential decrypt failed: ${reason}`; + continue; + } + } + scanned.push({ runId, record: restored }); + } + await syncDirectoryDurable(runsDir); + await syncDirectoryDurable(dataDir); + return scanned; +} diff --git a/vendor/intx/workflow-host/src/index.ts b/vendor/intx/workflow-host/src/index.ts index 8e30d2a718..1182b9c89c 100644 --- a/vendor/intx/workflow-host/src/index.ts +++ b/vendor/intx/workflow-host/src/index.ts @@ -37,7 +37,6 @@ export { export { createInMemorySpawnChild, createInMemorySpawnSuspendableChild, - type ChildTerminalStatus, type RunChildWorkflow, type RunSuspendableChild, } from "./adapters/spawn-child"; @@ -116,7 +115,6 @@ export { export { ControlPayload, DEFAULT_EVENT_BUFFER_LIMIT, - EventPayload, FrameEnvelope, IPC_CRYPTO, MailboxNotifyHeaders, @@ -143,15 +141,17 @@ export { type EventChannelReceiverOpts, type EventChannelSender, type EventChannelSenderOpts, + type EventPayload, type FrameReader, type FrameWriter, type NdjsonReader, type NdjsonWriter, + type ReceivedEvent, } from "./ipc"; export { EVENT_CHANNEL_FD, - createChildMailboxReader, + createChildMailboxCallBridge, createChildMailboxMutationBridge, createChildOutboundMailBridge, createChildSubstrateWriteBridge, @@ -164,8 +164,8 @@ export { parseSpawnTimeEnv, runWorkflowChild, runWorkflowChildFromProcessEnv, + type ChildMailboxCallBridge, type ChildMailboxMutationBridge, - type ChildMailboxReader, type ChildOutboundMailBridge, type ChildStepInvoker, type ChildSubstrateWriteBridge, @@ -206,3 +206,14 @@ export { type SignalChannelHandle, type SignalChannelOpts, } from "./seams/index"; +export { + createDurableConversationRegistry, + createDurableConversationStore, + reconstructDurableConversation, + type DurableConversationRegistry, + type DurableConversationRegistryOpts, + type DurableConversationStore, + type DurableConversationStoreOpts, + type ReconstructedConversation, +} from "./conversation-state"; +export { readRunGrants, runGrantsPath } from "./run-grants"; diff --git a/vendor/intx/workflow-host/src/ipc/control-channel.ts b/vendor/intx/workflow-host/src/ipc/control-channel.ts index e37a12a15d..0451e96db8 100644 --- a/vendor/intx/workflow-host/src/ipc/control-channel.ts +++ b/vendor/intx/workflow-host/src/ipc/control-channel.ts @@ -1,881 +1,26 @@ -// Control channel: NDJSON over stdio, Ed25519-signed per direction. -// -// Two Ed25519 keypairs flow per spawn: -// - Supervisor's keypair. The supervisor holds the private half and -// signs every downstream (supervisor->child) frame. The matching -// public half is passed to the child in spawn-time env -// (`HOST_PUBKEY`) and the child verifies downstream frames -// against it. The supervisor's PRIVATE KEY NEVER LEAVES THE -// SUPERVISOR'S ADDRESS SPACE. -// - Child's keypair. The child mints it at startup, holds the -// private half in its own address space, and signs every upstream -// (child->supervisor) frame. The matching public half rides as -// `childPublicKey` on the upstream `ready` frame's payload; the -// supervisor extracts it on receive and uses it to verify -// subsequent upstream frames. The CHILD'S PRIVATE KEY NEVER -// LEAVES THE CHILD'S ADDRESS SPACE. -// -// Upstream `ready` bootstraps the supervisor's view of the child's -// public key. The supervisor's receiver opens in bootstrap mode: -// the first frame's envelope is parsed structurally so the -// supervisor can extract `childPublicKey` from the payload, then the -// signature is verified against that key. Subsequent upstream frames -// verify against the same key. A child-signed frame whose claimed -// `childPublicKey` does not match the bootstrap value (or any -// non-`ready` first frame) crashes the receiver. -// -// Wire format: one signed envelope per line. Each line is the JSON -// serialization of `{ envelope: { seq, channelId, payload }, sig: -// }`. The signature covers the canonical bytes of the -// envelope sub-object (see `envelope.ts`). -// -// Payload union: every legal payload's discriminator lives in this -// module. The mixing-the-two-channels failure mode -- a "control" -// frame whose payload is structurally an InferenceEvent -- is -// prevented at the type level by the disjoint discriminated union. +// Aggregate control-channel API and combined payload schema for inspection. +// Process-specific callers import control-sender or control-receiver directly; +// importing this module constructs the full payload union. -import { type } from "arktype"; - -import { hexDecode, hexEncode } from "@intx/types"; -import { - BoundedApprovalSnapshot, - ControlParkKind, - InferenceSource, - InterchangeType, -} from "@intx/types/runtime"; -import { CredentialDelivery } from "@intx/types/sidecar"; - -import { - decodeEnvelope, - encodeEnvelope, - FrameEnvelope, - SignedEnvelope, -} from "./envelope"; -import { signEd25519, verifyEd25519 } from "./crypto"; - -/** - * Wire-shape of one per-step credentials entry the supervisor pushes - * inside a `grants-updated` frame. Mirrors `CredentialsSnapshotStep` - * in `supervisor/credentials.ts` -- duplicated here as an arktype - * validator so the control-channel module stays free of a - * compile-time import on the supervisor module (the IPC module sits - * underneath the supervisor and child modules in the dependency - * graph). The contentHash pins the per-step grants so the child can - * detect a stale push and ignore an out-of-order one. - */ -export const CredentialsSnapshotStepPayload = type({ - stepId: "string", - address: "string", - grants: "unknown[]", - contentHash: "string", -}); - -export const CredentialsSnapshotPayload = type({ - steps: CredentialsSnapshotStepPayload.array(), -}); - -/** - * Wire shape of a `sources-updated` frame's `data`: the full ordered - * inference-source failover chain plus the default source id. Carried - * inline like the grants snapshot -- a single-producer, single-consumer - * supervisor->child push, so a substrate round-trip would only add - * latency. No per-source hash rides along; a source list is flat, with no - * per-item pin for a receiver to cross-check. - * - * The `narrow` pins two frame-structural invariants at this boundary so - * every consumer can trust them without re-checking: source ids are - * unique, and the first element is the default source. The head-is-default - * rule is what keeps the two rotation paths in agreement -- a warm agent's - * `setSources` activates the matched default index, while a cold rebuild - * pins element 0 -- so they pick the same active source only when the - * default is the head. - */ -export const SourcesUpdatedData = type({ - sources: InferenceSource.array().atLeastLength(1), - defaultSource: "string > 0", -}).narrow((data, ctx) => { - const seen = new Set(); - for (const source of data.sources) { - if (seen.has(source.id)) { - return ctx.mustBe( - `a source list with unique ids; "${source.id}" appears more than once`, - ); - } - seen.add(source.id); - } - const head = data.sources[0]; - if (head === undefined || head.id !== data.defaultSource) { - return ctx.mustBe( - "a source list whose first element is the default source", - ); - } - return true; -}); - -/** - * Wire projection of an attachment on an outbound mail message. The - * runtime `MessageAttachment.data` is raw bytes; the NDJSON control - * channel is text, so the bytes ride base64-encoded under `dataBase64`. - * The child encodes on send; the supervisor decodes before handing the - * `OutboundMessage` to the host transport. - */ -export const OutboundAttachmentPayload = type({ - name: "string", - contentType: "string", - dataBase64: "string", -}); - -/** - * Wire projection of `@intx/types/runtime`'s `OutboundMessage`. Mirrors - * that type field-for-field with two adjustments for the NDJSON wire: - * attachment bytes are base64 strings (see `OutboundAttachmentPayload`), - * and every optional field is spelled with the `"?"` suffix so an - * absent field round-trips as absent rather than `null`. The supervisor - * reconstructs the runtime `OutboundMessage` from this shape before - * invoking `MailBusBindings.sendOutbound`. - * - * Duplicated here as an arktype validator (rather than importing the - * TypeScript `OutboundMessage` type) so the IPC module validates the - * child-supplied payload at the wire boundary -- the child is a separate - * process and its frames are untrusted input the receiver must parse. - */ -export const OutboundMessagePayload = type({ - to: "string | string[]", - "cc?": "string | string[]", - "subject?": "string", - type: InterchangeType, - "content?": "string", - "payload?": "Record", - "summary?": "string", - "attachments?": OutboundAttachmentPayload.array(), - "inReplyTo?": "string", - "references?": "string[]", - "correlationId?": "string", - "sessionId?": "string", - "tenantId?": "string", -}); - -export type OutboundMessagePayload = typeof OutboundMessagePayload.infer; - -/** - * Wire shape of the parsed `MessageHeaders` the supervisor rides inline on a - * `mailbox.notify` frame. Mirrors `@intx/types/runtime`'s `MessageHeaders` - * field-for-field so a child watcher gets the arrived message's envelope for - * its `exists` `MailboxEvent` without a substrate round-trip. Only `to` is - * required; the rest are optional, matching the runtime type. - * - * Duplicated here as an arktype validator (rather than importing the TypeScript - * `MessageHeaders` type) so the IPC module validates the header block at the - * wire boundary, exactly as `OutboundMessagePayload` does for outbound mail. - */ -export const MailboxNotifyHeaders = type({ - "from?": "string", - to: "string[]", - "cc?": "string[]", - "date?": "string", - "messageId?": "string", - "inReplyTo?": "string", - "references?": "string[]", - "subject?": "string", - "listId?": "string", - "interchangeType?": InterchangeType, - "interchangeCorrelationId?": "string", - "interchangeTenantId?": "string", - "interchangeAgentId?": "string", - "interchangeSessionId?": "string", - "interchangeOfferingId?": "string", - "interchangeSchemaVersion?": "string", - "traceparent?": "string", - "tracestate?": "string", -}); - -export type MailboxNotifyHeaders = typeof MailboxNotifyHeaders.infer; - -/** - * Discriminated union of every control-channel payload kind. The - * `type` discriminator namespaces the control-plane vocabulary so a - * future addition (e.g. `connector-bind`) lands by extending this - * union and not by widening the envelope shape. Inference events - * NEVER appear here; they ride the event channel. - */ -export const ControlPayload = type.or( - { - type: "'trigger.fire'", - data: { - runId: "string", - messageId: "string", - receivedAt: "number", - // The run's inbound-mail input, resolved by the supervisor (the sole - // mail owner) before the frame: a decoded `Mail` (headers plus part - // descriptors that reference the part bytes committed to the workflow-run - // substrate). The child hands this straight to the runtime as the trigger - // payload. Refs, not raw mail bytes, ride here; the committed part files - // stay in the substrate. Typed `unknown` -- `Mail` is a nested structural - // type validated at the consumption boundary by `isMail`. - payload: "unknown", - }, - }, - { - type: "'signal.deliver'", - data: { - runId: "string", - signalName: "string", - signalId: "string", - // The resume decision in FINAL form -- the child commits it as the - // SignalReceived payload verbatim. Each sender owns any - // provenance-specific preparation BEFORE this frame: the dispatch loop - // resolves an inbound mail to a decoded `Mail` (headers plus part - // references, like the turn-1 trigger), while `deliverSignal` ships a - // structured signal payload unchanged -- so this field stays - // polymorphic. Do NOT ship raw inbound mail bytes through here. - payload: "unknown", - }, - }, - { - type: "'drain'", - data: { - deadlineMs: "number", - }, - }, - { - type: "'shutdown'", - data: { - reason: "string", - }, - }, - { - type: "'grants-updated'", - data: { - /** - * Full credentialsSnapshot the supervisor assembled. The child - * replaces its in-memory snapshot wholesale on receive so the - * authorize closure binds to the new per-step grants on the - * next step invocation. Carried inline rather than by reference - * because the snapshot is per-step grants payload -- the - * supervisor is the only producer and the child is the only - * consumer, so the substrate round-trip would just add latency. - */ - snapshot: CredentialsSnapshotPayload, - /** - * Per-step content hashes the supervisor expects the snapshot - * to pin to. Surfaced separately so receivers can cheap-compare - * a push against the snapshot they already have without rehashing - * each step's grants. Optional; the receiver does not require it - * but uses it for the staleness cross-check when present. - */ - "stepHashes?": "Record", - }, - }, - { - type: "'sources-updated'", - data: SourcesUpdatedData, - }, - { - // Refreshed credential material for the deployment's inference sources and - // tools. The child MERGES this into its in-memory cell (see - // `mergeCredentialDelivery`): `delivery.materials` upsert by credentialId - // and `delivery.bindings` upsert by (consumer, handle); `revoke` names - // credentialIds to drop, and dropping one drops every binding referencing - // it. Merge rather than wholesale-replace because the cell has several - // independently-scoped producers (the deploy frame, an inference rotation, - // a tool-grant push), each carrying only its own slice -- a swap would let - // one evict another's credentials. Revocation is therefore explicit, never - // by omission. Carried inline like the grants and sources snapshots. The - // secret rides this frame and the in-memory cell only; it is never - // persisted. - type: "'credentials-updated'", - data: { - delivery: CredentialDelivery, - "revoke?": "string[]", - }, - }, - { - type: "'ready'", - data: { - childPid: "number", - /** - * Hex-encoded Ed25519 public key the child minted at startup. - * The supervisor extracts this on receive and uses it to verify - * every subsequent upstream control frame's signature. The - * child's private key never leaves the child's address space. - */ - childPublicKey: "string", - }, - }, - { - // Child-initiated request to recycle the workflow-process. The - // child emits this when its own self-check decides it needs to be - // recycled (an internal consistency error it can't recover from, - // a watchdog tripping); the supervisor receives it on its - // upstream control-channel reader and funnels it into the same - // `triggerRecycle` code path the operator and policy origins use. - // The `reason` rides verbatim into the supervisor's reason field; - // the supervisor does not interpret it beyond logging and - // attaching it to the recycle attempt. - type: "'recycle.request'", - data: { - reason: "string", - }, - }, - { - // Child-initiated `writeTreePreservingPrefix` request. The child - // does not hold a substrate write authority for the workflow-run - // repo (single-writer at the ref tip belongs to the supervisor); - // its workflow-run substrate proxy forwards every write through - // this frame. The supervisor receives the request, runs its own - // wrapped `writeTreePreservingPrefix`, and reaches back to the - // child for the merge bytes via `substrate.merge.request` so the - // child's merge closure (which knows about seq computation, - // duplicate detection, etc.) keeps producing the prospective tree. - // The supervisor resolves the child's awaiter with - // `substrate.write.response`. - type: "'substrate.write.request'", - data: { - requestId: "string > 0", - repoId: { - kind: "string", - id: "string > 0", - }, - ref: "string > 0", - preservePrefix: "string > 0", - message: "string > 0", - }, - }, - { - // Supervisor-initiated request for the child's merge bytes. Fired - // from inside the supervisor's `writeTreePreservingPrefix` merge - // callback while the per-repo lock is held; the child receives the - // existing prefix entries (base64-encoded bytes), invokes its merge - // closure, and replies with the prospective tree on - // `substrate.merge.response`. Carrying the entries inline preserves - // the lock window: the supervisor blocks inside the merge callback - // until the response lands. - type: "'substrate.merge.request'", - data: { - requestId: "string > 0", - existing: type({ - path: "string > 0", - contentBase64: "string", - }).array(), - }, - }, - { - // Child's merge result. `requestId` correlates with the - // `substrate.write.request` that started the write; the supervisor - // resumes its merge callback with the supplied entries (or - // propagates the structured failure). - type: "'substrate.merge.response'", - data: { - requestId: "string > 0", - result: type( - { - ok: "true", - files: type({ - path: "string > 0", - contentBase64: "string", - }).array(), - }, - "|", - { - ok: "false", - reason: "string > 0", - }, - ), - }, - }, - { - // Supervisor's terminal reply to a child's `substrate.write.request`. - // The `requestId` echoes the child's allocated correlation id so - // the child's pending-id map resolves the awaiter. A successful - // write surfaces `commitSha`; the child's proxy returns that to its - // caller. A failed write (substrate rejection, validatePush - // violation, the supervisor's pack-push wrap's downstream - // `HubLink.pushWorkflowRunPack` rejection) surfaces a structured - // `{ ok: false, reason }` the child's proxy rethrows. - type: "'substrate.write.response'", - data: { - requestId: "string > 0", - result: type( - { - ok: "true", - commitSha: "string > 0", - }, - "|", - { - ok: "false", - reason: "string > 0", - }, - ), - }, - }, - { - // Child-initiated outbound-mail request (OUTBOUND half of mailbox - // ownership, §3a). The workflow-process child never holds the - // agent's signing key and never calls `transport.send` itself. When - // a step agent produces a reply or invokes a mail-send tool, the - // child forwards the structured outbound message plus the sender - // (agent) address up over the control channel; the supervisor - // performs the actual signed send through the host's real transport - // (`MailBusBindings.sendOutbound`), which signs with the sender's - // `CryptoProvider` exactly as the in-process path does. The - // supervisor is the sole mail owner and the only process that can - // emit signed mail on the agent's behalf. - // - // `requestId` correlates the supervisor's `outbound.result` reply so - // the child's mail-tool `send()` resolves with the real - // `SendReceipt` (or rejects with the supervisor's structured - // failure). The message is carried as a JSON-projected - // `OutboundMessage`; attachment bytes ride base64-encoded so the - // NDJSON wire stays text-safe. - type: "'outbound.message'", - data: { - requestId: "string > 0", - senderAddress: "string > 0", - "mailbox?": "string", - message: OutboundMessagePayload, - }, - }, - { - // Supervisor's terminal reply to a child's `outbound.message`. The - // `requestId` echoes the child's correlation id so the child's - // pending mail-tool awaiter resolves. A successful send surfaces the - // `SendReceipt` (messageId + status); a failed send (unregistered - // sender, signing failure, transport rejection) surfaces a - // structured `{ ok: false, reason }` the child's transport rethrows - // so the mail-tool call fails loudly rather than dropping the send. - type: "'outbound.result'", - data: { - requestId: "string > 0", - result: type( - { - ok: "true", - messageId: "string > 0", - status: "'delivered' | 'queued'", - }, - "|", - { - ok: "false", - reason: "string > 0", - }, - ), - }, - }, - { - // Child-initiated terminal-run notification. The workflow-process - // child emits this when one of its runs reaches a terminal phase - // (`RunCompleted`, `RunFailed`, `RunCancelled`) so the supervisor's - // dispatch loop and drain accumulators can settle without re-reading - // the workflow-run substrate from the supervisor process. The child - // commits the terminal event to its own substrate through the - // workflow-run pack-push pipeline; this frame is the peer-channel - // notification that mirrors the commit so the supervisor's - // in-process consumers do not have to round-trip the substrate. - // - // The `seq` mirrors the on-disk EventBase.seq the child assigned at - // commit time. The supervisor does not authoritatively verify the - // commit landed -- the pack-push response covers that contract -- - // but the field is carried so a downstream consumer can correlate - // the notification with the substrate blob. - type: "'terminal.event'", - data: { - runId: "string > 0", - seq: "number >= 0", - kind: "'RunCompleted' | 'RunFailed' | 'RunCancelled'", - at: "string > 0", - "error?": { - message: "string", - }, - }, - }, - { - // Child-initiated control-plane suspension notification. The - // workflow-process child emits this when a workflow agent step parks - // on a reserved `signalName(correlationId)` channel (`env.onPark`), - // so the supervisor can register the correlation out-of-band before - // the parked run can be resumed. The supervisor stamps the - // deployment identity it owns (`runId` + `agentAddress`) and - // forwards a `signal.correlation.register` frame to the hub, which - // co-writes the run's routing + approval rows. Mirrors - // `terminal.event`: a peer-channel notification the supervisor fans - // out, distinct from the substrate commit the run also produces. - // - // `signalName` is deliberately NOT carried: it is a pure function of - // `correlationId` (`signalName(correlationId)`), recomputed by every - // consumer that needs it, so the two cannot drift. - type: "'park.notify'", - data: { - runId: "string > 0", - correlationId: "string > 0", - parkKind: ControlParkKind, - // Approver-facing snapshot of the parked tool call, size-capped at this - // process boundary. Optional: only an ask-rail suspension carries one. - "snapshot?": BoundedApprovalSnapshot, - }, - }, - { - // Supervisor-initiated request: enumerate the child's currently-parked - // approval correlations. The supervisor fires this after a - // re-establishment (child respawn, or hub-link reconnect fanned out to - // this deployment) so it can re-register at the hub every correlation - // whose original `park.notify`-driven register may have been lost while - // the hub was down. Modeled on `substrate.merge.request`: a - // supervisor->child request the child answers on `parked-correlations. - // response`, correlated by `requestId`. Carries no filter -- one child - // owns one deployment, so the child enumerates everything parked and the - // supervisor re-emits all; the hub co-write is idempotent. - type: "'parked-correlations.request'", - data: { - requestId: "string > 0", - }, - }, - { - // Child's reply to `parked-correlations.request`. Each entry mirrors - // `park.notify`'s data -- the child-supplied half of a - // `SuspensionRegistration` the supervisor stamps its deployment identity - // onto -- so the supervisor's re-emit path shares one transform with the - // `park.notify` arm. Only reduced-state approval parks appear here: the - // child enumerates steps whose reduced phase is `awaiting-signal` on a - // control-plane `signalName(correlationId)` channel, each of which carries - // a durable snapshot by construction (a snapshot-less correlated suspend - // reduces to `failed`, not `awaiting-signal`). `snapshot` is therefore - // required, and size-capped at this process boundary like `park.notify`. - type: "'parked-correlations.response'", - data: { - requestId: "string > 0", - parked: type({ - runId: "string > 0", - correlationId: "string > 0", - parkKind: ControlParkKind, - // Snapshot is required for approval parks and absent for input parks. - "snapshot?": BoundedApprovalSnapshot, - }).array(), - }, - }, - { - // Child reports self-discovered runs after reconnect or recycle. - // The supervisor seeds its cohort tracking from these runIds so - // drain accumulators and dispatch routing account for runs the - // supervisor did not personally trigger.fire. - type: "'resumed.runs'", - data: { - runIds: type("string > 0").array(), - }, - }, - { - // Supervisor-to-child one-way notification that new mail landed in a - // deployment mailbox (INBOUND half of mailbox ownership, §3b). One-way - // like `grants-updated`/`sources-updated`: no correlation id, no response. - // The supervisor -- the sole mail owner -- commits the arrived message to - // the workflow-run substrate mailbox, then fires this frame so the child's - // warm-agent `watch`/`mail_wait` observes the arrival decoupled from the - // FIFO trigger dispatch that resolves a run's first input. `headers` rides - // inline so a watcher gets the `exists` `MailboxEvent`'s envelope without a - // substrate round-trip. The child reads the latest committed mailbox state - // regardless, so the frame carries no commit pin. - type: "'mailbox.notify'", - data: { - runId: "string > 0", - mailbox: "string > 0", - uid: "number >= 1", - headers: MailboxNotifyHeaders, - }, - }, - { - // Child-initiated mailbox-mutation request (INBOUND half of mailbox - // ownership, §3b). The supervisor is the sole writer to the - // workflow-run mailbox: a step agent reads its INBOX locally but - // every mutation -- flag writes and `expunge` -- routes up here so - // the supervisor applies it to its owned store. A child flushing the - // same ref would race the supervisor's in-memory mirror and break - // uid / modseq monotonicity. - // - // `data` is discriminated on `op`: an `addFlags` / `removeFlags` - // carries the target `uid` and the `flags` to change, so the wire - // boundary rejects a flag frame that omits them; an `expunge` sweeps - // every `\Deleted` message in the mailbox and the child constructs it - // with neither. `requestId` correlates the supervisor's - // `mailbox.mutate.response` reply. - type: "'mailbox.mutate.request'", - data: type( - { - requestId: "string > 0", - runId: "string > 0", - mailbox: "string > 0", - op: "'addFlags' | 'removeFlags'", - uid: "number >= 1", - flags: "string[]", - }, - "|", - { - requestId: "string > 0", - runId: "string > 0", - mailbox: "string > 0", - op: "'expunge'", - }, - ), - }, - { - // Supervisor's terminal reply to a child's `mailbox.mutate.request`. - // The `requestId` echoes the child's correlation id so the child's - // pending mail-tool awaiter resolves. The reply is sent only after - // the supervisor flushes the mutation, so the child's next committed - // read observes it -- the same flush-before-signal ordering - // `mailbox.notify` relies on. A successful `expunge` carries the - // `expungedUids` it swept so the agent tool can report the count; a - // flag write carries no operand echo. A failed mutation (unknown - // uid, substrate fault) surfaces a structured `{ ok: false, reason }` - // the child's bridge rethrows so the mail-tool call fails loudly. - type: "'mailbox.mutate.response'", - data: { - requestId: "string > 0", - result: type( - { - ok: "true", - "expungedUids?": "number[]", - }, - "|", - { - ok: "false", - reason: "string > 0", - }, - ), - }, - }, -); +import { createControlPayloadSchema } from "./control-payloads"; +export const ControlPayload = createControlPayloadSchema(); export type ControlPayload = typeof ControlPayload.infer; -export interface NdjsonWriter { - write(line: string): Promise | void; -} - -export interface NdjsonReader { - read(): AsyncIterableIterator; -} - -export interface ControlChannelSenderOpts { - privateKeySeed: Uint8Array; - channelId: string; - writer: NdjsonWriter; -} - -export interface ControlChannelSender { - send(payload: ControlPayload): Promise; - readonly seq: number; -} - -/** - * Construct the supervisor-side control-channel sender. The - * supervisor's Ed25519 seed lives in closure. The matching public - * key flows to the child through spawn-time env -- never the seed. - */ -export function createControlChannelSender( - opts: ControlChannelSenderOpts, -): ControlChannelSender { - let seq = 0; - // Serialize sends. Signing is async, so without a lock two concurrent - // callers could each assign seq, suspend on `signEd25519`, and resume in - // signature-resolution order — writing frames out of seq order, which the - // receiver rejects as a gap and crashes the channel. The promise chain - // makes each send await the previous send's completion before it assigns - // seq, signs, and writes, keeping that critical section atomic. - let tail: Promise = Promise.resolve(); - return { - get seq() { - return seq; - }, - send(payload: ControlPayload): Promise { - const previous = tail; - let release: () => void = () => undefined; - tail = new Promise((resolve) => { - release = resolve; - }); - return (async () => { - await previous; - try { - seq += 1; - const envelope: FrameEnvelope = { - seq, - channelId: opts.channelId, - payload, - }; - const envelopeBytes = encodeEnvelope(envelope); - const sig = await signEd25519(envelopeBytes, opts.privateKeySeed); - const signed: SignedEnvelope = { - envelope, - sig: hexEncode(sig), - }; - await opts.writer.write(JSON.stringify(signed) + "\n"); - } finally { - release(); - } - })(); - }, - }; -} - -export interface ControlChannelReceiverOpts { - /** - * Public key used to verify every inbound frame. When `Uint8Array` - * the value is fixed at construction time (the child's downstream - * receiver uses the supervisor's pubkey from `HOST_PUBKEY`). When - * `{ bootstrapFromReady: true }` the receiver opens in - * bootstrap mode: the first frame must be `ready` and must carry - * a `childPublicKey` hex-encoded Ed25519 public key in its payload. - * The receiver extracts the key, verifies the first frame's - * signature against it, then continues verifying subsequent frames - * against the same key. The supervisor's upstream receiver opens - * in bootstrap mode so the child can publish its own public key - * over the wire without the supervisor ever holding the matching - * private half. - */ - publicKey: Uint8Array | { bootstrapFromReady: true }; - channelId: string; - reader: NdjsonReader; - /** - * Invoked when any invariant is violated: signature failure, - * channelId mismatch, non-monotonic seq, malformed payload. The - * receiver's contract is to crash on any such violation. The - * caller wires this to a process-exit path; tests inject a - * recorder to assert on the failure mode. - */ - onCrash: (reason: string) => void; -} - -/** - * Construct the child-side control-channel receiver. Yields one - * verified, in-order `ControlPayload` per call. Any frame that - * fails verification, carries a non-current channelId, or arrives - * out of order calls `onCrash` and ends the iterator. - */ -export async function* receiveControlChannel( - opts: ControlChannelReceiverOpts, -): AsyncGenerator { - let highestSeq = 0; - let activePublicKey: Uint8Array | null = - opts.publicKey instanceof Uint8Array ? opts.publicKey : null; - const bootstrapping = activePublicKey === null; - for await (const line of opts.reader.read()) { - if (line.length === 0) continue; - - let raw: unknown; - try { - raw = JSON.parse(line); - } catch (cause) { - opts.onCrash( - `control channel received non-JSON line: ${errorMessage(cause)}`, - ); - return; - } - - const signed = SignedEnvelope(raw); - if (signed instanceof type.errors) { - opts.onCrash( - `control channel envelope failed validation: ${signed.summary}`, - ); - return; - } - - let envelopeBytes: Uint8Array; - try { - envelopeBytes = encodeEnvelope(signed.envelope); - } catch (cause) { - opts.onCrash( - `control channel envelope re-encode failed: ${errorMessage(cause)}`, - ); - return; - } - - let sigBytes: Uint8Array; - try { - sigBytes = hexDecode(signed.sig); - } catch (cause) { - opts.onCrash( - `control channel signature decode failed: ${errorMessage(cause)}`, - ); - return; - } - - if (activePublicKey === null) { - // Bootstrap mode: the first frame must be `ready`. Extract the - // child's public key from the payload, then verify the - // first frame's signature against it. The receiver crashes if - // the payload is not a `ready` frame or carries a malformed - // `childPublicKey`. - const candidate = ControlPayload(signed.envelope.payload); - if (candidate instanceof type.errors) { - opts.onCrash( - `control channel bootstrap payload failed validation: ${candidate.summary}`, - ); - return; - } - if (candidate.type !== "ready") { - opts.onCrash( - `control channel bootstrap expected a ready frame, got ${candidate.type}`, - ); - return; - } - let bootstrapKey: Uint8Array; - try { - bootstrapKey = hexDecode(candidate.data.childPublicKey); - } catch (cause) { - opts.onCrash( - `control channel bootstrap childPublicKey decode failed: ${errorMessage(cause)}`, - ); - return; - } - activePublicKey = bootstrapKey; - } - - const ok = await verifyEd25519(envelopeBytes, sigBytes, activePublicKey); - if (!ok) { - opts.onCrash( - `control channel signature did not verify (seq=${String(signed.envelope.seq)}, channelId=${signed.envelope.channelId}${bootstrapping ? "; bootstrap" : ""})`, - ); - return; - } - - if (signed.envelope.channelId !== opts.channelId) { - opts.onCrash( - `control channel channelId mismatch: expected ${opts.channelId}, got ${signed.envelope.channelId} at seq=${String(signed.envelope.seq)}`, - ); - return; - } - - if (signed.envelope.seq <= highestSeq) { - opts.onCrash( - `control channel out-of-order seq: expected > ${String(highestSeq)}, got ${String(signed.envelope.seq)}`, - ); - return; - } - if (signed.envelope.seq !== highestSeq + 1) { - opts.onCrash( - `control channel seq gap: expected ${String(highestSeq + 1)}, got ${String(signed.envelope.seq)}`, - ); - return; - } - highestSeq = signed.envelope.seq; - - const payload = ControlPayload(signed.envelope.payload); - if (payload instanceof type.errors) { - opts.onCrash( - `control channel payload failed validation: ${payload.summary}`, - ); - return; - } - - yield payload; - } -} - -function errorMessage(cause: unknown): string { - if (cause instanceof Error) return cause.message; - return String(cause); -} - -/** - * Re-export the envelope decoder for callers that need to inspect - * a control frame's envelope without going through the receiver - * iterator (testing harnesses that fuzz the wire format). - */ -export { decodeEnvelope }; +export { + SourcesUpdatedData, + OutboundAttachmentPayload, + OutboundMessagePayload, + MailboxNotifyHeaders, +} from "./control-payloads"; +export { + createControlChannelSender, + type ControlChannelSender, + type ControlChannelSenderOpts, + type NdjsonReader, + type NdjsonWriter, +} from "./control-sender"; +export { + receiveControlChannel, + type ControlChannelReceiverOpts, +} from "./control-receiver"; diff --git a/vendor/intx/workflow-host/src/ipc/control-payloads-cache-worker.ts b/vendor/intx/workflow-host/src/ipc/control-payloads-cache-worker.ts new file mode 100644 index 0000000000..58cf1c0cf7 --- /dev/null +++ b/vendor/intx/workflow-host/src/ipc/control-payloads-cache-worker.ts @@ -0,0 +1,47 @@ +// A fresh worker isolates both the payload cache and the ArkType module mock +// from tests that load the aggregate control-channel schema in the same worker. +import { mock } from "bun:test"; +import { parentPort } from "node:worker_threads"; +import * as arktype from "arktype"; + +const constructed: string[] = []; +const observedType = new Proxy(arktype.type, { + apply(target, thisArg: unknown, args: unknown[]) { + const definition = args[0]; + if ( + typeof definition === "object" && + definition !== null && + "type" in definition && + (definition.type === "'shutdown'" || definition.type === "'drain'") + ) { + constructed.push(definition.type); + } + const result: unknown = Reflect.apply(target, thisArg, args); + return result; + }, +}); +const observedArktype = { ...arktype, type: observedType }; +await mock.module("arktype", () => observedArktype); + +const { parseControlPayload } = await import("./control-payloads"); +const afterImport = [...constructed]; +parseControlPayload({ type: "unknown", data: {} }); +const afterUnknown = [...constructed]; +parseControlPayload({ type: "shutdown", data: { reason: "done" } }); +const afterFirst = [...constructed]; +parseControlPayload({ type: "shutdown", data: { reason: "again" } }); +const afterRepeat = [...constructed]; +parseControlPayload({ type: "drain", data: { deadlineMs: 100 } }); +const afterSecond = [...constructed]; +parseControlPayload({ type: "drain", data: { deadlineMs: 200 } }); + +if (!parentPort) throw new Error("Payload cache observation requires a worker"); +parentPort.postMessage({ + afterImport, + afterUnknown, + afterFirst, + afterRepeat, + afterSecond, + afterSecondRepeat: [...constructed], +}); +parentPort.close(); diff --git a/vendor/intx/workflow-host/src/ipc/control-payloads.ts b/vendor/intx/workflow-host/src/ipc/control-payloads.ts new file mode 100644 index 0000000000..4fd0b9ad0a --- /dev/null +++ b/vendor/intx/workflow-host/src/ipc/control-payloads.ts @@ -0,0 +1,1075 @@ +import { type } from "arktype"; +import { + BoundedApprovalSnapshot, + ControlParkKind, + InferenceSource, + InterchangeType, + MessageTransportCondition, + SignatureStatus, +} from "@intx/types/runtime-core"; +import { CredentialDelivery } from "@intx/types/credential-delivery"; + +/** + * Wire-shape of one per-step credentials entry the supervisor pushes + * inside a `grants-updated` frame. Mirrors `CredentialsSnapshotStep` + * in `supervisor/credentials.ts` -- duplicated here as an arktype + * validator so the control-channel module stays free of a + * compile-time import on the supervisor module (the IPC module sits + * underneath the supervisor and child modules in the dependency + * graph). The contentHash pins the per-step grants so the child can + * detect a stale push and ignore an out-of-order one. + */ +export const CredentialsSnapshotStepPayload = type({ + stepId: "string", + address: "string", + grants: "unknown[]", + contentHash: "string", +}); + +export const CredentialsSnapshotPayload = type({ + steps: CredentialsSnapshotStepPayload.array(), +}); + +/** + * Wire shape of a `sources-updated` frame's `data`: the full ordered + * inference-source failover chain plus the default source id. Carried + * inline like the grants snapshot -- a single-producer, single-consumer + * supervisor->child push, so a substrate round-trip would only add + * latency. No per-source hash rides along; a source list is flat, with no + * per-item pin for a receiver to cross-check. + * + * The `narrow` pins two frame-structural invariants at this boundary so + * every consumer can trust them without re-checking: source ids are + * unique, and the first element is the default source. The head-is-default + * rule is what keeps the two rotation paths in agreement -- a warm agent's + * `setSources` activates the matched default index, while a cold rebuild + * pins element 0 -- so they pick the same active source only when the + * default is the head. + */ +export const SourcesUpdatedData = type({ + sources: InferenceSource.array().atLeastLength(1), + defaultSource: "string > 0", +}).narrow((data, ctx) => { + const seen = new Set(); + for (const source of data.sources) { + if (seen.has(source.id)) { + return ctx.mustBe( + `a source list with unique ids; "${source.id}" appears more than once`, + ); + } + seen.add(source.id); + } + const head = data.sources[0]; + if (head === undefined || head.id !== data.defaultSource) { + return ctx.mustBe( + "a source list whose first element is the default source", + ); + } + return true; +}); + +/** + * Wire projection of an attachment on an outbound mail message. The + * runtime `MessageAttachment.data` is raw bytes; the NDJSON control + * channel is text, so the bytes ride base64-encoded under `dataBase64`. + * The child encodes on send; the supervisor decodes before handing the + * `OutboundMessage` to the host transport. + */ +export const OutboundAttachmentPayload = type({ + name: "string", + contentType: "string", + dataBase64: "string", +}); + +/** + * Wire projection of `@intx/types/runtime`'s `OutboundMessage`. Mirrors + * that type field-for-field with two adjustments for the NDJSON wire: + * attachment bytes are base64 strings (see `OutboundAttachmentPayload`), + * and every optional field is spelled with the `"?"` suffix so an + * absent field round-trips as absent rather than `null`. The supervisor + * reconstructs the runtime `OutboundMessage` from this shape before + * invoking `MailBusBindings.sendOutbound`. + * + * Duplicated here as an arktype validator (rather than importing the + * TypeScript `OutboundMessage` type) so the IPC module validates the + * child-supplied payload at the wire boundary -- the child is a separate + * process and its frames are untrusted input the receiver must parse. + */ +export const OutboundMessagePayload = type({ + to: "string | string[]", + "cc?": "string | string[]", + "subject?": "string", + type: InterchangeType, + "content?": "string", + "payload?": "Record", + "summary?": "string", + "attachments?": OutboundAttachmentPayload.array(), + "inReplyTo?": "string", + "references?": "string[]", + "correlationId?": "string", + "sessionId?": "string", + "tenantId?": "string", +}); + +export type OutboundMessagePayload = typeof OutboundMessagePayload.infer; + +/** + * Wire shape of the parsed `MessageHeaders` the supervisor rides inline on a + * `mailbox.notify` frame. Mirrors `@intx/types/runtime`'s `MessageHeaders` + * field-for-field so a child watcher gets the arrived message's envelope for + * its `exists` `MailboxEvent` without a substrate round-trip. Only `to` is + * required; the rest are optional, matching the runtime type. + * + * Duplicated here as an arktype validator (rather than importing the TypeScript + * `MessageHeaders` type) so the IPC module validates the header block at the + * wire boundary, exactly as `OutboundMessagePayload` does for outbound mail. + */ +export const MailboxNotifyHeaders = type({ + "from?": "string", + to: "string[]", + "cc?": "string[]", + "date?": "string", + "messageId?": "string", + "inReplyTo?": "string", + "references?": "string[]", + "subject?": "string", + "listId?": "string", + "interchangeType?": InterchangeType, + "interchangeCorrelationId?": "string", + "interchangeTenantId?": "string", + "interchangeAgentId?": "string", + "interchangeSessionId?": "string", + "interchangeOfferingId?": "string", + "interchangeSchemaVersion?": "string", + "traceparent?": "string", + "tracestate?": "string", +}); + +export type MailboxNotifyHeaders = typeof MailboxNotifyHeaders.infer; + +/** + * Recursive mailbox values carried on `mailbox.call`. Date fields of a search + * stay strings: reviving them here would turn a bad date into a control-channel + * failure, and the handler revives the tree before it runs the query. + */ +const mailboxCall = type.module({ + SearchQuery: { + "from?": "string", + "to?": "string", + "cc?": "string", + "bcc?": "string", + "header?": { + field: "string", + contains: "string", + }, + "before?": "string", + "after?": "string", + "on?": "string", + "sentBefore?": "string", + "sentAfter?": "string", + "sentOn?": "string", + "hasFlags?": "string[]", + "missingFlags?": "string[]", + "body?": "string", + "text?": "string", + "largerThan?": "number", + "smallerThan?": "number", + "and?": "SearchQuery[]", + "or?": "SearchQuery[]", + "not?": "SearchQuery", + }, + Thread: { + ref: { + uid: "number >= 1", + mailbox: "string > 0", + }, + children: "Thread[]", + }, + BodyStructure: { + contentType: "string", + "size?": "number", + "disposition?": "string", + "parts?": "BodyStructure[]", + }, +}); + +const MailboxCallMessageRef = type({ + uid: "number >= 1", + mailbox: "string > 0", +}); + +/** + * Caller-supplied message coordinates. They are not bounded: a frame this + * schema rejects crashes the child, and the supervisor is what refuses a + * mailbox name or a uid. Refs the supervisor emits stay on + * `MailboxCallMessageRef`, which admits only a positive uid and a non-empty + * mailbox. + */ +const MailboxCallRequestRef = type({ + uid: "number", + mailbox: "string", +}); + +const MailboxCallOp = type.enumerated( + "search", + "thread", + "fetchHeaders", + "fetchStructure", + "fetchPart", + "fetchFull", + "sync", + "getMailboxStatus", + "append", + "listMailboxes", + "createMailbox", + "deleteMailbox", + "move", + "copy", + "createList", + "listMembers", + "subscribe", + "unsubscribe", + "watch", + "readMailPart", +); + +const MailboxCallStructuredPayload = type({ + type: InterchangeType, + version: "string", + body: "Record", +}); + +/** Decoded part bytes. `encoding` is omitted when the part was 7bit. */ +const MailboxCallPart = type({ + contentType: "string", + contentBase64: "string", + "encoding?": "string", + "filename?": "string", + "disposition?": "'inline' | 'attachment'", +}); + +/** + * A fetched attachment. `part` is the IMAP section `fetchPart` addresses, so + * it has to survive this schema; a header-only attachment object would drop + * it on the way to the caller. + */ +const MailboxCallAttachment = type({ + name: "string", + contentType: "string", + dataBase64: "string", + "part?": "string", +}); + +const MailboxCallFetchedMessage = type({ + ref: MailboxCallMessageRef, + headers: MailboxNotifyHeaders, + flags: "string[]", + signatureStatus: SignatureStatus, + "content?": "string", + "payload?": MailboxCallStructuredPayload, + "attachments?": MailboxCallAttachment.array(), +}); + +const MailboxCallSyncResult = type({ + vanished: "number[]", + changed: type({ + uid: "number >= 1", + flags: "string[]", + }).array(), + newMessages: MailboxCallMessageRef.array(), + fullResyncRequired: "boolean", +}); + +const MailboxCallStatus = type({ + total: "number >= 0", + unseen: "number >= 0", + recent: "number >= 0", + uidNext: "number >= 1", + uidValidity: "number", + highestModSeq: "number >= 0", +}); + +const MailboxCallMailbox = type({ + name: "string > 0", + "role?": "string", + "delimiter?": "string", +}); + +const MailboxCallListInfo = type({ + address: "string", + name: "string", + memberCount: "number >= 0", + createdAt: "string", +}); + +/** + * Operands the child forwards are unbounded. A frame this schema rejects + * crashes the child, and the supervisor refuses a mailbox, a uid, or a part + * it does not own. Results the supervisor emits stay bounded. + */ +function createMailboxCallRequestData() { + return type.or( + { + requestId: "string > 0", + runId: "string > 0", + op: "'search'", + mailbox: "string", + query: mailboxCall.SearchQuery, + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'thread'", + mailbox: "string", + algorithm: "'references' | 'orderedsubject'", + "query?": mailboxCall.SearchQuery, + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'fetchHeaders' | 'fetchStructure' | 'fetchFull'", + ref: MailboxCallRequestRef, + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'fetchPart'", + ref: MailboxCallRequestRef, + partPath: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'sync'", + mailbox: "string", + uidNext: "number", + uidValidity: "number", + highestModSeq: "number", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'getMailboxStatus' | 'watch'", + mailbox: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'append'", + mailbox: "string", + headers: MailboxNotifyHeaders, + "content?": "string", + "payload?": MailboxCallStructuredPayload, + "flags?": "string[]", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'listMailboxes'", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'createMailbox' | 'deleteMailbox'", + name: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'move' | 'copy'", + ref: MailboxCallRequestRef, + toMailbox: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'createList'", + address: "string", + name: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'listMembers'", + address: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'subscribe' | 'unsubscribe'", + listAddress: "string", + subscriberAddress: "string", + }, + { + requestId: "string > 0", + runId: "string > 0", + op: "'readMailPart'", + partRef: "string", + }, + ); +} + +/** + * `op` is echoed so `value` is checked under that operation. A shared value + * union would let a structure object match a part fetch and drop + * `contentBase64`. Operations with no result omit `value` rather than sending + * null. Part and attachment bytes are base64. + */ +function createMailboxCallResponseData() { + return type.or( + { + requestId: "string > 0", + ok: "false", + op: MailboxCallOp, + reason: "string > 0", + "condition?": MessageTransportCondition, + }, + { + requestId: "string > 0", + ok: "true", + op: "'watch' | 'deleteMailbox' | 'move' | 'copy' | 'subscribe' | 'unsubscribe'", + }, + { + requestId: "string > 0", + ok: "true", + op: "'search'", + value: MailboxCallMessageRef.array(), + }, + { + requestId: "string > 0", + ok: "true", + op: "'thread'", + value: mailboxCall.Thread.array(), + }, + { + requestId: "string > 0", + ok: "true", + op: "'fetchHeaders'", + value: MailboxNotifyHeaders, + }, + { + requestId: "string > 0", + ok: "true", + op: "'fetchStructure'", + value: mailboxCall.BodyStructure, + }, + { + requestId: "string > 0", + ok: "true", + op: "'fetchPart'", + value: MailboxCallPart, + }, + { + requestId: "string > 0", + ok: "true", + op: "'fetchFull'", + value: MailboxCallFetchedMessage, + }, + { + requestId: "string > 0", + ok: "true", + op: "'sync'", + value: MailboxCallSyncResult, + }, + { + requestId: "string > 0", + ok: "true", + op: "'getMailboxStatus'", + value: MailboxCallStatus, + }, + { + requestId: "string > 0", + ok: "true", + op: "'append'", + value: MailboxCallMessageRef, + }, + { + requestId: "string > 0", + ok: "true", + op: "'listMailboxes'", + value: MailboxCallMailbox.array(), + }, + { + requestId: "string > 0", + ok: "true", + op: "'createMailbox'", + value: MailboxCallMailbox, + }, + { + requestId: "string > 0", + ok: "true", + op: "'createList'", + value: MailboxCallListInfo, + }, + { + requestId: "string > 0", + ok: "true", + op: "'listMembers'", + value: "string[]", + }, + { + requestId: "string > 0", + ok: "true", + op: "'readMailPart'", + value: { + contentBase64: "string", + }, + }, + ); +} + +// A message's discriminant selects its validator. Keep each schema factory lazy +// so a process constructs only the payload schemas it actually receives. +const controlPayloadFactories = defineControlPayloadFactories({ + "trigger.fire": () => + type({ + type: "'trigger.fire'", + data: { + runId: "string", + messageId: "string", + receivedAt: "number", + // The run's inbound-mail input, resolved by the supervisor (the sole + // mail owner) before the frame: a decoded `Mail` (headers plus part + // descriptors that reference the part bytes committed to the workflow-run + // substrate). The child hands this straight to the runtime as the trigger + // payload. Refs, not raw mail bytes, ride here; the committed part files + // stay in the substrate. Typed `unknown` -- `Mail` is a nested structural + // type validated at the consumption boundary by `isMail`. + payload: "unknown", + }, + }), + "signal.deliver": () => + type({ + type: "'signal.deliver'", + data: { + runId: "string", + signalName: "string", + signalId: "string", + // Addresses one onTrigger relay park. Omitted, the child commits + // a name-FIFO delivery. + "correlationId?": "string > 0", + // The resume decision in FINAL form -- the child commits it as the + // SignalReceived payload verbatim. Each sender owns any + // provenance-specific preparation BEFORE this frame: the dispatch loop + // resolves an inbound mail to a decoded `Mail` (headers plus part + // references, like the turn-1 trigger), while `deliverSignal` ships a + // structured signal payload unchanged -- so this field stays + // polymorphic. Do NOT ship raw inbound mail bytes through here. + payload: "unknown", + }, + }), + drain: () => + type({ + type: "'drain'", + data: { + deadlineMs: "number", + }, + }), + shutdown: () => + type({ + type: "'shutdown'", + data: { + reason: "string", + }, + }), + "cancel.prepare": () => + type({ + type: "'cancel.prepare'", + data: { requestId: "string", runId: "string", reason: "string" }, + }), + "cancel.prepared": () => + type({ + type: "'cancel.prepared'", + data: { requestId: "string", "error?": "string" }, + }), + "cancel.committed": () => + type({ + type: "'cancel.committed'", + data: { requestId: "string", "error?": "string" }, + }), + "grants-updated": () => + type({ + type: "'grants-updated'", + data: { + /** + * Full credentialsSnapshot the supervisor assembled. The child + * replaces its in-memory snapshot wholesale on receive so the + * authorize closure binds to the new per-step grants on the + * next step invocation. Carried inline rather than by reference + * because the snapshot is per-step grants payload -- the + * supervisor is the only producer and the child is the only + * consumer, so the substrate round-trip would just add latency. + */ + snapshot: CredentialsSnapshotPayload, + /** + * Per-step content hashes the supervisor expects the snapshot + * to pin to. Surfaced separately so receivers can cheap-compare + * a push against the snapshot they already have without rehashing + * each step's grants. Optional; the receiver does not require it + * but uses it for the staleness cross-check when present. + */ + "stepHashes?": "Record", + }, + }), + "sources-updated": () => + type({ + type: "'sources-updated'", + data: SourcesUpdatedData, + }), + "credentials-updated": () => + type({ + // Refreshed credential material for the deployment's inference sources + // and binding consumers. The child MERGES this into its in-memory cell + // (see `mergeCredentialDelivery`): `delivery.materials` upsert by + // credentialId and `delivery.bindings` upsert by (consumer, handle); + // `revoke` names credentialIds to drop, and dropping one drops every + // binding referencing it. Merge rather than wholesale-replace because the + // cell has several independently-scoped producers (the deploy frame, an + // inference rotation, a credential-grant push), each carrying only its + // own slice -- a swap would let one evict another's credentials. + // Revocation is therefore explicit, never by omission. Carried inline + // like the grants and sources snapshots. The secret rides this frame and + // the in-memory cell only; it is never persisted. + type: "'credentials-updated'", + data: { + delivery: CredentialDelivery, + "revoke?": "string[]", + }, + }), + ready: () => + type({ + type: "'ready'", + data: { + childPid: "number", + /** + * Hex-encoded Ed25519 public key the child minted at startup. + * The supervisor extracts this on receive and uses it to verify + * every subsequent upstream control frame's signature. The + * child's private key never leaves the child's address space. + */ + childPublicKey: "string", + }, + }), + "recycle.request": () => + type({ + // Child-initiated request to recycle the workflow-process. The + // child emits this when its own self-check decides it needs to be + // recycled (an internal consistency error it can't recover from, + // a watchdog tripping); the supervisor receives it on its + // upstream control-channel reader and funnels it into the same + // `triggerRecycle` code path the operator and policy origins use. + // The `reason` rides verbatim into the supervisor's reason field; + // the supervisor does not interpret it beyond logging and + // attaching it to the recycle attempt. + type: "'recycle.request'", + data: { + reason: "string", + }, + }), + "substrate.write.request": () => + type({ + // Child-initiated `writeTreePreservingPrefix` request. The child + // does not hold a substrate write authority for the workflow-run + // repo (single-writer at the ref tip belongs to the supervisor); + // its workflow-run substrate proxy forwards every write through + // this frame. The supervisor receives the request, runs its own + // wrapped `writeTreePreservingPrefix`, and reaches back to the + // child for the merge bytes via `substrate.merge.request` so the + // child's merge closure (which knows about seq computation, + // duplicate detection, etc.) keeps producing the prospective tree. + // The supervisor resolves the child's awaiter with + // `substrate.write.response`. + type: "'substrate.write.request'", + data: { + requestId: "string > 0", + repoId: { + kind: "string", + id: "string > 0", + }, + ref: "string > 0", + preservePrefix: "string > 0", + message: "string > 0", + }, + }), + "substrate.merge.request": () => + type({ + // Supervisor-initiated request for the child's merge bytes. Fired + // from inside the supervisor's `writeTreePreservingPrefix` merge + // callback while the per-repo lock is held; the child receives the + // existing prefix entries (base64-encoded bytes), invokes its merge + // closure, and replies with the prospective tree on + // `substrate.merge.response`. Carrying the entries inline preserves + // the lock window: the supervisor blocks inside the merge callback + // until the response lands. + type: "'substrate.merge.request'", + data: { + requestId: "string > 0", + existing: type({ + path: "string > 0", + contentBase64: "string", + }).array(), + }, + }), + "substrate.merge.response": () => + type({ + // Child's merge result. `requestId` correlates with the + // `substrate.write.request` that started the write; the supervisor + // resumes its merge callback with the supplied entries (or + // propagates the structured failure). + type: "'substrate.merge.response'", + data: { + requestId: "string > 0", + result: type( + { + ok: "true", + files: type({ + path: "string > 0", + contentBase64: "string", + }).array(), + }, + "|", + { + ok: "false", + reason: "string > 0", + }, + ), + }, + }), + "substrate.write.response": () => + type({ + // Supervisor's terminal reply to a child's `substrate.write.request`. + // The `requestId` echoes the child's allocated correlation id so + // the child's pending-id map resolves the awaiter. A successful + // write surfaces `commitSha`; the child's proxy returns that to its + // caller. A failed write (substrate rejection, validatePush + // violation, the supervisor's pack-push wrap's downstream + // `HubLink.pushWorkflowRunPack` rejection) surfaces a structured + // `{ ok: false, reason }` the child's proxy rethrows. + type: "'substrate.write.response'", + data: { + requestId: "string > 0", + result: type( + { + ok: "true", + commitSha: "string > 0", + }, + "|", + { + ok: "false", + reason: "string > 0", + }, + ), + }, + }), + "outbound.message": () => + type({ + // Child-initiated outbound-mail request (OUTBOUND half of mailbox + // ownership, §3a). The workflow-process child never holds the + // agent's signing key and never calls `transport.send` itself. When + // a step agent produces a reply or invokes a mail-send tool, the + // child forwards the structured outbound message plus the sender + // (agent) address up over the control channel; the supervisor + // performs the actual signed send through the host's real transport + // (`MailBusBindings.sendOutbound`), which signs with the sender's + // `CryptoProvider` exactly as the in-process path does. The + // supervisor is the sole mail owner and the only process that can + // emit signed mail on the agent's behalf. + // + // `requestId` correlates the supervisor's `outbound.result` reply so + // the child's mail-tool `send()` resolves with the real + // `SendReceipt` (or rejects with the supervisor's structured + // failure). The message is carried as a JSON-projected + // `OutboundMessage`; attachment bytes ride base64-encoded so the + // NDJSON wire stays text-safe. + // + // `completeReferences` is set only by a connector reply. The + // supervisor then fills References from the committed mailbox when + // `inReplyTo` is set and `references` is absent. A send without the + // flag is left alone: `mail_send` is the same shape, and completing + // it would replace its one-parent chain with the full ancestry. + type: "'outbound.message'", + data: { + requestId: "string > 0", + senderAddress: "string > 0", + "mailbox?": "string", + "completeReferences?": "boolean", + message: OutboundMessagePayload, + }, + }), + "outbound.result": () => + type({ + // Supervisor's terminal reply to a child's `outbound.message`. The + // `requestId` echoes the child's correlation id so the child's + // pending mail-tool awaiter resolves. A successful send surfaces the + // `SendReceipt` (messageId + status); a failed send (unregistered + // sender, signing failure, transport rejection) surfaces a + // structured `{ ok: false, reason }` the child's transport rethrows + // so the mail-tool call fails loudly rather than dropping the send. + type: "'outbound.result'", + data: { + requestId: "string > 0", + result: type( + { + ok: "true", + messageId: "string > 0", + status: "'delivered' | 'queued'", + }, + "|", + { + ok: "false", + reason: "string > 0", + }, + ), + }, + }), + "terminal.event": () => + type({ + // Child-initiated terminal-run notification. The workflow-process + // child emits this when one of its runs reaches a terminal phase + // (`RunCompleted`, `RunFailed`, `RunCancelled`) so the supervisor's + // dispatch loop and drain accumulators can settle without re-reading + // the workflow-run substrate from the supervisor process. The child + // commits the terminal event to its own substrate through the + // workflow-run pack-push pipeline; this frame is the peer-channel + // notification that mirrors the commit so the supervisor's + // in-process consumers do not have to round-trip the substrate. + // + // The `seq` mirrors the on-disk EventBase.seq the child assigned at + // commit time. The supervisor does not authoritatively verify the + // commit landed -- the pack-push response covers that contract -- + // but the field is carried so a downstream consumer can correlate + // the notification with the substrate blob. + type: "'terminal.event'", + data: { + runId: "string > 0", + seq: "number >= 0", + kind: "'RunCompleted' | 'RunFailed' | 'RunCancelled'", + at: "string > 0", + "error?": { + message: "string", + }, + }, + }), + "park.notify": () => + type({ + // Child-initiated control-plane suspension notification. The + // workflow-process child emits this when a workflow agent step parks + // on a reserved `signalName(correlationId)` channel (`env.onPark`), + // so the supervisor can register the correlation out-of-band before + // the parked run can be resumed. The supervisor stamps the + // deployment identity it owns (`runId` + `agentAddress`) and + // forwards a `signal.correlation.register` frame to the hub, which + // co-writes the run's routing + approval rows. Mirrors + // `terminal.event`: a peer-channel notification the supervisor fans + // out, distinct from the substrate commit the run also produces. + // + // `signalName` is deliberately NOT carried: it is a pure function of + // `correlationId` (`signalName(correlationId)`), recomputed by every + // consumer that needs it, so the two cannot drift. + type: "'park.notify'", + data: { + runId: "string > 0", + correlationId: "string > 0", + parkKind: ControlParkKind, + // Approver-facing snapshot of the parked tool call, size-capped at this + // process boundary. Optional: only an ask-rail suspension carries one. + "snapshot?": BoundedApprovalSnapshot, + }, + }), + "parked-correlations.request": () => + type({ + // Supervisor-initiated request: enumerate the child's currently-parked + // approval correlations. The supervisor fires this after a + // re-establishment (child respawn, or hub-link reconnect fanned out to + // this deployment) so it can re-register at the hub every correlation + // whose original `park.notify`-driven register may have been lost while + // the hub was down. Modeled on `substrate.merge.request`: a + // supervisor->child request the child answers on `parked-correlations. + // response`, correlated by `requestId`. Carries no filter -- one child + // owns one deployment, so the child enumerates everything parked and the + // supervisor re-emits all; the hub co-write is idempotent. + type: "'parked-correlations.request'", + data: { + requestId: "string > 0", + }, + }), + "parked-correlations.response": () => + type({ + // Child's reply to `parked-correlations.request`. Each entry mirrors + // `park.notify`'s data -- the child-supplied half of a + // `SuspensionRegistration` the supervisor stamps its deployment identity + // onto -- so the supervisor's re-emit path shares one transform with the + // `park.notify` arm. Only reduced-state approval parks appear here: the + // child enumerates steps whose reduced phase is `awaiting-signal` on a + // control-plane `signalName(correlationId)` channel, each of which carries + // a durable snapshot by construction (a snapshot-less correlated suspend + // reduces to `failed`, not `awaiting-signal`). `snapshot` is therefore + // required, and size-capped at this process boundary like `park.notify`. + type: "'parked-correlations.response'", + data: { + requestId: "string > 0", + parked: type({ + runId: "string > 0", + correlationId: "string > 0", + parkKind: ControlParkKind, + // Snapshot is required for approval parks and absent for input parks. + "snapshot?": BoundedApprovalSnapshot, + }).array(), + }, + }), + "resumed.runs": () => + type({ + // Child reports self-discovered runs after reconnect or recycle. + // The supervisor seeds its cohort tracking from these runIds so + // drain accumulators and dispatch routing account for runs the + // supervisor did not personally trigger.fire. + type: "'resumed.runs'", + data: { + runIds: type("string > 0").array(), + }, + }), + "mailbox.notify": () => + type({ + // Supervisor-to-child one-way notification that new mail landed in a + // deployment mailbox (INBOUND half of mailbox ownership, §3b). One-way + // like `grants-updated`/`sources-updated`: no correlation id, no response. + // The supervisor -- the sole mail owner -- commits the arrived message to + // the workflow-run substrate mailbox, then fires this frame so a registered + // `watch` observes the arrival decoupled from the FIFO trigger dispatch + // that resolves a run's first input. `headers` rides inline and is copied + // onto the `exists` event. The frame is not a copy of the message: a later + // fetch is its own mailbox call. The frame carries no commit pin. + type: "'mailbox.notify'", + data: { + runId: "string > 0", + mailbox: "string > 0", + uid: "number >= 1", + headers: MailboxNotifyHeaders, + }, + }), + "mailbox.mutate.request": () => + type({ + // Child-initiated mailbox-mutation request (INBOUND half of mailbox + // ownership, §3b). The supervisor is the sole writer to the + // workflow-run mailbox. Flag writes and `expunge` route up here so + // the supervisor applies them to its owned store. A child flushing the + // same ref would race the supervisor's in-memory mirror and break + // uid / modseq monotonicity. + // + // `data` is discriminated on `op`: an `addFlags` / `removeFlags` + // carries the target `uid` and the `flags` to change, so the wire + // boundary rejects a flag frame that omits them; an `expunge` sweeps + // every `\Deleted` message in the mailbox and the child constructs it + // with neither. The mailbox name and the flag uid are not bounded. A + // frame this schema rejects crashes the child, and the supervisor + // refuses a mailbox other than INBOX and an unknown uid. `requestId` + // correlates the supervisor's `mailbox.mutate.response` reply. + type: "'mailbox.mutate.request'", + data: type( + { + requestId: "string > 0", + runId: "string > 0", + mailbox: "string", + op: "'addFlags' | 'removeFlags'", + uid: "number", + flags: "string[]", + }, + "|", + { + requestId: "string > 0", + runId: "string > 0", + mailbox: "string", + op: "'expunge'", + }, + ), + }), + "mailbox.mutate.response": () => + type({ + // Supervisor's terminal reply to a child's `mailbox.mutate.request`. + // The `requestId` echoes the child's correlation id so the child's + // pending mail-tool awaiter resolves. The reply is sent only after + // the supervisor flushes the mutation, so the child's next committed + // read observes it -- the same flush-before-signal ordering + // `mailbox.notify` relies on. A successful `expunge` carries the + // `expungedUids` it swept so the agent tool can report the count; a + // flag write carries no operand echo. A failed mutation (unknown + // uid, substrate fault) surfaces a structured `{ ok: false, reason }` + // the child's bridge rethrows so the mail-tool call fails loudly. + type: "'mailbox.mutate.response'", + data: { + requestId: "string > 0", + result: type( + { + ok: "true", + "expungedUids?": "number[]", + }, + "|", + { + ok: "false", + reason: "string > 0", + // Present when the supervisor refused the mailbox itself. Absent + // for a failure that names no IMAP condition, such as an unknown uid. + "condition?": MessageTransportCondition, + }, + ), + }, + }), + "mailbox.call.request": () => + type({ + // A child asks the supervisor to answer one mailbox operation. The + // supervisor owns the deployment mailbox, so the answer — including a + // refusal — is its decision. `data` is discriminated on `op`, and each + // operation carries only the operands it uses. `requestId` correlates + // the `mailbox.call.response`. + type: "'mailbox.call.request'", + data: createMailboxCallRequestData(), + }), + "mailbox.call.response": () => + type({ + // Reply to `mailbox.call.request`, sent after the operation finishes. + // `op` echoes the request. See `createMailboxCallResponseData()`. + type: "'mailbox.call.response'", + data: createMailboxCallResponseData(), + }), +}); + +function defineControlPayloadFactories< + T extends { [K in keyof T]: () => { infer: { type: K } } }, +>(factories: T): T { + return factories; +} + +type ControlPayloadFactory = + (typeof controlPayloadFactories)[keyof typeof controlPayloadFactories]; +type ControlPayloadSchema = ReturnType; +export type ControlPayload = ControlPayloadSchema["infer"]; + +const factories = new Map( + Object.entries(controlPayloadFactories), +); +const schemas = new Map(); +const PayloadHeader = type({ type: type.enumerated(...factories.keys()) }); + +function getControlPayloadSchema(kind: string): ControlPayloadSchema { + const cached = schemas.get(kind); + if (cached) return cached; + const create = factories.get(kind); + if (!create) throw new Error("Unknown control payload kind: " + kind); + const schema = create(); + schemas.set(kind, schema); + return schema; +} + +export function parseControlPayload( + value: unknown, +): ControlPayload | type.errors { + const header = PayloadHeader(value); + if (header instanceof type.errors) return header; + return getControlPayloadSchema(header.type)(value); +} + +// The public inspection schema remains available to hosts and tooling. Child +// receivers use parseControlPayload and never construct this combined union. +export function createControlPayloadSchema() { + return type.or(...[...factories.keys()].map(getControlPayloadSchema)); +} diff --git a/vendor/intx/workflow-host/src/ipc/control-receiver.ts b/vendor/intx/workflow-host/src/ipc/control-receiver.ts new file mode 100644 index 0000000000..4dc101aaa8 --- /dev/null +++ b/vendor/intx/workflow-host/src/ipc/control-receiver.ts @@ -0,0 +1,200 @@ +// Control channel: NDJSON over stdio, Ed25519-signed per direction. +// +// Two Ed25519 keypairs flow per spawn: +// - Supervisor's keypair. The supervisor holds the private half and +// signs every downstream (supervisor->child) frame. The matching +// public half is passed to the child in spawn-time env +// (`HOST_PUBKEY`) and the child verifies downstream frames +// against it. The supervisor's PRIVATE KEY NEVER LEAVES THE +// SUPERVISOR'S ADDRESS SPACE. +// - Child's keypair. The child mints it at startup, holds the +// private half in its own address space, and signs every upstream +// (child->supervisor) frame. The matching public half rides as +// `childPublicKey` on the upstream `ready` frame's payload; the +// supervisor extracts it on receive and uses it to verify +// subsequent upstream frames. The CHILD'S PRIVATE KEY NEVER +// LEAVES THE CHILD'S ADDRESS SPACE. +// +// Upstream `ready` bootstraps the supervisor's view of the child's +// public key. The supervisor's receiver opens in bootstrap mode: +// the first frame's envelope is parsed structurally so the +// supervisor can extract `childPublicKey` from the payload, then the +// signature is verified against that key. Subsequent upstream frames +// verify against the same key. A child-signed frame whose claimed +// `childPublicKey` does not match the bootstrap value (or any +// non-`ready` first frame) crashes the receiver. +// +// Wire format: one signed envelope per line. Each line is the JSON +// serialization of `{ envelope: { seq, channelId, payload }, sig: +// }`. The signature covers the canonical bytes of the +// envelope sub-object (see `envelope.ts`). +// +// Payload schemas live in control-payloads.ts. The receiver selects the schema +// for the message's discriminator; inference events use the separate event +// channel and cannot satisfy a control payload schema. + +import { type } from "arktype"; +import { hexDecode } from "@intx/types/hex"; +import { encodeEnvelope, SignedEnvelope } from "./envelope"; +import { verifyEd25519 } from "./crypto"; +import { parseControlPayload, type ControlPayload } from "./control-payloads"; +import type { NdjsonReader } from "./control-sender"; + +export interface ControlChannelReceiverOpts { + /** + * Public key used to verify every inbound frame. When `Uint8Array` + * the value is fixed at construction time (the child's downstream + * receiver uses the supervisor's pubkey from `HOST_PUBKEY`). When + * `{ bootstrapFromReady: true }` the receiver opens in + * bootstrap mode: the first frame must be `ready` and must carry + * a `childPublicKey` hex-encoded Ed25519 public key in its payload. + * The receiver extracts the key, verifies the first frame's + * signature against it, then continues verifying subsequent frames + * against the same key. The supervisor's upstream receiver opens + * in bootstrap mode so the child can publish its own public key + * over the wire without the supervisor ever holding the matching + * private half. + */ + publicKey: Uint8Array | { bootstrapFromReady: true }; + channelId: string; + reader: NdjsonReader; + /** + * Invoked when any invariant is violated: signature failure, + * channelId mismatch, non-monotonic seq, malformed payload. The + * receiver's contract is to crash on any such violation. The + * caller wires this to a process-exit path; tests inject a + * recorder to assert on the failure mode. + */ + onCrash: (reason: string) => void; +} + +/** + * Construct the child-side control-channel receiver. Yields one + * verified, in-order `ControlPayload` per call. Any frame that + * fails verification, carries a non-current channelId, or arrives + * out of order calls `onCrash` and ends the iterator. + */ +export async function* receiveControlChannel( + opts: ControlChannelReceiverOpts, +): AsyncGenerator { + let highestSeq = 0; + let activePublicKey: Uint8Array | null = + opts.publicKey instanceof Uint8Array ? opts.publicKey : null; + const bootstrapping = activePublicKey === null; + for await (const line of opts.reader.read()) { + if (line.length === 0) continue; + + let raw: unknown; + try { + raw = JSON.parse(line); + } catch (cause) { + opts.onCrash( + `control channel received non-JSON line: ${errorMessage(cause)}`, + ); + return; + } + + const signed = SignedEnvelope(raw); + if (signed instanceof type.errors) { + opts.onCrash( + `control channel envelope failed validation: ${signed.summary}`, + ); + return; + } + + let envelopeBytes: Uint8Array; + try { + envelopeBytes = encodeEnvelope(signed.envelope); + } catch (cause) { + opts.onCrash( + `control channel envelope re-encode failed: ${errorMessage(cause)}`, + ); + return; + } + + let sigBytes: Uint8Array; + try { + sigBytes = hexDecode(signed.sig); + } catch (cause) { + opts.onCrash( + `control channel signature decode failed: ${errorMessage(cause)}`, + ); + return; + } + + if (activePublicKey === null) { + // Bootstrap mode: the first frame must be `ready`. Extract the + // child's public key from the payload, then verify the + // first frame's signature against it. The receiver crashes if + // the payload is not a `ready` frame or carries a malformed + // `childPublicKey`. + const candidate = parseControlPayload(signed.envelope.payload); + if (candidate instanceof type.errors) { + opts.onCrash( + `control channel bootstrap payload failed validation: ${candidate.summary}`, + ); + return; + } + if (candidate.type !== "ready") { + opts.onCrash( + `control channel bootstrap expected a ready frame, got ${candidate.type}`, + ); + return; + } + let bootstrapKey: Uint8Array; + try { + bootstrapKey = hexDecode(candidate.data.childPublicKey); + } catch (cause) { + opts.onCrash( + `control channel bootstrap childPublicKey decode failed: ${errorMessage(cause)}`, + ); + return; + } + activePublicKey = bootstrapKey; + } + + const ok = await verifyEd25519(envelopeBytes, sigBytes, activePublicKey); + if (!ok) { + opts.onCrash( + `control channel signature did not verify (seq=${String(signed.envelope.seq)}, channelId=${signed.envelope.channelId}${bootstrapping ? "; bootstrap" : ""})`, + ); + return; + } + + if (signed.envelope.channelId !== opts.channelId) { + opts.onCrash( + `control channel channelId mismatch: expected ${opts.channelId}, got ${signed.envelope.channelId} at seq=${String(signed.envelope.seq)}`, + ); + return; + } + + if (signed.envelope.seq <= highestSeq) { + opts.onCrash( + `control channel out-of-order seq: expected > ${String(highestSeq)}, got ${String(signed.envelope.seq)}`, + ); + return; + } + if (signed.envelope.seq !== highestSeq + 1) { + opts.onCrash( + `control channel seq gap: expected ${String(highestSeq + 1)}, got ${String(signed.envelope.seq)}`, + ); + return; + } + highestSeq = signed.envelope.seq; + + const payload = parseControlPayload(signed.envelope.payload); + if (payload instanceof type.errors) { + opts.onCrash( + `control channel payload failed validation: ${payload.summary}`, + ); + return; + } + + yield payload; + } +} + +function errorMessage(cause: unknown): string { + if (cause instanceof Error) return cause.message; + return String(cause); +} diff --git a/vendor/intx/workflow-host/src/ipc/control-sender.ts b/vendor/intx/workflow-host/src/ipc/control-sender.ts new file mode 100644 index 0000000000..843ee130b2 --- /dev/null +++ b/vendor/intx/workflow-host/src/ipc/control-sender.ts @@ -0,0 +1,111 @@ +// Control-channel sender: Ed25519-signed NDJSON over stdio. Each line carries +// { envelope: { seq, channelId, payload }, sig: }, with the +// signature covering the envelope's canonical bytes. Each direction uses its +// sender's private key; control-receiver.ts verifies against the peer's key. + +import { hexEncode } from "@intx/types/hex"; +import { + encodeEnvelope, + type FrameEnvelope, + type SignedEnvelope, +} from "./envelope"; +import { signEd25519 } from "./crypto"; +import type { ControlPayload } from "./control-payloads"; + +export interface NdjsonWriter { + write(line: string): Promise | void; +} + +export interface NdjsonReader { + read(): AsyncIterableIterator; +} + +export interface ControlChannelSenderOpts { + privateKeySeed: Uint8Array; + channelId: string; + writer: NdjsonWriter; +} + +export interface ControlChannelSender { + send(payload: ControlPayload): Promise; + readonly seq: number; +} + +/** + * JSON has no NaN or Infinity: `JSON.stringify` emits `null` for both. + * The receiver then rejects the frame and crashes the child. Refuse + * before `seq` advances. A skipped sequence is itself a crash on the + * next frame. + */ +function rejectNonFiniteNumbers(value: unknown, path: string): void { + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error( + `control channel: cannot encode non-finite number at ${path}`, + ); + } + return; + } + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + rejectNonFiniteNumbers(value[index], `${path}[${String(index)}]`); + } + return; + } + if (typeof value === "object" && value !== null) { + for (const [key, child] of Object.entries(value)) { + rejectNonFiniteNumbers(child, `${path}.${key}`); + } + } +} + +/** + * Construct the supervisor-side control-channel sender. The + * supervisor's Ed25519 seed lives in closure. The matching public + * key flows to the child through spawn-time env -- never the seed. + */ +export function createControlChannelSender( + opts: ControlChannelSenderOpts, +): ControlChannelSender { + let seq = 0; + // Serialize sends. Signing is async, so without a lock two concurrent + // callers could each assign seq, suspend on `signEd25519`, and resume in + // signature-resolution order — writing frames out of seq order, which the + // receiver rejects as a gap and crashes the channel. The promise chain + // makes each send await the previous send's completion before it assigns + // seq, signs, and writes, keeping that critical section atomic. + let tail: Promise = Promise.resolve(); + return { + get seq() { + return seq; + }, + send(payload: ControlPayload): Promise { + const previous = tail; + let release: () => void = () => undefined; + tail = new Promise((resolve) => { + release = resolve; + }); + return (async () => { + await previous; + try { + rejectNonFiniteNumbers(payload, "payload"); + seq += 1; + const envelope: FrameEnvelope = { + seq, + channelId: opts.channelId, + payload, + }; + const envelopeBytes = encodeEnvelope(envelope); + const sig = await signEd25519(envelopeBytes, opts.privateKeySeed); + const signed: SignedEnvelope = { + envelope, + sig: hexEncode(sig), + }; + await opts.writer.write(JSON.stringify(signed) + "\n"); + } finally { + release(); + } + })(); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/ipc/crypto.ts b/vendor/intx/workflow-host/src/ipc/crypto.ts index 7052dff2c8..550dec4ccd 100644 --- a/vendor/intx/workflow-host/src/ipc/crypto.ts +++ b/vendor/intx/workflow-host/src/ipc/crypto.ts @@ -23,8 +23,8 @@ import { signEd25519 as ed25519Sign, verifyEd25519 as ed25519Verify, -} from "@intx/crypto"; -import { hexEncode } from "@intx/types"; +} from "@intx/crypto/keys"; +import { hexEncode } from "@intx/types/hex"; const ED25519_SIGNATURE_BYTES = 64; const ED25519_KEY_BYTES = 32; diff --git a/vendor/intx/workflow-host/src/ipc/event-channel.ts b/vendor/intx/workflow-host/src/ipc/event-channel.ts index 30588f22e4..00f10d02a4 100644 --- a/vendor/intx/workflow-host/src/ipc/event-channel.ts +++ b/vendor/intx/workflow-host/src/ipc/event-channel.ts @@ -17,34 +17,25 @@ // silent drop, and a blocking writer would deadlock against the // reactor's emit path. // -// Mixing failure mode: the payload union here covers InferenceEvent -// shapes only. A "control message" structurally shaped as `drain` or -// `recycle` will not satisfy this union and the receiver will crash. -// The discriminated arktype validators in `control-channel.ts` and -// here are disjoint by construction. +// The receiver validates every payload with `parseInferenceEvent`, which +// applies the shared `InferenceEvent` schema. A control-shaped payload such +// as `drain` or `recycle` fails validation; the receiver reports a crash +// and stops. +// The per-kind control validators in `control-payloads.ts` reject these +// inference-event shapes, keeping the channels' payload schemas disjoint. import { type } from "arktype"; -import { hexDecode, hexEncode } from "@intx/types"; -import { InferenceEvent } from "@intx/types/runtime"; - +import { hexDecode } from "@intx/types/hex"; import { - decodeEnvelope, - encodeEnvelope, - FrameEnvelope, - MacedEnvelope, -} from "./envelope"; -import { signHmac, verifyHmac } from "./crypto"; + parseInferenceEvent, + type InferenceEvent, +} from "@intx/types/inference-events"; -/** - * The event channel additionally carries the two bracket events the - * reactor emits per message run. They are part of the InferenceEvent - * union upstream; re-exporting the union directly keeps this channel - * structurally identical to what the hub's `agent.event` frame - * carries today. - */ -export const EventPayload = InferenceEvent; -export type EventPayload = typeof EventPayload.infer; +import { decodeEnvelope, encodeEnvelope, MacedEnvelope } from "./envelope"; +import { verifyHmac } from "./crypto"; + +export type EventPayload = InferenceEvent; /** * One verified event-channel frame. `childRunId` is the occurrence the @@ -56,89 +47,10 @@ export type ReceivedEvent = { readonly childRunId?: string; }; -export interface FrameWriter { - write(bytes: Uint8Array): Promise | void; -} - export interface FrameReader { read(): AsyncIterableIterator; } -export interface EventChannelSenderOpts { - hmacKey: Uint8Array; - channelId: string; - writer: FrameWriter; -} - -export interface EventChannelSender { - send(payload: EventPayload, childRunId?: string): Promise; - readonly seq: number; -} - -/** - * Construct the child-side event-channel sender. The shared HMAC - * key lives in closure. The child mints monotonic seq values per - * channelId; the receiver enforces strict monotonicity. - */ -export function createEventChannelSender( - opts: EventChannelSenderOpts, -): EventChannelSender { - let seq = 0; - // Serialize sends. `signHmac` is async, so without a lock two - // concurrent callers could each assign seq, suspend on the signer, and - // resume in HMAC-resolution order — writing frames out of seq order, - // which the receiver rejects as a gap and crashes the channel. The - // production caller fires events without awaiting (`void send(event)`), - // so this is the live case. The promise chain makes each send await the - // previous send's completion before it assigns seq, signs, and writes, - // keeping that critical section atomic. Mirrors the control channel's - // sender, whose Ed25519 signing has the same shape. - let tail: Promise = Promise.resolve(); - return { - get seq() { - return seq; - }, - send(payload: EventPayload, childRunId?: string): Promise { - const previous = tail; - let release: () => void = () => undefined; - tail = new Promise((resolve) => { - release = resolve; - }); - return (async () => { - await previous; - try { - seq += 1; - const envelope: FrameEnvelope = { - seq, - channelId: opts.channelId, - payload, - ...(childRunId !== undefined ? { childRunId } : {}), - }; - const envelopeBytes = encodeEnvelope(envelope); - const tag = await signHmac(envelopeBytes, opts.hmacKey); - const maced: MacedEnvelope = { - envelope, - mac: hexEncode(tag), - }; - // Newline-delimit each frame. The channel rides a byte-stream - // pipe (fd3) where the kernel may coalesce successive writes into - // one read or split one write across reads -- the one-write-equals- - // one-frame assumption does not hold under the burst of events a - // real step emits. `JSON.stringify` never emits a literal newline - // (newlines inside strings are escaped as `\n`), so `\n` is an - // unambiguous frame terminator the receiver splits on, mirroring - // the control channel's NDJSON discipline. - await opts.writer.write( - new TextEncoder().encode(`${JSON.stringify(maced)}\n`), - ); - } finally { - release(); - } - })(); - }, - }; -} - export interface EventChannelReceiverOpts { hmacKey: Uint8Array; channelId: string; @@ -155,7 +67,7 @@ export interface EventChannelReceiverOpts { /** * Construct the supervisor-side event-channel receiver. Yields one - * verified, in-order event per call, carrying the occurrence + * verified, in-order `ReceivedEvent` per call — the `EventPayload` plus * `childRunId` when the sender stamped it. Any frame that fails * HMAC verification, carries a non-current channelId, arrives out * of order, or arrives faster than the consumer drains -- triggers @@ -305,7 +217,7 @@ export async function* receiveEventChannel( } highestSeq = maced.envelope.seq; - const payload = EventPayload(maced.envelope.payload); + const payload = parseInferenceEvent(maced.envelope.payload); if (payload instanceof type.errors) { crashed = true; opts.onCrash( diff --git a/vendor/intx/workflow-host/src/ipc/event-sender.ts b/vendor/intx/workflow-host/src/ipc/event-sender.ts new file mode 100644 index 0000000000..571ffbe3df --- /dev/null +++ b/vendor/intx/workflow-host/src/ipc/event-sender.ts @@ -0,0 +1,88 @@ +import { hexEncode } from "@intx/types/hex"; +import type { EventPayload } from "./event-channel"; + +import { + encodeEnvelope, + type FrameEnvelope, + type MacedEnvelope, +} from "./envelope"; +import { signHmac } from "./crypto"; + +export interface FrameWriter { + write(bytes: Uint8Array): Promise | void; +} + +export interface EventChannelSenderOpts { + hmacKey: Uint8Array; + channelId: string; + writer: FrameWriter; +} + +export interface EventChannelSender { + send(payload: EventPayload, childRunId?: string): Promise; + readonly seq: number; +} + +/** + * Construct the child-side event-channel sender. The shared HMAC + * key lives in closure. The child mints monotonic seq values per + * channelId; the receiver enforces strict monotonicity. + */ +export function createEventChannelSender( + opts: EventChannelSenderOpts, +): EventChannelSender { + let seq = 0; + // Serialize sends. `signHmac` is async, so without a lock two + // concurrent callers could each assign seq, suspend on the signer, and + // resume in HMAC-resolution order — writing frames out of seq order, + // which the receiver rejects as a gap and crashes the channel. The + // production caller fires events without awaiting (`void send(event)`), + // so this is the live case. The promise chain makes each send await the + // previous send's completion before it assigns seq, signs, and writes, + // keeping that critical section atomic. Mirrors the control channel's + // sender, whose Ed25519 signing has the same shape. + let tail: Promise = Promise.resolve(); + return { + get seq() { + return seq; + }, + send(payload: EventPayload, childRunId?: string): Promise { + const previous = tail; + let release: () => void = () => undefined; + tail = new Promise((resolve) => { + release = resolve; + }); + return (async () => { + await previous; + try { + seq += 1; + const envelope: FrameEnvelope = { + seq, + channelId: opts.channelId, + payload, + ...(childRunId !== undefined ? { childRunId } : {}), + }; + const envelopeBytes = encodeEnvelope(envelope); + const tag = await signHmac(envelopeBytes, opts.hmacKey); + const maced: MacedEnvelope = { + envelope, + mac: hexEncode(tag), + }; + // Newline-delimit each frame. The channel rides a byte-stream + // pipe (fd3) where the kernel may coalesce successive writes into + // one read or split one write across reads -- the one-write-equals- + // one-frame assumption does not hold under the burst of events a + // real step emits. `JSON.stringify` never emits a literal newline + // (newlines inside strings are escaped as `\n`), so `\n` is an + // unambiguous frame terminator the receiver splits on, mirroring + // the control channel's NDJSON discipline. + await opts.writer.write( + new TextEncoder().encode(`${JSON.stringify(maced)}\n`), + ); + } finally { + release(); + } + })(); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/ipc/index.ts b/vendor/intx/workflow-host/src/ipc/index.ts index 8944f716ca..b80ecda0ee 100644 --- a/vendor/intx/workflow-host/src/ipc/index.ts +++ b/vendor/intx/workflow-host/src/ipc/index.ts @@ -118,10 +118,9 @@ // The event channel carries InferenceEvents and the bracket // events -- shapes whose authority is the deployment's identity // (which the supervisor and child share via the HMAC key) and -// whose rate is high. The two channels' typed payload unions are -// DISJOINT BY CONSTRUCTION (the discriminated `ControlPayload` -// union in `control-channel.ts` does not overlap the -// `EventPayload` union in `event-channel.ts`). A control payload +// whose rate is high. The per-kind validators in `control-payloads.ts` +// and `parseInferenceEvent` used by `event-channel.ts` accept disjoint +// payload shapes. A control payload // that included an inference-event shape would defeat the split; // an inference-event payload that included a `drain` or `recycle` // discriminator would let a compromised child issue control-plane @@ -155,16 +154,18 @@ export { export { DEFAULT_EVENT_BUFFER_LIMIT, - EventPayload, - createEventChannelSender, receiveEventChannel, - type EventChannelSender, - type EventChannelSenderOpts, + type EventPayload, type EventChannelReceiverOpts, type FrameReader, - type FrameWriter, type ReceivedEvent, } from "./event-channel"; +export { + createEventChannelSender, + type EventChannelSender, + type EventChannelSenderOpts, + type FrameWriter, +} from "./event-sender"; export { FrameEnvelope, diff --git a/vendor/intx/workflow-host/src/local-credentials.ts b/vendor/intx/workflow-host/src/local-credentials.ts new file mode 100644 index 0000000000..909c5e254d --- /dev/null +++ b/vendor/intx/workflow-host/src/local-credentials.ts @@ -0,0 +1,140 @@ +// The credentials snapshot a local run authorizes against. +// +// A deployed source-ref run authorizes every step against one flat grant set: +// the hub projects the probe's capability walk into runtime `tool:` and +// `effect:` rules, adds a `credential:` use rule for each resolved binding, +// and the sidecar gives every step id the same set. This module builds that +// set from the live definition, with no hub and no sidecar, so a local run +// evaluates tool, effect and credential grants against the same rules as a +// deployed step. Requirement grants need hub state, so the caller passes them. +// Nothing here imports a Node module. + +import type { DirectorRegistry, ToolDeclaration } from "@intx/agent"; +import type { GrantRule } from "@intx/authz"; +import { type } from "arktype"; + +import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import { serializeGrantWalk } from "@intx/types/grant-snapshot"; +import { WireGrantRule } from "@intx/types/grant-wire"; +import { executableStepIds, type WorkflowDefinition } from "@intx/workflow"; +import { + credentialUseGrantRules, + runtimeGrantRules, + walkCapabilities, + type RuntimeGrantRule, +} from "@intx/workflow-deploy/capabilities"; + +import { + resolveSnapshotStepGrants, + type CredentialsSnapshotRef, +} from "./child/credentials-backed-authorize"; +import type { CredentialsSnapshot } from "./supervisor/credentials"; + +export type LocalCredentialsSnapshotOpts = { + /** The live definition the run executes. */ + readonly definition: WorkflowDefinition; + /** The director registry the run resolves directors against. */ + readonly directors: DirectorRegistry; + /** + * The tool declarations of each plugin an agent declares, keyed by plugin + * name. Required when any agent declares a plugin: the walk refuses a plugin + * it has no declarations for. + */ + readonly pluginDefinitions?: ReadonlyMap; + /** + * The credential delivery the run's material cell holds. Each binding adds + * the `credential:` use rule its consumer needs. + */ + readonly delivery?: CredentialDelivery; + /** + * Further grants every step holds, such as the requirement grants a hub + * resolves from its principals. + */ + readonly grants?: readonly GrantRule[]; +}; + +/** + * Build the credentials snapshot a deployed source-ref run of `definition` + * would authorize against. Every step id the definition can execute (top + * level, loop bodies and inline onTrigger and childWorkflow bodies) holds the + * same grants, as in a deployed run. A local childWorkflow child therefore + * authorizes against the parent's full set, where a deployed child gets a + * narrower capped set. + * + * Throws when the walk cannot resolve a director, as the deploy probe does. + */ +export function createLocalCredentialsSnapshot( + opts: LocalCredentialsSnapshotOpts, +): CredentialsSnapshot { + const walk = walkCapabilities( + opts.definition, + opts.directors, + opts.pluginDefinitions ?? new Map(), + ); + const [unresolved] = walk.unresolvedDirectors; + if (unresolved !== undefined) { + throw new Error(`unresolvable director: ${unresolved}`); + } + const rules = [ + ...runtimeGrantRules(serializeGrantWalk(walk, undefined)), + ...credentialUseGrantRules(opts.delivery?.bindings ?? []), + ]; + const grants: readonly GrantRule[] = [ + ...rules.map(localGrantRule), + ...(opts.grants ?? []), + ]; + return { + steps: executableStepIds(opts.definition).map((stepId) => ({ + stepId, + address: stepId, + grants, + // A local snapshot is never pushed over a control channel, so it carries + // no content hash to cross-check. + contentHash: "", + })), + }; +} + +function localGrantRule(rule: RuntimeGrantRule, index: number): GrantRule { + return { + id: `local-grant:${String(index)}`, + ...rule, + roleId: null, + principalId: null, + }; +} + +/** + * The grants of `stepId` in the active snapshot, in the `GrantRule` shape + * Gate 2 reads: the `resolveGrants` of a step's credential wiring, and the + * `resolveStepGrants` of an action credentials factory. A map iteration's + * scoped id resolves to its base step. The snapshot types its grants as + * `unknown[]` and a caller can pass any snapshot, so the grants are + * validated as grant rules here. Throws when no snapshot is active, the + * snapshot has no entry for the step, or an entry is not a grant rule. + */ +export function snapshotStepGrants( + ref: CredentialsSnapshotRef, + stepId: string, +): readonly GrantRule[] { + const snapshot = ref.current; + if (snapshot === null) { + throw new Error( + `local credentials: no credentials snapshot is active for step ${stepId}`, + ); + } + const grants = WireGrantRule.array()( + resolveSnapshotStepGrants( + snapshot, + stepId, + (lookupStepId) => + `local credentials: the credentials snapshot has no entry for step ${lookupStepId}`, + ), + ); + if (grants instanceof type.errors) { + throw new Error( + `local credentials: the grants of step ${stepId} are not grant rules: ${grants.summary}`, + ); + } + return grants; +} diff --git a/vendor/intx/workflow-host/src/local.ts b/vendor/intx/workflow-host/src/local.ts new file mode 100644 index 0000000000..ddca308d32 --- /dev/null +++ b/vendor/intx/workflow-host/src/local.ts @@ -0,0 +1,46 @@ +// @intx/workflow-host/local -- run a definition outside a sidecar with the +// real step invoker. +// +// `runLocal` in `@intx/workflow` drives a definition in process; its agent +// steps run through the `invokeStep` a caller supplies. This entry exports the +// step invoker the deployed host uses, the builders that give a step its +// tools and credentials, and the authorize closure and grant evaluator the +// deployed host authorizes with, plus a snapshot builder that derives a +// deployed run's grants from the definition. A local run then executes steps +// with the deployed step invoker and authorizes them with the same grants and +// evaluator as a deployed source-ref run; `createLocalCredentialsSnapshot` +// names where a local run differs. +// +// Nothing this entry imports is a Node or Bun module, so it bundles for a +// browser. Some code it reaches uses the `Buffer` global at run time (MIME +// header encoding in `@intx/mime`); a browser caller installs one, as +// `@intx/storage-isogit/browser` does. The package root also exports the +// supervisor, the run child and the substrate adapters, which assume a Node +// filesystem and process. +export { + createWorkflowStepInvoker, + type StepEnvBase, + type WorkflowStepInvokerOpts, +} from "./adapters/step-invoker"; +export { + attachStepCredentialWiring, + attachStepTools, + createToolBearingAgentFactory, + deriveToolMarkFloorGrants, + type StepToolMaterialization, +} from "./child/step-tools"; +export { + createCredentialsBackedAuthorize, + createGrantEvaluator, + type CredentialsSnapshotRef, + type GrantEvaluator, +} from "./child/credentials-backed-authorize"; +export { + createLocalCredentialsSnapshot, + snapshotStepGrants, + type LocalCredentialsSnapshotOpts, +} from "./local-credentials"; +export type { + CredentialsSnapshot, + CredentialsSnapshotStep, +} from "./supervisor/credentials"; diff --git a/vendor/intx/workflow-host/src/probe/child.ts b/vendor/intx/workflow-host/src/probe/child.ts new file mode 100644 index 0000000000..e4c3cf654e --- /dev/null +++ b/vendor/intx/workflow-host/src/probe/child.ts @@ -0,0 +1,137 @@ +import { serializeGrantWalk } from "@intx/types/grant-snapshot"; +import { hexEncode } from "@intx/types/hex"; +import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; +import { + collectDeclaredPluginNames, + projectLiveToInert, +} from "@intx/workflow/projection"; + +import { encodeEnvelope, type FrameEnvelope } from "../ipc/envelope"; +import { signHmac } from "../ipc/crypto"; +import { + loadWorkflowDefinitionFromClosure, + loadWorkflowDirectorRegistryFromClosure, + loadWorkflowPluginToolDefinitionsFromClosure, +} from "../workflow-definition-loader"; +import { parseProbeChildEnv } from "./env"; +import { enrichProbeError } from "./errors"; +import type { ProbeResultPayload, WalkCapabilities } from "./protocol"; + +/** + * One line the child writes to a sink. Production wraps `process.stdout`; + * tests inject a capture. The bytes are handed to the OS before the child + * exits so the result is not truncated. + */ +export type ProbeChildLineWriter = (line: string) => Promise; + +export interface RunProbeChildOpts { + /** Raw env the child reads its anchors from (defaults to `process.env`). */ + rawEnv?: Readonly>; + /** Result-line sink (defaults to a drained `process.stdout` write). */ + writeLine?: ProbeChildLineWriter; +} + +/** + * The airlocked child's whole job: read the materialized package dir and + * IPC anchors from its fresh env, load+evaluate the workflow entry, run + * the capability walk plus the live->inert projector, and ship the inert + * projection + advisory grant set + wire hash back inside one + * HMAC-signed result frame. + * + * An evaluation failure (malformed code, an entry that throws, a package + * with no `interchange.workflow`) is caught and shipped as an `ok: false` + * frame so the host reaps cleanly and answers `workflow.probe.error` + * with the reason -- rather than the child crashing and the host seeing a + * bare "exited without result". + */ +export async function runWorkflowProbeChildFromProcessEnv( + walkCapabilities: WalkCapabilities, + opts: RunProbeChildOpts = {}, +): Promise { + const rawEnv = opts.rawEnv ?? process.env; + const writeLine = opts.writeLine ?? defaultStdoutWriteLine; + const { channelId, hmacKey, packageDir } = parseProbeChildEnv(rawEnv); + + let payload: ProbeResultPayload; + try { + payload = await computeProbePayload(packageDir, walkCapabilities); + } catch (err) { + payload = { ok: false, error: enrichProbeError(err) }; + } + + const envelope: FrameEnvelope = { seq: 0, channelId, payload }; + const envelopeBytes = encodeEnvelope(envelope); + const mac = hexEncode(await signHmac(envelopeBytes, hmacKey)); + await writeLine(`${JSON.stringify({ envelope, mac })}\n`); +} + +async function computeProbePayload( + packageDir: string, + walkCapabilities: WalkCapabilities, +): Promise { + const definition = await loadWorkflowDefinitionFromClosure({ packageDir }); + const projection = projectLiveToInert(definition); + const wireHash = await computeWireDefinitionHash(projection); + // Compose the director registry from the SAME closure the run-child will, + // so the `director:` grants advertised here match what the runtime + // resolves. Built-ins-only when the closure ships no `interchange.directors`. + const directors = await loadWorkflowDirectorRegistryFromClosure({ + packageDir, + }); + // Load the static tool `definitions` each declared plugin package + // contributes from the SAME closure, so the walk emits `tool:` + // grants for plugin-contributed tools (Tier-2 governance). A plugin + // package reaches an agent only through `env.plugins`, so its tool grant + // surface is invisible to the walk otherwise -- the run-child would then + // load the plugin from the closure and the reactor would fail closed on + // an un-approved `tool:`. Loading here (over the frozen closure the + // run-child also materializes from) keeps the approved snapshot and the + // runtime plugin in lockstep. + const pluginToolDefinitions = + await loadWorkflowPluginToolDefinitionsFromClosure({ + packageDir, + plugins: collectDeclaredPluginNames(definition), + }); + const walk = walkCapabilities(definition, directors, pluginToolDefinitions); + // Fail closed on an unresolved director: the runtime does not re-gate + // `director:` against the approved grant set, so this advertisement is + // the only approval checkpoint for a director. Shipping an ok probe whose + // grant set silently omits a director the runtime would still try to + // resolve would let the operator approve an incomplete manifest. Mirrors + // the live-authored approval gate (`createApprovalSetGate`). + const [unresolved] = walk.unresolvedDirectors; + if (unresolved !== undefined) { + return { ok: false, error: `unresolvable director: ${unresolved}` }; + } + return { + ok: true, + projection, + grants: collectDeploymentGrants(walk), + grantWalkSnapshot: serializeGrantWalk(walk, definition.grantRequirements), + wireHash, + }; +} + +/** + * Flatten the per-step walk output into the deployment-wide advisory + * grant set: the deduplicated, sorted union of every step's grant + * strings. Sorting makes the shipped set order-independent. + */ +function collectDeploymentGrants(walk: ReturnType): string[] { + const grants = new Set(); + for (const declarations of walk.perStep.values()) { + for (const grant of declarations.grants) { + grants.add(grant); + } + } + return [...grants].sort(); +} + +function defaultStdoutWriteLine(line: string): Promise { + return new Promise((resolve, reject) => { + process.stdout.write(line, (err) => { + if (err) reject(err); + else resolve(); + }); + }); +} diff --git a/vendor/intx/workflow-host/src/probe/env.ts b/vendor/intx/workflow-host/src/probe/env.ts new file mode 100644 index 0000000000..e89b078f62 --- /dev/null +++ b/vendor/intx/workflow-host/src/probe/env.ts @@ -0,0 +1,71 @@ +import { type } from "arktype"; + +import { hexDecode, hexEncode } from "@intx/types/hex"; + +const IPC_HMAC_KEY_BYTES = 32; + +// Env keys the host sets on the child's fresh spawn env. The child reads +// exactly these plus PATH/HOME/TMPDIR (for exec + tmp); nothing else +// crosses the airlock. +const PROBE_CHANNEL_ID_ENV = "PROBE_IPC_CHANNEL_ID"; +const PROBE_HMAC_KEY_ENV = "PROBE_IPC_HMAC_KEY"; +const PROBE_PACKAGE_DIR_ENV = "PROBE_PACKAGE_DIR"; + +export function buildProbeChildEnv(args: { + packageDir: string; + channelId: string; + hmacKey: Uint8Array; +}): Record { + // A fresh, minimal env: exactly the IPC anchors and the materialized + // package dir, plus the OS handles the shebang needs to exec `bun` and + // land temp files on the host's temp root. No `process.env` spread, so + // no sidecar secret or ambient input crosses the airlock. + const env: Record = { + [PROBE_CHANNEL_ID_ENV]: args.channelId, + [PROBE_HMAC_KEY_ENV]: hexEncode(args.hmacKey), + [PROBE_PACKAGE_DIR_ENV]: args.packageDir, + }; + const path = process.env["PATH"]; + if (path !== undefined) env["PATH"] = path; + const home = process.env["HOME"]; + if (home !== undefined) env["HOME"] = home; + const tmpdir = process.env["TMPDIR"]; + if (tmpdir !== undefined) env["TMPDIR"] = tmpdir; + return env; +} + +interface ProbeChildEnv { + readonly channelId: string; + readonly hmacKey: Uint8Array; + readonly packageDir: string; +} + +const NonEmptyString = type("string > 0"); + +export function parseProbeChildEnv( + rawEnv: Readonly>, +): ProbeChildEnv { + const channelId = requireEnv(rawEnv, PROBE_CHANNEL_ID_ENV); + const packageDir = requireEnv(rawEnv, PROBE_PACKAGE_DIR_ENV); + const hmacKeyHex = requireEnv(rawEnv, PROBE_HMAC_KEY_ENV); + const hmacKey = hexDecode(hmacKeyHex); + if (hmacKey.length !== IPC_HMAC_KEY_BYTES) { + throw new Error( + `workflow probe child env: ${PROBE_HMAC_KEY_ENV} must decode to ${String(IPC_HMAC_KEY_BYTES)} bytes, got ${String(hmacKey.length)}`, + ); + } + return { channelId, hmacKey, packageDir }; +} + +function requireEnv( + rawEnv: Readonly>, + key: string, +): string { + const value = NonEmptyString(rawEnv[key]); + if (value instanceof type.errors) { + throw new Error( + `workflow probe child env: required key ${key} is unset or empty`, + ); + } + return value; +} diff --git a/vendor/intx/workflow-host/src/probe/errors.ts b/vendor/intx/workflow-host/src/probe/errors.ts new file mode 100644 index 0000000000..449a56b98b --- /dev/null +++ b/vendor/intx/workflow-host/src/probe/errors.ts @@ -0,0 +1,29 @@ +export function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +// Node/Bun's module-not-found message shape. The specifier is the missing +// package the workflow entry imported at evaluation time. +const MISSING_MODULE_RE = /Cannot find (?:module|package) ['"]([^'"]+)['"]/; + +/** + * Enrich a probe evaluation failure whose cause is a module that could not be + * resolved from the workflow's dependency closure. The evaluator is the layer + * that KNOWS what the workflow imports at run time (it actually ran the + * import), so a missing specifier here means the closure did not carry it -- + * the common cause is a runtime import declared only under `devDependencies` + * (which the closure does not materialize), whether a workspace-local member or + * an external package. Rewrite the opaque "Cannot find module" into that + * actionable diagnostic. A non-resolution failure passes through unchanged. + */ +export function enrichProbeError(err: unknown): string { + const message = errorMessage(err); + const match = MISSING_MODULE_RE.exec(message); + const specifier = match?.[1]; + if (specifier === undefined) return message; + return ( + `workflow entry could not resolve ${JSON.stringify(specifier)} from its dependency closure; ` + + `if the workflow imports it at run time, declare it under "dependencies" rather than "devDependencies" ` + + `(a devDependencies-only import is not materialized into the closure). ${message}` + ); +} diff --git a/vendor/intx/workflow-host/src/probe/index.ts b/vendor/intx/workflow-host/src/probe/index.ts new file mode 100644 index 0000000000..6a8908ad95 --- /dev/null +++ b/vendor/intx/workflow-host/src/probe/index.ts @@ -0,0 +1,397 @@ +// Host-side workflow-probe handler: spawns and supervises a one-shot probe child. +// +// A `workflow.probe.request` frame asks this sidecar to inspect a +// code-sourced workflow WITHOUT deploying it. The inspection evaluates +// author code (the workflow package's `interchange.workflow` entry), so +// it must never run in the sidecar host's address space. This module +// spawns a ONE-SHOT child process behind the IPC airlock that loads and +// evaluates the entry, runs the capability walk plus the live->inert +// projector, and ships the inert projection + advisory grant set + wire +// hash back over an HMAC-authenticated result frame (reusing the same +// per-frame HMAC discipline `@intx/workflow-host`'s event channel uses). +// +// Reaping is sidecar-owned and independent of the hub's probe timeout. +// The hub timeout only rejects the hub-side promise; it does not kill +// the child. `runOneShotProbeChild` owns a self-contained lifecycle with +// its OWN deadline and reaps the child on every exit path -- eval +// success, eval throw, malformed code, and a probe that outruns the +// self-owned deadline -- so a wedged or runaway child can never survive +// the probe call. +// +// The frozen dependency closure is materialized sidecar-side (host, +// no eval) through the injected `MaterializeWorkflowClosure` seam; only +// the load+evaluate+walk+project step runs in the child. The child +// receives the materialized package directory in its fresh, minimal env +// -- no ambient inputs, no sidecar keys. + +import { type } from "arktype"; + +import { getLogger } from "@intx/log"; +import { hexDecode } from "@intx/types/hex"; +import { + WorkflowProjectionDefinition, + type WorkflowProbeRequestFrame, +} from "@intx/types/sidecar"; + +import { MacedEnvelope, encodeEnvelope } from "../ipc/envelope"; +import { generateChannelId, generateHmacKey, verifyHmac } from "../ipc/crypto"; +import { DEFAULT_KILL_TIMEOUT_MS } from "../supervisor/child-termination"; +import { buildProbeChildEnv } from "./env"; +import { errorMessage } from "./errors"; +import { ProbeResultPayload, type WorkflowProbeResult } from "./protocol"; + +export type { WorkflowProbeResult } from "./protocol"; + +const logger = getLogger(["sidecar", "workflow-probe"]); + +/** + * Self-owned upper bound on how long a single probe child may run before + * the sidecar reaps it and fails the probe. Independent of the hub's + * `probeTimeoutMs`: the hub timeout only rejects the hub-side promise, + * whereas this deadline is what actually kills a wedged child. + */ +export const DEFAULT_PROBE_CHILD_TIMEOUT_MS = 30_000; + +/** + * Self-owned SIGTERM->SIGKILL escalation window when reaping the child. + * Mirrors the supervisor's `DEFAULT_KILL_TIMEOUT_MS` so a probe child + * that ignores SIGTERM is force-killed on the same schedule a supervised + * child is. + */ +export const DEFAULT_PROBE_CHILD_KILL_TIMEOUT_MS = DEFAULT_KILL_TIMEOUT_MS; + +// --------------------------------------------------------------------------- +// Closure materialization seam +// --------------------------------------------------------------------------- + +/** + * A materialized workflow package closure: the directory holding the + * workflow package's `package.json` (with its `node_modules/` laid out + * so the entry's bare-specifier imports resolve), plus a `cleanup` the + * handler always calls once the child has been reaped. + */ +export interface MaterializedWorkflowClosure { + readonly packageDir: string; + cleanup(): Promise; +} + +/** + * Host-side materializer for a probe frame's frozen closure. Fetches, + * verifies, extracts, and lays out the workflow package (and its + * dependency closure) into a resolvable tree, returning the package + * directory the child loads from. This runs on the sidecar host -- it is + * I/O, not author-code evaluation -- so the airlocked child only performs + * the load+evaluate step. The production materializer is host-supplied so + * `@intx/workflow-host` stays free of a `@intx/tool-packaging` dependency. + */ +export type MaterializeWorkflowClosure = ( + frame: WorkflowProbeRequestFrame, +) => Promise; + +// --------------------------------------------------------------------------- +// Child spawn seam +// --------------------------------------------------------------------------- + +/** + * Minimal handle over a spawned probe child. The probe needs only the + * child's stdout (the single result line), a kill primitive, and the + * `exited` promise for reaping -- no control/event channels, because the + * probe carries no bidirectional control traffic. + */ +export interface ProbeChildHandle { + readonly pid: number; + readonly stdout: ReadableStream; + kill(signal?: number | string): void; + readonly exited: Promise; +} + +/** + * Spawner the handler invokes to launch the one-shot probe child. + * The sidecar process passes a `Bun.spawn` spawner. Tests pass a spawner + * that records the spawned pid so they can assert the child was reaped. + */ +export type ProbeChildSpawner = (args: { + binaryPath: string; + env: Record; +}) => ProbeChildHandle; + +// --------------------------------------------------------------------------- +// Executor (host side) +// --------------------------------------------------------------------------- + +export interface WorkflowProbeExecutorOpts { + /** Host-side materializer for the frame's frozen closure. */ + materialize: MaterializeWorkflowClosure; + /** Child spawner. The sidecar process passes its `Bun.spawn` spawner. */ + spawnProbeChild: ProbeChildSpawner; + /** Path of `bin/workflow-probe-child`. The sidecar process resolves it. */ + binaryPath: string; + /** + * Self-owned deadline before the child is reaped and the probe fails. + * Independent of the hub's `probeTimeoutMs`. + */ + childTimeoutMs?: number; + /** SIGTERM->SIGKILL escalation window when reaping. */ + killTimeoutMs?: number; +} + +/** + * Build the sidecar's workflow-probe executor. The returned object + * satisfies the hub-agent `WorkflowProbeExecutor` seam: `probe(frame)` + * returns the inert projection + advisory grant set + wire hash, and + * throws when any step fails so the link answers `workflow.probe.error`. + * + * `probe` materializes the frozen closure, spawns a one-shot airlocked + * child to evaluate the workflow, and reaps that child on every exit + * path independent of the hub's probe timeout. + */ +export function createWorkflowProbeExecutor(opts: WorkflowProbeExecutorOpts): { + probe(frame: WorkflowProbeRequestFrame): Promise; +} { + const spawnProbeChild = opts.spawnProbeChild; + const binaryPath = opts.binaryPath; + const childTimeoutMs = opts.childTimeoutMs ?? DEFAULT_PROBE_CHILD_TIMEOUT_MS; + const killTimeoutMs = + opts.killTimeoutMs ?? DEFAULT_PROBE_CHILD_KILL_TIMEOUT_MS; + + async function probe( + frame: WorkflowProbeRequestFrame, + ): Promise { + const materialized = await opts.materialize(frame); + try { + return await runOneShotProbeChild({ + packageDir: materialized.packageDir, + spawnProbeChild, + binaryPath, + childTimeoutMs, + killTimeoutMs, + }); + } finally { + await materialized.cleanup(); + } + } + + return { probe }; +} + +interface RunOneShotProbeChildArgs { + readonly packageDir: string; + readonly spawnProbeChild: ProbeChildSpawner; + readonly binaryPath: string; + readonly childTimeoutMs: number; + readonly killTimeoutMs: number; +} + +/** + * Spawn a single probe child, drive it to its one result frame, and reap + * it on every exit path. The `finally` guarantees the child is killed + * whether the read succeeds, the child ships an error frame, the child + * exits without a frame (malformed code / crash), or the self-owned + * deadline fires first. + */ +async function runOneShotProbeChild( + args: RunOneShotProbeChildArgs, +): Promise { + const channelId = generateChannelId(); + const hmacKey = generateHmacKey(); + const env = buildProbeChildEnv({ + packageDir: args.packageDir, + channelId, + hmacKey, + }); + const handle = args.spawnProbeChild({ binaryPath: args.binaryPath, env }); + + let reaped = false; + async function reap(): Promise { + if (reaped) return; + reaped = true; + await reapChild(handle, args.killTimeoutMs); + } + + // Attach a catch so a post-reap stdout read error (the kill closes the + // pipe mid-read) resolves to null instead of surfacing as an unhandled + // rejection on the losing race branch. + const linePromise: Promise = readResultLine( + handle.stdout, + ).catch((err: unknown) => { + logger.debug`probe child ${String(handle.pid)} stdout read errored: ${errorMessage(err)}`; + return null; + }); + + const deadline = createDeadline(args.childTimeoutMs); + try { + // Race the result line against the deadline ONLY. Child exit is + // deliberately not a race arm: a child writes its result line and then + // exits promptly, so `handle.exited` and the buffered-line read both become + // ready, and an exit arm winning that race would discard an already-written + // result and fail the probe spuriously. Exit is not a distinct outcome the + // line read misses -- when the child exits its stdout write end closes, so + // `readResultLine` settles either with the trailing line (returned below) + // or null (the "closed its output" case). A child that neither writes nor + // exits is still caught by the deadline. + const outcome = await Promise.race([ + linePromise.then((line) => ({ kind: "line" as const, line })), + deadline.promise.then(() => ({ kind: "timeout" as const })), + ]); + + if (outcome.kind === "timeout") { + throw new Error( + `workflow probe child ${String(handle.pid)} did not produce a result within ${String(args.childTimeoutMs)}ms`, + ); + } + if (outcome.line === null) { + throw new Error( + `workflow probe child ${String(handle.pid)} closed its output without producing a result`, + ); + } + return await parseProbeResult(outcome.line, channelId, hmacKey); + } finally { + deadline.cancel(); + await reap(); + } +} + +/** + * Reap a probe child: SIGTERM, then SIGKILL if the exit does not land + * within `killTimeoutMs`. SIGKILL is unignorable, so `exited` is + * guaranteed to settle -- a child that traps or ignores SIGTERM cannot + * wedge this call. A kill against an already-exited child is a no-op. + */ +async function reapChild( + handle: ProbeChildHandle, + killTimeoutMs: number, +): Promise { + try { + handle.kill("SIGTERM"); + } catch (err) { + logger.debug`probe child ${String(handle.pid)} SIGTERM raised (already exited?): ${errorMessage(err)}`; + } + const deadline = createDeadline(killTimeoutMs); + const first = await Promise.race([ + handle.exited.then(() => "exited" as const), + deadline.promise.then(() => "deadline" as const), + ]); + deadline.cancel(); + if (first === "exited") return; + logger.warn`workflow probe child ${String(handle.pid)} did not exit on SIGTERM within ${String(killTimeoutMs)}ms; escalating to SIGKILL`; + try { + handle.kill("SIGKILL"); + } catch (err) { + logger.debug`probe child ${String(handle.pid)} SIGKILL raised (already exited?): ${errorMessage(err)}`; + } + await handle.exited.catch(() => { + // A non-zero exit on SIGKILL is the expected outcome; reaping treats + // child exit as success regardless of code. + }); +} + +/** + * Authenticate and parse the child's single result frame. Verifies the + * HMAC over the re-encoded envelope BEFORE trusting any field (mirroring + * the event channel's receiver), binds the frame to this spawn's + * channelId, then narrows the payload. A `ok: false` payload is turned + * into a throw so the probe fails with the child's reason. + */ +async function parseProbeResult( + line: string, + channelId: string, + hmacKey: Uint8Array, +): Promise { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch (cause) { + throw new Error("workflow probe child result is not valid JSON", { cause }); + } + const maced = MacedEnvelope(raw); + if (maced instanceof type.errors) { + throw new Error( + `workflow probe child result envelope failed validation: ${maced.summary}`, + ); + } + const envelopeBytes = encodeEnvelope(maced.envelope); + const macBytes = hexDecode(maced.mac); + const ok = await verifyHmac(envelopeBytes, macBytes, hmacKey); + if (!ok) { + throw new Error( + `workflow probe child result HMAC did not verify (channelId=${maced.envelope.channelId})`, + ); + } + if (maced.envelope.channelId !== channelId) { + throw new Error( + `workflow probe child result carried a foreign channelId ${JSON.stringify(maced.envelope.channelId)}`, + ); + } + const payload = ProbeResultPayload(maced.envelope.payload); + if (payload instanceof type.errors) { + throw new Error( + `workflow probe child result payload failed validation: ${payload.summary}`, + ); + } + if (!payload.ok) { + throw new Error(`workflow probe evaluation failed: ${payload.error}`); + } + const projection = WorkflowProjectionDefinition(payload.projection); + if (projection instanceof type.errors) { + throw new Error( + `workflow probe child projection failed validation: ${projection.summary}`, + ); + } + return { + projection, + grants: payload.grants, + grantWalkSnapshot: payload.grantWalkSnapshot, + wireHash: payload.wireHash, + }; +} + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +/** + * Read one newline-delimited line from a byte stream. Resolves the first + * complete line, or `null` when the stream closes without one (the child + * exited before writing). Releases the reader lock on every exit. + */ +async function readResultLine( + stream: ReadableStream, +): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder("utf-8"); + let pending = ""; + try { + for (;;) { + const { value, done } = await reader.read(); + if (value !== undefined) { + pending += decoder.decode(value, { stream: true }); + const nl = pending.indexOf("\n"); + if (nl >= 0) { + return pending.slice(0, nl).replace(/\r$/, ""); + } + } + if (done) { + const trailing = pending.replace(/\r?\n$/, ""); + return trailing.length > 0 ? trailing : null; + } + } + } finally { + reader.releaseLock(); + } +} + +function createDeadline(ms: number): { + promise: Promise; + cancel: () => void; +} { + let handle: ReturnType | undefined; + const promise = new Promise((resolve) => { + handle = setTimeout(resolve, ms); + }); + return { + promise, + cancel(): void { + if (handle !== undefined) clearTimeout(handle); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/probe/protocol.ts b/vendor/intx/workflow-host/src/probe/protocol.ts new file mode 100644 index 0000000000..488c3dd414 --- /dev/null +++ b/vendor/intx/workflow-host/src/probe/protocol.ts @@ -0,0 +1,70 @@ +import { type } from "arktype"; + +import type { DirectorRegistry, ToolDeclaration } from "@intx/agent"; +import type { GrantEffect } from "@intx/types"; +import { GrantWalkSnapshot } from "@intx/types/grant-snapshot"; +import type { WorkflowProjectionDefinition } from "@intx/types/sidecar"; +import type { WorkflowDefinition } from "@intx/workflow/definition"; + +/** + * Capability walk the probe child runs over the evaluated definition. + * The process that boots the probe passes the real walk. This module + * does not import it. + */ +export type WalkCapabilities = ( + workflow: WorkflowDefinition, + registry: DirectorRegistry, + pluginDefs: ReadonlyMap, +) => { + readonly perStep: ReadonlyMap< + string, + { + readonly grants: readonly string[]; + readonly grantEffects: ReadonlyMap; + } + >; + readonly unresolvedDirectors: readonly string[]; +}; + +// --------------------------------------------------------------------------- +// Result payload wire (child -> host) +// --------------------------------------------------------------------------- + +/** + * The child's single result payload, carried inside the HMAC-signed + * envelope. `ok: true` ships the inert projection, the advisory grant + * set, the un-flattened grant walk snapshot, and the wire hash; `ok: + * false` ships the failure reason (eval throw, malformed code) so the + * host can reject the probe with a meaningful message rather than a bare + * "child exited" surface. + */ +export const ProbeResultPayload = type({ + ok: "true", + projection: "unknown", + grants: "string[]", + grantWalkSnapshot: GrantWalkSnapshot, + wireHash: "string > 0", +}).or({ + ok: "false", + error: "string", +}); +export type ProbeResultPayload = typeof ProbeResultPayload.infer; + +/** + * The inert answer a probe execution produces: the workflow's inert + * needs-surface projection, the advisory grant set derived from it, the + * un-flattened grant walk snapshot the set is derived from, and the + * projection's content hash. Structurally the `WorkflowProbeResult` the + * hub-agent probe seam consumes. + * + * `grantWalkSnapshot` carries the per-step grant declarations (grant + * strings plus each step's tool-grant `grantEffects` map) and the + * definition's full `grantRequirements` -- the grouping and effect data + * the flattened `grants` union discards. + */ +export interface WorkflowProbeResult { + readonly projection: WorkflowProjectionDefinition; + readonly grants: string[]; + readonly grantWalkSnapshot: GrantWalkSnapshot; + readonly wireHash: string; +} diff --git a/vendor/intx/workflow-host/src/run-grants.ts b/vendor/intx/workflow-host/src/run-grants.ts new file mode 100644 index 0000000000..ce114bf759 --- /dev/null +++ b/vendor/intx/workflow-host/src/run-grants.ts @@ -0,0 +1,92 @@ +// Per-run grants read primitives shared by the two sidecar modules that +// resolve a run's authorization from disk: the supervisor wiring +// (`workflow-host-wiring.ts`), which reads a run's grants at the +// `onRunStart` barrier, and the in-process child runtime +// (`substrate-factory.ts`), which reads the PARENT run's grants +// to bind a spawned child's authorize. Both destinations read the same +// `runs//grants.json` file in a deployment's `workflow-run` repo, +// so the read lives in one place. +// +// The module's dependency surface is deliberately narrow -- node fs/path, +// arktype, and the substrate `RepoStore` type -- so importing it into the +// child subprocess module does not drag the supervisor wiring +// (`workflow-host-wiring.ts`) into the child binary. + +import { readFile } from "node:fs/promises"; +import { join as pathJoin } from "node:path"; + +import { type } from "arktype"; + +import type { RepoStore } from "@intx/hub-sessions/substrate"; + +import { isErrnoNotFound } from "./supervisor/credentials"; + +/** + * Path inside a deployment's `workflow-run` repo that carries a single + * run's grants. It sits under the run's own `runs//` subtree -- + * sibling to that run's `events/` blobs -- so a run's grants live and are + * reclaimed with the rest of the run's state. + */ +export function runGrantsPath(runId: string): string { + return `runs/${runId}/grants.json`; +} + +/** + * Envelope the per-run grants file carries: the canonical `{ grants: [] }` + * shape the deploy-time snapshot writes and validates. The inner entries + * stay `unknown` -- the child's authorize layer narrows each against its + * own grant-rule validator, exactly as the deploy-time snapshot's grants + * are surfaced untyped. + */ +export const RunGrantsFile = type({ + grants: "unknown[]", +}).onUndeclaredKey("ignore"); + +/** + * Read a single run's grants from `runs//grants.json` inside the + * deployment's `workflow-run` repo. The read is a working-tree read via + * `getRepoDir` (the same substrate access pattern the deploy-time + * per-step read uses), so it observes the tip the grants handler wrote. + * + * Returns the run's grants when the file exists. Returns `undefined` + * -- distinct from an empty grants array -- when the file is ABSENT + * (ENOENT), so the caller can distinguish "this run got no per-run + * grants file" from "this run's grants are the empty set". A file that + * exists but is malformed (invalid JSON or a shape the envelope rejects) + * THROWS: the file's presence implies a grants frame was delivered, and a + * structural failure is a boundary bug, not a default. + */ +export async function readRunGrants(args: { + repoStore: RepoStore; + anchorRunId: string; + runId: string; +}): Promise { + const dir = args.repoStore.getRepoDir({ + kind: "workflow-run", + id: args.anchorRunId, + }); + const filePath = pathJoin(dir, runGrantsPath(args.runId)); + let raw: string; + try { + raw = await readFile(filePath, "utf8"); + } catch (cause) { + if (isErrnoNotFound(cause)) return undefined; + throw cause; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + throw new Error( + `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} is not valid JSON`, + { cause }, + ); + } + const validated = RunGrantsFile(parsed); + if (validated instanceof type.errors) { + throw new Error( + `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} failed validation: ${validated.summary}`, + ); + } + return validated.grants; +} diff --git a/vendor/intx/workflow-host/src/seams/scheduler.ts b/vendor/intx/workflow-host/src/seams/scheduler.ts index 1abd960ff7..40b200389b 100644 --- a/vendor/intx/workflow-host/src/seams/scheduler.ts +++ b/vendor/intx/workflow-host/src/seams/scheduler.ts @@ -37,6 +37,12 @@ // matches the filename's seq, so the scheduler mints the next seq // inside the `writeTreePreservingPrefix` merge step and writes both // into the envelope and the filename. +// +// The commit flushes that run's pending buffer before the merge +// numbers `TimerFired`. An already-fired timer, or a log that already +// holds a terminal event, returns the tree unchanged. This is not the +// runtime `commit()`: `handleTimerFired` advances `lastSeq` even when +// the timer is already gone, so a duplicate would move the tip. import { type } from "arktype"; @@ -51,6 +57,10 @@ import { WORKFLOW_RUN_EVENTS_DIR, WORKFLOW_RUN_RUNS_PREFIX, } from "@intx/hub-sessions/substrate"; +import { + withRunCommitBarrier, + type RepoStore as RuntimeRepoStore, +} from "@intx/workflow/runtime"; /** * Substrate-shape envelope for the workflow-event blob committed to @@ -86,6 +96,13 @@ export type SchedulerOpts = { * the `runs//events/.json` writes the scheduler emits. */ repoStore: RepoStore; + /** + * Runtime store that owns the commit chain. A timer flushes that + * run's pending buffer before the substrate merge numbers + * `TimerFired`. One store serves every run in this process; the + * barrier selects the buffer by run id. + */ + runtimeStore: RuntimeRepoStore; /** * Principal the scheduler presents to the substrate. The substrate * gates every operation behind `authorize`; the scheduler's @@ -494,47 +511,65 @@ async function commitTimerFired( ); } const prefix = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/`; - await opts.repoStore.writeTreePreservingPrefix( - opts.principal, - owningRepoId, - opts.ref, - { - preservePrefix: prefix, - merge: async (existing) => { - let maxSeq = -1; - let alreadyFired = false; - for (const [filepath, contents] of existing) { - const name = filepath.slice(prefix.length); - const seq = requireEventSeq(name, `${prefix}${name}`); - if (seq > maxSeq) maxSeq = seq; - try { - const parsed: unknown = JSON.parse( - new TextDecoder().decode(contents), - ); - if (isMatchingTimerFired(parsed, timerId)) { - alreadyFired = true; + await withRunCommitBarrier({ repoStore: opts.runtimeStore }, runId, () => + opts.repoStore.writeTreePreservingPrefix( + opts.principal, + owningRepoId, + opts.ref, + { + preservePrefix: prefix, + merge: async (existing) => { + let maxSeq = -1; + let alreadyFired = false; + let terminal = false; + for (const [filepath, contents] of existing) { + const name = filepath.slice(prefix.length); + const seq = requireEventSeq(name, `${prefix}${name}`); + if (seq > maxSeq) maxSeq = seq; + try { + const parsed: unknown = JSON.parse( + new TextDecoder().decode(contents), + ); + if (isMatchingTimerFired(parsed, timerId)) { + alreadyFired = true; + } + if (isTerminalOnDiskEvent(parsed)) { + terminal = true; + } + } catch { + // Skip on parse failure -- a corrupt blob would have + // been rejected by validatePush at write time; treat as + // a non-matching entry. } - } catch { - // Skip on parse failure -- a corrupt blob would have - // been rejected by validatePush at write time; treat as - // a non-matching entry. } - } - const out: Record = {}; - for (const [filepath, contents] of existing) { - out[filepath] = new TextDecoder().decode(contents); - } - if (alreadyFired) return out; - const nextSeq = maxSeq + 1; - out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({ - seq: nextSeq, - type: "TimerFired", - timerId, - }); - return out; + const out: Record = {}; + for (const [filepath, contents] of existing) { + out[filepath] = new TextDecoder().decode(contents); + } + // An already-fired timer and a log that already reached a + // terminal event both leave the tree unchanged. Appending + // past RunFailed, RunCompleted, or RunCancelled would make + // the next resume disagree with the terminal result. + if (alreadyFired || terminal) return out; + const nextSeq = maxSeq + 1; + out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({ + seq: nextSeq, + type: "TimerFired", + timerId, + }); + return out; + }, + message: `TimerFired ${timerId} for run ${runId}`, }, - message: `TimerFired ${timerId} for run ${runId}`, - }, + ), + ); +} + +function isTerminalOnDiskEvent(parsed: unknown): boolean { + if (typeof parsed !== "object" || parsed === null) return false; + const type = (parsed as { type?: unknown }).type; + return ( + type === "RunCompleted" || type === "RunFailed" || type === "RunCancelled" ); } diff --git a/vendor/intx/workflow-host/src/seams/signal-channel.ts b/vendor/intx/workflow-host/src/seams/signal-channel.ts index a6ec5fa2a0..9770d664a9 100644 --- a/vendor/intx/workflow-host/src/seams/signal-channel.ts +++ b/vendor/intx/workflow-host/src/seams/signal-channel.ts @@ -4,23 +4,32 @@ // wiring. Its `deliver` commits a `SignalReceived` blob to the run's // event log; commit-success is the dedup gate (the state machine's // `observedSignalIds` rejects a duplicate `signalId` on the next -// transition). `awaitNext` consults the state-machine `RunState` for -// signals that were committed and reduced into `unconsumedSignals` -// before the awaiter subscribed, then falls back to a per-name -// `subscribeKind` tail of the run's events ref. +// transition). `awaitNext` consults the state-machine `RunState` +// before subscribing. A name awaiter reads the head of +// `unconsumedSignals`. A correlated awaiter reads +// `unconsumedByCorrelation` for an item whose signalName matches. +// Otherwise it waits on a per-slot `subscribeKind` tail. // -// Per-name FIFO across concurrent awaiters: the channel maintains an -// awaiter queue per signal name. A single `subscribeKind` loop per -// name pulls events from the substrate; each matching event resolves -// the head of that queue. The loop starts on first `awaitNext` for a -// name and tears down when the queue empties or `stop()` is called. +// Slots do not share a tail. The name slot key is `name\0${name}` +// and a correlated slot key is `corr\0${correlationId}\0${name}`. +// Each slot has its own awaiter queue and its own `subscribeKind` +// loop, started on the first `awaitNext` for that slot and torn +// down when that queue empties or `stop()` is called. A delivery +// resolves only the slot whose correlationId equals the event's; +// both absent selects the name slot. // -// Commit-ordering invariant: `deliver` commits the `SignalReceived` -// event blob first, then returns. Resolution of an awaiter happens -// from the `subscribeKind` loop only after the substrate has surfaced -// the commit. The awaiter is never resolved from inside `deliver`'s -// call site, so resume-after-crash sees a coherent log: an awaiter -// that resolved must have a corresponding committed `SignalReceived`. +// Commit-ordering invariant: `deliver` flushes the run's pending +// buffer through `runtimeStore`, then commits the `SignalReceived` +// blob, then returns. The flush runs under the run's commit barrier +// so a sibling event already validated into the buffer keeps its +// sequence and the signal takes the next durable sequence. A log that +// already holds a terminal event is left unchanged, so a delivery that +// loses the race does not append after it. Resolution +// of an awaiter happens from the `subscribeKind` loop only after the +// substrate has surfaced the commit. The awaiter is never resolved +// from inside `deliver`'s call site, so resume-after-crash sees a +// coherent log: an awaiter that resolved must have a corresponding +// committed `SignalReceived`. import { type } from "arktype"; @@ -32,6 +41,10 @@ import type { } from "@intx/hub-sessions/substrate"; import { subscribeKind } from "@intx/hub-sessions/substrate"; import type { RunState, SignalChannel } from "@intx/workflow"; +import { + withRunCommitBarrier, + type RepoStore as RuntimeRepoStore, +} from "@intx/workflow/runtime"; /** * Substrate-shape envelope for the `SignalReceived` event blob @@ -51,6 +64,7 @@ export const SignalReceivedEnvelope = type({ signalName: "string", signalId: "string", payload: "unknown", + "correlationId?": "string > 0", }); export type SignalReceivedEnvelope = typeof SignalReceivedEnvelope.infer; @@ -63,6 +77,13 @@ export type SignalChannelOpts = { * must accept that path layout. */ repoStore: RepoStore; + /** + * Runtime store that owns this run's commit chain. `deliver` flushes + * its pending buffer before the substrate merge numbers the signal. + * One store serves every run in the deployment; the barrier selects + * the buffer by `runId`. + */ + runtimeStore: RuntimeRepoStore; /** * Principal the channel presents to the substrate. The substrate * gates every operation behind `authorize`; the principal must be @@ -99,12 +120,22 @@ export type SignalChannelOpts = { }; type Awaiter = { - resolve: (value: { payload: unknown; signalId: string }) => void; + resolve: (value: { + payload: unknown; + signalId: string; + correlationId?: string; + }) => void; reject: (cause: unknown) => void; signal?: AbortSignal; onAbort?: () => void; }; +function slotOf(name: string, correlationId: string | undefined): string { + return correlationId === undefined + ? `name\0${name}` + : `corr\0${correlationId}\0${name}`; +} + type NameSubscription = { abort: AbortController; done: Promise; @@ -112,7 +143,7 @@ type NameSubscription = { export type SignalChannelHandle = SignalChannel & { /** - * Tear down every per-name subscription and reject every pending + * Tear down every slot subscription and reject every pending * awaiter. Idempotent. After `stop()` the channel holds no * substrate watcher handles. */ @@ -128,17 +159,27 @@ export function createWorkflowHostSignalChannel( function peekState( name: string, - ): { payload: unknown; signalId: string } | null { + correlationId: string | undefined, + ): { payload: unknown; signalId: string; correlationId?: string } | null { const state = opts.readState(); + if (correlationId !== undefined) { + const queued = state.unconsumedByCorrelation.get(correlationId); + const head = queued?.find((item) => item.signalName === name); + if (head === undefined) return null; + return { + payload: head.payload, + signalId: head.id, + correlationId, + }; + } const queue = state.unconsumedSignals.get(name); if (queue === undefined || queue.length === 0) return null; const head = queue[0]; if (head === undefined) return null; - // The state-machine `unconsumedSignals` is drained by the - // runtime body's next `SignalAwaited` reduction, not by the - // channel. The channel reads the head; the caller commits the - // SignalAwaited that consumes it. This separation keeps the - // channel free of state-machine mutation responsibilities. + // The state-machine name queue is drained by the runtime body's + // next name-addressed `SignalAwaited`, not by the channel. The + // channel reads the head; the caller commits the SignalAwaited + // that consumes it. return { payload: head.payload, signalId: head.id }; } @@ -169,8 +210,12 @@ export function createWorkflowHostSignalChannel( return queue.length; } - function startNameSubscription(name: string): void { - if (subscriptions.has(name)) return; + function startNameSubscription( + name: string, + correlationId: string | undefined, + ): void { + const slot = slotOf(name, correlationId); + if (subscriptions.has(slot)) return; const abort = new AbortController(); // The teardown must run synchronously *before* the awaiter wakes // when this loop drains, otherwise the awaiter's continuation @@ -181,8 +226,8 @@ export function createWorkflowHostSignalChannel( // aborting *before* resolving the awaiter keeps the per-name // invariant intact across multi-round resolution cycles. const teardown = (): void => { - if (subscriptions.get(name)?.abort === abort) { - subscriptions.delete(name); + if (subscriptions.get(slot)?.abort === abort) { + subscriptions.delete(slot); } abort.abort(); }; @@ -203,18 +248,18 @@ export function createWorkflowHostSignalChannel( for await (const entry of iter) { if (stopped) break; if (!matchesAwaiter(entry, name)) continue; + if (entry.event.correlationId !== correlationId) continue; const observed = opts.readState().observedSignalIds; if (observed.has(entry.event.signalId)) continue; - const next = shiftAwaiter(name); + const next = shiftAwaiter(slot); if (next === null) { // No awaiter left -- the queue was drained between the // event landing and this loop reaching it. The reduced - // state machine queues the event into - // `unconsumedSignals`; the next `awaitNext` for the same - // name picks it up via the state-reader path. + // state machine keeps the event; the next `awaitNext` for + // the same slot picks it up via the state-reader path. break; } - if (awaiterCount(name) === 0) { + if (awaiterCount(slot) === 0) { // Resolving the last awaiter on this subscription. Tear // down before the resolve so any awaitNext call inside // the awaiter's continuation installs a fresh @@ -223,12 +268,18 @@ export function createWorkflowHostSignalChannel( next.resolve({ payload: entry.event.payload, signalId: entry.event.signalId, + ...(entry.event.correlationId !== undefined + ? { correlationId: entry.event.correlationId } + : {}), }); return; } next.resolve({ payload: entry.event.payload, signalId: entry.event.signalId, + ...(entry.event.correlationId !== undefined + ? { correlationId: entry.event.correlationId } + : {}), }); } } finally { @@ -239,7 +290,7 @@ export function createWorkflowHostSignalChannel( teardown(); } })(); - subscriptions.set(name, { abort, done }); + subscriptions.set(slot, { abort, done }); } async function stopSubscription(name: string): Promise { @@ -253,69 +304,84 @@ export function createWorkflowHostSignalChannel( } return { - async deliver(name, payload, signalId) { + async deliver(name, payload, signalId, correlationId) { if (stopped) { throw new Error("signal channel: deliver after stop"); } const id = signalId ?? opts.newId(); const at = opts.clock().toISOString(); const prefix = `runs/${opts.runId}/events/`; - await opts.repoStore.writeTreePreservingPrefix( - opts.principal, - opts.repoId, - opts.ref, - { - preservePrefix: prefix, - merge: async (existing) => { - let maxSeq = -1; - let duplicate = false; - for (const [filepath, contents] of existing) { - const fname = filepath.slice(prefix.length); - const match = /^(0|[1-9][0-9]*)\.json$/.exec(fname); - if (match === null) continue; - const seqStr = match[1]; - if (seqStr === undefined) continue; - const seq = Number.parseInt(seqStr, 10); - if (seq > maxSeq) maxSeq = seq; - try { - const parsed: unknown = JSON.parse( - new TextDecoder().decode(contents), - ); - if (isMatchingSignalId(parsed, id)) { - duplicate = true; + await withRunCommitBarrier( + { repoStore: opts.runtimeStore }, + opts.runId, + () => + opts.repoStore.writeTreePreservingPrefix( + opts.principal, + opts.repoId, + opts.ref, + { + preservePrefix: prefix, + merge: async (existing) => { + let maxSeq = -1; + let duplicate = false; + let terminal = false; + for (const [filepath, contents] of existing) { + const fname = filepath.slice(prefix.length); + const match = /^(0|[1-9][0-9]*)\.json$/.exec(fname); + if (match === null) continue; + const seqStr = match[1]; + if (seqStr === undefined) continue; + const seq = Number.parseInt(seqStr, 10); + if (seq > maxSeq) maxSeq = seq; + try { + const parsed: unknown = JSON.parse( + new TextDecoder().decode(contents), + ); + if (isMatchingSignalId(parsed, id)) { + duplicate = true; + } + if (isTerminalOnDiskEvent(parsed)) { + terminal = true; + } + } catch { + // A corrupt blob is rejected by validatePush at write + // time. Treat as non-matching here. + } } - } catch { - // A corrupt blob is rejected by validatePush at write - // time. Treat as non-matching here. - } - } - const out: Record = {}; - for (const [filepath, contents] of existing) { - out[filepath] = new TextDecoder().decode(contents); - } - if (duplicate) return out; - const nextSeq = maxSeq + 1; - // The workflow-run kind handler's `EventEnvelope` - // validator requires `seq: number` on every event blob; - // the same `nextSeq` we use to mint the filename also - // carries into the body so a reader that hydrates the - // envelope (state-machine resume, audit reads) sees a - // self-describing event without consulting the filename. - out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({ - type: "SignalReceived", - seq: nextSeq, - signalName: name, - signalId: id, - payload, - at, - }); - return out; - }, - message: `SignalReceived ${id} (${name}) for run ${opts.runId}`, - }, + const out: Record = {}; + for (const [filepath, contents] of existing) { + out[filepath] = new TextDecoder().decode(contents); + } + // A repeated signalId and a log that already reached a + // terminal event both leave the tree unchanged. Appending + // past RunFailed/RunCompleted/RunCancelled would make the + // next resume disagree with the terminal result already + // reported. + if (duplicate || terminal) return out; + const nextSeq = maxSeq + 1; + // The workflow-run kind handler's `EventEnvelope` + // validator requires `seq: number` on every event blob; + // the same `nextSeq` we use to mint the filename also + // carries into the body so a reader that hydrates the + // envelope (state-machine resume, audit reads) sees a + // self-describing event without consulting the filename. + out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({ + type: "SignalReceived", + seq: nextSeq, + signalName: name, + signalId: id, + payload, + at, + ...(correlationId !== undefined ? { correlationId } : {}), + }); + return out; + }, + message: `SignalReceived ${id} (${name}) for run ${opts.runId}`, + }, + ), ); }, - async awaitNext(name, signal) { + async awaitNext(name, signal, correlationId) { if (stopped) { throw new Error("signal channel: awaitNext after stop"); } @@ -323,43 +389,46 @@ export function createWorkflowHostSignalChannel( throw new Error("aborted"); } // Drain the state-machine queue first. A signal that was - // committed before the awaiter subscribed lives in - // `unconsumedSignals` after log replay. - const queued = peekState(name); + // committed before the awaiter subscribed lives in the name + // queue, or under its correlation, after log replay. + const queued = peekState(name, correlationId); if (queued !== null) return queued; - return new Promise<{ payload: unknown; signalId: string }>( - (resolve, reject) => { - const awaiter: Awaiter = { - resolve, - reject, - ...(signal !== undefined ? { signal } : {}), - }; - if (signal !== undefined) { - const onAbort = (): void => { - const list = awaiters.get(name); - if (list !== undefined) { - const idx = list.indexOf(awaiter); - if (idx >= 0) list.splice(idx, 1); - if (list.length === 0) { - awaiters.delete(name); - void stopSubscription(name); - } + const slot = slotOf(name, correlationId); + return new Promise<{ + payload: unknown; + signalId: string; + correlationId?: string; + }>((resolve, reject) => { + const awaiter: Awaiter = { + resolve, + reject, + ...(signal !== undefined ? { signal } : {}), + }; + if (signal !== undefined) { + const onAbort = (): void => { + const list = awaiters.get(slot); + if (list !== undefined) { + const idx = list.indexOf(awaiter); + if (idx >= 0) list.splice(idx, 1); + if (list.length === 0) { + awaiters.delete(slot); + void stopSubscription(slot); } - reject(new Error("aborted")); - }; - awaiter.onAbort = onAbort; - signal.addEventListener("abort", onAbort, { once: true }); - } - let list = awaiters.get(name); - if (list === undefined) { - list = []; - awaiters.set(name, list); - } - list.push(awaiter); - startNameSubscription(name); - }, - ); + } + reject(new Error("aborted")); + }; + awaiter.onAbort = onAbort; + signal.addEventListener("abort", onAbort, { once: true }); + } + let list = awaiters.get(slot); + if (list === undefined) { + list = []; + awaiters.set(slot, list); + } + list.push(awaiter); + startNameSubscription(name, correlationId); + }); }, async stop() { if (stopped) return; @@ -381,6 +450,14 @@ export function createWorkflowHostSignalChannel( }; } +function isTerminalOnDiskEvent(parsed: unknown): boolean { + if (typeof parsed !== "object" || parsed === null) return false; + const type = (parsed as { type?: unknown }).type; + return ( + type === "RunCompleted" || type === "RunFailed" || type === "RunCancelled" + ); +} + function isMatchingSignalId(parsed: unknown, signalId: string): boolean { if (typeof parsed !== "object" || parsed === null) return false; const obj = parsed as { diff --git a/vendor/intx/workflow-host/src/supervisor/cancel-signing.ts b/vendor/intx/workflow-host/src/supervisor/cancel-signing.ts index f3c65b82c9..c4a257a910 100644 --- a/vendor/intx/workflow-host/src/supervisor/cancel-signing.ts +++ b/vendor/intx/workflow-host/src/supervisor/cancel-signing.ts @@ -33,7 +33,7 @@ import type { WorkflowRunSupervisorPrincipal, } from "@intx/hub-sessions/substrate"; import type { CancelOrigin } from "@intx/workflow"; -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; import type { PrincipalSigner, @@ -161,7 +161,7 @@ export async function commitCancelRequested( { preservePrefix: prefix, merge: async (existing) => { - let maxSeq = -1; + let maxSeq = 0; for (const filepath of existing.keys()) { const name = filepath.slice(prefix.length); const seq = parseEventSeq(name); diff --git a/vendor/intx/workflow-host/src/supervisor/credentials.ts b/vendor/intx/workflow-host/src/supervisor/credentials.ts index 4254edaccc..bcb3748673 100644 --- a/vendor/intx/workflow-host/src/supervisor/credentials.ts +++ b/vendor/intx/workflow-host/src/supervisor/credentials.ts @@ -29,7 +29,7 @@ import { type } from "arktype"; -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; import type { Principal, RepoId, diff --git a/vendor/intx/workflow-host/src/supervisor/recycle.ts b/vendor/intx/workflow-host/src/supervisor/recycle.ts index 3d374f6afb..29eeca6c36 100644 --- a/vendor/intx/workflow-host/src/supervisor/recycle.ts +++ b/vendor/intx/workflow-host/src/supervisor/recycle.ts @@ -5,19 +5,13 @@ // Recycle is the supervisor's "same deploy tree, fresh process" path. // It tears the existing workflow-process child down and stands a new // one up against the SAME deploy tree (same materialized source -// closure, same per-step credential repos). It is STRICTLY ORTHOGONAL -// TO REDEPLOY: -// -// - Recycle = same deploy tree, fresh process. -// - Redeploy = new deploy tree. +// closure, same per-step credential repos). A deployment keeps its +// deploy tree for its life; a different tree means a new deployment. // // Recycle does not refetch the deploy tree, does not consult an -// updated workflow definition, does not re-resolve agents. If a -// deploy-tree change is needed the host runs redeploy, which is a -// different code path with different authorization and a different -// rollback shape. The recycle module must never grow a "maybe also -// refetch the deploy tree" mode -- that would erase the orthogonality -// and let a recycle silently turn into a redeploy. +// updated workflow definition, does not re-resolve agents. The recycle +// module must never grow a "maybe also refetch the deploy tree" mode -- +// that would let a recycle silently change what a deployment runs. // // Six-step sequence (locked): // @@ -96,15 +90,16 @@ import { generateKeyPair } from "@intx/crypto"; import { createControlChannelSender, - generateChannelId, - generateHmacKey, - receiveControlChannel, - receiveEventChannel, type ControlChannelSender, - type ControlPayload, +} from "../ipc/control-sender"; +import { generateChannelId, generateHmacKey } from "../ipc/crypto"; +import { receiveControlChannel } from "../ipc/control-receiver"; +import { + receiveEventChannel, type EventPayload, type ReceivedEvent, -} from "../ipc/index"; +} from "../ipc/event-channel"; +import type { ControlPayload } from "../ipc/control-payloads"; import { assembleCredentialsSnapshot, @@ -515,11 +510,9 @@ export async function triggerRecycle( /** * Reap a respawned child that was spawned and wired but never installed - * on `state`. Such a child is invisible to the supervisor's - * recycle-failure teardown -- that path reaps the PRIOR cohort - * (`state.handle` during `recycling`) -- so a respawn that fails after - * the spawn must reap the new child here or it leaks its OS process and - * both IPC channels. + * on `state`. Recycle owns startup-failure cleanup even when invoked + * without a supervisor. A supervisor also tracks this handle so a + * concurrent shutdown can stop it before the startup failure settles. * * Kill FIRST, then finalize the pumps: process death drives EOF on both * channels, which unparks `waitForReady`'s in-flight `iter.next()` diff --git a/vendor/intx/workflow-host/src/supervisor/spawn-env.ts b/vendor/intx/workflow-host/src/supervisor/spawn-env.ts index 698d5fd375..9d24e2f5b7 100644 --- a/vendor/intx/workflow-host/src/supervisor/spawn-env.ts +++ b/vendor/intx/workflow-host/src/supervisor/spawn-env.ts @@ -7,7 +7,7 @@ // broke every recycle -- is no longer expressible: the required keys are // built as an exactly-typed record, so omitting one is a compile error. -import { hexEncode } from "@intx/types"; +import { hexEncode } from "@intx/types/hex"; import type { RequiredSpawnEnvKey } from "../child/env-bootstrap"; diff --git a/vendor/intx/workflow-host/src/supervisor/supervisor.ts b/vendor/intx/workflow-host/src/supervisor/supervisor.ts index b94d2eda52..eb886fb49d 100644 --- a/vendor/intx/workflow-host/src/supervisor/supervisor.ts +++ b/vendor/intx/workflow-host/src/supervisor/supervisor.ts @@ -64,36 +64,51 @@ import { type WorkflowRunSupervisorPrincipal, type WorkflowRunWorkflowProcessPrincipal, } from "@intx/hub-sessions/substrate"; +import { base64Decode, base64Encode } from "@intx/types/base64"; +import { deriveMessageId } from "@intx/types/message-id"; +import { deriveWorkflowRunId } from "@intx/types/workflow-run-id"; +import { signalName } from "@intx/types/signals"; +import { RepoId } from "@intx/types/repo"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import { inboundMessageToRaw } from "@intx/mail-memory"; import { - base64Decode, - base64Encode, - deriveMessageId, - deriveWorkflowRunId, - signalName, -} from "@intx/types"; -import { RepoId, type CredentialDelivery } from "@intx/types/sidecar"; -import type { - ApprovalSnapshot, - InferenceSource, - Mail, - MessageHeaders, - OutboundMessage, + executeSearch, + executeThread, + fetchFull, + fetchHeaders, + fetchPart, + fetchStructure, + type StoredEnvelope, +} from "@intx/mailbox"; +import { + isMessageTransportError, + type ApprovalSnapshot, + type InferenceSource, + type InboundMessage, + type Mail, + type MessageHeaders, + type MessagePart, + type MessageTransportCondition, + type OutboundMessage, + type SearchQuery, } from "@intx/types/runtime"; -import type { StoredEnvelope } from "@intx/mailbox"; import type { CancelOrigin } from "@intx/workflow"; import { createControlChannelSender, - generateChannelId, - generateHmacKey, - receiveControlChannel, - receiveEventChannel, type ControlChannelSender, - type ControlPayload, +} from "../ipc/control-sender"; +import { generateChannelId, generateHmacKey } from "../ipc/crypto"; +import { receiveControlChannel } from "../ipc/control-receiver"; +import { + receiveEventChannel, type EventPayload, type ReceivedEvent, - type OutboundMessagePayload, -} from "../ipc/index"; +} from "../ipc/event-channel"; +import type { + ControlPayload, + OutboundMessagePayload, +} from "../ipc/control-payloads"; import { assembleCredentialsSnapshot, @@ -105,11 +120,16 @@ import { buildChildSpawnEnv } from "./spawn-env"; import { compactRunEvents } from "./run-event-compaction"; import { recoverInterruptedCompactions } from "./run-event-recovery"; import { decodeMail } from "@intx/mime"; -import { commitMail, InvalidMailError } from "../adapters/mail-part-store"; +import { + commitMail, + createMailPartReader, + InvalidMailError, +} from "../adapters/mail-part-store"; import { mergeCredentialDelivery } from "../child/credential-cell"; import { createSubstrateMailboxStore, MAILBOX_INBOX_DIR, + type MailboxSyncResult, type SubstrateMailboxStore, } from "../adapters/substrate-mailbox-store"; import { @@ -243,7 +263,8 @@ export interface WorkflowSupervisor { * Used by the host directly for `supervisor-operator` and `hub- * admin` origins; the `self` origin is invoked indirectly by the * supervisor when the child requests cancellation over the - * control IPC. + * control IPC. An active child's runtime writer is flushed and paused before + * the signed append. The caller owns the deadline for an unresponsive child. */ requestCancel(opts: CancelRequestOpts): Promise; /** @@ -412,6 +433,11 @@ export type DeliverSignalOpts = { signalId: string; /** Opaque signal payload the awaiter resolves with. */ payload: unknown; + /** + * The onTrigger relay park this delivery addresses. Omitted, the + * delivery is name-FIFO and does not complete a correlated park. + */ + correlationId?: string; }; export type DeliverSourcesOpts = { @@ -451,6 +477,255 @@ export type RecycleOpts = { origin?: RecycleOrigin; }; +type MailboxCallRequest = Extract< + ControlPayload, + { type: "mailbox.call.request" } +>["data"]; + +type MailboxCallResponse = Extract< + ControlPayload, + { type: "mailbox.call.response" } +>["data"]; + +type MailboxCallSuccess = Extract; + +type MailboxCallSearchQuery = Extract< + MailboxCallRequest, + { op: "search" } +>["query"]; + +/** + * Search dates arrive as strings. A bad one is a failed call, reported by the + * handler, rather than a schema rejection of the control frame. + */ +function reviveSearchDates(query: MailboxCallSearchQuery): SearchQuery { + const revived: SearchQuery = {}; + if (query.from !== undefined) revived.from = query.from; + if (query.to !== undefined) revived.to = query.to; + if (query.cc !== undefined) revived.cc = query.cc; + if (query.bcc !== undefined) revived.bcc = query.bcc; + if (query.header !== undefined) { + revived.header = { + field: query.header.field, + contains: query.header.contains, + }; + } + if (query.before !== undefined) + revived.before = reviveSearchDate(query.before); + if (query.after !== undefined) revived.after = reviveSearchDate(query.after); + if (query.on !== undefined) revived.on = reviveSearchDate(query.on); + if (query.sentBefore !== undefined) { + revived.sentBefore = reviveSearchDate(query.sentBefore); + } + if (query.sentAfter !== undefined) { + revived.sentAfter = reviveSearchDate(query.sentAfter); + } + if (query.sentOn !== undefined) + revived.sentOn = reviveSearchDate(query.sentOn); + if (query.hasFlags !== undefined) revived.hasFlags = query.hasFlags; + if (query.missingFlags !== undefined) + revived.missingFlags = query.missingFlags; + if (query.body !== undefined) revived.body = query.body; + if (query.text !== undefined) revived.text = query.text; + if (query.largerThan !== undefined) revived.largerThan = query.largerThan; + if (query.smallerThan !== undefined) revived.smallerThan = query.smallerThan; + if (query.and !== undefined) revived.and = query.and.map(reviveSearchDates); + if (query.or !== undefined) revived.or = query.or.map(reviveSearchDates); + if (query.not !== undefined) revived.not = reviveSearchDates(query.not); + return revived; +} + +function reviveSearchDate(value: string): Date { + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + throw new Error(`Invalid search date ${JSON.stringify(value)}`); + } + return date; +} + +function missingMailbox(name: string): string { + return `Mailbox "${name}" does not exist`; +} + +/** + * The supervisor's mailbox policy. A refusal is the whole answer: the caller + * does not open the store. A name other than INBOX is not uniformly + * NONEXISTENT — creating a mailbox is CANNOT for every name, and deleting + * INBOX is CANNOT while deleting any other name is NONEXISTENT. + */ +function refuseMailboxCall( + data: MailboxCallRequest, +): { condition: MessageTransportCondition; reason: string } | undefined { + switch (data.op) { + case "createMailbox": + return { + condition: "CANNOT", + reason: `Cannot create mailbox "${data.name}"`, + }; + case "deleteMailbox": + if (data.name !== MAILBOX_INBOX_DIR) { + return { condition: "NONEXISTENT", reason: missingMailbox(data.name) }; + } + return { + condition: "CANNOT", + reason: `Cannot delete mailbox "${data.name}"`, + }; + case "move": + case "copy": + return refuseMoveOrCopy(data.ref.mailbox, data.toMailbox); + case "createList": + case "listMembers": + case "subscribe": + case "unsubscribe": + return { + condition: "CANNOT", + reason: "Distribution list management is not implemented", + }; + case "search": + case "thread": + case "sync": + case "getMailboxStatus": + case "append": + case "watch": + if (data.mailbox !== MAILBOX_INBOX_DIR) { + return { + condition: "NONEXISTENT", + reason: missingMailbox(data.mailbox), + }; + } + return undefined; + case "fetchHeaders": + case "fetchStructure": + case "fetchPart": + case "fetchFull": + if (data.ref.mailbox !== MAILBOX_INBOX_DIR) { + return { + condition: "NONEXISTENT", + reason: missingMailbox(data.ref.mailbox), + }; + } + return undefined; + case "listMailboxes": + case "readMailPart": + return undefined; + } +} + +function refuseMoveOrCopy( + fromMailbox: string, + toMailbox: string, +): { condition: MessageTransportCondition; reason: string } { + if (fromMailbox !== MAILBOX_INBOX_DIR) { + return { condition: "NONEXISTENT", reason: missingMailbox(fromMailbox) }; + } + if (toMailbox !== MAILBOX_INBOX_DIR) { + return { condition: "NONEXISTENT", reason: missingMailbox(toMailbox) }; + } + return { + condition: "CANNOT", + reason: `Cannot move/copy a message within "${MAILBOX_INBOX_DIR}"`, + }; +} + +function mailboxCallFailure( + data: MailboxCallRequest, + reason: string, + condition?: MessageTransportCondition, +): Extract { + if (condition === undefined) { + return { requestId: data.requestId, ok: false, op: data.op, reason }; + } + return { + requestId: data.requestId, + ok: false, + op: data.op, + reason, + condition, + }; +} + +function projectFetchedMessage( + message: InboundMessage, +): Extract["value"] { + const value: Extract["value"] = { + ref: message.ref, + headers: message.headers, + flags: message.flags, + signatureStatus: message.signatureStatus, + }; + if (message.content !== undefined) value.content = message.content; + if (message.payload !== undefined) value.payload = message.payload; + if (message.attachments !== undefined) { + value.attachments = message.attachments.map((attachment) => { + const projected: { + name: string; + contentType: string; + dataBase64: string; + part?: string; + } = { + name: attachment.name, + contentType: attachment.contentType, + dataBase64: base64Encode(attachment.data), + }; + if (attachment.part !== undefined) projected.part = attachment.part; + return projected; + }); + } + return value; +} + +/** + * The store reports one delta of new arrivals and flag changes together, and + * its flags are a `Set` (which JSON would encode as `{}`). Split on the + * caller's `uidNext` and copy the flags out as arrays. The caller receives + * the finished sync result. + */ +function syncResultForCaller( + mailbox: string, + uidNext: number, + result: MailboxSyncResult, +): Extract["value"] { + if (result.resync) { + return { + vanished: [], + changed: [], + newMessages: result.messages.map((message) => ({ + uid: message.uid, + mailbox, + })), + fullResyncRequired: true, + }; + } + const newMessages: { uid: number; mailbox: string }[] = []; + const changed: { uid: number; flags: string[] }[] = []; + for (const message of result.changed) { + if (message.uid >= uidNext) { + newMessages.push({ uid: message.uid, mailbox }); + } else { + changed.push({ uid: message.uid, flags: Array.from(message.flags) }); + } + } + return { + vanished: [...result.vanished], + changed, + newMessages, + fullResyncRequired: false, + }; +} + +function projectPart( + part: MessagePart, +): Extract["value"] { + const value: Extract["value"] = { + contentType: part.contentType, + contentBase64: base64Encode(part.content), + }; + if (part.encoding !== undefined) value.encoding = part.encoding; + if (part.filename !== undefined) value.filename = part.filename; + if (part.disposition !== undefined) value.disposition = part.disposition; + return value; +} + /** * Construct a per-deployment supervisor. All host-specific * dependencies are pulled in via `bindings`; nothing in the @@ -460,6 +735,10 @@ export function createWorkflowSupervisor( bindings: WorkflowSupervisorBindings, ): WorkflowSupervisor { let state: SupervisorState = { phase: "idle" }; + let shutdownPromise: Promise | null = null; + // Replacement processes belong to shutdown before their ready handshake + // transfers ownership to the active state. + const uninstalledChildren = new Set(); // The live credential delivery to seed the child on every spawn and every // pre-trigger barrier. Initialized from the deploy-time delivery and MUTATED // by `deliverCredentials` on every runtime update, so a mid-life revocation or @@ -777,9 +1056,9 @@ export function createWorkflowSupervisor( * Cached per-spawn context the recycle path needs to respawn the * child against the same deploy tree. Populated on `spawn(opts)`; * cleared on `shutdown`. The recycle path never mutates the - * `stepOrder` or `definitionHash` -- the orthogonality with redeploy - * lives at this field: a deploy-tree change would land via a - * different code path that minted a new supervisor. + * `stepOrder` or `definitionHash`: a deployment keeps them for its + * life, and a different deploy tree means a new deployment with its + * own supervisor. */ let spawnContext: SpawnContext | null = null; let recyclePolicy: RecyclePolicy | null = null; @@ -1037,43 +1316,41 @@ export function createWorkflowSupervisor( // cannot saturate the host. const crashCount = crashTimestamps.length; logger.error`workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms; stopping the deployment (${reason})`; + // The RunFailed tombstone is the SOLE durable, externally-queryable + // signal of the crash-loop (the `crash-looping` phase is in-memory + // only). Shutdown commits it once teardown has quiesced the drain + // accumulators, so no escalation commit races this write, and before + // it resolves or reports the self-termination, so a host that then + // reads the deployment's history finds it. Best-effort: the deployment + // is already terminal, so the write not landing costs observability, + // not correctness. await shutdownInternal({ reason: `crash-loop: ${reason}`, terminalPhase: "crash-looping", selfTerminated: true, + terminalCommit: async () => { + // `anchorRunId` and the tombstone's `runId` are DISTINCT ids and + // must not be conflated. `bindings.anchorRunId` is the workflow-run + // repo slug (`deriveWorkflowRunRepoId`), which the supervisor + // principal's authz check keys on (`repoId.id === anchorRunId`). + // The RunFailed must land on the deployment's ONE top-level run, + // whose id is the local part of the deployment's mail address + // (`deriveWorkflowRunId`) -- the same id the dispatch loop writes + // every run event under. For a domain like `integration.interchange` + // the two ids differ (the repo slug carries a domain suffix), so + // writing the tombstone under the repo slug would strand it in a run + // subtree no reader consults. + await commitRunFailed({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + ref: bindings.workflowRunRef, + anchorRunId: bindings.anchorRunId, + runId: deriveWorkflowRunId(bindings.deploymentMailAddress), + at: new Date(nowMs).toISOString(), + message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`, + }); + }, }); - // Commit the RunFailed tombstone AFTER teardown: shutdownInternal has - // quiesced the drain accumulators (stop + await disposed), so the - // run-event tree is settled and no escalation commit races this write. - // This RunFailed is the SOLE durable, externally-queryable signal of - // the crash-loop (the `crash-looping` phase is in-memory only), so a - // failure to write it is logged loudly rather than swallowed. Best- - // effort: the deployment is already terminal, so the write not landing - // costs observability, not correctness. - try { - // `anchorRunId` and the tombstone's `runId` are DISTINCT ids and must - // not be conflated. `bindings.anchorRunId` is the workflow-run repo - // slug (`deriveWorkflowRunRepoId`), which the supervisor principal's - // authz check keys on (`repoId.id === anchorRunId`). The RunFailed must - // land on the deployment's ONE top-level run, whose id is the local - // part of the deployment's mail address (`deriveWorkflowRunId`) -- the - // same id the dispatch loop writes every run event under. For a domain - // like `integration.interchange` the two ids differ (the repo slug - // carries a domain suffix), so writing the tombstone under the repo - // slug would strand it in a run subtree no reader consults. - await commitRunFailed({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, - runId: deriveWorkflowRunId(bindings.deploymentMailAddress), - at: new Date(nowMs).toISOString(), - message: `workflow-process crash-looped: ${String(crashCount)} unexpected exits within ${String(crashLoopWindowMs)}ms`, - }); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.error`crash-loop RunFailed commit failed; deployment has no durable failure tombstone: ${message}`; - } return; } const thisBackoffMs = respawnBackoffMs; @@ -1163,6 +1440,22 @@ export function createWorkflowSupervisor( return mailboxStore; } + // The long-lived mirror drops a message's raw bytes on flush and cannot + // read them back: its committed-read snapshot was pinned when the mirror + // opened. A fetch, a search, or a reply's References lookup opens a fresh + // snapshot, which resolves the committed `.eml`. Pending mirror writes + // are flushed first so that snapshot includes them. + async function openCommittedMailbox(): Promise { + const writer = await getMailboxStore(); + if (writer.pendingWrites) await writer.flush(); + return createSubstrateMailboxStore({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + principal: mailboxWritePrincipal, + ref: bindings.workflowRunRef, + }); + } + function storedEnvelopeFromHeaders( headers: MessageHeaders, messageId: string, @@ -1405,6 +1698,12 @@ export function createWorkflowSupervisor( cohortBroadcaster: TerminalBroadcaster, ): Promise { for await (const payload of iter) { + if (payload.type === "cancel.prepared") { + const pending = pendingCancellations.get(payload.data.requestId); + if (pending?.broadcaster === cohortBroadcaster) + pending.resolve(payload.data.error); + continue; + } if (payload.type === "recycle.request") { logger.info`workflow-process self-initiated recycle.request: ${payload.data.reason}`; // Run the recycle off the iterator's loop so the iterator can @@ -1428,11 +1727,13 @@ export function createWorkflowSupervisor( // for this very write -- if the loop were blocked here, the // merge response could not be consumed and the write would // deadlock). - void handleSubstrateWriteRequest(payload.data).catch((cause) => { - const message = - cause instanceof Error ? cause.message : String(cause); - logger.error`substrate.write.request handler crashed: ${message}`; - }); + ownDetachedWrite( + handleSubstrateWriteRequest(payload.data).catch((cause) => { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.error`substrate.write.request handler crashed: ${message}`; + }), + ); continue; } if (payload.type === "substrate.merge.response") { @@ -1463,10 +1764,23 @@ export function createWorkflowSupervisor( // expunge. Run it off the iterator's loop so the iterator keeps // draining while the store flushes; the handler owns the // `mailbox.mutate.response` reply that resolves the child's awaiter. - void handleMailboxMutation(payload.data).catch((cause) => { + ownDetachedWrite( + handleMailboxMutation(payload.data).catch((cause) => { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.error`mailbox.mutate.request handler crashed: ${message}`; + }), + ); + continue; + } + if (payload.type === "mailbox.call.request") { + // A mailbox read, append, or refusal. Run it off the iterator so + // the pump keeps draining; the handler owns the + // `mailbox.call.response` that answers the request. + void handleMailboxCall(payload.data).catch((cause) => { const message = cause instanceof Error ? cause.message : String(cause); - logger.error`mailbox.mutate.request handler crashed: ${message}`; + logger.error`mailbox.call.request handler crashed: ${message}`; }); continue; } @@ -1593,6 +1907,37 @@ export function createWorkflowSupervisor( ) => void; }; const pendingMerges = new Map(); + const pendingCancellations = new Map< + string, + { + broadcaster: TerminalBroadcaster; + resolve: (error: string | undefined) => void; + } + >(); + let cancellationSeq = 0; + const cancellationCommits = new Set>(); + // Repository writes started off the control pump. Each promise settles + // without rejecting; its handler logs any failure. + const detachedWrites = new Set>(); + + function ownDetachedWrite(write: Promise): void { + detachedWrites.add(write); + void write.then(() => { + detachedWrites.delete(write); + }); + } + + // Every dispatch loop still running. A recycle moves `state` to the + // replacement cohort while the retired loop finishes its last message. + const dispatchLoops = new Set>(); + + function ownDispatchLoop(loop: Promise): void { + dispatchLoops.add(loop); + const release = () => { + dispatchLoops.delete(loop); + }; + void loop.then(release, release); + } /** * Reject every pending merge round-trip and every park-notify @@ -1604,6 +1949,9 @@ export function createWorkflowSupervisor( * channel will never invoke. */ function rejectCohortAwaiters(reason: string): void { + for (const pending of pendingCancellations.values()) + pending.resolve(`cohort aborted: ${reason}`); + pendingCancellations.clear(); for (const [requestId, entry] of pendingMerges) { pendingMerges.delete(requestId); entry.resolve({ ok: false, reason: `cohort aborted: ${reason}` }); @@ -1814,6 +2162,11 @@ export function createWorkflowSupervisor( * transport rejection) surfaces back to the child as a structured * `{ ok: false, reason }` so the agent's mail-tool call fails loudly * rather than silently dropping the send. + * + * The child sends only as its deployment's mail address. The host + * transport can hold signing keys for other deployments on the same + * host, so a child naming any other sender is refused here, before the + * transport would sign as that deployment. */ async function handleOutboundMessage( data: Extract["data"], @@ -1830,7 +2183,40 @@ export function createWorkflowSupervisor( return; } try { + if (data.senderAddress !== bindings.deploymentMailAddress) { + logger.warn`outbound.message requestId=${data.requestId} named sender ${data.senderAddress}; refused, the child sends only as ${bindings.deploymentMailAddress}`; + throw new Error( + `the child sends mail only as ${bindings.deploymentMailAddress}, not ${data.senderAddress}`, + ); + } const message = outboundMessageFromPayload(data.message); + // A connector reply arrives with `inReplyTo` and no References, and + // it is the only send that sets `completeReferences`. `mail_send` + // is that same shape without the flag; completing it would grow its + // one-parent chain into the full ancestry when the parent happens + // to be in this mailbox. An already-present `references`, including + // an empty array, is the caller's chain. A parent that is not in + // the mailbox leaves `references` unset, and the transport still + // derives `[inReplyTo]`. The lookup releases the mailbox lock + // before the send. + if ( + data.completeReferences === true && + message.inReplyTo !== undefined && + message.references === undefined + ) { + const inReplyTo = message.inReplyTo; + const references = await runMailboxExclusive(async () => { + const store = await openCommittedMailbox(); + const parent = store.messages.find( + (m) => m.envelope.messageId === inReplyTo, + ); + if (parent === undefined) return undefined; + return [...parent.envelope.references, parent.envelope.messageId]; + }); + if (references !== undefined && references.length > 0) { + message.references = references; + } + } const receipt = await bindings.mailBus.sendOutbound( data.senderAddress, message, @@ -1898,6 +2284,7 @@ export function createWorkflowSupervisor( result: { ok: false, reason: `unknown mailbox "${data.mailbox}"; only ${MAILBOX_INBOX_DIR} is writable`, + condition: "NONEXISTENT", }, }, }); @@ -1964,6 +2351,238 @@ export function createWorkflowSupervisor( } } + /** + * Answer one mailbox operation from the deployment the supervisor owns. + * Refusals are decided here and do not open the store. Reads, sync, status, + * and append run under `runMailboxExclusive` and reply after the lock, on + * the sender captured before it. `readMailPart` reads committed part blobs + * through the same principal and ref that wrote them, and does not take the + * mailbox lock. `fetchFull` verifies with no key, so the signature status + * stays `unknown`. `append` stores the shared text/plain encoding and does + * not notify. + */ + async function handleMailboxCall(data: MailboxCallRequest): Promise { + const controlSender = activeControlSender(); + if (controlSender === null) { + logger.warn`mailbox.call.request received outside running phase; requestId=${data.requestId} dropped (child awaiter will fail on pipe close)`; + return; + } + const refusal = refuseMailboxCall(data); + if (refusal !== undefined) { + await controlSender.send({ + type: "mailbox.call.response", + data: mailboxCallFailure(data, refusal.reason, refusal.condition), + }); + return; + } + try { + const answer = await answerMailboxCall(data); + await controlSender.send({ + type: "mailbox.call.response", + data: answer, + }); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + const condition = isMessageTransportError(cause) + ? cause.condition + : undefined; + await controlSender.send({ + type: "mailbox.call.response", + data: mailboxCallFailure(data, reason, condition), + }); + } + } + + async function answerMailboxCall( + data: MailboxCallRequest, + ): Promise { + switch (data.op) { + case "listMailboxes": + return { + requestId: data.requestId, + ok: true, + op: "listMailboxes", + value: [{ name: MAILBOX_INBOX_DIR }], + }; + case "watch": + return { requestId: data.requestId, ok: true, op: "watch" }; + case "readMailPart": { + // The blobs were committed under this principal and ref. A reader + // opened against any other identity looks at a different tree. + const reader = createMailPartReader({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + principal: mailboxWritePrincipal, + ref: bindings.workflowRunRef, + }); + const bytes = await reader.read(data.partRef); + return { + requestId: data.requestId, + ok: true, + op: "readMailPart", + value: { contentBase64: base64Encode(bytes) }, + }; + } + case "search": { + const query = reviveSearchDates(data.query); + return runMailboxExclusive(async () => { + const store = await openCommittedMailbox(); + const value = await executeSearch(data.mailbox, store, query); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "search", + value, + }; + return success; + }); + } + case "thread": { + const query = + data.query === undefined ? undefined : reviveSearchDates(data.query); + return runMailboxExclusive(async () => { + const store = await openCommittedMailbox(); + const value = + query === undefined + ? await executeThread(data.mailbox, store, data.algorithm) + : await executeThread(data.mailbox, store, data.algorithm, query); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "thread", + value, + }; + return success; + }); + } + case "fetchHeaders": + case "fetchStructure": + case "fetchFull": + case "fetchPart": { + const ref = data.ref; + return runMailboxExclusive(async () => { + const store = await openCommittedMailbox(); + if (data.op === "fetchHeaders") { + const value = await fetchHeaders(ref, store); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "fetchHeaders", + value, + }; + return success; + } + if (data.op === "fetchStructure") { + const value = await fetchStructure(ref, store); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "fetchStructure", + value, + }; + return success; + } + if (data.op === "fetchPart") { + const value = projectPart( + await fetchPart(ref, data.partPath, store), + ); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "fetchPart", + value, + }; + return success; + } + const value = projectFetchedMessage( + await fetchFull(ref, store, () => undefined), + ); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "fetchFull", + value, + }; + return success; + }); + } + case "sync": + return runMailboxExclusive(async () => { + const store = await getMailboxStore(); + const result = store.sync({ + uidValidity: data.uidValidity, + highestModSeq: data.highestModSeq, + }); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "sync", + value: syncResultForCaller(data.mailbox, data.uidNext, result), + }; + return success; + }); + case "getMailboxStatus": + return runMailboxExclusive(async () => { + const store = await getMailboxStore(); + const unseen = store.messages.filter( + (message) => !message.flags.has(MAILBOX_FLAG_SEEN), + ).length; + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "getMailboxStatus", + value: { + total: store.messages.length, + unseen, + recent: 0, + uidNext: store.uidNext, + uidValidity: store.uidValidity, + highestModSeq: store.highestModSeq, + }, + }; + return success; + }); + case "append": { + const appended: { + headers: MessageHeaders; + content?: string; + payload?: InboundMessage["payload"]; + } = { headers: data.headers }; + if (data.content !== undefined) appended.content = data.content; + if (data.payload !== undefined) appended.payload = data.payload; + const flags = data.flags === undefined ? [] : data.flags; + // Encode before the lock. A missing Message-ID, or a missing or + // unparseable Date, throws here, so the message never enters the + // mirror. An Invalid Date that reached append would survive a failed + // flush and wedge every later write. + const stored = inboundMessageToRaw(appended); + return runMailboxExclusive(async () => { + const store = await getMailboxStore(); + const uid = store.append(stored.raw, stored.envelope, flags); + await store.flush(); + const success: MailboxCallSuccess = { + requestId: data.requestId, + ok: true, + op: "append", + value: { uid, mailbox: data.mailbox }, + }; + return success; + }); + } + case "createMailbox": + case "deleteMailbox": + case "move": + case "copy": + case "createList": + case "listMembers": + case "subscribe": + case "unsubscribe": + // Refusals are sent before this switch. Reaching one means the + // refusal table and this switch disagree. + throw new Error(`mailbox call ${data.op} has no answer`); + } + } + async function handleSubstrateWriteRequest( data: Extract["data"], ): Promise { @@ -2133,15 +2752,17 @@ export function createWorkflowSupervisor( // per-event. The fold commit carries no newly-added terminal event, // so it does not re-fire this terminal-write coupling. for (const { runId } of newlyTerminalRuns) { - void compactRunEvents({ - substrate: bindings.repoStore, - repoId: validatedRepoId, - ref: data.ref, - anchorRunId: bindings.anchorRunId, - runId, - }).catch((cause) => { - logger.warn`compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`; - }); + ownDetachedWrite( + compactRunEvents({ + substrate: bindings.repoStore, + repoId: validatedRepoId, + ref: data.ref, + anchorRunId: bindings.anchorRunId, + runId, + }).catch((cause) => { + logger.warn`compaction of run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }), + ); } } catch (cause) { // Clean up any merge awaiter that the substrate may not have @@ -2540,6 +3161,7 @@ export function createWorkflowSupervisor( startingPhaseBroadcaster, replayDone, ); + ownDispatchLoop(dispatchLoop); // Surface dispatch-loop failures via the logger; the loop's own // catch already swallows per-iteration faults, but a structural // failure (e.g. the cohort abort handler itself throws) lands @@ -2576,8 +3198,8 @@ export function createWorkflowSupervisor( // Cache the spawn context for the recycle path. The recycle path // reuses the same stepOrder/definitionHash/onInferenceEvent on - // every respawn -- those are the strict-orthogonality anchors - // with redeploy, and the supervisor never mutates them. + // every respawn -- a deployment keeps them for its life, and the + // supervisor never mutates them. const now = bindings.recyclePolicyNow ?? defaultNow; spawnContext = { stepOrder: opts.stepOrder, @@ -3449,25 +4071,86 @@ export function createWorkflowSupervisor( async function requestCancel( opts: CancelRequestOpts, ): Promise { - const result = await commitCancelRequested({ - substrate: bindings.repoStore, - repoId: bindings.workflowRunRepoId, - ref: bindings.workflowRunRef, - anchorRunId: bindings.anchorRunId, - runId: opts.runId, - origin: opts.origin, - reason: opts.reason, - at: opts.at, - signAsPrincipal: bindings.signAsPrincipal, + if ( + state.phase === "stopping" || + state.phase === "stopped" || + state.phase === "crash-looping" + ) { + throw new Error("Cannot cancel a stopped workflow supervisor"); + } + const commitCancellation = async () => { + const committing = commitCancelRequested({ + substrate: bindings.repoStore, + repoId: bindings.workflowRunRepoId, + ref: bindings.workflowRunRef, + anchorRunId: bindings.anchorRunId, + runId: opts.runId, + origin: opts.origin, + reason: opts.reason, + at: opts.at, + signAsPrincipal: bindings.signAsPrincipal, + }); + cancellationCommits.add(committing); + try { + const result = await committing; + return { commitSha: result.commitSha, seq: result.seq }; + } finally { + cancellationCommits.delete(committing); + } + }; + const cohort = state; + if ( + cohort.phase !== "starting" && + cohort.phase !== "running" && + cohort.phase !== "recycling" + ) + return commitCancellation(); + const sender = cohort.controlSender; + + const requestId = `cancel-${String((cancellationSeq += 1))}`; + const prepared = Promise.withResolvers(); + pendingCancellations.set(requestId, { + broadcaster: cohort.terminalBroadcaster, + resolve: prepared.resolve, }); - return { commitSha: result.commitSha, seq: result.seq }; + try { + await sender.send({ + type: "cancel.prepare", + data: { requestId, runId: opts.runId, reason: opts.reason }, + }); + const error = await prepared.promise; + if (error !== undefined) throw new Error(error); + cohort.terminalCohortAbort.signal.throwIfAborted(); + if (activeControlSender() !== sender) + throw new Error("Cancellation's workflow child was replaced"); + const result = await commitCancellation(); + await sender + .send({ type: "cancel.committed", data: { requestId } }) + .catch((cause: unknown) => { + logger.warn`CancelRequested committed for ${opts.runId}, but the child wakeup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + }); + return result; + } catch (cause) { + await sender + .send({ + type: "cancel.committed", + data: { + requestId, + error: cause instanceof Error ? cause.message : String(cause), + }, + }) + .catch(() => undefined); + throw cause; + } finally { + pendingCancellations.delete(requestId); + } } async function shutdown(): Promise { await shutdownInternal({ reason: "shutdown requested" }); } - async function shutdownInternal(opts: { + type ShutdownOptions = { reason: string; // Terminal phase the teardown lands in. Defaults to `stopped` (a clean // shutdown); the crash-loop latch passes `crash-looping` so the terminal @@ -3479,23 +4162,42 @@ export function createWorkflowSupervisor( // fire below. The terminal phase alone cannot carry this: a self-terminated // and a host-requested teardown both land in `stopped`. selfTerminated?: boolean; - }): Promise { + // A durable record of the teardown, written once the child has exited and + // the writers shutdown owns have settled, and before shutdown resolves or + // reports a self-termination. A host that finds the supervisor gone reads + // history that already holds it. + terminalCommit?: () => Promise; + }; + + function shutdownInternal(opts: ShutdownOptions): Promise { + if (shutdownPromise !== null) return shutdownPromise; if ( state.phase === "idle" || state.phase === "stopped" || state.phase === "crash-looping" ) - return; + return Promise.resolve(); + // Publish the completion before teardown can call back into the supervisor. + const completion = Promise.withResolvers(); + shutdownPromise = completion.promise; + void performShutdown(opts).then( + () => completion.resolve(undefined), + completion.reject, + ); + return shutdownPromise; + } + + async function performShutdown(opts: ShutdownOptions): Promise { const prior = state; state = { phase: "stopping" }; // shutdownInternal is designed to be TOTAL: when a child is up it must // always kill it and always reach `stopped`, no matter which teardown // step throws. Rather than depend on every step being individually // non-throwing (an approach that has already leaked an escape hatch), - // the whole teardown body runs inside one `try`, and the two - // load-bearing actions -- the child kill and the `phase = "stopped"` - // transition -- live in the `finally`, so a throw anywhere above them - // still runs both. This is the documented shutdown carve-out to the + // the whole teardown body runs inside one `try`. It requests the child + // kill before waiting for teardown; `finally` requests it if that point + // was never reached and always completes the terminal transition. + // This is the documented shutdown carve-out to the // fail-loud rule: leaking the child or wedging the supervisor in // `stopping` is strictly worse than logging and continuing, so the // steps that can throw surface at `logger.warn` and execution proceeds. @@ -3504,6 +4206,34 @@ export function createWorkflowSupervisor( // by construction; they sit inside the `try` regardless so the // invariant survives if that ever changes.) const accumulatorsToDispose = [...drainAccumulators.values()]; + const childrenToStop = new Set(uninstalledChildren); + if ( + prior.phase === "starting" || + prior.phase === "running" || + prior.phase === "recycling" + ) + childrenToStop.add(prior.handle); + const childTerminations: Promise[] = []; + let killRequested = false; + function killChildren(): void { + if (killRequested) return; + killRequested = true; + // Escalate to SIGKILL: the child runs workflow code, which can trap + // SIGTERM and would otherwise hold the forced stop open indefinitely. + for (const handle of childrenToStop) { + childTerminations.push( + killChildHandle(handle, DEFAULT_KILL_TIMEOUT_MS, { + setTimer: readySetTimer, + clearTimer: readyClearTimer, + logger, + }).catch((cause: unknown) => { + const message = + cause instanceof Error ? cause.message : String(cause); + logger.warn`child kill threw during shutdown: ${message}`; + }), + ); + } + } try { // Stop every armed drainTimeout accumulator before tearing the child // down. An accumulator left running would otherwise fire its @@ -3548,6 +4278,17 @@ export function createWorkflowSupervisor( // some other actor woke it. wakeDispatch(); } + // A dispatch may be blocked writing to an unresponsive child. Killing + // before awaiting its loop releases that pipe and the cancellation wait. + killChildren(); + // An already-started signed append may finish after the child exits. + // Own that write through teardown before the host releases its bindings. + await Promise.allSettled([...cancellationCommits]); + // Own the pump's detached writes the same way, so every write the child + // requested lands before shutdown resolves. New merges fail once + // stopping, so what remains is local I/O. A finishing write can start a + // fold. + while (detachedWrites.size > 0) await Promise.all([...detachedWrites]); // Await every accumulator's `disposed()` so a pending escalation // commit or terminal-event watcher coroutine cannot outlive the // supervisor and fire against torn-down bindings. @@ -3558,16 +4299,10 @@ export function createWorkflowSupervisor( }), ), ); - if ( - (prior.phase === "running" || prior.phase === "recycling") && - prior.dispatchLoop !== null - ) { - await prior.dispatchLoop.catch(() => { - /* swallowed: dispatch-loop failures are surfaced by the - loop's own logger; the shutdown path only waits for the - loop's last iteration to settle. */ - }); - } + // Includes a loop a recycle has retired from `state`, so its last + // consumption write lands before shutdown resolves. Each loop's own + // logger surfaces its failure. + await Promise.allSettled([...dispatchLoops]); if ( (prior.phase === "starting" || prior.phase === "running" || @@ -3659,38 +4394,37 @@ export function createWorkflowSupervisor( // whatever happened above, so a throwing teardown step can neither // leak the child nor wedge the supervisor in `stopping`. The kill is // itself guarded so a throw here cannot re-escape the `finally`. + killChildren(); + await Promise.all(childTerminations); + await Promise.all( + [...childrenToStop].map((handle) => + handle.exited.catch(() => { + /* A non-zero child exit is expected during shutdown. */ + }), + ), + ); if ( prior.phase === "starting" || prior.phase === "running" || prior.phase === "recycling" ) { - try { - prior.handle.kill(); - } catch (cause) { - const message = - cause instanceof Error ? cause.message : String(cause); - logger.warn`child kill threw during shutdown: ${message}`; - } - await prior.handle.exited.catch(() => { - /* swallowed: the host has already been told the deployment is - coming down; an error surfaced from the spawner is the - process exiting with a non-zero code, which is what the - shutdown path expects. */ - }); await prior.eventPump.catch(() => { /* swallowed for the same reason as above. */ }); } state = { phase: opts.terminalPhase ?? "stopped" }; } + if (opts.terminalCommit !== undefined) { + try { + await opts.terminalCommit(); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`terminal commit failed; the deployment has no durable record of why it stopped (${opts.reason}): ${message}`; + } + } // Surface a self-termination to the host after the terminal transition is - // committed. The already-terminal early-return at the top dedups the common - // case, but it does NOT cover the `stopping` window, so two self-terminating - // callers interleaving through teardown can each fire (e.g. an onChildCrash - // during `recycling` plus the recycle-failure catch). The sink is therefore - // idempotent-required, not exactly-once; the reclaim it drives absorbs a - // repeat by design. Wrapped so a throwing sink cannot re-escape here and - // break the documented shutdown totality. + // committed. Concurrent shutdown callers share this teardown. Catch sink + // failures so they cannot escape a completed shutdown. if (opts.selfTerminated === true) { try { bindings.onSelfTerminate?.({ @@ -3884,7 +4618,22 @@ export function createWorkflowSupervisor( try { attempt = await triggerRecycle( { - bindings, + bindings: { + ...bindings, + subprocessSpawner: (spawnArgs) => { + if (state.phase !== "recycling") { + throw new Error( + `Cannot spawn a replacement in supervisor phase ${state.phase}`, + ); + } + const handle = bindings.subprocessSpawner(spawnArgs); + uninstalledChildren.add(handle); + const forgetExitedChild = () => + uninstalledChildren.delete(handle); + void handle.exited.then(forgetExitedChild, forgetExitedChild); + return handle; + }, + }, stepOrder: priorContext.stepOrder, definitionHash: priorContext.definitionHash, warmKeep: priorContext.warmKeep, @@ -3923,26 +4672,10 @@ export function createWorkflowSupervisor( credentialsSnapshot, controlIncoming, }) => { - // Phase guard: a `shutdown()` that landed during the - // kill/respawn gap (between `subprocessSpawner` and this - // callback) has flipped `state.phase` to `stopping` or - // `stopped`. The new child is now an orphan -- the - // supervisor was supposed to be tearing down, not - // installing a fresh cohort. Kill the new wiring's - // handle and bail out without registering it on - // `state`. `shutdownInternal`'s own teardown path has - // already disposed the prior cohort; there is nothing - // for this callback to do. + // Shutdown already owns the uninstalled child's kill and exit. + // A late ready frame must only drain its IPC resources, never + // install it as a new running cohort. if (state.phase !== "recycling") { - // Kill the orphan child and release its event-channel / - // upstream-control resources so they cannot survive as - // unowned promises. Without this, the eventPump and - // controlIncoming iterator would have no `state` - // bookkeeping to drive their cleanup -- a rejection - // inside `pumpEvents` would surface as an unhandled - // rejection, and the upstream control iterator's - // exit would never be observed. - wiring.handle.kill("SIGTERM"); void wiring.eventPump.catch((cause: unknown) => { const message = cause instanceof Error ? cause.message : String(cause); @@ -3992,6 +4725,7 @@ export function createWorkflowSupervisor( newBroadcaster, null, ); + ownDispatchLoop(newDispatchLoop); void newDispatchLoop.catch((cause) => { const message = cause instanceof Error ? cause.message : String(cause); @@ -4014,6 +4748,7 @@ export function createWorkflowSupervisor( replayDone: null, sweepDone: prior.sweepDone, }; + uninstalledChildren.delete(wiring.handle); // Bump the generation and arm the exit-watcher for the // respawned child atomically with this running transition, so // the predecessor's watcher (already stale by generation) never @@ -4177,6 +4912,9 @@ export function createWorkflowSupervisor( signalName: opts.signalName, signalId: opts.signalId, payload: opts.payload, + ...(opts.correlationId !== undefined + ? { correlationId: opts.correlationId } + : {}), }, }); } @@ -4357,9 +5095,11 @@ type ActiveState = { * `starting`-phase ActiveState carries `null` because the loop is * not started until the child emits `ready`; once `spawn()` * transitions to `running` the field carries the live loop - * promise. `shutdownInternal` awaits this promise after aborting - * the cohort so a dispatch-loop iteration that is mid-await - * settles before the supervisor tears the bindings down. + * promise. The recycle that replaces this cohort awaits it. + * `shutdownInternal` waits on every running loop through the + * supervisor's `dispatchLoops`, including one a recycle has already + * cleared from this field, so an iteration that is mid-await settles + * before the supervisor tears the bindings down. */ dispatchLoop: Promise | null; /** diff --git a/vendor/intx/workflow-host/src/supervisor/types.ts b/vendor/intx/workflow-host/src/supervisor/types.ts index 928ac57f26..53c565aa99 100644 --- a/vendor/intx/workflow-host/src/supervisor/types.ts +++ b/vendor/intx/workflow-host/src/supervisor/types.ts @@ -321,25 +321,25 @@ export interface WorkflowSupervisorBindings { * while recycling, or a recycle failure (both `stopped`) -- but NOT when the * host drives it down through the public `shutdown()`, and NOT for a failure * of the initial spawn handshake (that is the deploy's to unwind). Production - * wires this to the sidecar so it reclaims the deployment address (drops the - * supervisor from its active map and releases the address's routing state) - * and the address becomes redeployable without a manual undeploy. + * wires this to the sidecar so it drops the supervisor, releases the + * address's routing state, and holds the deployment as stopped, reporting why + * it ended, until the Hub undeploys it. * * The handler MUST be idempotent. Firing is not exactly-once: two * self-terminating callers interleaving through teardown (e.g. a channel * crash while recycling plus the recycle-failure catch) can each fire. The - * sidecar's reclaim absorbs a repeat because its `activeSupervisors.has` - * guard makes the second run a no-op. + * sidecar absorbs a repeat because it acts only while the incarnation still + * has its supervisor, which makes the second run a no-op. * * Unlike `onSuspensionRegister`, this sink does NOT share the same * log-and-continue contract on the host side. A missed suspension has an - * independent recovery path (`reEmitParkedCorrelations`); a missed reclaim - * does not -- the address stays stranded until an operator undeploys. So - * the host's handler is engineered to be total, and a failure there is - * logged loudly rather than swallowed. The supervisor still invokes this - * best-effort (a throwing sink cannot break the terminal transition), but a - * host that copies `onSuspensionRegister`'s quiet-swallow semantics onto its - * reclaim handler reintroduces the stranding bug. + * independent recovery path (`reEmitParkedCorrelations`); a missed stop + * does not -- the deployment stays reported live with nothing running it + * until an operator undeploys. So the host's handler is engineered to be + * total, and a failure there is logged loudly rather than swallowed. The + * supervisor still invokes this best-effort (a throwing sink cannot break + * the terminal transition), but a host that copies `onSuspensionRegister`'s + * quiet-swallow semantics onto this handler reintroduces the stranding bug. */ onSelfTerminate?: (info: { phase: "stopped" | "crash-looping"; @@ -368,12 +368,12 @@ export interface WorkflowSupervisorBindings { anchorRunId: string; }) => Promise; /** - * Decrypted credential material for the deployment's tools, delivered to the - * child on the pre-trigger barrier alongside the grants. Absent when the - * deployment binds no credentials. A rotation flows through - * `deliverCredentials`, not this static binding. + * Decrypted credential material for the deployment's binding consumers and + * inference sources, delivered to the child on the pre-trigger barrier + * alongside the grants. Absent when the deployment binds no credentials. A + * rotation flows through `deliverCredentials`, not this static binding. */ - credentialDelivery?: import("@intx/types/sidecar").CredentialDelivery; + credentialDelivery?: import("@intx/types/credential-delivery").CredentialDelivery; /** Subprocess spawner the supervisor invokes per spawn. */ subprocessSpawner: SubprocessSpawner; /** diff --git a/vendor/intx/workflow-host/src/testing/change-notifier.ts b/vendor/intx/workflow-host/src/testing/change-notifier.ts new file mode 100644 index 0000000000..e0a7d6efc6 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/change-notifier.ts @@ -0,0 +1,74 @@ +// Level-triggered wait over a mutable test double. +// +// A test that needs a double to have reached some state -- two entries +// consumed, an entry moved to processing -- has no event to await, because +// the state lives in a map the double mutates in place. The usual stand-in is +// a predicate re-evaluated on a timer, which makes the tick interval and the +// deadline part of whether the test passes. +// +// This inverts it: the double reports that it changed, and the waiter +// re-evaluates the predicate only then. The predicate is still a predicate -- +// there is no event to name when the subject is "the map now looks like this" +// -- but nothing is re-read on a schedule and no duration decides the +// outcome. +// +// `until` evaluates the predicate on entry to every pass of its loop, so a +// state the double already reached resolves it and a caller never has to know +// whether it armed the wait in time. Only `notify` is an edge: one with no +// waiter registered is dropped, which is why the double must call it after +// every mutation rather than only when it thinks someone is watching. +// +// It is deliberately NOT a consolidation of the doubles themselves. The five +// inbox doubles in this package are genuinely different implementations +// rather than cosmetic variants, and a merged superset could not be shown to +// preserve what each test relies on. Sharing the waiting is the part that was +// worth sharing. + +export type ChangeNotifier = { + /** Call from the double after any mutation a waiter might care about. */ + notify(): void; + /** + * Resolve once `predicate` holds. Evaluated immediately, then again after + * each `notify`. Carries no deadline: a predicate that never holds is + * caught by the lane timeout, per "Synchronizing on State, Not Time" in + * CONVENTIONS.md. + */ + until(predicate: () => boolean): Promise; +}; + +export function createChangeNotifier(): ChangeNotifier { + let waiters: (() => void)[] = []; + return { + notify() { + // Clearing the list is what bounds it. Every pass of the wait loop + // below registers a resolver, and the pass that returns leaves its own + // resolver behind unsettled; without the clear those accumulate for the + // life of the notifier. Measured over 50 notifies with one waiter: 1 + // entry with the clear, 51 without. + // + // Clearing BEFORE the wake rather than after is not what makes a + // re-arming waiter wait for the next change instead of this one. Waking + // a settled resolver is a no-op, and the loop re-registers only after + // `await changed` resumes, which is a microtask boundary -- by then this + // function has finished iterating. A mutant that wakes the live list and + // clears afterwards passes every test that uses this helper. + const waking = waiters; + waiters = []; + for (const waiter of waking) waiter(); + }, + async until(predicate) { + for (;;) { + // Re-checked on every pass, which is what makes this usable when the + // change already happened before the wait was created. The registering + // executor runs synchronously, so no notify can land between the check + // and the registration in either order -- the loop is the invariant, + // not the order of those two lines. + const changed = new Promise((resolve) => { + waiters.push(resolve); + }); + if (predicate()) return; + await changed; + } + }, + }; +} diff --git a/vendor/intx/workflow-host/src/testing/index.ts b/vendor/intx/workflow-host/src/testing/index.ts new file mode 100644 index 0000000000..6852e37366 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/index.ts @@ -0,0 +1,36 @@ +// @intx/workflow-host/testing -- in-memory doubles for the IPC transports. +// +// These exist so a test can drive a supervisor or a workflow-process child +// over the real channel code without spawning a process. They hold their +// frames in an array and have no durability, no backpressure, and no +// framing beyond the newline terminator the event channel expects, so a +// production caller reaching for this subpath is almost certainly looking +// for the real transports in the package root instead. +export { + createMemoryFrameStream, + createMemoryNdjsonStream, + type MemoryFrameStream, + type MemoryNdjsonStream, +} from "./memory-streams"; +export { + createSupervisorReaper, + type ReapableSupervisor, + type SupervisorReaper, +} from "./supervisor-reaper"; +export { createMockMailBus, type MockMailBus } from "./mail-bus"; +export { createSpawnObserver, type SpawnObserver } from "./spawn-observer"; +export { + parseTriggerFireRunIds, + readPayloadsOfType, + waitForTriggerFireRunIds, + waitForUpstreamPayload, + waitForUpstreamPayloads, + type UpstreamFrameSource, +} from "./upstream-frames"; +export { createChangeNotifier, type ChangeNotifier } from "./change-notifier"; +export { + createLogCapture, + type CapturedLogRecord, + type LogCapture, +} from "./log-capture"; +export { createStubRepoStore, type StubRepoStoreOpts } from "./stub-repo-store"; diff --git a/vendor/intx/workflow-host/src/testing/log-capture.ts b/vendor/intx/workflow-host/src/testing/log-capture.ts new file mode 100644 index 0000000000..b91d26db29 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/log-capture.ts @@ -0,0 +1,208 @@ +// Capture LogTape records for one test file, so a test can await a log line +// instead of polling for it. +// +// Some decisions announce themselves only in the log. A guard that returns +// without touching any observable state leaves the record as the sole +// evidence it ran, and a test that wants to assert on such a decision has no +// other signal to await. Waiting on the record is what "Synchronizing on +// State, Not Time" in CONVENTIONS.md asks for, and it replaces a poll whose +// deadline had to be guessed against whatever the code under test was +// waiting on. +// +// The capture is per-call, not module-level. The unit pass gives a worker one +// module registry for every file it runs, so a module-level record array +// would accumulate another file's records and a module-level waiter list +// would let another file's log line resolve this file's wait. +// `createChangeNotifier` and `createSupervisorReaper` are factories for the +// same reason. +// +// `configureSync` is process-global, so `install` saves the configuration it +// replaces and `restore` puts it back. Wire the three lifecycle methods to +// `beforeAll`, `afterAll`, and `beforeEach`. + +import { configureSync, getConfig } from "@intx/log"; + +/** One captured record, with its message template already interpolated. */ +export type CapturedLogRecord = { + readonly category: readonly string[]; + readonly level: string; + readonly message: string; +}; + +type RecordWaiter = { + needle: string; + /** The level the record must carry, or null to accept any level. */ + level: string | null; + resolve: (record: CapturedLogRecord) => void; + abandon: () => void; +}; + +export type LogCapture = { + /** + * Replace the process-wide logging configuration with the capturing one. + * Pass straight to `beforeAll`. Throws if a capture is already installed + * through this object. + */ + install(): void; + /** + * Put back the configuration `install` replaced. Pass straight to + * `afterAll`. Throws if nothing was installed. + */ + restore(): void; + /** + * Drop the captured records and abandon every armed waiter. Pass straight + * to `beforeEach`. + */ + reset(): void; + /** Every record captured since the last `reset`, in log order. */ + records(): readonly CapturedLogRecord[]; + /** The messages of the captured `error` records, in log order. */ + errors(): string[]; + /** + * Resolve with the first record at any level whose message contains + * `needle`, whether it was logged before this call or arrives after it. + * Carries no deadline: a record that never arrives is caught by the lane + * timeout, per "Synchronizing on State, Not Time" in CONVENTIONS.md. + * + * `needle` must identify the record uniquely within its own test. Matching + * on a message some other test also emits is what makes this a barrier in + * name only: `reset` clears the records a previous test logged, but it + * cannot exclude the ones still arriving from work that test left running, + * and nothing in a record says which test caused it. A wait satisfied by + * such a straggler returns before the awaited work has happened, and the + * assertion behind it reads pre-barrier state and passes. Where two tests + * exercise the same path, vary an input the record carries -- see the + * crash-reason token in `substrate-write.test.ts`. + * + * A test that triggers fire-and-forget work owes the next test the same + * courtesy: await that work's own completion record before returning, so + * it cannot straggle across the boundary in the first place. + */ + waitForRecord(needle: string): Promise; + /** As `waitForRecord`, restricted to records at the `error` level. */ + waitForError(needle: string): Promise; +}; + +export function createLogCapture(): LogCapture { + const captured: CapturedLogRecord[] = []; + // Tests awaiting a record that has not been logged yet. The sink resolves + // these as each record lands. + const waiters: RecordWaiter[] = []; + // Non-null exactly while this capture is installed, so it doubles as the + // installed flag. + let savedConfig: ReturnType = null; + + function matches( + record: CapturedLogRecord, + needle: string, + level: string | null, + ): boolean { + if (level !== null && level !== record.level) return false; + return record.message.includes(needle); + } + + function waitFor( + needle: string, + level: string | null, + ): Promise { + const already = captured.find((record) => matches(record, needle, level)); + if (already !== undefined) return Promise.resolve(already); + return new Promise((resolve, reject) => { + waiters.push({ + needle, + level, + resolve, + abandon: () => { + const what = level === null ? "record" : `${level} record`; + reject( + new Error(`no ${what} matching ${needle} arrived before teardown`), + ); + }, + }); + }); + } + + return { + install() { + if (savedConfig !== null) { + throw new Error("this log capture is already installed"); + } + const prior = getConfig(); + // A null configuration means this file loaded without `@intx/log` + // having installed its default sink, which cannot happen -- importing + // the package runs the install. Failing here rather than at `restore` + // keeps the worker from running a whole suite it cannot unwind: there + // would be nothing to put back, and the install cannot re-fire to + // repair it. + if (prior === null) { + throw new Error( + "no logging configuration was present for the capture to replace", + ); + } + savedConfig = prior; + configureSync({ + reset: true, + sinks: { + capture: (record) => { + const message = record.message + .map((part) => + typeof part === "string" ? part : JSON.stringify(part), + ) + .join(""); + const entry = { + category: record.category, + level: record.level, + message, + }; + captured.push(entry); + // Hand the record to anyone waiting for it. Take only the + // matching waiters, leaving the rest armed for their own needles. + for (const waiter of waiters.splice(0)) { + if (matches(entry, waiter.needle, waiter.level)) { + waiter.resolve(entry); + } else { + waiters.push(waiter); + } + } + }, + }, + loggers: [ + { category: [], lowestLevel: "debug", sinks: ["capture"] }, + { + category: ["logtape", "meta"], + lowestLevel: "warning", + sinks: ["capture"], + }, + ], + }); + }, + + restore() { + const prior = savedConfig; + if (prior === null) { + throw new Error( + "this log capture was restored without being installed", + ); + } + savedConfig = null; + configureSync({ reset: true, ...prior }); + }, + + reset() { + captured.length = 0; + // Reject rather than drop: a waiter outstanding from a previous test + // (its test was killed mid-wait) would otherwise be discarded still + // armed, and its promise could never settle. + for (const waiter of waiters.splice(0)) waiter.abandon(); + }, + + records: () => captured.slice(), + + errors: () => + captured.filter((r) => r.level === "error").map((r) => r.message), + + waitForRecord: (needle) => waitFor(needle, null), + + waitForError: (needle) => waitFor(needle, "error"), + }; +} diff --git a/vendor/intx/workflow-host/src/testing/mail-bus.ts b/vendor/intx/workflow-host/src/testing/mail-bus.ts new file mode 100644 index 0000000000..0323fb8b0e --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/mail-bus.ts @@ -0,0 +1,105 @@ +// In-memory MailBusBindings, for supervisor tests that need mail delivered to +// a deployment address without a real bus. +// +// Seven copies of this had accumulated in five variants. Beyond cosmetics +// they differed in what they let a test observe: one kept a register/ +// unregister history, one kept only the current set. Both observables are +// here, because a test asking "is it registered now" and one asking "was it +// ever unregistered" are asking different questions and the second cannot be +// answered from the first. + +import type { MailBusBindings } from "../supervisor/types"; + +export type MockMailBus = MailBusBindings & { + /** Addresses currently registered, in registration order. */ + registered(): readonly string[]; + /** + * Every registration and unregistration in order, as `register:
` + * and `unregister:
`. Distinguishes "never registered" from + * "registered and then released", which `registered()` cannot. + */ + registrationHistory(): readonly string[]; + /** Deliver a raw message to every handler subscribed for `address`. */ + deliver(address: string, message: Uint8Array): void; + /** + * Resolve once `address` is registered, whether it already is or is + * registered later. + * + * The registration is the event a test waiting for a deployment to come up + * actually wants; polling `registered()` on a timer was standing in for it. + */ + awaitRegistered(address: string): Promise; +}; + +export function createMockMailBus(): MockMailBus { + const registered: string[] = []; + const history: string[] = []; + const subscribers = new Map< + string, + Set<(rawMessage: Uint8Array) => Promise> + >(); + let waiters: (() => void)[] = []; + + function announce(): void { + const waiting = waiters; + waiters = []; + for (const waiter of waiting) waiter(); + } + + return { + registerAddress(address: string) { + registered.push(address); + history.push(`register:${address}`); + announce(); + }, + unregisterAddress(address: string) { + const idx = registered.lastIndexOf(address); + if (idx >= 0) registered.splice(idx, 1); + subscribers.delete(address); + history.push(`unregister:${address}`); + // Every mutation of the registration set is announced, which is what + // makes an arbitrary predicate over that set awaitable: a waiter re-reads + // on any change rather than on the subset of changes someone remembered + // to report. `awaitRegistered` is the only waiter today, and an + // unregister can only falsify its predicate, so this wake settles nothing + // for it -- a fact about that one predicate, not a gap in the reporting. + announce(); + }, + subscribeMailForAddress( + address: string, + handler: (rawMessage: Uint8Array) => Promise, + ) { + let set = subscribers.get(address); + if (set === undefined) { + set = new Set(); + subscribers.set(address, set); + } + set.add(handler); + return () => { + const current = subscribers.get(address); + current?.delete(handler); + }; + }, + sendOutbound() { + throw new Error("sendOutbound not exercised in this test"); + }, + registered: () => registered.slice(), + registrationHistory: () => history.slice(), + deliver(address: string, message: Uint8Array) { + const set = subscribers.get(address); + if (set === undefined) return; + for (const handler of set) void handler(message).catch(() => undefined); + }, + async awaitRegistered(address: string) { + for (;;) { + // Re-read on every pass, so an address registered before this call + // resolves it rather than leaving it waiting for a re-registration. + const changed = new Promise((resolve) => { + waiters.push(resolve); + }); + if (registered.includes(address)) return; + await changed; + } + }, + }; +} diff --git a/vendor/intx/workflow-host/src/testing/memory-streams.ts b/vendor/intx/workflow-host/src/testing/memory-streams.ts new file mode 100644 index 0000000000..e07a1a7416 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/memory-streams.ts @@ -0,0 +1,273 @@ +// In-memory stand-ins for the two IPC transports, for tests that drive a +// supervisor or a workflow-process child without spawning one. +// +// Seventeen copies of these two factories accumulated across this package, +// `apps/sidecar`, and the deploy tests, in nine variants apiece. The +// divergences were almost entirely cosmetic, but the copies disagreed on one +// thing that matters: whether `inject` appends the newline terminator that +// delimits a wire frame. +// +// The two doubles answer that differently, because the two channels carry the +// terminator differently. The event channel is byte-oriented and its +// terminator is part of the payload, so `createMemoryFrameStream` separates +// three roles and a caller says which one it means: +// +// `writer` what production writes through; bytes go in verbatim +// `inject` one complete frame arriving on the wire, terminator supplied +// `injectRaw` bytes verbatim, for a deliberately malformed wire +// +// The control channel is line-oriented: one NDJSON line is one frame, and the +// terminator is the line break the buffer does not store. So +// `createMemoryNdjsonStream` has no third role -- `writer.write` and `inject` +// are the same function -- and it strips one trailing newline, so a caller +// that spells the terminator and one that omits it buffer the same line. A +// malformed control frame is spelled as a malformed line, which `inject` +// passes through unchanged apart from that newline, so there is no separate +// raw role for a caller to reach for. +// +// A reader supports ONE `read()` iteration. Both channels are single-consumer +// by construction -- the supervisor starts one pump per stream per child -- +// and the buffer is drained by whoever iterates, so a second iteration would +// silently steal frames from the first. `read()` enforces that rather than +// trusting it, because the way it gets violated is invisible: a spawner +// double that hands the same streams to a respawned child re-reads them, and +// the second iteration returns immediately off the closed stream instead of +// receiving the new child's frames. The child then looks like it died on its +// handshake, which is indistinguishable from the failure such a test is +// usually written to examine. + +import type { NdjsonReader, NdjsonWriter } from "../ipc/control-channel"; +import type { FrameReader } from "../ipc/event-channel"; +import type { FrameWriter } from "../ipc/event-sender"; + +/** + * Observers of the buffer growing, distinct from the reader's own waiter: a + * test watches for a line arriving so it can re-read `flushed` when there is + * something new, rather than on a timer. + */ +type WriteObservers = { list: (() => void)[] }; + +function announce(observers: WriteObservers): void { + const waiting = observers.list; + observers.list = []; + for (const observer of waiting) observer(); +} + +/** + * Resolves on the next item to reach the buffer, from either direction. + * + * Edge-triggered: it does not see an item already buffered. A caller reads + * `flushed`, arms this, and re-checks in a loop, with the read and the arm in + * ONE synchronous block -- an `await` between them is what loses an item. + * Which of the two comes first inside that block decides nothing, because + * nothing can land between two synchronous statements. Both orders are in use + * here (`waitForUpstreamPayload` arms first, the park-notify waits in + * `apps/sidecar` read first) and both are safe for that reason. + */ +function nextWriteOf(observers: WriteObservers): Promise { + return new Promise((resolve) => { + observers.list.push(resolve); + }); +} + +export type MemoryNdjsonStream = { + writer: NdjsonWriter; + reader: NdjsonReader; + inject(line: string): void; + flushed(): readonly string[]; + nextWrite(): Promise; + /** How many lines the consumer has taken off the buffer and finished with. */ + readCount(): number; + /** + * Resolves once the consumer has finished with at least `count` lines. + * + * The write report says a line arrived; this says the pump reading the + * stream is done with it. A test that injects a frame the consumer is + * supposed to IGNORE needs the second one: without it, the assertion that + * nothing happened can run before the consumer has even looked. + * + * "Finished with" is the consumer asking for the next line, which a + * `for await` loop does only after its body returns -- so the guarantee is + * exactly as strong as the loop body is. A body that dispatches work with + * `void` and returns has finished in this sense while its handler is still + * in flight, and a consumer that stops iterating never asks again, so its + * last line stays uncounted. + * + * Level-triggered on a count rather than edge-triggered on the next read, + * so a caller that arms it after the read already happened is not left + * waiting for another one. + */ + awaitReadCount(count: number): Promise; + close(): void; +}; + +export function createMemoryNdjsonStream(): MemoryNdjsonStream { + const buffer: string[] = []; + const observers: WriteObservers = { list: [] }; + const readObservers: WriteObservers = { list: [] }; + let reads = 0; + let waiter: (() => void) | null = null; + let done = false; + let reading = false; + + function wake(): void { + const w = waiter; + waiter = null; + if (w) w(); + } + + function push(line: string): void { + buffer.push(line.replace(/\n$/, "")); + wake(); + announce(observers); + } + + const reader: NdjsonReader = { + read(): AsyncIterableIterator { + if (reading) { + throw new Error( + "createMemoryNdjsonStream: a second read() on one stream; hand each spawn its own streams, as a real spawn does", + ); + } + reading = true; + return (async function* () { + let handedOut = false; + for (;;) { + if (handedOut) { + // Counted here, on re-entry, rather than beside the `yield`. A + // `for await` loop calls `next()` only after its body has run to + // completion, so re-entry is the point at which the previous line + // is finished with. Counting at the `yield` instead resolves a + // waiter whose continuation is queued AHEAD of the consumer's, so + // the one assertion the count exists for -- that a frame was + // ignored -- would run before the consumer had looked at it. + handedOut = false; + reads += 1; + announce(readObservers); + } + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("buffer shift returned undefined"); + } + handedOut = true; + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + + return { + writer: { write: push }, + reader, + inject: push, + flushed: () => buffer.slice(), + nextWrite: () => nextWriteOf(observers), + readCount: () => reads, + async awaitReadCount(count: number) { + for (;;) { + // Re-checked on every pass: level-triggered on the count, so a read + // that happened before this call resolves it. An edge wait on the next + // read deadlocks here, because the consumer usually reads an injected + // frame before a test can ask to be told about it. + const read = nextWriteOf(readObservers); + if (reads >= count) return; + await read; + } + }, + close() { + done = true; + wake(); + // A closed stream takes nothing further, so release anyone still + // watching rather than leaving them parked on a write that cannot come. + announce(observers); + announce(readObservers); + }, + }; +} + +export type MemoryFrameStream = { + writer: FrameWriter; + reader: FrameReader; + inject(bytes: Uint8Array): void; + injectRaw(bytes: Uint8Array): void; + flushed(): readonly Uint8Array[]; + nextWrite(): Promise; + close(): void; +}; + +export function createMemoryFrameStream(): MemoryFrameStream { + const buffer: Uint8Array[] = []; + const observers: WriteObservers = { list: [] }; + let waiter: (() => void) | null = null; + let done = false; + let reading = false; + + function wake(): void { + const w = waiter; + waiter = null; + if (w) w(); + } + + function pushRaw(bytes: Uint8Array): void { + buffer.push(bytes); + wake(); + announce(observers); + } + + const reader: FrameReader = { + read(): AsyncIterableIterator { + if (reading) { + throw new Error( + "createMemoryFrameStream: a second read() on one stream; hand each spawn its own streams, as a real spawn does", + ); + } + reading = true; + return (async function* () { + for (;;) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("frame buffer shift returned undefined"); + } + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + + return { + writer: { write: pushRaw }, + reader, + inject(bytes: Uint8Array) { + // The event channel is newline-delimited: its sender terminates every + // frame, and the receiver splits on that terminator. A caller passing + // envelope bytes is describing one frame, so the terminator belongs + // here rather than at each call site. + const framed = new Uint8Array(bytes.length + 1); + framed.set(bytes, 0); + framed[bytes.length] = 0x0a; + pushRaw(framed); + }, + injectRaw: pushRaw, + flushed: () => buffer.slice(), + nextWrite: () => nextWriteOf(observers), + close() { + done = true; + wake(); + announce(observers); + }, + }; +} diff --git a/vendor/intx/workflow-host/src/testing/spawn-observer.ts b/vendor/intx/workflow-host/src/testing/spawn-observer.ts new file mode 100644 index 0000000000..ceb046e212 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/spawn-observer.ts @@ -0,0 +1,47 @@ +// Records the environment each spawn was invoked with, and lets a caller wait +// for the first one. +// +// A test that needs the spawn-time env -- almost always to read the IPC +// channel id and mint a child-side sender -- has to wait for the spawner to +// be called. Twenty-two sites did that by assigning a mutable binding inside +// the spawner and re-reading it on a one-millisecond timer until it stopped +// being undefined. The spawner being called IS the event, so it is reported +// rather than inferred. + +export type SpawnObserver = { + /** Call from inside the spawner double with the env it received. */ + record(env: Record): void; + /** + * Resolve with the first recorded env, whether the spawn has happened yet + * or not. A respawn records again; this always resolves with the first. + */ + first(): Promise>; + /** Every env recorded so far, in spawn order. */ + all(): readonly Record[]; +}; + +export function createSpawnObserver(): SpawnObserver { + const envs: Record[] = []; + let waiters: (() => void)[] = []; + return { + record(env) { + envs.push(env); + const waiting = waiters; + waiters = []; + for (const waiter of waiting) waiter(); + }, + async first() { + for (;;) { + // Re-read on every pass, so a spawn that happened before this call + // resolves it rather than leaving it waiting for another one. + const spawned = new Promise((resolve) => { + waiters.push(resolve); + }); + const head = envs[0]; + if (head !== undefined) return head; + await spawned; + } + }, + all: () => envs.slice(), + }; +} diff --git a/vendor/intx/workflow-host/src/testing/stub-repo-store.ts b/vendor/intx/workflow-host/src/testing/stub-repo-store.ts new file mode 100644 index 0000000000..7612e10254 --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/stub-repo-store.ts @@ -0,0 +1,56 @@ +// A RepoStore that implements only what a test declares it needs, and throws +// for everything else. +// +// The throw is the point. These stubs assert, by failing, that the code under +// them touches nothing beyond the named surface -- so the narrow ones are not +// an incomplete version of the broad ones, and substituting a working store +// would silently permit paths that currently fail loudly. +// +// Only the two capability sets that were byte-identical across several files +// live here. The rest of this repo's stub stores are genuinely different +// fidelities, up to a 218-line in-memory implementation, and merging those +// would trade a real assertion for a smaller diff. + +import path from "node:path"; + +import type { RepoId, RepoStore } from "@intx/hub-sessions"; + +export type StubRepoStoreOpts = { + /** + * Permit `writeTreePreservingPrefix`, returning a fixed commit and no + * newly-terminal runs. Off by default, so a test that does not ask for it + * fails on the call rather than accepting it. + */ + writeTree?: boolean; +}; + +export function createStubRepoStore( + baseDir: string, + opts: StubRepoStoreOpts = {}, +): RepoStore { + const stub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(baseDir, repoId.kind, repoId.id); + }, + ...(opts.writeTree === true + ? { + writeTreePreservingPrefix: async () => ({ + commitSha: "deadbeefcafef00d", + newlyTerminalRuns: [], + }), + } + : {}), + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; every method the opts did not enable surfaces a precise failure via the proxy + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error( + `stub RepoStore: ${String(prop)} not implemented for this test`, + ); + }; + }, + }); +} diff --git a/vendor/intx/workflow-host/src/testing/supervisor-reaper.ts b/vendor/intx/workflow-host/src/testing/supervisor-reaper.ts new file mode 100644 index 0000000000..1aeb1379af --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/supervisor-reaper.ts @@ -0,0 +1,79 @@ +// Per-file registry of the supervisors a test built, so a lifecycle hook can +// tear them down whatever way the test ended. +// +// A supervisor is normally torn down by a `shutdown()` at the end of the test +// that built it, which an earlier failed assertion skips -- and some tests +// never had one. What that abandons is not inert: a dispatch suspended in +// `waitForRunTerminalOrPark` holds that function's armed five-minute +// backstop, whose clearing `finally` cannot run while the race it guards is +// pending. Aborting the cohort is what settles that race, so reaping is what +// disarms the timer. The unit pass shares a module registry per worker, so an +// un-reaped supervisor outlives its file and any failure it causes is charged +// to an unrelated test. +// +// The registry is per-call rather than module-level: two test files importing +// one shared array would reap each other's supervisors if their lifecycles +// ever overlapped. + +/** The part of a supervisor a reaper needs. */ +export type ReapableSupervisor = { + shutdown(): Promise; +}; + +export type SupervisorReaper = { + /** Register a supervisor and return it, for use at the construction site. */ + track(supervisor: T): T; + /** + * Shut down every supervisor registered since the last call. Pass straight + * to `afterEach`. + * + * The teardowns overlap: every `shutdown()` is called before any is awaited, + * so they settle in whatever order they finish rather than registration + * order. Nothing here sequences one supervisor's teardown after another's. + * + * `shutdown` early-returns once the phase is terminal. That guard does not + * cover `stopping`, so a teardown already in flight is re-entered here -- + * harmlessly, because every branch of the teardown is guarded on the prior + * phase and a second pass through `stopping` does no work. Either way the + * tests that tear themselves down are unaffected. + */ + reap(): Promise; +}; + +export function createSupervisorReaper(): SupervisorReaper { + const live: ReapableSupervisor[] = []; + return { + track(supervisor) { + live.push(supervisor); + return supervisor; + }, + async reap() { + // Every `shutdown()` is CALLED before any of them is awaited, so no + // supervisor's teardown waits on its neighbours': one that REJECTS and + // one that never SETTLES both leave the rest to run to completion. + // Awaiting them one at a time would not -- a teardown that never + // settled would mean the supervisors behind it never started theirs, + // each keeping its child and its armed backstop. + // + // A non-settling `shutdown()` still blocks this function's own + // resolution, because `allSettled` waits for that one too -- so the + // `afterEach` calling it does not complete, and the lane timeout is what + // fails the file. Abandoning the pending one would take a deadline, + // which "Synchronizing on State, Not Time" in CONVENTIONS.md rules out, + // and `shutdown()` offers no other signal to race. What this covers is + // the leak, not the hang. + // + // `shutdown` is documented as total, so a throw is a defect and fails + // the test rather than landing in a log nobody reads. + const settled = await Promise.allSettled( + live.splice(0).map((supervisor) => supervisor.shutdown()), + ); + const failures = settled.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) { + throw new AggregateError(failures, "supervisor teardown threw"); + } + }, + }; +} diff --git a/vendor/intx/workflow-host/src/testing/upstream-frames.ts b/vendor/intx/workflow-host/src/testing/upstream-frames.ts new file mode 100644 index 0000000000..57169b188f --- /dev/null +++ b/vendor/intx/workflow-host/src/testing/upstream-frames.ts @@ -0,0 +1,116 @@ +// Reading and awaiting the control frames a supervisor writes upstream. +// +// A test that drives a supervisor asserts on the frames it emitted, and has +// to wait for the one it cares about first. That wait was written as a +// deadline plus a one-millisecond tick that re-decoded the whole buffer -- +// two wall-clock numbers deciding a run whose subject is what a frame +// CONTAINS, never how quickly it appears. Waiting on the write removes the +// window instead of widening it. + +import { type } from "arktype"; + +import { ControlPayload } from "../ipc/control-channel"; +import { SignedEnvelope } from "../ipc/envelope"; + +/** + * Decode every payload of `type_` from `lines`, in arrival order. + * + * Frames that fail envelope or payload validation are skipped, so a test + * asserting on one payload kind is not fooled by an unrelated frame sharing + * the stream. + */ +export function readPayloadsOfType( + lines: readonly string[], + type_: T, +): Extract[] { + const out: Extract[] = []; + for (const line of lines) { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + continue; + } + const signed = SignedEnvelope(raw); + if (signed instanceof type.errors) continue; + const payload = ControlPayload(signed.envelope.payload); + if (payload instanceof type.errors) continue; + if (payload.type !== type_) continue; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the arktype narrow above pins the discriminator; the cast walks the union to the matching branch + out.push(payload as Extract); + } + return out; +} + +/** The runIds carried on every `trigger.fire` frame in `lines`, in order. */ +export function parseTriggerFireRunIds(lines: readonly string[]): string[] { + return readPayloadsOfType(lines, "trigger.fire").map((p) => p.data.runId); +} + +/** The part of a stream double this module reads. */ +export type UpstreamFrameSource = { + flushed(): readonly string[]; + nextWrite(): Promise; +}; + +/** + * Resolve with the first payload of `type_` that `match` accepts, whether it + * is already buffered or arrives later. + * + * The buffer is re-read only when a line actually arrives, and the wait + * carries no deadline: a frame that never comes is caught by the lane + * timeout, per "Synchronizing on State, Not Time" in CONVENTIONS.md. + */ +export async function waitForUpstreamPayload( + stream: UpstreamFrameSource, + type_: T, + match: ( + payload: Extract, + ) => boolean = () => true, +): Promise> { + for (;;) { + // Re-read on every pass, so a frame already in the buffer resolves this + // rather than leaving it waiting for the next write. + const arrived = stream.nextWrite(); + const found = readPayloadsOfType(stream.flushed(), type_).find(match); + if (found !== undefined) return found; + await arrived; + } +} + +/** + * Resolve once at least `count` payloads of `type_` have been written, with + * every matching payload in arrival order. + * + * The count form exists because most waits here are for the Nth frame rather + * than for a particular one; a caller wanting a specific frame should use + * `waitForUpstreamPayload` with a predicate. + */ +export async function waitForUpstreamPayloads( + stream: UpstreamFrameSource, + type_: T, + count = 1, +): Promise[]> { + for (;;) { + const arrived = stream.nextWrite(); + const found = readPayloadsOfType(stream.flushed(), type_); + if (found.length >= count) return found; + await arrived; + } +} + +/** + * Resolve once at least `count` `trigger.fire` frames have been written, with + * every runId seen in order. + */ +export async function waitForTriggerFireRunIds( + stream: UpstreamFrameSource, + count: number, +): Promise { + for (;;) { + const arrived = stream.nextWrite(); + const ids = parseTriggerFireRunIds(stream.flushed()); + if (ids.length >= count) return ids; + await arrived; + } +} diff --git a/vendor/intx/workflow-host/src/workflow-definition-loader.ts b/vendor/intx/workflow-host/src/workflow-definition-loader.ts index 63619bb4b6..a1953663e1 100644 --- a/vendor/intx/workflow-host/src/workflow-definition-loader.ts +++ b/vendor/intx/workflow-host/src/workflow-definition-loader.ts @@ -11,8 +11,8 @@ // validated at this boundary before being returned. // // Materialization is deliberately NOT done here. `@intx/workflow-host` -// stays free of a `@intx/tool-packaging` dependency (the sidecar owns -// that layer, see `apps/sidecar/src/tool-materialization.ts`), so the +// stays free of a `@intx/tool-packaging` dependency (that layer lives +// in `packages/tool-packaging/src/tool-materialization.ts`), so the // caller runs the closure machinery and hands the resulting package // directory in. This module only performs the import + evaluate + // validate step, which is the part that must run inside the child's @@ -32,9 +32,9 @@ import { type AnnotatedPluginFactory, type DirectorRegistry, type ToolDeclaration, -} from "@intx/agent"; +} from "@intx/agent/authoring"; import { PackageJSON, isContainedEntryPath } from "@intx/types/package-json"; -import { workflowDefinitionEnvelopeSchema } from "@intx/hub-sessions/substrate"; +import { workflowDefinitionEnvelopeSchema } from "@intx/types/workflow-definition"; import type { WorkflowDefinition } from "@intx/workflow/definition"; import type { ActionHandler, LoopFn, LoopFnRegistry } from "@intx/workflow"; diff --git a/vendor/intx/workflow-host/tsconfig.json b/vendor/intx/workflow-host/tsconfig.json index 7e6f0a1928..5cae9bf3d5 100644 --- a/vendor/intx/workflow-host/tsconfig.json +++ b/vendor/intx/workflow-host/tsconfig.json @@ -18,6 +18,12 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, + { + "path": "../harness/tsconfig.src.json" + }, { "path": "../hub-sessions/tsconfig.src.json" }, @@ -36,6 +42,9 @@ { "path": "../types/tsconfig.src.json" }, + { + "path": "../workflow-deploy/tsconfig.src.json" + }, { "path": "../workflow/tsconfig.src.json" } diff --git a/vendor/intx/workflow-host/tsconfig.src.json b/vendor/intx/workflow-host/tsconfig.src.json index af6aafe358..e3b02e66a0 100644 --- a/vendor/intx/workflow-host/tsconfig.src.json +++ b/vendor/intx/workflow-host/tsconfig.src.json @@ -22,6 +22,12 @@ { "path": "../agent/tsconfig.src.json" }, + { + "path": "../authz/tsconfig.src.json" + }, + { + "path": "../harness/tsconfig.src.json" + }, { "path": "../hub-sessions/tsconfig.src.json" }, @@ -40,6 +46,9 @@ { "path": "../types/tsconfig.src.json" }, + { + "path": "../workflow-deploy/tsconfig.src.json" + }, { "path": "../workflow/tsconfig.src.json" } diff --git a/vendor/intx/workflow/README.md b/vendor/intx/workflow/README.md index 7e58932982..873c164755 100644 --- a/vendor/intx/workflow/README.md +++ b/vendor/intx/workflow/README.md @@ -11,6 +11,9 @@ persisted, scheduled, or spawned — those are the host's job. Multi-entry exports: +- `@intx/workflow/projection` — inspect declared plugins, project a live + definition to inert data, hash that projection, and derive an `inlineBodyRef` + without loading the workflow execution runtime. - `@intx/workflow/definition` — `WorkflowDefinition`, `defineWorkflow`, `hashDefinition`, the `stepId` shape rule. The on-disk form a workflow lives in. It also carries the canonical step walk @@ -26,7 +29,9 @@ Multi-entry exports: - `@intx/workflow/runlocal` — an in-memory adapter for tests. The scheduler, RepoStore, blob substrate, and spawn-child callback all exist purely in process memory so tests can drive the runtime - without a substrate. + without a substrate. Its `actionCredentials` option gives action + handlers mediated credentials; `createActionCredentialsFactory` from + `@intx/harness` builds one. For a production host (workflow-run repo backing, scheduler that honors wall-clock fire times, signal channel that observes commits, @@ -188,12 +193,16 @@ work that belongs at an `action`/LoopFn seam. Two facts govern what a `loop` body can and cannot survive. -First, crashes. An `action` runs at most once. A mid-invocation crash fails the -run and the effect is never re-run -- and this is true everywhere, not just in a -loop: a top-level action or agent step that crashes mid-invocation also settles -`RunFailed` (it is not re-invoked on resume). So a crash inside a loop iteration -fails the run exactly as a crash in any other step does; loops are not special -here. +First, crashes. An `action` runs at most once. A mid-invocation crash fails that +iteration and the effect is never re-run -- and this is true everywhere, not +just in a loop: a top-level action or agent step that crashes mid-invocation +also settles `RunFailed` (it is not re-invoked on resume). The loop's response +depends on `onIterationFailure`. Absent, or `"end"`, the failed iteration fails +the run, the same as any other step. `"tolerate"` records a genuine iteration +failure and continues with the same input, without calling `while` or `carry`; +the crashed action is still not re-run. A cancelled iteration, or a failure +that is an abort teardown, fails the loop either way. If every attempt is +tolerated, the loop exhausts and publishes `final: null`. Second, suspension. The body-ban forbids a loop body from containing a `sleep` or an `onTrigger`. It does NOT forbid an `awaitSignal`, a `childWorkflow`, or a @@ -209,38 +218,48 @@ A loop body may also spawn a `childWorkflow` grandchild: the child is lifted to a ref and runs as its own child run, depth-counted against the tree-wide spawn ceiling exactly like any other child. -A `childWorkflow` child cannot hold a human gate, and neither can anything -inside it. The relay that makes a loop iteration's pause durable is severed at -the `childWorkflow` boundary: a child run carries no address of its own, and -the spawning step waits for its terminal rather than driving it across parks, -so no decision can be routed back down. A body nested inside the child -inherits that answer, so this holds at any depth rather than only for a gate -written directly in the child. - -An untimed wait beneath that boundary fails the step that asked for it, naming -the gate, rather than waiting on a signal nobody can send. A gate carrying a -`timeout` is unaffected, because its own timer resolves it without anything -upstream. +A `childWorkflow` child carries no address of its own. The spawning step +drives the child across its parks and relays each one onto the parent: an +approval, carrying the body's snapshot, an author `awaitSignal`, and the +input park a trigger budget takes between turns. A gate inside a loop in +the child climbs the same way, one layer at a time, until it reaches the +deployment's addressable run. The hub approval is registered on that run. +The spawn step awaits one signal at a time. A second park that is already +live in the child stays queued until the decision for the first is relayed +down. A queued park whose gate has already moved on is not given a signal +that arrived while it waited. + +A parked child survives a process restart. Resume reattaches the same +child run from its log and reestablishes the relay on the spawn step. The +child stays unaddressable; the parent is the resume root. + +On drain, the relay sits on the spawn step. That step defaults to +`"wait"`, so the drain signal does not abort it, whether the child is +parked or still mid-step. An explicit `"cancel"` still aborts the child +through `localAbort`. The child's own `drainBehavior` is not what drain +consults. + +An untimed park is still refused in a run whose env says nothing upstream +can answer it. That run writes no suspension. A child may already have +recorded its own await, because a `childWorkflow` spawner forces the +child's flag true; the parent's refusal then tears the child down. A gate +carrying a `timeout` is unaffected, because its own timer resolves it +without anything upstream. Where the spawn step sits at a workflow root, an `onFailure` handler on it -absorbs this failure like any other child failure, so a workflow that routes -around child failures swallows it; leave that step unrouted if you want to see -it. A spawn step in a `loop` body cannot carry `onFailure` at all -- routing is -honored only at a workflow root, and a loop body is not one -- so the failure -surfaces there regardless. - -To hold a human gate, keep the `awaitSignal` in a run the control plane can -address: the deployment's own run, whether directly or in a `loop` body or -`onTrigger` section body of it, whose container relays the decision down. What -no container can do is reach across the `childWorkflow` boundary. - -Of those homes the `onTrigger` section body carries a caveat the relay does not -cause. A section body's run id omits the enclosing run, so a second top-level -run of the same deployment finds the first run's body log already terminal and -short-circuits it: the body never executes, nobody is asked, and the run still -reports a clean terminal status (tracked as INTR-552). A gate a repeating -deployment depends on belongs in the deployment's own run or in a `loop` body, -whose iteration run ids re-root per run. +routes a child that ended `failed` and does not route one that ended +`cancelled`. A spawn step in a `loop` body cannot carry `onFailure` at all -- +routing is honored only at a workflow root, and a loop body is not one. + +To hold a human gate inside a child, write the `awaitSignal` in the child. +The deployment run is what an operator addresses. A signal posted there is +relayed down. The child run id is not an address. + +Of those homes the `onTrigger` section body carries the same relay. Its run id +is `____` (`sectionBodyRunId`), so two runs +do not share a body log. A body already recorded as `__` +stays on that id. A `workflow_run` row already written under the old id still +collides at the hub until that row is gone. A loop body may contain a nested `loop`. An inner loop resolves its body ref from the same top-level bodies map (a loop iteration inherits its parent's env), @@ -264,12 +283,12 @@ layer. Practical guidance: use a loop to repeat a self-contained unit -- which may park on an `awaitSignal`, spawn a `childWorkflow`, or run a nested `loop` -- until a -pure `while`/`carry` says stop. Keep the gate in the body itself: an untimed -`awaitSignal` inside a `childWorkflow` the body spawns, at any depth, fails the -step rather than waiting. Model a `sleep` delay at a top-level step or an +pure `while`/`carry` says stop. An untimed `awaitSignal` inside a `childWorkflow` +the body spawns waits, and climbs through the spawn step, the same way a gate in +the loop body itself does. Model a `sleep` delay at a top-level step or an onTrigger section, not in a loop body. Keep a loop body's action idempotent -where practical, since a mid-invocation crash fails the run and the effect is -never re-run. +where practical, since a mid-invocation crash is not re-run: the iteration +fails, and the loop fails the run unless `onIterationFailure` is `"tolerate"`. These primitives are composed end to end by the interchange-demo dispatch orchestrator -- an outer per-level `loop` wrapping a Phase-5 verification `loop` diff --git a/vendor/intx/workflow/VENDORED-FROM b/vendor/intx/workflow/VENDORED-FROM index 9971fbad04..96a5868392 100644 --- a/vendor/intx/workflow/VENDORED-FROM +++ b/vendor/intx/workflow/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow) -Commit: 9febf699e1c8a01fd329927e5a14be7f94e9919e (origin/main, v0.4.0) +Commit: 0a4eb59b4276e6912082c9e3259b69ce4447869f (origin/main, v0.5.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages). One source delta (corbitsdev/workbench#709): run.ts's per-step timeout is an execution budget - trigger-idle input parks disarm the step timer across the wait and re-arm a fresh budget on resume, while the approval park stays armed (snapshot-bound, still bounded by `step.timeout`). Absorbed upstream at this re-vendor: the selectors.ts project-selector own-key/undefined guards. diff --git a/vendor/intx/workflow/package.json b/vendor/intx/workflow/package.json index 3d8480301a..9910d00b4e 100644 --- a/vendor/intx/workflow/package.json +++ b/vendor/intx/workflow/package.json @@ -1,7 +1,7 @@ { "name": "@intx/workflow", "description": "Definition surface, state machine, and abstract runtime for multi-step agent workflows", - "version": "0.4.0", + "version": "0.5.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -24,6 +24,14 @@ "./runlocal": { "types": "./src/runlocal/index.ts", "default": "./src/runlocal/index.ts" + }, + "./testing": { + "types": "./src/testing/index.ts", + "default": "./src/testing/index.ts" + }, + "./projection": { + "types": "./src/projection.ts", + "default": "./src/projection.ts" } }, "scripts": { diff --git a/vendor/intx/workflow/src/definition/primitives.ts b/vendor/intx/workflow/src/definition/primitives.ts index d53738a1cd..f0a2526718 100644 --- a/vendor/intx/workflow/src/definition/primitives.ts +++ b/vendor/intx/workflow/src/definition/primitives.ts @@ -7,18 +7,22 @@ // reuse the same constructor result under different keys without // repeating themselves. // -// `drainBehavior` defaults differ per primitive: `sleep` and -// `childWorkflow` default to `"cancel"` because long compute should not -// block a redeploy past `drainTimeout`; `awaitSignal` defaults to -// `"wait"` because human-in-the-loop pauses are the canonical case the -// spec calls out and operators do not want them silently cancelled at -// redeploy. A `step` defaults to `"cancel"` when its trigger budget is +// `drainBehavior` defaults differ per primitive: `sleep` defaults to +// `"cancel"` because long compute should not block a redeploy past +// `drainTimeout`. `childWorkflow` defaults to `"wait"`: drain consults +// the spawn step for the whole time it is in flight, including a parked +// relay and a child that is still mid-step, and a local abort would tear +// that child down. `awaitSignal` defaults to `"wait"` because +// human-in-the-loop pauses are the canonical case the spec calls out and +// operators do not want them silently cancelled at redeploy. A `step` +// defaults to `"cancel"` when its trigger budget is // `1` (batch) and `"wait"` when the budget is larger (multi-turn or // unbounded) -- a long-lived step is definitionally interactive. // `map`'s outer node carries no `drainBehavior` -- its inner step // carries its own. import type { AgentDefinition, BaseEnv } from "@intx/agent"; +import { ToolCredentialHandle } from "@intx/types/package-json"; import type { Type } from "arktype"; // Type-only import: an action handler's `ctx` parameter is the runtime's @@ -77,6 +81,13 @@ export interface StepPrimitive extends PrimitiveBase { kind: "step"; agent: AgentDefinition; input?: Selector; + /** + * Per-call inference options for this step's model call, resolved beside + * `input`. Must resolve to `null`, `undefined`, or an object limited to + * `PerCallInferenceOptions` keys; anything else fails the step as a + * selector error. Absent means the agent's defaults. + */ + inference?: Selector; reads?: readonly Selector[]; writes?: readonly Selector[]; retry?: RetryPolicy; @@ -140,11 +151,11 @@ export interface GatePrimitive extends PrimitiveBase { * run and is valid anywhere. * * An untimed gate has the one exit, so it can only end where something outside - * the run can deliver -- a run the control plane can address, or a container - * relaying a decision down. In a run tree with neither, the runtime refuses the - * gate at the park, before any suspension is recorded, rather than waiting on a - * signal that can never arrive. A `childWorkflow` child is the boundary that - * produces such a tree; see {@link ChildWorkflowPrimitive}. + * the run can deliver. A run the control plane can address delivers directly. + * A loop body, an `onTrigger` section, and a `childWorkflow` child relay the + * park onto that run. The runtime refuses the gate at the park, before any + * suspension is recorded, only when this run's env says nothing upstream can + * answer it. A timed gate is exempt: its own timer resolves it. * * The same refusal covers the parks the runtime takes on a step's behalf, so * this rule is not confined to a gate an author writes here. @@ -171,21 +182,36 @@ export interface SleepPrimitive extends PrimitiveBase { * approved surface and no separately-deployed asset is read. `input` * selects the child run's launch payload. * - * The child is driven to its terminal, not across parks, and carries no - * address of its own, so nothing upstream can answer a park inside it. Every - * untimed park beneath this boundary -- at any depth, including inside a - * `loop` body in the child -- is refused at the park and fails the child. That - * is an untimed {@link AwaitSignalPrimitive}, and also the parks the runtime - * takes on a step's behalf without the author writing one: an agent step - * suspending on a tool declared `approval: "ask"`, and the input park a step - * with a trigger budget takes between deliveries. A child containing no - * `awaitSignal` is not thereby safe. Give such a gate a `timeout`, or hold it - * in a run the control plane can address. + * The child carries no address of its own. The spawning step drives it + * across parks and relays each one onto the parent run: an approval, with + * the body's snapshot, an author {@link AwaitSignalPrimitive}, and the input + * park a trigger budget takes between turns. A gate inside a `loop` in the + * child climbs the same way, one layer at a time, until it reaches the + * deployment's addressable run. The hub approval is registered on that run. + * The spawn step awaits one signal at a time. A second park that is already + * live in the child stays queued until the decision for the first is relayed + * down. A queued park whose gate has already moved on is not given a signal + * that arrived while it waited. + * + * A parked child survives a process restart. Resume reattaches the same + * child run from its log and reestablishes the relay on the spawn step. + * The child stays unaddressable; the parent is the resume root. On drain + * the relay sits on this step. The step defaults to `"wait"`, so the drain + * signal does not abort it. That covers the whole time the step is in + * flight, including a parked relay and a child that is still mid-step. An + * explicit `"cancel"` still aborts the child through `localAbort`. The + * child's own `drainBehavior` is not what drain consults. + * + * A run whose env says nothing upstream can answer refuses an untimed + * relay before that run writes a suspension. The child may already have + * recorded its own await, because its spawner forces the flag true; the + * refusal then tears the child down. A gate carrying a `timeout` is + * exempt: its own timer resolves it. * - * The refusal surfaces to the parent as an ordinary child failure. Routing it - * with `onFailure` is available only where `onFailure` is honored at all: on a - * direct entry of a workflow root. A spawn inside a `loop` body may not carry - * `onFailure`, so there the failure surfaces unrouted. + * A child that ends `failed` or `cancelled` fails the spawn step. + * `onFailure` routes a failed child and does not route a cancelled one, and + * only where `onFailure` is honored at all: a direct entry of a workflow + * root. A spawn inside a `loop` body may not carry `onFailure`. */ export interface ChildWorkflowPrimitive extends PrimitiveBase { kind: "childWorkflow"; @@ -251,6 +277,18 @@ export interface ActionPrimitive extends PrimitiveBase { drainBehavior?: DrainBehavior; /** See {@link StepPrimitive.onFailure}. */ onFailure?: string; + /** + * The credential handles this action declares for its handler. The scope + * is per handler: inside `perform`'s `run`, the handler may resolve any + * handle a step running that handler declares and a root binding names for + * it. Deploy refuses a declared handle with no binding, and a binding for a + * handle no such step declares. `action()` validates each handle's + * format and rejects a duplicate, rejects a non-empty list without an + * `effect.requires` capability, and omits the field when the list is + * empty. The handles are part of the approved projection and the + * definition hash, in the order the author wrote them. + */ + credentials?: readonly string[]; } /** @@ -262,12 +300,15 @@ export interface ActionPrimitive extends PrimitiveBase { * * A deployed handler is a BARE MODULE EXPORT of the module named by the * package's `interchange.actions` field, resolved by export name. It - * receives exactly these three parameters and nothing else: no injected - * services, and no opportunity to close over host configuration, because - * nothing in the deployment constructs it. `ctx` carries one method, - * `perform`. Per-deployment configuration must therefore arrive through - * `input`, which in practice means the author selects it out of the - * trigger payload via the action's {@link ActionPrimitive.input} selector. + * receives exactly these three parameters and nothing else, and has no + * opportunity to close over host configuration, because nothing in the + * deployment constructs it. `ctx` carries `perform`, and `perform` gives each + * `run` the one host service: the consumer-scoped `credentials` capability, + * which resolves the handles the action declares and works only until that + * `run` settles. + * Per-deployment configuration must therefore arrive through `input`, which + * in practice means the author selects it out of the trigger payload via the + * action's {@link ActionPrimitive.input} selector. */ export type ActionHandler = ( input: unknown, @@ -300,15 +341,25 @@ export type ActionHandler = ( * `{ outcome, iterations, carry, final }`: * * - `outcome` is `"converged"` or `"exhausted"`, the arm the loop routed to. - * - `iterations` is how many iterations ran. - * - `carry` is the LAST iteration's INPUT. The loop settles the instant - * `while` goes false, before `carry` runs on that iteration, so this is - * the state the converging iteration worked from, not a state derived - * from its result. - * - `final` is the LAST iteration's OUTPUT -- the body's per-step output - * record, keyed by step id, exactly as `while` received it. This is where - * a `while` that judges the iteration output leaves its answer; the - * scoped iteration step ids are not selector paths. + * - `iterations` counts every attempt, including a tolerated failure. + * - `carry` is the last attempt's INPUT. The loop settles the instant + * `while` goes false, before `carry` runs on that iteration, so on + * convergence this is the state that iteration worked from. A tolerated + * failure does not call `carry`, so the input does not advance. + * - `final` is the last completed attempt's OUTPUT -- the body's per-step + * output record, keyed by step id -- or `null` when every attempt was a + * tolerated failure. A tolerated failure records no output, so `final` + * and `carry` can describe different attempts. The scoped iteration step + * ids are not selector paths, so `final` is how a downstream step reaches + * inside a completed iteration. + * + * `onIterationFailure` is absent or `"end"` unless the author opts in. + * `"tolerate"` records a genuine iteration failure and continues with the + * same input, without calling `while` or `carry`. The attempt still counts + * toward `maxIterations`. A cancelled attempt, or a failure that is an + * abort teardown, fails the loop either way. The field stays off the + * primitive when unset, so a default loop's hash is unchanged. See + * {@link BodyFailurePolicy}. * * That record is persisted on the loop's `StepCompleted`, and a run that * crashes after the loop settles replays the persisted value rather than @@ -331,6 +382,7 @@ export interface LoopPrimitive extends PrimitiveBase { input?: Selector; maxIterations: number; onExhausted: string; + onIterationFailure?: BodyFailurePolicy; drainBehavior?: DrainBehavior; } @@ -405,6 +457,8 @@ export type Primitive = export interface StepOpts { agent: AgentDefinition; input?: Selector; + /** See {@link StepPrimitive.inference}. */ + inference?: Selector; reads?: readonly Selector[]; writes?: readonly Selector[]; retry?: RetryPolicy; @@ -497,6 +551,7 @@ export function step( agent, drainBehavior, ...(opts.input !== undefined ? { input: opts.input } : {}), + ...(opts.inference !== undefined ? { inference: opts.inference } : {}), ...(opts.reads !== undefined ? { reads: opts.reads } : {}), ...(opts.writes !== undefined ? { writes: opts.writes } : {}), ...(opts.retry !== undefined ? { retry: opts.retry } : {}), @@ -612,7 +667,9 @@ export interface ChildWorkflowOpts { } export function childWorkflow(opts: ChildWorkflowOpts): ChildWorkflowPrimitive { - const drainBehavior: DrainBehavior = opts.drainBehavior ?? "cancel"; + // Stored on the primitive, so the value is part of the definition hash. + // Omitting the option selects `"wait"`. + const drainBehavior: DrainBehavior = opts.drainBehavior ?? "wait"; return { kind: "childWorkflow", id: "", @@ -650,10 +707,35 @@ export interface ActionOpts { after?: readonly string[]; /** See {@link StepPrimitive.onFailure}. */ onFailure?: string; + /** See {@link ActionPrimitive.credentials}. */ + credentials?: readonly string[]; } export function action(opts: ActionOpts): ActionPrimitive { const drainBehavior: DrainBehavior = opts.drainBehavior ?? "cancel"; + const credentials = opts.credentials ?? []; + const seen = new Set(); + for (const handle of credentials) { + if (!ToolCredentialHandle.allows(handle)) { + throw new Error( + `action ${JSON.stringify(opts.handler)} declares credential handle ${JSON.stringify(handle)}, which does not match ${ToolCredentialHandle.expression}`, + ); + } + if (seen.has(handle)) { + throw new Error( + `action ${JSON.stringify(opts.handler)} declares credential handle ${JSON.stringify(handle)} more than once`, + ); + } + seen.add(handle); + } + // Credentials resolve only inside `perform`'s `run`, and `perform` accepts + // only a capability the action requires. Without one, the declared handles + // could never be used. + if (credentials.length > 0 && (opts.effect?.requires.length ?? 0) === 0) { + throw new Error( + `action ${JSON.stringify(opts.handler)} declares credential handles but no effect.requires; its credentials resolve only inside perform, which needs a declared capability`, + ); + } return { kind: "action", id: "", @@ -664,6 +746,7 @@ export function action(opts: ActionOpts): ActionPrimitive { ...(opts.timeout !== undefined ? { timeout: opts.timeout } : {}), ...(opts.after !== undefined ? { after: opts.after } : {}), ...(opts.onFailure !== undefined ? { onFailure: opts.onFailure } : {}), + ...(credentials.length > 0 ? { credentials } : {}), }; } @@ -674,6 +757,7 @@ export interface LoopOpts { input?: Selector; maxIterations: number; onExhausted: string; + onIterationFailure?: BodyFailurePolicy; drainBehavior?: DrainBehavior; after?: readonly string[]; } @@ -693,6 +777,13 @@ export function loop(opts: LoopOpts): LoopPrimitive { onExhausted: opts.onExhausted, drainBehavior, ...(opts.input !== undefined ? { input: opts.input } : {}), + // Conditional passthrough (NOT resolved to a default like drainBehavior): + // an absent policy must leave the field off so a default loop's inert + // projection -- and therefore its approval hash -- is unchanged. An + // explicit "end" stays on the object; only absence is omitted. + ...(opts.onIterationFailure !== undefined + ? { onIterationFailure: opts.onIterationFailure } + : {}), ...(opts.after !== undefined ? { after: opts.after } : {}), }; } diff --git a/vendor/intx/workflow/src/definition/shorthand.ts b/vendor/intx/workflow/src/definition/shorthand.ts index 78fe50b95e..86bd9f1e3a 100644 --- a/vendor/intx/workflow/src/definition/shorthand.ts +++ b/vendor/intx/workflow/src/definition/shorthand.ts @@ -28,7 +28,7 @@ import type { GrantRequirement, SidecarCapabilityPolicy, } from "@intx/types"; -import type { InboundMailPolicy } from "@intx/types/runtime"; +import type { InboundMailPolicy } from "@intx/types/inbound-mail-policy"; import { step } from "./primitives"; import type { Primitive } from "./primitives"; diff --git a/vendor/intx/workflow/src/definition/step-walk.ts b/vendor/intx/workflow/src/definition/step-walk.ts index f6746cfe5e..77775d485c 100644 --- a/vendor/intx/workflow/src/definition/step-walk.ts +++ b/vendor/intx/workflow/src/definition/step-walk.ts @@ -297,10 +297,10 @@ export function walkNestedWorkflowSteps( * consumer that must keep those apart walks with an explicit descent and reads * each entry's `tree` instead. * - * Exported for consumers that want the id set alone. Nothing in this repository - * calls it: the deploy gate needs each step's ancestor chain and the sidecar - * needs a specific descent, so both walk directly rather than through this - * convenience. + * Exported for consumers that want the id set alone, such as the local + * credentials snapshot in `@intx/workflow-host/local`. The deploy gate needs + * each step's ancestor chain and the sidecar needs a specific descent, so both + * walk directly rather than through this convenience. */ export function executableStepIds( definition: WorkflowDefinition, diff --git a/vendor/intx/workflow/src/definition/workflow.ts b/vendor/intx/workflow/src/definition/workflow.ts index eafcf9a77b..269a176ffb 100644 --- a/vendor/intx/workflow/src/definition/workflow.ts +++ b/vendor/intx/workflow/src/definition/workflow.ts @@ -7,14 +7,14 @@ // from its record key and applies the default-input convention so the // runtime sees a fully-specified definition with no implicit shape. -import { canonicalizeForHash } from "@intx/agent"; +import { canonicalizeForHash } from "@intx/agent/canonicalize"; import type { AgentDefinition, BaseEnv } from "@intx/agent"; import type { CredentialBinding, GrantRequirement, SidecarCapabilityPolicy, } from "@intx/types"; -import type { InboundMailPolicy } from "@intx/types/runtime"; +import type { InboundMailPolicy } from "@intx/types/inbound-mail-policy"; import { normalizeSingularShorthand } from "./shorthand"; import { @@ -53,11 +53,10 @@ export interface WorkflowDefinition { grantRequirements?: readonly GrantRequirement[]; /** * The credential bindings a launch resolves against tenant-owned - * credentials, each mapping a tool package's declared handle to a - * concrete provider and authorizing the delegation against the - * binding's authority. The launch reads these from the folded body and - * materializes a consumer-scoped `credential:{id}` / `use` grant per - * binding. + * credentials. Each maps a tool package's or an action handler's + * declared handle to a provider. Ownership authorizes the use, and the + * launch stamps a consumer-scoped `credential:{id}` / `use` grant with + * `origin = 'system'`. */ credentialBindings?: readonly CredentialBinding[]; sidecarPlacement?: SidecarCapabilityPolicy; @@ -436,11 +435,13 @@ function validateStepIds(steps: Record): void { // derives from it: an inline-body ref (`__`, and under // nesting `__`), a loop iteration body run id // (`____`), and an onTrigger section body run id - // (`__`, which is parsed back). A `__` inside a step id - // would make one of those ids ambiguous with a different chain -- and they - // key the durable store, so the collision is silent shared-state - // corruption. A body step id feeds those same joins, which is why this pass - // has to reach a body rather than trust it to have normalized itself. + // (`____` via `sectionBodyRunId`, which is + // parsed back, as is a body already recorded as `__`). + // A `__` inside a step id would make one of those ids ambiguous with a + // different chain -- and they key the durable store, so the collision is + // silent shared-state corruption. A body step id feeds those same joins, + // which is why this pass has to reach a body rather than trust it to have + // normalized itself. if (stepId.includes("__")) { throw new Error( `step id ${JSON.stringify(stepId)} must not contain "__"; ` + @@ -548,6 +549,18 @@ function validateOnFailureStraddlers(steps: Record): void { if (!normalLive.has(straddlerId)) continue; const straddler = steps[straddlerId]; if (straddler === undefined) continue; + if ( + straddler.kind === "step" && + straddler.inference !== undefined && + unitOutputRefs(straddler.inference, unitId).length > 0 + ) { + throw new Error( + `step ${straddlerId} straddles onFailure unit ${unitId} and its ` + + `handler ${handlerId} but its inference selector reads ` + + `steps.${unitId}.output; the failure sentinel is not valid ` + + `inference options`, + ); + } const selector = straddlerOutputSelector(straddler); if (selector === undefined) continue; const refs = unitOutputRefs(selector, unitId); @@ -793,6 +806,8 @@ function validateLoopBody( // input), so per-iteration "route and proceed" is incoherent -- a routed // body step yields no output to carry. Reject onFailure on any body // step, including a map's inner step reached through this walk. + // `onIterationFailure` is the loop's own policy for a failed iteration. + // It is not a body `onFailure`, and tolerate must not be rewritten as one. assertNoRoutableFailure(bodyStepId, bodyPrimitive); } // The body-only bans above run first so their message wins over a generic @@ -973,29 +988,29 @@ function buildDependencyAdjacency( } /** - * Reject a definition that lets two schedulable-concurrent `awaitSignal` steps - * await the same signal name. `handleSignalReceived` is a FIFO single-consumer - * scan keyed on the name, so two gates awaiting one name simultaneously make a - * delivery's binding order-dependent, and on resume the consumed signal cannot - * be durably re-bound to a specific gate. INTR-277 added a runtime fail-loud - * guard for this topology; rejecting it here makes it unauthorable. + * Reject a definition that lets two schedulable-concurrent name-FIFO + * `awaitSignal` steps await the same signal name. A name-only delivery + * binds one such park, so two of them live at once make the binding + * order-dependent. The runtime guard rejects that topology for a + * name-addressed relay. A correlated park is outside this check. * - * A loop or inline onTrigger body's awaitSignal park relays up into the parent - * run over the body's own author signal name, so a container step contributes - * its body's await names into the parent's concurrency set (recursively through - * nested loop / inline onTrigger bodies). A childWorkflow runs as a separate - * run with its own signal namespace, so its awaiters never collide with the - * parent's and are excluded. + * A loop body's awaitSignal relays onto the parent under the body's + * author signal name and stays name-FIFO, so the loop contributes those + * names here, including through nested loops. An inline childWorkflow + * relays the same way and contributes its body's names, including a + * child nested in a loop. An onTrigger relay is correlated, so its await + * names are not added. Two sections, a section beside a parent gate, and + * a section beside a loop may share an author name. + * `validateOnTriggerBody` re-enters `validateSteps` on an inline body, so + * two name-FIFO awaiters inside one section body are still rejected. * * Two static-analysis limitations remain, both backstopped by the runtime - * guard: - * - Conservative over-reject: two same-name awaiters on a gate's mutually - * exclusive then/else branches are rejected, because deciding they can - * never both be live is a dominator analysis whose permissive-direction - * error would re-admit the hazard. - * - Under-reject: a `{ ref }` onTrigger body is a separately-deployed asset - * not visible here, so an await name it relays is not collected; a - * collision with such a body is caught by the runtime guard, not here. + * guard for name-FIFO awaiters. Two same-name awaiters on a gate's + * mutually exclusive then/else branches are rejected: deciding they can + * never both be live is a dominator analysis whose permissive-direction + * error would re-admit the hazard. A `{ ref }` childWorkflow is a + * separate definition, so an await name it relays is not collected, and + * the runtime guard catches that collision. */ function validateConcurrentAwaitSignalNames( steps: Record, @@ -1034,11 +1049,15 @@ function validateConcurrentAwaitSignalNames( } /** - * Collect the author signal names a loop or inline onTrigger body relays up to - * the parent run, recursing through nested loop and inline onTrigger bodies. A - * flat `steps` record keys gate branches and onTimeout targets as siblings, so - * enumerating the record covers them without a graph walk. A childWorkflow body - * runs in a separate signal namespace and is not relayed, so it is skipped. + * Collect the author signal names a loop body or an inline childWorkflow + * relays onto the parent run. The walk recurses through nested loops and + * nested inline children. Those names stay on the parent's name-FIFO + * concurrency check. An onTrigger body is not collected: its relay park + * is correlated and may share an author name with other parks. A flat + * `steps` record keys gate branches and onTimeout targets as siblings, so + * enumerating the record covers them without a graph walk. A `{ ref }` + * childWorkflow stays opaque: its names are not in this definition, and + * the runtime relay refuses a second name-FIFO awaiter of the same name. */ function collectRelayedAwaitSignalNames( primitive: Primitive, @@ -1046,8 +1065,11 @@ function collectRelayedAwaitSignalNames( let body: Record | undefined; if (primitive.kind === "loop") { body = primitive.body.steps; - } else if (primitive.kind === "onTrigger" && "inline" in primitive.body) { - body = primitive.body.inline.steps; + } else if ( + primitive.kind === "childWorkflow" && + "inline" in primitive.definition + ) { + body = primitive.definition.inline.steps; } if (body === undefined) return []; const names: { name: string; bodyStepId: string }[] = []; diff --git a/vendor/intx/workflow/src/index.ts b/vendor/intx/workflow/src/index.ts index 868e87ba8e..a58c1c3a8f 100644 --- a/vendor/intx/workflow/src/index.ts +++ b/vendor/intx/workflow/src/index.ts @@ -6,9 +6,9 @@ export type { export * from "./state-machine/index"; export * from "./definition/index"; export { collectDeclaredPluginNames } from "./declared-plugins"; +export { inlineBodyRef } from "./inline-body-ref"; export { collectSidecarCapabilityRules } from "./sidecar-capabilities"; export { - inlineBodyRef, rewriteInlineOnTriggerBodies, rewriteInlineChildWorkflowBodies, enumerateInlineLoopBodies, @@ -38,7 +38,13 @@ export { } from "./live-inert-projector"; export { runtimeRun, + bodyParkedSignals, + loopIterationCursor, + loopOccurrenceReattach, + onTriggerOccurrenceReattach, + resumeResidualOf, createSuspendableChildHandle, + createTerminalChildHandle, createLoopIterationHandle, MAX_CHILD_SPAWN_DEPTH, ChildSpawnDepthExceededError, @@ -51,6 +57,8 @@ export { scopedStepId, baseStepId, loopBodyRunId, + sectionBodyRunId, + type ActionCredentialsFactory, type ActionInvokeRequest, type ActionInvokeResult, type ActionInvoker, @@ -62,10 +70,16 @@ export { type LoopFn, type LoopFnRegistry, type ParkedApprovalOp, + type PerformScope, type ReadParkedApprovalOps, type RepoStore, type RunResult, type RuntimeRunOptions, + type LoopIterationCursor, + type OccurrenceReattach, + type ResumeResidual, + type StepKindLookup, + type RuntimeWorkflowRun, type Scheduler, type SignalChannel, type SpawnChildWorkflow, @@ -83,6 +97,8 @@ export { runLocal, createDefaultActionInvoker, createInMemoryEffectLedger, + unavailableActionCredentials, + unavailableCredentialCapability, createSpawnLoopIteration, createInMemoryRepoStore, createInMemoryScheduler, diff --git a/vendor/intx/workflow/src/inline-body-ref.ts b/vendor/intx/workflow/src/inline-body-ref.ts new file mode 100644 index 0000000000..24da2fdd2b --- /dev/null +++ b/vendor/intx/workflow/src/inline-body-ref.ts @@ -0,0 +1,21 @@ +/** + * Derive an inline body's ref from its owning workflow id and the step id that + * carries it. This is the SINGLE owner of the `__` scheme, + * shared by every inline-body kind -- onTrigger sections, childWorkflow + * children, and loop bodies. Refs never collide because each `__`-joined + * segment is atomic (a step id may not itself contain `__`, enforced in + * `normalize`) and a given step id is exactly one primitive kind. Atomic + * segments matter once a ref spans more than two segments: a loop body's + * `childWorkflow` grandchild is registered under + * `____`, so without the `__`-free rule + * that ref could alias a top-level step literally named + * `__`. The live rewrite/enumeration + * and the source-ref hub's inert-body enumerator both mint refs through it, + * and the source-ref run child re-derives the same ref when it rewrites the + * re-evaluated closure (`run-child`). A hub that records a body's sources under + * this ref and a run child that looks them up must agree byte-for-byte, so the + * scheme lives in one place rather than being re-spelled at each site. + */ +export function inlineBodyRef(workflowId: string, stepId: string): string { + return `${workflowId}__${stepId}`; +} diff --git a/vendor/intx/workflow/src/live-inert-projector.ts b/vendor/intx/workflow/src/live-inert-projector.ts index 5c3b88d1c3..50e3d0617f 100644 --- a/vendor/intx/workflow/src/live-inert-projector.ts +++ b/vendor/intx/workflow/src/live-inert-projector.ts @@ -38,7 +38,7 @@ import type { } from "@intx/agent"; import type { ToolPackagePin } from "@intx/types/tool-packages"; import type { CredentialBinding, SidecarCapabilityPolicy } from "@intx/types"; -import type { InboundMailPolicy } from "@intx/types/runtime"; +import type { InboundMailPolicy } from "@intx/types/inbound-mail-policy"; import type { ActionPrimitive, AwaitSignalPrimitive, @@ -117,6 +117,7 @@ export interface InertStepStep { readonly agent: InertAgent; readonly after?: readonly string[]; readonly input?: Selector; + readonly inference?: Selector; readonly reads?: readonly Selector[]; readonly writes?: readonly Selector[]; readonly retry?: RetryPolicy; @@ -144,6 +145,7 @@ export interface InertLoop { readonly input?: Selector; readonly maxIterations: number; readonly onExhausted: string; + readonly onIterationFailure?: BodyFailurePolicy; readonly drainBehavior?: DrainBehavior; readonly after?: readonly string[]; } @@ -205,15 +207,15 @@ export interface InertWorkflowDefinition { readonly stepOrder: readonly string[]; readonly steps: Record; readonly state?: { readonly schema?: unknown }; - // The definition's credential bindings -- each maps a tool package's - // declared handle to a concrete provider/locator. These are pure plain - // data (no secret material; the secret is resolved fresh at deploy), so - // they project verbatim. They are part of the hashed surface on purpose: - // a binding names WHICH provider-backed credential the code may request, - // so it belongs to what the operator approves. On the code-sourced path - // the inert projection is the ONLY carrier of the bindings the deploy - // resolves -- a moved or tampered registry that served code with - // different bindings would change this projection and fail re-verify. + // The definition's credential bindings -- each maps a tool package's or an + // action handler's declared handle to a concrete provider/locator. These are + // pure plain data (no secret material; the secret is resolved fresh at + // deploy), so they project verbatim. They are part of the hashed surface on + // purpose: a binding names WHICH provider-backed credential the code may + // request, so it belongs to what the operator approves. On the code-sourced + // path the inert projection is the ONLY carrier of the bindings the deploy + // resolves -- a moved or tampered registry that served code with different + // bindings would change this projection and fail re-verify. readonly credentialBindings?: readonly CredentialBinding[]; readonly sidecarPlacement?: SidecarCapabilityPolicy; // The author-declared inbound-mail admission policy, projected verbatim (it @@ -353,6 +355,7 @@ function projectStepPrimitive(step: StepPrimitive): InertStepStep { agent: projectAgent(step.agent), ...(step.after !== undefined ? { after: [...step.after] } : {}), ...(step.input !== undefined ? { input: step.input } : {}), + ...(step.inference !== undefined ? { inference: step.inference } : {}), ...(step.reads !== undefined ? { reads: [...step.reads] } : {}), ...(step.writes !== undefined ? { writes: [...step.writes] } : {}), ...(step.retry !== undefined ? { retry: step.retry } : {}), @@ -386,6 +389,9 @@ function projectLoop(primitive: LoopPrimitive): InertLoop { ...(primitive.input !== undefined ? { input: primitive.input } : {}), maxIterations: primitive.maxIterations, onExhausted: primitive.onExhausted, + ...(primitive.onIterationFailure !== undefined + ? { onIterationFailure: primitive.onIterationFailure } + : {}), ...(primitive.drainBehavior !== undefined ? { drainBehavior: primitive.drainBehavior } : {}), @@ -430,6 +436,9 @@ function projectAction(primitive: ActionPrimitive): ActionPrimitive { ...(primitive.onFailure !== undefined ? { onFailure: primitive.onFailure } : {}), + ...(primitive.credentials !== undefined + ? { credentials: [...primitive.credentials] } + : {}), }; } diff --git a/vendor/intx/workflow/src/ontrigger-bodies.ts b/vendor/intx/workflow/src/ontrigger-bodies.ts index 10087b0f6e..ddb90f3526 100644 --- a/vendor/intx/workflow/src/ontrigger-bodies.ts +++ b/vendor/intx/workflow/src/ontrigger-bodies.ts @@ -16,6 +16,7 @@ import type { Primitive, WorkflowDefinition } from "./definition/index"; import type { LoopFnRegistry } from "./runtime/env"; +import { inlineBodyRef } from "./inline-body-ref"; export interface ExtractedOnTriggerBody { /** The body's ref -- `__` -- and the id of `definition`. */ @@ -31,28 +32,6 @@ export interface OnTriggerBodyRewrite { readonly bodies: readonly ExtractedOnTriggerBody[]; } -/** - * Derive an inline body's ref from its owning workflow id and the step id that - * carries it. This is the SINGLE owner of the `__` scheme, - * shared by every inline-body kind -- onTrigger sections, childWorkflow - * children, and loop bodies. Refs never collide because each `__`-joined - * segment is atomic (a step id may not itself contain `__`, enforced in - * `normalize`) and a given step id is exactly one primitive kind. Atomic - * segments matter once a ref spans more than two segments: a loop body's - * `childWorkflow` grandchild is registered under - * `____`, so without the `__`-free rule - * that ref could alias a top-level step literally named - * `__`. The live rewrite/enumeration - * here and the source-ref hub's inert-body enumerator both mint refs through it, - * and the source-ref run child re-derives the same ref when it rewrites the - * re-evaluated closure (`run-child`). A hub that records a body's sources under - * this ref and a run child that looks them up must agree byte-for-byte, so the - * scheme lives in one place rather than being re-spelled at each site. - */ -export function inlineBodyRef(workflowId: string, stepId: string): string { - return `${workflowId}__${stepId}`; -} - /** * Replace each inline onTrigger body with a `{ ref }` and return the extracted * body definitions (each body's id is its ref). Pure and side-effect-free: no diff --git a/vendor/intx/workflow/src/projection.ts b/vendor/intx/workflow/src/projection.ts new file mode 100644 index 0000000000..0eb88d53b8 --- /dev/null +++ b/vendor/intx/workflow/src/projection.ts @@ -0,0 +1,8 @@ +/** Inspect workflow declarations without loading the execution runtime. */ +export { collectDeclaredPluginNames } from "./declared-plugins"; +export { inlineBodyRef } from "./inline-body-ref"; +export { + projectLiveToInert, + computeLiveDefinitionHash, + type InertWorkflowDefinition, +} from "./live-inert-projector"; diff --git a/vendor/intx/workflow/src/runlocal/index.ts b/vendor/intx/workflow/src/runlocal/index.ts index 59b05cd257..8c3669d4f1 100644 --- a/vendor/intx/workflow/src/runlocal/index.ts +++ b/vendor/intx/workflow/src/runlocal/index.ts @@ -2,6 +2,8 @@ export { runLocal, createDefaultActionInvoker, createInMemoryEffectLedger, + unavailableActionCredentials, + unavailableCredentialCapability, createSpawnLoopIteration, type RunLocalOptions, } from "./run-local"; diff --git a/vendor/intx/workflow/src/runlocal/run-local.ts b/vendor/intx/workflow/src/runlocal/run-local.ts index 10f1ce505c..3d565607ba 100644 --- a/vendor/intx/workflow/src/runlocal/run-local.ts +++ b/vendor/intx/workflow/src/runlocal/run-local.ts @@ -12,6 +12,7 @@ import { createDefaultDirectorRegistry, type DirectorRegistry, } from "@intx/agent"; +import type { HostCredentialCapability } from "@intx/types"; import type { AuthorizeContext, @@ -24,13 +25,17 @@ import { } from "../ontrigger-bodies"; import { runtimeRun, type RuntimeRunOptions } from "../runtime/run"; import { createNoopDrainController } from "../runtime/drain"; +import { guardCredentialCapability } from "../runtime/credential-guard"; import { createEffectContext } from "../runtime/effect-context"; import { createLoopIterationHandle } from "../runtime/loop-iteration-handle"; +import { createSuspendableChildHandle } from "../runtime/suspendable-child-handle"; import type { + ActionCredentialsFactory, ActionInvoker, EffectLedger, LoopFnRegistry, StepInvoker, + RepoStore, SpawnChildWorkflow, SpawnSuspendableChild, WorkflowRun, @@ -58,6 +63,17 @@ export interface RunLocalOptions extends RuntimeRunOptions { * ledger across a re-run construct the env directly instead. */ invokeAction?: ActionInvoker; + /** + * Build each action invocation's credentials capability. The default is + * `unavailableActionCredentials`: an action that declares credentials fails + * before its handler runs, and any other action's `resolve` fails closed. + * `createActionCredentialsFactory` from `@intx/harness` builds one over a + * material cell, a grants resolver and a provider registry, the same builder + * the deployed host uses. A `childWorkflow` child inherits it. It applies + * only to the default action-runner, so passing it with `invokeAction` is + * refused. + */ + actionCredentials?: ActionCredentialsFactory; /** Resolve an action `handler` ref to a handler function. */ actionResolver?: (ref: string) => ActionHandler; /** Resolve a loop's `while`/`carry` refs to pure functions. */ @@ -85,9 +101,9 @@ export interface RunLocalOptions extends RuntimeRunOptions { * rather than defaulted, because an absent value would have to mean * permissive and a call site that forgot it would silently inherit a park * nothing can answer. Pass true for a run the control plane can address: a - * top-level local run, whose caller holds the handle that delivers. The - * terminal-child spawner passes false, since nothing can address a child - * run. + * top-level local run, whose caller holds the handle that delivers. A + * spawned child forces true for itself: its parent relays each park down, + * and inheriting a parent false would refuse a gate the parent can answer. */ hasUpstreamSignalResolver: boolean; } @@ -102,14 +118,45 @@ export function runLocal( definition: WorkflowDefinition, options: RunLocalOptions, ): WorkflowRun { + const { rewritten, env } = assembleRunLocalEnv(definition, options); + return runtimeRun(rewritten, env, extractRuntimeOptions(options)); +} + +/** + * The per-run env `runLocal` and the child spawner share. The top-level call + * builds a fresh repo, channel, blob store, effect ledger, and drain. A + * spawned child writes its event log into the store it is given, under its + * own run id, and still builds a fresh channel, blob store, effect ledger, + * and drain. The top-level run drives this env with `runtimeRun`. The + * spawner hands the same shape to `createSuspendableChildHandle` and does + * not also drive it. + */ +function assembleRunLocalEnv( + definition: WorkflowDefinition, + options: RunLocalOptions, + sharedRepoStore?: RepoStore, +): { rewritten: WorkflowDefinition; env: WorkflowRuntimeEnv } { const directors = options.directors ?? createDefaultDirectorRegistry(); const authorize = options.authorize; const invokeStep: StepInvoker = options.invokeStep ?? createDefaultStepInvoker(authorize); + if ( + options.invokeAction !== undefined && + options.actionCredentials !== undefined + ) { + throw new Error( + "runLocal: actionCredentials applies only to the default action-runner; a custom invokeAction must build its own credentials", + ); + } const effects = createInMemoryEffectLedger(); const invokeAction: ActionInvoker = options.invokeAction ?? - createDefaultActionInvoker(authorize, effects, options.actionResolver); + createDefaultActionInvoker( + authorize, + effects, + options.actionResolver, + options.actionCredentials ?? unavailableActionCredentials, + ); const clock = options.clock ?? defaultClock; const newId = options.newId ?? defaultNewId; @@ -139,7 +186,7 @@ export function runLocal( } } - const repoStore = createInMemoryRepoStore(); + const repoStore = sharedRepoStore ?? createInMemoryRepoStore(); const env: WorkflowRuntimeEnv = { repoStore, scheduler: createInMemoryScheduler({ repoStore, clock }), @@ -153,6 +200,7 @@ export function runLocal( spawnChild: createInMemorySpawnChild( childBodies, inheritChildOptions(options), + repoStore, ), clock, newId, @@ -168,7 +216,7 @@ export function runLocal( env.loopFns = options.loopFns; } - return runtimeRun(rewritten, env, extractRuntimeOptions(options)); + return { rewritten, env }; } function extractRuntimeOptions(options: RunLocalOptions): RuntimeRunOptions { @@ -201,15 +249,17 @@ function extractRuntimeOptions(options: RunLocalOptions): RuntimeRunOptions { * * Everything omitted here is either per-run identity the spawn callback * supplies itself (`runId`, `triggerPayload`, `depth`) or per-run - * substrate the child builds fresh (its own event log, blob store, signal - * channel, and effect ledger), matching how the deployed host scopes each - * child run's substrate under its own `childRunId`. + * substrate the child builds fresh (its blob store, signal channel, and + * effect ledger). The event log is the parent's store, scoped by + * `childRunId`, the same way the deployed host scopes each child under + * one workflow-run repo. */ type InheritedChildOptions = Pick< RunLocalOptions, | "authorize" | "invokeStep" | "invokeAction" + | "actionCredentials" | "actionResolver" | "loopFns" | "directors" @@ -226,6 +276,9 @@ function inheritChildOptions(options: RunLocalOptions): InheritedChildOptions { ...(options.invokeAction !== undefined ? { invokeAction: options.invokeAction } : {}), + ...(options.actionCredentials !== undefined + ? { actionCredentials: options.actionCredentials } + : {}), ...(options.actionResolver !== undefined ? { actionResolver: options.actionResolver } : {}), @@ -271,18 +324,31 @@ function createDefaultStepInvoker(authorize: WorkflowAuthorizeFn): StepInvoker { } /** - * Default action invoker. Resolves the handler ref, builds an - * EffectContext against the supplied ledger and authorize, and runs the - * handler. Failing loudly when no resolver is wired mirrors - * `createNoopSpawnChild`: a silent stub would let action workflows pass - * tests against effects that never ran. + * Default action invoker. Resolves the handler ref, builds the action's + * credentials capability and an EffectContext against the supplied ledger and + * authorize, and runs the handler. Failing loudly when no resolver is wired + * mirrors `createNoopSpawnChild`: a silent stub would let action workflows + * pass tests against effects that never ran. + * + * The capability is built before the handler runs, so a factory that throws + * fails the action before any effect. It is disposed when the handler settles. + * A dispose failure fails the invocation, so a handle that does not release + * its resources surfaces; when the handler also threw, both errors surface. */ export function createDefaultActionInvoker( authorize: WorkflowAuthorizeFn, effects: EffectLedger, resolver: ((ref: string) => ActionHandler) | undefined, + credentialsFactory: ActionCredentialsFactory, ): ActionInvoker { - return async ({ handler, input, requires, authzContext, signal }) => { + return async ({ + handler, + input, + requires, + credentials, + authzContext, + signal, + }) => { // Refuse before anything is constructed, so a cancelled run resolves no // handler and touches no ledger. An action is single-attempt with // observable side effects and there is no retry to reconsider the @@ -297,18 +363,92 @@ export function createDefaultActionInvoker( ); } const fn = resolver(handler); - const ctx = createEffectContext({ - authorize, - effects, - requires, + const capability = credentialsFactory({ + handler, + credentials, authzContext, - input, }); - const output = await fn(input, ctx, signal); + // The handler's `perform` scopes see only `resolve`; `dispose` stays with + // the invoker. The view, and every handle it returns, refuses once + // disposal starts, so a `perform` the handler left pending cannot shape a + // handle that nothing would dispose, or use one it already holds. + let disposed = false; + const dispose = (): Promise => { + disposed = true; + return capability.dispose(); + }; + let output: unknown; + try { + const ctx = createEffectContext({ + handler, + authorize, + effects, + requires, + authzContext, + input, + credentials: guardCredentialCapability( + capability, + () => !disposed, + (h, verb) => + `action ${handler} cannot ${verb} credential handle "${h}": its credentials capability is disposed`, + ), + }); + output = await fn(input, ctx, signal); + } catch (handlerError) { + try { + await dispose(); + } catch (disposeError) { + throw new AggregateError( + [handlerError, disposeError], + `action ${handler} failed and its credentials capability failed to dispose`, + ); + } + throw handlerError; + } + await dispose(); return { output }; }; } +/** + * A credentials capability for a host that delivers no credential to the + * named handler: `resolve` fails closed for every handle and `dispose` releases + * nothing. + */ +export function unavailableCredentialCapability( + handler: string, +): HostCredentialCapability { + return { + resolve(handle) { + return Promise.reject( + new Error( + `action ${handler} cannot resolve credential handle "${handle}": no credential source is configured for action handlers`, + ), + ); + }, + dispose() { + return Promise.resolve(); + }, + }; +} + +/** + * The credentials factory for a host with no credential source. An action + * that declares credentials fails before its handler runs; any other action + * gets a capability whose `resolve` fails closed. + */ +export const unavailableActionCredentials: ActionCredentialsFactory = ({ + handler, + credentials, +}) => { + if (credentials.length > 0) { + throw new Error( + `action ${handler} declares credential handle(s) ${credentials.join(", ")}, but no credential source is configured for action handlers`, + ); + } + return unavailableCredentialCapability(handler); +}; + export function createInMemoryEffectLedger(): EffectLedger { const store = new Map(); return { @@ -365,6 +505,7 @@ export function createSpawnLoopIteration( export function createInMemorySpawnChild( bodies: ReadonlyMap, inherited: InheritedChildOptions, + sharedRepoStore?: RepoStore, ): SpawnChildWorkflow { return async ({ definitionRef, @@ -373,15 +514,15 @@ export function createInMemorySpawnChild( signal, depth, maxChildSpawnDepth, + resumeFromEvents, }) => { // Four awaits separate the child run id allocation from this call, one of - // them a durable flush, so a cancel can land before the spawner runs. The - // abort bridge below subscribes to an edge and would miss one already - // past, leaving the child uncancelled and this function awaiting a - // terminal that never comes. Refusing outright also avoids writing a whole - // child log subtree for a run already known to be cancelled, and matches - // what the deployed spawn adapter does, so a local rehearsal does not - // diverge from production. + // them a durable flush, so a cancel can land before the spawner runs. + // The signal handed in is already aborted in that case: the parent drive + // bridges the step abort onto it before the call, including an abort that + // has already fired. Refusing outright avoids writing a child log for a + // run already known to be cancelled, and matches what the deployed spawn + // adapter does, so a local rehearsal does not diverge from production. if (signal.aborted) { throw abortReason(signal); } @@ -396,36 +537,31 @@ export function createInMemorySpawnChild( `childWorkflow ${definitionRef} has no lifted definition; the inline child should have been extracted before the run started`, ); } - // Recursively invoke runLocal for the resolved child against the - // parent-allocated childRunId so the parent's audit log and the - // child's own log agree on identity. Carry the depth (already checked - // one rung up) and the tree-wide ceiling so the child's own spawns keep - // counting against the same bound, and the inherited env overrides so - // the child runs under the same authorize and invokers as its parent. - const child = runLocal(resolved, { - ...inherited, - triggerPayload: input, - runId: childRunId, + // Build the child's own env and hand it to the shared handle. The parent + // drives the handle. Carry the depth (already checked one rung up) and + // the tree-wide ceiling so the child's own spawns keep counting against + // the same bound, and the inherited authorize and invokers so a strict + // parent env is the env the child runs under. The resolver flag is forced + // true: the parent relays, and a parent false must not refuse a gate the + // relay can answer. The in-memory channel has no stop. + const { rewritten, env } = assembleRunLocalEnv( + resolved, + { + ...inherited, + hasUpstreamSignalResolver: true, + }, + sharedRepoStore, + ); + return createSuspendableChildHandle(env, { + definition: rewritten, + childRunId, + input, depth, maxChildSpawnDepth, - // Terminal: the parent awaits this child's terminal rather than driving - // it across parks, and nothing upstream can address the child run, so a - // park inside it could never be answered. - hasUpstreamSignalResolver: false, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), + signal, + surfaceInputPark: true, }); - const onParentAbort = (): void => { - void child.cancel("supervisor-operator", "parent cancelled"); - }; - signal.addEventListener("abort", onParentAbort); - try { - const result = await child.complete; - return { terminalStatus: result.terminalStatus }; - } finally { - // Drop the listener when the child has settled so the parent's - // per-step abort signal does not retain a reference to a child - // that no longer needs cancellation. - signal.removeEventListener("abort", onParentAbort); - } }; } diff --git a/vendor/intx/workflow/src/runlocal/signal-channel.ts b/vendor/intx/workflow/src/runlocal/signal-channel.ts index 5962caa747..0773e23327 100644 --- a/vendor/intx/workflow/src/runlocal/signal-channel.ts +++ b/vendor/intx/workflow/src/runlocal/signal-channel.ts @@ -1,23 +1,31 @@ -// In-process FIFO single-consumer signal channel. +// In-process signal channel. // -// Mirrors the state machine's signal semantics: a `SignalReceived` -// before any `SignalAwaited` is queued under the signal name and -// consumed by the next awaiter for that name. Per-signal dedup by -// `signalId` is enforced by the state machine; this layer only -// tracks the dispatch queue. +// A delivery pairs with the earliest awaiter of the same author name +// whose correlationId is equal (`undefined` with `undefined`). A +// correlated delivery does not wake a name-only awaiter, and a +// name-only delivery does not wake a correlated one. An unmatched +// delivery stays queued under the author name until such an awaiter +// arrives. Per-signal dedup by `signalId` is the state machine's +// job; this layer only tracks the dispatch queue. import type { SignalChannel } from "../runtime/env"; interface Awaiter { - resolve: (value: { payload: unknown; signalId: string }) => void; + resolve: (value: { + payload: unknown; + signalId: string; + correlationId?: string; + }) => void; reject: (cause: unknown) => void; signal?: AbortSignal; onAbort?: () => void; + correlationId?: string; } interface QueuedDelivery { payload: unknown; signalId: string; + correlationId?: string; } export interface InMemorySignalChannelOptions { @@ -32,17 +40,27 @@ export function createInMemorySignalChannel( const queued = new Map(); return { - async deliver(name, payload, signalId) { + async deliver(name, payload, signalId, correlationId) { const id = signalId ?? newId(); + const delivery: QueuedDelivery = { + payload, + signalId: id, + ...(correlationId !== undefined ? { correlationId } : {}), + }; const queueAwaiters = awaiters.get(name); - if (queueAwaiters && queueAwaiters.length > 0) { - const next = queueAwaiters.shift(); + const match = + queueAwaiters?.findIndex( + (awaiter) => awaiter.correlationId === correlationId, + ) ?? -1; + if (queueAwaiters !== undefined && match >= 0) { + const next = queueAwaiters[match]; + queueAwaiters.splice(match, 1); if (queueAwaiters.length === 0) awaiters.delete(name); - if (next) { + if (next !== undefined) { if (next.signal && next.onAbort) { next.signal.removeEventListener("abort", next.onAbort); } - next.resolve({ payload, signalId: id }); + next.resolve(delivery); return; } } @@ -51,49 +69,55 @@ export function createInMemorySignalChannel( q = []; queued.set(name, q); } - q.push({ payload, signalId: id }); + q.push(delivery); }, - async awaitNext(name, signal) { + async awaitNext(name, signal, correlationId) { const q = queued.get(name); - if (q && q.length > 0) { - const next = q.shift(); + const queuedMatch = + q?.findIndex((item) => item.correlationId === correlationId) ?? -1; + if (q !== undefined && queuedMatch >= 0) { + const next = q[queuedMatch]; + q.splice(queuedMatch, 1); if (q.length === 0) queued.delete(name); - if (next) { + if (next !== undefined) { return next; } } - return new Promise<{ payload: unknown; signalId: string }>( - (resolve, reject) => { - const awaiter: Awaiter = { - resolve, - reject, - ...(signal !== undefined ? { signal } : {}), - }; - if (signal !== undefined) { - if (signal.aborted) { - reject(new Error("aborted")); - return; - } - const onAbort = (): void => { - const list = awaiters.get(name); - if (list) { - const idx = list.indexOf(awaiter); - if (idx >= 0) list.splice(idx, 1); - if (list.length === 0) awaiters.delete(name); - } - reject(new Error("aborted")); - }; - awaiter.onAbort = onAbort; - signal.addEventListener("abort", onAbort, { once: true }); + return new Promise<{ + payload: unknown; + signalId: string; + correlationId?: string; + }>((resolve, reject) => { + const awaiter: Awaiter = { + resolve, + reject, + ...(signal !== undefined ? { signal } : {}), + ...(correlationId !== undefined ? { correlationId } : {}), + }; + if (signal !== undefined) { + if (signal.aborted) { + reject(new Error("aborted")); + return; } - let list = awaiters.get(name); - if (!list) { - list = []; - awaiters.set(name, list); - } - list.push(awaiter); - }, - ); + const onAbort = (): void => { + const list = awaiters.get(name); + if (list) { + const idx = list.indexOf(awaiter); + if (idx >= 0) list.splice(idx, 1); + if (list.length === 0) awaiters.delete(name); + } + reject(new Error("aborted")); + }; + awaiter.onAbort = onAbort; + signal.addEventListener("abort", onAbort, { once: true }); + } + let list = awaiters.get(name); + if (!list) { + list = []; + awaiters.set(name, list); + } + list.push(awaiter); + }); }, }; } diff --git a/vendor/intx/workflow/src/runtime/abort-bridge.ts b/vendor/intx/workflow/src/runtime/abort-bridge.ts index b1e588a0a3..3af39b371e 100644 --- a/vendor/intx/workflow/src/runtime/abort-bridge.ts +++ b/vendor/intx/workflow/src/runtime/abort-bridge.ts @@ -21,14 +21,9 @@ // with it. // // Consolidated here so the next bridge is a call rather than a fresh chance to -// get it wrong. Four sites stay hand-written and are not this shape: +// get it wrong. Three sites stay hand-written and are not this shape: // `createStepAbort` also consults drain and returns early on it, and -// `waitForTimer`, `createInMemorySpawnChild` and `createDefaultActionInvoker` -// throw rather than react. The spawner's listener is the one outer-abort -// bridge here still built from the edge alone; it is safe because nothing -// awaits between its entry throw and the registration, so the window a level -// check closes elsewhere never opens there. Its exclusion above names that -// entry throw, not the listener. +// `waitForTimer` and `createDefaultActionInvoker` throw rather than react. /** * Run `onAbort` when `outer` aborts, including when it already has. diff --git a/vendor/intx/workflow/src/runtime/abort-window.test-helpers.ts b/vendor/intx/workflow/src/runtime/abort-window.test-helpers.ts new file mode 100644 index 0000000000..be2ad28c8d --- /dev/null +++ b/vendor/intx/workflow/src/runtime/abort-window.test-helpers.ts @@ -0,0 +1,111 @@ +// Helpers for exercising the windows between a durable commit and the abort +// bridge that follows it. +// +// Each of those windows opens on an `await`, so a test has to land its abort +// precisely inside one. Driving that from a timer would be a race; these +// helpers instead trigger the abort from the durable write itself, which makes +// the placement deterministic. + +import { createDefaultDirectorRegistry } from "@intx/agent"; + +import type { WorkflowDefinition } from "../definition/index"; +import { createInMemoryBlobSubstrate } from "../runlocal/blob-substrate"; +import { createInMemoryRepoStore } from "../runlocal/repo-store"; +import { createInMemoryScheduler } from "../runlocal/scheduler"; +import { createInMemorySignalChannel } from "../runlocal/signal-channel"; +import { createNoopDrainController } from "./drain"; +import type { RepoStore, WorkflowRuntimeEnv } from "./env"; +import type { WorkflowEvent } from "../state-machine/index"; +import { createTerminalChildHandle } from "./suspendable-child-handle"; + +/** + * A repo store that aborts `teardown` as soon as an event of `kind` has been + * durably written, placing the abort inside the window between that write and + * whatever bridge the runtime builds next. + * + * Both write methods are wrapped. `RepoStore` exposes `append` and + * `appendBatch` as independent operations, so hooking only the batch form + * would leave a test targeting a single-event write silently unarmed -- it + * would pass without ever placing its abort, which is the one failure mode a + * deterministic helper must not have. + */ +export function abortOnDurableEvent( + teardown: AbortController, + kind: WorkflowEvent["kind"], +): RepoStore { + const inner = createInMemoryRepoStore(); + const fireIfCarried = (events: readonly WorkflowEvent[]): void => { + if (events.some((e) => e.kind === kind)) teardown.abort(); + }; + return { + ...inner, + append: async (runId, event) => { + const result = await inner.append(runId, event); + fireIfCarried([event]); + return result; + }, + appendBatch: async (runId, events) => { + const result = await inner.appendBatch(runId, events); + fireIfCarried(events); + return result; + }, + }; +} + +/** + * Resolve `"settled"` when `promise` finishes either way, or `"pending"` once + * `ms` elapses. A wedged run is indistinguishable from a slow one, so a bound + * is the only way to assert the difference. + */ +export async function settlesWithin( + promise: Promise, + ms: number, +): Promise<"settled" | "pending"> { + const pending = Symbol("pending"); + let timer: ReturnType | undefined; + try { + const outcome = await Promise.race([ + promise.then( + () => "settled" as const, + () => "settled" as const, + ), + new Promise((resolve) => { + timer = setTimeout(() => { + resolve(pending); + }, ms); + }), + ]); + return outcome === pending ? "pending" : "settled"; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +/** + * An env wired to in-memory implementations, with the supplied repo store so a + * test can place an abort against a durable write. + */ +export function buildAbortWindowEnv( + def: WorkflowDefinition, + opts: { repoStore: RepoStore }, +): WorkflowRuntimeEnv { + const clock = (): Date => new Date(); + return { + repoStore: opts.repoStore, + scheduler: createInMemoryScheduler({ repoStore: opts.repoStore, clock }), + signalChannel: createInMemorySignalChannel(), + blobs: createInMemoryBlobSubstrate(), + directors: createDefaultDirectorRegistry(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + invokeStep: async () => ({ output: null }), + spawnChild: async () => createTerminalChildHandle("completed"), + clock, + newId: (prefix) => `${prefix}-${Math.random().toString(36).slice(2, 8)}`, + drain: createNoopDrainController(def), + hasUpstreamSignalResolver: true, + }; +} diff --git a/vendor/intx/workflow/src/runtime/commit-chain.ts b/vendor/intx/workflow/src/runtime/commit-chain.ts index f2cb827474..6287e38e0e 100644 --- a/vendor/intx/workflow/src/runtime/commit-chain.ts +++ b/vendor/intx/workflow/src/runtime/commit-chain.ts @@ -29,17 +29,19 @@ // Because the seq the chain assigns and every `reloadState` caller's // next-seq computation must account for buffered-but-unflushed // events, `reloadState` folds the durable log together with the -// pending buffer. The buffer is only ever non-empty for events the -// current runtime process committed synchronously since the last -// flush; the run body flushes before it parks (so an external writer -// -- a separate-process scheduler committing `TimerFired`, or a -// control-plane `cancel`) only ever advances the durable log while -// the buffer is empty. `commit` (immediate) flushes any pending -// buffer before its own event so a buffered run body and an external -// immediate writer never compute a colliding seq. +// pending buffer. The buffer holds events this process validated +// since the last flush. A sibling primitive can buffer another event +// after a park flush, so the buffer is not empty just because one +// step has parked. Signal delivery, the host scheduler, and a +// control-plane cancel take `withRunCommitBarrier`, which flushes +// that buffer before their own write. The host scheduler still tails +// the durable `TimerSet`. `commit` (immediate) +// flushes any pending buffer before its own event so a buffered run +// body and an immediate writer never compute a colliding seq. import { applyEvent, + isTerminalRunPhase, resumeFromLog, type RunState, type WorkflowEvent, @@ -71,6 +73,34 @@ function getBuffer(runId: string): WorkflowEvent[] { return buf; } +const bufferWaiters = new Map void)[]>(); + +function wake(waiters: Map void)[]>, runId: string): void { + const list = waiters.get(runId); + if (list === undefined) return; + waiters.delete(runId); + for (const resolve of list) resolve(); +} + +/** Resolves when the next event is pushed into this run's buffer. */ +export function whenRunNextBuffers(runId: string): Promise { + return new Promise((resolve) => { + const list = bufferWaiters.get(runId) ?? []; + list.push(resolve); + bufferWaiters.set(runId, list); + }); +} + +/** + * Events buffered for this run and not yet flushed. A copy: the caller + * must not be able to push into the buffer `flush` clears. + */ +export function bufferedEvents(runId: string): readonly WorkflowEvent[] { + const buf = pendingBuffers.get(runId); + if (buf === undefined) return []; + return buf.slice(); +} + /** * Reconstruct the run's current state from the durable log folded * with any pending (buffered-but-unflushed) events. Used inside the @@ -98,12 +128,109 @@ async function readStateWithPending( * the per-runId chain lock so the flushed seqs are contiguous on the * durable tip. No-op when the buffer is empty. */ +const repoWriteFailure = Symbol("repoWriteFailure"); + +/** + * The durable append threw, so this batch is not on disk. Settlement + * must rethrow it: a substitute terminal event would replace the + * prefix a re-fire adopts after a crash mid-flush. + */ +export function markRepoWriteFailure(cause: unknown): void { + if (typeof cause === "object" && cause !== null) { + Object.defineProperty(cause, repoWriteFailure, { value: true }); + } +} + +export function isRepoWriteFailure(cause: unknown): boolean { + return ( + typeof cause === "object" && cause !== null && repoWriteFailure in cause + ); +} + +// A flush that threw dropped its batch: the events are neither durable +// nor still buffered. The writer that flushed them rejects on its own +// promise, and a barrier's finally drops the chain, so a later link +// would otherwise see an empty buffer and succeed. Later links rethrow +// the first cause. `dropChain` forgets it when the run settles. +const failedFlushes = new Map(); +const flushWaiters = new Map void)[]>(); + +/** + * The cause is the failed flush recorded for this run. The marker on + * the error is not enough: a nested run stamps the same marker, and + * that error rejects into the parent with the marker still set. + */ +export function isRunFlushFailure(runId: string, cause: unknown): boolean { + const recorded = failedFlushes.get(runId); + return recorded !== undefined && recorded === cause; +} + +/** The failed flush recorded for this run, if this process has seen one. */ +export function flushFailureCause(runId: string): unknown { + return failedFlushes.get(runId); +} + +/** + * Resolves when this run's buffer flush throws. Already resolved when + * the failure was recorded before the caller started waiting. A parked + * step never rejects, so the drive loop races this with its step + * promises. + */ +export function whenRunFlushFails(runId: string): Promise { + if (failedFlushes.has(runId)) return Promise.resolve(); + return new Promise((resolve) => { + const list = flushWaiters.get(runId) ?? []; + list.push(resolve); + flushWaiters.set(runId, list); + }); +} + +function throwIfFlushFailed(runId: string): void { + const cause = failedFlushes.get(runId); + if (cause !== undefined) throw cause; +} + +/** + * A `SignalReceived` whose id is already on this run. Host `deliver` + * appends that blob, and the runtime commits the same id again; the + * in-memory channel does not append, so the first commit still records + * it. A finished run is not this case: `applyEvent` must still reject + * the event before anything is buffered. + */ +function signalAlreadyRecorded(state: RunState, event: WorkflowEvent): boolean { + return ( + event.kind === "SignalReceived" && + !isTerminalRunPhase(state.phase) && + state.observedSignalIds.has(event.signalId) + ); +} + +function followChain(runId: string, body: () => Promise): Promise { + const prev = commitChains.get(runId) ?? Promise.resolve(); + const next = (async () => { + await prev.catch(() => undefined); + throwIfFlushFailed(runId); + return body(); + })(); + commitChains.set(runId, next); + return next; +} + async function flushBuffer(env: CommitEnv, runId: string): Promise { const buf = pendingBuffers.get(runId); if (buf === undefined || buf.length === 0) return; const events = buf.slice(); buf.length = 0; - await env.repoStore.appendBatch(runId, events); + try { + await env.repoStore.appendBatch(runId, events); + } catch (cause) { + markRepoWriteFailure(cause); + if (!failedFlushes.has(runId)) { + failedFlushes.set(runId, cause); + wake(flushWaiters, runId); + } + throw cause; + } } /** @@ -120,25 +247,29 @@ async function flushBuffer(env: CommitEnv, runId: string): Promise { * (segment boundary). Use this for the run body's intra-segment * events; use `commit` for events that must persist immediately * (the segment-boundary suspension/terminal events flushed via the - * run body's explicit `flushChain`, and external writers such as the - * scheduler's `TimerFired` and the control-plane `cancel`). + * run body's explicit `flushChain`, the run-local scheduler's + * `TimerFired`, and the control-plane `cancel`). */ export async function commitBuffered( env: CommitEnv, runId: string, event: WorkflowEvent, + guard?: (state: RunState) => void, ): Promise { - const prev = commitChains.get(runId) ?? Promise.resolve(); - const next = (async (): Promise => { - await prev.catch(() => undefined); + return followChain(runId, async () => { const fresh = await readStateWithPending(env, runId); + if (signalAlreadyRecorded(fresh, event)) return fresh; + // A throw here leaves the buffer unchanged. A check that must exclude + // a concurrent writer of this same event belongs in the guard: the + // chain is the lock those writers share, and the fresh read above is + // the state both of them see. + guard?.(fresh); const adjustedEvent: WorkflowEvent = { ...event, seq: fresh.lastSeq + 1 }; const nextState = applyEvent(fresh, adjustedEvent); getBuffer(runId).push(adjustedEvent); + wake(bufferWaiters, runId); return nextState; - })(); - commitChains.set(runId, next); - return next; + }); } /** @@ -158,10 +289,14 @@ export async function commit( runId: string, event: WorkflowEvent, ): Promise { - const prev = commitChains.get(runId) ?? Promise.resolve(); - const next = (async (): Promise => { - await prev.catch(() => undefined); + return followChain(runId, async () => { const fresh = await readStateWithPending(env, runId); + if (signalAlreadyRecorded(fresh, event)) { + // This event is not appended. Events already buffered still have + // to land: immediate commit drains the buffer either way. + await flushBuffer(env, runId); + return fresh; + } const adjustedEvent: WorkflowEvent = { ...event, seq: fresh.lastSeq + 1 }; // Validate the transition before appending so a state-machine // rejection leaves the log clean. A subsequent commit on the same @@ -171,9 +306,7 @@ export async function commit( getBuffer(runId).push(adjustedEvent); await flushBuffer(env, runId); return nextState; - })(); - commitChains.set(runId, next); - return next; + }); } /** @@ -185,13 +318,43 @@ export async function commit( * No-op when the buffer is empty. */ export async function flushChain(env: CommitEnv, runId: string): Promise { - const prev = commitChains.get(runId) ?? Promise.resolve(); - const next = (async (): Promise => { - await prev.catch(() => undefined); + await followChain(runId, async () => { await flushBuffer(env, runId); - })(); - commitChains.set(runId, next); - await next; + }); +} + +/** + * Queue `body` behind this run's commit chain, then discard the pending + * buffer before `body` runs. Links already on the chain finish first, + * including a barrier `write()` that has started. `body` appends through + * the repo store. `commit`, `commitBuffered`, `flushChain`, and + * `withRunCommitBarrier` queue another link that waits on this one. + */ +export function runAfterDiscardingBuffer( + runId: string, + body: () => Promise, +): Promise { + return followChain(runId, async () => { + pendingBuffers.delete(runId); + return body(); + }); +} + +/** Flush and exclude runtime commits while an external writer advances the log. */ +export function withRunCommitBarrier( + env: CommitEnv, + runId: string, + write: () => Promise, +): Promise { + const next = followChain(runId, async () => { + await flushBuffer(env, runId); + return write(); + }); + return next.finally(() => { + if (commitChains.get(runId) !== next) return; + commitChains.delete(runId); + if (pendingBuffers.get(runId)?.length === 0) pendingBuffers.delete(runId); + }); } export async function reloadState( @@ -214,4 +377,7 @@ export async function reloadState( export function dropChain(runId: string): void { commitChains.delete(runId); pendingBuffers.delete(runId); + failedFlushes.delete(runId); + flushWaiters.delete(runId); + bufferWaiters.delete(runId); } diff --git a/vendor/intx/workflow/src/runtime/credential-guard.ts b/vendor/intx/workflow/src/runtime/credential-guard.ts new file mode 100644 index 0000000000..10e5e5c752 --- /dev/null +++ b/vendor/intx/workflow/src/runtime/credential-guard.ts @@ -0,0 +1,74 @@ +// A credentials capability that works only while a window is open. +// +// An action handler gets its credentials inside `perform`'s `run`, so every +// authenticated call is gated by the effect grant and recorded in the ledger. +// The window is guarded in two layers with this one wrapper: the invoker's +// view closes when the invoker disposes the invocation's capability, and each +// `perform` scope closes when its `run` settles. Every handle the guarded +// capability returns refuses use once its window closes, so a handle kept past +// its `run`, or used by a `perform` the handler left pending, fails closed. +// +// This is a guard against author mistakes, not an isolation boundary: a +// handler can still call `globalThis.fetch` itself. + +import type { CredentialCapability, MediatedCredential } from "@intx/types"; + +/** The message a guarded capability refuses with once its window closes. */ +export type CredentialGuardRefusal = ( + handle: string, + verb: "resolve" | "use", +) => string; + +/** + * Wrap `capability` so `resolve`, and every handle it returns, refuses once + * `isOpen` returns false. + */ +export function guardCredentialCapability( + capability: CredentialCapability, + isOpen: () => boolean, + refusal: CredentialGuardRefusal, +): CredentialCapability { + return { + async resolve(handle) { + if (!isOpen()) throw new Error(refusal(handle, "resolve")); + const mediated = await capability.resolve(handle); + if (!isOpen()) throw new Error(refusal(handle, "resolve")); + return guardMediatedCredential(mediated, isOpen, () => + refusal(handle, "use"), + ); + }, + }; +} + +/** + * Wrap one shaped handle so its use refuses once `isOpen` returns false. + * + * `dispose` releases nothing. The capability shares one shaped handle across + * every scope that resolves the same handle, so a forwarded `dispose` would + * break a scope that still uses it. The invoker owns disposal. + */ +function guardMediatedCredential( + mediated: MediatedCredential, + isOpen: () => boolean, + refusal: () => string, +): MediatedCredential { + switch (mediated.kind) { + case "http": + return { + kind: "http", + fetch: (input, init) => + isOpen() + ? mediated.fetch(input, init) + : Promise.reject(new Error(refusal())), + dispose() { + /* the invoker owns disposal */ + }, + }; + default: { + const _exhaustive: never = mediated.kind; + throw new Error( + `cannot guard a mediated credential of kind ${String(_exhaustive)}`, + ); + } + } +} diff --git a/vendor/intx/workflow/src/runtime/dag.ts b/vendor/intx/workflow/src/runtime/dag.ts index df5a366381..5d03046831 100644 --- a/vendor/intx/workflow/src/runtime/dag.ts +++ b/vendor/intx/workflow/src/runtime/dag.ts @@ -14,6 +14,55 @@ import { } from "../state-machine/index"; import { baseStepId } from "./step-scope"; +/** + * The only fact the resume predicates read off a definition: each step's + * `kind`. `WorkflowDefinition` satisfies it. A frozen projection does too, + * once its steps have been narrowed to `{ kind: string }`, so the same + * question covers a live definition and an approved projection. + */ +export type StepKindLookup = { + readonly steps: Readonly>; +}; + +/** + * How `executeRunBody` treats one reduced step. `continue` is a residual + * the runtime re-offers. `crashed-invocation` is an in-flight agent step + * or action, which the runtime settles as `StepFailed` rather than + * re-invoking. `unsupported` is what throws `RuntimeResumeUnsupportedError`. + * A terminal phase is `continue` because it is not a residual. This is the + * resume guard's if-chain; `executeRunBody` is the other caller and must + * not grow a second copy. + */ +export type ResumeResidual = "continue" | "crashed-invocation" | "unsupported"; + +export function resumeResidualOf( + def: StepKindLookup, + stepId: string, + phase: StepPhase, +): ResumeResidual { + if ( + isResumableLoopStep(def, stepId, phase) || + isResumableAwaitingSignalStep(def, stepId, phase) || + isResumableReceivedAwaitSignalStep(def, stepId, phase) || + isResumableOnTriggerStep(def, stepId, phase) || + isResumableSleepStep(def, stepId, phase) || + isResumableChildWorkflowStep(def, stepId, phase) + ) { + return "continue"; + } + if (isCrashedInvocationStep(def, stepId, phase)) { + return "crashed-invocation"; + } + if ( + phase === "in-flight" || + phase === "awaiting-signal" || + phase === "awaiting-timer" + ) { + return "unsupported"; + } + return "continue"; +} + /** * A loop container `` (or its synthetic iteration step * `[i]`) left non-terminal in a seed log is resumable in both of @@ -30,7 +79,7 @@ import { baseStepId } from "./step-scope"; * resolve the kind. */ export function isResumableLoopStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -64,7 +113,7 @@ export function isResumableLoopStep( * in exactly one place, mirroring `isResumableLoopStep`. */ export function isResumableAwaitingSignalStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -95,7 +144,7 @@ export function isResumableAwaitingSignalStep( * signal payload it never received. */ export function isResumableReceivedAwaitSignalStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -115,7 +164,7 @@ export function isResumableReceivedAwaitSignalStep( * its container to resolve the kind, mirroring the loop carve-out. */ export function isResumableOnTriggerStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -138,11 +187,11 @@ export function isResumableOnTriggerStep( * routed through the EffectContext; the barrier is what makes the action * non-re-invocable at this layer. * - * Container/coordination primitives left `in-flight` -- a `map` outer - * step, a `childWorkflow`, etc. -- are deliberately excluded: they have a - * re-arm surface the in-process runtime body lacks (rebuilding the map - * iteration state), so they stay `RuntimeResumeUnsupportedError` and the host - * owns recovery. (An `awaitSignal` gate left `in-flight`, timed or not, is + * A `map` outer step left `in-flight` is deliberately excluded: rebuilding + * its iteration state is a re-arm surface this runtime body lacks, so it + * throws `RuntimeResumeUnsupportedError`, which the run records as + * `RunFailed`. A `childWorkflow` is admitted by + * `isResumableChildWorkflowStep`. (An `awaitSignal` gate left `in-flight`, timed or not, is * instead admitted by `isResumableReceivedAwaitSignalStep`, which reconstructs * its outcome from the log.) A synthetic map/loop inner id (`[i]`) is not a * definition key, so it resolves to `undefined` and is excluded here; @@ -151,7 +200,7 @@ export function isResumableOnTriggerStep( * unsupported with its container. */ export function isCrashedInvocationStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -173,13 +222,13 @@ export function isCrashedInvocationStep( * * The kind guard is exact for a reason: a retrying `step`/`action` also parks * in `awaiting-timer` during backoff (`handleAttemptScheduled`), and that - * residual has no sleep-resume path -- it must stay `RuntimeResumeUnsupportedError` - * and let the host own recovery. Only a `sleep` primitive is admitted here. The + * residual has no sleep-resume path -- it throws `RuntimeResumeUnsupportedError`, + * which the run records as `RunFailed`. Only a `sleep` primitive is admitted here. The * resume guard and `nextSchedulable` both key on this single predicate so the * carve-out lives in exactly one place, mirroring the other predicates. */ export function isResumableSleepStep( - def: WorkflowDefinition, + def: StepKindLookup, stepId: string, phase: StepPhase, ): boolean { @@ -187,6 +236,24 @@ export function isResumableSleepStep( return def.steps[stepId]?.kind === "sleep"; } +/** + * A `childWorkflow` spawn step left non-terminal in a seed log is resumable + * in both of its live phases. `in-flight` is a spawn whose child has not + * yet parked the parent, or whose relay decision was consumed before the + * crash. `awaiting-signal` is the parent proxy of a child park. `runChildWorkflow` + * reattaches the recorded child run and reestablishes that relay. The step + * has no synthetic id, so the kind is read off the step itself. The resume + * guard and `nextSchedulable` both key on this predicate. + */ +export function isResumableChildWorkflowStep( + def: StepKindLookup, + stepId: string, + phase: StepPhase, +): boolean { + if (phase !== "in-flight" && phase !== "awaiting-signal") return false; + return def.steps[stepId]?.kind === "childWorkflow"; +} + export function nextSchedulable( def: WorkflowDefinition, state: RunState, @@ -209,9 +276,11 @@ export function nextSchedulable( // already-logged `SignalReceived` (`isResumableReceivedAwaitSignalStep`, // the crash-after-signal-before-StepCompleted window), an onTrigger // section container in-flight or awaiting-signal - // (`isResumableOnTriggerStep`, re-derived by `runOnTrigger`), and a `sleep` + // (`isResumableOnTriggerStep`, re-derived by `runOnTrigger`), a `sleep` // step awaiting-timer or in-flight (`isResumableSleepStep`, re-adopted by - // `runSleep`). The resume guard keys on the SAME predicates so the two + // `runSleep`), and a `childWorkflow` spawn step in-flight or + // awaiting-signal (`isResumableChildWorkflowStep`, reattached by + // `runChildWorkflow`). The resume guard keys on the SAME predicates so the two // views agree. if (state.phase !== "running") { return []; @@ -233,7 +302,8 @@ export function nextSchedulable( !isResumableAwaitingSignalStep(def, stepId, existing.phase) && !isResumableReceivedAwaitSignalStep(def, stepId, existing.phase) && !isResumableOnTriggerStep(def, stepId, existing.phase) && - !isResumableSleepStep(def, stepId, existing.phase) + !isResumableSleepStep(def, stepId, existing.phase) && + !isResumableChildWorkflowStep(def, stepId, existing.phase) ) { continue; } diff --git a/vendor/intx/workflow/src/runtime/drain.ts b/vendor/intx/workflow/src/runtime/drain.ts index 8eafd13c3b..279a732a8d 100644 --- a/vendor/intx/workflow/src/runtime/drain.ts +++ b/vendor/intx/workflow/src/runtime/drain.ts @@ -55,7 +55,10 @@ export interface DrainController { * * The default-by-kind table mirrors the constructors in * `definition/primitives.ts`: - * - `sleep`, `childWorkflow` default to `"cancel"`. + * - `sleep` defaults to `"cancel"`. + * - `childWorkflow` defaults to `"wait"`. Drain consults the spawn step + * for the whole time it is in flight, including a parked relay and a + * child that is still mid-step. * - `awaitSignal` defaults to `"wait"` (the human-in-the-loop case). * - `step` defaults to `"cancel"` when its trigger budget is `1` * (batch) and to `"wait"` when the budget is larger (multi-turn @@ -89,7 +92,7 @@ export function resolveDrainBehavior( case "sleep": return primitive.drainBehavior ?? "cancel"; case "childWorkflow": - return primitive.drainBehavior ?? "cancel"; + return primitive.drainBehavior ?? "wait"; case "onTrigger": // A live event-driven section is definitionally interactive; // draining means "stop feeding new events", not "abort the paused diff --git a/vendor/intx/workflow/src/runtime/effect-context.ts b/vendor/intx/workflow/src/runtime/effect-context.ts index 071e36ef4a..0ece319fa2 100644 --- a/vendor/intx/workflow/src/runtime/effect-context.ts +++ b/vendor/intx/workflow/src/runtime/effect-context.ts @@ -13,25 +13,39 @@ // the ledger would dedup distinct effects against each other. import { canonicalizeForHash } from "@intx/agent"; +import type { CredentialCapability } from "@intx/types"; import type { AuthorizeContext, WorkflowAuthorizeFn, } from "../authorize-context"; +import { guardCredentialCapability } from "./credential-guard"; import type { EffectContext, EffectLedger } from "./env"; export interface EffectContextConfig { + /** The action handler ref, named in a refusal. */ + handler: string; authorize: WorkflowAuthorizeFn; effects: EffectLedger; requires: readonly string[]; authzContext: AuthorizeContext; input: unknown; + /** The invocation's credentials, handed to each `run` through its scope. */ + credentials: CredentialCapability; } export function createEffectContext( config: EffectContextConfig, ): EffectContext { - const { authorize, effects, requires, authzContext, input } = config; + const { + handler, + authorize, + effects, + requires, + authzContext, + input, + credentials, + } = config; const allowed = new Set(requires); const { runId, stepId } = authzContext; if (runId === undefined || stepId === undefined) { @@ -43,7 +57,7 @@ export function createEffectContext( async perform({ effectId, capability, run }) { if (!allowed.has(capability)) { throw new Error( - `action effect uses capability ${capability} which is not in its declared requires set`, + `action ${handler} effect uses capability ${capability} which is not in its declared requires set`, ); } // Fail closed: only an explicit allow proceeds. deny, ask, and a @@ -68,7 +82,23 @@ export function createEffectContext( if (hit !== undefined) { return hit.output; } - const output = await run(); + // The scope closes when `run` settles, before the record, so a handle + // the handler keeps cannot be used after the effect is recorded. + let open = true; + const scope = { + credentials: guardCredentialCapability( + credentials, + () => open, + (handle, verb) => + `action ${handler} cannot ${verb} credential handle "${handle}" outside the run of its effect ${effectId}`, + ), + }; + let output: unknown; + try { + output = await run(scope); + } finally { + open = false; + } await effects.record(effectKey, output); return output; }, diff --git a/vendor/intx/workflow/src/runtime/env.ts b/vendor/intx/workflow/src/runtime/env.ts index ee42982c6c..54864cc2e6 100644 --- a/vendor/intx/workflow/src/runtime/env.ts +++ b/vendor/intx/workflow/src/runtime/env.ts @@ -8,7 +8,15 @@ // source-level test in `run.test.ts` asserts the discipline. import type { AgentDefinition, BaseEnv, DirectorRegistry } from "@intx/agent"; -import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime"; +import type { + CredentialCapability, + HostCredentialCapability, +} from "@intx/types"; +import type { + ApprovalSnapshot, + ControlParkKind, + PerCallInferenceOptions, +} from "@intx/types/runtime"; import type { AuthorizeContext, @@ -114,28 +122,37 @@ export interface Scheduler { * for its name on subscription. * * The shape is in-process callback-based (`deliver` / `awaitNext`). - * A production signal source that delivers via a mail bus has to - * translate "mail arrives" into "the right awaiter's promise - * resolves," and on resume the channel's in-memory queue is empty - * even though the state machine's `unconsumedSignals` carries the - * queued signals. The choice of resolution -- rehydrate the channel - * from the log, change `awaitNext` to consult the state-machine - * queue, or replace this interface with a log-tail subscription - * shape -- is a substrate-shaped decision that depends on the - * production mail substrate. + * The production channel tails the run's log from the moment + * `awaitNext` is called, so it never sees a signal committed before + * that; the runtime reads such a signal from the log after starting + * the wait. */ export interface SignalChannel { - /** Inject a signal. The state machine handles dedup by `signalId`. */ - deliver(name: string, payload: unknown, signalId?: string): Promise; /** - * Wait for the next signal of the given name. Resolves with the - * payload (and the assigned `signalId`) when the signal arrives. - * Rejects on `signal.abort()` if `signal` is supplied. + * Inject a signal. The state machine handles dedup by `signalId`. + * `correlationId`, when present, addresses one correlated park. A + * name-only delivery leaves that park waiting. + */ + deliver( + name: string, + payload: unknown, + signalId?: string, + correlationId?: string, + ): Promise; + /** + * Wait for the next signal of the given name. When `correlationId` + * is set, only a delivery of that id resolves; a name-only delivery + * stays queued for a name wait. When it is omitted, only a name-only + * delivery resolves. Resolves with the payload and the assigned + * `signalId`. Rejects on `signal.abort()` if `signal` is supplied. + * Must see every signal delivered after the call, since the runtime + * only checks the log for ones delivered before it. */ awaitNext( name: string, signal?: AbortSignal, - ): Promise<{ payload: unknown; signalId: string }>; + correlationId?: string, + ): Promise<{ payload: unknown; signalId: string; correlationId?: string }>; } /** @@ -159,6 +176,12 @@ export interface StepInvokeRequest { agent: AgentDefinition; /** The materialized input the runtime resolved from the step's `input` selector. */ input: unknown; + /** + * Per-call inference options resolved from the step's `inference` + * selector. Applies to this invocation's send alone. Absent means the + * agent's defaults. + */ + inference?: PerCallInferenceOptions; /** Workflow-runtime context for every authz call inside the step. */ authzContext: AuthorizeContext; /** Cancelled when the step is being torn down (timeout, cancellation). */ @@ -223,6 +246,8 @@ export interface ActionInvokeRequest { input: unknown; /** Capabilities the action declared it may exercise (its `effect.requires`). */ requires: readonly string[]; + /** Credential handles the action declared (its `credentials`). */ + credentials: readonly string[]; /** Workflow-runtime context for every authz call inside the handler. */ authzContext: AuthorizeContext; /** Cancelled when the action is being torn down (timeout, cancellation). */ @@ -261,15 +286,49 @@ export interface EffectLedger { * refuses any `capability` not in the action's declared `requires` set, * calls `env.authorize` before the effect, and on a ledger hit returns * the recorded result without running `run`. + * + * `perform` passes `run` a {@link PerformScope} that carries the handler's + * credentials, so every authenticated call that `run` awaits is gated by the + * effect grant and recorded in the ledger. A call that `run` starts and does + * not await can finish after the record, so its result is not recorded. On a + * ledger hit `run` does not run, so a replay resolves no credential. */ export interface EffectContext { perform(opts: { effectId: string; capability: string; - run: () => Promise; + run: (scope: PerformScope) => Promise; }): Promise; } +/** + * What `perform` gives one `run`. The scope opens when `run` starts and closes + * when `run` settles. After it closes, `credentials.resolve` rejects and every + * handle the scope returned refuses use. Each `perform` call has its own scope. + * + * `credentials` is the handler's consumer-scoped credentials capability. Its + * consumer is `action:`, so the scope of the bindings is per handler, + * not per step. It resolves a handle bound to that consumer to an + * origin-pinned mediated credential, and fails closed for any other handle. + */ +export interface PerformScope { + readonly credentials: CredentialCapability; +} + +/** + * Builds the credentials capability for one action invocation. The action + * invoker calls it before the handler runs and disposes the capability when + * the handler settles. Each `perform` scope over it closes earlier, when its + * `run` settles. A factory that throws fails the action before any + * effect. The factory owns the consumer identity, the declared-versus-bound + * check, and the authorization gate. + */ +export type ActionCredentialsFactory = (req: { + readonly handler: string; + readonly credentials: readonly string[]; + readonly authzContext: AuthorizeContext; +}) => HostCredentialCapability; + /** * Blob substrate. The default-threshold (1 MiB) spill is implemented * by `recordOutput`; consumers receive the same shape (`{ ref }`) @@ -304,7 +363,14 @@ export interface BlobSubstrate { * inline child -- to a concrete `WorkflowDefinition` using whatever in-memory * lookup the runtime supplies (a lifted-body map in `runLocal`, the parent's * re-evaluated closure map in production), constructs the child run against - * the supplied id, and returns the terminal status. + * the supplied id, and returns a live handle. `runChildWorkflow` drives that + * handle across the child's parks and relays each one onto the parent run. + * The child run stays unaddressable. A parked child survives a process + * restart: the caller passes `resumeFromEvents` from the child log and the + * parent reestablishes the relay. The spawn step defaults to `"wait"`, so + * the drain signal does not abort it, whether the child is parked or still + * mid-step. An explicit `"cancel"` still aborts the child through + * `localAbort`. * * The runtime body does not carry a definition lookup of its own. */ @@ -324,16 +390,24 @@ export type SpawnChildWorkflow = (input: { depth: number; /** The tree-wide spawn-depth ceiling, threaded to the child run. */ maxChildSpawnDepth: number; -}) => Promise<{ - terminalStatus: "completed" | "failed" | "cancelled"; -}>; + /** + * Re-adopt a child that was mid-flight when the process crashed. Drive + * `runtimeRun` from this durable log instead of `input`. Omit the field + * when the child log is empty: an empty array is a resume with no + * payload, and a child that died after `ChildSpawned` and before its own + * `RunStarted` must start from `input`. A re-park does not re-fire + * `onPark` or `onSignalPark`. + */ + resumeFromEvents?: readonly WorkflowEvent[]; +}) => Promise; /** * An approval park surfaced from a suspendable child body: the reserved * correlation the child parked on and the approver-facing snapshot, so the * parent (runOnTrigger) can proxy the approval up its OWN run's park - * machinery on the same correlation. A child body's `input` parks are the - * section's re-arm, not surfaced here. + * machinery on the same correlation. An input park is not an approval: a + * handle built with `surfaceInputPark` returns it as its own event, and + * every other handle rejects it. */ export type SuspendableChildPark = { correlationId: string; @@ -345,15 +419,23 @@ export type SuspendableChildPark = { * time the child parks on an approval (the caller proxies it up and, once * granted, calls `resume`), parks on an author `awaitSignal` gate (surfaced as * `signal-park` so the caller proxies it up as a signal-relay await and - * `deliverSignal`s the resolved signal back), or reaches a terminal. `resume` + * `deliverSignal`s the resolved signal back), surfaces an input park when the + * handle was built to, or reaches a terminal. `resume` * delivers the granted decision to the child's reserved correlation channel; * `deliverSignal` delivers a signal on the body's own author-chosen name. * Both unblock the body's awaiter so it continues. + * + * `next` returns whatever was queued, including a sibling park, and it does + * not notice that the gate has since moved on. A caller that relays one + * signal at a time reads `readParkRelease` before it accepts a signal that + * is already waiting, and uses `waitForParkRelease` to learn that the open + * park ended without consuming a sibling. */ export type SuspendableChildHandle = { next(): Promise< | { kind: "park"; park: SuspendableChildPark } - | { kind: "signal-park"; name: string } + | { kind: "signal-park"; name: string; timeout?: number } + | { kind: "input-park"; correlationId: string } | { kind: "terminal"; terminalStatus: "completed" | "failed" | "cancelled" } >; resume(correlationId: string, decision: unknown): Promise; @@ -368,17 +450,65 @@ export type SuspendableChildHandle = { payload: unknown, signalId: string, ): Promise; + /** + * One read of the child log. `open` means a step is still + * `awaiting-signal` on `name`, or that await has not been recorded yet. + * `released` means the await was recorded and no step is awaiting that + * name. A signal already waiting on the caller must not be relayed once + * the read is `released` or `terminal`: `next` can still return the park + * event after the gate has moved on. + * + * Present on the same handles as {@link waitForParkRelease}. + */ + readParkRelease?(name: string): Promise< + | { kind: "open" } + | { kind: "released" } + | { + kind: "terminal"; + terminalStatus: "completed" | "failed" | "cancelled"; + } + >; + /** + * Resolve when no step in the child run is `awaiting-signal` on `name`, + * or the child run has reached a terminal. A sibling park does not + * resolve it and is left queued for a later `next`. Abort rejects. + * + * Present on a handle whose caller can have several parks live at once. + * A loop iteration and an onTrigger body are driven one step at a time, + * so their callers keep racing `next` and do not need this. + */ + waitForParkRelease?( + name: string, + abort: AbortSignal, + ): Promise< + | { kind: "released" } + | { + kind: "terminal"; + terminalStatus: "completed" | "failed" | "cancelled"; + } + >; + /** + * Resolves once a re-park of `name` has entered `awaitNext`. A caller + * that already holds the decision delivers after this, so a channel + * that tails from head sees the blob. Rejects if the child settles + * before the gate re-arms. + * + * Present on the live suspendable handle. A terminal handle has no + * channel and omits it. + */ + whenSignalAwaiter?(name: string): Promise; }; /** - * Spawn a body sub-DAG as a child run that MAY suspend on an approval park. - * Unlike `spawnChild` (which awaits a terminal), this returns a live handle - * the caller drives across parks: the child's control-plane approval parks - * surface via `handle.next()` so the caller proxies them up its own run's - * park machinery, and `handle.resume` relays the granted decision back into - * the child. onTrigger runs every event's body through this seam, so a body - * step's approval is serviced without the body child ever needing to reach a - * terminal to unblock the parent. + * Spawn a body sub-DAG as a child run that may suspend. `spawnChild` is the + * `childWorkflow` seam and returns the same handle shape. This callback is + * the loop and `onTrigger` seam. The caller drives the handle: approval parks + * and author-signal parks surface via `handle.next()`, the caller proxies + * them up its own run, and `resume` relays a granted decision back into the + * child. The handle this seam builds rejects an input park. onTrigger runs + * every event's body through this seam, so a body step's approval is + * serviced without the body child ever needing to reach a terminal to + * unblock the parent. * * `resumeFromEvents` re-adopts a body child that was mid-flight when the * process crashed: the seam drives `runtimeRun` from the supplied durable log @@ -470,6 +600,12 @@ export type WorkflowPark = { export type WorkflowSignalPark = { runId: string; name: string; + /** + * Set when the gate carries a timeout. Absent means the gate is untimed + * and needs a delivery from upstream. A parent tells the two apart from + * this field: its own timer can still resolve a timed gate. + */ + timeout?: number; }; /** @@ -542,12 +678,11 @@ export interface WorkflowRuntimeEnv { * {@link SpawnSuspendableChild}). Optional: a host that does not wire it * does not support onTrigger sections, and `runOnTrigger` fails loudly. * - * This is a second child-drive seam alongside the terminal-only - * `spawnChild` that `childWorkflow` uses, rather than one unified - * park-aware seam. The two are kept separate because unifying them would - * require migrating `childWorkflow`'s terminal-only path onto the - * park-aware drive; that is a deliberate, separate decision, not an - * oversight. + * Both seams return a {@link SuspendableChildHandle}. `runChildWorkflow` + * drives `spawnChild` across the child's parks, including an input park. + * This callback is driven across approval and author-signal parks. An + * input park here is rejected: a loop or `onTrigger` body has no separate + * input channel of its own. */ spawnSuspendableChild?: SpawnSuspendableChild; /** @@ -590,10 +725,10 @@ export interface WorkflowRuntimeEnv { * * An untimed park waits on a signal that has to arrive from outside the * run. That only happens when the run is reachable from the control plane: - * either it is the deployment's addressable run, or it is a suspendable - * child whose container relays a decision back down. A terminal - * `childWorkflow` child is neither -- it has no address of its own and no - * relay above it -- so a park there can never be answered. + * either it is the deployment's addressable run, or it is a child whose + * parent relays a decision back down. A `childWorkflow` spawner forces + * this true, because the parent relays. A run that sets it false refuses + * an untimed park, whether or not a parent exists. * * The seam that spawns a run is the only layer that knows which case it is * in, so it states the fact here and the runtime enforces it at the park. @@ -637,6 +772,15 @@ export interface WorkflowRuntimeEnv { * off any suspendable-child path parks silently as before. */ onSignalPark?: (park: WorkflowSignalPark) => void; + /** + * Fired when a re-park enters `awaitNext`, after that call has registered + * its subscriber and before the runtime waits on it. A host channel tails + * from head and does not replay a blob that landed before this + * subscription. A re-park does not re-fire `onSignalPark`, so the caller + * that already holds the decision waits for this registration and + * delivers after it. Fresh parks do not fire it. + */ + onSignalAwaiter?: (park: { runId: string; name: string }) => void; /** * Optional read-only recovery hook: enumerate the durable pending approval * operations a step left behind, keyed by `{ runId, stepId, attempt }`. The @@ -665,6 +809,20 @@ export interface WorkflowRun { signal(name: string, payload: unknown, signalId?: string): Promise; } +/** + * The handle `runtimeRun` gives a workflow host. A host that writes the run's + * log itself can commit a `CancelRequested` outside the runtime and then have + * the runtime act on it. + */ +export interface RuntimeWorkflowRun extends WorkflowRun { + /** + * Stop the run for a `CancelRequested` its log already holds. The request + * is durable, so the run aborts even if the child cancellation cascade + * fails; the run's cancelling branch retries that cascade. + */ + applyCommittedCancellation(): Promise; +} + export interface RunResult { runId: string; terminalStatus: TerminalRunPhase; diff --git a/vendor/intx/workflow/src/runtime/errors.ts b/vendor/intx/workflow/src/runtime/errors.ts index 832751575c..8e525d628d 100644 --- a/vendor/intx/workflow/src/runtime/errors.ts +++ b/vendor/intx/workflow/src/runtime/errors.ts @@ -10,17 +10,16 @@ * entry in the reconstructed `state.steps` is either aligned on a step * boundary or left in one of the resumable carve-outs (an in-flight * `loop` container, or an `awaitSignal` step still `awaiting-signal` - * [re-parked] or -- with no timeout -- left `in-flight` by an - * already-logged `SignalReceived`), or is a crash-mid-invocation step - * (an agent `step` or `action` left `in-flight`) which the runtime - * settles as a terminal `StepFailed` rather than re-arming. A log that - * stops while a step is `awaiting-timer`, mid-`map`, or otherwise - * `in-flight` (a `childWorkflow`, or a timeout-bearing `awaitSignal` - * left `in-flight`) has no schedulable primitive to advance it -- the - * runtime cannot re-arm the timer scheduler entry or rebuild the - * inner-map iteration state from the log alone. The host (supervisor) - * owns the recovery decision: crash the workflow process and let a fresh - * deploy re-drive the live log. + * [re-parked] or left `in-flight` after a mover already logged, a + * `SignalReceived` or a timed gate's `TimerFired`), or is a + * crash-mid-invocation step (an agent `step` or `action` left + * `in-flight`) which the runtime settles as a terminal `StepFailed` + * rather than re-arming. A log that stops while a step is + * `awaiting-timer`, mid-`map`, or otherwise `in-flight` with no + * resumable carve-out has no schedulable primitive to advance it. + * The runtime cannot re-arm the timer scheduler entry or rebuild the + * inner-map iteration state from the log alone. The run records that + * throw as `RunFailed`. * * Surfacing the limitation as a structured error keeps the contract * honest instead of stalling with an opaque "no schedulable @@ -42,3 +41,18 @@ export class RuntimeResumeUnsupportedError extends Error { this.awaitedPrimitive = awaitedPrimitive; } } + +/** + * Thrown when a seeded resume names blob refs and the env's blob + * substrate is ephemeral. The seed is left as it was: writing a + * terminal event would make a later resume on the substrate that + * holds those blobs see a failed run. + */ +export class EphemeralResumeSeedError extends Error { + constructor(blobOutputCount: number) { + super( + `resume requires the BlobSubstrate that recorded the seed log's blob refs (${String(blobOutputCount)} blob output(s) present); the runLocal in-memory substrate is ephemeral and starts empty. Pass the originating env, or use a durable substrate.`, + ); + this.name = "EphemeralResumeSeedError"; + } +} diff --git a/vendor/intx/workflow/src/runtime/index.ts b/vendor/intx/workflow/src/runtime/index.ts index 73d4dd81cf..6dbc9f0858 100644 --- a/vendor/intx/workflow/src/runtime/index.ts +++ b/vendor/intx/workflow/src/runtime/index.ts @@ -1,4 +1,5 @@ export type { + ActionCredentialsFactory, ActionInvokeRequest, ActionInvokeResult, ActionInvoker, @@ -8,10 +9,12 @@ export type { LoopFn, LoopFnRegistry, ParkedApprovalOp, + PerformScope, PrimitiveKind, ReadParkedApprovalOps, RepoStore, RunResult, + RuntimeWorkflowRun, Scheduler, SignalChannel, SpawnChildWorkflow, @@ -32,8 +35,27 @@ export { } from "./effect-context"; export { runtimeRun, type RuntimeRunOptions } from "./run"; +export { + bodyParkedSignals, + loopIterationCursor, + loopOccurrenceReattach, + onTriggerOccurrenceReattach, + type LoopIterationCursor, + type OccurrenceReattach, +} from "./run"; +export { + bufferedEvents, + commitBuffered, + dropChain, + reloadState, + whenRunNextBuffers, + withRunCommitBarrier, +} from "./commit-chain"; -export { createSuspendableChildHandle } from "./suspendable-child-handle"; +export { + createSuspendableChildHandle, + createTerminalChildHandle, +} from "./suspendable-child-handle"; export { createLoopIterationHandle } from "./loop-iteration-handle"; @@ -52,9 +74,20 @@ export { type DrainController, } from "./drain"; -export { nextSchedulable, isRunDone, hasFailedStep } from "./dag"; +export { + nextSchedulable, + isRunDone, + hasFailedStep, + resumeResidualOf, +} from "./dag"; +export type { ResumeResidual, StepKindLookup } from "./dag"; -export { scopedStepId, baseStepId, loopBodyRunId } from "./step-scope"; +export { + scopedStepId, + baseStepId, + loopBodyRunId, + sectionBodyRunId, +} from "./step-scope"; export { evaluate as evaluateSelector, diff --git a/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts b/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts index 4108fcbce9..1c722517b8 100644 --- a/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts +++ b/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts @@ -86,5 +86,6 @@ export function createLoopIterationHandle( : {}), signal: args.signal, ...(args.cleanup !== undefined ? { cleanup: args.cleanup } : {}), + surfaceInputPark: false, }); } diff --git a/vendor/intx/workflow/src/runtime/run.ts b/vendor/intx/workflow/src/runtime/run.ts index 48ab6478ad..1d186ede51 100644 --- a/vendor/intx/workflow/src/runtime/run.ts +++ b/vendor/intx/workflow/src/runtime/run.ts @@ -8,7 +8,7 @@ // us swap the env implementations underneath without re-validating the // body. -import { correlationIdFromSignalName, signalName } from "@intx/types"; +import { correlationIdFromSignalName, signalName } from "@intx/types/signals"; import type { ApprovalSnapshot, ControlParkKind } from "@intx/types/runtime"; import type { @@ -33,31 +33,36 @@ import { stepTriggerBudget, validateRetryTriggerCombination, } from "../definition/index"; -import { evaluate, type SelectorContext } from "./selectors"; +import { + evaluate, + resolveStepInference, + type SelectorContext, +} from "./selectors"; import { hasFailedStep, - isCrashedInvocationStep, - isResumableAwaitingSignalStep, - isResumableLoopStep, - isResumableOnTriggerStep, - isResumableReceivedAwaitSignalStep, - isResumableSleepStep, isRunDone, nextSchedulable, + resumeResidualOf, } from "./dag"; import { commit as commitDurableToChain, commitBuffered as commitBufferedToChain, dropChain, flushChain, + isRepoWriteFailure, + flushFailureCause, + isRunFlushFailure, + markRepoWriteFailure, + whenRunFlushFails, reloadState as reloadStateInChain, + runAfterDiscardingBuffer, } from "./commit-chain"; import type { RunResult, + RuntimeWorkflowRun, SpawnSuspendableChild, SuspendableChildHandle, WorkflowPark, - WorkflowRun, WorkflowRuntimeEnv, } from "./env"; import { shouldAbortForDrain } from "./drain"; @@ -66,12 +71,17 @@ import { assertSpawnDepthWithinLimit, resolveMaxChildSpawnDepth, } from "./child-depth"; -import { RuntimeResumeUnsupportedError } from "./errors"; -import { loopBodyRunId, scopedStepId } from "./step-scope"; -import { inlineBodyRef } from "../ontrigger-bodies"; import { + EphemeralResumeSeedError, + RuntimeResumeUnsupportedError, +} from "./errors"; +import { loopBodyRunId, scopedStepId, sectionBodyRunId } from "./step-scope"; +import { inlineBodyRef } from "../inline-body-ref"; +import { + applyEvent, controlParkKindOf, decideTerminalRunFlip, + emptyState, isTerminalRunPhase, resumeFromLog, TransitionError, @@ -102,9 +112,10 @@ export interface RuntimeRunOptions { * agent `step` or a deterministic `action` -- `in-flight` is a crash * mid-invocation: the runtime settles it as a terminal `StepFailed` * (at-most-once refusal) instead of re-invoking it. A step left - * `awaiting-timer`, mid-`map`, or otherwise `in-flight` (a - * `childWorkflow`) stays unsupported and surfaces as - * `RuntimeResumeUnsupportedError`. + * `awaiting-timer`, mid-`map`, or otherwise `in-flight` with no + * resumable carve-out stays unsupported and surfaces as + * `RuntimeResumeUnsupportedError`. A `childWorkflow` spawn step is one + * of the carve-outs. * * When omitted, the runtime reduces canonical state from the durable * log for `runId`: an empty log starts fresh and emits `RunStarted`; @@ -118,9 +129,9 @@ export interface RuntimeRunOptions { * This run's nesting depth in the childWorkflow spawn chain. The * top-level triggered run is depth 0; a `childWorkflow` child is one * deeper than the run that spawned it. Carried on the spawn seam, not in - * the durable log, so a resumed non-root child that re-enters at depth 0 - * does not re-check depth -- resume of an in-flight `childWorkflow` is - * `RuntimeResumeUnsupportedError` anyway. Default 0. + * the durable log. The spawning parent passes the depth when it + * reattaches a child, and that child counts from the depth it is given. + * Default 0. */ depth?: number; /** @@ -168,17 +179,19 @@ export interface RuntimeRunOptions { * or a deterministic `action` -- left `in-flight` is a crash * mid-invocation and settles as a terminal `StepFailed` rather * than re-invoking. A step left `awaiting-timer`, mid-`map`, or - * otherwise `in-flight` (a `childWorkflow`) is unsupported: the - * runtime body has no surface for re-arming the timer scheduler - * entry or the inner-map iteration state from the log alone, so it - * surfaces as `RuntimeResumeUnsupportedError` and the host - * (supervisor) owns the recovery decision. + * otherwise `in-flight` is unsupported. A `childWorkflow` left + * `in-flight` or `awaiting-signal` is a carve-out: resume reattaches + * the child from its log. The runtime body has no surface for + * re-arming the timer scheduler entry or the inner-map iteration + * state from the log alone, so those throw + * `RuntimeResumeUnsupportedError`. That throw is recorded as + * `RunFailed`, and the run is terminal. */ export function runtimeRun( definition: WorkflowDefinition, env: WorkflowRuntimeEnv, options: RuntimeRunOptions = {}, -): WorkflowRun { +): RuntimeWorkflowRun { const runId = options.runId ?? env.newId("run"); // A run id is a durable-store path segment and a mail-address local part, so // an unconstrained caller-supplied id is a path-escape and addressing hazard. @@ -256,6 +269,22 @@ export function runtimeRun( } cancelController.abort(); }, + async applyCommittedCancellation() { + try { + await emitChildCancelCascade(env, runId, await reloadState(env, runId)); + } catch (cause) { + if ( + !(cause instanceof TransitionError) || + cause.code !== "terminal-phase" + ) { + throw cause; + } + } finally { + // A parked body waits on this abort, so a failed read or cascade must + // not withhold it. + cancelController.abort(); + } + }, async signal(name, payload, signalId) { await env.signalChannel.deliver(name, payload, signalId); }, @@ -277,6 +306,12 @@ async function executeRun( cancelController, options, ); + } catch (cause) { + // The body escaped. A nested `commitDurable` would wait on this + // settlement forever, so the terminal event is appended directly + // once the chain's current link finishes and the pending buffer + // is discarded. The buffer is what a folding read just rejected. + return await settleThrownBody(env, runId, cause); } finally { // Always drop the per-runId commit chain entry, even on a thrown // body, so long-running processes accumulating many workflows do @@ -286,6 +321,197 @@ async function executeRun( } } +function thrownMessage(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause); +} + +async function durableState( + env: WorkflowRuntimeEnv, + runId: string, +): Promise { + return resumeFromLog(runId, await env.repoStore.read(runId)); +} + +function rethrowOriginal(cause: unknown, failure: unknown): never { + throw new Error(thrownMessage(cause), { cause: failure }); +} + +/** + * Child-cancel cascade plus `RunCancelled`, applied in memory. The + * caller appends the batch itself. `commit` from inside the settlement + * link waits on that link. + */ +function cancellingTerminalEvents( + state: RunState, + at: string, +): { events: WorkflowEvent[]; next: RunState } { + const events: WorkflowEvent[] = []; + let current = state; + for (const [childRunId, childState] of state.children) { + if (childState.cancelRequested) continue; + if (childState.terminalStatus !== undefined) continue; + const event: WorkflowEvent = { + kind: "ChildCancelRequested", + seq: current.lastSeq + 1, + at, + childRunId, + }; + current = applyEvent(current, event); + events.push(event); + } + const cancelled: WorkflowEvent = { + kind: "RunCancelled", + seq: current.lastSeq + 1, + at, + }; + current = applyEvent(current, cancelled); + events.push(cancelled); + return { events, next: current }; +} + +/** + * The log is already terminal. Strict output resolution can still + * throw. Discovery skips a terminal log, so that throw would leave + * the host without the event. Unreadable outputs are omitted. + */ +async function resultFromDurableTerminal( + env: WorkflowRuntimeEnv, + runId: string, + state: RunState, +): Promise { + try { + return await buildResultFromLog(env, runId, state); + } catch { + const events = await env.repoStore.read(runId); + const outputs: Record = {}; + for (const event of events) { + if (event.kind !== "StepCompleted") continue; + try { + outputs[event.stepId] = await env.blobs.resolveRef(event.output.ref); + } catch { + // This output is unreadable. Omit it. + } + } + return { + runId, + terminalStatus: decideTerminalRunFlip(state.phase), + outputs, + events, + }; + } +} + +async function appendEvents( + env: WorkflowRuntimeEnv, + runId: string, + events: readonly WorkflowEvent[], + next: RunState, + cause: unknown, + settleCancelling = true, +): Promise { + try { + await env.repoStore.appendBatch(runId, events); + } catch (failure) { + const again = await durableState(env, runId); + if (isTerminalRunPhase(again.phase)) { + return resultFromDurableTerminal(env, runId, again); + } + // A cancel written on the supervisor store can land while this + // append is in flight. The phase is then `cancelling` and nothing + // in the body is left to record `RunCancelled`. One attempt does. + // That attempt appends with `settleCancelling` false, so a second + // failure rejects instead of recursing for as long as the store + // keeps failing. + if (settleCancelling && again.phase === "cancelling") { + return appendCancellingTerminal(env, runId, again, cause); + } + rethrowOriginal(cause, failure); + } + return resultFromDurableTerminal(env, runId, next); +} + +async function appendRunFailed( + env: WorkflowRuntimeEnv, + runId: string, + state: RunState, + cause: unknown, +): Promise { + const at = env.clock().toISOString(); + const failed: WorkflowEvent = { + kind: "RunFailed", + seq: state.lastSeq + 1, + at, + error: { message: thrownMessage(cause) }, + }; + let next: RunState; + try { + next = applyEvent(state, failed); + } catch (failure) { + const again = await durableState(env, runId); + if (isTerminalRunPhase(again.phase)) { + return resultFromDurableTerminal(env, runId, again); + } + if (again.phase === "cancelling") { + return appendCancellingTerminal(env, runId, again, cause); + } + rethrowOriginal(cause, failure); + } + return appendEvents(env, runId, [failed], next, cause); +} + +async function appendCancellingTerminal( + env: WorkflowRuntimeEnv, + runId: string, + state: RunState, + cause: unknown, +): Promise { + const at = env.clock().toISOString(); + let built: { events: WorkflowEvent[]; next: RunState }; + try { + built = cancellingTerminalEvents(state, at); + } catch (failure) { + const again = await durableState(env, runId); + if (isTerminalRunPhase(again.phase)) { + return resultFromDurableTerminal(env, runId, again); + } + rethrowOriginal(cause, failure); + } + return appendEvents(env, runId, built.events, built.next, cause, false); +} + +async function settleThrownBody( + env: WorkflowRuntimeEnv, + runId: string, + cause: unknown, +): Promise { + // The store rejected the write. The durable prefix is what recovery + // adopts; appending RunFailed here would hide that crash. A seeded + // resume that names blob refs on an ephemeral substrate is the same + // kind of refusal: the seed is still runnable once those blobs are + // present. + if (isRepoWriteFailure(cause) || cause instanceof EphemeralResumeSeedError) { + throw cause; + } + return runAfterDiscardingBuffer(runId, async () => { + const state = await durableState(env, runId); + if (isTerminalRunPhase(state.phase)) { + try { + return await buildResultFromLog(env, runId, state); + } catch (failure) { + // The seed is already terminal, and rebuilding its result failed + // (a fresh ephemeral substrate cannot resolve the seed's blob + // refs). Surface the error the body threw. + rethrowOriginal(cause, failure); + } + } + if (state.phase === "pending") throw cause; + if (state.phase === "cancelling") { + return appendCancellingTerminal(env, runId, state, cause); + } + return appendRunFailed(env, runId, state, cause); + }); +} + // Intra-segment commit: validates the transition and assigns the seq // in memory (unchanged from before batching) but DEFERS the durable // write into the per-runId buffer. The buffer is flushed in one @@ -298,8 +524,9 @@ function commit( env: WorkflowRuntimeEnv, runId: string, event: WorkflowEvent, + guard?: (state: RunState) => void, ): Promise> { - return commitBufferedToChain(env, runId, event); + return commitBufferedToChain(env, runId, event, guard); } // Segment-boundary commit: buffers the event then flushes the whole @@ -326,9 +553,10 @@ function commitDurable( // Called at a suspension boundary AFTER buffering the suspension // marker (`SignalAwaited`/`TimerSet`) so the marker -- and everything // before it in the segment -- is durable BEFORE the run parks. The -// out-of-process scheduler tails the durable `TimerSet`; resume after -// a crash-while-suspended reconstructs the awaiting state from the -// durable log. A buffered suspension marker that never flushed would +// host scheduler tails the durable `TimerSet`. A crash-while-suspended +// reconstructs the awaiting state from that log. `TimerFired`'s +// sequence is assigned when the timer fires, after anything buffered +// past this flush. A buffered suspension marker that never flushed would // be lost on a park, so this flush is load-bearing, not an // optimisation knob. async function flush(env: WorkflowRuntimeEnv, runId: string): Promise { @@ -388,7 +616,14 @@ async function executeRunBody( } continue; } - await env.repoStore.append(runId, event); + try { + await env.repoStore.append(runId, event); + } catch (cause) { + // Same as a failed flush: settlement must not hide a write the + // store rejected. + markRepoWriteFailure(cause); + throw cause; + } } // Seed-contract guard, before any blob resolution. A resume that @@ -408,9 +643,7 @@ async function executeRunBody( e.kind === "StepCompleted" && e.output.ref.startsWith("blob:"), ); if (seedBlobRefs.length > 0 && env.blobs.ephemeral) { - throw new Error( - `resume requires the BlobSubstrate that recorded the seed log's blob refs (${String(seedBlobRefs.length)} blob output(s) present); the runLocal in-memory substrate is ephemeral and starts empty. Pass the originating env, or use a durable substrate.`, - ); + throw new EphemeralResumeSeedError(seedBlobRefs.length); } // Establish canonical state from the durable log itself, not from the @@ -453,12 +686,11 @@ async function executeRunBody( // `StepCompleted` landed, and the invoked primitive is // non-deterministic and unrecorded, so it cannot be replayed // exactly-once. Rather than re-invoke it (at-most-once refusal), settle - // it as a terminal `StepFailed`. Every OTHER non-terminal residual -- - // an `in-flight` coordination container (mid-`map`, `childWorkflow`), - // or an `awaiting-signal`/`awaiting-timer` step -- still surfaces - // `RuntimeResumeUnsupportedError`: those have a live re-arming surface - // the host owns (rebuild the map state, re-park on a later signal), so - // declining honestly is correct there. + // it as a terminal `StepFailed`. Every OTHER non-terminal residual -- an + // `in-flight` `map`, or an `awaiting-signal`/`awaiting-timer` step with + // no carve-out -- still surfaces `RuntimeResumeUnsupportedError`. The + // log alone cannot re-arm that step, so the throw is recorded as + // `RunFailed`. // // The pass runs whenever canonical state is `running`, whether the // residual arrived via a `resumeFromEvents` seed OR was adopted from a @@ -490,18 +722,12 @@ async function executeRunBody( // - a `sleep` step still `awaiting-timer` (runSleep re-adopts its // unfired durable timer and re-parks) or left `in-flight` by its // fired timer (runSleep completes it without re-parking -- the - // crash-after-TimerFired-before-StepCompleted window). - if ( - isResumableLoopStep(definition, stepId, stepState.phase) || - isResumableAwaitingSignalStep(definition, stepId, stepState.phase) || - isResumableReceivedAwaitSignalStep( - definition, - stepId, - stepState.phase, - ) || - isResumableOnTriggerStep(definition, stepId, stepState.phase) || - isResumableSleepStep(definition, stepId, stepState.phase) - ) { + // crash-after-TimerFired-before-StepCompleted window); + // - a `childWorkflow` spawn step `in-flight` or `awaiting-signal` + // (runChildWorkflow reattaches the recorded child and + // reestablishes its relay). + const residual = resumeResidualOf(definition, stepId, stepState.phase); + if (residual === "continue") { // The onTrigger container carries no crash-mid-invocation risk (it never // self-completes); `runOnTrigger` re-derives its cursor from the log and // re-links a body parked mid-approval. Leave it for `nextSchedulable` to @@ -522,7 +748,7 @@ async function executeRunBody( // pre-recovery behavior. Both settlings happen AFTER this loop -- // committing inline would leave `state` stale and merely relocate the // stall to the main loop. - if (isCrashedInvocationStep(definition, stepId, stepState.phase)) { + if (residual === "crashed-invocation") { const parkedOps = env.readParkedApprovalOps !== undefined ? await env.readParkedApprovalOps({ @@ -556,13 +782,21 @@ async function executeRunBody( } continue; } - // Every other non-terminal residual keeps declining: the host owns - // the recovery decision (crash, alert, or redeploy). - if ( - stepState.phase === "in-flight" || - stepState.phase === "awaiting-signal" || - stepState.phase === "awaiting-timer" - ) { + // Every other non-terminal residual still throws. The throw is + // recorded as `RunFailed`. + if (residual === "unsupported") { + // `resumeResidualOf` returns this only for a non-terminal phase. + // The constructor's phase is that same set, so a terminal phase + // here is a broken guard. + if ( + stepState.phase !== "in-flight" && + stepState.phase !== "awaiting-signal" && + stepState.phase !== "awaiting-timer" + ) { + throw new Error( + `resume residual for step ${stepId} is unsupported in phase ${stepState.phase}`, + ); + } throw new RuntimeResumeUnsupportedError( stepId, stepState.phase, @@ -768,12 +1002,34 @@ async function executeRunBody( // this map to abort in-flight cancel-mode steps when drain fires // after the step was already scheduled. const stepAborts = new Map(); + // A failed flush drops its batch and the step runner rejects with + // that same error. The loop below would otherwise treat the + // rejection as a settled step and schedule the step again, or wait + // forever on a sibling that is already parked. Remember the first + // error and rethrow it once every in-flight step has stopped. + let repoWriteFailure: unknown; + // One waiter for the whole loop. A flush that already failed resolves + // it immediately, so the next race observes it without a new registration. + const flushFailed = whenRunFlushFails(runId); + async function throwRememberedWriteFailure(): Promise { + if (repoWriteFailure === undefined) { + repoWriteFailure = flushFailureCause(runId); + } + const cause = repoWriteFailure; + if (cause === undefined) return; + for (const controller of stepAborts.values()) { + if (!controller.signal.aborted) controller.abort(); + } + await Promise.allSettled(stepPromises.values()); + throw cause; + } // Tick loop: schedule everything ready, await any in-flight to // settle, repeat until done. Cancellation aborts every in-flight // executor; we still loop to commit `CancelPropagated` and the // terminal `RunCancelled`. while (!isRunDone(definition, state)) { + await throwRememberedWriteFailure(); if (cancelController.signal.aborted && state.phase !== "cancelling") { state = await reloadState(env, runId); } @@ -821,7 +1077,11 @@ async function executeRunBody( .then((output) => { stepOutputs[primitive.id] = output; }) - .catch(() => { + .catch((cause: unknown) => { + if (isRunFlushFailure(runId, cause)) { + if (repoWriteFailure === undefined) repoWriteFailure = cause; + return; + } // Errors are committed as StepFailed inside the primitive // runner; the main loop notices the failed phase on the // next state reload. @@ -864,6 +1124,7 @@ async function executeRunBody( } if (stepPromises.size === 0) { + await throwRememberedWriteFailure(); if (ready.length === 0) { throw new Error( `workflow ${definition.id} run ${runId} stalled with no schedulable primitives`, @@ -872,16 +1133,21 @@ async function executeRunBody( // Promises were scheduled this tick but already completed // synchronously; reload state and continue. state = await reloadState(env, runId); + await throwRememberedWriteFailure(); continue; } - // Wait for at least one in-flight primitive to settle. Each - // primitive's runner already swallows its own errors into - // StepFailed events so the race resolves cleanly. - await Promise.race( - Array.from(stepPromises.values()).map((p) => p.catch(() => undefined)), - ); + // Wait for at least one in-flight primitive to settle. Step + // failures are already `StepFailed`. A parked step never settles, + // so the flush waiter is what wakes the loop when an external + // writer drops the buffer. + await Promise.race([ + ...Array.from(stepPromises.values()).map((p) => p.catch(() => undefined)), + flushFailed, + ]); + await throwRememberedWriteFailure(); state = await reloadState(env, runId); + await throwRememberedWriteFailure(); for (const stepId of justSettled) { stepPromises.delete(stepId); } @@ -1195,6 +1461,12 @@ async function runPrimitiveSafe( maxChildSpawnDepth, ); } catch (cause) { + // Only this run's failed flush. A nested run stamps the same + // marker on its own append error, and that error rejects the + // child. The parent still records this step's failure; rethrowing + // the marker here would drop the buffered ChildCompleted and + // leave the parent without a terminal event. + if (isRunFlushFailure(runId, cause)) throw cause; let state = await reloadState(env, runId); const stepState = state.steps.get(primitive.id); if (!stepState) { @@ -1320,6 +1592,7 @@ async function runPrimitiveSafe( }; state = await commit(env, runId, routed); void state; + await flushParkedChildWorkflowFailure(env, runId, primitive.kind); return { failed: true, stepId: primitive.id, error: { message } }; } const failed: WorkflowEvent = { @@ -1332,6 +1605,7 @@ async function runPrimitiveSafe( retriesExhausted: true, }; state = await commit(env, runId, failed); + await flushParkedChildWorkflowFailure(env, runId, primitive.kind); } } void state; @@ -1463,6 +1737,10 @@ async function runStep( // same `null` so an audit reader cannot diverge from the agent's // actual input. const input = rawInput === undefined ? null : rawInput; + const inference = + step.inference !== undefined + ? resolveStepInference(step.inference, selectorCtx) + : undefined; if (!stepStartedEmitted) { const { ref: inputRef } = await env.blobs.recordOutput( `${step.id}.input`, @@ -1488,33 +1766,31 @@ async function runStep( // before subscribing to its edge; an abort raised during that commit // would otherwise never reach the invoker, which would then work on // behalf of a run that is already cancelled. - // `step.timeout` is an execution budget: it bounds active agent - // invocations, not parked-idle waits. Each input park below disarms - // it before waiting and re-arms a fresh budget on resume; the - // approval park stays armed (an approval wait holds a mandatory - // snapshot and stays bounded by the execution timer, unlike - // trigger-idle input). Disarming never affects cancellation: the - // outer-abort listener aborts `stepAbort` independently of the timer. const stepAbort = new AbortController(); const onOuter = () => { stepAbort.abort(); }; bridgeAbort(abort, onOuter); + // `step.timeout` bounds each active invocation turn, not the idle gap + // between triggers on a multi-trigger (`onTrigger` / `maxTriggers > 1`) + // step. Disarm across input parks and re-arm a fresh budget before the + // next `invokeStep`. Approval parks keep the timer running (they are mid- + // turn suspensions). let timer: ReturnType | undefined; + const disarmStepTimer = (): void => { + if (timer !== undefined) { + clearTimeout(timer); + timer = undefined; + } + }; const armStepTimer = (): void => { - if (timer !== undefined) clearTimeout(timer); + disarmStepTimer(); if (step.timeout !== undefined) { timer = setTimeout(() => { stepAbort.abort(); }, step.timeout); } }; - const disarmStepTimer = (): void => { - if (timer !== undefined) { - clearTimeout(timer); - timer = undefined; - } - }; armStepTimer(); try { @@ -1541,13 +1817,10 @@ async function runStep( // step; a resume that suspends AGAIN re-parks through the normal // `{ suspend }` arm below. if (resumeFromPark !== undefined) { + if (resumeFromPark.parkKind === "input") { + disarmStepTimer(); + } const parkState = await reloadState(env, runId); - // An input re-park is trigger-idle, not execution: disarm the - // execution timer across the wait and re-arm a fresh budget on - // resume. An approval re-park keeps the timer armed (see the - // approval-park note below). - const resumeIsInput = resumeFromPark.parkKind === "input"; - if (resumeIsInput) disarmStepTimer(); const decision = await parkOnSignal( env, runId, @@ -1558,7 +1831,9 @@ async function runStep( parkState, stepAbort.signal, ); - if (resumeIsInput) armStepTimer(); + if (resumeFromPark.parkKind === "input") { + armStepTimer(); + } resume = { correlationId: resumeFromPark.correlationId, decision, @@ -1593,6 +1868,7 @@ async function runStep( const result = await env.invokeStep({ agent: step.agent, input, + ...(inference !== undefined ? { inference } : {}), authzContext: { stepId: step.id, attempt, @@ -1615,12 +1891,9 @@ async function runStep( const hasMoreTriggers = triggerBudget === "unbounded" || servicedTriggers < triggerBudget; if (!hasMoreTriggers) break; + disarmStepTimer(); const inputCorrelationId = env.newId("corr"); const rearmState = await reloadState(env, runId); - // The step is trigger-idle until its next trigger arrives: disarm - // the execution timer across the park and re-arm a fresh budget - // for the next turn on resume. - disarmStepTimer(); const decision = await parkOnSignal( env, runId, @@ -1628,6 +1901,9 @@ async function runStep( stepId: step.id, signalName: signalName(inputCorrelationId), parkKind: "input", + claimsRunInputChannel: { + sectionInputStepIds: sectionInputStepIds(definition), + }, }, rearmState, stepAbort.signal, @@ -1646,9 +1922,7 @@ async function runStep( // reduced state passed to `parkOnSignal` reads the step as // `in-flight`, and its re-park guard emits a fresh `SignalAwaited` // rather than treating this as a re-park of an already-awaiting - // gate. The execution timer stays armed across this approval park: - // an approval wait holds a mandatory snapshot and stays bounded by - // `step.timeout`; only trigger-idle input parks disarm (above). + // gate. const parkState = await reloadState(env, runId); const decision = await parkOnSignal( env, @@ -1960,6 +2234,7 @@ async function runAction( handler: primitive.handler, input, requires: primitive.effect?.requires ?? [], + credentials: primitive.credentials ?? [], authzContext: { stepId: primitive.id, attempt: 1, runId }, signal: actionAbort.signal, }); @@ -2046,11 +2321,16 @@ async function runLoop( }); } - // Replay fully-done iterations (child terminal AND step completed) from - // the log to re-derive the cursor, the threaded input, and whether the - // loop already reached its outcome before the crash (the post-routing - // window). while/carry are pure, so replaying them over recorded inputs - // and outputs reproduces the pre-crash decisions. + // Replay finished attempts to re-derive the cursor and the threaded input. + // A completed attempt (child completed AND scoped step completed) replays + // while/carry, which are pure. A tolerated failure has no scoped + // StepCompleted: skip it without while/carry, keep the same input, and + // count the attempt, so the frontier below is the attempt planLoopResume + // must re-link. A terminal attempt the policy will not continue throws + // here, before that planner runs and before any drive: planLoopResume + // returns undefined for a terminal child, and occurrenceResume is cleared + // after the first drive, so driving the terminal attempt would spawn the + // later park with no resume token. const iterationZeroInput = primitive.input !== undefined ? evaluate(primitive.input, selectorCtx) @@ -2060,29 +2340,56 @@ async function runLoop( let iteration = 0; let terminated = false; let outcome: "converged" | "exhausted" = "exhausted"; - // The output of the most recent iteration, boxed so a legitimately - // `undefined` iteration output stays distinguishable from "no iteration - // has run yet". The settle path below needs it: the loop breaks the - // moment `while` goes false, BEFORE `carry` runs, so `currentInput` is - // the converging iteration's input and this is its output. + // The output of the most recent COMPLETED attempt, boxed so a legitimately + // `undefined` iteration output stays distinguishable from "no attempt has + // completed". A tolerated failure does not set this. The settle path + // publishes it as `final`, or `null` when every attempt was tolerated. let lastIteration: { output: unknown } | undefined; - while (isIterationDone(state, runId, primitive.id, iteration)) { + const policy = iterationFailurePolicyOf(primitive); + while (true) { const doneStepId = scopedStepId(primitive.id, iteration); - const doneInput = await resolveIterationInput(env, log, doneStepId); - const doneOutput = await resolveIterationOutput(env, log, doneStepId); - lastIteration = { output: doneOutput }; - iteration += 1; - if (!whileFn(doneOutput, doneInput)) { - outcome = "converged"; - terminated = true; - break; + const cursor = loopIterationCursor( + state, + runId, + primitive.id, + iteration, + policy, + ); + if (cursor.kind === "completed") { + const doneInput = await resolveIterationInput(env, log, doneStepId); + const doneOutput = await resolveIterationOutput(env, log, doneStepId); + lastIteration = { output: doneOutput }; + iteration += 1; + if (!whileFn(doneOutput, doneInput)) { + outcome = "converged"; + terminated = true; + break; + } + if (iteration >= primitive.maxIterations) { + outcome = "exhausted"; + terminated = true; + break; + } + currentInput = carryFn(doneOutput, doneInput); + continue; } - if (iteration >= primitive.maxIterations) { - outcome = "exhausted"; - terminated = true; - break; + if (cursor.kind === "tolerated") { + iteration += 1; + if (iteration >= primitive.maxIterations) { + outcome = "exhausted"; + terminated = true; + break; + } + continue; + } + if (cursor.kind === "terminal") { + // Durable half only. The recovering process's abort signal is not a + // reason to reclassify a recorded genuine failure as teardown. + throw new Error( + `loop ${primitive.id} iteration ${String(iteration)} ended ${cursor.terminalStatus}`, + ); } - currentInput = carryFn(doneOutput, doneInput); + break; } // Prefer the resume iteration's own recorded input (an in-flight @@ -2126,26 +2433,61 @@ async function runLoop( // only on the recovered iteration of a crash resume, where it re-links a // body parked or mid-relay at the crash; it is cleared after this drive so // every later iteration spawns fresh. - const { terminalStatus } = await driveSuspendableOccurrence( - env, - runId, - primitive.id, - { - childRunId, - bodyRef, - input: currentInput, - resume: occurrenceResume, - spawnSuspendableChild: spawnLoopIteration, - depth, - maxChildSpawnDepth, - abort, - }, - ); + const driven = await driveSuspendableOccurrence(env, runId, primitive.id, { + childRunId, + bodyRef, + input: currentInput, + resume: occurrenceResume, + spawnSuspendableChild: spawnLoopIteration, + depth, + maxChildSpawnDepth, + abort, + signalRelayAddressing: "name", + }); occurrenceResume = undefined; + // Trust the recorded child, not only the drive's return. A child log + // that is missing is re-executed, and that fresh terminal can disagree + // with a ChildCompleted the parent already recorded. `abortedTeardown` + // lives only on the record. `abort.aborted` is this process's live + // signal: an abort during the drive's commit/flush has no durable flag + // yet, and a tolerate loop must still end rather than continue. + const recorded = (await reloadState(env, runId)).children.get(childRunId); + const terminalStatus = recorded?.terminalStatus ?? driven.terminalStatus; + const abortedTeardown = recorded?.abortedTeardown === true; + const tolerated = + terminalStatus === "failed" && + !abort.aborted && + !abortedTeardown && + policy === "tolerate"; + if (terminalStatus !== "completed" && !tolerated) { + // A failed or cancelled iteration the policy will not continue is a + // real failure, not an exhaustion. Throw so runPrimitiveSafe lands + // StepFailed (or CancelPropagated when the run is cancelling) on the + // loop node. Throwing skips routeLoopOutcome, so neither branch is + // pruned; both the normal dependents and onExhausted then run before + // the run settles failed, per the engine's "a failed dependency is + // resolved" scheduling (the same as a failed gate). The + // mutually-exclusive routing holds only on the success path. + throw new Error( + `loop ${primitive.id} iteration ${String(i)} ended ${terminalStatus}`, + ); + } + if (tolerated) { + // Leave the scoped step in-flight. StepFailed would make hasFailedStep + // fail the run after a tolerated exhaust, and StepCompleted would make + // the replay walk treat the failure as a success. + if (i + 1 >= primitive.maxIterations) { + outcome = "exhausted"; + break; + } + continue; + } + // The drive returns only the terminal status; the iteration's step outputs // live in the child's durable log, so hydrate them here -- the scoped - // StepCompleted records them, and while/carry read them. + // StepCompleted records them, and while/carry read them. Only a completed + // attempt gets that record. A tolerated failure has no output to carry. const output = await hydrateChildOutputs(env, childRunId); lastIteration = { output }; @@ -2155,20 +2497,6 @@ async function runLoop( } await flush(env, runId); - if (terminalStatus !== "completed") { - // A failed or cancelled iteration is a real failure, not an - // exhaustion. Throw so runPrimitiveSafe lands StepFailed (or - // CancelPropagated when the run is cancelling) on the loop node. - // Note: throwing skips routeLoopOutcome, so neither branch is - // pruned; both the normal dependents and onExhausted then run - // before the run settles failed, per the engine's "a failed - // dependency is resolved" scheduling (the same as a failed gate). - // The mutually-exclusive routing holds only on the success path. - throw new Error( - `loop ${primitive.id} iteration ${String(i)} ended ${terminalStatus}`, - ); - } - if (!whileFn(output, currentInput)) { outcome = "converged"; break; @@ -2180,13 +2508,14 @@ async function runLoop( currentInput = carryFn(output, currentInput); } - if (lastIteration === undefined) { + if (iterations === 0) { // Unreachable through `loop()`, which rejects a non-positive - // maxIterations, so every settle follows at least one iteration -- + // maxIterations, so every settle follows at least one attempt -- // replayed or driven. A definition that reached here by another road - // has no last iteration to report; fail loud rather than publish a - // `final` the loop never produced. Checked BEFORE the route so the - // throw cannot leave one branch of the loop's dependents pruned. + // has no attempt to report; fail loud rather than publish a record the + // loop never produced. Checked BEFORE the route so the throw cannot + // leave one branch of the loop's dependents pruned. A positive count + // whose every attempt was tolerated is a real exhaust: `final` is null. throw new Error( `loop ${primitive.id} settled ${outcome} without running an iteration`, ); @@ -2196,7 +2525,7 @@ async function runLoop( outcome, iterations, carry: currentInput, - final: lastIteration.output, + final: lastIteration === undefined ? null : lastIteration.output, }; await emitStepCompletedWithValue(env, runId, primitive.id, output); return output; @@ -2221,8 +2550,23 @@ type SuspendableOccurrenceResume = // emitted its relay await before the crash (in-flight, no durable relay). The // re-adopted body re-parks silently, so there is no await to reestablish and // no `onSignalPark` to surface it -- drive the container relay FRESH over the - // recovered name instead. - | { kind: "signal-relay-drive-fresh"; name: string }; + // recovered name instead. `timeoutAt` is the body's deadline, when the gate + // is timed, so the fresh relay stays timed. + | { kind: "signal-relay-drive-fresh"; name: string; timeoutAt?: string }; + +type ChildRelayPull = "next" | "hold"; + +type ChildDriveResult = + | Awaited> + | { kind: "held" }; + +/** + * How this container addresses a body `awaitSignal` it relays. + * `"name"` keeps the name FIFO (a loop that is itself the addressable + * run). `"correlation"` mints an id on each fresh relay park so a + * delivery for one park cannot satisfy the next. + */ +type SignalRelayAddressing = "name" | "correlation"; /** * Drive one occurrence's suspendable-child body to terminal, durably: commit @@ -2254,6 +2598,7 @@ async function driveSuspendableOccurrence( depth: number; maxChildSpawnDepth: number; abort: AbortSignal; + signalRelayAddressing: SignalRelayAddressing; }, ): Promise<{ terminalStatus: "completed" | "failed" | "cancelled" }> { const { @@ -2265,6 +2610,7 @@ async function driveSuspendableOccurrence( depth, maxChildSpawnDepth, abort, + signalRelayAddressing, } = args; let before = await reloadState(env, runId); @@ -2302,7 +2648,7 @@ async function driveSuspendableOccurrence( // `pending` carries a body event already pulled by a signal-relay drive // below (its `next()` raced the signal), so the loop consumes it rather // than calling `next()` a second time and dropping it. - let pending: Awaited> | undefined; + let pending: ChildDriveResult | undefined; if (resume?.kind === "approval") { // Re-link the parent to a body re-spawned from its log and parked on the @@ -2339,6 +2685,7 @@ async function driveSuspendableOccurrence( // The container's signal-relay await is durable; the re-spawned body // re-parks on the name silently, so re-drive the race from the recovered // await seq (no re-emit) and continue with the body event it yields. + const relayLog = await env.repoStore.read(runId); pending = await raceContainerSignalRelay( env, runId, @@ -2346,7 +2693,9 @@ async function driveSuspendableOccurrence( child, resume.name, resume.awaitSeq, + correlationIdAt(relayLog, resume.awaitSeq), abort, + "next", ); } else if (resume?.kind === "signal-relay-drive-fresh") { // The container never emitted its relay await before the crash, so there is @@ -2354,13 +2703,22 @@ async function driveSuspendableOccurrence( // onSignalPark). Drive the container relay FRESH over the recovered name -- // emit the relay await and race -- so the container ends up awaiting the // signal that will arrive, and continue with the body event it yields. + // The body's deadline rides along, so the fresh relay stays timed when + // the gate is timed. + const freshTimeout = remainingSignalTimeout( + resume.timeoutAt, + env.clock().getTime(), + ); pending = await driveContainerSignalRelay( env, runId, containerStepId, child, resume.name, + signalRelayAddressing, abort, + "next", + freshTimeout, ); } @@ -2371,6 +2729,11 @@ async function driveSuspendableOccurrence( terminalStatus = bodyEvent.terminalStatus; break; } + if (bodyEvent.kind === "held") { + throw new Error( + `suspendable body ${childRunId} drive held a park; this drive reads the queue`, + ); + } if (bodyEvent.kind === "park") { // A body step parked on an approval. Proxy it up on the SAME // correlation via THIS run's own park machinery, so the whole @@ -2396,6 +2759,16 @@ async function driveSuspendableOccurrence( await child.resume(bodyEvent.park.correlationId, decision); continue; } + if (bodyEvent.kind === "input-park") { + // Unreachable. Loop and onTrigger handles are built with + // `surfaceInputPark: false`, so `next` aborts the body and throws + // before this arm. The throw here does not abort: the handle already + // did, and a second teardown would race the one that owns the child. + throw new Error( + `suspendable body ${childRunId} surfaced an input park; a loop or ` + + `onTrigger body cannot re-arm an input park`, + ); + } // A body step parked on an author `awaitSignal` gate. Proxy it up as a // signal-relay await on THIS container run over the SAME author name and // relay the resolved signal back into the body. The drive returns the @@ -2407,7 +2780,10 @@ async function driveSuspendableOccurrence( containerStepId, child, bodyEvent.name, + signalRelayAddressing, abort, + "next", + bodyEvent.timeout, ); } @@ -2445,8 +2821,10 @@ async function driveSuspendableOccurrence( * a body run ends non-`completed` (terminal-is-final) or the run is * cancelled/aborted. * - * Each event's body is a full sub-run under `runs/__/`, so - * per-event detail lives in its own log; the parent log carries only the + * Each event's body is a full sub-run under + * `runs/____/` (`sectionBodyRunId`). A body + * already recorded as `runs/__/` stays on that id. Per-event + * detail lives in its own log; the parent log carries only the * container `StepStarted`, a `ChildSpawned`/`ChildCompleted` pair per event, * and the input-park `SignalAwaited`/`SignalReceived` re-arm -- all existing * event kinds, so the state machine is untouched. @@ -2525,7 +2903,7 @@ async function runOnTrigger( // The body already ended non-`completed` before the crash; end the // section the same way the steady-state loop does. throw new Error( - `onTrigger ${primitive.id} body run ${primitive.id}__${String(plan.eventIndex)} ended ${plan.terminalStatus}`, + `onTrigger ${primitive.id} body run ${recordedSectionBodyRunId(runId, primitive.id, plan.eventIndex, initial.children)} ended ${plan.terminalStatus}`, ); case "reestablish-approval": eventIndex = plan.eventIndex; @@ -2604,7 +2982,12 @@ async function runOnTrigger( } while (true) { - const childRunId = `${primitive.id}__${String(eventIndex)}`; + const childRunId = recordedSectionBodyRunId( + runId, + primitive.id, + eventIndex, + (await reloadState(env, runId)).children, + ); let resume: SuspendableOccurrenceResume | undefined; if (resumeApproval !== undefined) { resume = { @@ -2646,6 +3029,7 @@ async function runOnTrigger( depth, maxChildSpawnDepth, abort, + signalRelayAddressing: "correlation", }, ); @@ -2696,63 +3080,87 @@ async function runOnTrigger( /** * Drive one body `signal-relay` await: proxy the body's author `awaitSignal` * gate up as a signal-relay await on the container over the SAME name, then - * relay the resolved signal back into the body. Returns the body's NEXT event - * (its `next()` is consumed here) for the caller's loop to continue with. + * relay the resolved signal back into the body. Returns the body's next + * event for the caller's loop to continue with. * - * BUFFER-FIFO (operator ruling): a signal is RELAYED, never dropped. Emitting - * the container's `SignalAwaited` runs the reducer's FIFO pairing: + * `bodyLeg` chooses how "the body moved on" is observed. `"next"` races + * `child.next()` and is the loop and onTrigger leg: those bodies are driven + * one step at a time, so the next event is this gate moving on. `"park-release"` + * is the childWorkflow leg. A child run parks sibling steps at the same time, + * and `next` would take the sibling off the queue and treat it as this park + * ending. That leg watches the named park leave the await instead, and only + * then reads the queue. + * + * BUFFER-FIFO (operator ruling): a signal is RELAYED, never dropped, on the + * `"next"` leg. Emitting the container's `SignalAwaited` runs the reducer's + * FIFO pairing: * (a) the emit pre-consumes a signal queued before it (container reduces to * in-flight) -- bind it from the log's pairing and relay, no await; or * (b) the container is left awaiting -- race the signal's arrival against the - * body producing its next event (a gate the body timed out itself) and - * against abort. Signal-first relays it directly; body-first retires the - * now-stale relay await (`SignalAwaitAbandoned`) unless a signal landed - * and was consumed during the race, in which case it is relayed - * idempotently. An untimed body gate has a single exit -- the signal - * always wins -- so this is byte-identical in shape to the approval proxy. + * body leg and against abort. Signal-first relays it directly; body-first + * retires the now-stale relay await (`SignalAwaitAbandoned`) unless a + * signal landed and was consumed during the race, in which case it is + * relayed idempotently. An untimed body gate has a single exit -- the + * signal always wins -- so this is byte-identical in shape to the + * approval proxy. + * + * The `"park-release"` leg reads the child park before it accepts a signal + * that is already waiting. A gate that has moved on is not given that + * signal. The parent's relay await is abandoned when it is still open, and + * the signal already paired into the parent log is left there. A gate that + * is still open takes that paired signal. The event is already committed, + * so the relay does not append it again. */ +function finishRelayPull( + child: SuspendableChildHandle, + pull: ChildRelayPull, +): Promise { + if (pull === "hold") return Promise.resolve({ kind: "held" }); + return child.next(); +} + async function driveContainerSignalRelay( env: WorkflowRuntimeEnv, runId: string, containerStepId: string, child: SuspendableChildHandle, name: string, + addressing: SignalRelayAddressing, abort: AbortSignal, -): Promise>> { + bodyLeg: "next" | "park-release", + timeout?: number, + pull: ChildRelayPull = "next", +): Promise { const state = await reloadState(env, runId); - // Fail-loud guard on the one residual divergence from the reducer's pairing: - // `boundSignalForContainerAwait` is container-scoped and faithful ONLY while - // THIS container is the sole step awaiting `name`. If ANY other step is - // already awaiting the SAME author name in this parent run -- another - // section's signal-relay proxy OR a plain author `awaitSignal` gate (which - // reduces with no parkKind) -- the reducer consumes a delivery by - // `state.steps` Map-insertion order across ALL steps (see - // `handleSignalReceived`), not this container's seq order, so the helper - // could mis-bind and relay a payload the reducer delivered to the other - // awaiter. Refuse the topology loudly rather than route a signal to the wrong - // step. Mirrors `hasForeignSameNameAwaiter` on the plain-gate resume path; a - // single section re-awaiting a name sequentially is always one active awaiter - // and stays faithful. Parity with the resume guard is why the parkKind is NOT - // filtered here -- a plain-gate sibling reduces to `"approval"` and would slip - // a signal-relay-only check. - for (const [otherStepId, otherStep] of state.steps) { - if ( - otherStepId !== containerStepId && - otherStep.phase === "awaiting-signal" && - otherStep.awaitingSignal?.name === name - ) { - throw new Error( - `onTrigger ${containerStepId} signal-relay: step ${otherStepId} is ` + - `already awaiting the same signal name ${name}; two steps ` + - `awaiting the same signal name concurrently is not supported`, - ); + // Name mode still assumes this container is the sole name-addressed + // awaiter of `name`: the name replay is container-scoped, while the + // reducer consumes by `state.steps` insertion order. A correlated park + // is not on that queue. Correlation mode addresses by id, so another + // step waiting on the same author name is not a conflict. + if (addressing === "name") { + for (const [otherStepId, otherStep] of state.steps) { + if ( + otherStepId !== containerStepId && + otherStep.phase === "awaiting-signal" && + otherStep.awaitingSignal?.name === name && + otherStep.awaitingSignal.correlationId === undefined + ) { + throw new Error( + `step ${containerStepId} signal-relay: step ${otherStepId} is ` + + `already awaiting the same signal name ${name}; two steps ` + + `awaiting the same signal name concurrently is not supported`, + ); + } } } - // Emit the container's signal-relay await over the body's author name. The - // commit's reducer pass decides (a) vs (b): a signal queued before it - // pre-consumes it (in-flight); otherwise the container is left awaiting. + // A fresh correlated park mints once. Resume of a park that is still + // open reuses the id already on that SignalAwaited and does not call + // this path. The body gate stays the author name; the id is only the + // address on this container's relay. + const correlationId = + addressing === "correlation" ? env.newId("corr") : undefined; const awaited: WorkflowEvent = { kind: "SignalAwaited", seq: state.lastSeq + 1, @@ -2760,6 +3168,15 @@ async function driveContainerSignalRelay( stepId: containerStepId, signalName: name, parkKind: "signal-relay", + ...(correlationId !== undefined ? { correlationId } : {}), + // The deadline is part of the relay await. A parent that resumes before + // it has recorded its own await reads this field and keeps a timed gate + // timed. An untimed relay omits it. + ...(timeout !== undefined + ? { + timeoutAt: new Date(env.clock().getTime() + timeout).toISOString(), + } + : {}), }; const awaitSeq = awaited.seq; await commit(env, runId, awaited); @@ -2769,7 +3186,10 @@ async function driveContainerSignalRelay( if (afterEmit.steps.get(containerStepId)?.phase === "in-flight") { // (a) PRE-CONSUME: the emit consumed a signal queued before it. Bind it // from the log's FIFO pairing and relay -- the body's gate is already - // satisfiable, so there is nothing to await. + // satisfiable, so there is nothing to await. The park-release leg + // checks the child first: `next` can still be holding this park after + // the gate left the await, and delivering then takes the signal path + // or throws once the child channel has stopped. const log = await env.repoStore.read(runId); const bound = boundSignalForContainerAwait( log, @@ -2779,36 +3199,54 @@ async function driveContainerSignalRelay( ); if (bound === undefined) { throw new Error( - `onTrigger ${containerStepId} signal-relay: the emit pre-consumed a ` + + `step ${containerStepId} signal-relay: the emit pre-consumed a ` + `queued ${name} signal but no paired SignalReceived is in the log`, ); } + if (bodyLeg === "park-release") { + const snapshot = await readChildPark(child, containerStepId, name); + if (snapshot.kind !== "open") { + return closeChildRelayWithoutDelivery( + env, + runId, + containerStepId, + child, + name, + snapshot, + pull, + ); + } + } await child.deliverSignal(name, bound.payload, bound.signalId); - return child.next(); + return finishRelayPull(child, pull); } // (b) The container is awaiting. If THIS container is itself a suspendable // child (its env carries an `onSignalPark` sink), surface the relay await up // to its parent so the parent relays a delivery down into this container's // owned channel -- the same way the body's leaf gate surfaced up to here. - // Nesting composes: a loop inside a loop (or inside an onTrigger section) - // proxies the park one layer at a time until it reaches the run whose channel - // has a real upstream. An unset sink means this run owns the outermost - // channel, which is true of two different runs: an addressable top-level run, - // whose caller delivers, and a terminal `childWorkflow` child, which nothing - // can address. Only the first has an upstream, so the no-op below is a - // genuine wait in one case and would be an endless one in the other. - // The second case never reaches here. `parkOnSignalResult` is the sole reader - // of `hasUpstreamSignalResolver` and the sole enforcement point: it refuses - // the body's untimed gate at the park, so the body fails instead of relaying - // a name up and this drive is never entered. That is why no guard is - // repeated here -- one invariant enforced in two places drifts apart, and the - // reader that must keep it is the one holding the durable park. + // Nesting composes: a loop inside a loop, an onTrigger section, or a + // `childWorkflow` proxies the park one layer at a time until it reaches the + // addressable run. An unset sink means this run owns that outermost channel. + // A loop or onTrigger body inherits this run's flag, so its untimed gate + // fails in `parkOnSignalResult` before it is surfaced. A `childWorkflow` + // forces the child's flag true. The spawn step refuses an untimed relay + // before calling this drive when this run's own flag is false, and this + // run writes no suspension for that relay. A timed gate still reaches + // this drive, and its timer can win the race, so the notify below carries + // that timeout for the next layer. A parked `childWorkflow` survives a + // process restart. Resume re-enters this drive for a park the parent never + // recorded, on the park-release leg. A relay the parent already recorded + // is raced from that record and does not emit a second `SignalAwaited`. // Fired on the fresh drive only, after the durable relay await above -- the // reestablish path re-drives `raceContainerSignalRelay` directly and each // level re-establishes from its own durable await, so it needs no re-fire. if (env.onSignalPark !== undefined) { - env.onSignalPark({ runId, name }); + env.onSignalPark({ + runId, + name, + ...(timeout !== undefined ? { timeout } : {}), + }); } // Race the signal's arrival against the body advancing on its own. @@ -2819,43 +3257,344 @@ async function driveContainerSignalRelay( child, name, awaitSeq, + correlationId, abort, + bodyLeg, + pull, ); } /** - * The awaiting arm of a container signal-relay: the container's - * `SignalAwaited(name, "signal-relay")` at `awaitSeq` is durable and the body - * is parked on `name`; race the signal's live arrival against the body - * producing its next event (a gate the body timed out itself) and against - * abort, then relay or retire. Shared by the fresh drive - * ({@link driveContainerSignalRelay}, which just emitted the await) and by - * crash-recovery (`reestablish-signal-relay`, whose re-spawned body re-parks on - * `name` silently, so the await is re-driven from its durable seq without a - * re-emit). Returns the body's NEXT event for the caller's loop. + * The container's relay await is stale: the body left the park, or the child + * run ended, before the signal arrived. Retire it when it is still open. + * A signal that landed and was consumed during the race is relayed + * idempotently instead, and the await is not abandoned. */ -async function raceContainerSignalRelay( +async function abandonRelayIfOpen( env: WorkflowRuntimeEnv, runId: string, containerStepId: string, - child: SuspendableChildHandle, name: string, - awaitSeq: number, - abort: AbortSignal, -): Promise>> { - // `awaitNext` rejects on `raceAbort`; `next()` returns a terminal when the - // body is cancelled, so abort is threaded through the awaitNext leg. - const raceAbort = new AbortController(); - const onOuterAbort = (): void => { - raceAbort.abort(); - }; - bridgeAbort(abort, onOuterAbort); - const pSignal = env.signalChannel - .awaitNext(name, raceAbort.signal) - .then((r) => ({ tag: "signal" as const, r })); - const pNext = child.next().then((ev) => ({ tag: "next" as const, ev })); - let outcome: + correlationId?: string, +): Promise { + const afterRace = await reloadState(env, runId); + const container = afterRace.steps.get(containerStepId); + if ( + container?.phase !== "awaiting-signal" || + container.awaitingSignal?.name !== name + ) { + return false; + } + const retire = await reloadState(env, runId); + const abandoned: WorkflowEvent = { + kind: "SignalAwaitAbandoned", + seq: retire.lastSeq + 1, + at: env.clock().toISOString(), + stepId: containerStepId, + signalName: name, + ...(correlationId !== undefined ? { correlationId } : {}), + }; + await commit(env, runId, abandoned); + await flush(env, runId); + return true; +} + +async function settleStaleSignalRelay( + env: WorkflowRuntimeEnv, + runId: string, + containerStepId: string, + child: SuspendableChildHandle, + name: string, + awaitSeq: number, + correlationId?: string, +): Promise { + if ( + await abandonRelayIfOpen(env, runId, containerStepId, name, correlationId) + ) { + return; + } + const log = await env.repoStore.read(runId); + const bound = boundSignalForContainerAwait( + log, + name, + containerStepId, + awaitSeq, + ); + if (bound !== undefined) { + await child.deliverSignal(name, bound.payload, bound.signalId); + } +} + +type ChildParkSnapshot = Awaited< + ReturnType> +>; + +function childParkReader( + child: SuspendableChildHandle, + containerStepId: string, + name: string, +): NonNullable { + const read = child.readParkRelease; + if (read === undefined) { + throw new Error( + `step ${containerStepId} cannot relay signal ${name}: the child ` + + `handle does not read a park release, so a signal already waiting ` + + `would be relayed after the gate moved on`, + ); + } + return read; +} + +async function readChildPark( + child: SuspendableChildHandle, + containerStepId: string, + name: string, +): Promise { + const read = childParkReader(child, containerStepId, name); + return read(name); +} + +/** + * The named child gate is no longer open. Retire the parent's relay await + * when it is still open, and do not deliver. A `SignalReceived` already in + * the parent log stays there: unwriting it is not this layer's job, and + * delivering it would take a path the gate has left. + */ +async function closeChildRelayWithoutDelivery( + env: WorkflowRuntimeEnv, + runId: string, + containerStepId: string, + child: SuspendableChildHandle, + name: string, + read: Exclude, + pull: ChildRelayPull, +): Promise { + await abandonRelayIfOpen(env, runId, containerStepId, name); + if (read.kind === "terminal") { + return { kind: "terminal", terminalStatus: read.terminalStatus }; + } + return finishRelayPull(child, pull); +} + +type ChildParkRelease = Awaited< + ReturnType> +>; + +/** + * Child-workflow arm of a container signal-relay. The spawn step awaits one + * signal, and the child may already have queued a sibling park. Racing + * `next` would consume that sibling and retire this relay while the child + * gate is still parked. Watch the named park instead. A sibling does not + * resolve the watch. The queue is read only after this park has a decision, + * has left the await, or the child run has ended. + */ +async function raceChildSignalRelease( + env: WorkflowRuntimeEnv, + runId: string, + containerStepId: string, + child: SuspendableChildHandle, + name: string, + abort: AbortSignal, + pull: ChildRelayPull, +): Promise { + const watch = child.waitForParkRelease; + if (watch === undefined) { + throw new Error( + `step ${containerStepId} cannot relay signal ${name}: the child ` + + `handle does not watch a park release, so a sibling park would look ` + + `like this one ended`, + ); + } + // A signal already on the channel resolves before the watch's first + // read. Snapshot first, and do not start the wait when the gate has + // already moved on, so that signal stays queued instead of being recorded + // as relayed. + const before = await readChildPark(child, containerStepId, name); + if (before.kind !== "open") { + return closeChildRelayWithoutDelivery( + env, + runId, + containerStepId, + child, + name, + before, + pull, + ); + } + const raceAbort = new AbortController(); + const detachRaceAbort = bridgeAbort(abort, () => { + raceAbort.abort(); + }); + const signalWait = env.signalChannel + .awaitNext(name, raceAbort.signal) + .then((r) => ({ tag: "signal" as const, r })); + const releaseWait = watch(name, raceAbort.signal).then((rel) => ({ + tag: "release" as const, + rel, + })); + // The channel tails from the moment `awaitNext` is called, so a signal + // committed before that wait is only in the log. Both waits are already + // started: a signal or a release during this read is still observed, and + // a signal that arrived earlier is taken here. That event is already + // committed. Relay it while the child gate is open, and do not append a + // second `SignalReceived`. A gate that has left the park is not given it. + let logged: { payload: unknown; signalId: string } | undefined; + try { + const log = await env.repoStore.read(runId); + logged = signalBoundToAwait(runId, log, containerStepId, name); + } catch (cause) { + raceAbort.abort(); + detachRaceAbort(); + await signalWait.catch(() => undefined); + await releaseWait.catch(() => undefined); + throw cause; + } + if (logged !== undefined) { + // Stop the watch before the delivery. The child's own SignalReceived + // would otherwise look like the gate timing out. + raceAbort.abort(); + detachRaceAbort(); + await signalWait.catch(() => undefined); + await releaseWait.catch(() => undefined); + const again = await readChildPark(child, containerStepId, name); + if (again.kind !== "open") { + return closeChildRelayWithoutDelivery( + env, + runId, + containerStepId, + child, + name, + again, + pull, + ); + } + await child.deliverSignal(name, logged.payload, logged.signalId); + return finishRelayPull(child, pull); + } + let outcome: | { tag: "signal"; r: { payload: unknown; signalId: string } } + | { tag: "release"; rel: ChildParkRelease }; + try { + outcome = await Promise.race([signalWait, releaseWait]); + } catch (cause) { + raceAbort.abort(); + await signalWait.catch(() => undefined); + await releaseWait.catch(() => undefined); + throw cause; + } finally { + detachRaceAbort(); + } + + if (outcome.tag === "signal") { + // Stop the watch before the delivery. The child's own SignalReceived + // would otherwise look like the gate timing out. Re-read before the + // commit: the gate can leave the await in the window after a snapshot + // that said it was open, and the channel has already taken the signal. + raceAbort.abort(); + await releaseWait.catch(() => undefined); + const again = await readChildPark(child, containerStepId, name); + if (again.kind !== "open") { + return closeChildRelayWithoutDelivery( + env, + runId, + containerStepId, + child, + name, + again, + pull, + ); + } + const beforeCommit = await reloadState(env, runId); + const received: WorkflowEvent = { + kind: "SignalReceived", + seq: beforeCommit.lastSeq + 1, + at: env.clock().toISOString(), + signalName: name, + signalId: outcome.r.signalId, + payload: outcome.r.payload, + }; + await commit(env, runId, received); + await flush(env, runId); + await child.deliverSignal(name, outcome.r.payload, outcome.r.signalId); + return finishRelayPull(child, pull); + } + + raceAbort.abort(); + await signalWait.catch(() => undefined); + // The watch won, so this gate has left the await. A host delivery can + // already have paired `SignalReceived` into the parent log while + // `awaitNext` is still pending. Relaying that signal takes the path + // the gate has left, and a settled child channel throws on delivery. + return closeChildRelayWithoutDelivery( + env, + runId, + containerStepId, + child, + name, + outcome.rel, + pull, + ); +} + +/** + * The awaiting arm of a container signal-relay: the container's + * `SignalAwaited(name, "signal-relay")` at `awaitSeq` is durable and the body + * is parked on `name`; race the signal's live arrival against the body + * leg and against abort, then relay or retire. Shared by the fresh drive + * ({@link driveContainerSignalRelay}, which just emitted the await) and by + * crash-recovery (`reestablish-signal-relay`, whose re-spawned body re-parks on + * `name` silently, so the await is re-driven from its durable seq without a + * re-emit). `"next"` is the loop and onTrigger leg and returns the body's + * next event. `"park-release"` is the childWorkflow leg. + */ +async function raceContainerSignalRelay( + env: WorkflowRuntimeEnv, + runId: string, + containerStepId: string, + child: SuspendableChildHandle, + name: string, + awaitSeq: number, + correlationId: string | undefined, + abort: AbortSignal, + bodyLeg: "next" | "park-release", + pull: ChildRelayPull = "next", +): Promise { + if (bodyLeg === "park-release") { + return raceChildSignalRelease( + env, + runId, + containerStepId, + child, + name, + abort, + pull, + ); + } + if (pull === "hold") { + throw new Error( + `step ${containerStepId} signal-relay on ${name} cannot hold the queue on the next leg`, + ); + } + // `awaitNext` rejects on `raceAbort`; `next()` returns a terminal when the + // body is cancelled, so abort is threaded through the awaitNext leg. + const raceAbort = new AbortController(); + const onOuterAbort = (): void => { + raceAbort.abort(); + }; + bridgeAbort(abort, onOuterAbort); + const waited = + correlationId === undefined + ? awaitSignalFor(env, runId, containerStepId, name, raceAbort.signal) + : awaitSignalFor(env, runId, containerStepId, name, raceAbort.signal, { + correlationId, + }); + const pSignal = waited.then((r) => ({ tag: "signal" as const, r })); + const pNext = child.next().then((ev) => ({ tag: "next" as const, ev })); + let outcome: + | { + tag: "signal"; + r: { payload: unknown; signalId: string; correlationId?: string }; + } | { tag: "next"; ev: Awaited> }; try { outcome = await Promise.race([pSignal, pNext]); @@ -2870,10 +3609,10 @@ async function raceContainerSignalRelay( } if (outcome.tag === "signal") { - // The awaited signal arrived live. Commit its SignalReceived -- mirroring - // parkOnSignal's awaiter-commits contract, so the container log carries it - // in every host; a production delivery that also committed it dedups by - // signalId -- then relay it into the body and take the body's next event. + // The awaited signal arrived live. Commit its SignalReceived so a + // channel that did not write the blob still records it. A host + // delivery already in the log is not appended again. Then relay it + // into the body and take the body's next event. const before = await reloadState(env, runId); const received: WorkflowEvent = { kind: "SignalReceived", @@ -2882,6 +3621,9 @@ async function raceContainerSignalRelay( signalName: name, signalId: outcome.r.signalId, payload: outcome.r.payload, + ...(outcome.r.correlationId !== undefined + ? { correlationId: outcome.r.correlationId } + : {}), }; await commit(env, runId, received); await flush(env, runId); @@ -2895,40 +3637,15 @@ async function raceContainerSignalRelay( // consumed during the race. raceAbort.abort(); await pSignal.catch(() => undefined); - const afterRace = await reloadState(env, runId); - const container = afterRace.steps.get(containerStepId); - if ( - container?.phase === "awaiting-signal" && - container.awaitingSignal?.name === name - ) { - // No signal was consumed: retire the stale relay await so the reducer stops - // treating the container as awaiting this name. - const retire = await reloadState(env, runId); - const abandoned: WorkflowEvent = { - kind: "SignalAwaitAbandoned", - seq: retire.lastSeq + 1, - at: env.clock().toISOString(), - stepId: containerStepId, - signalName: name, - }; - await commit(env, runId, abandoned); - await flush(env, runId); - } else { - // A signal landed and was consumed during the race (its SignalReceived - // reduced the container off awaiting-signal). Relay it idempotently: the - // body has moved past its gate, but its run-lifetime dedup on the ORIGINAL - // signalId absorbs a relay it no longer needs. Do NOT abandon. - const log = await env.repoStore.read(runId); - const bound = boundSignalForContainerAwait( - log, - name, - containerStepId, - awaitSeq, - ); - if (bound !== undefined) { - await child.deliverSignal(name, bound.payload, bound.signalId); - } - } + await settleStaleSignalRelay( + env, + runId, + containerStepId, + child, + name, + awaitSeq, + correlationId, + ); return outcome.ev; } @@ -2972,7 +3689,9 @@ type OnTriggerResumePlan = * (terminal-is-final). Single source for the default so the steady-state drive * loop and the resume planner cannot drift. */ -function bodyFailurePolicyOf(primitive: OnTriggerPrimitive): BodyFailurePolicy { +function bodyFailurePolicyOf(primitive: { + readonly onBodyFailure?: BodyFailurePolicy; +}): BodyFailurePolicy { return primitive.onBodyFailure ?? "end"; } @@ -2986,7 +3705,7 @@ function bodyFailurePolicyOf(primitive: OnTriggerPrimitive): BodyFailurePolicy { * drive-fresh on `author`; the onTrigger planner refuses on either, since it has * no fresh-relay drive for a body still awaiting a signal. */ -function bodyParkedSignals(childState: RunState): { +export function bodyParkedSignals(childState: RunState): { author: string[]; controlPlane: string[]; } { @@ -3006,24 +3725,123 @@ function bodyParkedSignals(childState: RunState): { return { author, controlPlane }; } +/** Decimal event index with no leading zero. `section__01` is not index 1. */ +const CANONICAL_EVENT_INDEX = /^(0|[1-9][0-9]*)$/; + +function canonicalEventIndex(text: string): number | undefined { + if (!CANONICAL_EVENT_INDEX.test(text)) return undefined; + return Number(text); +} + +/** + * Event index carried by a section body run id. The minted form + * (`sectionBodyRunId`) is tried before the legacy `__` + * form, so a parent run id that itself begins with `__` still + * parses as the minted id. + */ +function sectionBodyEventIndex( + parentRunId: string, + sectionId: string, + childRunId: string, +): number | undefined { + const mintedPrefix = `${parentRunId}__${sectionId}__`; + if (childRunId.startsWith(mintedPrefix)) { + return canonicalEventIndex(childRunId.slice(mintedPrefix.length)); + } + const legacyPrefix = `${sectionId}__`; + if (childRunId.startsWith(legacyPrefix)) { + return canonicalEventIndex(childRunId.slice(legacyPrefix.length)); + } + return undefined; +} + +/** + * The body run id to drive for one event of this section. A child already + * recorded under the minted key or the legacy key is returned as that key. + * Neither key mints `sectionBodyRunId`. Both keys is a corrupt log: throw + * before any spawn. `spawnedBy` keeps a same-shaped id another step recorded + * from counting as this section's body. + */ +function recordedSectionBodyRunId( + parentRunId: string, + sectionId: string, + eventIndex: number, + children: RunState["children"], +): string { + const minted = sectionBodyRunId(parentRunId, sectionId, eventIndex); + const legacy = `${sectionId}__${String(eventIndex)}`; + const hasMinted = children.get(minted)?.spawnedBy === sectionId; + const hasLegacy = children.get(legacy)?.spawnedBy === sectionId; + if (hasMinted && hasLegacy) { + throw new Error( + `onTrigger ${sectionId} event ${String(eventIndex)} records both ${minted} and ${legacy}`, + ); + } + if (hasMinted) return minted; + if (hasLegacy) return legacy; + return minted; +} + +/** + * Highest event index this section has spawned, or `undefined` when it has + * spawned nothing (the crash after the container `StepStarted` and before + * the first `ChildSpawned`). A child this section spawned whose id is neither + * body-run form throws: that log is not a fresh section. + */ +function highestSectionEventIndex( + parentRunId: string, + sectionId: string, + children: RunState["children"], +): number | undefined { + let eventIndex: number | undefined; + for (const [childRunId, child] of children) { + if (child.spawnedBy !== sectionId) continue; + const parsed = sectionBodyEventIndex(parentRunId, sectionId, childRunId); + if (parsed === undefined) { + throw new Error( + `onTrigger ${sectionId} child ${childRunId} is not a section body run id`, + ); + } + if (eventIndex === undefined || parsed > eventIndex) eventIndex = parsed; + } + return eventIndex; +} + async function planOnTriggerResume( env: WorkflowRuntimeEnv, primitive: OnTriggerPrimitive, state: RunState, log: readonly WorkflowEvent[], ): Promise { - const prefix = `${primitive.id}__`; - let eventIndex = -1; - for (const childRunId of state.children.keys()) { - if (!childRunId.startsWith(prefix)) continue; - const parsed = Number.parseInt(childRunId.slice(prefix.length), 10); - if (Number.isInteger(parsed) && parsed > eventIndex) eventIndex = parsed; - } - if (eventIndex === -1) { + return finishOnTriggerResume(primitive, state, log, (childRunId) => + reloadState(env, childRunId), + ); +} + +async function finishOnTriggerResume( + primitive: { + readonly id: string; + readonly onBodyFailure?: BodyFailurePolicy; + }, + state: RunState, + log: readonly WorkflowEvent[], + loadChild: (childRunId: string) => Promise, +): Promise { + const eventIndex = highestSectionEventIndex( + state.runId, + primitive.id, + state.children, + ); + if (eventIndex === undefined) { // The container `StepStarted` is durable but no body was ever spawned. return { kind: "fresh" }; } - const childRunId = `${prefix}${String(eventIndex)}`; + const childRunId = recordedSectionBodyRunId( + state.runId, + primitive.id, + eventIndex, + state.children, + ); const child = state.children.get(childRunId); if (child === undefined) { throw new Error( @@ -3182,16 +4000,17 @@ async function planOnTriggerResume( // forever. onTrigger has no fresh-relay drive (loop's `signal-relay-drive- // fresh` is out of scope here), so keep failing loud. Any body NOT awaiting a // signal re-adopts and its OWN resume classifier decides its terminal: a - // `sleep` resumes via `isResumableSleepStep`; a mid-flight `childWorkflow` or - // `map` REJECTS, and the seam re-throws that rejection as a loud section - // failure -- never a silently-tolerated terminal (see the tolerate regression - // test). A crashed agent step instead SETTLES a terminal `StepFailed`: under + // `sleep` resumes via `isResumableSleepStep`. A mid-flight `childWorkflow` + // reattaches from its log. A mid-flight `map` REJECTS, and the seam + // re-throws that rejection as a loud section failure -- never a + // silently-tolerated terminal (see the tolerate regression test). A + // crashed agent step instead SETTLES a terminal `StepFailed`: under // the default `end` policy this fails the section as the pre-branch throw did, // and under `tolerate` the section absorbs the crashed body and re-arms. That // last case is the one genuine behavior change here, and it is the intended // reading of `tolerate` -- a real body failure it is meant to swallow, not a // hang or a lost run. - const childState = await reloadState(env, childRunId); + const childState = await loadChild(childRunId); const { author, controlPlane } = bodyParkedSignals(childState); const parkedSignals = [...author, ...controlPlane]; if (parkedSignals.length > 0) { @@ -3229,6 +4048,24 @@ async function planLoopResume( state: RunState, log: readonly WorkflowEvent[], iteration: number, +): Promise { + return finishLoopResume( + primitive, + runId, + state, + log, + iteration, + (childRunId) => reloadState(env, childRunId), + ); +} + +async function finishLoopResume( + primitive: { readonly id: string }, + runId: string, + state: RunState, + log: readonly WorkflowEvent[], + iteration: number, + loadChild: (childRunId: string) => Promise, ): Promise { const childRunId = loopBodyRunId(runId, primitive.id, iteration); const child = state.children.get(childRunId); @@ -3309,7 +4146,7 @@ async function planLoopResume( // and drive the container relay FRESH. (On an inconsistent store the child // log is gone, so no gate is found and the forward drive re-runs the // iteration -- the existing inconsistent-store behavior.) - const childState = await reloadState(env, childRunId); + const childState = await loadChild(childRunId); const { author: authorAwaits } = bodyParkedSignals(childState); if (authorAwaits.length > 1) { throw new Error( @@ -3320,11 +4157,132 @@ async function planLoopResume( } const parkedName = authorAwaits[0]; if (parkedName !== undefined) { - return { kind: "signal-relay-drive-fresh", name: parkedName }; + let timeoutAt: string | undefined; + for (const step of childState.steps.values()) { + if ( + step.phase === "awaiting-signal" && + step.awaitingSignal?.name === parkedName + ) { + timeoutAt = step.awaitingSignal.timeoutAt; + break; + } + } + return { + kind: "signal-relay-drive-fresh", + name: parkedName, + ...(timeoutAt !== undefined ? { timeoutAt } : {}), + }; } return undefined; } +/** + * Whether a crash-recovered container can be re-attached, judged by the same + * planners the runtime drives. `rejected` is a planner throw (caught here so + * a caller can release instead of letting it escape). `reenter` is a child + * the forward drive re-adopts that is not itself a park. A drive-fresh loop + * park with a control-plane signal beside the author signal is `rejected` + * even though the runtime planner would return the author name: more than + * the author gate is waiting, and that is not re-attached. + */ +export type OccurrenceReattach = + | { readonly kind: "parked"; readonly childRunId?: string } + | { readonly kind: "reenter"; readonly childRunId: string } + | { readonly kind: "open" } + | { readonly kind: "rejected" }; + +export async function loopOccurrenceReattach(args: { + readonly loopId: string; + readonly runId: string; + readonly iteration: number; + readonly state: RunState; + readonly log: readonly WorkflowEvent[]; + readonly childState: RunState; +}): Promise { + const childRunId = loopBodyRunId(args.runId, args.loopId, args.iteration); + try { + const plan = await finishLoopResume( + { id: args.loopId }, + args.runId, + args.state, + args.log, + args.iteration, + async () => args.childState, + ); + if (plan === undefined) { + const child = args.state.children.get(childRunId); + if (child === undefined || child.terminalStatus !== undefined) { + return { kind: "open" }; + } + return { kind: "reenter", childRunId }; + } + if (plan.kind === "signal-relay-drive-fresh") { + const { author, controlPlane } = bodyParkedSignals(args.childState); + if (author.length !== 1 || controlPlane.length > 0) { + return { kind: "rejected" }; + } + } + return { kind: "parked", childRunId }; + } catch { + return { kind: "rejected" }; + } +} + +export async function onTriggerOccurrenceReattach(args: { + readonly sectionId: string; + readonly onBodyFailure?: BodyFailurePolicy; + readonly state: RunState; + readonly log: readonly WorkflowEvent[]; + readonly childState: RunState; +}): Promise { + try { + const plan = await finishOnTriggerResume( + { + id: args.sectionId, + ...(args.onBodyFailure !== undefined + ? { onBodyFailure: args.onBodyFailure } + : {}), + }, + args.state, + args.log, + async () => args.childState, + ); + // A recorded body keeps the id the log stored. `sectionBodyRunId` + // mints the parent-prefixed form, which is an empty log when the + // child was committed as `__`. + const recordedChild = (eventIndex: number): string => + recordedSectionBodyRunId( + args.state.runId, + args.sectionId, + eventIndex, + args.state.children, + ); + switch (plan.kind) { + case "reawait-input": + return { kind: "parked" }; + case "reestablish-approval": + case "reestablish-signal-relay": + case "relay-grant": + case "relay-signal-grant": + return { + kind: "parked", + childRunId: recordedChild(plan.eventIndex), + }; + case "readopt-in-flight-body": + return { + kind: "reenter", + childRunId: recordedChild(plan.eventIndex), + }; + case "fresh": + case "advance-with-input": + case "terminal-is-final": + return { kind: "open" }; + } + } catch { + return { kind: "rejected" }; + } +} + /** * Recover the correlation and delivered decision of the container's most recent * approval park from the durable log. Used only for the narrow crash window @@ -3479,9 +4437,9 @@ function recoverDeliveredSignalRelay( * the wrong payload. * * Two reducer behaviors the pairing depends on, both replayed here: - * - dedup by `signalId`: a redelivered `SignalReceived` is a no-op in the - * reducer (`observedSignalIds`) yet still lands in the log, so the replay - * must skip an already-seen id rather than double-advance the FIFO. + * - dedup by `signalId`: the reducer skips an id already in + * `observedSignalIds`. Replay does the same. A second blob is not + * required; a log that already has one still skips that id. * - retire an abandoned awaiter: `SignalAwaitAbandoned` drops the awaiter, so * a later signal queues rather than pairing with the retired await; the * replay pops the container's active waiter on abandon to match. The @@ -3492,13 +4450,14 @@ function recoverDeliveredSignalRelay( * reconstructs the reducer's queue/waiters from `emptyState`, so a windowed * suffix would start mid-stream and mis-pair; a non-full log fails loud below. * - * ASSUMPTION: a UNIQUE container awaiter per name in the parent run. The - * pairing is container-scoped (it filters `SignalAwaited`/`SignalAwaitAbandoned` - * to `containerStepId`), which matches the reducer ONLY while this container is - * the sole step awaiting the name. Two sections concurrently awaiting the same - * name would let the reducer consume by `state.steps` Map-insertion order - * across all steps; `driveContainerSignalRelay` refuses that topology loudly at - * the proxy-park, so this helper never sees it. + * ASSUMPTION for a name-addressed await: a UNIQUE container awaiter per name + * in the parent run. The name replay filters `SignalAwaited` / + * `SignalAwaitAbandoned` to `containerStepId` and ignores events that carry + * `correlationId`, which matches the reducer only while this container is the + * sole name-addressed step awaiting the name. A correlated park is not on + * that queue. An await that itself carries `correlationId` binds by that id + * instead, and a name mismatch against the park holding the id dedups without + * queueing, matching `handleSignalReceived`. */ // Exported for direct unit tests: the reducer-FIFO-replay binding is the // correctness crux of the signal-relay pass, and its correction cases (dedup, @@ -3518,14 +4477,27 @@ export function boundSignalForContainerAwait( `mis-pair the reducer's FIFO)`, ); } + const correlationId = correlationIdAt(log, awaitSeq); + if (correlationId !== undefined) { + return bindCorrelatedContainerAwait( + log, + name, + containerStepId, + awaitSeq, + correlationId, + ); + } const queue: { payload: unknown; signalId: string }[] = []; const waiters: number[] = []; const pairing = new Map(); const observed = new Set(); for (const event of log) { - if (event.kind === "SignalReceived" && event.signalName === name) { + if (event.kind === "SignalReceived") { if (observed.has(event.signalId)) continue; observed.add(event.signalId); + if (event.correlationId !== undefined || event.signalName !== name) { + continue; + } const waiter = waiters.shift(); if (waiter !== undefined) { pairing.set(waiter, { @@ -3537,6 +4509,7 @@ export function boundSignalForContainerAwait( } } else if ( event.kind === "SignalAwaited" && + event.correlationId === undefined && event.signalName === name && event.stepId === containerStepId ) { @@ -3548,6 +4521,7 @@ export function boundSignalForContainerAwait( } } else if ( event.kind === "SignalAwaitAbandoned" && + event.correlationId === undefined && event.signalName === name && event.stepId === containerStepId ) { @@ -3557,6 +4531,151 @@ export function boundSignalForContainerAwait( return pairing.get(awaitSeq); } +function correlationIdAt( + log: readonly WorkflowEvent[], + awaitSeq: number, +): string | undefined { + for (const event of log) { + if (event.kind === "SignalAwaited" && event.seq === awaitSeq) { + return event.correlationId; + } + } + return undefined; +} + +/** + * Bind a container await that carries `correlationId`. Dedup is still + * run-wide. A delivery for this id pairs only when `signalName` matches; + * a delivery for this id against a park held under a different name is + * deduped and not queued. Events for other ids do not enter this queue. + */ +function bindCorrelatedContainerAwait( + log: readonly WorkflowEvent[], + name: string, + containerStepId: string, + awaitSeq: number, + correlationId: string, +): { payload: unknown; signalId: string } | undefined { + const queue: { payload: unknown; signalId: string; signalName: string }[] = + []; + const waiters: { seq: number; signalName: string; stepId: string }[] = []; + const pairing = new Map(); + const observed = new Set(); + for (const event of log) { + if (event.kind === "SignalReceived") { + if (observed.has(event.signalId)) continue; + observed.add(event.signalId); + if (event.correlationId !== correlationId) continue; + const waiterIndex = waiters.findIndex( + (waiter) => waiter.signalName === event.signalName, + ); + const waiter = waiterIndex >= 0 ? waiters[waiterIndex] : undefined; + if (waiter !== undefined) { + waiters.splice(waiterIndex, 1); + pairing.set(waiter.seq, { + payload: event.payload, + signalId: event.signalId, + }); + } else if (waiters.length === 0) { + queue.push({ + payload: event.payload, + signalId: event.signalId, + signalName: event.signalName, + }); + } + } else if ( + event.kind === "SignalAwaited" && + event.correlationId === correlationId + ) { + const headIndex = queue.findIndex( + (item) => item.signalName === event.signalName, + ); + const head = headIndex >= 0 ? queue[headIndex] : undefined; + if (head !== undefined) { + queue.splice(headIndex, 1); + if (event.stepId === containerStepId && event.signalName === name) { + pairing.set(event.seq, { + payload: head.payload, + signalId: head.signalId, + }); + } + } else { + waiters.push({ + seq: event.seq, + signalName: event.signalName, + stepId: event.stepId, + }); + } + } else if ( + event.kind === "SignalAwaitAbandoned" && + (event.correlationId === undefined || + event.correlationId === correlationId) + ) { + const waiterIndex = waiters.findIndex( + (waiter) => + waiter.stepId === event.stepId && + waiter.signalName === event.signalName, + ); + if (waiterIndex >= 0) waiters.splice(waiterIndex, 1); + } + } + return pairing.get(awaitSeq); +} + +/** + * The loop's iteration-failure policy, defaulting an absent field to `"end"`. + * One read for the replay walk and the post-drive check so they cannot drift. + * Absent stays off the primitive; this is the only place that fills it in. + */ +function iterationFailurePolicyOf(primitive: LoopPrimitive): BodyFailurePolicy { + return primitive.onIterationFailure ?? "end"; +} + +/** + * One iteration of the replay walk, without `while` or `carry`. `runLoop` + * still threads input on `completed`. A classifier uses the same cursor to + * name the frontier child, including a tolerated failure it must skip. + */ +export type LoopIterationCursor = + | { readonly kind: "completed" } + | { readonly kind: "tolerated" } + | { + readonly kind: "terminal"; + readonly terminalStatus: "failed" | "cancelled"; + } + | { readonly kind: "frontier" }; + +export function loopIterationCursor( + state: RunState, + runId: string, + loopId: string, + iteration: number, + policy: BodyFailurePolicy, +): LoopIterationCursor { + if (isIterationDone(state, runId, loopId, iteration)) { + return { kind: "completed" }; + } + const child = state.children.get(loopBodyRunId(runId, loopId, iteration)); + const step = state.steps.get(scopedStepId(loopId, iteration)); + if ( + child?.terminalStatus === "failed" && + child.abortedTeardown !== true && + policy === "tolerate" && + step?.phase !== "completed" + ) { + return { kind: "tolerated" }; + } + const terminalStatus = child?.terminalStatus; + if ( + terminalStatus === "cancelled" || + (terminalStatus === "failed" && + (child?.abortedTeardown === true || policy !== "tolerate")) + ) { + return { kind: "terminal", terminalStatus }; + } + return { kind: "frontier" }; +} + function isIterationDone( state: RunState, runId: string, @@ -3565,7 +4684,10 @@ function isIterationDone( ): boolean { const child = state.children.get(loopBodyRunId(runId, loopId, iteration)); const step = state.steps.get(scopedStepId(loopId, iteration)); - return child?.terminalStatus !== undefined && step?.phase === "completed"; + // Completed only. Any other terminal child plus a completed scoped step is + // the crash window where StepCompleted landed before the failure throw, and + // replaying it as a success would continue a loop that had already failed. + return child?.terminalStatus === "completed" && step?.phase === "completed"; } function findStepInputRef( @@ -3727,9 +4849,9 @@ async function waitForTimer( // the scheduler-committed `TimerFired`. Flush the buffered segment // (the `TimerSet` -- and, on the retry path, the preceding // `StepFailed`/`AttemptScheduled`) to durable storage BEFORE - // computing `subscribeFromSeq` and subscribing, so the out-of-process - // scheduler can tail the durable `TimerSet` and a crash-while-waiting - // leaves a resumable pre-suspension log. + // computing `subscribeFromSeq` and subscribing, so the scheduler can + // tail the durable `TimerSet` and a crash-while-waiting leaves a + // resumable pre-suspension log. await flush(env, runId); const subscribeFromSeq = (await reloadState(env, runId)).lastSeq + 1; const ac = new AbortController(); @@ -4119,9 +5241,15 @@ export function reconstructGateOutcome( const observed = new Set(); let awaiting = false; for (const event of log) { - if (event.kind === "SignalReceived" && event.signalName === signalName) { + if (event.kind === "SignalReceived") { if (observed.has(event.signalId)) continue; observed.add(event.signalId); + if ( + event.correlationId !== undefined || + event.signalName !== signalName + ) { + continue; + } if (awaiting) { return { timedOut: false, @@ -4132,6 +5260,7 @@ export function reconstructGateOutcome( queue.push({ payload: event.payload, signalId: event.signalId }); } else if ( event.kind === "SignalAwaited" && + event.correlationId === undefined && event.signalName === signalName && event.stepId === selfStepId ) { @@ -4175,18 +5304,20 @@ function gateTimerId( } /** - * Whether any `awaitSignal` gate OTHER than `selfStepId` awaited `signalName` - * anywhere in the run. `reconstructGateOutcome` replays the reducer FIFO scoped - * to a SINGLE awaiter of the name, so a second same-name awaiter -- even one - * that already COMPLETED -- breaks that assumption: the log can no longer say - * which gate consumed which delivery. The in-flight short-circuit refuses that - * topology rather than risk binding a payload to the wrong gate; only a run - * where `selfStepId` is the sole awaiter of the name is provably unambiguous. A - * completed same-name sibling is the case a phase-scoped in-flight count would - * miss, so the predicate keys on the durable `SignalAwaited` marker, which - * outlives the sibling's completion. + * Whether any name-addressed `awaitSignal` gate OTHER than `selfStepId` + * awaited `signalName` anywhere in the run. A `SignalAwaited` that carries + * `correlationId` is not on the name queue, so it is not a foreign awaiter. + * `reconstructGateOutcome` replays the reducer name FIFO scoped to a SINGLE + * name-addressed awaiter of the name, so a second same-name awaiter -- even + * one that already COMPLETED -- breaks that assumption: the log can no longer + * say which gate consumed which delivery. The in-flight short-circuit refuses + * that topology rather than risk binding a payload to the wrong gate; only a + * run where `selfStepId` is the sole name-addressed awaiter of the name is + * provably unambiguous. A completed same-name sibling is the case a + * phase-scoped in-flight count would miss, so the predicate keys on the + * durable `SignalAwaited` marker, which outlives the sibling's completion. */ -function hasForeignSameNameAwaiter( +export function hasForeignSameNameAwaiter( log: readonly WorkflowEvent[], signalName: string, selfStepId: string, @@ -4194,6 +5325,7 @@ function hasForeignSameNameAwaiter( for (const event of log) { if ( event.kind === "SignalAwaited" && + event.correlationId === undefined && event.signalName === signalName && event.stepId !== selfStepId ) { @@ -4241,6 +5373,88 @@ function findAwaitedSignalNameForStep( return step.awaitingSignal?.name; } +/** + * The signal the reducer gave `stepId`'s latest await on `signalName`, or + * `undefined` while that await is unanswered. Folds the log through the reducer + * itself, so the binding cannot drift from the run's own state. + */ +function signalBoundToAwait( + runId: string, + log: readonly WorkflowEvent[], + stepId: string, + signalName: string, +): { payload: unknown; signalId: string } | undefined { + let state = emptyState(runId); + let bound: { payload: unknown; signalId: string } | undefined; + for (const event of log) { + const queued = state.unconsumedSignals.get(signalName)?.[0]; + const wasAwaiting = + findAwaitedSignalNameForStep(state, stepId) === signalName; + state = applyEvent(state, event); + const awaiting = findAwaitedSignalNameForStep(state, stepId) === signalName; + if ( + event.kind === "SignalAwaited" && + event.stepId === stepId && + event.signalName === signalName + ) { + bound = + awaiting || queued === undefined + ? undefined + : { payload: queued.payload, signalId: queued.id }; + } else if (event.kind === "SignalReceived" && wasAwaiting && !awaiting) { + bound = { payload: event.payload, signalId: event.signalId }; + } + } + return bound; +} + +/** + * Wait on the signal channel for the signal answering `stepId`'s await. The + * channel only sees signals delivered after `awaitNext` is called, but the + * caller read the step as awaiting before that; a signal delivered in between + * is already in the log, so it is taken from there. `onWaiting` runs only + * when the log has no such signal, after the subscriber exists and before + * the wait blocks. + */ +async function awaitSignalFor( + env: WorkflowRuntimeEnv, + runId: string, + stepId: string, + signalName: string, + abort: AbortSignal, + opts?: { + correlationId?: string; + onWaiting?: () => void; + }, +): Promise<{ payload: unknown; signalId: string; correlationId?: string }> { + const correlationId = opts?.correlationId; + const wait = new AbortController(); + const detach = bridgeAbort(abort, () => { + wait.abort(); + }); + const waiting = + correlationId === undefined + ? env.signalChannel.awaitNext(signalName, wait.signal) + : env.signalChannel.awaitNext(signalName, wait.signal, correlationId); + const waitSettled = waiting.then( + () => undefined, + () => undefined, + ); + try { + const log = await env.repoStore.read(runId); + const bound = signalBoundToAwait(runId, log, stepId, signalName); + if (bound !== undefined) { + return correlationId === undefined ? bound : { ...bound, correlationId }; + } + opts?.onWaiting?.(); + return await waiting; + } finally { + detach(); + wait.abort(); + await waitSettled; + } +} + /** * Signal-park core, shared by `runAwaitSignal` and the step-suspend arm * in `runStep`. Given a step already marked started, it emits the @@ -4285,6 +5499,15 @@ type ParkResult = { timedOut: false; payload: unknown } | { timedOut: true }; * here is a wiring bug, so surface it loudly rather than mis-route a control * decision. */ +function unanswerableParkMessage(stepId: string, name: string): string { + return ( + `step ${stepId} waits on ${name} with no timeout, but ` + + `nothing outside this run can deliver it. This run has no upstream ` + + `resolver, so an untimed wait cannot be answered. Give the gate a ` + + `timeout, or run it where a parent relays the park` + ); +} + function requireSignalPayload(result: ParkResult): unknown { if (result.timedOut) { throw new Error( @@ -4312,33 +5535,32 @@ async function parkOnSignalResult( * the durable `SignalAwaited` without re-deriving the kind. */ parkKind?: ControlParkKind; + /** + * Set when this park is the run's one deployment input: a trigger-budget + * re-arm, or a childWorkflow relay of that park. An onTrigger section's + * event re-arm is a separate channel, and several of those stay open + * together. `sectionInputStepIds` names those sections so an open + * section park does not hold this channel. + */ + claimsRunInputChannel?: { + sectionInputStepIds: ReadonlySet; + }; }, state: ReturnType, abort: AbortSignal, ): Promise { // An untimed park waits for a signal from outside this run, so it can only - // be answered where something upstream can deliver one. A terminal child has - // no address of its own and no container relaying decisions down to it, so - // the wait would never end: the child parks, the spawner waits on a terminal - // that never comes, and an approval nobody can see holds the whole tree. - // - // A body nested inside such a child inherits the same answer, so this fires - // for a gate at any depth beneath the boundary rather than only a direct - // one. The advice below must therefore not send an author to a loop or - // onTrigger body: that is where they already are. + // be answered where something upstream can deliver one. This env has no + // upstream resolver, so the wait would never end. The flag is the sole + // reason: a child whose parent relays sets it true, and a run that sets it + // false refuses here whether or not a parent exists. // // Refuse before anything durable is written, so the run fails at the step // that asked for the impossible and no suspension is recorded for a - // correlation the control plane will never hear about. A timed gate is - // exempt -- its own timer resolves it in process, needing nothing upstream. + // correlation nobody will hear about. A timed gate is exempt -- its own + // timer resolves it in process, needing nothing upstream. if (!env.hasUpstreamSignalResolver && opts.timeout === undefined) { - throw new Error( - `step ${opts.stepId} waits on ${opts.signalName} with no timeout, but ` + - `nothing outside this run can deliver it. A childWorkflow child is ` + - `run to its terminal rather than driven across parks, so no gate ` + - `beneath that boundary can be answered, at any depth. Give the gate ` + - `a timeout, or move it into a run the control plane can address`, - ); + throw new Error(unanswerableParkMessage(opts.stepId, opts.signalName)); } // Re-emit `SignalAwaited` only when the gate is not already awaiting it. // On a re-park resume the gate is already `awaiting-signal` (StepStarted @@ -4363,7 +5585,8 @@ async function parkOnSignalResult( // so the suspendable-child seam's `onSignalPark` sink fires after the flush, // mirroring the `parkToNotify` deferral below. Set only on the author branch. let signalParkToNotify: string | undefined; - if (state.steps.get(opts.stepId)?.phase !== "awaiting-signal") { + const repark = state.steps.get(opts.stepId)?.phase === "awaiting-signal"; + if (!repark) { const awaited: WorkflowEvent = { kind: "SignalAwaited", seq: state.lastSeq + 1, @@ -4381,7 +5604,21 @@ async function parkOnSignalResult( // approval one after a crash/reconnect (see parked-correlations). ...(opts.parkKind !== undefined ? { parkKind: opts.parkKind } : {}), }; - state = await commit(env, runId, awaited); + const claim = opts.claimsRunInputChannel; + state = await commit( + env, + runId, + awaited, + claim !== undefined + ? (fresh) => { + assertNoOpenInputPark( + fresh, + opts.stepId, + claim.sectionInputStepIds, + ); + } + : undefined, + ); // Only a park on a reserved `signalName(correlationId)` channel (the // agent-step suspend arm) carries a correlation the resolver routes a // decision back on; a plain `awaitSignal` gate parked on an @@ -4477,11 +5714,11 @@ async function parkOnSignalResult( // (and, when a timeout is set, tail the durable log for the // scheduler-committed `TimerFired`). Flush the buffered // `SignalAwaited` (+ `TimerSet`) to durable storage BEFORE parking so - // (a) the out-of-process scheduler can tail the durable `TimerSet` - // and arm the timeout, (b) a crash-while-suspended leaves a - // complete pre-suspension log that resume reconstructs the - // awaiting-signal state from, and (c) the control-plane suspension is - // durable in the log before the host is notified below, so the hub is + // (a) the scheduler can tail the durable `TimerSet` and arm the + // timeout, (b) a crash-while-suspended leaves a complete + // pre-suspension log that resume reconstructs the awaiting-signal + // state from, and (c) the control-plane suspension is durable in + // the log before the host is notified below, so the hub is // never told about a correlation the log cannot reconstruct on resume. // `subscribeFromSeq` above was computed from the in-memory tip; the // flush makes the durable tip match, so the timer-watch subscription @@ -4502,7 +5739,11 @@ async function parkOnSignalResult( // gate is surfaced up so the section can proxy it. Fired after the flush like // `onPark` so the `SignalAwaited` is durable before the section observes it. if (signalParkToNotify !== undefined) { - env.onSignalPark?.({ runId, name: signalParkToNotify }); + env.onSignalPark?.({ + runId, + name: signalParkToNotify, + ...(opts.timeout !== undefined ? { timeout: opts.timeout } : {}), + }); } // Drain observation point #4: signal-park entry. If drain has @@ -4566,9 +5807,26 @@ async function parkOnSignalResult( })(); } try { - const received = await env.signalChannel.awaitNext( + // `awaitNext` registers its subscriber before this returns. A host + // channel tails from head and does not replay a blob that landed + // earlier. A signal already in the log is taken from there. Otherwise + // a re-park notifies `onSignalAwaiter` and then waits: the re-park does + // not re-fire `onSignalPark`, so the caller delivers on that + // notification while the wait is outstanding. A fresh park omits the + // notify; its delivery arrives later, once this subscription exists. + const received = await awaitSignalFor( + env, + runId, + opts.stepId, opts.signalName, combinedAbort.signal, + repark + ? { + onWaiting: () => { + env.onSignalAwaiter?.({ runId, name: opts.signalName }); + }, + } + : undefined, ); let next = await reloadState(env, runId); const signalReceived: WorkflowEvent = { @@ -4901,119 +6159,1084 @@ function computeDelayToUntil( return Math.max(0, fireAt - now); } -async function runChildWorkflow( - parent: WorkflowDefinition, +/** + * Close a child whose drive threw while the child was still parked. + * Abort the controller passed into the spawner, then drain `next` until + * the child reports a terminal or the drain itself throws. Commit + * `ChildCompleted` from that terminal, or `failed` when none arrived. + * A child the parent log already closed is left as it is. + */ +async function closeFailedChild( env: WorkflowRuntimeEnv, parentRunId: string, - primitive: ChildWorkflowPrimitive, - selectorCtx: SelectorContext, + childRunId: string, + childAbort: AbortController, + handle: SuspendableChildHandle | undefined, +): Promise { + if (!childAbort.signal.aborted) childAbort.abort(); + let drained: "completed" | "failed" | "cancelled" | undefined; + if (handle !== undefined) { + try { + for (;;) { + const event = await handle.next(); + if (event.kind === "terminal") { + drained = event.terminalStatus; + break; + } + } + } catch { + // The child did not settle. The parent record still has to close. + } + } + const afterThrow = await reloadState(env, parentRunId); + if (afterThrow.children.get(childRunId)?.terminalStatus !== undefined) { + return; + } + const childFailed: WorkflowEvent = { + kind: "ChildCompleted", + seq: afterThrow.lastSeq + 1, + at: env.clock().toISOString(), + childRunId, + terminalStatus: drained ?? "failed", + }; + await commit(env, parentRunId, childFailed); +} + +/** + * Publish a childWorkflow step failure that landed while a sibling may + * still be parked. Those commits are buffered, and the parked sibling + * has already passed its segment boundary, so nothing later flushes + * them. The supervisor reads the durable log. + */ +async function flushParkedChildWorkflowFailure( + env: WorkflowRuntimeEnv, + runId: string, + kind: Primitive["kind"], +): Promise { + if (kind !== "childWorkflow") return; + await flush(env, runId); +} + +/** + * Park the parent on a control-plane relay and race that wait against the + * named park leaving the child. A sibling park stays queued. `next` would + * treat it as this park ending, and the spawn step can await only one + * signal. A child that ends while the parent is parked wins the race, and + * the caller continues from that terminal without reading the queue. A + * release while the child run is still going fails the spawn: an approval + * or input relay cannot be retired with `SignalAwaitAbandoned`, and parking + * the next signal on a step that is still awaiting this one would wait on + * the old name. A failure of the park itself (nothing upstream can answer + * it, or a second input park) is not a body event. `next` is not started + * on that path: the child is still parked, and the failure path's later + * `next` takes the one wake slot. + */ +async function awaitChildPark( + env: WorkflowRuntimeEnv, + parentRunId: string, + stepId: string, + handle: SuspendableChildHandle, + correlationId: string, + parkKind: "approval" | "input", + approvalSnapshot: ApprovalSnapshot | undefined, abort: AbortSignal, - depth: number, - maxChildSpawnDepth: number, -): Promise { - // Bound the spawn chain BEFORE committing StepStarted/ChildSpawned. A - // reject here lands a clean StepFailed on this spawn step (runPrimitiveSafe - // synthesizes it) and never writes a phantom child-run log. The child runs - // one rung deeper; the ceiling is tree-wide (threaded from this run). - const childDepth = depth + 1; - assertSpawnDepthWithinLimit(childDepth, primitive.id, maxChildSpawnDepth); - // Post-extraction the child definition is the internal `{ ref }` handle: the - // deploy step lifts the authored inline child to a standalone definition and - // the host resolves it from an in-memory closure map keyed by this ref. An - // inline child reaching the runtime is a deploy-step bug -- the same - // contract `runOnTrigger` enforces on its body. - if (!("ref" in primitive.definition)) { + sectionSteps: ReadonlySet, + pull: ChildRelayPull = "next", +): Promise { + const watch = handle.waitForParkRelease; + if (watch === undefined) { throw new Error( - `childWorkflow ${primitive.id} reached the runtime with an inline ` + - `definition; the deploy step must lift the child to an internal ref`, + `step ${stepId} cannot relay a ${parkKind} park: the child handle ` + + `does not watch a park release, so a sibling park would look like ` + + `this one ended`, ); } - const definitionRef = primitive.definition.ref; - const childInput = - primitive.input !== undefined - ? evaluate(primitive.input, selectorCtx) - : null; - // Allocate the child run-id locally and commit StepStarted + - // ChildSpawned *before* invoking the spawn callback so the parent - // audit log records the spawn ahead of any child-side work. A - // crash between the spawn-launch and the post-await commit would - // otherwise leave the parent log with no record that the child - // was spawned at all, and a concurrent cancel sweep iterating - // state.children would not find the child to issue - // ChildCancelRequested against. - const childRunId = env.newId("run"); - await emitStepStartedWithValue(env, parentRunId, primitive.id, { - definitionRef, - input: childInput, - ...(primitive.drainBehavior !== undefined - ? { drainBehavior: primitive.drainBehavior } - : {}), + const readPark = handle.readParkRelease; + if (readPark === undefined) { + throw new Error( + `step ${stepId} cannot relay a ${parkKind} park: the child handle ` + + `does not read a park release, so a decision already waiting would ` + + `be resumed after the gate moved on`, + ); + } + const name = signalName(correlationId); + const before = await readPark(name); + if (before.kind === "terminal") { + return { kind: "terminal", terminalStatus: before.terminalStatus }; + } + if (before.kind === "released") { + throw childRelayRetired(stepId, parkKind, name); + } + const parkAbort = new AbortController(); + const detachParkAbort = bridgeAbort(abort, () => { + parkAbort.abort(); }); - let state = await reloadState(env, parentRunId); - const spawned: WorkflowEvent = { - kind: "ChildSpawned", - seq: state.lastSeq + 1, - at: env.clock().toISOString(), - stepId: primitive.id, - childRunId, - childDefinitionRef: definitionRef, - }; - state = await commit(env, parentRunId, spawned); - // Segment boundary: the parent is about to hand off to and AWAIT a - // sub-run (which commits its own events -- including its terminal -- - // to the same workflow-run repo while this await blocks). Flush the - // parent's buffered pre-spawn events (RunStarted .. ChildSpawned) to - // durable storage BEFORE the child runs so the parent's audit log - // records the spawn ahead of any child-side work and a concurrent - // cancel sweep iterating state.children finds the child to cascade - // against -- the exact invariant the ChildSpawned-before-spawn - // ordering above exists to uphold. Without this flush the parent's - // RunStarted would not become durable until the parent's own - // terminal, so its runs// subtree would materialize - // AFTER its children's. - await flush(env, parentRunId); - // Wrap the spawn callback so a throw still lands a closing - // ChildCompleted event for the orphan. Without this, ChildSpawned - // would persist in state.children with `terminalStatus: undefined` - // and a future resume would treat it as a live child to cascade - // cancellation to. The catch commits ChildCompleted with status - // "failed" so state.children stays coherent, then rethrows so - // runPrimitiveSafe lands StepFailed on the parent's spawn step. - let child: { terminalStatus: "completed" | "failed" | "cancelled" }; + const watchAbort = new AbortController(); + const detachWatchAbort = bridgeAbort(abort, () => { + watchAbort.abort(); + }); + const parkRearm = await reloadState(env, parentRunId); + const decision = parkOnSignal( + env, + parentRunId, + { + stepId, + signalName: name, + parkKind, + ...(approvalSnapshot !== undefined ? { approvalSnapshot } : {}), + ...(parkKind === "input" + ? { claimsRunInputChannel: { sectionInputStepIds: sectionSteps } } + : {}), + }, + parkRearm, + parkAbort.signal, + ); + const release = watch(name, watchAbort.signal); + let outcome: + | { tag: "decision"; payload: unknown } + | { tag: "release"; rel: ChildParkRelease }; try { - child = await env.spawnChild({ - definitionRef, - childRunId, - input: childInput, - parentRunId, - parentStepId: primitive.id, - signal: abort, - depth: childDepth, - maxChildSpawnDepth, - }); + outcome = await Promise.race([ + decision.then((payload) => ({ tag: "decision" as const, payload })), + release.then((rel) => ({ tag: "release" as const, rel })), + ]); } catch (cause) { - let afterThrow = await reloadState(env, parentRunId); - const childFailed: WorkflowEvent = { - kind: "ChildCompleted", - seq: afterThrow.lastSeq + 1, - at: env.clock().toISOString(), - childRunId, - terminalStatus: "failed", - }; - afterThrow = await commit(env, parentRunId, childFailed); - void afterThrow; + parkAbort.abort(); + watchAbort.abort(); + await decision.catch(() => undefined); + await release.catch(() => undefined); throw cause; + } finally { + detachParkAbort(); + detachWatchAbort(); } - state = await reloadState(env, parentRunId); - const childCompleted: WorkflowEvent = { - kind: "ChildCompleted", - seq: state.lastSeq + 1, - at: env.clock().toISOString(), - childRunId, - terminalStatus: child.terminalStatus, + if (outcome.tag === "release") { + parkAbort.abort(); + await decision.catch(() => undefined); + if (outcome.rel.kind === "terminal") { + return { + kind: "terminal", + terminalStatus: outcome.rel.terminalStatus, + }; + } + throw childRelayRetired(stepId, parkKind, name); + } + // Stop the watch before resume. The child's own receipt would otherwise + // look like the park ending without a decision. Re-read before resume: + // a decision already waiting can win the race after the gate has left + // the await. + watchAbort.abort(); + await release.catch(() => undefined); + const again = await readPark(name); + if (again.kind === "terminal") { + return { kind: "terminal", terminalStatus: again.terminalStatus }; + } + if (again.kind === "released") { + throw childRelayRetired(stepId, parkKind, name); + } + await handle.resume(correlationId, outcome.payload); + return finishRelayPull(handle, pull); +} + +function childRelayRetired( + stepId: string, + parkKind: "approval" | "input", + name: string, +): Error { + return new Error( + `step ${stepId} ${parkKind} relay on ${name} ended while the child ` + + `run was still going; a control-plane relay cannot be retired`, + ); +} + +/** + * Step ids whose input park is an onTrigger event channel, not the run's + * deployment input. Several sections re-arm at once. A trigger-budget + * re-arm and a childWorkflow relay of that park do not count those parks + * as holding the one deployment channel. + */ +function sectionInputStepIds( + definition: WorkflowDefinition, +): ReadonlySet { + const ids = new Set(); + for (const [id, step] of Object.entries(definition.steps)) { + if (step.kind === "onTrigger") ids.add(id); + } + return ids; +} + +/** + * The run has one deployment input channel. The caller is claiming it. + * Another holder fails the commit. Section event channels are not holders. + */ +function assertNoOpenInputPark( + state: RunState, + stepId: string, + sectionSteps: ReadonlySet, +): void { + for (const [otherId, step] of state.steps) { + if (otherId === stepId || sectionSteps.has(otherId)) continue; + if ( + step.phase === "awaiting-signal" && + step.awaitingSignal !== undefined && + controlParkKindOf(step.awaitingSignal) === "input" + ) { + throw new Error( + `step ${stepId} input park: step ${otherId} is already awaiting ` + + `an input park; a run holds one input channel`, + ); + } + } +} + +/** + * One park the child log still holds, in child-log order. An author name is + * anything that is not a reserved correlation. A reserved name is an + * approval or an input park. A signal-relay recorded on a reserved name is + * corrupt: a relay uses the author's name, and the outer spawn step climbs + * that name one level. + */ +type OpenChildPark = + | { + kind: "author"; + name: string; + awaitSeq: number; + timeoutAt?: string; + } + | { + kind: "approval" | "input"; + name: string; + correlationId: string; + awaitSeq: number; + }; + +/** + * What the parent still has to do for one child park, or for a relay the + * parent is holding that the child has left. `hold` drives return when the + * relay is parked or already satisfied; they do not read the child's queue. + * A resumed child does not surface parks that are already in its log, so + * reading the queue after the first of two open parks would wait forever. + */ +type ChildRelay = + | { kind: "signal-abandon"; name: string } + | { kind: "signal-reestablish"; name: string; awaitSeq: number } + | { + kind: "signal-deliver"; + name: string; + payload: unknown; + signalId: string; + } + | { kind: "signal-fresh"; name: string; timeout?: number } + | { + kind: "control-reestablish"; + parkKind: "approval" | "input"; + correlationId: string; + } + | { + kind: "control-deliver"; + parkKind: "approval" | "input"; + correlationId: string; + decision: unknown; + } + | { kind: "input-fresh"; correlationId: string }; + +type ChildWorkflowResumePlan = + | { kind: "fresh"; stepStarted: boolean } + | { + kind: "reattach"; + childRunId: string; + relays: ChildRelay[]; + seed: boolean; + } + | { + kind: "terminal"; + childRunId: string; + terminalStatus: "completed" | "failed" | "cancelled"; + }; + +function recordedChildSpawn( + parentLog: readonly WorkflowEvent[], + state: RunState, + stepId: string, + definitionRef: string, +): + | { + childRunId: string; + terminalStatus: "completed" | "failed" | "cancelled" | undefined; + } + | undefined { + const spawned: Extract[] = []; + for (const event of parentLog) { + if (event.kind === "ChildSpawned" && event.stepId === stepId) { + spawned.push(event); + } + } + if (spawned.length > 1) { + throw new Error( + `childWorkflow ${stepId} has ${String(spawned.length)} ChildSpawned events`, + ); + } + const logged = spawned[0]; + const fromState: { + childRunId: string; + terminalStatus: "completed" | "failed" | "cancelled" | undefined; + }[] = []; + for (const child of state.children.values()) { + if (child.spawnedBy !== stepId) continue; + fromState.push({ + childRunId: child.childRunId, + terminalStatus: child.terminalStatus, + }); + } + if (fromState.length > 1) { + throw new Error( + `childWorkflow ${stepId} has ${String(fromState.length)} child records`, + ); + } + const recorded = fromState[0]; + if (logged === undefined && recorded === undefined) return undefined; + if (logged === undefined || recorded === undefined) { + throw new Error( + `childWorkflow ${stepId} child record disagrees with the log`, + ); + } + if (logged.childRunId !== recorded.childRunId) { + throw new Error( + `childWorkflow ${stepId} child run ${logged.childRunId} disagrees with state child ${recorded.childRunId}`, + ); + } + if (logged.childDefinitionRef !== definitionRef) { + throw new Error( + `childWorkflow ${stepId} was spawned as ${logged.childDefinitionRef}, not ${definitionRef}`, + ); + } + return { + childRunId: logged.childRunId, + terminalStatus: recorded.terminalStatus, }; - state = await commit(env, parentRunId, childCompleted); - void state; +} + +/** + * The last `SignalAwaited` per child step, in seq order, kept only while + * that step is still `awaiting-signal` on the same name. An earlier await + * the step has left does not match, so an abandoned park is not relayed. + */ +function openChildParks( + childLog: readonly WorkflowEvent[], + childState: RunState, +): OpenChildPark[] { + const lastByStep = new Map< + string, + Extract + >(); + for (const event of childLog) { + if (event.kind !== "SignalAwaited") continue; + lastByStep.set(event.stepId, event); + } + const ordered = [...lastByStep.values()].sort((a, b) => a.seq - b.seq); + const parks: OpenChildPark[] = []; + for (const event of ordered) { + const step = childState.steps.get(event.stepId); + if (step?.phase !== "awaiting-signal") continue; + const awaiting = step.awaitingSignal; + if (awaiting === undefined || awaiting.name !== event.signalName) { + continue; + } + const correlationId = correlationIdFromSignalName(event.signalName); + if (correlationId === undefined) { + parks.push({ + kind: "author", + name: event.signalName, + awaitSeq: event.seq, + ...(awaiting.timeoutAt !== undefined + ? { timeoutAt: awaiting.timeoutAt } + : {}), + }); + continue; + } + // Absent parkKind on an author name means a plain gate, not an + // approval. Only a reserved name is classified through the legacy + // default, and a signal-relay never belongs on one. + const parkKind = controlParkKindOf(awaiting); + if (parkKind === "signal-relay") { + throw new Error( + `child step ${event.stepId} awaits reserved name ${event.signalName} as a signal-relay`, + ); + } + parks.push({ + kind: parkKind, + name: event.signalName, + correlationId, + awaitSeq: event.seq, + }); + } + return parks; +} + +function childSawSignal( + childLog: readonly WorkflowEvent[], + awaitSeq: number, + signalId: string, +): boolean { + for (const event of childLog) { + if ( + event.kind === "SignalReceived" && + event.signalId === signalId && + event.seq < awaitSeq + ) { + return true; + } + } + return false; +} + +/** + * Milliseconds left until `timeoutAt`. A deadline that has already passed + * is still a timed gate: zero lets the in-process timer finish it. An + * absent or unreadable deadline is an untimed park. + */ +function remainingSignalTimeout( + timeoutAt: string | undefined, + nowMs: number, +): number | undefined { + if (timeoutAt === undefined) return undefined; + const deadline = Date.parse(timeoutAt); + if (!Number.isFinite(deadline)) return undefined; + const remaining = deadline - nowMs; + if (!(remaining > 0)) return 0; + return remaining; +} + +function lastAwaitSeq( + log: readonly WorkflowEvent[], + stepId: string, + name: string, +): number | undefined { + let seq: number | undefined; + for (const event of log) { + if ( + event.kind === "SignalAwaited" && + event.stepId === stepId && + event.signalName === name + ) { + seq = event.seq; + } + } + return seq; +} + +/** + * The signal paired with the parent's last await of `name`, if that await + * already consumed one. The full parent log is required: the pairing + * replays the reducer's queue from seq 1. + */ +function deliveredForAwait( + parentLog: readonly WorkflowEvent[], + stepId: string, + name: string, +): { payload: unknown; signalId: string } | undefined { + const awaitSeq = lastAwaitSeq(parentLog, stepId, name); + if (awaitSeq === undefined) return undefined; + return boundSignalForContainerAwait(parentLog, name, stepId, awaitSeq); +} + +function relayName(relay: ChildRelay): string { + switch (relay.kind) { + case "signal-abandon": + case "signal-reestablish": + case "signal-deliver": + case "signal-fresh": + return relay.name; + case "control-reestablish": + case "control-deliver": + case "input-fresh": + return signalName(relay.correlationId); + default: { + const unexpected: never = relay; + throw new Error(`unexpected child relay ${String(unexpected)}`); + } + } +} + +function orderChildRelays( + relays: readonly ChildRelay[], + parentAwaitName: string | undefined, +): ChildRelay[] { + const abandon: ChildRelay[] = []; + const current: ChildRelay[] = []; + const rest: ChildRelay[] = []; + for (const relay of relays) { + if (relay.kind === "signal-abandon") { + abandon.push(relay); + continue; + } + if (parentAwaitName !== undefined && relayName(relay) === parentAwaitName) { + current.push(relay); + continue; + } + rest.push(relay); + } + return [...abandon, ...current, ...rest]; +} + +function parentSignalAwait( + state: RunState, + stepId: string, +): { name: string; parkKind?: ControlParkKind } | undefined { + const step = state.steps.get(stepId); + if (step?.phase !== "awaiting-signal") return undefined; + const awaiting = step.awaitingSignal; + if (awaiting === undefined) { + throw new Error( + `childWorkflow ${stepId} is awaiting a signal with no signal name`, + ); + } + return awaiting; +} + +function planChildRelays( + stepId: string, + childRunId: string, + parentLog: readonly WorkflowEvent[], + parentState: RunState, + childLog: readonly WorkflowEvent[], + childState: RunState, + nowMs: number, +): ChildRelay[] { + const parks = openChildParks(childLog, childState); + const parentAwait = parentSignalAwait(parentState, stepId); + const childTerminal = isTerminalRunPhase(childState.phase); + const relays: ChildRelay[] = []; + for (const park of parks) { + if (parentAwait !== undefined && parentAwait.name === park.name) { + const parentKind = controlParkKindOf(parentAwait); + if (park.kind === "author") { + if (parentKind !== "signal-relay") { + throw new Error( + `childWorkflow ${stepId} author park ${park.name} is awaited as ${parentKind}`, + ); + } + const awaitSeq = lastAwaitSeq(parentLog, stepId, park.name); + if (awaitSeq === undefined) { + throw new Error( + `childWorkflow ${stepId} is awaiting ${park.name} with no SignalAwaited in the log`, + ); + } + relays.push({ kind: "signal-reestablish", name: park.name, awaitSeq }); + continue; + } + if (parentKind !== park.kind) { + throw new Error( + `childWorkflow ${stepId} ${park.kind} park ${park.name} is awaited as ${parentKind}`, + ); + } + relays.push({ + kind: "control-reestablish", + parkKind: park.kind, + correlationId: park.correlationId, + }); + continue; + } + const delivered = deliveredForAwait(parentLog, stepId, park.name); + if ( + delivered !== undefined && + !childSawSignal(childLog, park.awaitSeq, delivered.signalId) + ) { + if (park.kind === "author") { + relays.push({ + kind: "signal-deliver", + name: park.name, + payload: delivered.payload, + signalId: delivered.signalId, + }); + } else { + relays.push({ + kind: "control-deliver", + parkKind: park.kind, + correlationId: park.correlationId, + decision: delivered.payload, + }); + } + continue; + } + if (park.kind === "approval") { + throw new Error( + `childWorkflow ${stepId} cannot resume approval park of child run ${childRunId}: the approval snapshot is not in the log`, + ); + } + if (park.kind === "author") { + const timeout = remainingSignalTimeout(park.timeoutAt, nowMs); + relays.push({ + kind: "signal-fresh", + name: park.name, + ...(timeout !== undefined ? { timeout } : {}), + }); + continue; + } + relays.push({ + kind: "input-fresh", + correlationId: park.correlationId, + }); + } + if (parentAwait !== undefined) { + const awaiting = parentAwait; + const stillOpen = parks.some((park) => park.name === awaiting.name); + if (!stillOpen) { + const parentKind = controlParkKindOf(awaiting); + if (parentKind === "signal-relay") { + relays.push({ kind: "signal-abandon", name: awaiting.name }); + } else if (!childTerminal) { + throw new Error( + `step ${stepId} ${parentKind} relay on ${awaiting.name} ended while child run ${childRunId} was still going; a control-plane relay cannot be retired`, + ); + } + } + } + return orderChildRelays(relays, parentAwait?.name); +} + +async function planChildWorkflowResume( + env: WorkflowRuntimeEnv, + stepId: string, + definitionRef: string, + prior: RunState, + parentLog: readonly WorkflowEvent[], +): Promise { + const step = prior.steps.get(stepId); + const recorded = recordedChildSpawn(parentLog, prior, stepId, definitionRef); + if (step === undefined) { + if (recorded !== undefined) { + throw new Error( + `childWorkflow ${stepId} has child run ${recorded.childRunId} but the spawn step is not in state`, + ); + } + return { kind: "fresh", stepStarted: false }; + } + if (step.phase !== "in-flight" && step.phase !== "awaiting-signal") { + throw new Error( + `childWorkflow ${stepId} cannot resume from phase ${step.phase}`, + ); + } + if (recorded === undefined) { + if (step.phase === "awaiting-signal") { + throw new Error( + `childWorkflow ${stepId} is awaiting a signal with no child run`, + ); + } + return { kind: "fresh", stepStarted: true }; + } + if (recorded.terminalStatus !== undefined) { + return { + kind: "terminal", + childRunId: recorded.childRunId, + terminalStatus: recorded.terminalStatus, + }; + } + const childLog = [...(await env.repoStore.read(recorded.childRunId))]; + if (childLog.length === 0) { + if (step.phase === "awaiting-signal") { + throw new Error( + `childWorkflow ${stepId} is awaiting a signal but child run ${recorded.childRunId} has an empty log`, + ); + } + return { + kind: "reattach", + childRunId: recorded.childRunId, + relays: [], + seed: false, + }; + } + const childState = resumeFromLog(recorded.childRunId, childLog); + const childPhase = childState.phase; + if ( + childPhase === "completed" || + childPhase === "failed" || + childPhase === "cancelled" + ) { + return { + kind: "terminal", + childRunId: recorded.childRunId, + terminalStatus: childPhase, + }; + } + return { + kind: "reattach", + childRunId: recorded.childRunId, + relays: planChildRelays( + stepId, + recorded.childRunId, + parentLog, + prior, + childLog, + childState, + env.clock().getTime(), + ), + seed: true, + }; +} + +async function drivePlannedChildRelay( + env: WorkflowRuntimeEnv, + parentRunId: string, + stepId: string, + handle: SuspendableChildHandle, + relay: ChildRelay, + abort: AbortSignal, + sectionSteps: ReadonlySet, +): Promise { + switch (relay.kind) { + case "signal-abandon": { + await abandonRelayIfOpen(env, parentRunId, stepId, relay.name); + return { kind: "held" }; + } + case "signal-reestablish": { + const relayLog = await env.repoStore.read(parentRunId); + return raceContainerSignalRelay( + env, + parentRunId, + stepId, + handle, + relay.name, + relay.awaitSeq, + correlationIdAt(relayLog, relay.awaitSeq), + abort, + "park-release", + "hold", + ); + } + case "signal-deliver": { + const snapshot = await readChildPark(handle, stepId, relay.name); + if (snapshot.kind === "terminal") { + return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; + } + if (snapshot.kind === "released") return { kind: "held" }; + if (handle.whenSignalAwaiter !== undefined) { + await handle.whenSignalAwaiter(relay.name); + } + await handle.deliverSignal(relay.name, relay.payload, relay.signalId); + return { kind: "held" }; + } + case "signal-fresh": { + if (relay.timeout === undefined && !env.hasUpstreamSignalResolver) { + throw new Error(unanswerableParkMessage(stepId, relay.name)); + } + const snapshot = await readChildPark(handle, stepId, relay.name); + if (snapshot.kind === "terminal") { + return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; + } + if (snapshot.kind === "released") return { kind: "held" }; + return driveContainerSignalRelay( + env, + parentRunId, + stepId, + handle, + relay.name, + "name", + abort, + "park-release", + relay.timeout, + "hold", + ); + } + case "control-reestablish": + return awaitChildPark( + env, + parentRunId, + stepId, + handle, + relay.correlationId, + relay.parkKind, + undefined, + abort, + sectionSteps, + "hold", + ); + case "control-deliver": { + const name = signalName(relay.correlationId); + const snapshot = await readChildPark(handle, stepId, name); + if (snapshot.kind === "terminal") { + return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; + } + if (snapshot.kind === "released") return { kind: "held" }; + if (handle.whenSignalAwaiter !== undefined) { + await handle.whenSignalAwaiter(name); + } + await handle.resume(relay.correlationId, relay.decision); + return { kind: "held" }; + } + case "input-fresh": { + const name = signalName(relay.correlationId); + const snapshot = await readChildPark(handle, stepId, name); + if (snapshot.kind === "terminal") { + return { kind: "terminal", terminalStatus: snapshot.terminalStatus }; + } + if (snapshot.kind === "released") return { kind: "held" }; + return awaitChildPark( + env, + parentRunId, + stepId, + handle, + relay.correlationId, + "input", + undefined, + abort, + sectionSteps, + "hold", + ); + } + default: { + const unexpected: never = relay; + throw new Error(`unexpected child relay ${String(unexpected)}`); + } + } +} + +async function runChildWorkflow( + parent: WorkflowDefinition, + env: WorkflowRuntimeEnv, + parentRunId: string, + primitive: ChildWorkflowPrimitive, + selectorCtx: SelectorContext, + abort: AbortSignal, + depth: number, + maxChildSpawnDepth: number, +): Promise { + // Bound the spawn chain BEFORE committing StepStarted/ChildSpawned. A + // reject here lands a clean StepFailed on this spawn step (runPrimitiveSafe + // synthesizes it) and never writes a phantom child-run log. The child runs + // one rung deeper; the ceiling is tree-wide (threaded from this run). + const childDepth = depth + 1; + assertSpawnDepthWithinLimit(childDepth, primitive.id, maxChildSpawnDepth); + const sectionSteps = sectionInputStepIds(parent); + // Post-extraction the child definition is the internal `{ ref }` handle: the + // deploy step lifts the authored inline child to a standalone definition and + // the host resolves it from an in-memory closure map keyed by this ref. An + // inline child reaching the runtime is a deploy-step bug -- the same + // contract `runOnTrigger` enforces on its body. + if (!("ref" in primitive.definition)) { + throw new Error( + `childWorkflow ${primitive.id} reached the runtime with an inline ` + + `definition; the deploy step must lift the child to an internal ref`, + ); + } + const definitionRef = primitive.definition.ref; + // Plan before spawn. An approval the parent never proxied has no snapshot + // in the log, and emitting a fresh one would throw after the child was + // already reattached. A child the parent already closed is finished from + // that record and is not spawned again. + const prior = await reloadState(env, parentRunId); + const parentLog = [...(await env.repoStore.read(parentRunId))]; + const plan = await planChildWorkflowResume( + env, + primitive.id, + definitionRef, + prior, + parentLog, + ); + let childRunId: string; + let child: + | { terminalStatus: "completed" | "failed" | "cancelled" } + | undefined; + if (plan.kind === "terminal") { + childRunId = plan.childRunId; + child = { terminalStatus: plan.terminalStatus }; + } else { + const childInput = + primitive.input !== undefined + ? evaluate(primitive.input, selectorCtx) + : null; + if (plan.kind === "fresh") { + // A StepStarted with no ChildSpawned is a crash before the spawn + // record. Keep that step and allocate a new child id. The first + // visit emits StepStarted, then ChildSpawned, before the spawner + // runs, so a cancel sweep can find the child. + if (!plan.stepStarted) { + await emitStepStartedWithValue(env, parentRunId, primitive.id, { + definitionRef, + input: childInput, + ...(primitive.drainBehavior !== undefined + ? { drainBehavior: primitive.drainBehavior } + : {}), + }); + } + childRunId = env.newId("run"); + const spawnedState = await reloadState(env, parentRunId); + const spawned: WorkflowEvent = { + kind: "ChildSpawned", + seq: spawnedState.lastSeq + 1, + at: env.clock().toISOString(), + stepId: primitive.id, + childRunId, + childDefinitionRef: definitionRef, + }; + await commit(env, parentRunId, spawned); + // Segment boundary: the parent is about to hand off to and AWAIT a + // sub-run (which commits its own events -- including its terminal -- + // to the same workflow-run repo while this await blocks). Flush the + // parent's buffered pre-spawn events (RunStarted .. ChildSpawned) to + // durable storage BEFORE the child runs so the parent's audit log + // records the spawn ahead of any child-side work and a concurrent + // cancel sweep iterating state.children finds the child to cascade + // against -- the exact invariant the ChildSpawned-before-spawn + // ordering above exists to uphold. Without this flush the parent's + // RunStarted would not become durable until the parent's own + // terminal, so its runs// subtree would materialize + // AFTER its children's. + await flush(env, parentRunId); + } else { + childRunId = plan.childRunId; + } + const childEvents = + plan.kind === "reattach" && plan.seed + ? [...(await env.repoStore.read(childRunId))] + : []; + // The signal handed to the spawner is a controller this drive can abort + // without aborting the parent run. The step abort is bridged onto it, so + // a parent cancel still tears the child down. A throw while the child is + // parked aborts that controller and drains the handle before the parent + // log records the child closed. Without the drain, ChildCompleted would + // say the child failed while its steps were still running. + const childAbort = new AbortController(); + const detachChildAbort = bridgeAbort(abort, () => { + childAbort.abort(); + }); + let handle: SuspendableChildHandle | undefined; + try { + try { + const spawnedHandle = await env.spawnChild({ + definitionRef, + childRunId, + input: childInput, + parentRunId, + parentStepId: primitive.id, + signal: childAbort.signal, + depth: childDepth, + maxChildSpawnDepth, + ...(childEvents.length > 0 ? { resumeFromEvents: childEvents } : {}), + }); + handle = spawnedHandle; + let terminalStatus: "completed" | "failed" | "cancelled" | undefined; + if (plan.kind === "reattach") { + for (const relay of plan.relays) { + const driven = await drivePlannedChildRelay( + env, + parentRunId, + primitive.id, + spawnedHandle, + relay, + childAbort.signal, + sectionSteps, + ); + if (driven.kind === "terminal") { + terminalStatus = driven.terminalStatus; + break; + } + if (driven.kind !== "held") { + throw new Error( + `childWorkflow ${primitive.id} resume relay returned ${driven.kind}; a held relay does not read the queue`, + ); + } + } + } + if (terminalStatus === undefined) { + let pending: ChildDriveResult | undefined; + for (;;) { + const bodyEvent = pending ?? (await spawnedHandle.next()); + pending = undefined; + if (bodyEvent.kind === "terminal") { + terminalStatus = bodyEvent.terminalStatus; + break; + } + if (bodyEvent.kind === "held") { + throw new Error( + `childWorkflow ${primitive.id} drive held a park; this drive reads the queue`, + ); + } + if (bodyEvent.kind === "park") { + pending = await awaitChildPark( + env, + parentRunId, + primitive.id, + spawnedHandle, + bodyEvent.park.correlationId, + "approval", + bodyEvent.park.approvalSnapshot, + childAbort.signal, + sectionSteps, + ); + continue; + } + if (bodyEvent.kind === "input-park") { + pending = await awaitChildPark( + env, + parentRunId, + primitive.id, + spawnedHandle, + bodyEvent.correlationId, + "input", + undefined, + childAbort.signal, + sectionSteps, + ); + continue; + } + if ( + bodyEvent.timeout === undefined && + !env.hasUpstreamSignalResolver + ) { + throw new Error( + unanswerableParkMessage(primitive.id, bodyEvent.name), + ); + } + pending = await driveContainerSignalRelay( + env, + parentRunId, + primitive.id, + spawnedHandle, + bodyEvent.name, + "name", + childAbort.signal, + "park-release", + bodyEvent.timeout, + ); + } + } + if (terminalStatus === undefined) { + throw new Error( + `childWorkflow ${primitive.id} drive ended without a terminal`, + ); + } + child = { terminalStatus }; + } catch (cause) { + await closeFailedChild( + env, + parentRunId, + childRunId, + childAbort, + handle, + ); + throw cause; + } + } finally { + detachChildAbort(); + } + } + if (child === undefined) { + throw new Error( + `childWorkflow ${primitive.id} drive ended without a terminal`, + ); + } + const settled = await reloadState(env, parentRunId); + if (settled.children.get(childRunId)?.terminalStatus === undefined) { + const childCompleted: WorkflowEvent = { + kind: "ChildCompleted", + seq: settled.lastSeq + 1, + at: env.clock().toISOString(), + childRunId, + terminalStatus: child.terminalStatus, + }; + await commit(env, parentRunId, childCompleted); + } if (child.terminalStatus !== "completed") { // A child run that ended `failed` or `cancelled` propagates to // the parent step as a failure. The runtime is the layer with diff --git a/vendor/intx/workflow/src/runtime/selectors.ts b/vendor/intx/workflow/src/runtime/selectors.ts index b2da59e1b4..c1775d880d 100644 --- a/vendor/intx/workflow/src/runtime/selectors.ts +++ b/vendor/intx/workflow/src/runtime/selectors.ts @@ -5,6 +5,9 @@ // Used by the step executor to materialize a step's `input` and to // resolve declared `reads` against the run-state subtree. +import { PerCallInferenceOptions } from "@intx/types/runtime-core"; +import { type } from "arktype"; + import { isFromSelector, isLiteralSelector, @@ -97,6 +100,33 @@ export function evaluate(selector: Selector, ctx: SelectorContext): unknown { throw new SelectorError("unknown selector shape", selector); } +/** + * Resolve a step's `inference` selector into per-call options. `null` and + * `undefined` mean the agent's defaults; any other non-object, or an object + * carrying a key outside `PerCallInferenceOptions`, is a selector error. + */ +export function resolveStepInference( + selector: Selector, + ctx: SelectorContext, +): PerCallInferenceOptions | undefined { + const value = evaluate(selector, ctx); + if (value === null || value === undefined) return undefined; + if (!isRecord(value)) { + throw new SelectorError( + "inference selector must resolve to an object, null, or undefined", + selector, + ); + } + const options = PerCallInferenceOptions(value); + if (options instanceof type.errors) { + throw new SelectorError( + `inference selector resolved to invalid options: ${options.summary}`, + selector, + ); + } + return options; +} + function resolvePath( path: string, ctx: SelectorContext, diff --git a/vendor/intx/workflow/src/runtime/step-scope.ts b/vendor/intx/workflow/src/runtime/step-scope.ts index 4b2b4477fb..e0049fd95c 100644 --- a/vendor/intx/workflow/src/runtime/step-scope.ts +++ b/vendor/intx/workflow/src/runtime/step-scope.ts @@ -13,9 +13,9 @@ // owner rather than a hand-rolled template and several divergent strip // regexes scattered across the runtime and the sidecar. // -// This module also owns the related loop-iteration body RUN id format -// (`loopBodyRunId`): a cross-run store key rather than an in-run step id, so it -// carries the container run id and is documented separately below. +// This module also owns the related body RUN id formats (`loopBodyRunId`, +// `sectionBodyRunId`): cross-run store keys rather than in-run step ids, so +// each carries the container run id and is documented separately below. /** * Encode a fan-out iteration's scoped step id from its base step id and @@ -45,14 +45,13 @@ export function baseStepId(stepId: string): string { * inner loop under two outer iterations gets distinct ids. Deterministic -- * crash-resume re-derives the same string rather than reversing it. * - * Injectivity does NOT require a `__`-free run id (a nested loop's own run id - * contains `__`, and a caller-supplied top-level run id may too). It holds - * because `loopId` contains no `__` (rejected at definition time in - * `normalize`) and `index` is always digits: the final two `__` in the output - * are therefore always the two separators, so the string decomposes to exactly - * one (runId, loopId, index) regardless of what the run id contains. The run - * id is separately constrained by `RUN_ID_PATTERN` for store-path and - * mail-address safety, not for injectivity. + * Unique for one fixed run id. `loopId` contains no `__` (rejected at + * definition time in `normalize`) and `index` is decimal digits, so two + * different `(loopId, index)` pairs under that same run id cannot encode the + * same string. It is not a unique decoding of every string `RUN_ID_PATTERN` + * allows: `loopBodyRunId("a_", "_b", 0)` and `loopBodyRunId("a__", "b", 0)` + * are both `a____b__0`. `RUN_ID_PATTERN` constrains the run id for store-path + * and mail-address safety. */ export function loopBodyRunId( runId: string, @@ -61,3 +60,23 @@ export function loopBodyRunId( ): string { return `${runId}__${loopId}__${String(index)}`; } + +/** + * Encode an onTrigger section body's run id from the parent run id, the + * section step id, and the zero-based event index. The body run id is a + * cross-run store key, so it carries the parent: + * `____`. + * + * Unique for one fixed parent run id. `sectionId` contains no `__` (rejected + * at definition time in `normalize`) and `index` is decimal digits, so two + * different `(sectionId, index)` pairs under that parent cannot encode the + * same string. It is not a unique decoding across every parent + * `RUN_ID_PATTERN` allows. + */ +export function sectionBodyRunId( + parentRunId: string, + sectionId: string, + eventIndex: number, +): string { + return `${parentRunId}__${sectionId}__${String(eventIndex)}`; +} diff --git a/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts b/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts index 5710e6e904..87b423bfa8 100644 --- a/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts +++ b/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts @@ -1,20 +1,20 @@ // Shared park-aware drive for a suspendable child body. // // Runs a body sub-DAG through `runtimeRun` and returns a `SuspendableChildHandle` -// the caller drives across the body's approval / author-`awaitSignal` parks -// (rather than awaiting a terminal). This is the SINGLE handle implementation -// behind the `SuspendableChildHandle` contract: every suspendable-child seam -- -// an onTrigger section body and a loop iteration -- builds its own host-shaped -// env (a fresh capped env for a section, the inherited parent env for a loop) -// and then hands it here, so the two share identical park/resume/signal -// semantics by construction. +// the caller drives across the body's parks (rather than awaiting a terminal). +// This is the SINGLE handle implementation behind the `SuspendableChildHandle` +// contract: an onTrigger section body, a loop iteration, and a `childWorkflow` +// child each build a host-shaped env and hand it here, so they share park, +// resume, and signal semantics by construction. // -// The helper OWNS `onPark` / `onSignalPark`: it spreads the caller's env and -// installs its own sinks to feed the FIFO the caller drains via `next()`. The -// caller must therefore pass an env that has not wired its own park sinks; a -// silent override would drop them, so this fails loud instead. +// The helper OWNS `onPark` / `onSignalPark` / `onSignalAwaiter`: it spreads +// the caller's env and installs its own sinks. Park sinks feed the FIFO the +// caller drains via `next()`. `onSignalAwaiter` records that a re-park has +// entered `awaitNext`, which `whenSignalAwaiter` waits for. The caller must +// pass an env that has not wired these sinks; a silent override would drop +// them, so this fails loud instead. -import { signalName } from "@intx/types"; +import { signalName } from "@intx/types/signals"; import type { WorkflowDefinition } from "../definition/index"; import type { @@ -22,9 +22,33 @@ import type { SuspendableChildPark, WorkflowRuntimeEnv, } from "./env"; +import { resumeFromLog, type WorkflowEvent } from "../state-machine/index"; import { bridgeAbort } from "./abort-bridge"; import { runtimeRun } from "./run"; -import type { WorkflowEvent } from "../state-machine/index"; + +/** + * A handle that has already reached a terminal. `next` returns that same + * terminal on every call, matching a live handle after `complete` settles. + * `resume` and `deliverSignal` reject: this handle has no channel, and a + * delivery after settle is a caller bug. + */ +export function createTerminalChildHandle( + terminalStatus: "completed" | "failed" | "cancelled", +): SuspendableChildHandle { + const terminal = { + kind: "terminal" as const, + terminalStatus, + }; + const rejectSettled = (): Promise => + Promise.reject( + new Error(`child handle has already settled ${terminalStatus}`), + ); + return { + next: () => Promise.resolve(terminal), + resume: () => rejectSettled(), + deliverSignal: () => rejectSettled(), + }; +} export function createSuspendableChildHandle( env: WorkflowRuntimeEnv, @@ -37,7 +61,7 @@ export function createSuspendableChildHandle( * through unchanged) and the tree-wide ceiling, so a `childWorkflow` * spawned inside the body counts against the same bound as the top-level * run rather than resetting at the body boundary. Mirrors the options - * `createSidecarRunChild` threads into a terminal child's `runtimeRun`. + * `createSidecarRunChild` threads into the handle that drives the child. */ depth: number; maxChildSpawnDepth: number; @@ -50,6 +74,14 @@ export function createSuspendableChildHandle( * than the helper reaching for it on `env.signalChannel`. */ cleanup?: () => void | Promise; + /** + * Whether an input park is returned to the caller. Required, with no + * default: the safe direction is to reject, and a caller that forgot + * the flag would otherwise climb a park the container cannot answer. + * `childWorkflow` passes true. A loop iteration and an `onTrigger` + * body pass false, and `next` aborts the body on the input park. + */ + surfaceInputPark: boolean; }, ): SuspendableChildHandle { const { @@ -61,13 +93,18 @@ export function createSuspendableChildHandle( resumeFromEvents, signal, cleanup, + surfaceInputPark, } = args; - if (env.onPark !== undefined || env.onSignalPark !== undefined) { + if ( + env.onPark !== undefined || + env.onSignalPark !== undefined || + env.onSignalAwaiter !== undefined + ) { throw new Error( `createSuspendableChildHandle: the env for ${childRunId} already wired ` + - `onPark/onSignalPark; this helper owns the park sinks, so a caller-set ` + - `sink would be silently overridden`, + `onPark/onSignalPark/onSignalAwaiter; this helper owns those sinks, ` + + `so a caller-set sink would be silently overridden`, ); } @@ -77,7 +114,8 @@ export function createSuspendableChildHandle( // mirroring the signal channel's single-consumer shape. type BodyEvent = | { kind: "park"; park: SuspendableChildPark } - | { kind: "signal-park"; name: string } + | { kind: "signal-park"; name: string; timeout?: number } + | { kind: "input-park"; correlationId: string } | { kind: "error"; error: Error }; const events: BodyEvent[] = []; let wake: (() => void) | null = null; @@ -89,6 +127,29 @@ export function createSuspendableChildHandle( } }; + // Names whose re-park has entered `awaitNext`. A host channel does not + // replay a blob that landed before that subscription, and a re-park does + // not re-fire `onSignalPark`, so a caller holding the decision waits here + // and delivers after the name is recorded. + const awaiterReady = new Set(); + const awaiterWaiters = new Map< + string, + { resolve: () => void; reject: (error: Error) => void }[] + >(); + const resolveAwaiter = (name: string): void => { + awaiterReady.add(name); + const waiters = awaiterWaiters.get(name); + if (waiters === undefined) return; + awaiterWaiters.delete(name); + for (const waiter of waiters) waiter.resolve(); + }; + const rejectAwaiterWaiters = (error: Error): void => { + for (const waiters of awaiterWaiters.values()) { + for (const waiter of waiters) waiter.reject(error); + } + awaiterWaiters.clear(); + }; + const runEnv: WorkflowRuntimeEnv = { ...env, onPark: (park) => { @@ -102,6 +163,11 @@ export function createSuspendableChildHandle( : {}), }, }); + } else if (surfaceInputPark && park.parkKind === "input") { + events.push({ + kind: "input-park", + correlationId: park.correlationId, + }); } else { events.push({ kind: "error", @@ -119,9 +185,16 @@ export function createSuspendableChildHandle( // via `deliverSignal`. Without this the body would park on the signal // channel with nothing upstream to route a delivery to it. onSignalPark: (park) => { - events.push({ kind: "signal-park", name: park.name }); + events.push({ + kind: "signal-park", + name: park.name, + ...(park.timeout !== undefined ? { timeout: park.timeout } : {}), + }); notify(); }, + onSignalAwaiter: (park) => { + resolveAwaiter(park.name); + }, }; // A suspendable body runs IN-PROCESS under the workflow-process child's own @@ -157,6 +230,13 @@ export function createSuspendableChildHandle( terminalStatus: "completed" | "failed" | "cancelled"; } | null = null; let failure: Error | null = null; + // Separate from `wake`. `next` has one consumer; a park-release watch is + // the other, and it must not take the event-queue slot. + const releaseWaiters = new Set<() => void>(); + const notifyReleaseWaiters = (): void => { + for (const wakeRelease of [...releaseWaiters]) wakeRelease(); + }; + let openParkWatch: string | null = null; void handle.complete .then((result) => { settled = { terminalStatus: result.terminalStatus }; @@ -168,8 +248,174 @@ export function createSuspendableChildHandle( detachParentAbort(); if (cleanup !== undefined) void cleanup(); notify(); + notifyReleaseWaiters(); + rejectAwaiterWaiters( + failure ?? + new Error( + `child run ${childRunId} settled before a resumed gate re-armed`, + ), + ); }); + // One read of the child log. `open` means a step is still awaiting + // `name`, or that await is not in the log yet. `next` can still return + // the park event after the gate has moved on, so a caller that already + // holds a signal reads this before accepting it. + const classifyPark = async ( + name: string, + ): Promise< + | { kind: "open"; lastSeq: number } + | { kind: "released" } + | { + kind: "terminal"; + terminalStatus: "completed" | "failed" | "cancelled"; + } + > => { + if (failure !== null) throw failure; + const log = await env.repoStore.read(childRunId); + if (failure !== null) throw failure; + if (settled !== null) { + return { kind: "terminal", terminalStatus: settled.terminalStatus }; + } + const state = resumeFromLog(childRunId, log); + if ( + state.phase === "completed" || + state.phase === "failed" || + state.phase === "cancelled" + ) { + return { kind: "terminal", terminalStatus: state.phase }; + } + for (const step of state.steps.values()) { + if ( + step.phase === "awaiting-signal" && + step.awaitingSignal?.name === name + ) { + return { kind: "open", lastSeq: state.lastSeq }; + } + } + const opened = log.some( + (event) => event.kind === "SignalAwaited" && event.signalName === name, + ); + if (!opened) return { kind: "open", lastSeq: state.lastSeq }; + return { kind: "released" }; + }; + + const readParkRelease = async ( + name: string, + ): Promise< + | { kind: "open" } + | { kind: "released" } + | { + kind: "terminal"; + terminalStatus: "completed" | "failed" | "cancelled"; + } + > => { + const read = await classifyPark(name); + if (read.kind === "open") return { kind: "open" }; + return read; + }; + + // Does not call `next`, so a sibling park stays queued. The park is + // durable before `next` returns it; a read that has not yet seen this + // name's `SignalAwaited` keeps waiting rather than reporting a release. + // The subscription is only a wake. A store that does not replay `from` + // still works. The host adapter's `subscribe` returns a generator that + // attaches its tail on the first pull, not when `subscribe` returns, so + // the pull happens before the second read. That read sees a release + // that landed while the previous snapshot was taken. A release that + // lands during that read wakes the tail. + const waitForParkRelease = async ( + name: string, + abort: AbortSignal, + ): Promise< + | { kind: "released" } + | { + kind: "terminal"; + terminalStatus: "completed" | "failed" | "cancelled"; + } + > => { + if (openParkWatch !== null) { + throw new Error( + `child ${childRunId} already watches ${openParkWatch} for release; ` + + `a drive relays one park at a time`, + ); + } + openParkWatch = name; + try { + if (abort.aborted) throw new Error("aborted"); + for (;;) { + if (abort.aborted) throw new Error("aborted"); + const read = await classifyPark(name); + if (read.kind !== "open") return read; + const fromSeq = read.lastSeq + 1; + const poke = new AbortController(); + const detachPoke = bridgeAbort(abort, () => { + poke.abort(); + }); + const onSettle = (): void => { + poke.abort(); + }; + releaseWaiters.add(onSettle); + let subscription: + | ReturnType + | undefined; + let pending: + | ReturnType< + ReturnType["next"] + > + | undefined; + try { + if (settled !== null || failure !== null || abort.aborted) { + continue; + } + subscription = env.repoStore.subscribe(childRunId, { + signal: poke.signal, + from: { seq: fromSeq }, + }); + // Pull before the second read. On the host store this is what + // attaches the tail; a commit during the read is then in the + // iterator even when that read's snapshot missed it. + pending = subscription.next(); + void pending.catch(() => undefined); + const again = await classifyPark(name); + if (again.kind !== "open") return again; + if (abort.aborted) throw new Error("aborted"); + // The log moved before this tail could see it. A store that + // does not replay `from` would wait forever on the old cursor. + if (again.lastSeq !== read.lastSeq) continue; + let sawEvent = false; + const first = await pending; + pending = undefined; + if (first.done !== true) sawEvent = true; + if (abort.aborted) throw new Error("aborted"); + if ( + !sawEvent && + settled === null && + failure === null && + !abort.aborted + ) { + throw new Error( + `child ${childRunId} park-release watch on ${name} ended before ` + + `the park released`, + ); + } + } finally { + releaseWaiters.delete(onSettle); + detachPoke(); + poke.abort(); + const abandoned = pending; + pending = undefined; + await subscription?.return?.(); + if (abandoned !== undefined) { + await abandoned.catch(() => undefined); + } + } + } + } finally { + openParkWatch = null; + } + }; + return { next: async () => { for (;;) { @@ -186,7 +432,16 @@ export function createSuspendableChildHandle( throw event.error; } if (event.kind === "signal-park") { - return { kind: "signal-park", name: event.name }; + return { + kind: "signal-park", + name: event.name, + ...(event.timeout !== undefined + ? { timeout: event.timeout } + : {}), + }; + } + if (event.kind === "input-park") { + return { kind: "input-park", correlationId: event.correlationId }; } return { kind: "park", park: event.park }; } @@ -205,5 +460,24 @@ export function createSuspendableChildHandle( deliverSignal: async (name, payload, signalId) => { await env.signalChannel.deliver(name, payload, signalId); }, + readParkRelease, + waitForParkRelease, + whenSignalAwaiter: (name) => { + if (awaiterReady.has(name)) return Promise.resolve(); + if (failure !== null) return Promise.reject(failure); + if (settled !== null) { + return Promise.reject( + new Error(`child run ${childRunId} settled before ${name} re-armed`), + ); + } + return new Promise((resolve, reject) => { + const waiters = awaiterWaiters.get(name); + if (waiters !== undefined) { + waiters.push({ resolve, reject }); + return; + } + awaiterWaiters.set(name, [{ resolve, reject }]); + }); + }, }; } diff --git a/vendor/intx/workflow/src/state-machine/events.ts b/vendor/intx/workflow/src/state-machine/events.ts index 0c04e7b8d7..878dca02ca 100644 --- a/vendor/intx/workflow/src/state-machine/events.ts +++ b/vendor/intx/workflow/src/state-machine/events.ts @@ -125,6 +125,12 @@ export interface SignalAwaited extends EventBase { * every reserved-channel park before this change was an approval. */ parkKind?: ControlParkKind; + /** + * Addresses this park by id. A delivery pairs only when this id and + * `signalName` both match. Absent on a name-FIFO park: plain gates, loop + * relays, and signal-relay parks logged before correlated addressing. + */ + correlationId?: string; } export interface SignalReceived extends EventBase { @@ -132,6 +138,12 @@ export interface SignalReceived extends EventBase { signalName: string; signalId: SignalId; payload: unknown; + /** + * Addresses a correlated park. Pairing requires this id and `signalName` + * to match the park. A name mismatch dedups `signalId` and leaves the + * park and both queues untouched. Absent on a name-FIFO delivery. + */ + correlationId?: string; } /** @@ -147,6 +159,11 @@ export interface SignalAwaitAbandoned extends EventBase { kind: "SignalAwaitAbandoned"; stepId: StepId; signalName: string; + /** + * The correlated park being retired. When both this and the live park + * carry an id, they must match. Absent on a name-addressed retire. + */ + correlationId?: string; } export interface TimerSet extends EventBase { diff --git a/vendor/intx/workflow/src/state-machine/index.ts b/vendor/intx/workflow/src/state-machine/index.ts index 934be55a22..a221d1007e 100644 --- a/vendor/intx/workflow/src/state-machine/index.ts +++ b/vendor/intx/workflow/src/state-machine/index.ts @@ -34,6 +34,7 @@ export { isTerminalRunPhase, isTerminalStepPhase, type ChildState, + type CorrelatedQueuedSignal, type PendingTimer, type QueuedSignal, type RunPhase, diff --git a/vendor/intx/workflow/src/state-machine/state.ts b/vendor/intx/workflow/src/state-machine/state.ts index 1c47819185..8481c73a94 100644 --- a/vendor/intx/workflow/src/state-machine/state.ts +++ b/vendor/intx/workflow/src/state-machine/state.ts @@ -44,6 +44,7 @@ export interface StepState { name: string; timeoutAt?: string; parkKind?: ControlParkKind; + correlationId?: string; }; awaitingTimerId?: TimerId; } @@ -96,6 +97,13 @@ export interface QueuedSignal { payload: unknown; } +/** A signal queued for one correlation id. `signalName` is matched on consume. */ +export interface CorrelatedQueuedSignal { + id: SignalId; + payload: unknown; + signalName: string; +} + export interface RunState { runId: RunId; phase: RunPhase; @@ -108,6 +116,12 @@ export interface RunState { observedSignalIds: Set; /** Queue per signal name for pre-await delivery. */ unconsumedSignals: Map; + /** + * Queue per correlation id. Name FIFO never reads or writes this map. + * A queued entry is consumed only by a `SignalAwaited` whose + * `correlationId` and `signalName` both match. + */ + unconsumedByCorrelation: Map; /** * Message-ids consumed by this run's `RunStarted` events. The kind * handler rejects a re-issued `RunStarted` whose message-id appears @@ -129,6 +143,7 @@ export function emptyState(runId: RunId): RunState { pendingTimers: new Map(), observedSignalIds: new Set(), unconsumedSignals: new Map(), + unconsumedByCorrelation: new Map(), consumedMessageIds: new Set(), }; } diff --git a/vendor/intx/workflow/src/state-machine/transition.ts b/vendor/intx/workflow/src/state-machine/transition.ts index a93561bc22..355f66a77b 100644 --- a/vendor/intx/workflow/src/state-machine/transition.ts +++ b/vendor/intx/workflow/src/state-machine/transition.ts @@ -16,12 +16,14 @@ // phases intact (a step that completed before cancellation cascaded // stays completed). // -// Signal delivery is FIFO single-consumer with run-lifetime dedup by -// `signalId`. A `SignalReceived` whose name has no awaiter is queued -// under that name and consumed by the next `SignalAwaited` for the -// same name. - -import type { ControlParkKind } from "@intx/types/runtime"; +// Signal delivery is single-consumer with run-lifetime dedup by +// `signalId`. A delivery with `correlationId` pairs only with the park +// whose id and author name both match, and queues under that id when +// no such park is waiting. A name mismatch against a park that holds +// the id dedups and does not queue. A delivery without `correlationId` +// is name FIFO: it skips parks that carry an id, and a signal with no +// name awaiter queues under that name for the next name-addressed +// `SignalAwaited`. import { CANCEL_ORIGINS } from "./events"; import type { @@ -328,9 +330,12 @@ function handleAttemptScheduled( function handleSignalAwaited(state: RunState, e: SignalAwaited): RunState { const step = requireStep(state, e, e.stepId); + if (e.correlationId !== undefined) { + return awaitCorrelated(state, step, e, e.correlationId); + } // Pre-await queued signal: consume the first unconsumed signal for // this name and keep the step in-flight. Otherwise transition to - // awaiting-signal. + // awaiting-signal. The name queue never holds a correlated delivery. const queue = state.unconsumedSignals.get(e.signalName); if (queue && queue.length > 0) { const remaining = queue.slice(1); @@ -353,6 +358,55 @@ function handleSignalAwaited(state: RunState, e: SignalAwaited): RunState { }; } const steps = new Map(state.steps); + steps.set(e.stepId, { + ...step, + phase: "awaiting-signal", + awaitingSignal: awaitingSignalFrom(e), + }); + return { ...state, steps, lastSeq: e.seq }; +} + +function awaitCorrelated( + state: RunState, + step: StepState, + e: SignalAwaited, + correlationId: string, +): RunState { + const queue = state.unconsumedByCorrelation.get(correlationId) ?? []; + const index = queue.findIndex((item) => item.signalName === e.signalName); + const matched = index >= 0 ? queue[index] : undefined; + if (matched !== undefined) { + const remaining = queue.filter((_, itemIndex) => itemIndex !== index); + const unconsumedByCorrelation = new Map(state.unconsumedByCorrelation); + if (remaining.length === 0) { + unconsumedByCorrelation.delete(correlationId); + } else { + unconsumedByCorrelation.set(correlationId, remaining); + } + const steps = new Map(state.steps); + steps.set(e.stepId, { + ...withoutAwaiters(step), + phase: "in-flight", + }); + return { + ...state, + steps, + unconsumedByCorrelation, + lastSeq: e.seq, + }; + } + const steps = new Map(state.steps); + steps.set(e.stepId, { + ...step, + phase: "awaiting-signal", + awaitingSignal: awaitingSignalFrom(e), + }); + return { ...state, steps, lastSeq: e.seq }; +} + +function awaitingSignalFrom( + e: SignalAwaited, +): NonNullable { // Carry the control-plane park kind through to the reduced state VERBATIM so // recovery can tell an `"input"` park from an `"approval"` one. It is absent // for a plain `awaitSignal` gate and for a legacy park committed before the @@ -360,33 +414,27 @@ function handleSignalAwaited(state: RunState, e: SignalAwaited): RunState { // single point that interprets a reserved-channel absence as a legacy // approval. The reducer does not itself default -- it must not stamp // `"approval"` onto a plain gate that is not one. - const awaitingSignal: { - name: string; - timeoutAt?: string; - parkKind?: ControlParkKind; - } = { + return { name: e.signalName, ...(e.timeoutAt !== undefined ? { timeoutAt: e.timeoutAt } : {}), ...(e.parkKind !== undefined ? { parkKind: e.parkKind } : {}), + ...(e.correlationId !== undefined + ? { correlationId: e.correlationId } + : {}), }; - steps.set(e.stepId, { - ...step, - phase: "awaiting-signal", - awaitingSignal, - }); - return { ...state, steps, lastSeq: e.seq }; } /** - * Owner of the single-consumer signal FIFO: the Map-insertion-order scan for - * the first awaiting step, the `observedSignalIds` dedup, and the queue-on-no- - * awaiter rule live here and nowhere else authoritative. Two out-of-band read- - * side replayers in `runtime/run.ts` reproduce these exact semantics off the - * durable log -- `reconstructGateOutcome` (per-gate) and - * `boundSignalForContainerAwait` (container-relay binder) -- so any change to - * the FIFO, dedup, or queue behavior here must be mirrored in both. The - * `reconstruct-gate-outcome` reducer-fidelity oracle cross-checks the replay - * against this reducer and fails on drift. + * Owner of single-consumer signal pairing: the awaiter scan, the + * `observedSignalIds` dedup, and the queue-on-no-awaiter rule live here and + * nowhere else authoritative. A correlated delivery matches id and author + * name and queues on `unconsumedByCorrelation`; a name delivery skips parks + * that carry an id. Two out-of-band read-side replayers in `runtime/run.ts` + * reproduce these semantics off the durable log -- `reconstructGateOutcome` + * (per-gate name replay) and `boundSignalForContainerAwait` (container-relay + * binder) -- so any change to pairing, dedup, or queue behavior here must be + * mirrored in both. The reducer-fidelity oracles cross-check the replays + * against this reducer and fail on drift. */ function handleSignalReceived(state: RunState, e: SignalReceived): RunState { // At-least-once delivery: duplicate signalIds are no-ops. @@ -396,36 +444,64 @@ function handleSignalReceived(state: RunState, e: SignalReceived): RunState { const observed = new Set(state.observedSignalIds); observed.add(e.signalId); - // FIFO single-consumer: find the first step awaiting this signal name. - let awaiterId: StepId | undefined; + if (e.correlationId !== undefined) { + return receiveCorrelated(state, e, observed, e.correlationId); + } + return receiveByName(state, e, observed); +} + +function receiveCorrelated( + state: RunState, + e: SignalReceived, + observed: Set, + correlationId: string, +): RunState { + let heldByOtherName = false; for (const [id, step] of state.steps) { - if ( - step.phase === "awaiting-signal" && - step.awaitingSignal?.name === e.signalName - ) { - awaiterId = id; - break; + if (step.phase !== "awaiting-signal") continue; + const awaiting = step.awaitingSignal; + if (awaiting?.correlationId !== correlationId) continue; + if (awaiting.name !== e.signalName) { + heldByOtherName = true; + continue; } + return consumeAwaiter(state, e, observed, id); + } + // The id is live on a different author name. Dedup stands; do not + // enqueue and do not clear that park. + if (heldByOtherName) { + return { ...state, observedSignalIds: observed, lastSeq: e.seq }; } + const unconsumedByCorrelation = new Map(state.unconsumedByCorrelation); + const queue = unconsumedByCorrelation.get(correlationId) ?? []; + unconsumedByCorrelation.set(correlationId, [ + ...queue, + { id: e.signalId, payload: e.payload, signalName: e.signalName }, + ]); + return { + ...state, + unconsumedByCorrelation, + observedSignalIds: observed, + lastSeq: e.seq, + }; +} - if (awaiterId !== undefined) { - const steps = new Map(state.steps); - const awaiter = state.steps.get(awaiterId); - if (!awaiter) { - throw new TransitionError( - "unknown-step", - `internal: awaiter ${awaiterId} disappeared between scan and set`, - e, - ); +function receiveByName( + state: RunState, + e: SignalReceived, + observed: Set, +): RunState { + // Name FIFO skips parks addressed by correlation. Those are not on + // this queue, and a name delivery must not complete them. + for (const [id, step] of state.steps) { + if ( + step.phase === "awaiting-signal" && + step.awaitingSignal?.name === e.signalName && + step.awaitingSignal.correlationId === undefined + ) { + return consumeAwaiter(state, e, observed, id); } - steps.set(awaiterId, { - ...withoutAwaiters(awaiter), - phase: "in-flight", - }); - return { ...state, steps, observedSignalIds: observed, lastSeq: e.seq }; } - - // No awaiter present -- queue the signal for a future SignalAwaited. const unconsumed = new Map(state.unconsumedSignals); const queue = unconsumed.get(e.signalName) ?? []; unconsumed.set(e.signalName, [ @@ -440,6 +516,28 @@ function handleSignalReceived(state: RunState, e: SignalReceived): RunState { }; } +function consumeAwaiter( + state: RunState, + e: SignalReceived, + observed: Set, + awaiterId: StepId, +): RunState { + const steps = new Map(state.steps); + const awaiter = state.steps.get(awaiterId); + if (!awaiter) { + throw new TransitionError( + "unknown-step", + `internal: awaiter ${awaiterId} disappeared between scan and set`, + e, + ); + } + steps.set(awaiterId, { + ...withoutAwaiters(awaiter), + phase: "in-flight", + }); + return { ...state, steps, observedSignalIds: observed, lastSeq: e.seq }; +} + /** * Retire a step's `signal-relay` await without delivering a payload. Emitted by * the onTrigger driver when the proxied body progressed by another exit before @@ -452,11 +550,13 @@ function handleSignalReceived(state: RunState, e: SignalReceived): RunState { * awaiter and drop the step to `in-flight`. * * Scope is strict. It touches ONLY a matching `signal-relay` park: an approval - * or input park, or a name mismatch, is a producer bug or a mis-target and is - * rejected, never silently cleared. It never touches `unconsumedSignals`, - * `observedSignalIds`, or `consumedMessageIds` -- a torn-down await does not - * consume or dedup a signal, and a late delivery on the same name still queues - * for the next await, matching a timed-out `awaitSignal`. + * or input park, a name mismatch, or a correlation-id mismatch is a producer + * bug or a mis-target and is rejected, never silently cleared. It never + * touches either unconsumed queue, `observedSignalIds`, or + * `consumedMessageIds`. A torn-down await does not consume or dedup a signal. + * A late name-only delivery still queues for the next name await. A late + * correlated delivery queues under its own id and does not satisfy a + * different correlation. */ function handleSignalAwaitAbandoned( state: RunState, @@ -469,10 +569,14 @@ function handleSignalAwaitAbandoned( return { ...state, lastSeq: e.seq }; } const awaiting = step.awaitingSignal; + const correlationMismatch = + e.correlationId !== undefined && + awaiting?.correlationId !== e.correlationId; if ( awaiting === undefined || controlParkKindOf(awaiting) !== "signal-relay" || - awaiting.name !== e.signalName + awaiting.name !== e.signalName || + correlationMismatch ) { throw new TransitionError( "step-phase", diff --git a/vendor/intx/workflow/src/testing/index.ts b/vendor/intx/workflow/src/testing/index.ts new file mode 100644 index 0000000000..8b497df600 --- /dev/null +++ b/vendor/intx/workflow/src/testing/index.ts @@ -0,0 +1,15 @@ +// @intx/workflow/testing -- waiters for tests that need a run to have reached +// a state before acting on it. +// +// Two observables, because one does not cover every park. The log waiters +// read a run's committed events through the ordinary `RepoStore` surface and +// hold no state of their own, so nothing about them is specific to the +// in-memory store; a production caller wanting to react to committed events +// should subscribe directly rather than through a helper written for test +// ergonomics. They cannot see a RE-PARK, which commits nothing, so the +// observed channel reports that one from the `SignalChannel` seam instead. +export { waitForEvent, waitForNthEvent } from "./log-waiters"; +export { + createObservedSignalChannel, + type ObservedSignalChannel, +} from "./signal-channel-observer"; diff --git a/vendor/intx/workflow/src/testing/log-waiters.ts b/vendor/intx/workflow/src/testing/log-waiters.ts new file mode 100644 index 0000000000..38419a382d --- /dev/null +++ b/vendor/intx/workflow/src/testing/log-waiters.ts @@ -0,0 +1,74 @@ +// Waiters that resolve on a run's committed events. +// +// The runtime interface documents tailing `repoStore.subscribe` as how a +// caller awaits a commit, and these are that pattern for tests: a test that +// needs a run to have reached some state waits for the event proving it +// rather than for an interval long enough to assume it. +// +// Subscribing from seq 0 replays what is already committed before delivering +// live events, so an event that landed before the call counts exactly as a +// later one does -- the property a poll got from re-reading the whole log on +// every tick. Neither waiter carries a deadline: an event that never arrives +// is caught by the lane timeout, per "Synchronizing on State, Not Time" in +// CONVENTIONS.md. + +import type { RepoStore } from "../runtime/env"; +import type { WorkflowEvent } from "../state-machine/index"; + +/** + * Resolve with the `count`-th event `match` accepts, counting from the start + * of the log. + * + * Note it is the count-th match, not the most recent one once `count` exist. + * The two differ whenever more than `count` matching events are already + * committed when the call is made. + */ +export async function waitForNthEvent( + repoStore: RepoStore, + runId: string, + match: (event: WorkflowEvent) => boolean, + count = 1, +): Promise { + if (!Number.isInteger(count) || count < 1) { + throw new Error(`waitForNthEvent: count must be a positive integer`); + } + const abort = new AbortController(); + let seen = 0; + try { + for await (const { event } of repoStore.subscribe(runId, { + signal: abort.signal, + from: { seq: 0 }, + })) { + if (!match(event)) continue; + seen += 1; + if (seen >= count) return event; + } + } finally { + // Ends the subscription whether the match arrived or the loop was left by + // a throw, so the store is not left holding a subscriber for this waiter. + // + // It reads as redundant against an in-memory store and is not. Leaving a + // `for await` by either exit makes the iteration protocol call the + // iterator's `return()`, and an implementation that honours it has already + // closed the subscription by the time this runs -- `createInMemoryRepoStore` + // does, so deleting this line changes nothing observable about that store. + // Honouring `return()` is not required of an iterator, and `RepoStore` is + // an interface: production wraps the substrate's `subscribeKind`. This line + // is what ends the subscription for an implementation that does not. The + // sibling test asserts this signal aborts, rather than asserting store + // state, for the same reason -- store state would look correct either way. + abort.abort(); + } + throw new Error( + `log for ${runId} ended after ${String(seen)} of ${String(count)} matching event(s)`, + ); +} + +/** Resolve once the run's log holds an event `match` accepts. */ +export async function waitForEvent( + repoStore: RepoStore, + runId: string, + match: (event: WorkflowEvent) => boolean, +): Promise { + return waitForNthEvent(repoStore, runId, match, 1); +} diff --git a/vendor/intx/workflow/src/testing/signal-channel-observer.ts b/vendor/intx/workflow/src/testing/signal-channel-observer.ts new file mode 100644 index 0000000000..8255bac653 --- /dev/null +++ b/vendor/intx/workflow/src/testing/signal-channel-observer.ts @@ -0,0 +1,115 @@ +// A `SignalChannel` that reports when the run parks on it. +// +// `waitForEvent` covers a FRESH park, because `parkOnSignal` commits a +// `SignalAwaited` and flushes it before waiting. It cannot cover a RE-PARK: a +// run resuming from a durable log finds its step already `awaiting-signal`, +// so `parkOnSignal` re-adopts the seed's `SignalAwaited` and skips the +// re-emit. Nothing new reaches the log, and a log waiter resolves off the +// seed -- before the run has re-parked -- which makes every "no second +// SignalAwaited was minted" assertion read pre-crash state and pass for the +// wrong reason. +// +// The one observable the re-park does produce is the channel registration +// itself: `parkOnSignal`'s tail calls `awaitNext(name)`. This wraps a channel +// to count those calls per signal name, so a test can wait for the park it +// cares about. +// +// Counting, rather than reporting the next call, is the load-bearing choice. +// A test does not control whether the run re-parks before or after it asks to +// be told, so an edge-triggered "resolve on the next `awaitNext`" deadlocks +// on exactly the interleaving that is most common -- the run parks first. The +// count is level-triggered: asking for a count already reached returns. +// `readCount`/`awaitReadCount` in `@intx/workflow-host/testing` is the same +// pair for the same reason. + +import { createInMemorySignalChannel } from "../runlocal/signal-channel"; +import type { SignalChannel } from "../runtime/env"; + +export type ObservedSignalChannel = SignalChannel & { + /** How many times the run has called `awaitNext` for this name so far. */ + awaitedCount(name: string): number; + /** + * Resolve once `awaitNext` has been called for `name` at least `count` + * times, including when it already has been. + */ + awaitAwaitedCount(name: string, count: number): Promise; +}; + +/** + * Wrap a fresh in-memory signal channel so a test can await the run parking + * on it. `deliver` and `awaitNext` behave exactly as the unwrapped channel + * does; the observation is additive. + */ +export function createObservedSignalChannel(): ObservedSignalChannel { + const inner = createInMemorySignalChannel(); + const counts = new Map(); + let waiters: (() => void)[] = []; + + function announce(): void { + // Clearing the list is what bounds it. Every pass of the wait loop below + // registers a resolver, and the pass that returns leaves its own resolver + // behind unsettled; clearing here is what discards those rather than + // accumulating one per park for the life of the channel. Measured over 50 + // parks with one waiter: 1 entry with the clear, 51 without. + // + // Clearing BEFORE the wake rather than after decides neither of those. It + // is not what makes a re-arming waiter wait for the next park instead of + // the current one, and it is not what bounds the list either. Waking a + // settled resolver is a no-op, and the loop re-registers only after this + // function has finished iterating, so both orders behave identically and + // both hold the list at 1 -- a mutant that wakes the live list and clears + // afterwards passes every test in the sibling test file. + const waking = waiters; + waiters = []; + for (const waiter of waking) waiter(); + } + + function countOf(name: string): number { + const seen = counts.get(name); + // A name with no entry is one the run has not awaited, which is zero. The + // map records observations; it is not a source of truth that can fail. + return seen === undefined ? 0 : seen; + } + + return { + deliver(name, payload, signalId, correlationId) { + if (correlationId === undefined) { + return inner.deliver(name, payload, signalId); + } + return inner.deliver(name, payload, signalId, correlationId); + }, + awaitNext(name, signal, correlationId) { + counts.set(name, countOf(name) + 1); + announce(); + if (correlationId === undefined) { + return inner.awaitNext(name, signal); + } + return inner.awaitNext(name, signal, correlationId); + }, + awaitedCount: countOf, + async awaitAwaitedCount(name, count) { + if (!Number.isInteger(count) || count < 1) { + throw new Error( + `awaitAwaitedCount: count must be a positive integer, got ${String(count)}`, + ); + } + for (;;) { + // Re-checking on every pass is the invariant: it is what lets a park + // that already happened satisfy the wait. Drop the check and await + // the next report instead, and the "satisfied by a park that happened + // BEFORE the call" test hangs -- which is the failure mode this shape + // exists to avoid. + // + // The ORDER of the registration and the check is not the invariant, + // and no comment should claim it is. The executor runs synchronously, + // so no `awaitNext` can land between them in either arrangement; + // swapping the two lines passes every test in the sibling test file. + const awaited = new Promise((resolve) => { + waiters.push(resolve); + }); + if (countOf(name) >= count) return; + await awaited; + } + }, + }; +}