From 6e9d7dd7917205a2f7c24c309cec5e1360383e67 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:03:12 -0700 Subject: [PATCH 01/31] chore(hub): comments say why, never what --- apps/hub/src/mailbox-persist.ts | 70 +++--------------- apps/hub/src/provisioners/process.ts | 76 +++++--------------- apps/hub/src/provisioners/sandbox-sidecar.ts | 49 ++++--------- apps/hub/src/server.ts | 19 +---- 4 files changed, 45 insertions(+), 169 deletions(-) diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index 5f6cae0fdb..e33bc1e466 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -1,11 +1,6 @@ -// Wires `@corbits/mailbox`'s `createMailboxPersist` onto the hub's own -// `persistMail` lookup (`vendor/intx/hub-sessions/src/hub-session-lookups.ts`) -// so every outbound agent frame also lands a durable `principal_mail` row in -// each addressed human participant's mailbox, not just `session_mail`. -// -// One seam lives here, host-owned by the package's own contract -// (`persist.ts` in `@corbits/mailbox`): `authorizeSender` decides which -// sender addresses may write at all. +// Wires @corbits/mailbox's createMailboxPersist onto the hub's own +// persistMail lookup so every outbound frame also lands a durable +// principal_mail row, not just session_mail. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; @@ -19,46 +14,10 @@ import { getLogger } from "@intx/log"; const logger = getLogger(["hub", "mailbox-persist"]); /** - * Resolve a `mail.outbound` frame's sender run address to the mailbox - * authorization the package needs: the tenant the resulting rows belong to, - * and the mail domain that scopes which recipients are even addressable. - * - * Mirrors exactly what `persistMail` itself already does to resolve a sender - * (`resolveRoutableAddress`, the same routable-address resolver) rather than - * re-deriving liveness some other way: a sender that is not a live run - * resolves to `undefined` there and to `null` here, which skips the mailbox - * write while the frame still goes upstream unchanged. - * - * This is a genuine double resolve of the same address across one frame -- - * `baseLookups.persistMail` (vendor-owned, `hub-session-lookups.ts`) resolves - * `senderAddress` for its own `session_mail` write, and this seam resolves it - * again for mailbox authorization. `createMailboxPersist` calls `upstream` - * and `authorizeSender` as two independent stages and hands neither's result - * to the other, so there is no seam to thread one resolution through without - * changing the vendor-owned `persistMail` signature itself, which the - * ground rules rule out. Accepted as the cost of two owners agreeing on one - * fact from two directions: one extra indexed lookup by address per frame, - * not per recipient. - */ -/** - * Wraps the hub's own `persistMail` (vendor's `baseLookups.persistMail`) - * so a run's `agent_session` and event collector exist before that write - * runs: the first inbound mail on a freshly triggered run is often the - * earliest point the run is mail-routable at all, since - * `workflow_run.principal_id` only reconciles onto the trigger that just - * fired. Resolves the frame's own sender address to its run - * (`resolveRoutableAddress`, the same resolver `persistMail` itself - * uses) rather than assuming the caller already knows it; a sender that - * is not a live run resolves to `undefined` and this is a no-op past - * that point. Best-effort: `ensureRunSession` failing must not swallow - * the mail upstream is about to persist regardless, so this only - * degrades to "no session yet" rather than ever throwing past itself. - * - * `eventCollectorsRef` is a forward reference (the same pattern this - * file's other composition-order refs use, e.g. `apps/hub/src/index.ts`'s - * `grantAllowanceGateRef`): the wrapped `eventCollectors` registry isn't - * constructed until after `lookups` is, so this only reads `.current` - * inside the returned closure, never at wiring time. + * Ensures a run's agent_session/event collector exist before the first + * mail-triggered write reaches it, since workflow_run.principal_id only + * reconciles onto the trigger that just fired. Best-effort: a failure + * here must not block the mail upstream is about to persist regardless. */ export function createHubPersistMailWithSessionEnsure( db: DB["db"], @@ -78,22 +37,15 @@ export function createHubPersistMailWithSessionEnsure; }; -/** - * The narrow OS port this backend needs: start a detached child, ask - * whether a pid is still alive, and signal it. Injected so the unit - * suite exercises the full ensure/destroy lifecycle — including - * generation fencing and directory cleanup — without starting a real - * sidecar. - */ +/** Injected so tests exercise the full ensure/destroy lifecycle without starting a real sidecar. */ export interface SidecarProcessRunner { spawn(args: SpawnSidecarProcessArgs): number; isAlive(pid: number): boolean; @@ -118,11 +106,9 @@ export interface SidecarProcessRunner { } /** - * Bun implementation. The child is spawned with its own stdio inherited - * from the hub so a local operator sees sidecar logs in the same - * terminal, and `unref`ed so a hub shutdown is not blocked waiting on - * it — a still-running sidecar is reconciled (or destroyed) on the next - * boot from its pid file. + * Stdio inherited so a local operator sees sidecar logs in the same + * terminal; unref()ed so a hub shutdown doesn't wait on it — a still-running + * sidecar is reconciled from its pid file on the next boot. */ export function createBunSidecarProcessRunner(): SidecarProcessRunner { return { @@ -169,24 +155,11 @@ const TERMINATION_GRACE_MS = 5_000; const TERMINATION_POLL_MS = 100; /** - * Implements `SidecarBackend` (see `./sandbox-sidecar`) by running one - * `apps/sidecar` process per allocation on the hub host itself, the only - * sidecar backend, so one server hosts many chats and workflows with no - * container runtime and no remote sandbox account. - * - * Layout under the hub's data dir, one directory per started unit: - * - * //gen-/sidecar.pid - * //gen-/data/ - * - * `data/` is that sidecar's own `SIDECAR_DATA_DIR`. The pid file is what - * makes this backend survive a hub restart: the child handle is gone, but - * `findUnitsByAllocation` still finds every live unit from disk, so the - * core can sweep a superseded generation and destroy can still clean up - * after a unit this process never started. Scoping a unit by generation - * (rather than one pid file per allocation) is what keeps that sweep - * honest — a new generation's pid never overwrites the record of the one - * it is replacing. + * One apps/sidecar process per allocation on the hub host — the only + * backend. Scoping each unit's directory by generation (not one pid file + * per allocation) is what lets this backend survive a hub restart: + * findUnitsByAllocation recovers every live unit from disk by pid file, + * and a new generation's pid never overwrites the one it supersedes. */ export function createProcessBackend( runner: SidecarProcessRunner, @@ -319,13 +292,7 @@ export function createProcessBackend( }; } -/** - * The sidecar's own boot config (`apps/sidecar/src/config.ts`) requires - * `SIDECAR_DATA_DIR`, `HUB_WS_URL`, `SIDECAR_ID`, `SIDECAR_TOKEN` and - * `PATH`, and forwards `HOME`/`TMPDIR` into each workflow-process child. - * Nothing else of the hub's environment is inherited: a sidecar learns - * everything else over the wire. - */ +/** A sidecar learns everything else over the wire; nothing else of the hub's env is inherited. */ function sidecarEnvFor(args: StartUnitArgs, sidecarDataDir: string): Record { const path = process.env["PATH"]; if (path === undefined || path === "") { @@ -453,11 +420,10 @@ const PROVISIONER_API_VERSION = 1 as const; export const PROCESS_PROVISIONER_ID = "process"; /** - * Which allocations this instance owns. Interchange adopts a probe's - * allocation for the deployment when the two provisioners share id, api - * version and binding fingerprint; that adopt path does not deploy the - * workflow after the sidecar reconnects at the current pin, so the hub - * runs a separate probe instance whose fingerprint never matches. + * Interchange adopts a probe's allocation for the deployment when both + * provisioners share id/apiVersion/fingerprint, which skips redeploying at + * the current pin — so the hub runs a probe instance whose fingerprint + * never matches. */ export type ProcessProvisionerRole = "deployment" | "probe"; @@ -469,15 +435,9 @@ export type CreateProcessSidecarProvisionerOpts = { }; /** - * The only sidecar backend: every allocation is a child process of the - * hub on the same host. Idempotence, generation fencing, and destroy - * tombstones come from `./sandbox-sidecar`'s shared core — this module - * supplies only the OS-level unit. - * - * The binding fingerprint pins the two facts that decide what a - * provisioned sidecar actually is: which entry point runs, and which hub - * it dials. A change to either is a different backend binding, so - * allocations bound to the old one are not silently treated as current. + * The binding fingerprint pins entry point and hub URL: a change to either + * is a different backend binding, so allocations bound to the old one + * aren't silently treated as current. */ export function createProcessSidecarProvisioner( opts: CreateProcessSidecarProvisionerOpts, diff --git a/apps/hub/src/provisioners/sandbox-sidecar.ts b/apps/hub/src/provisioners/sandbox-sidecar.ts index 47c779523b..932a1a46d7 100644 --- a/apps/hub/src/provisioners/sandbox-sidecar.ts +++ b/apps/hub/src/provisioners/sandbox-sidecar.ts @@ -1,8 +1,5 @@ -// Backend-agnostic sidecar allocation lifecycle: state store and -// ensure/destroy convergence for a SidecarProvisioner. A backend supplies -// only the SidecarBackend port (start/stop/find) and identity -// (id/apiVersion/bindingFingerprint); this module owns everything else a -// SidecarProvisioner needs to behave correctly under concurrent and +// Backend-agnostic lifecycle: a backend supplies only start/stop/find and +// identity; this module owns correctness under concurrent and // out-of-order ensure()/destroy() calls. import { randomBytes } from "node:crypto"; import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; @@ -39,13 +36,7 @@ export interface SidecarBackend { findUnitsByAllocation(allocationId: string): Promise; } -/** - * A backend throws this from startUnit()/stopUnit() to report a - * classified failure — its own error code and whether it is worth - * retrying — rather than a generic Error. The core forwards code/message/ - * retryable straight into the rejected EnsureSidecarResult/ - * DestroySidecarResult. - */ +/** Lets a backend report a classified, retryable-or-not failure instead of a generic Error. */ export class BackendOperationError extends Error { readonly code: string; readonly retryable: boolean; @@ -59,12 +50,10 @@ export class BackendOperationError extends Error { } /** - * The isolation a sidecar backend gives the code it runs, as a ladder: a - * container also isolates the process, and a VM also isolates the - * container. A backend declares every rung it reaches as `available` and - * every rung above it as `blocked`, so a deployment that requires - * `isolation:vm` fails closed on the process backend instead of quietly - * landing on a shared kernel. + * A ladder: a container also isolates the process, a VM also isolates the + * container. A backend declares every rung it reaches as available so a + * deployment requiring isolation:vm fails closed rather than landing quietly + * on a shared kernel. */ export const SIDECAR_ISOLATION_LEVELS = ["process", "container", "vm"] as const; @@ -372,10 +361,8 @@ export type CreateSidecarProvisionerOpts = { readonly apiVersion: 1; readonly bindingFingerprint: string; /** - * Capabilities this backend declares to the hub's capability policy. - * Empty declares nothing, which matches any deployment that states no - * capability requirement -- the behaviour every allocation had before - * Interchange replaced the placement model with capability selection. + * Empty matches any deployment stating no requirement — the behaviour + * every allocation had before capability selection replaced placement. */ readonly capabilities: readonly SidecarCapabilityDeclaration[]; readonly backend: SidecarBackend; @@ -441,13 +428,9 @@ export function createSidecarProvisioner(opts: CreateSidecarProvisionerOpts): Si false, ); } - // Removes any other unit still labeled for this allocation: an older - // generation's unit this ensure() is superseding, a duplicate left - // behind by a concurrent ensure() that lost the race to record its - // unit, or a unit from a prior process that crashed between starting - // the unit and recordUnit(). All three look identical from here — "a - // unit for this allocation that isn't the one we just recorded" — so - // one sweep by allocation covers all of them. + // Covers three cases indistinguishable from here (superseded generation, + // a losing concurrent ensure(), a crash between startUnit and + // recordUnit) with one sweep by allocation. await sweepObsoleteUnits(backend, request.allocationId, externalRef); return { kind: "accepted", externalRef }; } @@ -513,13 +496,7 @@ export function createSidecarProvisioner(opts: CreateSidecarProvisionerOpts): Si }; } -/** - * Stops every unit labeled for this allocation other than the one just - * confirmed current, best-effort: a sweep failure is logged to stderr - * rather than failing the ensure() it runs alongside, since the - * allocation itself is already correctly ensured by this point and an - * orphaned unit is a leak, not a correctness bug. - */ +/** Best-effort: a sweep failure only leaks a unit, so it's logged rather than failing ensure(). */ async function sweepObsoleteUnits( backend: SidecarBackend, allocationId: string, diff --git a/apps/hub/src/server.ts b/apps/hub/src/server.ts index c025521ef0..c8cf4f9ff3 100644 --- a/apps/hub/src/server.ts +++ b/apps/hub/src/server.ts @@ -48,13 +48,8 @@ import { upgradeWebSocket, websocket } from "hono/bun"; import { setup, getLogger } from "@intx/log"; import { Hono } from "hono"; -// --------------------------------------------------------------------- -// Everything above this line is upstream Interchange's own -// apps/hub/src/server.ts, verbatim. Corbits libraries are mounted in one -// delimited block below, after createApp(...) and before the function's -// return -- see AGENTS.md's "Workbench is a plain Interchange tenant" -// ruling. Nothing else in this file is Workbench-specific. -// --------------------------------------------------------------------- +// Everything above this line is upstream Interchange's server.ts, verbatim; +// see AGENTS.md's "plain Interchange tenant" ruling. import { buildMailFrame, createInMemoryMailboxEventBus, @@ -470,14 +465,8 @@ export async function createHubServer({ }), }); - // --------------------------------------------------------------------- // Corbits mount block -- everything Workbench adds to the stock hub. - // Each library gets its own Hono routed under the tenant - // prefix (or "/" when the library's own routes already carry that - // prefix), reusing the app's own db/grantStore/credentialCipher/router. - // See AGENTS.md: "any other hub mount is cutover debt with a Linear - // issue, never a pattern to extend." - // --------------------------------------------------------------------- + // See AGENTS.md: any other hub mount is cutover debt, never a pattern to extend. const TENANT_PREFIX = "/api/tenants/:tenantId"; const corbitsDatabaseUrl = `postgres://${encodeURIComponent(process.env["DB_USER"] ?? "postgres")}:${encodeURIComponent(process.env["DB_PASSWORD"] ?? "postgres")}@${process.env["DB_HOST"] ?? "localhost"}:${String(Number(process.env["DB_PORT"] ?? 5432))}/${process.env["DB_NAME"] ?? "interchange"}`; const { db: mailboxDb } = createMailboxDb(corbitsDatabaseUrl); @@ -665,9 +654,7 @@ export async function createHubServer({ }), ); - // --------------------------------------------------------------------- // End of Corbits mount block. - // --------------------------------------------------------------------- log.info("Starting server on port {port}", { port }); From 993f0e627c5a7006211f60db4bd486059cda2f26 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:06:15 -0700 Subject: [PATCH 02/31] chore(sidecar): comments say why, never what (part 1) --- apps/sidecar/src/signing-keypair.ts | 18 +- apps/sidecar/src/source-asset-delivery.ts | 57 +---- .../src/step-credential-capabilities.ts | 108 ++------- apps/sidecar/src/tool-materialization.ts | 229 +++--------------- docs/sidecar-tool-package-durability.md | 32 +++ 5 files changed, 110 insertions(+), 334 deletions(-) create mode 100644 docs/sidecar-tool-package-durability.md diff --git a/apps/sidecar/src/signing-keypair.ts b/apps/sidecar/src/signing-keypair.ts index 8d3932dc48..0c4eda726a 100644 --- a/apps/sidecar/src/signing-keypair.ts +++ b/apps/sidecar/src/signing-keypair.ts @@ -28,20 +28,10 @@ async function exists(filePath: string): Promise { } /** - * Load the sidecar's persisted Ed25519 signing keypair, minting a fresh - * one on first boot. - * - * The 32-byte seed in `ed25519.private` is the sole source of truth for - * the sidecar's identity; the public key is always derived from it. The - * `ed25519.public` file is an identity anchor, not a cache: on every load - * it is cross-checked against the seed-derived public key. A mismatch - * means either the seed or the public file was corrupted or swapped (a - * bad backup restore, disk bitrot, an operator fat-finger). Rather than - * trusting the file and advertising a public key the sidecar cannot sign - * with -- which would make every signature fail verification at the hub, - * far from the root cause -- we fail loudly here at boot. We cannot tell - * which of the two files rotted, so we halt and let an operator decide - * (restore the seed, or remove the directory to mint a fresh identity). + * The seed file is the sole source of identity; the public file is cross + * checked against it on every load, not trusted as a cache, so a corrupted + * or swapped file fails loud at boot instead of silently failing signature + * verification at the hub later. */ export async function loadOrMintSidecarKeypair(signingDir: string): Promise { const privateKeyPath = path.join(signingDir, PRIVATE_KEY_FILENAME); diff --git a/apps/sidecar/src/source-asset-delivery.ts b/apps/sidecar/src/source-asset-delivery.ts index 28e87e13a0..7170834a1f 100644 --- a/apps/sidecar/src/source-asset-delivery.ts +++ b/apps/sidecar/src/source-asset-delivery.ts @@ -1,14 +1,6 @@ -// Sidecar-side delivery of a workflow closure's source assets. -// -// A `WorkflowSourceAssetMount` carries a git pack for one hub asset. How that -// pack is materialized depends on how the closure references the asset: -// - a tarball-format entry reads a `.tgz` blob from a plain-file checkout -// (`applyAssetPack`), keyed by an `assetId -> mountPath` map; and -// - a source-format entry checks a subtree out of the git objects, so the -// pack is indexed into a RETAINED `.git` (a "gitDir"), keyed by an -// `assetId -> gitDir` map the loader hands `materializeGitEntry`. -// One asset can be referenced both ways; a source-format workflow closure -// references only source entries, so it produces only gitDirs. +// Sidecar-side delivery of a workflow closure's source assets. A tarball +// entry checks out a plain-file mount; a source entry indexes the pack +// into a retained gitDir. One asset can be referenced both ways. import fsp from "node:fs/promises"; import path from "node:path"; @@ -28,18 +20,9 @@ const logger = getLogger(["sidecar", "source-asset-delivery"]); const SAFE_ASSET_ID = /^[a-zA-Z0-9_.-]+$/; /** - * Index a delivered asset pack into `gitDir` and RETAIN the object store, so a - * source subtree can be checked out from it. Builds into a sibling temp `.git` - * and RENAMES it into place, so the durable store is complete-or-absent: a crash - * mid-materialization leaves only the temp, never a partial `gitDir` that the - * dir-exists check `resolveDeploymentAssetMounts` runs on restore would trust. - * The rename is same-filesystem (the temp is a sibling under `gitDir`'s parent). - * - * On a rename conflict (a stale `gitDir` from a torn prior attempt) the freshly - * built store wins: the existing dir is removed and the temp renamed over it, so - * a re-delivery at a new commit never keeps the old content. A secondary rm - * failure is logged so it does not silently mask state; the primary error is - * rethrown. + * Builds into a sibling temp .git and renames into place so the durable + * store is complete-or-absent: a crash mid-materialization never leaves a + * partial gitDir the restore dir-exists check would trust. */ export async function indexAssetPackIntoGitDir(args: { pack: Uint8Array; @@ -68,16 +51,12 @@ export async function indexAssetPackIntoGitDir(args: { try { await fsp.rename(tempDir, gitDir); } catch (err) { - // Only a "destination already exists" failure means a torn prior attempt we - // may supersede; any other rename error (EXDEV, EACCES, ENOSPC, EIO) must - // NOT destroy a possibly-good prior store -- clean up only the fresh temp - // and surface it. + // Only "destination exists" means a torn prior attempt to supersede; + // any other rename error must not destroy a possibly-good prior store. if (!isDestinationExistsError(err)) { await cleanupTemp(); throw err; } - // The final path already holds a store (a torn prior attempt): rebuild - // wins, so drop the stale store and rename the fresh one over it. await fsp.rm(gitDir, { recursive: true, force: true }); try { await fsp.rename(tempDir, gitDir); @@ -119,24 +98,15 @@ export function assetReferenceFormats( /** The absolute gitDir a source asset's objects are indexed into. */ export function sourceAssetGitDir(gitDirRoot: string, assetId: string): string { - // Reject an all-dots assetId (".", "..", ...) before the join. SAFE_ASSET_ID - // permits "." as a character, so a bare ".." would otherwise escape the - // per-asset dir (`path.join(root, "..")` is root's parent) and "." would - // resolve to the shared root itself. Mirrors `applyAssetPack`'s all-dots - // segment guard. + // SAFE_ASSET_ID permits "." as a character, so an all-dots id must be + // rejected separately or it escapes/aliases the per-asset dir. if (!SAFE_ASSET_ID.test(assetId) || /^\.+$/.test(assetId)) { throw new Error(`source-asset delivery: unsafe assetId ${JSON.stringify(assetId)}`); } return path.join(gitDirRoot, assetId); } -/** - * The single cap on the total inline (base64) source-asset payload a workflow - * closure may deliver in one frame. Both the probe and the deploy pass this to - * `materializeWorkflowAssets`; a git-sourced asset that grows past it is the - * signal to move that path's asset delivery to a streamed transfer. One - * constant so the two paths cannot drift. - */ +/** Shared by probe and deploy so the two paths cannot drift; exceeding it signals moving to a streamed transfer. */ export const MAX_INLINE_ASSET_PAYLOAD_BYTES = 32 * 1024 * 1024; /** @@ -173,10 +143,7 @@ export async function materializeWorkflowAssets(args: { ); } seen.add(asset.assetId); - // The frame delivers one mount per asset the closure references, so a - // delivered asset with no closure entry is a hub/frame inconsistency. - // Fail loud rather than silently ignore it (while still counting its - // payload toward the cap above). + // An asset with no closure entry is a hub/frame inconsistency; fail loud. const refs = formats.get(asset.assetId); if (refs === undefined) { throw new Error( diff --git a/apps/sidecar/src/step-credential-capabilities.ts b/apps/sidecar/src/step-credential-capabilities.ts index 354aae5031..6ca875d6b0 100644 --- a/apps/sidecar/src/step-credential-capabilities.ts +++ b/apps/sidecar/src/step-credential-capabilities.ts @@ -1,26 +1,9 @@ -// Per-consumer assembly of the runtime `credentials` capability for a step's -// tool bundles. -// -// A step can carry tool factories from several packages, and each package is a -// distinct credential consumer (`toolConsumer(packageName)`). Gate 2 -- the -// `{ tool }` condition on a `credential:{id}` / `use` grant -- is baked into -// each capability at construction, so a single capability shared across a -// step's packages would be a confused-deputy hole (package A's grant would -// authorize package B's resolve). This module therefore builds ONE capability -// per package, keyed by package name, and the bundle-invocation site layers -// each package's capability onto only that package's bundles. -// -// Two invariants this module owns, both fail-closed: -// 1. Material is read from the LIVE delivery cell on every use, never -// snapshotted at shape time. A rotation updates the cell in place; a -// re-push that revokes a consumer's access DROPS the material, and an -// already-shaped handle must starve the instant that happens. This live -// read is the revocation-enforcement surface -- Gate 2 stays a -// launch/first-resolve gate (Seam A memoizes it), so mid-deploy -// revocation takes effect here, at the material, not at the gate. -// 2. The declared-vs-bound reconcile runs per consumer against that -// consumer's OWN bound set, so package A's declaration can never be -// satisfied by a handle bound for package B. +// One credential capability per package (not one shared per step), since +// sharing across packages would let package A's grant authorize package +// B's resolve. Two fail-closed invariants: material is read from the live +// delivery cell on every use (not snapshotted), so revocation takes effect +// at the material rather than only at the launch-time grant gate; and the +// declared-vs-bound reconcile runs per consumer against its own bound set. import { toolConsumer } from "@intx/authz"; import { @@ -40,46 +23,27 @@ import type { CredentialDelivery } from "@intx/types/sidecar"; import type { StepToolFactory } from "./tool-materialization"; -/** - * The mutable cell the control channel writes each credential delivery into. - * Held structurally (rather than importing `CredentialMaterialRef` from - * `@intx/workflow-host`) so this module stays free of the transport package and - * a test can drive it with a plain object. - */ +/** Held structurally so this module stays free of the transport package and a test can use a plain object. */ export interface CredentialMaterialCell { readonly current: CredentialDelivery | null; } -/** - * Everything the per-consumer assembly needs beyond the step's factories: the - * live material cell, the step's grants for Gate 2, and the provider registry - * that shapes a credential into a mediated handle. The sidecar assembles this - * at the invoke-step boundary -- the material cell and grants ride in from the - * run child, the providers are sidecar-static. - */ +/** Assembled at the invoke-step boundary: material cell and grants ride in from the run child; providers are sidecar-static. */ export interface StepCredentialWiring { readonly materialCell: CredentialMaterialCell; /** - * Resolves the step's grants for Gate 2. A thunk rather than a resolved - * array, so the grants are read only when a capability is actually built: a - * step with no credential-consuming package never touches them. This matters - * on the resume path -- a run resumed by self-discovery does not pass through - * the pre-trigger grants barrier, so the credentials snapshot a Gate-2 read - * needs may not be present yet; a toolless resume must not fault on that. + * A thunk, not a resolved array: a resumed run may not yet have the + * grants snapshot a Gate-2 read needs, and a toolless resume must not + * fault on that. */ readonly resolveGrants: () => readonly GrantRule[]; readonly providers: CredentialProviderRegistry; } /** - * Build the credential capabilities for a step's tool factories, one per - * distinct package, keyed by package name. - * - * A package with neither a declared nor a bound credential gets no entry: its - * bundles keep the base capabilities bag untouched, so `resolve("credentials")` - * fails closed as "not provided by host" rather than handing back an empty - * sub-registry. Every returned capability owns a `dispose` the caller must run - * on teardown. + * A package with neither declared nor bound credentials gets no entry, so + * resolve("credentials") fails closed rather than handing back an empty + * sub-registry. Every returned capability owns a dispose the caller must run. */ export function buildCredentialCapabilities( factories: readonly StepToolFactory[], @@ -87,9 +51,6 @@ export function buildCredentialCapabilities( ): Map { const byPackage = new Map(); const seenPackages = new Set(); - // Resolve the step's grants at most once, and only if some package actually - // needs a capability -- a step with no credential-consuming package never - // reads them (see StepCredentialWiring.resolveGrants). let grants: readonly GrantRule[] | undefined; for (const stf of factories) { @@ -99,9 +60,7 @@ export function buildCredentialCapabilities( const consumer = toolConsumer(stf.packageName); const bindings = buildConsumerBindings(consumer, wiring.materialCell); - // Fail the launch closed if the package declares a handle no binding - // resolves for it -- the tool needs a credential the definition never - // bound. Checked against THIS consumer's bound set only (invariant 2). + // Fail closed against this consumer's own bound set only. reconcileDeclaredCredentials(consumer, stf.declaredCredentials, new Set(bindings.keys())); if (stf.declaredCredentials.length === 0 && bindings.size === 0) { @@ -123,12 +82,7 @@ export function buildCredentialCapabilities( return byPackage; } -/** - * Join the delivery's descriptors and materials into the binding map one - * consumer's capability is built from: every descriptor addressed to this - * consumer, resolved to the material that backs its credential. A descriptor - * whose material is absent is a malformed delivery and fails closed at build. - */ +/** A descriptor addressed to this consumer with no matching material is a malformed delivery and fails closed. */ function buildConsumerBindings( consumer: string, cell: CredentialMaterialCell, @@ -144,16 +98,12 @@ function buildConsumerBindings( (entry) => entry.credentialId === descriptor.credentialId, ); if (material === undefined) { - // The hub sends a material for every descriptor's credential; a - // descriptor with no material is a delivery bug. Refuse it at build - // rather than let the handle resolve to nothing at first use. throw new Error( `credential delivery is malformed: descriptor for handle "${descriptor.handle}" (consumer ${consumer}) references credential ${descriptor.credentialId} but the delivery carries no material for it`, ); } - // Capture the provider key and origin at build time (they identify the - // credential and its pinned origin); the secret is read live per use. + // provider/origin are pinned at build time; the secret is read live. bindings.set(descriptor.handle, { credentialId: descriptor.credentialId, providerKey: material.providerKey, @@ -171,12 +121,7 @@ function buildConsumerBindings( return bindings; } -/** - * The rotation/revocation indirection a shaped handle reads through: it looks - * the credential's material up in the LIVE cell on every call (invariant 1), - * failing closed when the material is gone (a re-push dropped it) and asserting - * the provider/origin have not drifted under the already-shaped handle. - */ +/** Reads the live cell on every call so a re-push's revocation takes effect immediately. */ function makeReadCurrentMaterial(args: { cell: CredentialMaterialCell; credentialId: string; @@ -198,10 +143,8 @@ function makeReadCurrentMaterial(args: { `credential material for ${credentialId} (consumer ${consumer}) is no longer delivered: a re-push dropped it (rotated away or revoked)`, ); } - // A rotation changes the SECRET, not the provider or origin. A live entry - // whose provider/origin diverged would mean the shaped handle now - // authenticates somewhere it was not pinned to; surface it rather than - // silently follow the change. + // A rotation changes only the secret; a provider/origin drift would mean + // the handle now authenticates somewhere it wasn't pinned to. if (material.providerKey !== providerKey || material.origin !== origin) { throw new Error( `credential ${credentialId} changed provider/origin under an already-shaped handle (${providerKey}@${origin} -> ${material.providerKey}@${material.origin}); a shaped handle cannot follow that change`, @@ -212,14 +155,9 @@ function makeReadCurrentMaterial(args: { } /** - * A generic inference credential resolver over the live cell: resolves a - * credential's current secret by `credentialId`, failing closed when the cell is - * empty or the credential is absent (a re-push dropped it -- rotated away or - * revoked). Unlike `makeReadCurrentMaterial` it is keyed by `credentialId` at - * call time -- an inference source's forward-only failover chain carries a - * distinct credential per entry -- and pins no provider/origin: an inference - * request authenticates to the source's own `baseURL` (resolved separately), - * so there is no shaped handle to protect against origin drift. + * Unlike makeReadCurrentMaterial, pins no provider/origin: an inference + * request authenticates to its own baseURL, so there's no handle to + * protect against origin drift. */ export function createInferenceCredentialResolver( cell: CredentialMaterialCell, diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index 23efead255..978da03b2a 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -1,19 +1,9 @@ -// Tool-package materialization for the sidecar. -// -// The deploy-tree reader, the `@intx/tool-packaging` loader -// construction, `applyAtomic`, and the active-deploy-id persistence -// ladder live here so the workflow-process child's substrate factory -// (`workflow-substrate-factory.ts`) reaches this implementation without -// the portable orchestration package depending on the sidecar's tool -// runtime. Keeping this in `apps/sidecar` (and out of -// `@intx/workflow-host`) preserves that package's host-agnostic -// layering: the child IS the sidecar binary, so the sidecar's tool -// runtime is already present in the child's address space. -// -// The module is deliberately free of any `@intx/harness` reactor -// dependency so the workflow-process child's boot graph does not pull -// in `@intx/harness`'s transport/reactor stack -- a forbidden-import -// guard enforces that boundary. +// Lives in apps/sidecar (not @intx/workflow-host) to keep that package +// host-agnostic; the child IS the sidecar binary, so its tool runtime is +// already present. Kept free of @intx/harness so the workflow-process +// child's boot graph doesn't pull in its reactor stack (enforced by a +// forbidden-import guard). +// See docs/sidecar-tool-package-durability.md for the deploy-id persist protocol. import fs from "node:fs"; import path from "node:path"; @@ -35,23 +25,13 @@ import { readRegistries, resolveHostPlatform } from "./sidecar-materialization-c const logger = getLogger(["sidecar", "harness-builder"]); -// Sentinel `previousDeployId` for an instance that has never applied -// a deploy successfully. Encoded as a literal string so the value -// travels through `applyAtomic`'s `ApplyAtomicFailure.previousDeployId` -// channel unchanged; the hub treats it as "no prior deploy" rather -// than as a real id. +// Sentinel the hub treats as "no prior deploy" rather than a real id. const NO_PRIOR_DEPLOY_ID = "none"; /** - * A loaded tool factory paired with the identity of the package it came - * from. `collectFactories` flattens every package's factories into one list - * but preserves this association per factory, because the per-bundle - * credential assembly downstream needs two things the bare factory (which - * carries only its bundle id) cannot supply: the package name, to derive the - * consumer identity (`toolConsumer(packageName)`) that Gate 2 checks, and the - * package's declared credential handles, to reconcile against the delivered - * bindings. `declaredCredentials` is the package-level set, repeated on each - * of that package's factories. + * Keeps package identity attached per factory because the per-bundle + * credential assembly needs the package name and declared handles that a + * bare factory (which carries only its bundle id) cannot supply. */ export interface StepToolFactory { readonly packageName: string; @@ -70,24 +50,12 @@ interface MaterializedToolPackages { // builder. export async function materializeToolPackages(args: { rawManifestBytes: string | undefined; - /** - * `assetId` → workspace-relative mount path, parsed from the deploy - * pack's `deploy/asset-mounts.json`. The loader resolves - * `kind: "asset"` manifest entries against this map. - */ + /** assetId -> workspace-relative mount path from deploy/asset-mounts.json. */ assetMounts: ReadonlyMap; storeDir: string; /** - * Workspace root the loader resolves `kind: "asset"` tarball mounts - * against (`//...`). Defaults to - * `/workspace` -- the default workspace layout, where the - * deploy flow stages asset packs into the same dir the agent runs in. - * - * The workflow-process child overrides this: it stages assets in the - * step's LEGACY agent dir workspace (where the hub's asset-pack push - * lands them) but roots the per-step apply-state + agent `env.workdir` - * under a distinct per-step store dir, so the loader's asset source - * and the apply-state root are two different directories. + * Defaults to /workspace. The workflow-process child overrides + * this to keep its asset source and apply-state root as separate directories. */ assetRoot?: string; agentAddress: string; @@ -98,33 +66,18 @@ export async function materializeToolPackages(args: { if (args.rawManifestBytes === undefined) { return { factories: [], pluginFactories: [] }; } - // Capture into a local so closures below see a `string` (the property - // narrowing on args.rawManifestBytes does not survive the nested - // function boundary, even though the property is readonly). + // Local capture: readonly-property narrowing doesn't survive the nested + // closure boundary below. const rawManifestBytes = args.rawManifestBytes; - // The apply-state tree (`tool-packages/`) shares the agent's - // storeDir root with the workspace tree the tool factories run - // against. Factories execute with `cwd: env.workdir`; the loader - // and the agent runtime both rely on factories not writing outside - // their workspace. A factory that walks `..` out of the workspace - // could touch the apply-state tree. The substrate trusts factories - // to honor that boundary — a misbehaving factory is treated as a - // security regression against the package, not a sandboxing gap to - // close at this layer. + // Shares storeDir with the factories' own workspace; the substrate trusts + // factories not to write outside cwd rather than sandboxing this boundary. const instanceDir = path.join(args.storeDir, "tool-packages"); await fs.promises.mkdir(instanceDir, { recursive: true }); const activeIdFile = path.join(instanceDir, "active-deploy-id"); const activeIdDirtyFile = `${activeIdFile}.dirty`; - // The dirty marker is written by `persistActiveDeployId`'s catch - // path when the commit persist (the normal write + fsync + dir - // fsync) failed on the prior apply and even the no-fsync fallback - // failed. Its presence means the staged deploy was committed but the - // recorded `active-deploy-id` is stale, and the marker carries the id - // that belongs there. Read it first so a degraded prior boot doesn't - // surface as `previousDeployId="none"` and silently demote the - // committed deploy to "fresh instance". + // See docs/sidecar-tool-package-durability.md for the dirty-marker protocol. let previousDeployId = NO_PRIOR_DEPLOY_ID; try { const dirtyRaw = (await fs.promises.readFile(activeIdDirtyFile, "utf-8")).trim(); @@ -148,15 +101,8 @@ export async function materializeToolPackages(args: { const attemptId = crypto.randomUUID(); const newDeployId = crypto.randomUUID(); - // Validate the manifest at the loader boundary. Both JSON parse - // failures and arktype schema failures route through the same - // `manifest.invalid` category so the hub sees a single failure - // shape for malformed manifests regardless of the specific defect. - // The helper persists the rejected bytes + the failure payload to - // the durable audit, then returns the Error for the caller to throw. - // Returning instead of throwing lets the caller use `throw await ...`, - // which TypeScript narrows control flow against without any - // dead-code suffix at the call site. + // Returns rather than throws so the caller can `throw await ...`, which + // TypeScript narrows control flow against with no dead-code suffix. const rejectManifestInvalid = async (reason: string): Promise => { const occurredAt = new Date().toISOString(); logger.warn`tool-package apply rejected for ${args.agentAddress}: manifest.invalid — ${reason}`; @@ -205,8 +151,7 @@ export async function materializeToolPackages(args: { instanceDir, assetRoot: args.assetRoot ?? path.join(args.storeDir, "workspace"), assetMounts: args.assetMounts, - // Step tool packages never source from git; only workflow-definition - // closures carry git entries. + // Only workflow-definition closures carry git entries. gitDirs: new Map(), attemptId, previousDeployId, @@ -214,41 +159,21 @@ export async function materializeToolPackages(args: { }); if (result.status === "failed") { logger.warn`tool-package apply rejected for ${args.agentAddress}: ${result.category} — ${result.message}`; - // A failed apply never wrote `active-deploy-id`: `applyAtomic` - // stages into a per-deploy-id directory and the commit is the - // persist below, which only runs on success. So the prior deploy is - // trivially still live and `result.previousDeployId` carries the - // unchanged prior id. There is no committed-but-failed swap to - // reconcile here — that case existed only under the old rename - // protocol. + // A failed apply never wrote active-deploy-id, so the prior deploy is + // trivially still live. await writeRejectedApplyAudit({ storeDir: args.storeDir, attemptId: result.attemptId, - // The raw bytes from disk, not the arktype-narrowed object. - // Re-serializing `validated` would drop unknown-tolerated - // fields, normalize key order, and lose whitespace — exactly - // the original-input evidence a future investigator needs to - // reproduce the failure against a newer validator. + // Raw bytes from disk, not the arktype-narrowed object — preserves + // whitespace/key order/unknown fields for replay against a newer validator. manifestBytes: rawManifestBytes, failure: result, }); throw new Error(`tool-package apply rejected (${result.category}): ${result.message}`); } - // Apply staged: the loader built the new deploy at - // `packages//`. Persisting the new active id is the - // commit — the single write that advances the live deploy from the - // prior id to this one. If the write or its fsync fails (disk full, - // EIO, EROFS), the on-disk state has diverged from the recorded id: - // `persistActiveDeployIdWithFallback` has already written the new id - // through its no-fsync / dirty-marker degradation ladder, so the new - // deploy is logically committed, but the id was not durably flushed. - // Record this as `apply.previous-rotation.failed` — for that category - // `previousDeployId` carries the **new** deploy id (the one now live) - // so the durable audit records the on-disk truth. Write the audit - // entry, then throw so the harness tears down: the durability gap - // means the next apply cannot trust `previousDeployId` until the next - // boot reconciles via the dirty marker. + // Persisting the id is the commit. On a degraded persist, throw so the + // harness tears down — see docs/sidecar-tool-package-durability.md. const persistOutcome = await persistActiveDeployIdWithFallback( instanceDir, activeIdFile, @@ -292,19 +217,10 @@ export async function materializeToolPackages(args: { * previousDeployId="none" and treat the committed deploy as belonging * to a fresh, deploy-less instance. * - * Dir-fsync is best-effort durability hardening, not part of the - * apply's structural success contract. Some filesystems (notably - * FAT/exFAT, certain network mounts) do not support fsync on a - * directory handle and will surface EINVAL / ENOTSUP. The deploy is - * already staged on disk and the deploy-id file's own fsync has - * landed; tearing the harness down at this point would force a - * restart for a degraded-durability condition the operator has no - * way to act on. Log it and continue. + * Dir-fsync is best-effort: some filesystems (FAT/exFAT, some network + * mounts) don't support it and surface EINVAL/ENOTSUP, which shouldn't + * force a restart when the deploy-id file's own fsync already landed. */ -// Version prefix for the active-deploy-id file. A future format -// change (e.g. carrying additional fields alongside the id) bumps -// this and the reader rejects unknown prefixes loudly instead of -// silently treating an unrecognized payload as a deploy id. const ACTIVE_DEPLOY_ID_VERSION = "v1"; const ACTIVE_DEPLOY_ID_PREFIX = `${ACTIVE_DEPLOY_ID_VERSION}:`; @@ -315,19 +231,9 @@ export function parseActiveDeployId(raw: string, sourcePath: string): string { `active-deploy-id file ${sourcePath} is empty; expected ${ACTIVE_DEPLOY_ID_PREFIX}`, ); } - // Pre-versioning files carried just the raw deploy id. Accept those - // for backward compatibility so an upgrade does not require an - // operator-driven state rewrite, but require the prefix on any - // file the new code writes. - // - // Deprecation horizon: this branch exists to ease the v1: cutover - // from pre-versioning files. Once every deployed sidecar has been - // restarted at least once on a version that writes the prefix, the - // un-prefixed branch can be deleted and an un-prefixed file should - // be rejected as garbage. The horizon is "remove after a few - // sidecar minor versions have shipped that write the prefix"; the - // exact cutover is an operational decision, not a code-encoded - // one. + // Accepts pre-versioning unprefixed files so upgrade needs no operator + // rewrite; delete this branch once every sidecar has restarted on a + // version that writes the prefix (see docs/sidecar-tool-package-durability.md). if (raw.startsWith(ACTIVE_DEPLOY_ID_PREFIX)) { const id = raw.slice(ACTIVE_DEPLOY_ID_PREFIX.length); if (id.length === 0) { @@ -379,19 +285,8 @@ async function persistActiveDeployId( } /** - * Remove a stale `.dirty` sibling of the active-deploy-id file. Called - * after either the fsync'd primary persist or the no-fsync fallback - * persist successfully writes a fresh active-deploy-id; in both cases a - * pre-existing marker carries a now-stale id that would otherwise - * shadow the recorded id on the next boot (the boot reader prefers the - * marker when present). - * - * Best-effort: a failure to remove the marker leaves the next boot - * reading the now-redundant marker (which carries a stale id), so the - * divergence is bounded to a noisier log path. ENOENT is the normal - * case when no prior failure had written a marker. - * - * Exported for direct unit testing of the cleanup contract. + * Best-effort: a leftover marker would otherwise shadow the recorded id on + * the next boot, since the boot reader prefers it when present. */ export async function clearDirtyMarker(activeIdFile: string, reason: string): Promise { try { @@ -403,22 +298,7 @@ export async function clearDirtyMarker(activeIdFile: string, reason: string): Pr } } -/** - * Wrap `persistActiveDeployId` with a degradation ladder: try the - * normal fsync'd write first, and on failure try a no-fsync write + - * write of a sibling `.dirty` marker so the next boot can reconcile. - * - * Returns `{ degraded: false }` on full success. Returns - * `{ degraded: true, error }` when the primary persist failed; the - * caller records an `apply.previous-rotation.failed` audit entry and - * throws on the back of it. - * - * If the fallback persist also fails, on-disk state will diverge from - * the recorded id for one boot cycle. Boot-time reconciliation reads - * the dirty marker and prefers it. - * - * Exported for direct unit testing of the dirty-marker contract. - */ +// See docs/sidecar-tool-package-durability.md for the degradation ladder. export async function persistActiveDeployIdWithFallback( instanceDir: string, activeIdFile: string, @@ -449,31 +329,7 @@ export async function persistActiveDeployIdWithFallback( } } -/** - * Persist the rejected manifest and the failure payload to the agent's - * on-disk audit trail. The destination is - * `/audit/rejected-applies//`, two files: - * - * - `manifest.json` — the manifest bytes that were rejected, written - * verbatim — for every failure category, these - * are the original on-disk bytes the sidecar - * read from `deploy/tool-packages-manifest.json` - * (corrupt JSON, wrong-shape JSON, or a - * validator-accepted manifest the loader later - * rejected). Persisting the raw bytes lets a - * future investigator replay the same input - * against a newer validator without losing - * whitespace, key order, or tolerated fields - * the validator narrowed away. - * - `error.json` — `{ attemptId, previousDeployId, category, - * message, package?, occurredAt }` - * - * The files live in the agent's storeDir so a future - * git-commit-of-audit-entries pass can pick them up without rerouting - * the data. v1 writes them directly to disk; the formal git-commit - * step (turning these into a signed audit-trail commit on the - * agent-state repo) is a separate plumbing piece. - */ +// See docs/sidecar-tool-package-durability.md for the audit trail format. async function writeRejectedApplyAudit(args: { storeDir: string; attemptId: string; @@ -489,15 +345,8 @@ async function writeRejectedApplyAudit(args: { }): Promise { const dir = path.join(args.storeDir, "audit", "rejected-applies", args.attemptId); await fs.promises.mkdir(dir, { recursive: true }); - // fsync the files and their parent directory before returning so the - // rejection's evidence is durable before the caller throws and tears - // the apply down. Otherwise, a crash after the throw begins - // propagating but before the audit bytes hit disk would leave a - // rejected apply the sidecar cannot prove the manifest for on replay. - // fsync failures are downgraded to warnings: the bytes are written - // either way, and a refusing fsync (rare networked-FS failure mode) - // should not turn into a second cascading failure on an already- - // failing apply. + // Fsynced before the caller throws so a crash right after doesn't lose + // the evidence; fsync failures downgrade to warnings, not a second failure. await fsyncWriteFile(path.join(dir, "manifest.json"), args.manifestBytes); await fsyncWriteFile(path.join(dir, "error.json"), JSON.stringify(args.failure, null, 2)); try { diff --git a/docs/sidecar-tool-package-durability.md b/docs/sidecar-tool-package-durability.md new file mode 100644 index 0000000000..e74e38d50a --- /dev/null +++ b/docs/sidecar-tool-package-durability.md @@ -0,0 +1,32 @@ +# Sidecar tool-package apply durability + +`apps/sidecar/src/tool-materialization.ts` applies a workflow's tool-package +manifest atomically and records which deploy is active. This covers the +durability protocol around that record. + +## active-deploy-id file + +Written as `v1:`. The `v1:` prefix lets a future format change be +rejected loudly instead of misread; pre-versioning files with no prefix are +still accepted for one upgrade cycle, then that branch should be deleted +once every sidecar has restarted on a version that writes the prefix. + +## Persist ladder + +Persisting the new id is the commit for a staged apply. If the fsync'd write +fails, a no-fsync fallback write runs; if that also fails, a sibling +`.dirty` marker records the id instead. On boot, the dirty marker (if +present) is read in preference to the recorded id, since it means a prior +apply committed on disk but the id was not durably flushed. Each stage is +best-effort past the first: a degraded write is logged, not escalated, +since the deploy itself is already staged on disk. + +## Rejected-apply audit + +A rejected manifest (JSON parse failure, schema failure, or loader +rejection) is written verbatim to +`/audit/rejected-applies//manifest.json` alongside an +`error.json` failure payload, fsynced before the caller throws. This +preserves the original bytes (whitespace, key order, unknown fields) for +replay against a newer validator, independent of anything the arktype +schema narrowed away. From 2095599254fdeb3669d84414255eec10bb704fe0 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:08:25 -0700 Subject: [PATCH 03/31] chore(sidecar): comments say why, never what (part 2) --- apps/sidecar/src/run-grants.ts | 48 +--- .../src/workflow-closure-materialization.ts | 112 ++------ apps/sidecar/src/workflow-probe-handler.ts | 239 +++--------------- 3 files changed, 66 insertions(+), 333 deletions(-) diff --git a/apps/sidecar/src/run-grants.ts b/apps/sidecar/src/run-grants.ts index 27e3ac25f4..dcc9b1353b 100644 --- a/apps/sidecar/src/run-grants.ts +++ b/apps/sidecar/src/run-grants.ts @@ -1,17 +1,7 @@ -// Per-run grants read primitives shared by the two sidecar modules that -// resolve a run's authorization from disk: the supervisor wiring -// (`workflow-host-wiring.ts`), which reads a run's grants at the -// `onRunStart` barrier, and the in-process child runtime -// (`workflow-substrate-factory.ts`), which reads the PARENT run's grants -// to bind a spawned child's authorize. Both destinations read the same -// `runs//grants.json` file in a deployment's `workflow-run` repo, -// so the read lives in one place. -// -// The module's dependency surface is deliberately narrow -- node fs/path, -// arktype, and the substrate `RepoStore` type -- so importing it into the -// child subprocess module does not drag the hub-agent deploy-router -// surface `workflow-host-wiring.ts` also depends on into the child -// binary. +// Shared by workflow-host-wiring.ts and workflow-substrate-factory.ts, both +// of which read runs//grants.json. Kept dependency-narrow (fs/path, +// arktype, RepoStore) so importing this into the child binary doesn't drag +// in the hub-agent deploy-router surface the wiring module also depends on. import { readFile } from "node:fs/promises"; import { join as pathJoin } from "node:path"; @@ -21,40 +11,20 @@ import { type } from "arktype"; import type { RepoStore } from "@intx/hub-sessions"; import { isErrnoNotFound } from "@intx/workflow-host"; -/** - * Path inside a deployment's `workflow-run` repo that carries a single - * run's grants. It sits under the run's own `runs//` subtree -- - * sibling to that run's `events/` blobs -- so a run's grants live and are - * reclaimed with the rest of the run's state. - */ +/** Sibling to the run's events/ blobs, so grants live and are reclaimed with the rest of the run's state. */ export function runGrantsPath(runId: string): string { return `runs/${runId}/grants.json`; } -/** - * Envelope the per-run grants file carries: the canonical `{ grants: [] }` - * shape the deploy-time snapshot writes and validates. The inner entries - * stay `unknown` -- the child's authorize layer narrows each against its - * own grant-rule validator, exactly as the deploy-time snapshot's grants - * are surfaced untyped. - */ +/** Entries stay unknown; the child's authorize layer narrows each against its own grant-rule validator. */ export const RunGrantsFile = type({ grants: "unknown[]", }).onUndeclaredKey("ignore"); /** - * Read a single run's grants from `runs//grants.json` inside the - * deployment's `workflow-run` repo. The read is a working-tree read via - * `getRepoDir` (the same substrate access pattern the deploy-time - * per-step read uses), so it observes the tip the grants handler wrote. - * - * Returns the run's grants when the file exists. Returns `undefined` - * -- distinct from an empty grants array -- when the file is ABSENT - * (ENOENT), so the caller can distinguish "this run got no per-run - * grants file" from "this run's grants are the empty set". A file that - * exists but is malformed (invalid JSON or a shape the envelope rejects) - * THROWS: the file's presence implies a grants frame was delivered, and a - * structural failure is a boundary bug, not a default. + * Returns undefined (distinct from an empty array) when the file is + * absent, so the caller can tell "no per-run grants file" from "empty + * grants set". A malformed existing file throws rather than defaulting. */ export async function readRunGrants(args: { repoStore: RepoStore; diff --git a/apps/sidecar/src/workflow-closure-materialization.ts b/apps/sidecar/src/workflow-closure-materialization.ts index 46d65b5072..599dbc8962 100644 --- a/apps/sidecar/src/workflow-closure-materialization.ts +++ b/apps/sidecar/src/workflow-closure-materialization.ts @@ -1,37 +1,11 @@ -// Host-side materializer for a workflow-probe frame's frozen closure. -// -// The airlocked probe child (`workflow-probe-handler.ts`) evaluates a -// code-sourced workflow's `interchange.workflow` entry, but the frozen -// dependency closure it evaluates against is materialized on the sidecar -// HOST first -- fetch + SRI-verify + extract + `node_modules` layout is -// I/O, not author-code evaluation, so it stays out of the child. This -// module builds the production `MaterializeWorkflowClosure` the probe -// executor injects: it lays out the frame's frozen closure and returns -// the workflow package directory the child loads from, without importing -// any author code on the host. -// -// Layering (greybeard): the concrete materializer lives here in -// `apps/sidecar` so `@intx/workflow-host` stays free of a -// `@intx/tool-packaging` dependency and `workflow-probe-handler.ts` -// stays free of one too -- the materializer is an injected seam. The -// portable packages only see the `MaterializeWorkflowClosure` callback -// this module produces. -// -// Phases 1-2 only, no `applyAtomic`: a probe is ephemeral and inert, so -// the durable-deploy lifecycle bookkeeping (`active-deploy-id`, the -// per-deploy-id retention ladder) is the wrong semantics. The closure is -// laid out under a per-probe scratch dir that `cleanup` removes once the -// child has been reaped. `createToolLoader(...).loadManifest(...)` is -// invoked with an EMPTIED `topLevel`: the loader's phase-3 import loop -// only imports packages named in `topLevel`, so an empty `topLevel` -// fetches + extracts + lays out every closure entry (the full `entries` -// set) while importing NONE of them. That runs exactly the eval-free -// `materializeClosure` phases the probe needs, using the loader's -// production registry fetcher -- the tarball fetcher `@intx/tool-packaging` -// owns is only reachable through `createToolLoader`, so the layout is -// driven through the loader rather than by calling `materializeClosure` -// with a fetcher this package would otherwise have to build (and thereby -// reach for the npm-registry machinery that package exists to contain). +// Materializes a probe's frozen closure on the sidecar host (I/O only, no +// author-code eval) before the airlocked child evaluates it. Lives in +// apps/sidecar so @intx/workflow-host stays free of a @intx/tool-packaging +// dependency; the portable packages only see the injected +// MaterializeWorkflowClosure seam. Phases 1-2 only (no applyAtomic): a +// probe is ephemeral, so the durable-deploy bookkeeping doesn't apply. +// loadManifest runs with an emptied topLevel so it lays out every closure +// entry without importing (evaluating) any of them. import { promises as fs } from "node:fs"; import path from "node:path"; @@ -65,12 +39,7 @@ export interface WorkflowClosureMaterializerConfig { readonly cacheMaxBytes: number; /** Byte cap for a single HTTP-registry tarball fetch. */ readonly registryMaxTarballBytes: number; - /** - * Byte cap for the total base64-encoded asset payload a probe frame may - * deliver inline. Measured against the base64 wire length (a conservative - * upper bound on the decoded bytes), it is enforced before any pack is - * decoded so an oversized frame fails loud rather than being materialized. - */ + /** Enforced against the base64 wire length, before decoding, so an oversized frame fails loud. */ readonly maxAssetPayloadBytes: number; /** Registry identifier -> URL + credentials the loader resolves entries against. */ readonly registries: ReadonlyMap; @@ -86,17 +55,7 @@ export interface WorkflowClosureMaterializerConfig { readonly fetchTarball?: TarballFetcher; } -/** - * Build the production `MaterializeWorkflowClosure` the workflow-probe - * executor injects. The returned function lays out a probe frame's frozen - * closure under a fresh scratch dir and returns the workflow package - * directory plus a `cleanup` that removes the scratch dir. - * - * @throws (from the returned materializer) if the closure does not pin - * exactly one top-level package, the source registry is not configured, - * the layout fails (fetch / integrity / extract), or the frame's `entry` - * disagrees with the materialized package's `interchange.workflow`. - */ +/** Lays out a probe frame's frozen closure under a fresh scratch dir removed by the returned cleanup. */ export function createWorkflowClosureMaterializer( config: WorkflowClosureMaterializerConfig, ): MaterializeWorkflowClosure { @@ -105,11 +64,8 @@ export function createWorkflowClosureMaterializer( return async function materialize( frame: WorkflowProbeRequestFrame, ): Promise { - // Gap 1: the closure's single top-level pin IS the workflow definition - // package (the hub resolved the closure for exactly that pin). Assert - // the cardinality and fail loud rather than silently picking `[0]`; a 0- - // or >1-pin closure is an incoherent request the materializer owns - // rejecting at this boundary. + // Fail loud rather than silently picking [0]: a 0- or >1-pin closure is + // an incoherent request this boundary owns rejecting. const topLevel = frame.closure.topLevel; if (topLevel.length !== 1) { throw new Error( @@ -123,13 +79,7 @@ export function createWorkflowClosureMaterializer( ); } - // Boundary check on the definition's source. The `registry` arm surfaces a - // missing source registry loudly before any I/O (the per-entry registry - // gates fire again inside the loader). An `asset` closure is materialized - // from the `assets` the frame delivers -- tarball entries from a plain-file - // mount, source entries from an indexed gitDir -- checked below. The - // `never` default makes a future source kind a compile error rather than a - // silent fallthrough. + // The never default makes a future source kind a compile error, not a silent fallthrough. switch (frame.source.kind) { case "registry": if (!config.registries.has(frame.source.registry)) { @@ -167,11 +117,8 @@ export function createWorkflowClosureMaterializer( ...(config.fetchTarball !== undefined ? { fetchTarball: config.fetchTarball } : {}), }); - // Materialize any inline-delivered assets under the probe scratch: - // tarball entries as plain files under the workspace root, source entries - // as an indexed gitDir the loader checks subtrees out of. Registry-sourced - // closures deliver none; both maps stay empty and the loader fetches over - // HTTP. + // Registry-sourced closures deliver no assets; both maps stay empty + // and the loader fetches over HTTP instead. const assetRoot = path.join(scratchDir, "workspace"); const gitDirRoot = path.join(scratchDir, "gitdirs"); const { assetMounts, gitDirs } = await materializeWorkflowAssets({ @@ -182,12 +129,8 @@ export function createWorkflowClosureMaterializer( maxAssetPayloadBytes: config.maxAssetPayloadBytes, }); - // Source-boundary check for the asset arm, the post-unpack analog of the - // registry arm's config check: the asset the definition is sourced from - // holds the workflow package, so it MUST be among the delivered assets -- - // as a gitDir for a source definition, as a mount for a tarball one. - // Surface a missing delivery here rather than as a downstream failure on - // the top-level entry. + // The asset the definition is sourced from must be among the delivered + // assets; surface a missing delivery here, not as a downstream failure. if (frame.source.kind === "asset") { const delivered = frame.source.package.format === "source" @@ -200,11 +143,8 @@ export function createWorkflowClosureMaterializer( } } - // Lay out phases 1-2 only. `topLevel` is emptied so the loader's - // phase-3 loop imports nothing -- no author code is evaluated on the - // host; the airlocked child owns the single import of the workflow - // entry. The full `entries` set is still fetched, SRI-verified, - // extracted, and laid out with its `node_modules` graph. + // topLevel emptied so the loader's phase-3 import loop runs nothing on + // the host; the airlocked child owns the single import. const layoutManifest: ToolPackageManifest = { schemaVersion: frame.closure.schemaVersion, topLevel: [], @@ -226,24 +166,14 @@ export function createWorkflowClosureMaterializer( logger.debug`materialized workflow-probe closure for ${workflowPin.name}@${workflowPin.version} at ${packageDir}`; return { packageDir, cleanup }; } catch (err) { - // On any failure before a closure handle is handed back, the executor - // never sees a `cleanup` to call, so the scratch dir is this function's - // to reclaim. + // The executor never sees a cleanup to call before a handle is returned. await cleanup(); throw err; } }; } -/** - * Gap 2: cross-check the probe frame's `entry` against the materialized - * package's own `interchange.workflow`. The child loader reads the entry - * path from the package's `package.json`, ignoring the frame's `entry`; - * left unchecked the frame field is an input that travels but is never - * validated. Comparing them host-side -- a `package.json` read, no author - * code -- surfaces a tampered or incoherent request before the child is - * ever spawned, and fails loud on mismatch. - */ +/** The child loader ignores frame.entry; comparing it host-side surfaces a tampered request before the child spawns. */ async function assertFrameEntryMatchesPackage( packageDir: string, frameEntry: string, diff --git a/apps/sidecar/src/workflow-probe-handler.ts b/apps/sidecar/src/workflow-probe-handler.ts index 4bd0fc6456..90042974a5 100644 --- a/apps/sidecar/src/workflow-probe-handler.ts +++ b/apps/sidecar/src/workflow-probe-handler.ts @@ -1,28 +1,8 @@ -// Sidecar workflow-probe handler: the airlocked one-shot probe child. -// -// A `workflow.probe.request` frame asks this sidecar to inspect a -// code-sourced workflow WITHOUT deploying it. The inspection evaluates -// author code (the workflow package's `interchange.workflow` entry), so -// it must never run in the sidecar host's address space. This module -// spawns a ONE-SHOT child process behind the IPC airlock that loads and -// evaluates the entry, runs the capability walk plus the live->inert -// projector, and ships the inert projection + advisory grant set + wire -// hash back over an HMAC-authenticated result frame (reusing the same -// per-frame HMAC discipline `@intx/workflow-host`'s event channel uses). -// -// Reaping is sidecar-owned and independent of the hub's probe timeout. -// The hub timeout only rejects the hub-side promise; it does not kill -// the child. `runOneShotProbeChild` owns a self-contained lifecycle with -// its OWN deadline and reaps the child on every exit path -- eval -// success, eval throw, malformed code, and a probe that outruns the -// self-owned deadline -- so a wedged or runaway child can never survive -// the probe call. -// -// The frozen dependency closure is materialized sidecar-side (host, -// no eval) through the injected `MaterializeWorkflowClosure` seam; only -// the load+evaluate+walk+project step runs in the child. The child -// receives the materialized package directory in its fresh, minimal env -// -- no ambient inputs, no sidecar keys. +// Evaluates a code-sourced workflow's entry to inspect it without +// deploying, so it must never run in the sidecar host's address space: +// this spawns a one-shot child behind an IPC airlock. Reaping is +// sidecar-owned with its own deadline, independent of the hub's probe +// timeout (which only rejects the hub-side promise, never kills the child). import { fileURLToPath } from "node:url"; @@ -54,20 +34,10 @@ const logger = getLogger(["sidecar", "workflow-probe"]); const IPC_HMAC_KEY_BYTES = 32; -/** - * Self-owned upper bound on how long a single probe child may run before - * the sidecar reaps it and fails the probe. Independent of the hub's - * `probeTimeoutMs`: the hub timeout only rejects the hub-side promise, - * whereas this deadline is what actually kills a wedged child. - */ +/** Independent of the hub's probeTimeoutMs, which only rejects the hub-side promise; this is what kills a wedged child. */ export const DEFAULT_PROBE_CHILD_TIMEOUT_MS = 30_000; -/** - * Self-owned SIGTERM->SIGKILL escalation window when reaping the child. - * Mirrors the supervisor's `DEFAULT_KILL_TIMEOUT_MS` so a probe child - * that ignores SIGTERM is force-killed on the same schedule a supervised - * child is. - */ +/** Mirrors the supervisor's kill timeout so a probe child is force-killed on the same schedule. */ export const DEFAULT_PROBE_CHILD_KILL_TIMEOUT_MS = DEFAULT_KILL_TIMEOUT_MS; // Env keys the host sets on the child's fresh spawn env. The child reads @@ -77,11 +47,7 @@ const PROBE_CHANNEL_ID_ENV = "PROBE_IPC_CHANNEL_ID"; const PROBE_HMAC_KEY_ENV = "PROBE_IPC_HMAC_KEY"; const PROBE_PACKAGE_DIR_ENV = "PROBE_PACKAGE_DIR"; -/** - * The child's `bin/workflow-probe-child` entry, resolved statically at - * module load so the spawn surface does not depend on any runtime env - * override. Mirrors the supervisor's `bin/workflow-child` resolution. - */ +/** Resolved statically at module load so the spawn surface doesn't depend on a runtime env override. */ const DEFAULT_PROBE_CHILD_BINARY: string = fileURLToPath( import.meta.resolve("../bin/workflow-probe-child"), ); @@ -90,14 +56,7 @@ const DEFAULT_PROBE_CHILD_BINARY: string = fileURLToPath( // Result payload wire (child -> host) // --------------------------------------------------------------------------- -/** - * The child's single result payload, carried inside the HMAC-signed - * envelope. `ok: true` ships the inert projection, the advisory grant - * set, the un-flattened grant walk snapshot, and the wire hash; `ok: - * false` ships the failure reason (eval throw, malformed code) so the - * host can reject the probe with a meaningful message rather than a bare - * "child exited" surface. - */ +/** ok: false carries the failure reason so the host rejects with a message rather than a bare "child exited". */ const ProbeResultPayload = type({ ok: "true", projection: "unknown", @@ -110,18 +69,7 @@ const ProbeResultPayload = type({ }); type ProbeResultPayload = typeof ProbeResultPayload.infer; -/** - * The inert answer a probe execution produces: the workflow's inert - * needs-surface projection, the advisory grant set derived from it, the - * un-flattened grant walk snapshot the set is derived from, and the - * projection's content hash. Structurally the `WorkflowProbeResult` the - * hub-agent probe seam consumes. - * - * `grantWalkSnapshot` carries the per-step grant declarations (grant - * strings plus each step's tool-grant `grantEffects` map) and the - * definition's full `grantRequirements` -- the grouping and effect data - * the flattened `grants` union discards. - */ +/** grantWalkSnapshot preserves per-step grouping and effect data the flattened `grants` union discards. */ export interface WorkflowProbeResult { readonly projection: WorkflowProjectionDefinition; readonly grants: string[]; @@ -133,26 +81,13 @@ export interface WorkflowProbeResult { // Closure materialization seam // --------------------------------------------------------------------------- -/** - * A materialized workflow package closure: the directory holding the - * workflow package's `package.json` (with its `node_modules/` laid out - * so the entry's bare-specifier imports resolve), plus a `cleanup` the - * handler always calls once the child has been reaped. - */ +/** node_modules/ is laid out so the entry's bare-specifier imports resolve; cleanup runs once the child is reaped. */ export interface MaterializedWorkflowClosure { readonly packageDir: string; cleanup(): Promise; } -/** - * Host-side materializer for a probe frame's frozen closure. Fetches, - * verifies, extracts, and lays out the workflow package (and its - * dependency closure) into a resolvable tree, returning the package - * directory the child loads from. This runs on the sidecar host -- it is - * I/O, not author-code evaluation -- so the airlocked child only performs - * the load+evaluate step. The production materializer is host-supplied so - * `@intx/workflow-host` stays free of a `@intx/tool-packaging` dependency. - */ +/** Runs on the sidecar host (I/O, not author-code eval), so the airlocked child only does load+evaluate. */ export type MaterializeWorkflowClosure = ( frame: WorkflowProbeRequestFrame, ) => Promise; @@ -161,12 +96,7 @@ export type MaterializeWorkflowClosure = ( // Child spawn seam // --------------------------------------------------------------------------- -/** - * Minimal handle over a spawned probe child. The probe needs only the - * child's stdout (the single result line), a kill primitive, and the - * `exited` promise for reaping -- no control/event channels, because the - * probe carries no bidirectional control traffic. - */ +/** No control/event channels: the probe carries no bidirectional control traffic. */ export interface ProbeChildHandle { readonly pid: number; readonly stdout: ReadableStream; @@ -174,23 +104,13 @@ export interface ProbeChildHandle { readonly exited: Promise; } -/** - * Spawner the handler invokes to launch the one-shot probe child. - * Production injects the `Bun.spawn`-backed `defaultProbeChildSpawner`; - * tests inject a spawner that records the spawned pid so they can assert - * the child was reaped. - */ +/** Tests inject a spawner that records the pid so they can assert the child was reaped. */ export type ProbeChildSpawner = (args: { binaryPath: string; env: Record; }) => ProbeChildHandle; -/** - * Real `Bun.spawn`-backed probe-child spawner. Constructs a fresh env - * (the caller assembles it; no `process.env` spread), pipes stdout for - * the result line, ignores stdin, and inherits stderr so child - * diagnostics land on the sidecar's stderr. - */ +/** No process.env spread — the caller assembles a fresh env. */ export const defaultProbeChildSpawner: ProbeChildSpawner = ({ binaryPath, env, @@ -238,16 +158,7 @@ export interface WorkflowProbeExecutorOpts { killTimeoutMs?: number; } -/** - * Build the sidecar's workflow-probe executor. The returned object - * satisfies the hub-agent `WorkflowProbeExecutor` seam: `probe(frame)` - * returns the inert projection + advisory grant set + wire hash, and - * throws when any step fails so the link answers `workflow.probe.error`. - * - * `probe` materializes the frozen closure, spawns a one-shot airlocked - * child to evaluate the workflow, and reaps that child on every exit - * path independent of the hub's probe timeout. - */ +/** Satisfies the hub-agent WorkflowProbeExecutor seam; throws so the link answers workflow.probe.error. */ export function createWorkflowProbeExecutor(opts: WorkflowProbeExecutorOpts): { probe(frame: WorkflowProbeRequestFrame): Promise; } { @@ -282,13 +193,7 @@ interface RunOneShotProbeChildArgs { readonly killTimeoutMs: number; } -/** - * Spawn a single probe child, drive it to its one result frame, and reap - * it on every exit path. The `finally` guarantees the child is killed - * whether the read succeeds, the child ships an error frame, the child - * exits without a frame (malformed code / crash), or the self-owned - * deadline fires first. - */ +/** The finally guarantees the child is killed on every exit path, including a self-owned deadline firing first. */ async function runOneShotProbeChild(args: RunOneShotProbeChildArgs): Promise { const channelId = generateChannelId(); const hmacKey = generateHmacKey(); @@ -318,15 +223,9 @@ async function runOneShotProbeChild(args: RunOneShotProbeChildArgs): Promise ({ kind: "line" as const, line })), deadline.promise.then(() => ({ kind: "timeout" as const })), @@ -354,10 +253,7 @@ function buildProbeChildEnv(args: { channelId: string; hmacKey: Uint8Array; }): Record { - // A fresh, minimal env: exactly the IPC anchors and the materialized - // package dir, plus the OS handles the shebang needs to exec `bun` and - // land temp files on the host's temp root. No `process.env` spread, so - // no sidecar secret or ambient input crosses the airlock. + // No process.env spread: no sidecar secret or ambient input crosses the airlock. const env: Record = { [PROBE_CHANNEL_ID_ENV]: args.channelId, [PROBE_HMAC_KEY_ENV]: hexEncode(args.hmacKey), @@ -372,12 +268,7 @@ function buildProbeChildEnv(args: { return env; } -/** - * Reap a probe child: SIGTERM, then SIGKILL if the exit does not land - * within `killTimeoutMs`. SIGKILL is unignorable, so `exited` is - * guaranteed to settle -- a child that traps or ignores SIGTERM cannot - * wedge this call. A kill against an already-exited child is a no-op. - */ +/** SIGKILL is unignorable, so `exited` always settles even if the child traps SIGTERM. */ async function reapChild(handle: ProbeChildHandle, killTimeoutMs: number): Promise { try { handle.kill("SIGTERM"); @@ -403,13 +294,7 @@ async function reapChild(handle: ProbeChildHandle, killTimeoutMs: number): Promi }); } -/** - * Authenticate and parse the child's single result frame. Verifies the - * HMAC over the re-encoded envelope BEFORE trusting any field (mirroring - * the event channel's receiver), binds the frame to this spawn's - * channelId, then narrows the payload. A `ok: false` payload is turned - * into a throw so the probe fails with the child's reason. - */ +/** Verifies HMAC before trusting any field, mirroring the event channel's receiver. */ async function parseProbeResult( line: string, channelId: string, @@ -461,11 +346,7 @@ async function parseProbeResult( // Child side // --------------------------------------------------------------------------- -/** - * One line the child writes to a sink. Production wraps `process.stdout`; - * tests inject a capture. The bytes are handed to the OS before the child - * exits so the result is not truncated. - */ +/** Bytes are handed to the OS before the child exits so the result isn't truncated. */ export type ProbeChildLineWriter = (line: string) => Promise; export interface RunProbeChildOpts { @@ -475,19 +356,7 @@ export interface RunProbeChildOpts { writeLine?: ProbeChildLineWriter; } -/** - * The airlocked child's whole job: read the materialized package dir and - * IPC anchors from its fresh env, load+evaluate the workflow entry, run - * the capability walk plus the live->inert projector, and ship the inert - * projection + advisory grant set + wire hash back inside one - * HMAC-signed result frame. - * - * An evaluation failure (malformed code, an entry that throws, a package - * with no `interchange.workflow`) is caught and shipped as an `ok: false` - * frame so the host reaps cleanly and answers `workflow.probe.error` - * with the reason -- rather than the child crashing and the host seeing a - * bare "exited without result". - */ +/** An eval failure ships as ok: false so the host reaps cleanly instead of seeing a bare crash. */ export async function runWorkflowProbeChildFromProcessEnv( opts: RunProbeChildOpts = {}, ): Promise { @@ -512,32 +381,21 @@ async function computeProbePayload(packageDir: string): Promise` grants advertised here match what the runtime - // resolves. Built-ins-only when the closure ships no `interchange.directors`. + // Composed from the SAME closure the run-child will use, so director: + // grants advertised here match what the runtime resolves. const directors = await loadWorkflowDirectorRegistryFromClosure({ packageDir, }); - // Load the static tool `definitions` each declared plugin package - // contributes from the SAME closure, so the walk emits `tool:` - // grants for plugin-contributed tools (Tier-2 governance). A plugin - // package reaches an agent only through `env.plugins`, so its tool grant - // surface is invisible to the walk otherwise -- the run-child would then - // load the plugin from the closure and the reactor would fail closed on - // an un-approved `tool:`. Loading here (over the frozen closure the - // run-child also materializes from) keeps the approved snapshot and the - // runtime plugin in lockstep. + // A plugin reaches an agent only through env.plugins, so without loading + // its tool definitions here the walk would miss tool: grants and the + // run-child would later fail closed on an un-approved tool. const pluginToolDefinitions = await loadWorkflowPluginToolDefinitionsFromClosure({ packageDir, plugins: collectDeclaredPluginNames(definition), }); const walk = walkCapabilities(definition, directors, pluginToolDefinitions); - // Fail closed on an unresolved director: the runtime does not re-gate - // `director:` against the approved grant set, so this advertisement is - // the only approval checkpoint for a director. Shipping an ok probe whose - // grant set silently omits a director the runtime would still try to - // resolve would let the operator approve an incomplete manifest. Mirrors - // the live-authored approval gate (`createApprovalSetGate`). + // Fail closed: the runtime never re-gates director:, so this + // advertisement is the only approval checkpoint for it. const [unresolved] = walk.unresolvedDirectors; if (unresolved !== undefined) { return { ok: false, error: `unresolvable director: ${unresolved}` }; @@ -551,11 +409,7 @@ async function computeProbePayload(packageDir: string): Promise(); for (const declarations of walk.perStep.values()) { @@ -566,15 +420,7 @@ function collectDeploymentGrants(walk: CapabilityWalkResult): string[] { return [...grants].sort(); } -/** - * Serialize the un-flattened capability walk into a plain-data - * `GrantWalkSnapshot`: the per-step grant declarations (each step's grant - * strings plus its tool-grant `grantEffects` map, converted from the - * walk's `Map` to a plain object) and the definition's full, unfiltered - * `grantRequirements`. Unlike `collectDeploymentGrants`, this preserves - * the per-step grouping and the effect data the flattened set discards. - * A definition that declares no requirements snapshots an empty list. - */ +/** Unlike collectDeploymentGrants, preserves per-step grouping and effect data the flattened set discards. */ function buildGrantWalkSnapshot( walk: CapabilityWalkResult, grantRequirements: readonly GrantRequirement[] | undefined, @@ -632,11 +478,7 @@ function defaultStdoutWriteLine(line: string): Promise { // Shared helpers // --------------------------------------------------------------------------- -/** - * Read one newline-delimited line from a byte stream. Resolves the first - * complete line, or `null` when the stream closes without one (the child - * exited before writing). Releases the reader lock on every exit. - */ +/** Resolves null when the stream closes without a complete line (the child exited before writing). */ async function readResultLine(stream: ReadableStream): Promise { const reader = stream.getReader(); const decoder = new TextDecoder("utf-8"); @@ -685,16 +527,7 @@ function errorMessage(err: unknown): string { // package the workflow entry imported at evaluation time. const MISSING_MODULE_RE = /Cannot find (?:module|package) ['"]([^'"]+)['"]/; -/** - * Enrich a probe evaluation failure whose cause is a module that could not be - * resolved from the workflow's dependency closure. The evaluator is the layer - * that KNOWS what the workflow imports at run time (it actually ran the - * import), so a missing specifier here means the closure did not carry it -- - * the common cause is a runtime import declared only under `devDependencies` - * (which the closure does not materialize), whether a workspace-local member or - * an external package. Rewrite the opaque "Cannot find module" into that - * actionable diagnostic. A non-resolution failure passes through unchanged. - */ +/** Rewrites "Cannot find module" into an actionable hint: the common cause is a devDependencies-only import. */ export function enrichProbeError(err: unknown): string { const message = errorMessage(err); const match = MISSING_MODULE_RE.exec(message); From 90297f845f66d50f8213fbf3fb35f8e3e0d6ab82 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:10:50 -0700 Subject: [PATCH 04/31] chore(sidecar): comments say why, never what (part 3) --- apps/sidecar/src/atomic-write.ts | 55 +-- apps/sidecar/src/workflow-run-pack-client.ts | 413 +++---------------- 2 files changed, 67 insertions(+), 401 deletions(-) diff --git a/apps/sidecar/src/atomic-write.ts b/apps/sidecar/src/atomic-write.ts index d0a32dda2e..4f897eafab 100644 --- a/apps/sidecar/src/atomic-write.ts +++ b/apps/sidecar/src/atomic-write.ts @@ -1,10 +1,6 @@ -// Atomic, durable file replacement for the sidecar's non-rebuildable -// on-disk records. Distinct from the cache's rebuildable temp+rename -// (no fsync, a lost write just forces a re-fetch) and from -// `fsyncWriteFile`'s in-place fsync write (no atomicity, a torn write -// leaves a half-file): this is the tier for a sole restore source that -// must survive both a process kill and a power loss without ever -// exposing a torn record. +// For a sole restore source that must survive a process kill or power +// loss without ever exposing a torn record — stronger than the cache's +// rebuildable temp+rename or fsyncWriteFile's non-atomic in-place write. import { open, rename, unlink } from "node:fs/promises"; import { dirname } from "node:path"; @@ -20,27 +16,10 @@ export interface AtomicWriteOptions { } /** - * Replace `path` with `contents` atomically and durably. The bytes land - * in a fresh per-write temp file that is fsynced and then `rename`d over - * `path`; because rename is atomic within a directory, a reader only - * ever observes the prior complete file or the new complete file, never - * a torn one. The fsync before the rename is what extends that - * guarantee past process death to OS crash / power loss: without it, the - * ext4 delayed-allocation window can surface the renamed path as a - * zero-length file after a power loss. - * - * The parent directory is fsynced after the rename so the new link is - * itself durable, but a filesystem that rejects directory fsync - * (FAT/exFAT, some network mounts) only degrades durability -- the file - * is already renamed and fsynced -- so that failure is logged, not - * thrown. - * - * `mode` is applied on the temp file's creation, so it takes effect on - * every write. A plain in-place overwrite of an existing file would - * silently keep the original file's mode instead. - * - * The temp file follows `createTarballCache`'s `.tmp..` - * naming convention for consistency across the sidecar's staged writes. + * fsync before rename extends atomicity past process death to power loss + * (without it, ext4 delayed allocation can surface a zero-length file). + * Directory fsync failure (FAT/exFAT, some network mounts) only degrades + * durability, so it's logged, not thrown. */ export async function writeFileAtomicDurable( path: string, @@ -58,10 +37,7 @@ export async function writeFileAtomicDurable( } await rename(tmp, path); } catch (cause) { - // The write failed and is about to rethrow; unlink the temp so a - // failed write leaves no orphan. Best-effort: the temp may never - // have been created, and a second failure here must not mask the - // original cause. + // Best-effort cleanup: must not mask the original cause on a second failure. await unlink(tmp).catch(() => undefined); throw cause; } @@ -79,18 +55,9 @@ export async function writeFileAtomicDurable( } /** - * Remove `path` durably: unlink it, then fsync the parent directory so the - * removal survives a power loss. A raw `unlink` leaves the directory-entry - * removal in the OS's delayed-metadata window, so a power loss can resurrect - * the file -- for a cache whose on-disk entry is the restore source, a - * resurrected entry re-stales the cache. Idempotent: a file already absent is a - * completed removal, so ENOENT on the unlink is success. - * - * The parent-directory fsync mirrors `writeFileAtomicDurable`, including its - * degrade: a filesystem that rejects directory fsync (FAT/exFAT, some network - * mounts) only weakens durability -- the file is already unlinked -- so that - * failure is logged, not thrown. It runs unconditionally so a prior removal - * that unlinked but died before the fsync is made durable on the next attempt. + * Fsyncs the parent dir after unlink so the removal survives a power loss — + * without it a resurrected file would re-stale a cache whose entry this is + * the restore source for. Idempotent: ENOENT on unlink is success. */ export async function removeFileAtomicDurable(path: string): Promise { try { diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index b70504a505..4722aaff05 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -1,17 +1,6 @@ -// Workflow-run pack push client. -// // Sits between the boot-edge substrate facade and the hub-link's -// `pushWorkflowRunPack` wire surface. The facade fires this client -// after a successful supervisor-authored `writeTreePreservingPrefix` -// against a `workflow-run` repo; the client builds the new pack via -// `RepoStore.createPack` under the supervisor principal and ships it -// over the hub link. The hub routes the pack to its workflow-run -// receiver because `repoId.kind === "workflow-run"`. -// -// The client does NOT mint a fresh signing key, transferId, or -// principal kind. The principal is the same `WorkflowRunSupervisorPrincipal` -// shape the supervisor uses for `writeTreePreservingPrefix`; the -// transferId is minted inside `HubLink.pushWorkflowRunPack`. +// pushWorkflowRunPack wire surface, fired after a successful +// writeTreePreservingPrefix against a workflow-run repo. import { type } from "arktype"; @@ -25,20 +14,9 @@ import type { HubLink } from "@intx/hub-agent"; const logger = getLogger(["interchange", "sidecar", "workflow-run-pack-client"]); export type WorkflowRunPackClient = { - /** - * Build a pack of the workflow-run repo at `ref` and ship it to the - * hub. Resolves on the hub's `repo.pack.ack`; rejects on - * `repo.pack.reject`, on a disconnect that cancels the in-flight - * transfer, or on a substrate-side `createPack` failure. The push - * failure shape is intentionally loud per the project's - * defensive-coding rule. - */ + /** Resolves on the hub's repo.pack.ack; rejects loud on reject, disconnect, or a substrate createPack failure. */ push(opts: { agentAddress: string; repoId: RepoId; ref: string }): Promise; - /** - * Seed the acknowledged-tip state after the Hub restores a ref into a fresh - * worker. This prevents a reconnect from trying to re-send an empty delta at - * the restored tip; the next real local commit remains incremental from it. - */ + /** Prevents a reconnect from re-sending an empty delta at the restored tip. */ markRestored(repoId: RepoId, ref: string, commitSha: string): void; }; @@ -52,18 +30,9 @@ export function createWorkflowRunPackClient( ): WorkflowRunPackClient { const { substrate, hubLink } = opts; - // Shadow of the substrate's shipped-tip cursor, keyed by `(repoId.id, ref)`. - // This client is the SOLE caller of `commitPackedTip` for the workflow-run - // kind, so its own record of "the last commitSha I acked" cannot drift from - // the substrate cursor. It exists to answer one question `createPack` cannot - // answer to its caller: is the current ref tip already shipped? When the tip - // equals the last acked sha there is nothing un-acked to ship, and building - // a pack would produce an empty delta whose declared tip is not in the pack - // -- which the hub rejects as `sha_mismatch`. Skipping the wire send in that - // case is what makes a re-drive of an already-shipped tip a clean no-op - // rather than a spurious rejection. The reconnect re-drive path - // (`notifyAddressRoutable`) can legitimately fire for a slot whose commits - // already landed on an earlier attempt, so this guard is load-bearing. + // Answers a question createPack can't: is the tip already shipped? Skips + // the wire send when it is, since an empty-delta pack the hub rejects as + // sha_mismatch would otherwise turn a re-drive into a spurious rejection. const lastAckedSha = new Map(); function ackKey(repoId: RepoId, ref: string): string { return `${repoId.id}/${ref}`; @@ -89,10 +58,7 @@ export function createWorkflowRunPackClient( kind: "supervisor", anchorRunId: repoId.id, }; - // Nothing to ship when the local tip is already the last acked tip: - // the run's commits landed on a prior push. Return without a wire send - // so a re-drive of an already-shipped tip is a clean no-op instead of - // an empty-delta pack the hub rejects. + // Nothing to ship if the run's commits already landed on a prior push. const tip = await substrate.resolveRef(principal, repoId, ref); if (tip !== null && tip === lastAckedSha.get(ackKey(repoId, ref))) { return; @@ -105,26 +71,15 @@ export function createWorkflowRunPackClient( ref, commitSha, }); - // `pushWorkflowRunPack` resolves only on the hub's - // `repo.pack.ack` and rejects on a reject or a reconnect that - // cancels the transfer. Advancing the substrate's shipped-tip - // cursor here — after the ack, never at build time — is what - // lets a cancelled transfer be re-shipped: a rejected push throws - // before this line, so the cursor stays put and the next - // `createPack` re-includes the un-acked commits. + // Advanced only after the ack (never at build time) so a cancelled + // transfer leaves the cursor put and gets re-shipped on retry. substrate.commitPackedTip(repoId, ref, commitSha); lastAckedSha.set(ackKey(repoId, ref), commitSha); }, }; } -/** - * Mapping registry the boot-edge substrate facade consults to resolve - * `repoId.id` (the workflow-run runId, which the deploy router - * derives by slugging the agent's mail address) back into the - * agentAddress carried on every outbound pack frame. Populated by the - * deploy router as each `agent.deploy` frame lands. - */ +/** Resolves repoId.id (the workflow-run runId) back into the agentAddress carried on outbound pack frames. */ export type DeploymentAddressRegistry = { record(runId: string, agentAddress: string): void; resolve(runId: string): string | null; @@ -146,40 +101,18 @@ export function createDeploymentAddressRegistry(): DeploymentAddressRegistry { }; } -/** - * Handler the multi-step deploy router installs on the - * `MultistepMailRouter` after a supervisor's `spawn` succeeds. The - * handler hands a delivered inbound mail off to the per-deployment - * supervisor's `routeInbound`, which dispatches into the workflow-host - * mail-bus the multi-step child's `awaitSignal` subscribes against. - */ +/** Dispatches into the workflow-host mail-bus the multi-step child's awaitSignal subscribes against. */ export type MultistepMailHandler = (message: Uint8Array) => Promise; /** - * Per-deployment-address mail handler registry the sidecar hub-link - * consults before falling back to `transport.deliver`. The multi-step - * deploy router registers a handler against the deployment's mail - * address after `wired.supervisor.spawn` succeeds, so an inbound - * `mail.inbound` frame for that address dispatches into the - * supervisor's mail-bus subscription rather than the - * never-provisioned-for-this-address transport mailbox. - * - * The registry is owned at the sidecar's host layer (not inside the - * workflow-host library) because the routing decision is between - * "legacy single-agent path" and "supervisor mail-bus path" -- two - * concrete sidecar host concerns. The workflow-host package stays - * agnostic to which transport surface its mail-bus rides on. + * Kept at the sidecar host layer (not inside workflow-host) since the + * routing decision — legacy single-agent path vs supervisor mail-bus — + * is a concrete host concern the portable package stays agnostic to. */ export type MultistepMailRouter = { register(address: string, handler: MultistepMailHandler): void; unregister(address: string): void; - /** - * Dispatch `message` to the handler registered for `address`. Returns - * `null` when no handler is registered (the caller logs and drops, and - * sends no ack). Otherwise returns the handler's durable settlement: a - * promise that resolves once the message is durably accepted and rejects - * when it was not, so the caller acks only on resolution. - */ + /** Returns null when unregistered (caller logs and drops, sends no ack). */ tryRoute(address: string, message: Uint8Array): Promise | null; }; @@ -200,17 +133,7 @@ export function createMultistepMailRouter(): MultistepMailRouter { }; } -/** - * Per-deployment signal-delivery handler the multi-step deploy router - * installs against the `MultistepSignalRouter` after a supervisor's - * `spawn` succeeds. The handler hands the signal off to the supervisor's - * `deliverSignal`, which sends a `signal.deliver` control IPC frame to - * the workflow-process child. Routing every workflow-run signal through - * the child keeps the workflow-run repo's single-writer invariant - * intact -- the child is the only writer of `runs//events/` on - * the sidecar side, so the pack-push pipeline that propagates the - * commit to the hub never races against a concurrent host-side write. - */ +/** Routing every signal through the child preserves the workflow-run repo's single-writer invariant. */ export type MultistepSignalHandler = (args: { runId: string; signalName: string; @@ -218,20 +141,7 @@ export type MultistepSignalHandler = (args: { payload: unknown; }) => Promise; -/** - * Per-deployment-address signal handler registry the sidecar hub-link - * consults on every inbound `signal.deliver` frame. The deploy router - * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds, for single-step and multi-step - * deployments alike; the handler dispatches the signal into the - * supervisor's `deliverSignal`. - * - * The registry lives at the sidecar's host layer (not inside the - * workflow-host library) for the same boundary reason as - * `MultistepMailRouter`: the routing decision is a concrete sidecar - * host concern, and the workflow-host package stays agnostic to which - * transport surface its supervisor handle rides on. - */ +// Same boundary reason as MultistepMailRouter. export type MultistepSignalRouter = { register(address: string, handler: MultistepSignalHandler): void; unregister(address: string): void; @@ -269,18 +179,9 @@ export function createMultistepSignalRouter(): MultistepSignalRouter { } /** - * Per-deployment grants-write handler the deploy router installs against - * the `MultistepGrantsRouter` after a supervisor's `spawn` succeeds. The - * handler writes the run's grants to `runs//grants.json` inside the - * deployment's `workflow-run` repo -- sibling to the run's - * `runs//events/` subtree. The write is awaited so the frame's FIFO - * completion means the grants are durable on disk before the next frame is - * processed. - * - * `senderIdentities` carries the run's authorized senders' hub-vouched keys, - * co-delivered on the same `run.grants` frame. The handler caches each one - * before the grants write, so a grant that lands durably is never missing the - * key its recipient needs to verify the sender's mail. + * The write is awaited so FIFO completion means grants are durable before + * the next frame processes. senderIdentities are cached before the write so + * a durable grant is never missing the key to verify the sender's mail. */ export type MultistepGrantsHandler = (args: { runId: string; @@ -288,21 +189,7 @@ export type MultistepGrantsHandler = (args: { senderIdentities?: readonly SenderIdentity[]; }) => Promise; -/** - * Per-deployment-address grants handler registry the sidecar hub-link - * consults on every inbound `run.grants` frame. The deploy router - * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds, for single-step and multi-step - * deployments alike; the handler writes the run's grants into the - * deployment's workflow-run repo. - * - * The registry lives at the sidecar's host layer (not inside the - * workflow-host library) for the same boundary reason as - * `MultistepMailRouter` / `MultistepSignalRouter`: the routing decision - * is a concrete sidecar host concern, and the workflow-host package - * stays agnostic to which transport surface its supervisor handle rides - * on. - */ +// Same boundary reason as MultistepMailRouter / MultistepSignalRouter. export type MultistepGrantsRouter = { register(address: string, handler: MultistepGrantsHandler): void; unregister(address: string): void; @@ -339,34 +226,10 @@ export function createMultistepGrantsRouter(): MultistepGrantsRouter { }; } -/** - * Per-deployment drain handler the multi-step deploy router installs - * against the `MultistepDrainRouter` after a supervisor's `spawn` - * succeeds. The handler hands the drain opts off to the supervisor's - * `drain`, which sends a `drain` control IPC frame to the - * workflow-process child and arms one `drainTimeout` accumulator per - * in-flight run. Cancel-mode in-flight steps abort as the child's - * controller signal flips; wait-mode steps continue. Each accumulator - * commits a signed `CancelRequested{origin: "supervisor-drain"}` - * against the workflow-run repo when the deadline expires. - */ +/** Each drainTimeout accumulator commits a signed CancelRequested against the workflow-run repo at deadline. */ export type MultistepDrainHandler = (args: { deadlineMs: number }) => Promise; -/** - * Per-deployment-address drain handler registry the sidecar hub-link - * consults on every inbound `drain.deliver` frame. The deploy router - * registers a handler against the deployment's mail address after - * `wired.supervisor.spawn` succeeds, for single-step and multi-step - * deployments alike; the handler dispatches into the supervisor's - * `drain`. - * - * The registry lives at the sidecar's host layer (not inside the - * workflow-host library) for the same boundary reason as - * `MultistepMailRouter` / `MultistepSignalRouter`: the routing decision - * is a concrete sidecar host concern, and the workflow-host package - * stays agnostic to which transport surface its supervisor handle - * rides on. - */ +// Same boundary reason as MultistepMailRouter / MultistepSignalRouter. export type MultistepDrainRouter = { register(address: string, handler: MultistepDrainHandler): void; unregister(address: string): void; @@ -395,34 +258,13 @@ export function createMultistepDrainRouter(): MultistepDrainRouter { }; } -/** - * Per-deployment sources-rotation handler the deploy router installs - * against the `MultistepSourcesRouter` after a supervisor's `spawn` - * succeeds -- but ONLY for a single-step (warm launched-agent) - * deployment. The handler hands the rotated list off to the supervisor's - * `deliverSources`, which sends a `sources-updated` control IPC frame to - * the workflow-process child, where the warm agent's live sources are - * swapped in place. A multi-step deployment has no single warm agent to - * rotate, so the router registers no handler for it and an inbound - * `sources.update` for a multi-step address is unrouted. - */ +/** Only for single-step (warm launched-agent) deployments; multi-step has no single warm agent to rotate. */ export type MultistepSourcesHandler = (args: { sources: InferenceSource[]; defaultSource: string; }) => Promise; -/** - * Per-deployment-address sources-rotation handler registry. Only a - * single-step warm deployment registers a handler (after - * `wired.supervisor.spawn` succeeds); a multi-step deployment never - * does, so `tryRoute` resolves a rotation only for a registered - * single-step address and returns `false` for any other. - * - * The registry lives at the sidecar's host layer for the same boundary - * reason as the mail/signal/drain routers: the routing decision is a - * concrete sidecar host concern, and the workflow-host package stays - * agnostic to which transport surface its supervisor handle rides on. - */ +// Same boundary reason as the mail/signal/drain routers. export type MultistepSourcesRouter = { register(address: string, handler: MultistepSourcesHandler): void; unregister(address: string): void; @@ -445,19 +287,10 @@ export function createMultistepSourcesRouter(): MultistepSourcesRouter { }, async tryRoute(frame) { const handler = handlers.get(frame.agentAddress); - // Registration check first: an unregistered (multi-step or torn-down) - // address is unrouted -- reported as `false`, its payload never - // inspected, because it would not be acted on regardless. if (handler === undefined) return false; - // Validate the rotation BEFORE dispatch. This is the only inbound - // router that validates its frame, and deliberately so: a bad list - // (duplicate ids, or a default that is not the head element) would - // reach the child's control-channel receiver and crash it on - // `SourcesUpdatedData`'s narrow -- the sources-updated frame is the - // only inbound frame carrying a crash-on-invalid narrow downstream, - // and the only one that is request/ack. Rejecting here throws, and - // the hub-link turns the throw into a truthful `session.error` - // instead of acking and detonating the child. + // Validated before dispatch, unlike other routers: a bad frame would + // crash the child's control-channel receiver on SourcesUpdatedData's + // narrow. Throwing here turns it into a session.error instead. const validated = SourcesUpdatedData({ sources: frame.sources, defaultSource: frame.defaultSource, @@ -475,31 +308,15 @@ export function createMultistepSourcesRouter(): MultistepSourcesRouter { } /** - * Per-deployment credential-delivery handler the deploy router installs - * against the `MultistepCredentialsRouter` after a supervisor's `spawn` - * succeeds. The handler hands the delivery to the supervisor's - * `deliverCredentials`, which sends a `credentials-updated` control IPC frame - * to the workflow-process child, where the material cell is swapped in place. - * - * Unlike sources rotation, this registers for ANY deployment (single- or - * multi-step): the material cell is per-child and read by every step's tool - * capabilities, so there is no single-warm-agent restriction. There is no - * durable persist -- credential material never touches disk (it is re-resolved - * by the hub on reconnect). + * Unlike sources rotation, registers for any deployment: the material cell + * is per-child. No durable persist — material never touches disk. */ export type MultistepCredentialsHandler = (args: { delivery: CredentialDelivery; revoke?: string[]; }) => Promise; -/** - * Per-deployment-address credential-delivery handler registry. Mirrors - * `MultistepSourcesRouter`: `credentials.update` is a REQUEST/ACK frame, so a - * registered address that throws surfaces as a `session.error` and an - * unregistered address returns `false` (unrouted). Lives at the sidecar host - * layer for the same boundary reason -- the workflow-host package stays - * agnostic to the transport surface its supervisor rides on. - */ +/** Mirrors MultistepSourcesRouter: request/ack, so a throw surfaces as session.error. */ export type MultistepCredentialsRouter = { register(address: string, handler: MultistepCredentialsHandler): void; unregister(address: string): void; @@ -522,14 +339,9 @@ export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { }, async tryRoute(frame) { const handler = handlers.get(frame.agentAddress); - // Registration check first: an unregistered (torn-down) address is - // unrouted -- reported as `false`, its payload never inspected. if (handler === undefined) return false; - // Validate the delivery BEFORE dispatch: a malformed delivery would - // reach the child's control-channel receiver and crash it on - // `CredentialsUpdateFrame`'s narrow. Rejecting here throws, and the - // hub-link turns the throw into a truthful `session.error` instead of - // acking and detonating the child. + // Validated before dispatch: a malformed delivery would crash the + // child's control-channel receiver instead of surfacing as session.error. const validated = CredentialDelivery(frame.delivery); if (validated instanceof type.errors) { throw new Error(validated.summary); @@ -544,61 +356,14 @@ export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { } /** - * Boot-edge facade around the substrate-shaped `RepoStore`. Forwards - * every method to the underlying store; intercepts the - * `writeTreePreservingPrefix` return path so a successful write - * against a `workflow-run` repo schedules a workflow-run pack push. - * Writes against any other `repoId.kind` (today, only `agent-state` - * via the deploy-applier path) flow through unchanged. - * - * Pack-push coalescing: the facade returns from - * `writeTreePreservingPrefix` as soon as the LOCAL commit lands and - * schedules an asynchronous pack push for `(repoId.id, ref)`. At - * most one push per (repoId, ref) is in flight at a time. Writes - * that arrive while a push is in flight are NOT enqueued as - * additional pushes; instead they mark the slot as "dirty", and the - * loop runs one more push after the current one settles. This means - * a burst of N writes against the same ref produces at most 2 - * pushes (the one already running when the burst starts, plus one - * more for everything that arrived during it), rather than N - * serial round-trips' worth of hub-ack latency. The push body - * captures the current local ref tip at the moment it runs, so the - * single pack it builds covers every commit landed since the prior - * ACKED tip -- the substrate's incremental `createPack` walks the - * chain from the cursor `commitPackedTip` last committed on an ack - * forward, so the receiver still sees every commit transition. - * - * Single-writer + FIFO correctness: the underlying substrate - * serialises local writes via `withRepoLock`, so commits land on - * disk in submission order. The hub's `receivePack` validates each - * commit's parent against its existing-commits set; as long as the - * pack carries the full chain from prior acked tip to current tip, - * every intermediate commit is validated by the receiver. Coalescing - * multiple local commits into one network push therefore preserves - * the receive-time CAS invariant while collapsing N hub round-trips - * into 1. - * - * Reconnect-safe re-shipping: the substrate advances its shipped-tip - * cursor on the ack (`push` calls `commitPackedTip` only after - * `pushWorkflowRunPack` resolves), never at build time. A transfer a - * reconnect cancels before its ack therefore leaves the cursor where - * it was, so the retry loop's next `createPack` re-includes the - * un-acked commits and the receiver gets a self-consistent chain - * rather than a pack whose base commit it never received. - * - * Failure surfacing: a failed push latches its error on the - * per-(repoId, ref) slot's `lastError` field. The next call to - * `writeTreePreservingPrefix` on that (repoId, ref) re-throws the - * latched error before doing its own work, keeping failures loud - * rather than swallowed by the fire-and-forget pipeline. The - * defensive-coding rule says errors must surface; this is how they - * surface from a coalescing writer. - * - * Flush: callers that need a hub-visible barrier (shutdown, - * integration tests that read hub-side state) call - * `flushWorkflowRunPushes(repoId, ref)` to await the per-(repoId, - * ref) slot to drain (both the in-flight push and any follow-up - * triggered by writes that arrived during it). + * Wraps RepoStore so a successful writeTreePreservingPrefix against a + * workflow-run repo schedules a coalesced pack push: at most one push per + * (repoId, ref) in flight, with writes arriving mid-push marking the slot + * dirty for one more run rather than queuing N pushes. The shipped-tip + * cursor advances only after the hub's ack, so a reconnect-cancelled + * transfer is re-shipped whole on the next createPack rather than leaving + * a gap. A failed push latches on the slot and re-throws on the next write + * to that (repoId, ref) instead of being swallowed. */ export type WorkflowRunPackPushingRepoStoreOpts = { underlying: RepoStore; @@ -606,48 +371,17 @@ export type WorkflowRunPackPushingRepoStoreOpts = { registry: DeploymentAddressRegistry; }; -/** - * The wrapped store plus a side-channel API for waiting on the - * per-(repoId.id, ref) pack-push pipeline to drain. The `RepoStore` - * shape is unchanged so call sites that consume `RepoStore` keep - * working; `flushWorkflowRunPushes` is opt-in for code that - * genuinely needs hub-side visibility (shutdown, integration tests, - * end-to-end benchmarks). Call sites that don't need it pay zero - * cost. - */ +/** RepoStore shape is unchanged; flushWorkflowRunPushes is opt-in for callers needing hub-side visibility. */ export type WorkflowRunPackPushingRepoStore = RepoStore & { - /** - * Await the pack-push pipeline for `(repoId.id, ref)` to drain. - * Resolves once no push is in flight and no follow-up push is - * pending; rejects if the most recent push failed (the same - * latched error the next `writeTreePreservingPrefix` call would - * surface). - */ + /** Resolves once the (repoId.id, ref) pipeline drains; rejects on the same latched error a write would surface. */ flushWorkflowRunPushes: (repoId: RepoId, ref: string) => Promise; /** - * Re-drive any workflow-run push for `agentAddress` that a disconnect - * cancelled. Called when the hub-link observes the deployment address - * become routable again after an authenticated reconnect. For each slot bound - * to `agentAddress` whose last push attempt failed (its `lastError` is - * latched), it re-arms the coalescing loop so a fresh `createPack` re-ships - * the un-acked commits -- the liveness path a synchronous single-step run - * lacks, because it has no later local write to re-set `dirty`. - * - * A re-ship that fails again re-latches without self-retrying, so a - * genuinely unrecoverable failure still surfaces loudly on the next local - * write rather than spinning. It is gated on the address being routable - * again (the caller only fires post-reconnect) so the re-ship cannot race - * ahead of the hub re-routing the address. + * Re-arms a slot a disconnect cancelled — the liveness path a synchronous + * single-step run lacks, since it has no later local write to re-set dirty. + * Gated on post-reconnect so the re-ship can't race ahead of hub re-routing. */ notifyAddressRoutable: (agentAddress: string) => void; - /** - * Block workflow-run pushes for `agentAddress` until the next - * `notifyAddressRoutable`. Called when the hub-link observes its WS drop: - * the address's hub route is gone until the authenticated reconnect restores - * it, so a push shipped in the interim is dropped by the hub as - * "unrouted". Holding the push at the block -- rather than shipping and - * failing -- is what lets the reconnect re-ship wait for route restoration. - */ + /** Holds pushes rather than shipping-and-failing, so a WS-drop address waits for reconnect before shipping. */ markAddressUnroutable: (agentAddress: string) => void; }; @@ -670,16 +404,8 @@ export function createWorkflowRunPackPushingRepoStore( return `${repoId.kind}/${repoId.id}/${ref}`; } - // Addresses whose hub route was dropped and has not been re-established by a - // authenticated reconnect. Absent means routable -- the steady state, and the - // first-connect state (a deployment routes via its `agent.deploy`, not a - // reconnect, so it is never blocked before its first push). An address is - // added on `markAddressUnroutable` (WS disconnect) and removed on - // `notifyAddressRoutable` (reconnect sent). A push for a blocked address - // is held: the coalescing loop pauses with `dirty` still set rather than - // shipping to a hub that has not yet re-routed the address -- which is what - // makes the reconnect re-ship wait for route restoration instead of racing - // ahead of it and being dropped as "unrouted". + // Absent means routable. A blocked address's push holds with dirty still + // set, rather than racing ahead of hub re-routing and being dropped. const blockedAddresses = new Set(); function notifySettled(slot: Slot): void { @@ -690,17 +416,11 @@ export function createWorkflowRunPackPushingRepoStore( function startLoop(slot: Slot, repoId: RepoId, ref: string): void { if (slot.inFlight !== null) return; - // Hold the push while the address is not routable (dropped, awaiting the - // authenticated reconnect). Leave `dirty` set and start no loop: the loop - // resumes when `notifyAddressRoutable` clears the block and re-arms it. - // Shipping now would race ahead of the hub re-routing the address, and - // the frames would be dropped as "unrouted". if (blockedAddresses.has(slot.agentAddress)) return; slot.inFlight = (async () => { while (slot.dirty) { - // Re-check routability each iteration: a disconnect mid-drain must - // pause the loop rather than push into a severed link. Leave `dirty` - // set so the post-reconnect resume re-ships. + // Re-checked each iteration: a disconnect mid-drain must pause + // rather than push into a severed link. if (blockedAddresses.has(slot.agentAddress)) break; slot.dirty = false; try { @@ -736,11 +456,7 @@ export function createWorkflowRunPackPushingRepoStore( }; slots.set(key, slot); } else { - // The agentAddress is derived from a stable per-deployment - // mapping; refreshing it on every call keeps the slot in sync - // if the registry ever re-resolves the same runId to - // a different address (today it does not, but the contract is - // "look up at push time", not "cache forever"). + // Refreshed every call, not cached: the contract is "look up at push time". slot.agentAddress = agentAddress; } slot.dirty = true; @@ -756,33 +472,16 @@ export function createWorkflowRunPackPushingRepoStore( } function markAddressUnroutable(agentAddress: string): void { - // The hub route for this address just dropped (WS disconnect). Block its - // pushes until the authenticated reconnect re-routes it. A push already - // in-flight when the link dropped rejects through `packSender.cancelAll` - // and latches its error; the block stops the coalescing loop from - // immediately re-shipping on the fresh (not-yet-registered) connection. blockedAddresses.add(agentAddress); } function notifyAddressRoutable(agentAddress: string): void { - // The authenticated reconnect re-routed this address on the hub. Clear the - // block and re-drive so a push the disconnect cancelled -- or one held - // while the block was up -- ships now. This is the liveness path a - // synchronous single-step run lacks: with all its events in one batch it - // has no later local write to re-arm the coalescing loop, so the drop - // would otherwise strand it forever. blockedAddresses.delete(agentAddress); for (const slot of slots.values()) { if (slot.agentAddress !== agentAddress) continue; - // Re-drive a slot that has pending work (`dirty`, e.g. a push held at - // the block) OR whose last attempt failed (`lastError` latched by the - // disconnect-cancel). A slot that is clean and already acked - // (`!dirty && lastError === null`) has nothing un-shipped -- the - // `packClient.push` empty-delta guard would skip it anyway, but not - // re-arming it avoids a pointless loop spin. Re-arming `dirty` and - // restarting is safe against double-ship: `startLoop` no-ops when a - // push is already in flight, and the per-(repoId, ref) serialization in - // the hub-link's `pushWorkflowRunPack` prevents overlapping transfers. + // Skip a clean, already-acked slot to avoid a pointless loop spin. + // Re-arming is safe against double-ship: startLoop no-ops if a push + // is already in flight. if (!slot.dirty && slot.lastError === null) continue; slot.dirty = true; startLoop(slot, slot.repoId, slot.ref); From d7de81506257b93b3667d44d7feb5d3a0fce204a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:12:49 -0700 Subject: [PATCH 05/31] chore(sidecar): comments say why, never what (part 4) --- .../src/sidecar-materialization-config.ts | 62 +--- apps/sidecar/src/step-agent-tools.ts | 332 +++--------------- 2 files changed, 67 insertions(+), 327 deletions(-) diff --git a/apps/sidecar/src/sidecar-materialization-config.ts b/apps/sidecar/src/sidecar-materialization-config.ts index 9ff1de0e98..ee5646959e 100644 --- a/apps/sidecar/src/sidecar-materialization-config.ts +++ b/apps/sidecar/src/sidecar-materialization-config.ts @@ -1,21 +1,12 @@ -// Shared sidecar-local materialization config helpers. -// -// The sidecar's per-step tool-package apply path -// (`tool-materialization.ts`) and the workflow-probe closure -// materializer (`workflow-closure-materialization.ts`) both resolve the -// same two boundary inputs before they construct a `@intx/tool-packaging` -// loader: the operator's registry map, and the host platform token pair -// asserted against npm's `os`/`cpu` namespaces. Centralizing them here -// keeps both call sites reading one source of truth rather than -// duplicating the env parsing and the platform allowlists. +// Shared by tool-materialization.ts and workflow-closure-materialization.ts +// so both loader-boundary inputs (registry map, host platform) come from +// one source of truth rather than duplicated env parsing and allowlists. import { type } from "arktype"; import type { HostPlatform, RegistryConfig } from "@intx/tool-packaging"; -// Boundary validator for the SIDECAR_TOOL_REGISTRIES env var. The -// env-wire shape carries `name` alongside the registry config so an -// operator can author the JSON as a flat array; the boundary collapses -// the array into a Map keyed by name before handing it to the loader. +// name rides alongside the config so an operator authors a flat JSON array; +// this collapses it into a Map keyed by name for the loader. const RegistryConfigEnvEntry = type({ name: "string", url: "string", @@ -31,13 +22,8 @@ export function readRegistries(): ReadonlyMap { if (raw === undefined) { return new Map([["npmjs", { url: "https://registry.npmjs.org" }]]); } - // Distinguish unset from empty. An operator setting the var to an - // empty string almost always indicates misconfig (CI secret - // expansion failed, a templater dropped the value). Falling through - // to the npmjs default at that point silently routes tool packages - // through public npm, which is precisely the misroute a custom - // registry pin was meant to prevent. Surface the gap loudly; the - // recovery is `unset SIDECAR_TOOL_REGISTRIES`, not `=""`. + // An empty string almost always means misconfig; falling through to the + // npmjs default would silently route packages through public npm. if (raw.trim() === "") { throw new Error( "SIDECAR_TOOL_REGISTRIES is set but empty — unset the variable to use the default npmjs registry", @@ -71,21 +57,9 @@ export function readRegistries(): ReadonlyMap { return out; } -// npm's `os` token namespace, mirrored from Node's `process.platform` -// enum. Any value outside this set means the host is running a Node -// build the loader's platform filter would silently mis-route — a -// pinned package whose `os` list excludes the host would not be -// excluded if `process.platform` is a token npm has never heard of. -// Validate at the boundary so an unknown platform fails the boot -// instead of producing a quiet, host-shaped mis-resolution at apply -// time. -// -// UPGRADE TAX: Node periodically adds platforms (and Bun ships -// extensions of its own). A Node/Bun major bump that lands a new -// `process.platform` value will fail boot until this allowlist is -// refreshed against the upstream enum. Sidecar operators upgrading -// the runtime should expect this as part of the cutover, not as a -// surprise regression. +// Validated at the boundary so an unknown process.platform token fails +// boot instead of silently mis-routing the loader's os filter. A Node/Bun +// major bump adding a new platform value will fail boot until refreshed. const KNOWN_PROCESS_PLATFORMS = new Set([ "aix", "android", @@ -100,12 +74,7 @@ const KNOWN_PROCESS_PLATFORMS = new Set([ "netbsd", ]); -// npm's `cpu` token namespace, mirrored from Node's `process.arch` -// enum. Same rationale as KNOWN_PROCESS_PLATFORMS — an unknown arch -// would mis-route the loader's filter without surfacing the gap. -// Same upgrade tax applies: Node has added `loong64` and `riscv64` -// in recent releases, and future arch additions will need to be -// added here when the sidecar is rebuilt against them. +// Same rationale as KNOWN_PROCESS_PLATFORMS, for process.arch. const KNOWN_PROCESS_ARCHS = new Set([ "arm", "arm64", @@ -135,14 +104,7 @@ function assertKnownHostArch(arch: NodeJS.Architecture): void { } } -/** - * Resolve the host platform token pair the `@intx/tool-packaging` loader - * filters manifest entries against, asserting each token is one npm - * recognizes before returning it. Both consumers (`tool-materialization.ts` - * and `workflow-closure-materialization.ts`) resolve the host through this - * one helper so the allowlists and the fail-loud gate live in a single - * place. - */ +/** Both consumers resolve the host through this one helper so the allowlists and gate live in one place. */ export function resolveHostPlatform(): HostPlatform { assertKnownHostPlatform(process.platform); assertKnownHostArch(process.arch); diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index e61b3b3aea..446088e2ce 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -1,29 +1,9 @@ -// Per-step tool materialization and agent construction for the -// workflow-process child. -// -// The child IS the sidecar binary, so the sidecar's tool runtime -// (`@intx/tool-packaging` loader, posix, the LSP plugin) is present in -// the child's address space. This module is the seam that runs that -// runtime when the real step-invoker builds a step's agent: it reads -// the step's deploy tree off disk, materializes the pinned tool-package -// closure via `materializeToolPackages`, builds the plugin chain, -// attaches the resulting tool factories to the step's -// `AgentDefinition`, and returns an `Agent` whose `close()` tears every -// plugin and tool bundle down. -// -// LSP lifecycle (the riskiest sub-item, design §6): the LSP plugin -// factory's `dispose` terminates the LSP subprocess. The -// `createWorkflowStepInvoker` adapter calls `agent.close()` in a -// `finally` on EVERY exit path (clean completion, abort, rejection), -// so wrapping `close()` to also run the plugin + bundle disposers is -// what guarantees the LSP subprocess dies with the step's agent -- -// no leak across steps, on abort, or on child recycle/drain (recycle -// kills the child process, which kills the LSP grandchild regardless). -// -// Layering (design §3d): everything here lives in `apps/sidecar`. The -// portable `@intx/workflow-host` package never gains a dependency on -// the tool runtime; it only sees the `agentFactory` callback this -// module produces. +// The child IS the sidecar binary, so its tool runtime is present in the +// child's address space; this module runs it when the real step-invoker +// builds a step's agent. Wrapping close() to run plugin/bundle disposers +// is what guarantees the LSP subprocess dies with the step's agent on +// every exit path. Lives in apps/sidecar so @intx/workflow-host stays +// free of the tool runtime dependency. import path from "node:path"; @@ -60,55 +40,24 @@ import { materializeToolPackages, type StepToolFactory } from "./tool-materializ const logger = getLogger(["sidecar", "workflow-child", "step-tools"]); -/** - * Cache and registry caps the per-step tool loader needs. Resolved at - * the sidecar boot edge from the existing `SIDECAR_CACHE_*` / - * `SIDECAR_REGISTRY_*` config keys and threaded into the child through - * the substrate config, so the child's per-step materialization is - * bounded by those boot-edge-resolved caps. - */ +/** Resolved at the boot edge and threaded into the child so per-step materialization stays bounded. */ export interface StepToolCacheConfig { readonly cacheMaxBytes: number; readonly registryMaxTarballBytes: number; } -/** - * Materialized tool runtime for one step's agent. Carried from - * `buildEnv` (which knows the step's identity) to the `agentFactory` - * (which knows the agent definition + env) via a symbol-keyed slot on - * the per-step env so the two callbacks of `createWorkflowStepInvoker` - * can cooperate without widening the portable adapter's surface. - */ +/** Carried via a symbol-keyed slot on the per-step env so the two step-invoker callbacks cooperate without widening its surface. */ export interface StepToolMaterialization { readonly factories: readonly StepToolFactory[]; readonly pluginFactories: readonly AnnotatedPluginFactory[]; } /** - * Derive the per-step tool-mark floor grants from a step's materialized - * tool factories. Each loaded factory carries its static - * `definitions` (name + optional `approval` mark) verbatim from the tool - * package; every declared tool contributes a `tool:` / `invoke` - * grant whose effect is the tool's floor (`ask` for an approval-gated - * tool, `allow` otherwise), computed through the same - * `toolApprovalEffect` mapping the deploy-time capability walk uses so a - * pinned tool floors identically hub-side and sidecar-side. - * - * The hub's capability walk reads only INLINE `agent.toolFactories`, so a - * tool that ships as a pinned package loads in the child and never - * produces a `tool:` grant on the run principal. These derived rows - * supply that missing floor: they join the per-step grants the authz path - * evaluates as ADDITIONAL rows, so `evaluateGrants` precedence still - * resolves an explicit `deny` (priority 2) over the derived `ask`/`allow` - * -- the floor only raises the minimum authority to the tool's mark, it - * never overrides a declared denial. - * - * The grant `id` is deterministic (`floor:tool:`) rather than - * random: `evaluateGrants` never dedupes or joins on `id` (it ranks by - * specificity then effect), so a stable id keeps the rows reproducible - * without a generator, and a floor row that coincides with a - * hub-supplied `tool:` row resolves by effect precedence regardless - * of the ids. + * The hub's capability walk only reads inline agent.toolFactories, so a + * pinned-package tool never gets a hub-side tool: grant; these + * derived rows supply that floor as additional grants, which an explicit + * deny still overrides by evaluateGrants precedence. The id is + * deterministic (floor:tool:) since evaluateGrants doesn't dedupe by id. */ export function deriveToolMarkFloorGrants(factories: readonly LoadedToolFactory[]): GrantRule[] { const rows: GrantRule[] = []; @@ -130,20 +79,11 @@ export function deriveToolMarkFloorGrants(factories: readonly LoadedToolFactory[ return rows; } -/** - * Symbol-keyed slot the `buildEnv` callback sets on the env it returns - * and the `agentFactory` reads. Object spread (`{ ...envBase, - * authorize }`) inside the step-invoker adapter copies own enumerable - * symbol-keyed properties, so the slot survives the spread that - * produces the env handed to `agentFactory`. - */ +// Symbol survives the step-invoker adapter's `{ ...envBase, authorize }` +// spread, which copies own enumerable symbol-keyed properties. const STEP_TOOLS = Symbol("intx.sidecar.step-tools"); -// The per-step env carries one private symbol-keyed slot the -// buildEnv/agentFactory pair cooperate over. The slot is read/written -// through `Reflect.get`/`Reflect.set` so neither site needs a type -// assertion: `BaseEnv` is an interface without a symbol index -// signature, so a structural cast would otherwise be required. +// Read/written via Reflect since BaseEnv has no symbol index signature. function setStepToolSlot(env: object, materialization: StepToolMaterialization): void { Reflect.set(env, STEP_TOOLS, materialization); } @@ -170,16 +110,7 @@ function isStepToolMaterialization(value: unknown): value is StepToolMaterializa ); } -/** - * Symbol-keyed slot carrying the per-step credential wiring the - * `agentFactory` assembles each bundle's consumer-scoped `credentials` - * capability from: the live material cell, the step's grants, and the - * provider registry. Set by `buildEnv` alongside the tool slot, read by - * `createToolBearingAgentFactory`. Absent for a build that threaded no - * credential context (a unit test using the bare factory) -- then no - * credentials capability is assembled and bundles keep the base capabilities - * bag. - */ +/** Absent for a build with no credential context (e.g. a unit test), in which case bundles keep the base bag. */ const STEP_CREDENTIAL_WIRING = Symbol("intx.sidecar.step-credential-wiring"); function setStepCredentialWiring(env: object, wiring: StepCredentialWiring): void { @@ -207,13 +138,7 @@ function isStepCredentialWiring(value: unknown): value is StepCredentialWiring { ); } -/** - * Attach a step's credential wiring to the per-step env so the tool-bearing - * `agentFactory` can assemble each bundle's `credentials` capability. Called - * by `buildEnv` for a step whose build threaded a credential context; omitted - * otherwise, which leaves the slot unset and the credentials capability - * unassembled. - */ +/** Omitted for a build with no credential context, leaving the slot unset. */ export function attachStepCredentialWiring( env: Omit, wiring: StepCredentialWiring, @@ -221,14 +146,7 @@ export function attachStepCredentialWiring( setStepCredentialWiring(env, wiring); } -/** - * Read the base `RuntimeCapabilities` bag `buildEnv` set on the per-step env - * (`env.capabilities`, currently `mail.transport`). `BaseEnv` does not type - * the key -- it is a runtime-only widening the step env carries -- so it is - * read reflectively and validated. Throws when a bundle has a credentials - * capability to layer but the env carries no base bag: that is a wiring - * inconsistency (the same `buildEnv` sets both), not a condition to paper over. - */ +/** Throws on a missing base bag: that's a wiring inconsistency (buildEnv sets both), not to paper over. */ function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { const value: unknown = Reflect.get(env, "capabilities"); if ( @@ -246,25 +164,9 @@ function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { } /** - * Resolve the on-disk directory holding a step's deploy tree. - * - * The deploy tree (`deploy/prompt.md`, `deploy/tool-packages-manifest.json`, - * `deploy/asset-mounts.json`) is shipped to the sidecar per step by the - * hub's `launchSession` deploy-pack push, which lands it in the LEGACY - * per-agent directory keyed by the step's sanitized mail address (see - * `@intx/hub-agent` `agentDir`). It is NOT in the substrate's - * `agent-state/` layout -- the multi-step deploy path never pushes - * step `agent-state` packs to the child's substrate. - * - * The step's mail address is `resolveStepAddress(...)`, the single owner - * of the head/step collapse: for a single-step deployment the lone step - * IS the head (the deployment mailbox itself), so the tree is read at the - * head; for multi-step it is `deriveStepAddress(runId, stepId, - * domain)`. The `runId`/`domain` are recovered from the deployment mailbox - * address the supervisor threaded into the child as `MAILBOX_ADDRESS` - * (`@`): the local part is the run id and the address domain - * is the deployment domain. `stepCount` is sourced from the host (via - * `substrateEnv`) so producer and consumer never derive divergent addresses. + * The deploy tree lands in the legacy per-agent directory keyed by the + * step's mail address, not the substrate's agent-state/ layout — the + * multi-step deploy path never pushes step agent-state packs there. */ export function stepDeployTreeDir(args: { dataDir: string; @@ -278,11 +180,8 @@ export function stepDeployTreeDir(args: { `sidecar workflow-child step tools: deployment mailbox address ${JSON.stringify(args.mailboxAddress)} is not a parseable run address; cannot locate the step's deploy tree`, ); } - // A `map` iteration runs under a scoped step id `[]`, but - // deploy stages one deploy tree per base step, so the scoped id resolves - // to its base address -- every iteration reads the base step's tree. - // `baseStepId` is the identity on an unscoped id, so a plain step is - // unaffected. + // A map iteration's scoped step id resolves to its base address, since + // deploy stages one tree per base step; baseStepId is identity otherwise. const stepAddress = resolveStepAddress({ runId: parsed.runId, stepId: baseStepId(args.stepId), @@ -292,19 +191,7 @@ export function stepDeployTreeDir(args: { return agentDir(args.dataDir, stepAddress); } -/** - * Read a step's deploy tree and materialize its pinned tool-package - * closure. The tarball cache and the tool instance dir are rooted under - * the supplied per-step `storeDir` (the Phase-1 per-step state root) so - * concurrent steps/agents in one child never collide on cache or - * apply-state paths. - * - * A deploy with no tool-package manifest yields empty factories -- the - * legitimate `rawManifestBytes === undefined` case. A manifest that is - * present but fails to load surfaces loudly through - * `materializeToolPackages` (the throw path), never a silent - * empty-tools fallback that would mask a broken deploy. - */ +/** Rooted under storeDir so concurrent steps in one child never collide on cache/apply-state paths. */ export async function materializeStepTools(args: { dataDir: string; mailboxAddress: string; @@ -322,25 +209,14 @@ export async function materializeStepTools(args: { }); const deployTree = await readDeployTree(deployTreeDir); - // Root the tarball cache per step so concurrent steps in one child - // do not race on the content-addressable cache root. The cache is - // content-addressed and therefore safe to share globally, but the - // design (§3d, point 3) calls for a per-step cacheRoot so a wedged - // or partially-written apply in one step cannot corrupt another's - // view. + // Per-step even though content-addressed, so a wedged apply in one step + // cannot corrupt another's view. const cacheRoot = path.join(args.storeDir, "tarball-cache"); - // Asset-mounted tool tarballs are staged by the hub's asset-pack push - // into the step's LEGACY agent dir workspace (the same dir the deploy - // tree lives in), not under the per-step store dir. Point the loader's - // asset resolution there while keeping the apply-state + cache rooted - // per step under `storeDir`. - // - // This `/workspace` (read-only staged assets, keyed by the - // BASE step) and the agent's read-write workdir `/workspace` - // (keyed by the SCOPED step) share a leaf name but are deliberately - // different roots -- a map iteration reads one shared deploy tree while - // each iteration writes its own scratch. Do not unify them. + // deployTreeDir/workspace (read-only, keyed by base step) and + // storeDir/workspace (read-write, keyed by scoped step) share a leaf name + // but are deliberately different roots — a map iteration reads one shared + // tree while each iteration writes its own scratch. Do not unify them. const assetRoot = path.join(deployTreeDir, "workspace"); const materialized = await materializeToolPackages({ @@ -359,16 +235,7 @@ export async function materializeStepTools(args: { }; } -/** - * Attach a step's materialized tool runtime to the per-step env so the - * tool-bearing `agentFactory` can consume it. Called by `buildEnv` - * after `materializeStepTools` resolves. - * - * The parameter is `Omit` because `buildEnv` - * yields exactly that shape (the step-invoker adapter adds `authorize` - * before the env reaches `agentFactory`); the symbol slot survives the - * adapter's `{ ...envBase, authorize }` spread. - */ +/** Omit because buildEnv yields exactly that shape before the adapter adds authorize. */ export function attachStepTools( env: Omit, materialization: StepToolMaterialization, @@ -376,24 +243,7 @@ export function attachStepTools( setStepToolSlot(env, materialization); } -/** - * Re-wrap a loaded tool factory so its bundle's `dispose` (when present) - * is captured via `onDispose`, forwarding the loader's `id`, `requires`, - * and `definitions` so the result is a real `AnnotatedToolFactory`, not a - * hand-shaped lookalike. The static `definitions` declaration is - * forwarded verbatim: this wrapper does not rename tools, so the names - * the deploy-time walk enumerates must survive the re-wrap unchanged. - * - * When `credentials` is supplied, the bundle's factory sees a per-bundle - * capabilities bag: the step's base bag layered with THIS package's - * consumer-scoped `credentials` capability, so the bundle resolves only the - * handles its own package is authorized for. When it is absent (the package - * declares and binds no credential), the base bag is passed through unchanged. - * - * Exported so the definitions-preservation contract is testable in - * isolation; the production path calls it from - * `createToolBearingAgentFactory`. - */ +/** definitions is forwarded verbatim: this wrapper must not rename tools the deploy-time walk enumerated. */ export function rewrapStepToolFactory( annotated: AnnotatedToolFactory, onDispose: (dispose: () => unknown) => void, @@ -415,43 +265,21 @@ export function rewrapStepToolFactory( }); } -/** - * Produce a copy of `factoryEnv` whose `capabilities` bag is the base bag - * layered with this bundle's consumer-scoped `credentials` capability. Object - * spread copies own enumerable properties -- including the runtime-only - * `capabilities` key `BaseEnv` does not type and the private tool slot -- so - * every other env surface survives; only `capabilities` is replaced. - */ +/** Spread preserves every other env surface (including the private tool slot); only capabilities is replaced. */ function layerCredentialsOntoEnv( factoryEnv: BaseEnv, credentials: HostCredentialCapability, ): BaseEnv { const base = requireCapabilitiesBag(factoryEnv); const layered = layerRuntimeCapabilities(base, { credentials }); - // Copy every env surface (spread carries own enumerable string AND symbol - // keys, so `transport`, `address`, and the private tool/credential slots - // survive), then replace the runtime-only `capabilities` key via Reflect -- - // a `{ ...factoryEnv, capabilities }` literal trips the excess-property - // check because BaseEnv does not type the key. + // Reflect.set, not a literal: BaseEnv doesn't type "capabilities", so a + // `{ ...factoryEnv, capabilities }` literal trips the excess-property check. const layeredEnv: BaseEnv = { ...factoryEnv }; Reflect.set(layeredEnv, "capabilities", layered); return layeredEnv; } -/** - * Build the `agentFactory` the workflow step-invoker uses. The returned - * factory reads the materialized tool runtime off the env (set by - * `buildEnv` via `attachStepTools`), augments the step's - * `AgentDefinition` with the loaded tool factories (wrapped to capture - * each bundle's disposer), constructs the plugin chain on `env.plugins`, - * builds the agent, and wraps - * `agent.close()` so every plugin instance and tool bundle is disposed - * when the step's agent closes. - * - * When the env carries no materialized tools (the `buildEnv` did not - * run materialization, e.g. a unit test using the bare factory), the - * factory falls back to `createAgent(def, env)` unchanged. - */ +/** Falls back to createAgent(def, env) unchanged when the env carries no materialized tools (e.g. a bare-factory test). */ export function createToolBearingAgentFactory(): ( def: AgentDefinition, env: EnvReq, @@ -465,24 +293,15 @@ export function createToolBearingAgentFactory(): ( return createAgent(def, env); } - // Assemble each package's consumer-scoped `credentials` capability once, - // when the step carries credential wiring (a bare test build carries none, - // yielding an empty map). Every factory in a package shares the one - // capability; a package that declares a handle no binding resolves fails - // the launch here, loudly, rather than at the tool's first resolve. This - // can throw (reconcile fail-closed, malformed delivery) before any handle - // is shaped, so nothing is orphaned on that path. + // Fails the launch here, loudly, rather than at the tool's first resolve. const credentialWiring = getStepCredentialWiring(env); const credentialCapabilities = credentialWiring === undefined ? new Map() : buildCredentialCapabilities(materialization.factories, credentialWiring); - // Wrap each loaded tool factory so its bundle's `dispose` (when - // present) is captured. Dedupe by closure identity: a factory whose - // bundle returns the same `dispose` on every invocation must not be - // torn down once per push. Each package's credentials capability joins - // the same teardown set so its shaped handles are released with the agent. + // Set, not array: dedupes by closure identity so a bundle whose dispose + // is the same on every invocation isn't torn down once per push. const capturedDisposers = new Set<() => unknown>(); for (const capability of credentialCapabilities.values()) { capturedDisposers.add(() => capability.dispose()); @@ -497,14 +316,9 @@ export function createToolBearingAgentFactory(): ( ), ); - // Run every captured disposer -- each credentials capability and each tool - // bundle -- guarding each so one failure does not strand the rest. Used on - // the success teardown AND on the construction-failure rollbacks below: the - // credentials capabilities are built before the plugin chain, so a plugin - // or agent build that throws must still release them (an http handle holds - // nothing, but a future key-file/socket handle would leak otherwise). - // Bundle disposers are idempotent, so re-running one `createAgent` already - // disposed on its own failure path is safe. + // Shared between success teardown and the construction-failure rollbacks + // below, since credentials capabilities are built before the plugin + // chain and must still be released if a later step throws. const runCapturedDisposers = async (): Promise => { const failures: unknown[] = []; for (const dispose of capturedDisposers) { @@ -518,13 +332,9 @@ export function createToolBearingAgentFactory(): ( return failures; }; - // Rebuild the def with the materialized tool factories. The - // serialized `def.toolFactories` carry only `{ id, requires }` - // metadata (the workflow projection strips closures on the wire), - // so the runnable factories come from materialization, not the - // incoming def. `defineAgent` owns the contravariance escape for - // the `BaseEnv`-typed loader factories (see its `EnvRequiredByAll` - // machinery). + // The serialized def.toolFactories carry only { id, requires } metadata + // (the workflow projection strips closures on the wire), so the runnable + // factories come from materialization, not the incoming def. const toolDef = defineAgent({ id: def.id, systemPrompt: def.systemPrompt, @@ -536,16 +346,9 @@ export function createToolBearingAgentFactory(): ( ...(def.tags !== undefined ? { tags: def.tags } : {}), }); - // Instantiate plugin factories one at a time so each successive - // factory sees the prior plugins' instances on `env.plugins`: - // posix's bundle reads `env.plugins` and threads ToolPlugin-shaped - // values into `createPosixTools`; the LSP plugin factory is what - // populates them. - // - // On a midway factory throw, every plugin instance already - // constructed releases what it acquired (the LSP plugin starts a - // subprocess) before the construction error propagates, so a - // partial-success chain never leaks an LSP subprocess. + // One at a time so each factory sees prior plugins' instances on + // env.plugins (posix reads them; LSP populates them). On a midway + // throw, every constructed instance is released so nothing leaks. const pluginInstances: unknown[] = []; let chainEnv: BaseEnv = env; try { @@ -558,9 +361,6 @@ export function createToolBearingAgentFactory(): ( }; } } catch (err) { - // Release the credentials capabilities (built above, before the plugin - // chain) and any bundle disposers captured so far, then the plugin - // instances this module owns. await runCapturedDisposers(); await disposeAll(pluginInstances, "plugin construction rollback"); throw err; @@ -570,26 +370,16 @@ export function createToolBearingAgentFactory(): ( try { agent = await createAgent(toolDef, chainEnv); } catch (err) { - // `createAgent` disposes the tool bundles it constructed on its - // own failure path, but the credentials capabilities and the plugin - // instances are this module's to own -- tear them down so a failed - // agent build does not leak a shaped handle or the LSP subprocess. + // createAgent disposes its own tool bundles on failure, but the + // credentials capabilities and plugin instances are this module's to own. await runCapturedDisposers(); await disposeAll(pluginInstances, "agent construction failure"); throw err; } return wrapAgentClose(agent, async () => { - // Captured disposers first (each credentials capability, then the tool - // bundles -- posix's bundle dispose chains through to the LSP plugin's - // `dispose`), then the plugin instances directly. Disposing the LSP - // plugin twice is safe: `lsp.dispose()` clears its client set and the - // posix bundle's dispose is idempotent. Running both guarantees the LSP - // subprocess is torn down even for a plugin no tool bundle consumed. - // Both loops run every disposer and collect failures rather than - // throwing mid-loop, so one failing disposer never strands the rest. - // A leaked or failing LSP subprocess must surface, not be swallowed: - // any collected failure fails the close, so the caller sees it. + // Disposing the LSP plugin twice is safe (idempotent); running both + // guarantees teardown even for a plugin no tool bundle consumed. const failures = [ ...(await runCapturedDisposers()), ...(await disposeAll(pluginInstances, "step teardown")), @@ -604,14 +394,7 @@ export function createToolBearingAgentFactory(): ( }; } -/** - * Return an `Agent` whose `close()` runs the original close and then - * the supplied teardown. The teardown runs AFTER the agent's own close - * so the reactor has stopped issuing tool calls before the tool/plugin - * resources are released. `close()` is idempotent at the agent layer; - * this wrapper guards its own teardown so a double `close()` does not - * double-dispose. - */ +/** Teardown runs after the agent's own close so the reactor has stopped issuing tool calls first. */ function wrapAgentClose(agent: Agent, teardown: () => Promise): Agent { let tornDown = false; return { @@ -650,12 +433,7 @@ async function disposeAll(instances: readonly unknown[], context: string): Promi return failures; } -/** - * Extract a callable `dispose` from a plugin instance whose static type - * is `unknown` (plugin factories return host-defined shapes the agent - * runtime does not interpret). Returns a bound disposer or `undefined` - * when the instance carries no `dispose` function. - */ +/** Plugin instance type is unknown (host-defined shapes the runtime doesn't interpret). */ function pluginDispose(value: unknown): (() => unknown) | undefined { if (value === null || typeof value !== "object") return undefined; if (!("dispose" in value)) return undefined; From 2523a82cc9d26e2ea1e74b0d4ba15ff056f2ac6b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:13:40 -0700 Subject: [PATCH 06/31] chore(sidecar): comments say why, never what (part 5) --- apps/sidecar/src/workflow-closure-apply.ts | 105 ++++----------------- 1 file changed, 20 insertions(+), 85 deletions(-) diff --git a/apps/sidecar/src/workflow-closure-apply.ts b/apps/sidecar/src/workflow-closure-apply.ts index fdf6e91a14..a64d97fe6a 100644 --- a/apps/sidecar/src/workflow-closure-apply.ts +++ b/apps/sidecar/src/workflow-closure-apply.ts @@ -1,29 +1,10 @@ -// Deploy-side application of a code-sourced workflow's frozen closure. -// -// When a deploy frame carries a `source` (the npm registry the workflow -// definition package is published to) plus the hub's frozen dependency -// `closure` (concrete versions + integrity SRIs), the sidecar materializes -// EXACTLY that closure and evaluates the pinned code to a validated -// `WorkflowDefinition` -- rather than trusting an inline serialized -// projection. The closure is applied byte-for-byte as the hub froze it; the -// sidecar never re-resolves the pin against the registry at apply time. -// -// This is the DURABLE deploy counterpart to the airlocked install-time probe: -// it reuses the same `@intx/tool-packaging` apply machinery -// (`createTarballCache` / `createToolLoader` / `applyAtomic`) that -// `tool-materialization.ts` uses for a step's tool closure, so the fetch + -// SRI-verify + extract + `node_modules` layout is not reimplemented here. -// -// A workflow-definition package declares `interchange.workflow` (the module -// whose evaluation produces the definition), NOT `interchange.tools`. -// `applyAtomic`'s load phase imports each TOP-LEVEL package's -// `interchange.tools` entry and rejects a package that has none -// (`package.entry.missing`), so the layout manifest handed to `applyAtomic` -// carries an EMPTY `topLevel`: every entry is still materialized and laid out -// (the dependency layout walks `entries`, and each dependency resolves against -// the frozen closure), but no tool factory is imported. The workflow entry -// itself is imported by `loadWorkflowDefinitionFromClosure` -- the correct load -// site for a workflow definition -- against the materialized package directory. +// The durable deploy counterpart to the airlocked install-time probe: the +// closure is applied byte-for-byte as the hub froze it, never re-resolved +// against the registry. topLevel is emptied for applyAtomic since a +// workflow-definition package declares interchange.workflow, not +// interchange.tools, which applyAtomic's load phase would otherwise reject +// as package.entry.missing; loadWorkflowDefinitionFromClosure imports the +// workflow entry instead. import path from "node:path"; @@ -47,15 +28,9 @@ const logger = getLogger(["sidecar", "workflow-closure-apply"]); export interface ApplyFrozenWorkflowClosureArgs { /** Names the registry the workflow definition package is published to. */ readonly source: WorkflowDefinitionSource; - /** - * The hub's frozen dependency closure for the definition's pin: concrete - * versions and integrity SRIs. Applied byte-for-byte; never re-resolved. - */ + /** Concrete versions and integrity SRIs; applied byte-for-byte, never re-resolved. */ readonly closure: ToolPackageManifest; - /** - * Durable per-deployment directory the closure is staged under - * (`/packages//store/...`). - */ + /** Durable per-deployment directory the closure is staged under. */ readonly instanceDir: string; /** Content-addressable tarball cache root shared across applies. */ readonly cacheRoot: string; @@ -65,30 +40,15 @@ export interface ApplyFrozenWorkflowClosureArgs { readonly registryMaxTarballBytes: number; /** Registry identifier -> URL + credentials the loader resolves entries against. */ readonly registries: ReadonlyMap; - /** - * Workspace root `kind: "asset"` closure entries mount against. A - * registry-sourced workflow definition closure carries no asset entries, so - * this defaults to `/workspace`. - */ + /** Defaults to /workspace; a registry-sourced closure carries no asset entries. */ readonly assetRoot?: string; - /** `assetId` -> mount path for tarball `asset` entries; empty by default. */ + /** assetId -> mount path for tarball asset entries; empty by default. */ readonly assetMounts?: ReadonlyMap; - /** - * `assetId` -> absolute indexed git directory for source-format `asset` - * entries. A registry- or tarball-sourced closure carries no source - * entries, so this defaults to an empty map. - */ + /** assetId -> indexed git directory for source-format entries; empty by default. */ readonly gitDirs?: ReadonlyMap; - /** - * Test seam for tarball fetching, forwarded to `createToolLoader`. - * Production omits it and the loader fetches from the configured registry. - */ + /** Test seam; production fetches from the configured registry. */ readonly fetchTarball?: TarballFetcher; - /** - * Test seam for the workflow entry's dynamic import, forwarded to - * `loadWorkflowDefinitionFromClosure`. Production omits it and the loader - * imports the materialized entry natively. - */ + /** Test seam; production imports the materialized entry natively. */ readonly importModule?: (importUrl: string) => Promise; } @@ -101,20 +61,7 @@ export interface AppliedWorkflowClosure { readonly deployDir: string; } -/** - * Materialize a workflow definition's frozen closure durably and load the - * pinned code to a validated `WorkflowDefinition`. - * - * The closure's single top-level pin IS the workflow definition package: the - * hub resolved the closure for exactly that pin. The frozen `entries` are - * applied verbatim (concrete versions + SRIs), so no registry re-resolution - * happens at apply time. - * - * @throws if the closure does not carry exactly one top-level pin, the source - * registry is not configured on this sidecar, the apply fails (integrity - * mismatch, fetch failure, extract failure, ...), or the pinned code does not - * evaluate to exactly one valid `WorkflowDefinition`. - */ +/** The frozen entries are applied verbatim; no registry re-resolution happens at apply time. */ export async function applyFrozenWorkflowClosure( args: ApplyFrozenWorkflowClosureArgs, ): Promise { @@ -130,14 +77,7 @@ export async function applyFrozenWorkflowClosure( ); } - // Boundary check on the definition's source. The `registry` arm surfaces a - // missing source registry loudly before any I/O (the per-entry registry - // gates fire again inside the loader). An `asset` closure materializes its - // entries from the durable stores the caller populated: tarball entries from - // `assetMounts`, source entries from `gitDirs`; the loader fails loud - // (`asset.mount.missing` / `git.materialization.failed`) if either is absent. - // The `never` default makes a future source kind a compile error rather than - // a silent fallthrough. + // The never default makes a future source kind a compile error, not a silent fallthrough. switch (args.source.kind) { case "registry": if (!args.registries.has(args.source.registry)) { @@ -168,9 +108,6 @@ export async function applyFrozenWorkflowClosure( ...(args.fetchTarball !== undefined ? { fetchTarball: args.fetchTarball } : {}), }); - // Apply EXACTLY the frozen entries. `topLevel` is emptied so `applyAtomic` - // imports no `interchange.tools` module (a workflow-definition package has - // none); the full `entries` set is still materialized and laid out. const layoutManifest: ToolPackageManifest = { schemaVersion: args.closure.schemaVersion, topLevel: [], @@ -185,8 +122,7 @@ export async function applyFrozenWorkflowClosure( assetMounts: args.assetMounts ?? new Map(), gitDirs: args.gitDirs ?? new Map(), attemptId: crypto.randomUUID(), - // This apply stands alone per deployment: there is no prior deploy under - // `instanceDir` to retain, so the sentinel disables the retention window. + // No prior deploy under instanceDir to retain, so this disables the retention window. previousDeployId: "none", newDeployId: crypto.randomUUID(), }); @@ -202,10 +138,9 @@ export async function applyFrozenWorkflowClosure( workflowPin.version, ); - // The workflow package's own integrity is the natural ESM-cache-bust token: - // Node keys its module cache by resolved URL, so a re-apply of changed bytes - // under the same name@version reimports rather than resolving to the prior - // instance. + // The package's own integrity is the cache-bust token: a re-apply of + // changed bytes under the same name@version reimports rather than + // resolving to the prior module-cache instance. const workflowEntry = args.closure.entries.find( (entry) => entry.name === workflowPin.name && entry.version === workflowPin.version, ); From 54c4caac69b08cdbffd189485ce87d33536bb8c9 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:18:44 -0700 Subject: [PATCH 07/31] chore(sidecar): comments say why, never what (part 6) --- apps/sidecar/src/tool-materialization.ts | 17 +- .../src/workflow-closure-materialization.ts | 9 +- apps/sidecar/src/workflow-run-pack-client.ts | 12 +- .../sidecar/src/workflow-substrate-factory.ts | 847 +++--------------- 4 files changed, 145 insertions(+), 740 deletions(-) diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index 978da03b2a..c035cba422 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -207,19 +207,10 @@ export async function materializeToolPackages(args: { } /** - * Write the active deploy id to `activeIdFile` and fsync both the - * file's own data/metadata and the parent directory entry. POSIX does - * not guarantee a parent-directory entry is durably linked from a - * file's own fsync alone, so the dir handle is opened and synced - * separately. Without that, a crash between a deploy's commit and the - * next boot could leave the staged deploy directory present while - * active-deploy-id is not yet visible — the next apply would then read - * previousDeployId="none" and treat the committed deploy as belonging - * to a fresh, deploy-less instance. - * - * Dir-fsync is best-effort: some filesystems (FAT/exFAT, some network - * mounts) don't support it and surface EINVAL/ENOTSUP, which shouldn't - * force a restart when the deploy-id file's own fsync already landed. + * POSIX doesn't guarantee a parent-directory entry is durably linked from a + * file's own fsync alone, so the dir handle is synced separately. Dir-fsync + * itself is best-effort: some filesystems reject it (FAT/exFAT), which + * shouldn't force a restart when the file's own fsync already landed. */ const ACTIVE_DEPLOY_ID_VERSION = "v1"; const ACTIVE_DEPLOY_ID_PREFIX = `${ACTIVE_DEPLOY_ID_VERSION}:`; diff --git a/apps/sidecar/src/workflow-closure-materialization.ts b/apps/sidecar/src/workflow-closure-materialization.ts index 599dbc8962..b2f6cee09f 100644 --- a/apps/sidecar/src/workflow-closure-materialization.ts +++ b/apps/sidecar/src/workflow-closure-materialization.ts @@ -1,11 +1,8 @@ // Materializes a probe's frozen closure on the sidecar host (I/O only, no // author-code eval) before the airlocked child evaluates it. Lives in -// apps/sidecar so @intx/workflow-host stays free of a @intx/tool-packaging -// dependency; the portable packages only see the injected -// MaterializeWorkflowClosure seam. Phases 1-2 only (no applyAtomic): a -// probe is ephemeral, so the durable-deploy bookkeeping doesn't apply. -// loadManifest runs with an emptied topLevel so it lays out every closure -// entry without importing (evaluating) any of them. +// apps/sidecar so @intx/workflow-host stays free of @intx/tool-packaging. +// loadManifest runs with an emptied topLevel so it lays out every entry +// without importing (evaluating) any of them. import { promises as fs } from "node:fs"; import path from "node:path"; diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index 4722aaff05..1281994974 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -356,14 +356,10 @@ export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { } /** - * Wraps RepoStore so a successful writeTreePreservingPrefix against a - * workflow-run repo schedules a coalesced pack push: at most one push per - * (repoId, ref) in flight, with writes arriving mid-push marking the slot - * dirty for one more run rather than queuing N pushes. The shipped-tip - * cursor advances only after the hub's ack, so a reconnect-cancelled - * transfer is re-shipped whole on the next createPack rather than leaving - * a gap. A failed push latches on the slot and re-throws on the next write - * to that (repoId, ref) instead of being swallowed. + * Wraps RepoStore so a workflow-run write schedules a coalesced pack push: + * at most one push per (repoId, ref) in flight, with mid-push writes + * marking the slot dirty for one more run. The cursor advances only after + * the hub's ack, and a failed push latches and re-throws on the next write. */ export type WorkflowRunPackPushingRepoStoreOpts = { underlying: RepoStore; diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index e0f41981eb..128cd79d3f 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -1,26 +1,9 @@ -// Substrate factory the sidecar's `bin/workflow-child` hands to -// `runWorkflowChildFromProcessEnv`. The factory closes over the -// production substrate (`createAgentRepoStore`-backed `RepoStore`), -// the host-process scheduler singleton (adapted to the runtime's -// `Scheduler` shape), and the sidecar's grant-rule evaluator. -// -// The factory consumes the workflow-host's typed `SubstrateFactoryEnv` -// -- the parsed `SpawnTimeEnv` plus a narrow `substrateConfig` -// record carrying only the keys the binary listed in -// `RunWorkflowChildFromProcessEnvOpts.substrateConfigKeys`. The -// factory does not read `process.env` itself; the binary owns the -// only crossing of that boundary. -// -// Single-writer architecture: the workflow-run repo's ref has exactly -// one writer at a time -- the supervisor. The child opens a bare -// `createAgentRepoStore` against the shared on-disk data dir for -// read-only operations (`getRepoDir`, `subscribe`, `resolveRef`, -// etc.) and exposes a proxy `RepoStore` whose -// `writeTreePreservingPrefix` forwards every write over the control -// IPC into the supervisor's substrate. The supervisor's substrate is -// wrapped with the boot-edge pack-push facade, so the hub push fires -// as part of the supervisor's normal write path -- the child does -// not open its own pack-push pipeline. +// Single-writer architecture: the workflow-run repo's ref has exactly one +// writer, the supervisor. The child's bare RepoStore is read-only; its +// proxy RepoStore forwards writeTreePreservingPrefix over control IPC into +// the supervisor's substrate, which is wrapped with the boot-edge +// pack-push facade, so the child never opens its own pack-push pipeline. +// The factory does not read process.env itself; the binary owns that boundary. import fs from "node:fs"; import path from "node:path"; @@ -148,28 +131,7 @@ import { type DurableConversationRegistry, } from "./conversation-state"; -// The child does not construct a workflow-run pack-push pipeline of -// its own. The supervisor owns the workflow-run repo's write -// contract; the supervisor's substrate is wrapped at the sidecar's -// boot edge with the pack-pushing facade so any successful workflow- -// run write fires the hub push automatically. The child's proxy -// `RepoStore` forwards `writeTreePreservingPrefix` over IPC into the -// supervisor's wrapped substrate. - -/** - * Required substrate-config keys the sidecar's binary forwards into - * the factory's `substrateConfig` slot. Listed here so the binary - * passes the same names to the helper; the helper enforces - * presence-and-non-empty against this allowlist before the factory - * runs. - * - * `HUB_WS_URL`, `SIDECAR_ID`, and `SIDECAR_TOKEN` carry the - * hub-connection trust anchors the child needs to ship workflow-run - * pack pushes back to the hub. The sidecar's deploy router populates - * these via the supervisor's `substrateEnv` plumbing - * (`multistepSubstrateEnv` on `createSidecarDeployRouter`), threaded - * from the boot edge's own env reads. - */ +/** Enforced presence-and-non-empty before the factory runs; HUB_WS_URL/SIDECAR_ID/SIDECAR_TOKEN are the hub trust anchors. */ export const SIDECAR_SUBSTRATE_CONFIG_KEYS = [ "SIDECAR_DATA_DIR", "WORKFLOW_RUN_REPO_ID", @@ -196,37 +158,17 @@ const SubstrateConfig = type({ SIDECAR_ID: "string > 0", SIDECAR_TOKEN: "string > 0", STEP_INFERENCE_SOURCES: "string > 0", - // JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` of every spawned - // body's plaintext inference sources, decrypted sidecar-side from the run - // record. Always serialized by the deploy router (at least "{}" when the - // deployment spawns no bodies), so a missing key child-side is a serialization - // bug and must fail loud here. + // Always serialized by the deploy router (at least "{}"), so a missing key + // child-side is a serialization bug and must fail loud here. WORKFLOW_BODY_SOURCES: "string > 0", - // Per-step tool-loader caps. The supervisor threads the boot edge's - // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` - // through `substrateEnv` so the child's per-step tool materialization is - // bounded by the sidecar's boot-edge-resolved caps. Validated as - // positive-finite-number strings at this boundary. SIDECAR_CACHE_MAX_BYTES: "string > 0", SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", - // JSON-encoded custom inference adapter manifest. Required: the boot - // edge always serializes it into `substrateEnv` (defaulting to "[]" - // when no custom adapters are configured), so a missing key child-side - // is a serialization bug and must fail loud here, exactly like the - // byte-cap fields. Validated as a non-empty string at this boundary; - // its JSON shape is re-validated against `AdapterManifest` in - // `parseAdapterManifest` before any module is imported. + // Required like the byte-cap fields; re-validated against AdapterManifest + // in parseAdapterManifest before any module is imported. SIDECAR_ADAPTER_MANIFEST: "string > 0", }).onUndeclaredKey("ignore"); -/** - * Parse a substrate-config cap entry (`SIDECAR_CACHE_MAX_BYTES` / - * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) into a positive finite number. - * The boot edge already validated these via the `config.ts` readers - * before serializing them into `substrateEnv`; this re-parse at the - * child boundary keeps the typed-config contract honest rather than - * trusting the wire blindly. - */ +/** Re-parsed at the child boundary rather than trusting the wire, even though the boot edge already validated it. */ function parseByteCap(raw: string, name: string): number { const n = Number(raw); if (!Number.isFinite(n) || n <= 0) { @@ -237,30 +179,13 @@ function parseByteCap(raw: string, name: string): number { return n; } -/** - * Per-step inference-source table parsed from the spawn-time - * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes - * `frame.workflow.sources` (a `Record`) as - * JSON and threads it through the supervisor's `substrateEnv`; the - * factory parses and validates the table once at construction time and - * seeds it into the run loop's mutable sources reference, which each - * `buildEnv` reads. Each value is the step's ordered - * failover chain -- element 0 is the active source, the tail are the - * reactor's forward-only failover targets -- so the list is non-empty. - */ +/** Each value is the step's ordered failover chain: element 0 active, the tail forward-only targets. */ const StepInferenceSourceTable = type({ "[string]": InferenceSource.array().atLeastLength(1), }); type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; -/** - * Parse and validate the JSON-encoded `STEP_INFERENCE_SOURCES` entry - * the supervisor threaded through `substrateEnv`. A malformed JSON - * payload, a non-object root, or a value that does not match - * `Record` is rejected at the boundary with - * a structured error rather than being deferred to a deep-stack - * `buildEnv` failure. - */ +/** Rejected at the boundary with a structured error rather than deferred to a deep-stack buildEnv failure. */ function parseStepInferenceSources(raw: string): StepInferenceSourceTable { let parsed: unknown; try { @@ -280,22 +205,13 @@ function parseStepInferenceSources(raw: string): StepInferenceSourceTable { return validated; } -/** - * Every spawned body's per-step inference-source table, keyed by the body's - * definition id. Parsed from the `WORKFLOW_BODY_SOURCES` env entry the deploy - * router serializes from the run record's decrypted body sources. Empty when the - * deployment spawns no bodies. - */ +/** Empty when the deployment spawns no bodies. */ const BodyInferenceSources = type({ "[string]": StepInferenceSourceTable, }); type BodyInferenceSources = typeof BodyInferenceSources.infer; -/** - * Parse and validate the JSON-encoded `WORKFLOW_BODY_SOURCES` entry. Mirrors - * `parseStepInferenceSources`: a malformed payload is rejected at the boundary - * rather than deep in a body spawn. - */ +/** Mirrors parseStepInferenceSources: rejected at the boundary rather than deep in a body spawn. */ function parseBodyInferenceSources(raw: string): BodyInferenceSources { let parsed: unknown; try { @@ -316,26 +232,10 @@ function parseBodyInferenceSources(raw: string): BodyInferenceSources { } /** - * Parse and validate the JSON-encoded `SIDECAR_ADAPTER_MANIFEST` entry - * the supervisor threaded through `substrateEnv` from the boot edge's - * `readAdapterManifest`. - * - * Trust boundary: the child's substrate config is operator-supplied - * (the supervisor's `Bun.spawn` env), so this re-validation is - * defense-in-depth at the deserialization boundary, NOT a trust - * upgrade. The manifest was already trusted operator config on the - * parent side; the same channel already carries the sidecar's signing - * private key, so it is not a lower-trust surface. Re-asserting the - * shape here keeps the typed-config contract honest rather than - * importing modules off an unvalidated wire value. - * - * Host contract for custom adapters: a manifest `specifier` must - * resolve from BOTH the sidecar's and this child's module-resolution - * roots (the child is a separate `bun` process spawned by the - * supervisor), and an adapter module MUST be import-side-effect-free — - * it is imported once per process by `loadAdapterRegistry`, and any - * top-level side effect would run independently in the parent and in - * every child. + * Defense-in-depth re-validation at the deserialization boundary, not a + * trust upgrade. A manifest specifier must resolve from both the sidecar's + * and the child's module-resolution roots, and an adapter module must be + * import-side-effect-free since it's imported once per process. */ export function parseAdapterManifest(raw: string): AdapterManifest { let parsed: unknown; @@ -356,23 +256,7 @@ export function parseAdapterManifest(raw: string): AdapterManifest { return validated; } -/** - * Resolve the per-step inference-source failover chain from the table a - * build reads. The supervisor's multi-step branch only invokes - * a step whose `stepId` appears in `frame.workflow.sources`; a lookup - * miss here is a programmer error in the supervisor, not a wire-side - * failure, and the resolver surfaces it with the missing base step id - * named (plus the scoped invocation id, for a map iteration). The - * returned list is the step's ordered chain (element 0 the active - * source, the tail the reactor's failover targets); the table's arktype - * guarantees it is non-empty. - * - * A `map` iteration runs under a scoped step id `[]`, but - * deploy pins one source per base step, so the scoped id is resolved to - * its base before the lookup -- every iteration shares the base step's - * pinned source. `baseStepId` is the identity on an unscoped id, so a - * plain step is unaffected. - */ +/** A lookup miss here is a supervisor programmer error, not a wire-side failure. A scoped map-iteration id resolves to its base first. */ function createStepInferenceSourceResolver( table: StepInferenceSourceTable, ): (stepId: string) => InferenceSource[] { @@ -405,38 +289,16 @@ function hexDecode(hex: string, name: string): Uint8Array { return out; } -/** - * Dependency overrides accepted by `createSidecarSubstrateFactory`. - * Production callers omit these to get the default-disk-backed bare - * store and the IPC-bridge-backed substrate proxy; tests inject an - * in-memory bare store and/or an explicit substrate-write bridge. - */ +/** Production omits these for the default disk-backed store; tests inject an in-memory stub. */ interface SidecarSubstrateFactoryDeps { - /** - * Override the bare-store constructor. Production callers omit this - * to get the `createAgentRepoStore`-backed `RepoStore` against - * `SIDECAR_DATA_DIR`; tests inject an in-memory recording stub. - * - * The bare store backs the child's read-only operations - * (`getRepoDir`, `subscribe`, `resolveRef`, `listRefs`, - * `resolveHead`, `createPack`). The child's workflow-run writes do - * NOT flow through this store; the proxy `RepoStore` forwards them - * over IPC into the supervisor's substrate. - */ + /** Backs the child's read-only operations only; writes go through the proxy RepoStore over IPC instead. */ createBareRepoStore?: (config: { dataDir: string; signingKey: { publicKey: Uint8Array; privateKey: Uint8Array }; }) => RepoStore; } -/** - * `CommitSigner` the per-step isogit stores use to sign every commit. - * The factory's Ed25519 signing keypair (the same key the child's bare - * `RepoStore` carries) is bound into an `sshsig`-shaped signer so the - * per-step agent-state commits are attributable to the sidecar's - * substrate identity, matching the signing surface the production - * `RepoStore` uses for workflow-run writes. - */ +/** Binds the same Ed25519 keypair the bare RepoStore carries, matching the production signing surface. */ function createStepStorageSigner(signingKey: { publicKey: Uint8Array; privateKey: Uint8Array; @@ -446,34 +308,12 @@ function createStepStorageSigner(signingKey: { } /** - * Root directory for a single step invocation's agent-state storage and - * workspace, derived from the sidecar data dir and the run/step/attempt - * coordinates the workflow runtime owns. - * - * The per-step agent storage is a distinct isogit repo, deliberately - * rooted OUTSIDE the workflow-run repo's working tree. The workflow-run - * repo's single writer is the supervisor, and its working tree carries - * the run-event log under `runs//events/...`; nesting a second - * git repo inside that tree would collide with the event subtree and - * with the supervisor's write contract. Rooting the per-step store under - * a dedicated `workflow-step-state/` sibling subtree keyed by the - * workflow-run repo id keeps every step's storage isolated per run and - * per step while never touching the run-event tree. - * - * Resume-attempt invariant: a suspended agent step commits its pending-op - * + turns under the `attempt-N` directory where N is the step's attempt - * at suspend time. On crash-resume the runtime (`runStep` in - * `packages/workflow`) recovers that attempt from the reduced state's - * `currentAttempt` and re-invokes with it, so this function reopens the - * SAME `attempt-N` store and the reactor's `rehydrateGates` finds the - * pending-op the delivered decision correlates against. This is why the - * resume path recovers `currentAttempt` rather than assuming attempt 1: - * attempt 1 is correct only for a step that never retried before - * suspending; a retried-then-suspended step lives under `attempt-2`+, and - * reopening `attempt-1` would rehydrate an empty store and hang on a - * decision that correlates nothing. `createSidecarStepBuildEnv` asserts - * this invariant loudly on the cold path (a resume opening a store that - * lacks the resumed correlationId's pending-op throws). + * Deliberately rooted outside the workflow-run repo's working tree so it + * never collides with the run-event log under runs//events/. + * Resume-attempt invariant: a crash-resume must reopen the SAME attempt-N + * store the step suspended under (recovered via currentAttempt, not + * assumed to be 1), or the reactor rehydrates an empty gate and hangs. + * createSidecarStepBuildEnv asserts this loudly on the cold path. */ export function stepStorageRoot(args: { dataDir: string; @@ -494,15 +334,7 @@ export function stepStorageRoot(args: { ); } -/** - * Root directory for a single workflow-run subtree's per-step scratch: - * `/workflow-step-state//runs//`. The cold - * (multi-step) path's per-step `stepStorageRoot` nests under this, so - * reclaiming this subtree on run completion drops every step/attempt the - * run produced in one `rm -rf`. Kept distinct from `stepStorageRoot` so - * the deletion granularity (a whole run, not a single step/attempt) is - * expressed at the call site that owns run-completion cleanup. - */ +/** Reclaiming this subtree on run completion drops every step/attempt the run produced in one rm -rf. */ function runStepStorageRoot(args: { dataDir: string; workflowRunRepoId: RepoId; @@ -518,19 +350,9 @@ function runStepStorageRoot(args: { } /** - * Stable per-agent scratch root for the WARM single-step agent's - * workspace + tool materialization (tarball-cache + apply-state). Keyed - * by the step identity exactly like the durable conversation store's - * `agent-conversation-state///` (conversation-state.ts), - * NOT by the arbitrary first-message runId. Keying it stably is what - * bounds the warm case: the cached agent reuses ONE workspace across - * every message in the child's lifetime, and that same workspace is - * re-derived (and so survives) across a child respawn, instead of - * stranding a fresh per-runId subtree each time. The whole subtree is - * reclaimed on undeploy, when the deployment's supervisor + child are - * already torn down. Rooted under a `warm/` sibling of the cold `runs/` - * subtree so the undeploy sweep of `workflow-step-state//` - * reclaims both with one removal and the two keyings never collide. + * Keyed by step identity, not runId, so the cached warm agent reuses one + * workspace across every message and it survives a child respawn. + * Reclaimed on undeploy under a warm/ sibling of the cold runs/ subtree. */ function warmStepStorageRoot(args: { dataDir: string; @@ -562,19 +384,7 @@ function findApprovalSnapshot( return pendingOperations.find((op) => op.correlationId === correlationId)?.approvalSnapshot; } -/** - * Resolve the full RFC 5322 References chain for a reply whose parent is - * `inReplyTo`, from the deployment's committed substrate mailbox. Opens a - * fresh committed snapshot, locates the message whose Message-Id equals - * `inReplyTo`, and returns that message's own References followed by its - * Message-Id -- the complete ancestry a reply threaded onto it must carry. - * - * Returns `undefined` when the parent is not in the mailbox (the first reply - * on a fresh thread, or an id that never arrived). The message identifiers are - * carried through verbatim; the transport's send path (`buildReferences` in - * `@intx/mail-memory`) owns filtering non-RFC identifiers out of the emitted - * `References` header, so this reader does not pre-validate them. - */ +/** Returns undefined when the parent is not in the mailbox (fresh thread, or an id that never arrived). */ async function resolveMailboxReferences( reader: ChildMailboxReader, inReplyTo: string, @@ -585,20 +395,7 @@ async function resolveMailboxReferences( return [...parent.envelope.references, parent.envelope.messageId]; } -/** - * Read a cold (multi-step) parked step's durable pending operations from its - * on-disk per-attempt isogit store. The store is written at suspend and - * survives while the run is non-terminal -- a parked step keeps the run - * in-flight, so the run-completion reclamation (`cleanupRunStorage`) never - * fires against it. - * - * Returns an empty list when the store directory is absent rather than - * manufacturing an empty repo on the read path: `createIsogitStore` calls - * `initAgentRepo`, which would `mkdir` and init a fresh repo for a - * non-existent dir. The `directoryExists` guard keeps the read a read -- on an - * existing store `initAgentRepo` finds a repo and commits nothing, so no - * signer is needed (`load()` never signs). - */ +/** directoryExists guard keeps this a pure read: createIsogitStore would otherwise mkdir+init a fresh repo. */ export async function readColdParkedPendingOperations(args: { dataDir: string; workflowRunRepoId: RepoId; @@ -631,18 +428,9 @@ export async function readColdParkedApprovalSnapshot(args: { } /** - * Read a warm (single-step) parked agent's durable pending operations from - * substrate state. A warm agent's pending operations live in its durable - * conversation store, mirrored to the workflow-run substrate under - * `agent-state//`. - * - * Reconstructs that state read-only -- deliberately NOT through - * `DurableConversationRegistry.acquire`, whose first acquire writes and - * commits a substrate restore into the live store and would front-run the warm - * agent's own restore ordering. A respawned child has not rebuilt the live - * store when re-registration runs (resume re-parks without re-invoking the - * step), so the substrate is the only place the pending operations live at that - * moment. Returns an empty list when no durable state exists for the agent. + * Deliberately not through DurableConversationRegistry.acquire, whose first + * acquire would write a substrate restore and front-run the warm agent's own + * restore ordering. */ export async function readWarmParkedPendingOperations(args: { substrate: RepoStore; @@ -668,13 +456,7 @@ export async function readWarmParkedApprovalSnapshot(args: { return findApprovalSnapshot(await readWarmParkedPendingOperations(args), args.correlationId); } -/** - * Project a parked step's durable pending operations down to the minimal - * approval records the resume classifier needs. Filters to `approval` (the only - * control-plane kind today) and keeps only the correlationId and the optional - * epoch-ms deadline; the runtime reconstructs the lost `SignalAwaited` from - * those alone, and must not see the reactor's pending-operation internals. - */ +/** The runtime reconstructs the lost SignalAwaited from correlationId + deadline alone; it must not see the reactor's internals. */ export function toParkedApprovalOps(pendingOperations: PendingOperation[]): ParkedApprovalOp[] { return pendingOperations .filter((op) => op.kind === "approval") @@ -688,139 +470,39 @@ export interface SidecarStepBuildEnvDeps { dataDir: string; workflowRunRepoId: RepoId; signer: (payload: string) => Promise; - /** - * Deployment mailbox address the supervisor threaded into the child - * (`MAILBOX_ADDRESS`). Used to locate each step's on-disk deploy tree - * for tool materialization (see `stepDeployTreeDir`) AND as the step - * agent's outbound mail `address`: the supervisor signs the agent's - * outbound mail as this address through the host transport (OUTBOUND - * half of mailbox ownership, §3a). For the single-agent deploy this is - * the run address the host registered the agent's `CryptoProvider` - * against. - */ + /** Locates each step's deploy tree and doubles as the step agent's outbound mail address. */ mailboxAddress: string; - /** - * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), - * threaded from the host through the spawn-time env. Selects the - * head/step collapse when locating a step's deploy tree - * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step - * deployment reads at the head, a multi-step deployment at the per-step - * address, matching the host's producer push. - */ + /** Selects the head/step collapse: single-step reads at the head, multi-step at the per-step address. */ stepCount: number; - /** - * Child-side outbound-mail bridge over the upstream control channel - * (OUTBOUND half of mailbox ownership, §3a). The per-step env builder - * wraps it in a supervisor-backed `MessageTransport` it supplies as - * the step agent's `env.transport`; the agent's mail tools call - * `transport.send`, which routes through the bridge to the supervisor - * for the actual signed send. The step agent never holds the signing - * key. - */ + /** Wrapped in a supervisor-backed transport; the step agent never holds the signing key. */ outboundMailBridge: ChildOutboundMailBridge; - /** - * Inbound local IMAP surface for the step agent's supervisor-backed - * transport (INBOUND half of mailbox ownership, §3b). Bundles the child - * mailbox reader (a fresh committed snapshot of the deployment's substrate - * `INBOX` per read), the shared mailbox watch registry (the same instance the - * child's control loop fires `mailbox.notify` into), and the sender-key - * resolver `fetchFull` verifies signatures against. When present, the warm - * agent's `mail_read` / `mail_search` / `mail_wait` resolve against the - * committed mailbox rather than throwing "not wired". Absent for a build that - * owns no inbound mailbox (a spawned child or an onTrigger body), whose - * transport inbound stays inert. - */ + /** Absent for a build with no inbound mailbox (a spawned child or onTrigger body), whose inbound stays inert. */ inbound?: SupervisorBackedTransportInbound; /** Per-step tool-loader caps (cache + registry tarball size). */ cache: StepToolCacheConfig; - /** - * Adapter registry the step agent resolves inference adapters through. - * The child builds this eagerly at boot from the validated - * `SIDECAR_ADAPTER_MANIFEST` (built-ins merged with operator custom - * adapters) and the env builder sets it on `env.deps`, so a step whose - * source names a custom provider resolves in the child exactly as it - * does on the sidecar main path. Without it the step agent would fall - * back to `createAgent`'s built-ins-only default and a custom-provider - * source would fail to resolve at run time. - */ + /** Built eagerly at boot so a custom-provider step source resolves the same way as the sidecar main path. */ adapters: AdapterRegistry; - /** - * Durable-conversation registry for the warm single-step agent - * (design §3c). When present, the env builder swaps the per-run isogit - * `ContextStore` for a per-agent durable store whose conversation is - * mirrored to the workflow-run substrate, and restores the prior - * conversation from the substrate before returning the env (so the - * agent's reactor `load()` and the warm cache's lazy build see the - * restored turns -- including the respawn-rebuild path). Absent for a - * multi-step deploy, whose per-step agents are not warm/long-lived and - * need no cross-run conversation durability. - */ + /** Present only for a warm single-step agent; a multi-step deploy's per-step agents need no cross-run durability. */ durableConversation?: DurableConversationRegistry; /** - * Record the tool-mark floor grants derived from the step's - * materialized tool factories, keyed by the step's base id. The env - * builder is the only place the child holds the loaded factories' - * static `definitions` (name + approval mark), so it derives the floor - * here; the grant evaluator (a sibling closure in the substrate - * factory) reads the recorded floor by base step id and merges it under - * the credentials snapshot's grants at authorization time. Keyed by - * base id (matching the credentials snapshot and inference-source - * table) so a `map` iteration's scoped id shares its base step's floor. - * The recording persists across a warm agent's messages: the env - * builder runs once (first build) but the floor it records must be - * available for every later tool call the warm agent makes. + * Recorded by base step id so the grant evaluator can authorize a pinned + * tool (invisible to the hub's capability walk) against its own static + * mark; persists across a warm agent's messages since the env builder + * runs once. */ recordToolMarkFloor: (baseStepId: string, grants: GrantRule[]) => void; /** - * Feed the step agent's OWN evaluated tool factories into the - * materialization slot, instead of reading a pinned tool-package manifest - * off the deploy tree. Set for the source-ref lineage, whose child holds - * the live `WorkflowDefinition` (re-verified against the approved wire hash - * at run start) and threads each step's live `agent.toolFactories` in as - * `req.agent.toolFactories`. Those are the same `AnnotatedToolFactory` - * callables `materializeStepTools` would produce, bare-named (no - * namespacing loader runs), so the existing tool-bearing agent factory - * consumes them unchanged. - * - * The source-ref lineage stages no `tool-packages-manifest.json`, so - * `materializeStepTools` would find nothing and return empty tools; this - * arm is what actually runs a source workflow's tools. No tool-mark floor - * is recorded on this lineage: a source tool's runtime name is the bare - * `definition.name` the probe's capability walk already emitted a - * `tool:` grant for into the frozen credentials snapshot, so the - * snapshot authorizes it directly (unlike a pinned tool, whose namespaced - * name the walk never saw and which the floor exists to compensate for). + * Set for the source-ref lineage: feeds the step agent's own live + * toolFactories in directly, since that lineage stages no + * tool-packages-manifest.json. No floor is recorded here since the + * probe's capability walk already granted the bare tool name. */ sourceTools: boolean; - /** - * Sidecar-local directory of the materialized workflow-definition closure - * (`env.spawn.closurePackageDir`), present only on the source-ref lineage. - * The source arm materializes each step agent's declared plugin packages - * (`req.agent.plugins`) from this already-laid-out closure -- resolving them - * through the workflow package's `node_modules/` graph, no re-download -- and - * feeds the resulting plugin factories into the per-step plugin chain. A - * plugin package contributes no agent-visible tool factory, so this closure - * load is the only channel that reaches an `env.plugins` a source workflow's - * posix bundle consumes. Undefined on a build that materializes pinned tool - * packages instead, which reads no closure. - */ + /** Present only on the source-ref lineage, to materialize its declared plugin packages with no re-download. */ closurePackageDir?: string; } -/** - * Materialize the plugin factories a source-ref step agent declares - * (`req.agent.plugins`) from the workflow's frozen closure. Returns an - * empty list when the agent declares no plugins. Fails closed when plugins - * are declared but the source-ref closure dir is absent -- that pairing is a - * wiring bug (the source arm always carries `closurePackageDir`), and - * silently dropping the plugins would run the workflow without the tools it - * declared and leave the reactor to fail closed on the un-loaded tool. - * - * The closure is already laid out on disk from the deploy-side apply, so - * this is a load-only pass -- no re-download -- resolving each plugin - * package through the workflow package's `node_modules/` graph and - * collecting its `definePlugin` factories. - */ +/** Fails closed when plugins are declared but closurePackageDir is absent — that pairing is a wiring bug. */ async function materializeSourcePluginFactories( deps: SidecarStepBuildEnvDeps, req: StepInvokeRequest, @@ -840,38 +522,11 @@ async function materializeSourcePluginFactories( }); } -/** - * Build the step-invoker `buildEnv` callback the workflow-host's - * adapter consumes. Pulled out of `createSidecarSubstrateFactory` so - * the per-step env construction is observable without standing up the - * full substrate. - * - * The closure reads the per-step source table from the mutable - * reference passed per build, derives the `stepId` / `runId` / `attempt` - * from the runtime's `AuthorizeContext`, resolves the per-step - * `InferenceSource` from that table, and stands up - * a per-step isogit `ContextStore` (also serving as the audit store) - * plus a per-step workspace directory rooted under the run. A - * construction failure (mkdir, isogit init) surfaces here rather than - * being papered over with a stub: the single-step path now always runs - * a real agent against real storage. - */ -/** - * The per-run credential inputs a build resolves the step's `credentials` - * wiring from. The live material cell and the grants resolver ride in from the - * run child (per-run state); the provider registry is sidecar-static and - * combined in at the invoke-step boundary. Absent when no credential material - * was threaded, which leaves the step's inference reader and tool credentials - * unwired. - */ +/** Pulled out of createSidecarSubstrateFactory so per-step env construction is observable in isolation. */ +/** Absent when no credential material was threaded, leaving the step's inference reader and tool credentials unwired. */ interface SidecarStepCredentialContext { readonly materialCell: CredentialMaterialCell; - /** - * The step's grants, resolved live by base step id. Typed `unknown[]` at - * this boundary (the run child owns no grant grammar); the sidecar casts to - * `GrantRule[]` here where the grammar is known, exactly as - * `evaluateGrantsAdapter` does. - */ + /** Typed unknown[] at this boundary; the sidecar casts to GrantRule[] where the grammar is known. */ readonly resolveStepGrants: (stepId: string) => readonly unknown[]; readonly providers: CredentialProviderRegistry; } @@ -888,11 +543,8 @@ export function createSidecarStepBuildEnv( sourcesRef: SourcesSnapshotRef, credentialContext?: SidecarStepCredentialContext, ): Promise => { - // Resolve against the live table each build so a source rotation that - // wrote `sourcesRef.current` before this build is reflected in the - // agent this build constructs. A warm agent that is already built does - // not pass through here again, so a rotation does not reach it through - // this path -- this ref covers only a build that has not happened yet. + // Resolved live each build; a warm agent already built doesn't pass + // through here again, so this ref covers only builds not yet done. const resolveStepInferenceSource = createStepInferenceSourceResolver(sourcesRef.current); const { stepId, runId, attempt } = req.authzContext; if (stepId === undefined) { @@ -921,19 +573,9 @@ export function createSidecarStepBuildEnv( ); } - // Root the per-step scratch (workspace + tool tarball-cache + - // apply-state). The cold (multi-step) path keys it per - // run/step/attempt: each run rebuilds the agent and its scratch, and - // the run's whole `runs//` subtree is reclaimed on run - // completion. The warm single-step path (`durableConversation` - // present) keys it STABLY per agent so the cached agent reuses one - // workspace across every message -- bounding the warm case to one - // dir per agent and letting that workspace survive child respawn -- - // and the subtree is reclaimed on undeploy. The two keyings live - // under disjoint `runs/` and `warm/` sub-roots so neither sweep - // touches the other's tree, and the durable conversation under - // `agent-conversation-state/` is a different root that neither - // sweep touches. + // Cold path keys per run/step/attempt; warm path (durableConversation + // present) keys stably per agent. Disjoint runs/ and warm/ sub-roots so + // neither sweep touches the other's tree. const storeDir = deps.durableConversation !== undefined ? warmStepStorageRoot({ @@ -948,44 +590,20 @@ export function createSidecarStepBuildEnv( stepId, attempt, }); - // Conversation storage. For the warm single-step agent the - // conversation must survive child respawn, so it is backed by a - // per-agent durable store whose content is mirrored to the - // workflow-run substrate (design §3c); building it here restores the - // prior conversation before the agent's reactor loads. A multi-step - // deploy (no durable registry) keeps the per-run isogit store: its - // per-step agents are not warm/long-lived and have no cross-run - // conversation to carry. The workdir + tools stay per-run in both - // cases -- only the conversation context is durable across runs. + // The warm agent's conversation must survive child respawn, so it's + // backed by a per-agent durable store mirrored to the substrate; a + // multi-step deploy keeps the per-run isogit store instead. const storage: ContextStore & AuditStore = deps.durableConversation !== undefined ? (await deps.durableConversation.acquire(stepId)).storage : await createIsogitStore(storeDir, deps.signer); - // Cold-path resume keying assertion (correct-by-construction guard for - // the resume-attempt invariant documented on `stepStorageRoot`). An - // APPROVAL resume re-invocation delivers the correlated decision to the - // reactor, which rehydrates its gate from THIS store's pending - // operations. A re-dispatchable ask-rail approval gate carries a - // `suspendedCall` (the call the approval re-runs); an async-tool pending - // marker shares `kind: "approval"` but has no `suspendedCall`, and on - // resume the reactor clears its gate WITHOUT re-running the approved - // call. So the resume must find a `suspendedCall`-bearing op for its - // correlationId, mirroring the reactor's own discriminator. Two failure - // modes make that false: the runtime reopened the wrong `attempt`'s - // store (`stepStorageRoot` above), so no pending op matches at all and - // the reactor comes up gateless -- a silent forever-hang; or the only - // match is an async marker, so the gate clears and the approved call is - // silently skipped. Make either loud here, at the single seam that both - // opened the store AND knows an approval resume must find its gate, - // rather than letting it surface as a hang or a skipped call. The - // warm path keys its durable store per agent (not per attempt) and - // rehydrates from a different lifecycle, so - // this assertion is cold-path only. An `"input"` resume is exempt by - // construction: its correlationId names the runtime-minted re-arm channel - // awaiting the step's next trigger, not a reactor gate, so no pending - // operation ever exists for it and the delivered decision is sent as a - // plain next turn (no correlation to rehydrate). + // Cold-path-only guard for the resume-attempt invariant on + // stepStorageRoot: an approval resume must find a suspendedCall-bearing + // pending op for its correlationId, or the runtime reopened the wrong + // attempt's store and the reactor would hang (gateless) or silently + // skip the approved call (matched only an async marker). Loud here at + // the one seam that opened the store and knows the invariant. if ( deps.durableConversation === undefined && req.resume !== undefined && @@ -1006,35 +624,12 @@ export function createSidecarStepBuildEnv( const workdir = path.join(storeDir, "workspace"); await fs.promises.mkdir(workdir, { recursive: true }); - // Assemble the step's tool runtime. Two arms: - // - // - Source-ref lineage (`sourceTools`): run the step agent's OWN - // evaluated tool factories. The child holds the live re-verified - // `WorkflowDefinition`, and the runtime threads each step's live agent - // in as `req.agent`, so `req.agent.toolFactories` are the real callable - // `AnnotatedToolFactory`s -- the same shape `materializeStepTools` - // produces (`LoadedToolFactory === AnnotatedToolFactory`), - // bare-named because no namespacing loader runs. The source deploy - // stages no `tool-packages-manifest.json`, so `materializeStepTools` - // would find nothing here; feeding the evaluated factories straight - // into the slot is what runs a source workflow's tools. `packageName` - // is the factory's bundle `id` (the credential Gate-2 key; a mail/plain - // tool declares no credentials, so it stays inert). Plugin factories -- - // which have no agent slot, so Option A cannot carry them -- are - // materialized separately from the frozen closure the child already - // holds (`materializeSourcePluginFactories`) and fed into the SAME - // `pluginFactories` slot `materializeStepTools` fills for pinned - // packages, so the existing per-step plugin chain wires them onto - // `env.plugins`. - // - // - Pinned tool packages (`materializeStepTools`): materialize the pinned - // tool-package closure (posix, LSP, mail, ...) from its on-disk deploy - // tree, rooted per step under `storeDir` so concurrent steps in one - // child never collide on the tarball cache or the apply-state tree. A - // step with no manifest yields empty tools (the legitimate - // `rawManifestBytes === undefined` case); a present-but-broken manifest - // surfaces loudly through `materializeStepTools` rather than degrading to - // empty tools. + // sourceTools: run the step agent's own evaluated toolFactories directly + // (the source deploy stages no manifest, so materializeStepTools would + // find nothing); plugin factories come separately from the frozen + // closure into the same pluginFactories slot. Otherwise: + // materializeStepTools loads the pinned tool-package closure, rooted + // per step so concurrent steps never collide on cache/apply-state. const materialization: StepToolMaterialization = deps.sourceTools === true ? { @@ -1054,20 +649,9 @@ export function createSidecarStepBuildEnv( cache: deps.cache, }); - // Derive and record the step's tool-mark floor from the just-loaded - // factories' static definitions. A pinned tool loads here in the - // child and never reached the hub's capability walk, so its - // `tool:` grant is absent from the credentials snapshot; the - // recorded floor is what lets the grant evaluator authorize a pinned - // tool against its own static mark. Keyed by base step id so the - // evaluator's `baseStepId(stepId)` lookup resolves for both a plain - // step and a `map` iteration's scoped id. - // - // Skipped on the source-ref lineage: a source tool's runtime name is the - // bare `definition.name` the capability walk already emitted a `tool:` - // grant for, so the credentials snapshot authorizes it directly with the - // correct effect -- no floor is needed, and recording one would blur that - // invariant. + // A pinned tool never reached the hub's capability walk, so its floor is + // recorded here for the grant evaluator; skipped on the source-ref + // lineage since the walk already granted the bare tool name directly. if (deps.sourceTools !== true) { deps.recordToolMarkFloor( baseStepId(stepId), @@ -1075,89 +659,45 @@ export function createSidecarStepBuildEnv( ); } - // Supervisor-backed transport for the step agent's mail tools (both - // halves of mailbox ownership, §3a OUTBOUND and §3b INBOUND). Outbound - // (`send`) routes over the control IPC to the supervisor, which performs - // the actual signed send through the host transport as `address`. - // `address` is the deployment mailbox address: the same identity the host - // registered the agent's `CryptoProvider` against, so the outbound mail - // carries the agent's signature with parity to the in-process path. - // Inbound (`deps.inbound`, present for the warm single-step agent) makes - // `mail_read` / `mail_search` / `mail_wait` resolve locally against a fresh - // committed snapshot of the deployment's substrate `INBOX`; a build that - // owns no inbound mailbox (a spawned child) leaves it undefined and the - // inbound methods stay inert. Both `transport` and `address` are the env - // keys `@intx/tools-mail`'s sidecar bundle declares in its `requires`. + // Outbound routes over control IPC to the supervisor for the actual + // signed send; inbound (deps.inbound, warm agent only) resolves locally + // against a fresh committed INBOX snapshot. const transport = createSupervisorBackedTransport( deps.outboundMailBridge, deps.mailboxAddress, deps.inbound, ); - // The host owns capability assembly: it builds the RuntimeCapabilities - // bag here (currently `mail.transport`) and puts it on `env.capabilities`, - // so a bundle consumes the assembled bag rather than re-wrapping a raw - // env key of its own. `@intx/tools-mail`'s sidecar bundle (`requires: - // ["capabilities", "address"]`) resolves `mail.transport` from it. const capabilities = createHarnessRuntimeCapabilities({ transport }); - // The step env carries `toolCwd`, `transport`, `address`, and the - // assembled `capabilities` beyond `BaseEnv`. `toolCwd` is the working - // tree the posix tools operate on; `capabilities` is what the mail bundle - // reads; `transport` remains a raw env surface for tool packages that - // read it directly. `address` is observability-only. These widen the - // returned `StepEnvBase` structurally, which the buildEnv return type - // (`StepEnvBase`) accepts (a wider object is assignable to the narrower - // type). + // Widens StepEnvBase structurally (a wider object is assignable to the + // narrower return type); address is observability-only. const env: StepEnvBase & { toolCwd: string; transport: MessageTransport; address: string; capabilities: RuntimeCapabilities; } = { - // Feed the reactor the step's full ordered failover chain and pin - // its initial source to element 0. The reactor resolves the initial - // source by id and fails over forward through `sources`, so this - // restores cross-source failover inside the workflow-child. + // Pins the initial source to element 0 so the reactor fails over + // forward through the full chain. sources, defaultSource: activeSource.id, storage, workdir, - // The per-step workspace is both the lock boundary and the tree the - // filesystem tools operate on; a deployed step gets a throwaway - // scratch dir, so the two coincide. toolCwd: workdir, audit: storage, directors: createDefaultDirectorRegistry(), - // Resolve inference adapters through the child's boot-built - // registry (built-ins + operator custom adapters), so a - // custom-provider step source resolves in the child the same way - // it does on the sidecar main path rather than hitting - // `createAgent`'s built-ins-only default. deps: createDependencies(deps.adapters), transport, address: deps.mailboxAddress, capabilities, }; - // Carry the materialized tool runtime to the tool-bearing - // `agentFactory` via the env's symbol-keyed slot. The step-invoker - // adapter spreads this env (`{ ...envBase, authorize }`) before - // handing it to `agentFactory`; object spread preserves own - // symbol-keyed properties, so the slot survives the spread. attachStepTools(env, materialization); - // Attach the credential wiring so the `agentFactory` can assemble each - // bundle's consumer-scoped `credentials` capability. Grants are wired as a - // THUNK, resolved (and cast to the sidecar's `GrantRule` grammar, the same - // cast `evaluateGrantsAdapter` makes) only when a package actually needs a - // capability -- a step with no credential-consuming package never reads - // them, so a self-discovery resume that precedes the grants barrier does not - // fault on a missing snapshot. Omitted when no credential context was - // threaded, in which case a credential-consuming tool fails closed and loud - // at its own `resolve("credentials")`. + // Grants are a thunk so a self-discovery resume preceding the grants + // barrier doesn't fault on a missing snapshot. if (credentialContext !== undefined) { - // Inference resolves its source's secret from the SAME live cell tool - // credentials resolve from, by `credentialId`, so the step's sources carry - // no inline key and the child never holds the cipher key. + // Same live cell tool credentials resolve from, so the child never + // holds the cipher key. env.readCurrentMaterial = createInferenceCredentialResolver(credentialContext.materialCell); attachStepCredentialWiring(env, { materialCell: credentialContext.materialCell, @@ -1643,21 +1183,9 @@ async function buildChildRunEnv(args: { definition: WorkflowDefinition; childRunId: string; parentRunId: string; - /** - * Per-run live inference-event sink, threaded from the parent run's event - * channel. Required for both paths -- a childWorkflow step and an onTrigger - * body step both run a real agent whose inference the hub stream must see, so - * a missing sink is a wiring defect, not a silent drop. - */ + /** Required for both paths (childWorkflow and onTrigger body); a missing sink is a wiring defect, not a silent drop. */ onEvent: (event: InferenceEvent) => void; - /** - * The parent run's live credential-material cell. Threaded so the child's - * inference resolves its source secret by `credentialId` against the run's - * current delivery -- reached live through the shared reference on a rotation. - * Absent when a non-sidecar executor carries no credential material, in which - * case no credential context is assembled and the child's inference reader - * stays unset. - */ + /** Absent when a non-sidecar executor carries no credential material, leaving the child's inference reader unset. */ materialCell?: CredentialMaterialCell; }): Promise<{ env: WorkflowRuntimeEnv; @@ -1677,23 +1205,14 @@ async function buildChildRunEnv(args: { onEvent, materialCell, } = args; - // A rung may itself embed a child (grandchild recursion) as an inline - // `childWorkflow`. Lift each to an internal `{ ref }` and run the rewritten - // definition whose children are refs -- the shape the runtime dispatches -- - // and keep the lifted definitions in an in-memory map the rung's own - // terminal resolver serves from, so a grandchild spawns with no on-disk read - // at any depth. Returned to the caller so `runtimeRun` drives the rewritten - // form. + // Lifts each inline grandchild childWorkflow to a { ref } and keeps the + // lifted definitions in-memory, so a grandchild spawns with no on-disk read. const { workflow: rewrittenDefinition, bodies: grandchildBodies } = rewriteInlineChildWorkflowBodies(definition); const grandchildMap = new Map(grandchildBodies.map((b) => [b.ref, b.definition])); - // Enumerate any `loop` bodies nested in this spawned body so a loop can run - // here (the runtime needs `env.loopFns` + `env.spawnLoopIteration`, wired - // below). A loop body stays inline on its primitive, so mint a ref-keyed copy - // for the loop-iteration host, keep the pre-rewrite form for the per-iteration - // grant cap, and merge each loop body's own childWorkflow grandchildren into - // `grandchildMap` so a grandchild spawned from a loop body resolves in-memory. - // Mirrors the top-level loop-body registration in run-child.ts. + // Mints a ref-keyed copy for the loop-iteration host and keeps the + // pre-rewrite form for the per-iteration grant cap. Mirrors the + // top-level loop-body registration in run-child.ts. const loopBodies = enumerateInlineLoopBodies(definition); const loopBodiesMap = new Map(); const loopBodyPreRewrite = new Map(); @@ -1705,10 +1224,8 @@ async function buildChildRunEnv(args: { grandchildMap.set(grandchild.ref, grandchild.definition); } } - // Resolve the loop while/carry fns from the closure so a loop in this body can - // run. Only load when the body contains a loop. If it declares a loop but no - // closure is wired, that is a wiring defect -- fail loud, mirroring the - // establish-time posture rather than deferring to a mid-run resolve failure. + // A loop declared with no closure wired is a wiring defect; fail loud + // rather than deferring to a mid-run resolve failure. let loopFns: LoopFnRegistry | undefined; if (loopBodies.length > 0) { if (deps.closurePackageDir === undefined) { @@ -1724,10 +1241,7 @@ async function buildChildRunEnv(args: { loopFns, ); } - // Read the parent's grants, cap them to what this body declares, and persist - // the capped set as the child's own `runs//grants.json` (the file - // a grandchild reads as its ceiling). `definition` here is the childWorkflow- - // inline form, so the cap keeps a grandchild's declared resources. + // Persisted as runs//grants.json, the file a grandchild reads as its ceiling. const childGrants = await capAndPersistChildGrants({ deps, directors, @@ -1735,21 +1249,10 @@ async function buildChildRunEnv(args: { childRunId, parentRunId, }); - // The child's credentials snapshot applies the capped grant set - // uniformly across every step the child definition declares, keyed on - // each step's id (the same shape the deploy-time and per-run snapshot - // assemblies produce). The in-process child has no per-step mail - // address, so the snapshot's `address` mirrors the step id -- - // `createCredentialsBackedAuthorize` reads only `grants`. - // - // "Every step the child definition declares" reaches past `stepOrder` into - // this body's `loop` bodies: a loop iteration runs under the inherited env, - // so its steps authorize against THIS snapshot under their own plain ids. A - // snapshot built from `stepOrder` alone leaves a loop nested one rung down - // with no entry, and the authorize throws on the body's first tool call. The - // capped set applies to them unchanged -- the cap is taken once, against the - // spawned body, and a loop inherits its enclosing run's authority rather than - // narrowing it again. + // The in-process child has no per-step mail address, so address mirrors + // the step id (createCredentialsBackedAuthorize reads only grants). Walks + // past stepOrder into loop bodies too, or a nested loop's authorize would + // throw on its first tool call for lacking a snapshot entry. const credentialStepIds = new Set(); walkWorkflowSteps({ definition: rewrittenDefinition, @@ -1785,55 +1288,36 @@ async function buildChildRunEnv(args: { newId: () => newId("sig"), clock, }); - // The child's `env.authorize` binds to the inherited credentials - // snapshot: each `(resource, action)` decision looks up the step's - // grants and delegates to the parent factory's grant evaluator. The - // runtime body stores this on the env; the child's `invokeStep` - // wrapper below is the seam that consults it per tool call, and an - // action step's `EffectContext` calls it directly for each effect. + // Each (resource, action) decision looks up the step's grants and + // delegates to the parent factory's grant evaluator. const credentialsRef: CredentialsSnapshotRef = { current: credentialsSnapshot, }; const authorize = createCredentialsBackedAuthorize(credentialsRef, deps.evaluateGrants); const drain = createNoopDrainController(rewrittenDefinition); - // Both the terminal childWorkflow path and the onTrigger BODY path run a real - // agent that resolves inference against its OWN per-step source table, keyed by - // the rewritten ref, and funnels live events to the parent run's channel. So - // `onEvent` is required for both; a missing one is a wiring defect, not a - // silent drop. + // Required for both childWorkflow and onTrigger body paths; a missing + // one is a wiring defect, not a silent drop. if (onEvent === undefined) { throw new Error( "sidecar child: onEvent is missing; child inference events would be silently dropped from the hub stream", ); } const childOnEvent = onEvent; - // Resolve fresh per spawn into a `sourcesRef` disjoint from the top-level's - // mutable table, so a top-level source rotation never leaks into a child. The - // sources are delivered plaintext through the spawn env (`deps.bodySources`), - // decrypted sidecar-side, so the child never holds the cipher key. + // Disjoint from the top-level's mutable table, so a source rotation there + // never leaks into a child; sources arrive plaintext, decrypted sidecar-side. const sourcesRef: SourcesSnapshotRef = { current: await resolveBodyStepSources(deps, rewrittenDefinition.id), }; - // Recursive `spawnChild`: a grandchild embedded inline in this rung is - // resolved from the in-memory map lifted above and flows back into this same - // `runChild` callback. Inject THIS run's event funnel (mirroring the rung-0 - // wrap in run-child.ts) so the grandchild's agent steps ride this run's - // channel; the runtime env keeps the narrow `SpawnChildWorkflow` (no event - // slot). Sub-namespace scoping holds at every depth via `childRunId`. + // Mirrors the rung-0 wrap in run-child.ts so a grandchild's agent steps + // ride this run's event channel. const spawnHost = createInMemorySpawnChild({ bodies: grandchildMap, runChild, }); const spawnChild: WorkflowRuntimeEnv["spawnChild"] = (spawnInput) => spawnHost(spawnInput, childOnEvent, materialCell); - // Assemble the per-step credential context from the run's live material cell - // (threaded in from the parent) when one is present, so the child's inference - // resolves its source secret by `credentialId` against the parent's current - // delivery. Every spawned child's steps are tool-bearing, so the context - // resolves the child's capped grants (mirroring the top level's snapshot - // lookup) and carries the sidecar-static providers for the tool `credentials` - // capability. Absent when no material was threaded (a non-sidecar executor), - // which leaves the child's inference reader unset. + // Absent when no material was threaded (a non-sidecar executor), leaving + // the child's inference reader unset. const childCredentialContext: SidecarStepCredentialContext | undefined = materialCell === undefined ? undefined @@ -1852,14 +1336,7 @@ async function buildChildRunEnv(args: { }, providers: deps.credentialProviders, }; - // Per-step invocation seam. The runtime body invokes `env.invokeStep` with - // the request alone; the wrapper forwards the child's credentials-backed - // authorize (so each tool call gates against the inherited grants), the run's - // `sourcesRef`, the event funnel, and the credential context (so the step's - // inference reads the run's live material). One invoker serves both the - // childWorkflow path and the onTrigger body path: each runs a real - // tool-bearing agent through the source-tools arm against the same - // `sourcesRef`. + // One invoker serves both the childWorkflow and onTrigger body paths. const invokeStep: WorkflowRuntimeEnv["invokeStep"] = (req) => deps.invokeStep(req, authorize, sourcesRef, childOnEvent, childCredentialContext); const env: WorkflowRuntimeEnv = { @@ -1876,23 +1353,11 @@ async function buildChildRunEnv(args: { drain, ...(loopFns !== undefined ? { loopFns } : {}), }; - // Wire loop-iteration spawning for a `loop` nested in this body. Assigned - // AFTER the env literal because the iteration host closes over `env`: a loop - // iteration re-enters THIS body env, so a nested loop composes and inherits - // the body's step invoker, capped grants, and in-memory spawnChild. - // - // This deliberately REPLICATES the top-level loop host in run-child.ts rather - // than sharing a helper. The two live in different packages and differ in the - // grants seam (here `capAndPersistChildGrants` is in scope and called - // directly; the top level injects it as a binding), and the loop-in-body - // crash-resume path is not yet proven identical to the top level's. Extract a - // shared helper only once a deployed loop-in-body resume test shows the two - // paths match. - // - // Boundary: the suspendable-child seam services APPROVAL parks only. A loop - // iteration in a body that awaits an externally-delivered signal, or re-arms - // an onTrigger, inherits that approvals-only limit and cannot park on an - // external input channel. + // Assigned after the env literal since the iteration host closes over env. + // Deliberately replicates the top-level loop host in run-child.ts rather + // than sharing a helper — extract one only once a deployed loop-in-body + // resume test shows the two paths match. The suspendable-child seam + // services approval parks only. if (loopFns !== undefined) { const loopIterationHost = createInMemorySpawnSuspendableChild({ bodies: loopBodiesMap, @@ -1903,9 +1368,7 @@ async function buildChildRunEnv(args: { `sidecar child: no pre-rewrite loop body for ref ${loopInput.definitionRef}`, ); } - // Cap the iteration's grants against the body's own grants (the - // iteration's parent IS the body run) and persist them before the - // iteration appends its first event. + // Capped against the body's own grants before the first event. await capAndPersistChildGrants({ deps, directors, @@ -1944,18 +1407,9 @@ async function buildChildRunEnv(args: { } /** - * Resolve a spawned body's per-step inference sources by definition id. The - * primary path is sidecar-mediated: the sources arrive plaintext in - * `deps.bodySources`, decrypted sidecar-side from the sealed run record, so the - * child holds no cipher key. - * - * LEGACY FALLBACK: a body whose id is absent from the delivered set -- a - * deployment restored from a record written before body sources moved into the - * record -- reads its on-disk plaintext `sources.json`. That file predates the - * change and is plaintext, so the fallback constructs no cipher and the child - * stays key-free. Removable once no restorable record predates the - * record-carried body sources (after the reconnect re-push has re-persisted - * every live deployment). + * Legacy fallback: a body absent from deps.bodySources (a record predating + * body sources moving into the record) reads its plaintext on-disk + * sources.json instead. Removable once no restorable record predates it. */ async function resolveBodyStepSources( deps: SidecarRunChildDeps, @@ -1973,14 +1427,7 @@ async function resolveBodyStepSources( return readChildStepInferenceSources(deps.dataDir, definitionId); } -/** - * Legacy fallback reader (see `resolveBodyStepSources`): a spawned body's - * plaintext per-step pins from `${dataDir}/assets/workflow//sources.json`, - * staged by a pre-record deploy. Parsed through the same - * `parseStepInferenceSources` boundary the top-level `STEP_INFERENCE_SOURCES` env - * entry uses. Only reached for a body the delivered set does not carry, so a - * missing or malformed file is a defect and surfaces loudly. - */ +/** See resolveBodyStepSources; only reached for a body the delivered set doesn't carry. */ async function readChildStepInferenceSources( dataDir: string, childRef: string, @@ -2010,31 +1457,10 @@ function defaultNewId(prefix: string): string { } /** - * Build a `SubstrateFactory` closed over the supplied dependency - * overrides. The production export `createSubstrate` is the - * default-deps call. - * - * Construction order: - * 1. Narrow the `substrateConfig` record against the typed schema. - * A missing or empty key already threw inside the helper; this - * pass enforces the exact shape the factory consumes. - * 2. Open a bare `RepoStore` via `createAgentRepoStore` against the - * sidecar's data dir and Ed25519 keypair. This store backs the - * child's read-only operations against the workflow-run repo; - * the on-disk repo is shared with the supervisor's substrate so - * reads see whatever the supervisor has committed. - * 3. Construct a proxy `RepoStore` whose - * `writeTreePreservingPrefix` forwards over the upstream control - * channel via the substrate-write bridge. The supervisor's - * handler runs its own substrate's `writeTreePreservingPrefix` - * (wrapped at the boot edge with the pack-push facade) under the - * per-repo lock and replies with the resulting `commitSha`. - * 4. Start the host-process scheduler singleton against the proxy - * substrate, then adapt it to the runtime's `Scheduler` shape. - * 5. Construct the production `invokeStep` and `spawnChild` - * adapters. - * 6. Return the `RunWorkflowChildBindings` the runtime body - * consumes, with the proxy store in the `substrate` slot. + * Opens a bare RepoStore for read-only operations against the shared + * on-disk workflow-run repo, then a proxy RepoStore whose writes forward + * over the control channel into the supervisor's own (pack-push-wrapped) + * substrate. */ export function createSidecarSubstrateFactory( deps: SidecarSubstrateFactoryDeps = {}, @@ -2053,13 +1479,8 @@ export function createSidecarSubstrateFactory( const stepInferenceSources = parseStepInferenceSources(validated.STEP_INFERENCE_SOURCES); - // Build the child's adapter registry eagerly at boot from the - // operator-supplied manifest. `loadAdapterRegistry` imports every - // custom module now, so a bad specifier crashes the child loudly at - // construction rather than silently degrading to built-ins-only at - // first resolve. The closure registry the sidecar built at its own - // boot edge cannot cross the fork; the child rebuilds an equivalent - // one from the serialized-and-revalidated manifest. + // Eager: a bad specifier crashes the child loudly at construction + // rather than silently degrading to built-ins-only at first resolve. const childAdapterRegistry = await loadAdapterRegistry( parseAdapterManifest(validated.SIDECAR_ADAPTER_MANIFEST), ); From ba0323ff2dcb7cb82b14503d0065ca002c2c59e6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:19:57 -0700 Subject: [PATCH 08/31] chore(sidecar): comments say why, never what (part 7) --- .../sidecar/src/workflow-substrate-factory.ts | 192 +++--------------- 1 file changed, 32 insertions(+), 160 deletions(-) diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 128cd79d3f..4c6fa94b68 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -308,12 +308,9 @@ function createStepStorageSigner(signingKey: { } /** - * Deliberately rooted outside the workflow-run repo's working tree so it - * never collides with the run-event log under runs//events/. - * Resume-attempt invariant: a crash-resume must reopen the SAME attempt-N - * store the step suspended under (recovered via currentAttempt, not - * assumed to be 1), or the reactor rehydrates an empty gate and hangs. - * createSidecarStepBuildEnv asserts this loudly on the cold path. + * Rooted outside the workflow-run working tree to avoid the run-event log. + * A crash-resume must reopen the same attempt-N it suspended under + * (recovered via currentAttempt), or the reactor hangs on an empty gate. */ export function stepStorageRoot(args: { dataDir: string; @@ -711,16 +708,7 @@ export function createSidecarStepBuildEnv( }; } -/** - * Real per-step invoker for a spawned child's steps -- a childWorkflow child - * and an onTrigger section body alike. It widens the workflow-runtime - * `StepInvoker` with the child's credentials-backed `authorize` (the seam that - * resolves each tool call against the run's grants), the child's own per-step - * inference `sourcesRef` (built fresh per spawn from the env-delivered body - * sources, disjoint from the top-level's mutable table), and the parent run's - * `onEvent` funnel (so the child's live inference events reach the hub - * timeline). - */ +/** Widens StepInvoker with the child's authorize, its own per-spawn sourcesRef, and the parent's onEvent funnel. */ export type SidecarChildStepInvoker = ( req: StepInvokeRequest, authorize: WorkflowAuthorizeFn, @@ -730,29 +718,10 @@ export type SidecarChildStepInvoker = ( ) => Promise; /** - * Inputs required to construct the sidecar's in-process child runtime. - * Lifted out of `createSidecarSubstrateFactory` so the implementation - * is exercisable in isolation (the co-located test wires a hand-built - * substrate/principal/scheduler/invokeStep against this surface). - * - * Sub-namespace scoping: the child runtime is invoked with - * `runId: childRunId`. The runtime body threads that id through every - * `repoStore.read/append/subscribe` call, every `blobs.recordOutput` - * call, and every `signalChannel.deliver/awaitNext` call. The host- - * adapter implementations (`createWorkflowRunRepoStore`, - * `createWorkflowRunBlobSubstrate`, `createWorkflowHostSignalChannel`) - * each compute their on-disk path as `runs//...` against the - * supplied workflow-run repo. The net effect is that the child's - * events land under `runs//events/.json` of the - * parent's workflow-run repo, sibling to the parent's own - * `runs//...` subtree. - * - * Substrate identity: the child reuses the parent's wrapped `RepoStore` - * (the workflow-run pack-pushing wrap installed by the factory) so a - * successful child write fires the same hub pack push the parent's - * writes do. The substrate's signing principal (a workflow-process - * principal scoped to the parent's anchorRunId) is reused verbatim - * because the child runs under the same supervisor authority. + * Lifted out of createSidecarSubstrateFactory so it's exercisable in + * isolation. The child runs with runId: childRunId, so its events land at + * runs//... sibling to the parent's own subtree, reusing the + * parent's pack-pushing substrate and signing principal verbatim. */ interface SidecarRunChildDeps { /** Wrapped workflow-run substrate (the factory's `substrate`). */ @@ -766,72 +735,22 @@ interface SidecarRunChildDeps { /** Host-process scheduler singleton; shared with the parent. */ scheduler: Scheduler; /** - * Step invoker the child runtime delegates per-step invocations to. - * - * The in-process child runs a WorkflowDefinition whose stepIds are - * disjoint from the parent's. The parent's - * `STEP_INFERENCE_SOURCES`-pinned `buildStepEnv` knows only the - * parent's stepIds and throws on any other id; routing the child's - * step invocations through that closure surfaces a misleading - * "no InferenceSource pinned" error for every child step. Callers - * therefore supply a SEPARATE invokeStep for the child. The substrate - * factory's default (`childInvokeStep` in - * `createSidecarSubstrateFactory`) runs a real, tool-bearing agent - * against the child's own staged sources; a test may inject its own. - * One invoker serves every spawned child's steps -- a childWorkflow - * child, an onTrigger section body, and a body's own grandchildren -- - * because each of them runs real inference and so must be able to call - * tools. - * - * The invoker receives the child's credentials-backed `authorize` - * alongside the request, mirroring the workflow-process child's - * `ChildStepInvoker`: the runtime body calls `env.invokeStep` with the - * request only, so the invoker -- not the runtime -- is the seam that - * gates each tool call against the run's grants. + * The child's stepIds are disjoint from the parent's, so routing through + * the parent's STEP_INFERENCE_SOURCES-pinned buildStepEnv would throw a + * misleading error; callers supply a separate invokeStep instead. */ invokeStep: SidecarChildStepInvoker; - /** - * Every spawned body's plaintext inference-source table, keyed by definition - * id, decrypted sidecar-side from the run record and delivered through the - * spawn env. Each body path resolves its own table from here by definition id, - * so the child never holds the sidecar cipher key. A body whose id is absent -- - * a deployment restored from a legacy record written before body sources moved - * into the record -- falls back to the on-disk `dataDir` file. - */ + /** Keyed by definition id so the child never holds the sidecar cipher key; a missing id falls back to dataDir. */ bodySources: BodyInferenceSources; - /** - * Sidecar data dir. The legacy fallback for a body absent from `bodySources` - * reads its `assets/workflow//sources.json`; the terminal - * childWorkflow / body paths resolve per-step storage under it. Required. - */ + /** Backs the legacy on-disk sources.json fallback and per-step storage roots. Required. */ dataDir?: string; - /** - * Grant evaluator the child's credentials-backed `authorize` delegates - * each `(resource, action)` decision to. The parent factory owns the - * sidecar's grant-rule grammar and supplies its adapter here so the - * child resolves authorization against the same evaluator the parent's - * steps use. - */ + /** Supplied by the parent factory so the child authorizes against the same evaluator the parent's steps use. */ evaluateGrants: GrantEvaluator; - /** - * Sidecar-static credential provider registry, shared with the top level. - * The child's build combines it with the run's live material cell and the - * child's capped grants to assemble each bundle's consumer `credentials` - * capability, and to resolve its inference source secret against the same - * material. - */ + /** Combined with the run's live material cell and capped grants to assemble each bundle's credentials capability. */ credentialProviders: CredentialProviderRegistry; /** Director registry the child runtime uses; defaults to the canonical built-ins. */ directors?: DirectorRegistry; - /** - * Sidecar-local directory of the materialized workflow-definition closure - * (`env.spawn.closurePackageDir`), the source-ref lineage's only closure. - * `buildChildRunEnv` loads the child body's declared plugin tool definitions - * from it so a plugin-contributed `tool:` the child loads is declared - * when capping the child's inherited grants. A child body that declares no - * plugin package needs no closure read; absent on a lineage that stages no - * closure. - */ + /** Present only on the source-ref lineage, to declare plugin-contributed tools when capping inherited grants. */ closurePackageDir?: string; /** Clock for timestamp generation; defaults to `() => new Date()`. */ clock?: () => Date; @@ -843,21 +762,9 @@ interface SidecarRunChildDeps { } /** - * Write a spawned child's inherited grants to its own - * `runs//grants.json` in the deployment's workflow-run repo. - * - * The write goes through the child proxy substrate's - * `writeTreePreservingPrefix` -- the only write path the proxy forwards - * to the supervisor. It names the shallow `runs//` prefix, whose - * merge input is only that level's flat blobs (the prefix read does not - * recurse), so the write rebuilds the level with just `grants.json`. This is - * safe ONLY because the sole caller (`capAndPersistChildGrants`) is write-once: - * it invokes this only when no grants file exists yet, i.e. at the run's birth - * before the runtime appends any event, so the `runs//` subtree is - * empty and nothing is dropped. Invoking it over a populated run would delete - * that run's committed `events/`/`blobs/` subtrees. A later event append names - * the nested `runs//events/` prefix, so its own subtree-delete does - * not reach `grants.json` one level up -- the sibling survives. + * Safe only because the sole caller is write-once, at the run's birth + * before any event append — the shallow runs// prefix rebuild + * would otherwise delete a populated run's events/blobs subtrees. */ async function writeChildRunGrants(args: { substrate: RepoStore; @@ -883,55 +790,26 @@ async function writeChildRunGrants(args: { } /** - * Construct the `RunChildWorkflow` callback the spawn-child adapter - * delegates to. The returned callback, when invoked with the parent - * runtime's attribution + the parent-allocated `childRunId` + the - * resolved `WorkflowDefinition`, builds a fresh `WorkflowRuntimeEnv` - * scoped to `childRunId`, invokes `runtimeRun`, and returns the - * child's terminal status. - * - * Abort propagation: the parent-supplied `signal` is wired to the child - * run's local-abort seam (`RuntimeRunOptions.localAbort`). If it aborts -- - * mid-flight or pre-aborted -- the child's own cancel controller aborts, an - * in-flight step fails, and the returned promise resolves with - * `terminalStatus: "failed"`. There is no durable `CancelRequested`: an - * in-process child writes through the proxy substrate and cannot sign the - * supervisor cancel one would require. - * - * Resource lifecycle: the child's per-run signal channel handle is - * `stop()`ped in a finally block so any background `subscribeKind` - * loop tied to the child's runId tears down before the callback - * returns. The blob substrate, repo store, and scheduler entries are - * either per-call (no handle to dispose) or shared with the parent - * (the scheduler). + * No durable CancelRequested: an in-process child aborts through the + * runtime's local-abort seam since it cannot sign the supervisor cancel a + * control-plane cancel would require. The signal channel handle is + * stop()ped in a finally block so no subscribeKind loop outlives the call. */ export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkflow { const directors = deps.directors ?? createDefaultDirectorRegistry(); const clock = deps.clock ?? defaultClock; const newId = deps.newId ?? defaultNewId; - // No `controlPlanePrincipal`: an in-process child tears down through the - // runtime's local-abort seam (see the `runtimeRun` call below), not a durable - // `CancelRequested`, so it never writes a control-plane cancel this repo store - // would have to sign. Run-body events use the workflow-process `principal`. - // Created once and shared across every child this factory spawns (the - // runtime scopes reads/subscribes by runId), so sibling and grandchild - // spawns route through one repo-store handle rather than a fresh one each. + // No controlPlanePrincipal: teardown goes through the local-abort seam, not + // a durable CancelRequested. Shared across every child this factory spawns. const repoStore = createWorkflowRunRepoStore({ substrate: deps.substrate, repoId: deps.workflowRunRepoId, principal: deps.principal, ref: deps.workflowRunRef, }); - // Self-referential `RunChildWorkflow` so a child env's recursive - // `spawnChild` (built via `createInMemorySpawnChild` in `buildChildRunEnv`) - // can route grandchild spawns back through the same adapter. Each invocation - // builds a per-runId env that itself wires a `spawnChild` slot whose - // `runChild` is this same `runChild` constant -- the recursion bottoms - // out when a rung's `WorkflowDefinition` has no `childWorkflow` - // primitive. Sub-namespace scoping continues to hold at every depth - // because `childRunId` flows verbatim into the per-rung - // `blobs`/`signalChannel`/`runtimeRun` calls, keeping every rung's - // events under `runs//...` of the parent's workflow-run repo. + // Self-referential so a child env's recursive spawnChild routes + // grandchild spawns back through this same adapter; recursion bottoms + // out when a rung's definition has no childWorkflow primitive. const runChild: RunChildWorkflow = async ( { definition, childRunId, input, parentRunId, signal, depth, maxChildSpawnDepth }, onEvent, @@ -955,16 +833,10 @@ export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkfl ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), }); try { - // Thread this rung's depth/ceiling into the child run so its own - // childWorkflow spawns keep counting against the tree-wide bound (the - // guard lives in the runtime's runChildWorkflow). Without this the - // in-process sidecar recursion would reset to depth 0 each rung. - // A parent abort tears the child down through `runtimeRun`'s local-abort - // seam, not a control-plane cancel: the in-process child writes through - // the workflow-run PROXY substrate, which cannot sign the supervisor - // `CancelRequested` a cancel needs (the kind handler refuses a - // workflow-process-signed cancel). Local teardown aborts the child's own - // controller directly, so an in-flight step fails and the child settles + // Threads this rung's depth/ceiling so nested spawns keep counting + // against the tree-wide bound rather than resetting to depth 0 per rung. + // A parent abort tears down through the local-abort seam, not a + // control-plane cancel the proxy substrate cannot sign. // `failed` under its own principal -- no durable cancel, no wedge while the // parent awaits the terminal below. const handle = runtimeRun(rewrittenDefinition, env, { From 40d5458cffbb4e12142d37bac34d1a6f4098b60d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:20:33 -0700 Subject: [PATCH 09/31] chore(sidecar): comments say why, never what (part 8) --- .../sidecar/src/workflow-substrate-factory.ts | 92 ++++--------------- 1 file changed, 16 insertions(+), 76 deletions(-) diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 4c6fa94b68..0a91f56e6a 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -856,59 +856,27 @@ export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkfl } /** - * Construct the `RunSuspendableChild` callback the suspendable-spawn - * adapter delegates to. The park-aware analog of - * {@link createSidecarRunChild}: instead of awaiting the child's terminal, - * it returns a live {@link SuspendableChildHandle} the caller (`runOnTrigger`) - * drives across the body's approval parks. - * - * Park surfacing: the built env's `onPark` sink translates the child body's - * control-plane parks into the handle's `next()` stream. An `"approval"` - * park is queued for the caller to proxy up on the same correlation; the - * caller's granted decision returns through `resume`, which delivers it on - * the child's own signal channel so the parked step unblocks. A body that - * parks on a control-plane `"input"` channel is a nested onTrigger re-arm - * the suspendable seam does not service -- the caller proxies approvals - * only, so nothing would ever deliver that input and the child would park - * forever. Rather than drop the park and hang, `onPark` surfaces it as a - * hard error on `next()` and tears the child down locally so the section run - * ends loudly (failed). - * - * Signal-channel lifecycle: unlike `createSidecarRunChild`, which stops the - * channel in a `finally` around a single awaited terminal, this keeps the - * channel alive across every park (so `resume` can deliver) and ties its - * teardown to the run's terminal -- the one lifecycle moment every path - * funnels through (normal completion, a parent-abort local teardown, an - * illegal-input-park local teardown, or a runtime failure). Tearing down - * per-`next()` would leak the channel when a parent abort makes `runOnTrigger` - * stop calling `next()` mid-park. - * - * Abort propagation: the parent-supplied `signal` tears the body down through - * `createSuspendableChildHandle`'s local-teardown seam -- the in-process body - * runs under the `workflow-process` principal and cannot sign the control-plane - * `CancelRequested` a durable cancel needs, so on abort its own step fails and - * the run settles `failed` (not `cancelled`), and the terminal-tied teardown - * runs. + * The park-aware analog of {@link createSidecarRunChild}: returns a live + * handle instead of awaiting a terminal. A body parking on a control-plane + * "input" channel (a nested onTrigger the suspendable seam can't service) + * surfaces as a hard error on next() rather than hanging forever. The + * signal channel stays alive across every park and is tied to the run's + * terminal, not per-next(), or a parent abort mid-park would leak it. */ export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): RunSuspendableChild { const directors = deps.directors ?? createDefaultDirectorRegistry(); const clock = deps.clock ?? defaultClock; const newId = deps.newId ?? defaultNewId; - // No `controlPlanePrincipal`: an in-process body tears down through the shared - // handle's local-abort seam (`createSuspendableChildHandle`), not a durable - // `CancelRequested`, so it never writes a control-plane cancel this repo store - // would have to sign. Run-body events use the workflow-process `principal`. + // No controlPlanePrincipal: teardown goes through the shared handle's + // local-abort seam, not a durable CancelRequested. const repoStore = createWorkflowRunRepoStore({ substrate: deps.substrate, repoId: deps.workflowRunRepoId, principal: deps.principal, ref: deps.workflowRunRef, }); - // A body's own `childWorkflow` grandchildren spawn terminal-only: the - // suspendable seam is exercised only by onTrigger sections, and - // `buildChildRunEnv` wires the body env's `spawnChild` (not - // `spawnSuspendableChild`), so a nested onTrigger inside a body fails loud - // rather than silently spawning. + // Body grandchildren spawn terminal-only: buildChildRunEnv wires spawnChild, + // not spawnSuspendableChild, so a nested onTrigger fails loud. const runChild = createSidecarRunChild(deps); return async ( @@ -939,15 +907,8 @@ export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): R definition, childRunId, parentRunId, - // The live event sink is always threaded (both a body step and a - // grandchild childWorkflow step run a real agent). The body's own steps - // and its childWorkflow grandchildren, built via the internal - // `createSidecarRunChild(deps)` above, both run through the tool-bearing - // `deps.invokeStep`. onEvent, - // The run's live credential-material cell so the body's inference resolves - // its source secret against the parent's current delivery; a grandchild - // spawned from the body inherits it through the recursive spawnChild. + // A grandchild spawned from the body inherits this through the recursive spawnChild. ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), }); @@ -965,32 +926,11 @@ export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): R } /** - * Read the parent run's grants, cap them to what `definition` declares, and - * persist the capped set as the child run's own `runs//grants.json` - * -- the file the next spawn hop (a childWorkflow grandchild) reads back as its - * ceiling. Returns the capped grants so a caller building a fresh child env can - * also key its credentials snapshot on them. - * - * `definition` MUST be the PRE-rewrite body, its childWorkflow grandchildren - * still INLINE: `collectDeclaredResources` skips a `{ ref }` body, so a - * rewritten definition would drop the grandchild's declared resources and - * under-authorize it. Every birth path materializes a run's grants file, so a - * missing parent file is a defect, not a run that legitimately holds none -- - * fail closed. The cap only removes rules (the parent stays the ceiling), - * matching the top level's "authority bounded by declared capabilities" model. - * - * The grants file is WRITE-ONCE per run: a run's authorization ceiling is fixed - * at its birth, so this reads back an existing `runs//grants.json` - * rather than rewriting it. Re-writing on a resume re-drive is not just - * redundant, it is CORRUPTING: `writeChildRunGrants` commits at the shallow - * `runs//` prefix through a substrate writer separate from the - * runtime's event-log writer, so a re-write racing the runtime's replay - * re-appends on the shared repo regresses another run's event seq (a - * single-writer-invariant violation), and its wholesale subtree-delete drops the - * run's committed `events/`/`blobs/` (the prefix read is non-recursive, so the - * merge cannot carry them forward). The read-back returns the same capped set - * the birth-time write persisted, so a fresh child-env caller keys its - * credentials snapshot on identical grants either way. + * definition must be the pre-rewrite body (grandchildren still inline), or + * collectDeclaredResources would under-authorize a rewritten { ref } body. + * Write-once per run: a resume re-drive reads back the existing grants + * file rather than rewriting it, since a re-write would race the runtime's + * replay and corrupt the shared repo's event seq. */ async function capAndPersistChildGrants(args: { deps: SidecarRunChildDeps; From aae8d674aab7596c6d3a14289a92010bf287ebed Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:21:55 -0700 Subject: [PATCH 10/31] chore(sidecar): comments say why, never what (part 9) --- .../sidecar/src/workflow-substrate-factory.ts | 218 ++++-------------- 1 file changed, 41 insertions(+), 177 deletions(-) diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 0a91f56e6a..1383c51843 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -856,12 +856,10 @@ export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkfl } /** - * The park-aware analog of {@link createSidecarRunChild}: returns a live - * handle instead of awaiting a terminal. A body parking on a control-plane - * "input" channel (a nested onTrigger the suspendable seam can't service) - * surfaces as a hard error on next() rather than hanging forever. The - * signal channel stays alive across every park and is tied to the run's - * terminal, not per-next(), or a parent abort mid-park would leak it. + * Park-aware analog of {@link createSidecarRunChild}: returns a live handle. + * A body parking on an unserviceable "input" channel is a hard error on + * next() rather than hanging; the signal channel stays alive across every + * park, tied to the run's terminal rather than per-next(). */ export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): RunSuspendableChild { const directors = deps.directors ?? createDefaultDirectorRegistry(); @@ -976,15 +974,7 @@ async function capAndPersistChildGrants(args: { return childGrants; } -/** - * Build the per-childRunId `WorkflowRuntimeEnv` a spawned child runs - * against: inherit the parent's grants, assemble the child's credentials - * snapshot, and wire the per-run repo store / blob substrate / signal - * channel plus a recursive `spawnChild`. Returned alongside the child's - * signal channel so the caller can `stop()` it once the child settles. - * Shared by the child-drive callers so the env construction lives in one - * place. - */ +/** Shared by both child-drive callers so env construction lives in one place; signal channel returned for caller stop(). */ async function buildChildRunEnv(args: { deps: SidecarRunChildDeps; directors: ReturnType; @@ -1327,16 +1317,9 @@ export function createSidecarSubstrateFactory( workflowRunRepoId, }); - // INBOUND half of mailbox ownership (§3b). One watch registry per child, - // created at boot and shared by BOTH the step agent's supervisor-backed - // transport (its `watch` registers callbacks here, backing `mail_wait`) and - // the child's control loop (which fires each `mailbox.notify` into it). The - // registry rides out on the returned bindings so `runWorkflowChild` routes - // notifications to this same instance. The child mailbox reader opens a - // fresh committed snapshot of the deployment's substrate `INBOX` per read, - // over the same substrate handle / repo id / principal / ref the mail-part - // reader uses, so a read taken after a `mailbox.notify` observes the message - // the supervisor just committed. + // One registry per child, shared by the step agent's transport (mail_wait) + // and the control loop's mailbox.notify firing, so a read after a notify + // observes what the supervisor just committed. const mailboxWatchRegistry = createMailboxWatchRegistry(); const transportInbound: SupervisorBackedTransportInbound = { reader: createChildMailboxReader({ @@ -1346,13 +1329,9 @@ export function createSidecarSubstrateFactory( ref: validated.WORKFLOW_RUN_REF, }), watchRegistry: mailboxWatchRegistry, - // The child holds no sender-key registry, so signature verification is not - // yet possible here; `fetchFull` reports every message's signature status - // as "unknown". A future sender-key surface would replace this resolver. + // No sender-key registry yet, so every message's signature status is "unknown". getCrypto: () => undefined, - // The write methods (setFlags / clearFlags / expunge) route through this - // bridge to the supervisor, the sole mailbox writer, instead of flushing - // the run ref from the child. + // Write methods route to the supervisor, the sole mailbox writer. mutationBridge: env.mailboxMutationBridge, }; @@ -1366,13 +1345,6 @@ export function createSidecarSubstrateFactory( await hostScheduler.start(); const scheduler = adaptHostScheduler(hostScheduler); - // The single-step / top-level path runs a real agent. The per-step - // env builder stands up real per-step storage/workdir/audit/directors - // rooted under the run (see `createSidecarStepBuildEnv`), resolving - // the per-step `InferenceSource` from the pinned table; the real - // step-invoker instantiates the step's agent via `createAgent`, - // delivers the resolved input as a synthesized inbound message, and - // captures the agent's reply as the step output. const stepToolCache: StepToolCacheConfig = { cacheMaxBytes: parseByteCap(validated.SIDECAR_CACHE_MAX_BYTES, "SIDECAR_CACHE_MAX_BYTES"), registryMaxTarballBytes: parseByteCap( @@ -1381,26 +1353,9 @@ export function createSidecarSubstrateFactory( ), }; - // The single-step / top-level path runs a real agent with REAL - // tools materialized in-child. The per-step env builder stands up - // real per-step storage/workdir/audit/directors rooted under the - // run (see `createSidecarStepBuildEnv`), resolves the per-step - // `InferenceSource`, and materializes the step's pinned - // tool-package closure (posix, LSP, mail, ...) from its on-disk - // deploy tree -- rooted per step so concurrent steps in one child - // never collide on the tarball cache or apply-state. The - // tool-bearing `agentFactory` below attaches those factories to the - // step's `AgentDefinition` and builds the plugin chain. - // Durable-conversation registry for the warm single-step agent - // (design §3c). Built only when the deployment is warm-kept: the - // sole long-lived agent's conversation must survive child respawn, - // so it is mirrored to the workflow-run substrate at a per-agent - // path. A multi-step deploy leaves this `undefined` -- its per-step - // agents are not warm/long-lived (§3b), so they carry no cross-run - // conversation and keep the per-run isogit store. The registry lives - // for the child's lifetime; on respawn the child rebuilds it empty - // and each store restores its prior snapshot from the substrate on - // first acquire. + // Built only when warm-kept, since the sole long-lived agent's + // conversation must survive child respawn; on respawn the registry + // rebuilds empty and each store restores from the substrate on first acquire. const conversationSigner = createStepStorageSigner(signingKey); const durableConversation: DurableConversationRegistry | undefined = env.spawn.warmKeep ? createDurableConversationRegistry({ @@ -1413,14 +1368,8 @@ export function createSidecarSubstrateFactory( }) : undefined; - // Per-step tool-mark floor grants, keyed by base step id. The step - // env builder derives and records each step's floor from its - // materialized (pinned) tool factories; the grant evaluator reads it - // by base step id and merges it under the credentials snapshot's - // grants so a pinned tool authorizes against its own static mark. The - // map lives for the factory's (child's) lifetime, so a warm agent's - // floor -- recorded on its single first build -- remains available - // for every later tool call it makes. + // Lives for the child's lifetime so a warm agent's floor, recorded on + // its single first build, remains available for every later call. const toolMarkFloorByStep = new Map(); const buildStepEnv = createSidecarStepBuildEnv({ @@ -1435,43 +1384,21 @@ export function createSidecarSubstrateFactory( recordToolMarkFloor: (stepId, grants) => { toolMarkFloorByStep.set(stepId, grants); }, - // Source-ref is the only deploy lineage: the child runs each step agent's - // own evaluated tool factories (fed from `req.agent.toolFactories`) from - // the materialized closure, never a pinned tool-package manifest off a - // deploy tree. + // Source-ref is the only deploy lineage: never a pinned tool-package manifest off a deploy tree. sourceTools: true, - // The materialized closure dir the source arm materializes each step - // agent's declared plugin packages from. Always present (source-ref only). + // Always present (source-ref only). closurePackageDir: env.spawn.closurePackageDir, - // Activate the warm agent's inbound mail surface: `mail_read` / - // `mail_search` / `mail_wait` resolve against the deployment's committed - // substrate `INBOX` through this bundle. The spawned-child build below - // omits it (a spawned child owns no warm inbound mailbox). + // Omitted below for the spawned-child build, which owns no warm inbound mailbox. inbound: transportInbound, ...(durableConversation !== undefined ? { durableConversation } : {}), }); - // The tool-bearing agent factory reads the materialized tool - // runtime off the per-step env (set by `buildStepEnv` via - // `attachStepTools`), attaches the loaded tool factories to the - // step's `AgentDefinition`, builds the plugin chain on - // `env.plugins`, and wraps `agent.close()` so every plugin (the LSP - // subprocess included) and tool bundle is torn down with the agent - // on every exit path. The factory is stateless across steps, so it - // is pinned once here and shared by every per-step invoker built - // below. + // Stateless across steps, so pinned once and shared by every per-step invoker. const stepAgentFactory = createToolBearingAgentFactory(); - // The credential provider registry that shapes a delivered credential into - // a mediated handle. Built once here from the sidecar-static built-ins (the - // origin-pinned http provider) plus the header-shaped presets - // (`@corbits/credential-header`, for a provider row whose API expects - // the raw secret in `authorization` or an `x-api-key` header instead of - // a Bearer-prefixed `authorization`) and the MCP streamable-HTTP - // provider (`@corbits/credential-mcp`, for a tenant-connected MCP - // server, including its keyless-connection sentinel) — shared by every - // per-step build; the per-run material and grants ride in separately at - // each invoke. + // Built once from the sidecar-static built-ins plus the header and MCP + // streamable-HTTP presets; per-run material and grants ride in + // separately at each invoke. const credentialProviders = createCredentialProviderRegistry([ ...builtinCredentialProviders(), xApiKeyCredentialProvider(), @@ -1479,29 +1406,12 @@ export function createSidecarSubstrateFactory( createMcpStreamableHttpCredentialProvider(), ]); - // Spawned-child step build env (INTR-310). Every spawned child's steps -- - // a childWorkflow child's and an onTrigger section body's alike -- run real, - // TOOL-BEARING agents through the same source-tools arm the top level uses: - // the child holds the live re-verified definition, so each step agent - // carries live `toolFactories` fed from `req.agent.toolFactories`, and the - // source arm materializes declared plugins from the shared closure. Tools - // are available wherever inference runs, so a body agent gets the same - // build a top-level step agent gets (see the tool-availability invariant in - // `packages/workflow/README.md`). - // - // Built COLD per invocation -- no `durableConversation`/warm hooks (a - // fan-out branch and a section body are each a fresh run per spawn) and no - // `inbound` (a spawned child owns no warm inbound mailbox; inbound-reading - // tools stay inert). The source arm records no tool-mark floor (a source - // tool's bare `tool:` grant is already in the credentials snapshot), - // so the floor recorder throw-asserts that invariant. - // - // The source arm also keeps the body's tools scoped correctly by - // construction: a body child runs under the PARENT deployment's - // `mailboxAddress`/`stepCount`, so a body step whose id collides with a - // parent step id would read the PARENT step's tools if tools were resolved - // off the deploy tree. Feeding each agent its own evaluated - // `req.agent.toolFactories` never consults that tree. + // Built cold per invocation (no durableConversation/warm hooks, no + // inbound): a spawned child runs its own evaluated toolFactories rather + // than resolving off the deploy tree, which also keeps its tools scoped + // correctly even when a body step id collides with a parent step id. + // The floor recorder throw-asserts no floor is ever recorded here, + // since the source arm's bare tool: grant is already in the snapshot. const coldChildBuildStepEnv = createSidecarStepBuildEnv({ dataDir: validated.SIDECAR_DATA_DIR, workflowRunRepoId, @@ -1517,19 +1427,9 @@ export function createSidecarSubstrateFactory( sourceTools: true, closurePackageDir: env.spawn.closurePackageDir, }); - // Spawned-child step invoker (INTR-310). It runs a real agent through - // `createWorkflowStepInvoker`, resolving inference against the child's own - // per-step `sourcesRef` (staged at deploy, read fresh per spawn) and - // funnelling live events to the parent run's channel. `buildChildRunEnv` - // threads in the run's `credentialContext` (the live material cell, the - // child's capped grants, the sidecar-static providers), so the build - // attaches each bundle's `credentials` capability and the step's inference - // resolves its source secret against the run's live material. Absent when no - // material was threaded, in which case a tool that declares a credential - // consumer fails closed and loud at its own `resolve("credentials")`, never - // silently. Wired as `childRunDeps.invokeStep`, so it covers every spawned - // child's steps: a childWorkflow child, an onTrigger section body, and a - // body's own childWorkflow grandchildren. + // Wired as childRunDeps.invokeStep, covering every spawned child's steps. + // Absent credentialContext means a credential-consuming tool fails + // closed and loud at its own resolve("credentials"), never silently. const childInvokeStep: SidecarChildStepInvoker = ( req, authorize, @@ -1545,47 +1445,14 @@ export function createSidecarSubstrateFactory( onEvent, })(req); - // Adapt the workflow-runtime `StepInvoker` shape onto the host's - // `ChildStepInvoker` shape. The host's `onEvent` is the child's - // per-run event-channel sink: the runtime body passes it per step, - // and the chain from here is `onEvent -> child event-channel sender - // -> supervisor -> publishWorkflowInferenceEvent -> hub timeline`. - // - // The `authorize` argument is the child's credentials-backed - // authorize closure (`createCredentialsBackedAuthorize`), threaded - // in from `run-child.ts`'s runtime env. The step agent's runtime - // gates EVERY tool call through `env.authorize` with - // `resource = tool:`, `action = "invoke"` (the inference - // layer's authz before-tool extension); using the credentials-backed - // authorize here means each tool call resolves against the per-step - // grant snapshot the supervisor assembled from the agent's - // `state/grants.json` and pushed over the control IPC. A tool the - // agent's grants do not allow is blocked; a granted tool runs. The - // operator gate at deploy time (the capability walk's `tool:` - // approval) and this runtime grant check are complementary: the walk - // bounds the toolset the deploy may carry, the grant snapshot decides - // which of those the agent may invoke at run time. - // - // A fresh `createWorkflowStepInvoker` is built per invocation so the - // adapter subscribes the step agent's event stream to THIS step's - // `onEvent`. The per-step env builder and the tool-bearing agent - // factory are pinned (closed over above); the event sink and the - // authorize closure vary per step. - // - // The `warmCache` (design §3b) is the run-loop's per-deployment - // warm-agent cache, present only for the single-step long-lived - // deployment the deploy projection marked a warm candidate. When - // supplied, the adapter builds the agent once and reuses it across - // messages; when absent, it keeps instantiate-send-teardown per - // step. Forwarding it here is the only warm-keep wiring this binding - // needs -- the adapter and the run-loop own the rest of the - // lifecycle. - // Run-boundary durability flush (design §3c). When the deployment is - // warm-kept, mirror the warm agent's conversation snapshot to the - // workflow-run substrate after each message's send settles. The key - // is the step identity, the same key the env builder filed the - // durable store under, so the hook resolves the right per-agent - // store. Absent for a multi-step deploy (no durable registry). + // The operator gate at deploy time (capability walk approval) and this + // runtime grant check (per-step snapshot pushed over control IPC) are + // complementary: the walk bounds the toolset, the snapshot decides which + // of those the agent may invoke. A fresh invoker is built per invocation + // so the adapter subscribes to THIS step's onEvent; the warmCache, when + // supplied, is the only warm-keep wiring this binding needs. + // Mirrors the warm agent's conversation to the substrate after each + // send settles; absent for a multi-step deploy (no durable registry). const onRunBoundary: ((key: string) => Promise) | undefined = durableConversation !== undefined ? async (key: string) => { @@ -1593,11 +1460,8 @@ export function createSidecarSubstrateFactory( } : undefined; - // Connector-thread seed (design §3c). When the deployment is warm-kept, - // route each mail-derived inbound message onto the warm agent's - // connector thread before its send, so the reply path has thread state. - // The key is the step identity, the same key the durable store is filed - // under. Absent for a multi-step deploy (no durable registry). + // Routes each mail-derived inbound message onto the warm agent's + // connector thread before send, so the reply path has thread state. const seedInbound: ((key: string, message: InboundMessage) => Promise) | undefined = durableConversation !== undefined ? async (key: string, message: InboundMessage) => { From b5bb32a01c1b498d70b4bf7b0c00eab4da0e6317 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:22:59 -0700 Subject: [PATCH 11/31] chore(sidecar): comments say why, never what (part 10) --- .../sidecar/src/workflow-substrate-factory.ts | 151 +++--------------- 1 file changed, 26 insertions(+), 125 deletions(-) diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index 1383c51843..cd5da44ac2 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -1445,12 +1445,6 @@ export function createSidecarSubstrateFactory( onEvent, })(req); - // The operator gate at deploy time (capability walk approval) and this - // runtime grant check (per-step snapshot pushed over control IPC) are - // complementary: the walk bounds the toolset, the snapshot decides which - // of those the agent may invoke. A fresh invoker is built per invocation - // so the adapter subscribes to THIS step's onEvent; the warmCache, when - // supplied, is the only warm-keep wiring this binding needs. // Mirrors the warm agent's conversation to the substrate after each // send settles; absent for a multi-step deploy (no durable registry). const onRunBoundary: ((key: string) => Promise) | undefined = @@ -1469,19 +1463,8 @@ export function createSidecarSubstrateFactory( } : undefined; - // Connector reply drain (design §3c). When the deployment is - // warm-kept, drive the warm agent's outbound replies through the shared - // connector reply drain: on each `connector.reply` the agent emits, - // compose a threaded reply from the durable store's connector thread, - // send it through the outbound bridge (`bridge.submit`, the same signed- - // send path the agent's own supervisor-backed transport uses), then - // advance the thread from the receipt. Established once per warm agent - // over its lifetime stream; the returned drain handle carries the lifetime - // `done` promise the step-invoker folds into the warm entry so eviction - // drains it, plus the per-turn settle barrier the warm step gates each - // reply turn on. The key is the step identity, the same key the durable - // store is filed under. Absent for a multi-step deploy (no durable - // registry). + // Composes each connector.reply through the durable thread and sends it + // over the same signed-send path the agent's own transport uses. const driveReplies: | ((key: string, stream: AgentEventStream) => ConnectorReplyDrain) | undefined = @@ -1491,14 +1474,7 @@ export function createSidecarSubstrateFactory( stream, composeReply: () => durableConversation.get(key).composeReply(), send: (message) => env.outboundMailBridge.submit(env.spawn.mailboxAddress, message), - // Build the full RFC 5322 References chain from the deployment's - // committed mailbox: locate the parent by its Message-Id and - // return its own References plus its Message-Id, so a reply - // carries the whole conversational ancestry rather than a - // single element. The reader opens a fresh committed snapshot, - // so a parent committed just before this reply is visible. A - // parent miss (the first reply on a fresh thread, a malformed - // id) returns undefined and the transport derives [inReplyTo]. + // A parent miss returns undefined and the transport derives [inReplyTo]. resolveReferences: (inReplyTo) => resolveMailboxReferences(transportInbound.reader, inReplyTo), onReplySent: (receipt) => durableConversation.get(key).onReplySent(receipt), @@ -1516,11 +1492,7 @@ export function createSidecarSubstrateFactory( ) => createWorkflowStepInvoker({ workflowAuthorize: authorize, - // Combine the per-run credential wiring (the live material cell and - // the step-grants resolver, ridden in from the run child) with the - // sidecar-static provider registry, so `buildStepEnv` attaches a - // complete credential context and the agentFactory can assemble each - // bundle's consumer-scoped `credentials` capability. + // Combines the per-run credential wiring with the sidecar-static providers. buildEnv: (buildReq) => buildStepEnv(buildReq, sourcesRef, { materialCell: credentialWiring.materialRef, @@ -1538,29 +1510,11 @@ export function createSidecarSubstrateFactory( })(req); const evaluateGrantsAdapter: GrantEvaluator = async ({ resource, action, stepId, grants }) => { - // Merge the step's pinned-tool floor grants (derived and recorded - // by the env builder from the step's materialized factories) under - // the credentials snapshot's grants. The floor supplies the - // `tool:` authority a pinned tool never got from the hub's - // capability walk. It is ADDITIVE: `evaluateGrants` ranks by - // specificity then effect, so a declared `deny` (priority 2) still - // beats the derived `ask`/`allow` and an explicit denial is - // honored -- the floor only raises the minimum authority to the - // tool's static mark. - // - // A missing floor entry (`?? []`) contributes no rows: this can - // only ever fail MORE closed (a pinned tool the hub also did not - // grant stays denied, the pre-#68 behavior), never open a hole, so - // it is safe as an additive default. The floor is keyed by base - // step id, so a `map` iteration's scoped id resolves to its base - // step's floor. + // Additive: evaluateGrants ranks by specificity then effect, so a + // declared deny still beats the derived ask/allow floor. A missing + // entry (?? []) can only fail more closed, never open a hole. const floor = toolMarkFloorByStep.get(baseStepId(stepId)) ?? []; const result = await evaluateGrants( - // The credentialsSnapshot's grants are typed as - // `readonly unknown[]` so the workflow-host package does not - // depend on the sidecar's grant-rule grammar. The sidecar owns - // that grammar; the cast surfaces here at the boundary where - // the typed grant shape is known. // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- credentialsSnapshot.steps[*].grants is typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar [...(grants as readonly GrantRule[]), ...floor], resource, @@ -1580,57 +1534,27 @@ export function createSidecarSubstrateFactory( principal, scheduler, invokeStep: childInvokeStep, - // Plaintext body sources decrypted sidecar-side from the run record; each - // body path resolves its own table from here by definition id. bodySources: parseBodyInferenceSources(validated.WORKFLOW_BODY_SOURCES), dataDir: validated.SIDECAR_DATA_DIR, evaluateGrants: evaluateGrantsAdapter, - // Shared with the top level's `buildStepEnv`: a spawned child's build - // combines it with the run's live material and capped grants. credentialProviders, - // The shared closure the child re-walks to cap its inherited grants at - // its declared capabilities. Source-ref only, so always present here. + // Source-ref only, so always present here. closurePackageDir: env.spawn.closurePackageDir, }; - // Terminal childWorkflow executor. `run-child` builds the in-memory - // resolver from this plus the lifted-body map it extracts after loading - // the parent's re-verified definition, so an owned inline child spawns - // with no on-disk asset read. + // run-child builds the in-memory resolver from this plus the lifted-body + // map, so an owned inline child spawns with no on-disk asset read. const runChild = createSidecarRunChild(childRunDeps); - // An onTrigger section runs each event's body as a suspendable child. - // Source-ref is the only deploy lineage: `run-child` builds the in-memory - // body resolver from this raw executor plus the lifted-body map it extracts - // after re-evaluating the parent's closure, so a body resolves in-process - // with no on-disk read and no separate per-body re-verify (the parent's - // re-verify already covers every inline body). + // A body resolves in-process with no on-disk read and no separate + // per-body re-verify, since the parent's re-verify covers every inline body. const runSuspendableChild = createSidecarSpawnSuspendableChild(childRunDeps); - // Per-run scratch reclamation for the cold (multi-step) path. The - // run-loop fires this once each run reaches its terminal status; it - // drops the run's whole `workflow-step-state//runs//` - // subtree (every step/attempt the run produced), which nothing - // reopens after terminal (resume reads the substrate run log, not - // local step state). - // - // Parked-step safety: reclamation keys on the RUN's terminal status, - // and a step parked on a signal (`awaiting-signal`) keeps the run - // non-terminal, so this never fires while a suspended step's - // `attempt-N` store still holds a live pending-op the resume path must - // reopen. Any future per-STEP reclamation must preserve that invariant - // -- it MUST exclude an `awaiting-signal` step, whose `attempt-N` store - // is the exact store a later crash-resume reopens to rehydrate the - // gate; dropping it would reproduce the empty-store hang the - // resume-attempt recovery closes. - // - // Built only for the cold path: a warm deploy - // roots its single agent's scratch per agent under the disjoint - // `warm/` sub-root (reclaimed on undeploy), and the run-loop's own - // `warmKeep` gate already suppresses the per-run call there, so - // leaving this undefined for warm deploys keeps the path-owning - // module's intent explicit. `rm -rf` semantics via `recursive + - // force` so a run that never wrote scratch (no buildEnv reached) is - // a no-op rather than an ENOENT throw. + // Fires once a run reaches terminal, dropping the whole per-run subtree. + // Reclamation keys on the run's terminal status, and a signal-parked step + // keeps the run non-terminal, so this never fires while a suspended + // step's attempt-N store still holds a pending-op resume must reopen. + // Built only for the cold path: a warm deploy's scratch lives under the + // disjoint warm/ sub-root, reclaimed on undeploy instead. const cleanupRunStorage: ((runId: string) => Promise) | undefined = env.spawn.warmKeep ? undefined : (runId: string) => @@ -1643,12 +1567,9 @@ export function createSidecarSubstrateFactory( { recursive: true, force: true }, ); - // Recover a parked correlation's approval snapshot for the child's - // re-registration enumeration. Wired unconditionally (unlike - // `cleanupRunStorage`, which is cold-only): a warm agent parks on approval - // just as a cold one does, and the branch on `warmKeep` selects the durable - // read -- cold reads the per-attempt isogit store, warm reconstructs the - // agent's durable conversation state from the substrate. + // Wired unconditionally, unlike cleanupRunStorage: warm parks the same + // way cold does, just reconstructed from the substrate instead of the + // per-attempt isogit store. const loadParkedApproval: LoadParkedApproval = ({ runId, stepId, attempt, correlationId }) => env.spawn.warmKeep ? readWarmParkedApprovalSnapshot({ @@ -1666,14 +1587,8 @@ export function createSidecarSubstrateFactory( correlationId, }); - // Enumerate a crashed step's durable pending approval operations for the - // resume classifier, off the same cold/warm durable read as - // `loadParkedApproval`. Where that binding is a lookup by a known - // correlationId (answering the supervisor's re-registration), this is the - // enumeration the classifier needs when the correlationId never reached the - // log -- the crash-across-park case: read the pending operations, project - // to the minimal approval records the runtime reconstructs `SignalAwaited` - // from. + // Covers the crash-across-park case where the correlationId never + // reached the log: enumerates pending ops for the resume classifier. const readParkedApprovalOps: ReadParkedApprovalOps = async ({ runId, stepId, attempt }) => toParkedApprovalOps( env.spawn.warmKeep @@ -1700,11 +1615,8 @@ export function createSidecarSubstrateFactory( initialSources: stepInferenceSources, runChild, runSuspendableChild, - // A loop iteration runs under the inherited env (not `buildChildRunEnv`), - // so it is the one body birth path that writes no grants file of its own. - // Materialize it here -- capping the container run's grants to the loop - // body's declared resources -- so the body's childWorkflow grandchild - // spawn is authorized. `definition` is the PRE-rewrite loop body. + // A loop iteration runs under the inherited env, so it's the one body + // birth path that writes no grants file of its own; materialized here instead. materializeLoopIterationGrants: async ({ parentRunId, childRunId, definition }) => { await capAndPersistChildGrants({ deps: childRunDeps, @@ -1718,8 +1630,6 @@ export function createSidecarSubstrateFactory( evaluateGrants: evaluateGrantsAdapter, loadParkedApproval, readParkedApprovalOps, - // The same registry the warm agent's transport registers `watch` - // callbacks into; `runWorkflowChild` routes each `mailbox.notify` to it. mailboxWatchRegistry, ...(cleanupRunStorage !== undefined ? { cleanupRunStorage } : {}), }; @@ -1727,14 +1637,5 @@ export function createSidecarSubstrateFactory( }; } -/** - * Production substrate factory. The sidecar's - * `bin/workflow-child` binary calls - * `runWorkflowChildFromProcessEnv(createSubstrate, { substrateConfigKeys: SIDECAR_SUBSTRATE_CONFIG_KEYS })` - * and the helper invokes this factory with the parsed env. The - * factory is the default-deps variant of - * `createSidecarSubstrateFactory`; deployments that need a recording - * hub sink (tests, alternate hosts) construct their own via - * `createSidecarSubstrateFactory`. - */ +/** The default-deps variant; deployments needing a recording hub sink construct their own via createSidecarSubstrateFactory. */ export const createSubstrate: SubstrateFactory = createSidecarSubstrateFactory(); From 286de6329c4d2660476fdad73b863fd5ebfa1f86 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:23:50 -0700 Subject: [PATCH 12/31] chore(sidecar): comments say why, never what (part 11) --- apps/sidecar/src/workflow-run-record.ts | 133 ++++++------------------ 1 file changed, 31 insertions(+), 102 deletions(-) diff --git a/apps/sidecar/src/workflow-run-record.ts b/apps/sidecar/src/workflow-run-record.ts index 934be40f44..55e363e44e 100644 --- a/apps/sidecar/src/workflow-run-record.ts +++ b/apps/sidecar/src/workflow-run-record.ts @@ -1,21 +1,8 @@ -// Sidecar-local persistence of the per-run record needed to re-establish a -// workflow run across a sidecar PROCESS restart. The record is co-located -// with the run's workflow-run substrate at -// `${dataDir}/workflow-runs//deployment.json`, so a single -// teardown reclaims both and a boot scan can enumerate the active runs -// beside the run state they resume. -// -// It carries the inputs that are otherwise frame/in-memory only: `sources` -// (each step's ordered inference-source failover chain, threaded to the child -// via the spawn env and durable nowhere else), `sessionId` (inference-event -// correlation), `hubPublicKey` (the head's deploy-pack / inbound verification -// key), `approvedWireHash` (the hub-approved wire hash the deploy frame -// carried, so a restore re-spawn feeds the child the same re-verify anchor -// without recomputing it), `lineage`, and -- for a source-ref deployment -- -// the `sourceRef` pin (source + frozen closure) a restore needs to -// re-materialize and re-evaluate the pinned code. The definition itself is -// re-materialized from that `sourceRef` closure on restore, and each step's -// grants live in its agent-state repo, so neither is duplicated here. +// Sidecar-local persistence to re-establish a workflow run across a sidecar +// process restart, co-located with the run's substrate so one teardown +// reclaims both. Carries the inputs that are otherwise frame/in-memory only +// (sources, sessionId, hubPublicKey, approvedWireHash, sourceRef); the +// definition and each step's grants are re-materialized elsewhere on restore. import { mkdir, readdir, readFile, rm } from "node:fs/promises"; import { dirname, join as pathJoin } from "node:path"; @@ -40,71 +27,43 @@ function isENOENT(cause: unknown): boolean { ); } -// The base fields every run record carries, spread into the full schema below. -// `version` is 2 for the unified credential format: `sources`/`bodySources` are -// non-secret config carrying a `credentialId` per source, and the run's secret -// material lives once in `credentials` (the delivery cell) with each material's -// secret sealed under the sidecar cipher. A pre-unification record (inline -// per-source secrets) does not satisfy this schema and is soft-skipped at the -// scan boundary; the hub re-pushes its deployment on reconnect. +// version 2 unifies credentials: sources/bodySources reference a +// credentialId, and secrets live once in the credentials cell, sealed under +// the sidecar cipher. A pre-unification record is soft-skipped at scan; the +// hub re-pushes its deployment on reconnect. const workflowRunRecordBase = { version: "1 | 2", agentAddress: "string > 0", definitionId: "string > 0", - // Top-level per-step inference sources. Non-secret config only: each source - // carries a `credentialId` referencing an entry in `credentials`, no inline - // secret, so this rides in the clear. + // Non-secret: each source carries a credentialId into credentials. sources: { "[string]": InferenceSource.array().atLeastLength(1), }, - // Per spawned-body inference-source config, keyed by the body's definition id - // (an onTrigger section or a childWorkflow child; a nested loop body folds its - // step sources into its container's table, so it is not a separate entry). - // Non-secret, like `sources`: each carries a `credentialId` into `credentials`. - // Optional: a deployment with no bodies carries none. + // Keyed by spawned-body definition id; a nested loop body folds into its + // container's table instead of a separate entry. "bodySources?": { "[string]": { "[string]": InferenceSource.array().atLeastLength(1), }, }, - // The run's credential-material cell: the ONE at-rest home for every secret, - // inference and tool alike. `bindings` and each material's `credentialId`/ - // `providerKey`/`origin` ride in the clear; only each material's `secret` is - // sealed under the sidecar cipher (see `transformDeliveryMaterials`). Optional: - // a deployment that binds no credentials and whose sources need none carries - // none. Restored + re-delivered to the child on the pre-trigger barrier. + // The one at-rest home for every secret; only each material's secret is + // sealed under the sidecar cipher (see transformDeliveryMaterials). "credentials?": CredentialDelivery, "sessionId?": "string > 0", "hubPublicKey?": "string > 0", } as const; /** - * The on-disk deployment record. `version` guards the schema shape so a future - * reader can reject or migrate a stale record rather than parse it blindly. - * Validated at read time (the boot scan) at the trust boundary. - * - * Source-ref is the only deploy lineage, so the record REQUIRES a `sourceRef` - * pin (its co-required source + frozen closure, so a restore can re-materialize - * the pinned closure) AND `approvedWireHash` (so the restored child re-verifies - * the evaluated closure against the hub-approved pin rather than against a - * sidecar recompute of the inert projection -- the latter would collapse the - * out-of-band-pin property the barrier exists for). A record missing either -- - * including a legacy live-authored record written before this collapse -- fails - * validation at the scan boundary and is soft-skipped as corruption, so the - * restore loop needs no bespoke source-ref guard. + * Requires sourceRef and approvedWireHash so the restored child re-verifies + * against the hub-approved pin rather than a sidecar recompute — a record + * missing either (including a legacy live-authored one) soft-skips as + * corruption at scan, needing no bespoke source-ref guard. */ export const WorkflowRunRecord = type({ ...workflowRunRecordBase, lineage: "'source-ref'", - // The hub-approved wire hash the restored child re-verifies the evaluated - // closure against. approvedWireHash: "string > 0", - // The source-ref pin a restore re-runs `applyFrozenWorkflowClosure` with: - // `source` names the registry the definition package is published to (its - // token is resolved from the sidecar's env at apply time, so no secret is - // persisted here) and `closure` is the frozen dependency tree (concrete - // versions + integrity SRIs) -- plain strings, no secrets. Both rode the - // signed frame and co-travel (see `SourceRefPin`). + // Re-run through applyFrozenWorkflowClosure on restore; plain strings, no secrets. sourceRef: SourceRefPin, }); export type WorkflowRunRecord = typeof WorkflowRunRecord.infer; @@ -113,19 +72,13 @@ function recordPath(dataDir: string, runId: string): string { return pathJoin(dataDir, "workflow-runs", runId, RECORD_FILENAME); } -// The AAD column binding a credential's sealed secret to its id, so a ciphertext -// cannot be swapped between credentials (or between runs) and still decrypt. +// Binds a sealed secret to its id, so a ciphertext cannot be swapped between credentials or runs and still decrypt. function credentialSecretColumn(credentialId: string): string { return `credential:${credentialId}:secret`; } -// Transform each credential material's `secret` in the delivery under -// `transform` (encrypt on write, decrypt on scan), binding each to its run and -// credential. The non-secret fields -- each material's `credentialId`/ -// `providerKey`/`origin` and the whole `bindings` array -- ride in the clear. -// Returns a fresh delivery so the caller's in-memory record is not mutated. A -// decrypt failure (a rotated/wrong key or a tampered blob) throws, which the -// scan caller treats as record corruption and soft-skips the whole run. +// Returns a fresh delivery; a decrypt failure throws and the scan caller +// treats it as corruption, soft-skipping the whole run. async function transformDeliveryMaterials( delivery: CredentialDelivery, runId: string, @@ -146,14 +99,9 @@ async function transformDeliveryMaterials( } /** - * Persist a run record. Written after the run's slug is claimed and before - * the child is spawned, so a crash mid-spawn leaves a record the boot scan - * re-drives. Idempotent: it overwrites any existing record for the same run. - * - * Every credential secret in the run's `credentials` cell -- inference and tool - * alike -- is sealed under the sidecar `cipher` before it touches disk, so the - * record carries ciphertext (version 2). The `sources`/`bodySources` config is - * non-secret (each references a credential by id) and rides in the clear. + * Written after the run's slug is claimed and before the child spawns, so a + * crash mid-spawn leaves a record the boot scan re-drives. Idempotent. + * Every credential secret is sealed under the sidecar cipher before disk. */ export async function writeWorkflowRunRecord( dataDir: string, @@ -174,13 +122,8 @@ export async function writeWorkflowRunRecord( } : {}), }; - // Atomic + durable: this is the sole restore source for the run's credential - // material, and a rotation overwrites the existing record in place, so an - // interrupted write must never expose a torn record the boot scan would then - // skip. Owner-only (0o600): the sealed secrets are ciphertext, but the record - // still names each source's provider/baseURL and the deployment's identity, so - // it stays off a shared host's world-readable set, matching the private-key - // writes elsewhere on the sidecar. + // Atomic + durable since this is the sole restore source; owner-only since + // the record still names each source's provider/baseURL in the clear. await writeFileAtomicDurable(path, JSON.stringify(sealed, null, 2), { mode: 0o600, }); @@ -203,16 +146,8 @@ export interface ScannedWorkflowRun { } /** - * Enumerate the persisted run records under `workflow-runs/` so a boot-time - * restore can re-establish each run. Soft-fails per record: a missing - * `deployment.json`, unparseable JSON, or a record that fails schema - * validation is logged and skipped rather than wedging the whole boot -- one - * corrupt record must not strand every other run. An absent `workflow-runs/` - * directory is the legitimate first-boot case and yields an empty list, not - * an error. - * - * The returned `runId` is the directory name; the caller cross-checks it - * against the record's own address before trusting it. + * Soft-fails per record so one corrupt record doesn't strand every other + * run; an absent workflow-runs/ directory is the legitimate first-boot case. */ export async function scanWorkflowRunRecords( dataDir: string, @@ -261,14 +196,8 @@ export async function scanWorkflowRunRecords( logger.warn`skipping workflow-runs/${runId}: ${RECORD_FILENAME} failed validation: ${record.summary}`; continue; } - // A version-2 record seals every credential secret in its `credentials` - // cell; unseal them under the sidecar cipher for the restored run. A decrypt - // failure -- a rotated/wrong key or a tampered blob -- is treated as - // corruption: log and soft-skip the WHOLE run so one undecryptable run does - // not wedge the boot scan (the deployment is left unrestored, its - // tools/inference dead until the hub re-pushes, the same baseline as any - // other corrupt record). A record with no `credentials` cell (a deployment - // that binds none) needs no unseal. + // A decrypt failure is treated as corruption: soft-skip the whole run + // rather than wedge the boot scan. let restored = record; if (record.version === 2 && record.credentials !== undefined) { try { From 97934eda521dd397f26439f838b965091127236b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:24:34 -0700 Subject: [PATCH 13/31] chore(sidecar): comments say why, never what (part 12) --- apps/sidecar/src/child-grant-filter.ts | 59 ++++---------------------- apps/sidecar/src/config.ts | 43 +++---------------- 2 files changed, 16 insertions(+), 86 deletions(-) diff --git a/apps/sidecar/src/child-grant-filter.ts b/apps/sidecar/src/child-grant-filter.ts index 13440ac464..6831d2457f 100644 --- a/apps/sidecar/src/child-grant-filter.ts +++ b/apps/sidecar/src/child-grant-filter.ts @@ -1,32 +1,13 @@ -// Cap a spawned child's inherited grants at the capabilities the child body -// itself declares. -// -// A spawned child runs under the authority of the run that spawned it, but it -// must never hold MORE than the child body's own definition declares. Without -// this cap a child authorizes against the full parent grant set, so a -// parent-held `effect:`/`credential:` grant (neither is tool-gated) or a -// bare-name `tool:` collision the child body never declared would authorize a -// capability the child was never written to use. The parent stays the ceiling -// (a child can never exceed it); the child body's declared capability set is -// the shape it is narrowed to -- the same "a run's authority is bounded by its -// declared capabilities" model the top level enforces through its own walk. +// Caps a spawned child's inherited grants at what the child body itself +// declares, so a parent-held effect:/credential: grant or a bare-name tool: +// collision never authorizes a capability the child was never written to use. import type { DirectorRegistry } from "@intx/agent"; import { matchPattern } from "@intx/authz"; import type { WorkflowDefinition } from "@intx/workflow"; import { walkCapabilities, type PluginToolDefinitions } from "@intx/workflow-deploy"; -/** - * The flat union of every grant-shape resource string the child body declares, - * across all its steps AND every inline nested body (loop, onTrigger, - * childWorkflow) the capability walk folds into its enclosing step. - * - * Pass the child's PRE-rewrite definition -- whose grandchildren are still - * inline. The walk skips a `{ ref }` body, so a rewritten definition (children - * lifted to refs) would omit a grandchild's declared resources; the union must - * include them, because the caller uses this set to cap the grants it persists - * as the ceiling a grandchild spawn filters against in turn. - */ +/** Pass the pre-rewrite definition: the walk skips a { ref } body, so a rewritten one would drop grandchild resources. */ export function collectDeclaredResources( definition: WorkflowDefinition, directors: DirectorRegistry, @@ -43,33 +24,11 @@ export function collectDeclaredResources( } /** - * Filter a parent's grant rules down to what a spawned child body declares. - * - * The result caps the child at the parent: it only ever REMOVES parent rules, - * never adds or widens one. The filter errs toward keeping, because dropping is - * the only unsafe direction -- - * - * - Every `deny` and `ask` rule is kept unconditionally. Both only ever - * restrict, so dropping one WEAKENS safety. An `ask` floor in particular is - * load-bearing: `evaluateGrants` ranks `ask` above `allow` at equal - * specificity so a workflow cannot declare its way under an approval gate, - * and filtering the `ask` out would punch straight through that gate. - * - An `allow` rule is kept only when its resource PATTERN covers at least one - * resource the child body declares (via `matchPattern`, so a wildcard grant - * like `tool:*` survives when the child declares any matching `tool:`). An - * `allow` for a resource the child never declares -- a parent-only - * `effect:`/`credential:`/`tool:` -- is dropped: that is the escalation this - * cap closes. - * - Any entry that is not an `allow` rule with a string `resource` is kept. A - * non-rule entry is inert at evaluation (`evaluateGrants` reads `.resource`), - * so keeping it never widens authority, and keeping-on-doubt can never - * wrongly drop a restriction. - * - * Action is deliberately not part of the coverage test. The declared set is - * resource strings; a parent rule whose action never matches a child query is - * inert (`evaluateGrants` re-gates on action at decision time), so testing the - * resource alone can only OVER-keep an inert `allow`, never wrongly drop one - * the child legitimately needs. + * Only ever removes parent rules, erring toward keeping since dropping is the + * only unsafe direction: deny/ask are kept unconditionally (dropping an ask + * floor would punch through an approval gate), an allow is kept only if its + * pattern covers a declared resource, and action is deliberately not part of + * the coverage test since evaluateGrants re-gates on it at decision time. */ export function filterGrantsToDeclaredResources( parentGrants: readonly unknown[], diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index bf3b4651f0..e767d90351 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -1,27 +1,13 @@ -// Boundary readers for the sidecar's env-config inputs. -// -// Centralizing the readers here keeps env-validation rules in one -// place — `SIDECAR_CACHE_MAX_BYTES` must be a positive finite -// number, with a 10 GiB default — even though the sidecar only -// invokes the reader at one site today (the orchestrator boot in -// `apps/sidecar/src/index.ts`). The harness builder receives the -// resolved value through `DefaultHarnessBuilderConfig` rather than -// re-reading env, so the boundary stays at the boot edge. +// Boundary readers for the sidecar's env-config inputs, centralized so +// validation rules live in one place at the boot edge. import { AdapterManifest } from "@intx/inference"; import { hexDecode } from "@intx/types"; const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; -// The operator key the sidecar seals its at-rest credential material under -// (inference-source apiKeys, and the tool credential material store). A -// SEPARATE key from the hub's `CREDENTIAL_ENCRYPTION_KEY`: the sidecar seals a -// local store on a host it does not control as tightly, so a sidecar -// disk-plus-key compromise must not also decrypt the hub's credential -// database, and the two keys rotate independently. Required at boot: a missing -// key fails loudly here rather than letting the sidecar run and persist secrets -// it cannot protect. 32 bytes, hex -- e.g. `openssl rand -hex 32`, the same -// shape as the hub's key. +// Separate from the hub's CREDENTIAL_ENCRYPTION_KEY, so a sidecar +// disk-plus-key compromise can't also decrypt the hub's database. export function readCredentialEncryptionKey(): Uint8Array { const raw = process.env["SIDECAR_CREDENTIAL_ENCRYPTION_KEY"]; if (raw === undefined || raw.trim() === "") { @@ -40,11 +26,7 @@ export function readCacheMaxBytes(): number { return n; } -// Mirrors the hub's `DEFAULT_HUB_MAX_TARBALL_BYTES`. The sidecar's -// HTTP-registry fetcher enforces this cap on every upstream registry -// tarball pull. An operator pointing the sidecar at a third-party -// registry whose curated tarballs run larger should raise the cap -// explicitly via `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`. +// Mirrors the hub's DEFAULT_HUB_MAX_TARBALL_BYTES. const DEFAULT_REGISTRY_MAX_TARBALL_BYTES = 10 * 1024 * 1024; export function readRegistryMaxTarballBytes(): number { @@ -57,19 +39,8 @@ export function readRegistryMaxTarballBytes(): number { return n; } -// Operator-configured custom inference adapter manifest. The value is -// TRUSTED operator input read only from this process's environment; -// `import(specifier)` is arbitrary code execution, so a specifier must -// never originate from deploy or tenant data — the agent deploy tree -// carries only a `provider` key, never a specifier. The shape is -// arktype-validated here (and re-validated at the workflow-child spawn -// boundary as defense in depth), but the value itself is trusted. -// -// Unset or whitespace-only means "no custom adapters", a valid -// configuration — the sidecar then resolves only the statically-linked -// built-ins. A present-but-malformed value fails loud at boot: an -// opaque `JSON.parse` SyntaxError is rethrown naming the env key, and -// the parsed value is asserted against `AdapterManifest`. +// Trusted operator input only: import(specifier) is arbitrary code +// execution, so a specifier must never originate from deploy or tenant data. export function readAdapterManifest(): AdapterManifest { const raw = process.env["SIDECAR_ADAPTER_MANIFEST"]; if (raw === undefined || raw.trim() === "") return []; From 18e01441643b3ffe7b2916c165b652e5bddada27 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:29:26 -0700 Subject: [PATCH 14/31] chore(sidecar): comments say why, never what (part 13) --- apps/sidecar/src/index.ts | 272 +++++++------------------------------- 1 file changed, 50 insertions(+), 222 deletions(-) diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 8ba4b9cd35..82708336c9 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -84,45 +84,15 @@ const cacheRoot = const cacheMaxBytes = readCacheMaxBytes(); const registryMaxTarballBytes = readRegistryMaxTarballBytes(); -// Operator-configured custom inference adapters, resolved once at the -// boot edge. `loadAdapterRegistry` merges the statically-linked -// built-ins with any custom adapters the manifest names, importing each -// custom module eagerly here so a bad specifier fails the sidecar at -// boot rather than at first inference. The SAME registry backs the -// deploy router's source-admission check below; the workflow child -// cannot receive this object across the fork, so the validated -// manifest is serialized into the child's spawn env (see -// `multistepSubstrateEnv`) and the child rebuilds an equivalent -// registry from it. Specifiers are operator-config-only — the agent -// deploy tree never contributes one. +// Custom modules import eagerly here so a bad specifier fails at boot, +// not first inference. The child rebuilds an equivalent registry from +// the serialized manifest since it can't receive this object across the fork. const adapterManifest = readAdapterManifest(); const adapters = await loadAdapterRegistry(adapterManifest); -// Phase 4.7 latency-gate hook. When `SIDECAR_LATENCY_BENCH_FILE` names a -// path, the deploy router wires the supervisor's `onDispatchTiming` -// observer to append a parseable line per per-message dispatch boundary -// to that file, which the benchmark harness reads after the run. A file -// (rather than stdout) is the channel because the spawn fixture caps its -// stdout drain buffer, and a few-hundred-message run emits more lines -// than that cap holds. Resolved here at the boot edge (the one layer -// that reads env) so no non-boundary site re-decides what the absent -// value means; an unset path leaves the observer unwired and the -// supervisor's dispatch path untouched. This is observability-only -- no -// control-flow effect -- and is the single benchmark-side instrumentation -// hook the 4.7 gate adds. The synchronous append (not the logger) is the -// raw measurement channel: it is not application logging and must not be -// formatted or level-gated. A failed append surfaces (it is not -// swallowed) so a broken benchmark channel does not silently yield an -// empty result set. -// Two line shapes share the channel, discriminated by `mark.kind`. Both -// lead with the per-message id (the mail's Message-ID), the key the -// per-message fits group on: -// roundtrip: ` ` (4.7 latency gate) -// leg: ` leg [runsFanOut consumedFanOut looseObjects gitBytes]` -// (D2 per-leg attribution; the trailing four counters are -// present only on the `end` phase). The leg shape is a -// strict superset prefixed with the literal `leg` token, so -// a reader can split on whitespace and branch on field 2. +// A file, not stdout, since the spawn fixture caps its stdout drain buffer +// below what a few-hundred-message run emits. Observability-only; a failed +// append surfaces rather than silently yielding an empty result set. const latencyBenchFile = process.env["SIDECAR_LATENCY_BENCH_FILE"]; const onDispatchTiming: ((mark: DispatchTimingMark) => void) | undefined = latencyBenchFile !== undefined && latencyBenchFile.trim() !== "" @@ -141,14 +111,8 @@ const onDispatchTiming: ((mark: DispatchTimingMark) => void) | undefined = } : undefined; -// D2 §10c forced-repack A/B toggle. When `SIDECAR_REPACK_EVERY_MESSAGES` -// names a positive integer, the supervisor forces a `git gc`/repack of -// the workflow-run repo every Nth dispatched message (under the -// single-writer discipline). Resolved here at the boot edge alongside the -// timing channel; absent or non-positive => the supervisor forks no -// `git gc` and the dispatch path is untouched. Measurement-only: this -// exists to discriminate pack-growth from tree-fan-out, never to run in -// production. +// Measurement-only: forces a repack every Nth message to discriminate +// pack-growth from tree-fan-out, never to run in production. const repackEveryRaw = process.env["SIDECAR_REPACK_EVERY_MESSAGES"]; let repackEveryMessages: { everyMessages: number } | undefined; if (repackEveryRaw !== undefined && repackEveryRaw.trim() !== "") { @@ -161,15 +125,8 @@ if (repackEveryRaw !== undefined && repackEveryRaw.trim() !== "") { repackEveryMessages = { everyMessages: parsed }; } -// Consumed-dedup retention horizon (ms). Resolved here at the boot edge -// -- the single layer that owns operator config -- and threaded into -// every supervisor the deploy router constructs. Absent or empty => -// the supervisor applies its 24h default. This is the OPERATOR-policy -// value: the longest window in which the same message could -// legitimately be re-submitted and still must be caught as a duplicate -// (the consumed/ dedup index retains at least this long before the -// retention watermark prunes it). It must be >= the maximum redelivery -// window of any at-least-once mail source if one is ever added. +// The longest window a re-submitted message must still be caught as a +// duplicate; must be >= the max redelivery window of any at-least-once source. const consumedRetentionRaw = process.env["CONSUMED_RETENTION_MS"]; let consumedRetentionMs: number | undefined; if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { @@ -182,11 +139,8 @@ if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { consumedRetentionMs = parsed; } -// Bound on the child's spawn-time `ready` handshake. Threaded to every -// per-deployment supervisor; on expiry the supervisor kills the child and -// rejects the spawn, so a child that spawns but never signals ready fails -// the deploy (or is skipped by boot-time restore) instead of hanging it. -// Absent, the supervisor applies its 30s default. +// On expiry the supervisor kills the child and rejects the spawn, so a +// child that never signals ready fails the deploy instead of hanging it. const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; let readyTimeoutMs: number | undefined; if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { @@ -199,13 +153,8 @@ if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { readyTimeoutMs = parsed; } -// Hub-link reconnect backoff (ms). Resolved here at the boot edge -- the -// single layer that owns operator config -- and forwarded to the -// orchestrator's hub link. Absent or empty => the link's 3s -// `DEFAULT_RECONNECT_DELAY_MS`. Production never sets this; the deploy-flow -// test harness sets a short value so reconnect-survival tests that do not -// assert the delay itself (they assert the reconnect's recovery semantics, -// not the backoff duration) do not burn 3s of wall clock per dropped link. +// Production never sets this; test harnesses set a short value so +// reconnect tests don't burn 3s of wall clock per dropped link. const reconnectDelayRaw = process.env["SIDECAR_RECONNECT_DELAY_MS"]; let reconnectDelayMs: number | undefined; if (reconnectDelayRaw !== undefined && reconnectDelayRaw.trim() !== "") { @@ -218,44 +167,29 @@ if (reconnectDelayRaw !== undefined && reconnectDelayRaw.trim() !== "") { reconnectDelayMs = parsed; } -// Sweep any tmp staging directories left behind by a `put` or -// `extractTarball` that crashed between staging and the final rename -// on a previous boot. Running here, before the orchestrator starts -// accepting apply work, keeps the cache root from accumulating -// orphans for the lifetime of the sidecar's data directory. +// Sweeps orphaned tmp staging dirs left by a crash between staging and +// rename on a previous boot. await createTarballCache({ rootDir: cacheRoot, maxBytes: cacheMaxBytes, }).sweepOrphans(); -// Load or mint the sidecar's local Ed25519 keypair. The supervisor -// principal signs every workflow-run commit with this key; the -// substrate's `signingCallback` signs every SSH-signed commit with -// it; the workflow-host child's substrate factory re-uses it via the -// `SIDECAR_SIGNING_*` spawn-time env vars. One key, one identity for -// the sidecar process. +// One key, one identity: signs every workflow-run commit and every +// SSH-signed commit, and is re-used by the child via spawn-time env vars. const SIDECAR_SIGNING_DIR = path.join(dataDir, ".sidecar-signing"); const sidecarSigningKey = await loadOrMintSidecarKeypair(SIDECAR_SIGNING_DIR); -// Construct the substrate-backed RepoStore at the boot edge. The -// supervisor consumes this through the deploy router; the supervisor's -// conversation-state writes reach `writeTreePreservingPrefix` on this -// store. The boot-edge facade below wraps the store so a successful -// workflow-run write fires the pack push hook before its Promise -// resolves. +// Wrapped below by the boot-edge facade so a successful workflow-run +// write fires the pack push hook before its Promise resolves. const agentRepoStore = createAgentRepoStore({ dataDir, signingKey: sidecarSigningKey, }); -// Cache of the hub-vouched public keys of senders this sidecar's deployments -// are authorized to receive mail from. The grants handler writes each key the -// hub co-delivers on a `run.grants` frame; the recipient's inbound-mail verify -// reads it back. The keyring is loaded here at boot so a restart keeps every -// previously-cached key. The durable-write primitive is injected so the cache -// write is atomic and fsynced -- at least as durable as the run-grants write it -// gates -- while the cache itself stays in @intx/hub-agent. +// Loaded here at boot so a restart keeps every previously-cached key; the +// durable-write primitive is injected so the write stays at least as +// durable as the run-grants write it gates. const senderKeyCache = await createSenderKeyCache({ dataDir, writeFileDurable: (filePath, contents) => @@ -263,74 +197,33 @@ const senderKeyCache = await createSenderKeyCache({ removeFileDurable: (filePath) => removeFileAtomicDurable(filePath), }); -// The read side of the same cache: the inbound signature verify resolves a -// sender address to the crypto that verifies its mail. Built here at the edge -// so the hub link stays source-opaque -- it resolves address -> crypto without -// holding the cache or knowing where the key came from. +// Built here so the hub link stays source-opaque, resolving address -> crypto +// without holding the cache or knowing where the key came from. const resolveSenderCrypto = createSenderCryptoResolver(senderKeyCache); -// Per-recipient-address registry of resolved inbound-mail admission policies. -// The workflow-host wiring resolves each hydrated deployment's authored -// `inboundMailPolicy` into this registry beside its mail-router registration -// and removes it on teardown; the read side below hands the hub-link's -// `mail.inbound` seam a total policy for every address. Built here at the edge -// so the hub link stays policy-source-opaque, mirroring `resolveSenderCrypto`. +// Mirrors resolveSenderCrypto's opacity for inbound-mail admission policies. const inboundMailPolicyRegistry = createInboundMailPolicyRegistry(); const lookupInboundMailPolicy = createInboundMailPolicyLookup(inboundMailPolicyRegistry); -// The deploy router records `(runId -> agentAddress)` here on -// every inbound `agent.deploy`; the facade resolves the mapping when -// firing the pack push so the outbound frames carry the right -// agentAddress for hub-side routing. +// The facade resolves this mapping when firing the pack push so outbound +// frames carry the right agentAddress for hub-side routing. const deploymentAddressRegistry = createDeploymentAddressRegistry(); -// Per-deployment-address mail handler registry the hub-link consults on -// every inbound `mail.inbound` frame. The deploy router's multi-step -// branch registers `wired.routeInbound` against the deployment's mail -// address once its supervisor spawns; the hub-link's `mail.inbound` -// handler calls `tryRoute` so an inbound deployment-address message -// lands on the supervisor's mail-bus subscription. Mail for an address -// with no registered handler has no receiver and is logged-and-dropped. +// Mail for an address with no registered handler is logged-and-dropped. const multistepMailRouter = createMultistepMailRouter(); -// Per-deployment-address signal handler registry the hub-link consults -// on every inbound `signal.deliver` frame. The multi-step deploy -// router registers a handler against the deployment's mail address -// once its supervisor spawns; the handler forwards the signal into the -// supervisor's `deliverSignal`, which sends a `signal.deliver` control -// IPC frame to the workflow-process child. The child commits the -// resulting `SignalReceived` event through its own substrate -- the -// single writer of the workflow-run repo on the sidecar side. +// The child commits the resulting SignalReceived through its own +// substrate, the single writer of the workflow-run repo sidecar-side. const multistepSignalRouter = createMultistepSignalRouter(); -// Per-deployment-address drain handler registry the hub-link consults -// on every inbound `drain.deliver` frame. The multi-step deploy router -// registers a handler against the deployment's mail address once its -// supervisor spawns; the handler forwards the drain into the -// supervisor's `drain`, which sends a `drain` control IPC frame to the -// workflow-process child and arms one drainTimeout accumulator per -// in-flight run. Cancel-mode in-flight steps abort on the child side; -// wait-mode steps continue. Accumulators commit a signed -// `CancelRequested{origin: "supervisor-drain"}` against the -// workflow-run repo when the deadline expires. +// Cancel-mode in-flight steps abort child-side; wait-mode steps continue. const multistepDrainRouter = createMultistepDrainRouter(); -// Per-deployment-address grants handler registry the hub-link consults -// on every inbound `run.grants` frame. The deploy router registers a -// handler against the deployment's mail address once its supervisor -// spawns; the handler writes the run's grants to `runs//grants.json` -// inside the deployment's workflow-run repo, sibling to that run's -// `runs//events/` subtree. The write is awaited so the frame's -// FIFO completion means the grants are durable on disk. +// The write is awaited so the frame's FIFO completion means grants are durable. const multistepGrantsRouter = createMultistepGrantsRouter(); -// Per-deployment-address sources-rotation handler registry. Only a -// single-step warm deployment registers a handler once its supervisor -// spawns; the handler forwards the rotated list into the supervisor's -// `deliverSources`, which sends a `sources-updated` control IPC frame to -// the child, where the warm agent's live sources are swapped in place. A -// multi-step deployment registers none, so a rotation resolved against -// its address is unrouted. +// Only a single-step warm deployment registers a handler; a multi-step +// deployment registers none, so a rotation against its address is unrouted. const multistepSourcesRouter = createMultistepSourcesRouter(); // Per-deployment credential-delivery handler registry. A deployment registers // its handler after `spawn` (any deployment with a supervisor, not only warm @@ -340,11 +233,7 @@ const multistepCredentialsRouter = createMultistepCredentialsRouter(); const transport = createInMemoryTransport(); -// The pack-push client closes over the substrate (for `createPack`) -// and a lazy hub-link binding (for `pushWorkflowRunPack`). The link -// reference is set once the orchestrator is constructed below; the -// closure here is consulted lazily because the -// `createSidecarOrchestrator` factory calls `createDeployRouter` +// Consulted lazily since createSidecarOrchestrator calls createDeployRouter // during its constructor, before the orchestrator handle is bound. let resolvedHubLink: HubLink | null = null; const workflowRunPackClient = createWorkflowRunPackClient({ @@ -368,11 +257,7 @@ const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ markRestored: workflowRunPackClient.markRestored, }); -// Wrap the substrate's RepoStore with the boot-edge facade so a -// successful supervisor write against a workflow-run repo fires the -// pack push hook before its Promise resolves. Non-workflow-run writes -// (today, the agent-state deploy-applier path) flow through -// unchanged. +// Non-workflow-run writes (today, the agent-state deploy-applier path) flow through unchanged. const wrappedRepoStore = createWorkflowRunPackPushingRepoStore({ underlying: agentRepoStore.repoStore, packClient: workflowRunPackClient, @@ -383,20 +268,9 @@ const hubWsUrl = requireEnv("HUB_WS_URL"); const sidecarId = requireEnv("SIDECAR_ID"); const sidecarToken = requireEnv("SIDECAR_TOKEN"); -// Multi-step substrate-config the deploy router threads into the -// workflow-process child's spawn-time env. The child's substrate -// factory consumes these via the typed `SubstrateConfig` validator so -// the per-step pack-push wrap can identify the deployment's hub-side -// trust anchors. Today the IPC bridge in `pack.push.request` carries -// the pack; the WebSocket-connection keys are reserved for a future -// child-local hub link without redoing the boot-edge wiring. -// -// `PATH`, `HOME`, and `TMPDIR` are propagated from the boot edge's own -// environment so the child's `#!/usr/bin/env bun` shebang can resolve -// `bun`, agent code can find a writable home, and tmp-file APIs land -// on the same temp root the host uses. The substrate-config validator -// ignores undeclared keys, so these are visible to the OS for binary -// lookup but invisible to the typed `SubstrateConfig` shape. +// PATH/HOME/TMPDIR propagate from the boot edge so the child's shebang can +// resolve bun and tmp-file APIs land on the host's temp root; the +// SubstrateConfig validator ignores these as undeclared keys. const multistepSubstrateEnv: Record = { SIDECAR_DATA_DIR: dataDir, SIDECAR_SIGNING_PUBLIC_KEY: hexEncode(sidecarSigningKey.publicKey), @@ -405,20 +279,10 @@ const multistepSubstrateEnv: Record = { SIDECAR_ID: sidecarId, SIDECAR_TOKEN: sidecarToken, PATH: requireEnv("PATH"), - // Tool-loader caps the child's per-step tool materialization uses. - // Resolved once at the boot edge and threaded into the child through - // the substrate config so the child does not re-read env at a - // non-boundary site. SIDECAR_CACHE_MAX_BYTES: String(cacheMaxBytes), SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(registryMaxTarballBytes), - // Serialize the parent's ALREADY-VALIDATED manifest (the object - // `readAdapterManifest` returned), not the raw env string, so the - // child rebuilds the same custom-adapter set. Always present (defaults - // to "[]" when no custom adapters are configured); the child treats a - // missing key as a serialization bug and fails loud. The child - // re-validates the shape before importing any module — defense in - // depth at the deserialization boundary, since the child env is - // operator-controlled via Bun.spawn. + // The already-validated manifest, not the raw env string, so the child + // rebuilds the same set; the child re-validates before importing any module. SIDECAR_ADAPTER_MANIFEST: JSON.stringify(adapterManifest), }; const hostHome = process.env["HOME"]; @@ -435,19 +299,9 @@ if (hostTmpdir !== undefined) { // than a second copy of the check. const buildHarness = createDefaultHarnessBuilder({ adapters }); -// Airlocked workflow-probe executor, assembled at the boot edge and -// injected through the orchestrator so the live sidecar answers -// `workflow.probe.request` with a real inert projection instead of the -// hub-link's rejecting placeholder. The host-side `MaterializeWorkflowClosure` -// lays out a probe frame's frozen closure under a per-probe scratch dir -// (rooted here so it shares the sidecar data dir's lifecycle); the executor -// spawns the one-shot child that evaluates the workflow entry against it. -// The cache/registry caps are the same boot-edge-resolved values the tool -// loader uses. -// The probe delivers source assets inline in one frame (see -// `WorkflowProbeRequestFrame`). A git-sourced asset that grows past the shared -// inline-payload cap is the signal to move the probe's asset delivery to the -// streamed transfer the deploy path uses. +// Answers workflow.probe.request with a real inert projection instead of +// the hub-link's rejecting placeholder; scratch dir rooted here to share +// the sidecar data dir's lifecycle. const workflowProbeExecutor = createWorkflowProbeExecutor({ materialize: createWorkflowClosureMaterializer({ cacheRoot, @@ -517,19 +371,10 @@ const orchestrator = createSidecarOrchestrator({ // workflow_run.public_key); the link stays source-opaque and reports whatever // this returns. getCachedSenderAddresses: () => senderKeyCache.rotatableAddresses(), - // When the hub-link re-announces a deployment address in an authenticated - // reconnect, re-drive any workflow-run pack the disconnect cancelled. The - // link fires this AFTER sending the reconnect frame, so the hub routes the - // address before it sees the re-shipped pack (both frame families queue on - // the hub's per-connection chain). This is the liveness half of - // reconnect recovery: without it, a synchronous single-step run whose only - // pack was interrupted mid-transfer never re-ships, because it has no later - // local write to re-arm the coalescing loop. + // Re-drives a pack the disconnect cancelled; fires after the reconnect frame + // since both frame families queue on the hub's per-connection chain. onWorkflowAddressesRoutable: (addresses) => { - // The deploy router is captured synchronously during orchestrator - // construction, before the link ever connects, so a routable callback can - // only fire once it exists; assert (like `getWorkflowAddresses`) rather - // than optional-chain so a wiring regression fails loud. + // Assert rather than optional-chain so a wiring regression fails loud. if (sidecarDeployRouter === undefined) { throw new Error( "sidecar boot: deploy router was not constructed before a reconnect made workflow addresses routable", @@ -537,25 +382,8 @@ const orchestrator = createSidecarOrchestrator({ } for (const address of addresses) { wrappedRepoStore.notifyAddressRoutable(address); - // Trigger B: re-register the deployment's parked correlations. A long hub - // outage can evict a `signal.correlation.register` frame from the link's - // bounded send queue; re-driving it on reconnect recovers the parked - // run's approvability. Two independent facts make this safe: - // - The hub-side co-write's `registerSignalCorrelation` throws only when - // the deployment has no `status = "deployed"` row. That row is written - // at DEPLOY time and persists across the outage -- the reconnect - // re-routes the address in the hub's in-memory index but writes no DB - // status, so the lookup already resolves. (The frame also lands after - // the address is wire-routable: the link fires this after the - // reconnect frame, and both frames queue on the hub's per-connection - // chain -- see the pack re-ship note above.) - // - A sidecar restart re-emits the parked set twice (child - // re-establishment fires Trigger A too), but the co-write dedups on - // the `correlationId` PK/unique constraints via `onConflictDoNothing`, - // so the duplicate is a no-op. The pre-existing non-transactional - // status-check window in `registerSignalCorrelation` that this - // widened concurrency exercises is tracked in INTR-338, out of scope - // here. + // Re-registers parked correlations a long outage may have evicted from + // the link's send queue; dedups hub-side on the correlationId constraint. sidecarDeployRouter.reEmitParkedCorrelations(address); } }, From a8b63e0e9bf18c6d082631d1e7eb7142ce14e1ce Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:32:45 -0700 Subject: [PATCH 15/31] chore(sidecar): comments say why, never what (part 14) --- apps/sidecar/src/conversation-state.ts | 391 +++--------------------- docs/sidecar-conversation-durability.md | 44 +++ 2 files changed, 92 insertions(+), 343 deletions(-) create mode 100644 docs/sidecar-conversation-durability.md diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index 164dfd6b1e..fe0b19f045 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -1,127 +1,8 @@ -// Durable conversation state for the warm single-step agent (design §3c, -// §4 Phase D1). -// -// A long-lived single-step agent holds its multi-turn conversation in -// the reactor's in-memory turn buffer, backed by a per-step isogit -// `ContextStore`. That store is rooted per run/attempt, so it is lost -// the moment the warm agent's child is killed and respawned: the -// rebuilt agent loads a fresh, empty per-run store and the conversation -// continuity is gone. -// -// This module makes the warm agent's conversation DURABLE in the -// workflow-run substrate (the single-writer proxy `RepoStore`, written -// through the supervisor). The durable copy lives under the workflow-run -// repo at a stable per-agent path (`agent-state//...`), sibling -// to the per-run event log under `runs//...` and NOT confused with -// it. On a new run (and after a child respawn, once the warm agent is -// rebuilt lazily) the conversation is restored from the substrate into -// the agent's local store BEFORE the agent's reactor loads, so multi-turn -// continuity holds across runs and across respawn. -// -// Two-tier on-disk layout (design §4, Phase D1). The prior design wrote -// the WHOLE conversation as a single `conversation.json` blob on every -// message. That re-serialized and re-hashed every prior turn per message, -// so the per-message durable cost grew O(N) in the turn count -- O(N^2) -// over a conversation. D1 replaces it with an append-only, bucket-sharded -// write-ahead log plus a periodic compacted checkpoint: -// -// agent-state// -// checkpoint.json compacted full snapshot (turns + metadata) -// checkpoint.meta.json { checkpointSeq: , turnCount, -// tokenUsage, pendingOperations, -// connectorState } -// wal//.json one per-boundary delta blob keyed by mirror -// BOUNDARY seq, carrying that boundary's -// 0-or-more new turns plus the freshest metadata -// -// The WAL is keyed by mirror BOUNDARY, not by turn: each `mirrorToSubstrate` -// writes exactly one entry, even when the boundary added no new turns. That -// makes metadata (pendingOperations, tokenUsage, connectorState) persist on -// EVERY boundary -- a turnless-but-metadata-mutating boundary still commits -// a zero-turn entry, so restore never reconstructs stale metadata. (Keying -// the entry by turn dropped metadata on turnless boundaries, which regressed -// the byte-for-byte metadata-equivalence invariant; per-boundary keying is -// the fix.) -// -// `bucket = floor(boundarySeq / WAL_BUCKET_SIZE)` (B = 128) bounds any -// single directory's tree-object size so no commit re-hashes a tree that -// grows with N (a flat `wal/.json` directory would itself be O(N) per -// commit). Compaction every CHECKPOINT_INTERVAL boundaries (K = 64, i.e. -// once the live WAL reaches K entries) folds the WAL into a fresh -// `checkpoint.json` (capturing the freshest metadata in checkpoint.meta) and -// truncates the WAL, so between checkpoints the WAL holds at most K entries -// and per-boundary durable cost is ~O(1) amortized. K and B are constants -// here; the design flags them as measurement-tunable. -// -// Restore = load `checkpoint.json` (folded turns + its metadata) then replay -// the WAL tail in boundary-seq order, concatenating each boundary's turns -// and taking the LATEST entry's metadata (the last WAL entry wins; the -// checkpoint's metadata is the base when the WAL is empty). This is pure -// state reconstruction from recorded outputs -- never re-inference. It -// rebuilds the EXACT turn list + metadata the old whole-blob mirror would -// have restored. -// -// Substrate-merge constraint (load-bearing, design §4 "Substrate-merge -// note"). `writeTreePreservingPrefix`'s `merge` callback receives only -// the DIRECT CHILDREN of `preservePrefix`, and the substrate's -// `clearPrefix` step recursively removes the whole `preservePrefix` -// subtree before writing the merge's returned set (paths outside the -// prefix pass through untouched). A WAL blob is two levels below -// `agent-state//`, so: -// -// - WAL append uses `preservePrefix = agent-state//wal//`. -// The bucket's existing blobs ARE direct children, so the merge -// pre-image is exactly that bucket and the append adds one entry -- -// no isogit side-read, and the checkpoint / other buckets are -// untouched (outside the prefix). -// - Checkpoint write + WAL truncate uses `preservePrefix = -// agent-state//`. The top-level checkpoint files are direct -// children; the merge returns ONLY those files and NO `wal/...` -// paths, so the recursive `clearPrefix` at `agent-state//` drops -// the entire WAL subtree in the same atomic commit. The truncate -// needs no nested read: omitting the WAL paths from the returned set -// IS the truncate. -// -// Persistence sink (the riskiest part, design §6). The connector router's -// `snapshot()` / `restore()` surface and the harness's -// `createWrappedStorageOverrides` are reused, but the persistence sink is -// repointed from the agent's local isogit store to the workflow-run -// substrate. Both the WAL append and the checkpoint write route through -// the proxy `writeTreePreservingPrefix`; because the supervisor is the -// single writer and serializes every write to the workflow-run ref under -// a per-repo lock, and the `agent-state//...` prefix is disjoint from -// the run-event prefix (`runs//events/`), the conversation write -// never races nor clobbers the run-event log -- both pass through the same -// single writer, and the preserve-prefix merge leaves every other subtree -// byte-for-byte intact. -// -// Timing (design §4 Phase D1, invariant 1). This is a STRUCTURE-only -// change. The mirror is still `await`ed synchronously at the same run -// boundary (`onRunBoundary` -> `mirrorToSubstrate`), so every turn is -// still durably committed before the next message is processed. D1 -// changes WHAT the write does (O(1) append instead of O(N) whole-blob), -// not WHEN it happens. The run log is NOT yet a durable backstop for the -// turn (it carries a constant ref, design rev-2 FACT 1), so the -// conversation copy here remains the sole durable copy of the agent's -// per-turn output -- which is exactly why D1 keeps the write synchronous. -// The async flusher, run-log enrichment, and crash reconciliation are -// later, conditional phases (D2-D4), not done here. -// -// Commit granularity (greybeard's pick, design §3c open question). The -// design calls for connector-state-change-driven commits via the router's -// `onStateChanged` hook. The warm-agent path drives the connector router -// through `seedInbound`: each mail-derived inbound message routes and -// commits its thread state before the agent's send, so `onStateChanged` -// fires and enqueues a change-driven mirror. The run-boundary mirror (per -// message) still runs unconditionally, so the two triggers are -// complementary -- the seed persists the connector state promptly, the -// boundary persists the turn delta. -// -// Defensive: a restore that finds a checkpoint or WAL but cannot -// parse/replay it THROWS (a lost or corrupt conversation on respawn is a -// correctness failure, not a silently-fresh start). A mirror write failure -// surfaces so a dropped durability write is visible rather than leaving -// the next respawn to read a stale snapshot. +// Makes a warm single-step agent's multi-turn conversation durable in the +// workflow-run substrate (not the agent's per-run isogit store, which is +// lost on child respawn), so continuity survives across runs and respawns. +// See docs/sidecar-conversation-durability.md for the on-disk layout, +// substrate-merge constraints, and timing/failure guarantees. import fs from "node:fs"; import path from "node:path"; @@ -168,30 +49,14 @@ const CHECKPOINT_INTERVAL = 64; */ const WAL_BUCKET_SIZE = 128; -/** - * Metadata carried alongside the checkpoint and stamped onto every WAL - * entry. Small and bounded -- it is NOT the O(N) cost; the turn array is. - * Stamping the latest metadata on each WAL entry lets the restore replay - * recover the exact non-turn reactor state without a separate metadata - * log: the last replayed entry's metadata wins. Because every mirror - * boundary writes exactly one WAL entry (even a turnless one), the latest - * metadata is ALWAYS captured durably -- a turnless boundary still commits - * its advanced metadata as a zero-turn entry. - */ +/** Metadata stamped onto every WAL entry so restore recovers reactor state without a separate log. */ const SnapshotMetadata = type({ pendingOperations: "unknown[]", tokenUsage: TokenUsage, connectorState: ConnectorThreadState.or("null"), }); -/** - * On-disk shape of the compacted checkpoint blob committed at - * `agent-state//checkpoint.json`. Carries the folded turn - * history (turns 0..checkpointSeq-1) plus the non-turn reactor metadata. - * Validated on read because it crosses back into the program from the - * substrate working tree -- a corrupt or partially-written checkpoint must - * surface at the boundary, never be half-applied into the agent. - */ +/** Compacted checkpoint blob; validated on read so a corrupt or partial file surfaces rather than half-applies. */ const CheckpointSnapshot = type({ turns: "unknown[]", pendingOperations: "unknown[]", @@ -199,17 +64,7 @@ const CheckpointSnapshot = type({ connectorState: ConnectorThreadState.or("null"), }); -/** - * On-disk shape of `agent-state//checkpoint.meta.json`. The - * checkpoint pointer the restore path reads first to learn the boundary seq - * the checkpoint folded to (`checkpointSeq`) -- and therefore which WAL - * boundary seqs remain to replay -- plus the folded turn count - * (`turnCount`, = `checkpoint.json`'s turn array length) and the freshest - * metadata at fold time. `checkpointSeq` counts MIRROR BOUNDARIES, not - * turns: a boundary may carry zero or many turns, so the boundary count and - * the turn count diverge in general (they coincide only when every boundary - * adds exactly one turn). - */ +/** Checkpoint pointer restore reads first; checkpointSeq counts mirror boundaries, not turns, since a boundary may carry zero or many turns. */ const CheckpointMeta = type({ checkpointSeq: "number", turnCount: "number", @@ -218,16 +73,7 @@ const CheckpointMeta = type({ connectorState: ConnectorThreadState.or("null"), }); -/** - * On-disk shape of one WAL entry blob at - * `agent-state//wal//.json`. One entry per MIRROR - * BOUNDARY (keyed by boundary `seq`, not turn index). Records the 0-or-more - * new turns that boundary added (the O(1) append payload -- it never - * carries prior turns) plus the latest non-turn metadata snapshot. The - * append is UNCONDITIONAL: a turnless boundary still writes one entry with - * `turns: []` so its advanced metadata is durably committed (the invariant - * the per-turn keying broke -- metadata must persist on EVERY boundary). - */ +/** One entry per mirror boundary (not per turn); a turnless boundary still writes `turns: []` so its metadata persists. */ const WalEntry = type({ seq: "number", turns: "unknown[]", @@ -287,55 +133,15 @@ export interface DurableConversationStore { * isogit store the non-warm path uses. */ readonly storage: ContextStore & AuditStore; - /** - * Pull the prior conversation from the substrate (checkpoint + WAL-tail - * replay) into the local store so the agent's reactor `load()` sees it. - * Called before the warm agent is built (lazy first build and respawn - * rebuild). Returns `true` when prior state was found and applied, - * `false` when none exists yet (the genuine first-ever run). A read that - * finds a checkpoint or WAL but cannot parse/replay it throws -- a - * corrupt durable copy is a correctness failure that must not silently - * start the agent fresh. - */ + /** Restore prior conversation from the substrate before the agent builds; throws on a corrupt copy rather than starting fresh. */ restoreFromSubstrate(): Promise; - /** - * Commit the local store's new turn(s) to the substrate as O(1) WAL - * appends, folding into a fresh checkpoint when the WAL reaches the - * compaction interval. Called synchronously at the run boundary (after - * the agent's send settles). A write failure surfaces. - */ + /** Commit new turns as an O(1) WAL append, compacting into a checkpoint at the compaction interval. */ mirrorToSubstrate(): Promise; - /** - * Advance the connector router from a received inbound message so the - * warm agent's reply path has thread state. Runs the router's pure - * `route()` then `commit()`: a `start` seeds threadRoot / lastMessageId / - * replyTo from the message; a `continue` advances lastMessageId / replyTo - * and carries prior speakers into `cc`. The advanced connector state is - * flushed into the local store's metadata so the run-boundary mirror - * persists it and a respawn restore re-seeds the router. A `passthrough` - * decision -- no active-thread match, or an unparseable sender -- advances - * nothing. Called before the warm agent's send so `composeReply()` can - * compose a threaded reply. A metadata write failure surfaces. - */ + /** Advances connector thread state from an inbound message so composeReply can address a threaded reply. */ seedInbound(message: InboundMessage): Promise; - /** - * Produce the threading headers for a reply on the active connector - * thread (the router's `composeReply`). Throws - * `NoActiveConnectorThreadError` when no thread has been seeded. The warm - * mail loop's reply drain reads this to address its outbound reply. - */ + /** Threading headers for a reply on the active connector thread; throws NoActiveConnectorThreadError with none seeded. */ composeReply(): ConnectorReplyParts; - /** - * Advance the connector thread after a reply was sent. Forwards to the - * router's `onReplySent` (which moves `lastMessageId` to the sent reply's - * Message-ID so the next inbound continuation matches) and flushes the - * advanced connector state into the local store's metadata the same way - * `seedInbound` does, so `lastMessageId` persists across turns and across a - * child respawn. Called by the warm mail loop's reply drain after its - * outbound send settles. Throws `NoActiveConnectorThreadError` when no - * thread is active -- advancing outbound state has no meaning without a - * seeded thread. A metadata write failure surfaces. - */ + /** Advances thread state after a reply is sent so the next inbound continuation matches lastMessageId. */ onReplySent(receipt: SendReceipt): Promise; } @@ -379,22 +185,11 @@ export async function createDurableConversationStore( // checkpointBoundarySeq) and when to compact. let checkpointBoundarySeq = 0; - // Serialize the shared-counter critical section. Both `mirrorToSubstrate` - // and `restoreFromSubstrate` read and advance the three counts above, and - // either can re-enter the other: `connectorRouter.restore()` can fire - // `onStateChanged` synchronously, which enqueues a mirror. A single - // per-instance tail runs every such op one-at-a-time regardless of entry - // point, so two overlapping runs cannot read the same boundary seq and - // emit two WAL entries at it. Modeled on `runRepoOp` in the hub-agent - // session manager: the stored tail swallows rejections so one failed op - // does not poison the chain, while each caller still observes its own op's - // result (or rejection) through the returned promise. - // - // This serializes mirror-vs-mirror and mirror-vs-restore only. It does NOT - // address the reactor-vs-mirror peek-snapshot window documented on - // `runMirror` below (nothing must append to the reactor's turn array - // between its last writeTurns and the mirror's peekTurns) -- that is a - // different concurrency axis and is out of scope here. + // Serializes the shared-counter critical section: mirror and restore can + // re-enter each other (connectorRouter.restore() can fire onStateChanged + // synchronously), so a single tail prevents two overlapping runs from + // emitting two WAL entries at the same boundary seq. Does not address the + // separate reactor-vs-mirror peek-snapshot window documented on runMirror. let stateOpTail: Promise = Promise.resolve(); function serializeStateOp(op: () => Promise): Promise { const result = stateOpTail.then(op, op); @@ -471,14 +266,9 @@ export async function createDurableConversationStore( // so a future change-driven mirror carries the right base. await baseStorage.writeTurns(reconstructed.turns); baseStorage.setConnectorState(reconstructed.connectorState); - // Establish the committed counts BEFORE restoring the connector state. - // `connectorRouter.restore()` can fire `onStateChanged` synchronously - // (when the restored state differs from current), which enqueues a - // mirror. Serialization already chains that mirror behind this restore, - // but setting the counts first keeps them correct even if that ordering - // guarantee is ever weakened. The counts reflect the substrate state - // `reconstructed` was read from, which is durable independent of the - // local-store commit below. + // Set counts before restoring connector state: restore() can fire + // onStateChanged synchronously and enqueue a mirror, so this keeps the + // counts correct even if the serialization ordering is ever weakened. mirroredBoundaryCount = reconstructed.boundaryCount; mirroredTurnCount = reconstructed.totalTurns; checkpointBoundarySeq = reconstructed.checkpointBoundarySeq; @@ -493,15 +283,7 @@ export async function createDurableConversationStore( return true; } - /** - * Append one WAL entry for a mirror boundary (its 0-or-more new turns + - * the current metadata) to its bucket. The merge pre-image is exactly - * that bucket's existing blobs (direct children of `wal//`), so - * the append is the bucket's blobs plus the one new entry -- O(bucket - * size), independent of N. The append is the SINGLE synchronous write per - * boundary; the entry is keyed by boundary `seq` so a turnless boundary - * still commits its metadata as a zero-turn entry. - */ + /** Appends one WAL entry to its bucket; O(bucket size), independent of total turn count N. */ async function appendWalEntry( boundarySeq: number, turns: unknown[], @@ -533,14 +315,7 @@ export async function createDurableConversationStore( ); } - /** - * Fold the full conversation into a fresh checkpoint and truncate the - * WAL in one atomic commit at `preservePrefix = agent-state//`. The - * merge returns ONLY the two checkpoint files and NO `wal/...` paths; - * because the substrate's `clearPrefix` recursively removes the whole - * `agent-state//` subtree before writing the returned set, omitting - * the WAL paths IS the truncate. - */ + /** Folds the conversation into a fresh checkpoint and truncates the WAL atomically by omitting wal/ paths from the merge return. */ async function writeCheckpoint( boundarySeq: number, turns: unknown[], @@ -556,11 +331,7 @@ export async function createDurableConversationStore( tokenUsage: metadata.tokenUsage, connectorState: metadata.connectorState, }; - // `metadata` is the freshest snapshot (the current local-store - // metadata, identical to the last appended WAL entry's metadata), so - // the fold captures the latest metadata into checkpoint.meta -- a - // restore from the post-fold checkpoint sees the same metadata the - // pre-fold WAL tail would have yielded. + // metadata is the freshest snapshot, so a post-fold restore sees the same metadata the pre-fold WAL tail would have yielded. const meta = { checkpointSeq: boundarySeq, turnCount: turns.length, @@ -584,18 +355,10 @@ export async function createDurableConversationStore( } async function runMirror(): Promise { - // Slice the new turns from the reactor's in-memory array (retained - // by the local single-writer store at the last writeTurns) instead - // of re-reading and re-parsing the whole turns.jsonl every - // boundary; only the bounded metadata.json is read from disk. This - // rests on a sequencing invariant the store cannot enforce: nothing - // mutates the reactor's turn array between its last writeTurns and - // this read. The mirror runs at onRunBoundary after send() settles, - // and the reactor only appends inside a cycle (each ending in - // writeTurns), so peekTurns() equals the on-disk state here. - // Serializing the mirror entry points keeps two mirrors from - // overlapping this read, but the reactor must still not append - // between its writeTurns and this peek -- that axis is not serialized. + // Reads the reactor's in-memory turn array rather than re-parsing + // turns.jsonl every boundary; relies on the reactor never appending + // between its last writeTurns and this peek, which serialization here + // does not itself enforce. const turns = baseStorage.peekTurns(); const metadata = await baseStorage.loadMetadata(); @@ -613,15 +376,9 @@ export async function createDurableConversationStore( mirroredTurnCount = reconstructed?.totalTurns ?? 0; } - // ONE WAL entry per mirror boundary, UNCONDITIONALLY -- even when no new - // turns were added since the last mirror. The entry carries the - // 0-or-more new turns plus the freshest metadata snapshot, so a - // turnless-but-metadata-mutating boundary (e.g. a throwing send that - // still advanced tokenUsage/pendingOperations, since onRunBoundary runs - // in a finally) still durably commits its metadata. The payload is the - // turn DELTA plus bounded metadata -- never the whole conversation, so - // the O(N^2) growth stays gone. This is the single synchronous write - // per boundary on the reply path. + // One WAL entry per boundary, unconditionally, so a turnless boundary + // (e.g. a throwing send that still advanced tokenUsage) still commits + // its metadata; the payload is the turn delta, never the whole conversation. const newTurns = turns.slice(mirroredTurnCount); const boundarySeq = mirroredBoundaryCount; await appendWalEntry(boundarySeq, newTurns, metadata); @@ -659,15 +416,9 @@ export async function createDurableConversationStore( } } - // Advance the connector router from a received inbound message and flush - // the resulting connector state into the local store's metadata. `commit` - // fires the router's `onStateChanged`, which enqueues a change-driven - // mirror behind this op on the shared serialization tail; because that - // mirror reads the connector state from the local store's metadata (not - // from the router), the metadata write below is what makes the seeded - // state reach the substrate. The write preserves the reactor's staged - // pendingOperations / tokenUsage -- a seed advances only connectorState. - // A passthrough decision advances nothing and writes nothing. + // commit() fires onStateChanged, enqueuing a change-driven mirror that reads + // connector state from local-store metadata, so the write below is what + // makes the seeded state reach the substrate. async function runSeed(message: InboundMessage): Promise { const decision = routeOrPassthrough(message); connectorRouter.commit(decision); @@ -684,17 +435,8 @@ export async function createDurableConversationStore( }); } - // Advance the connector thread after a reply was sent and flush the - // resulting connector state into the local store's metadata. `onReplySent` - // moves `lastMessageId` to the reply's Message-ID and fires the router's - // `onStateChanged`, which enqueues a change-driven mirror behind this op on - // the shared serialization tail; because that mirror reads the connector - // state from the local store's metadata (not from the router), the metadata - // write below is what makes the advanced state reach the substrate. The - // write preserves the reactor's staged pendingOperations / tokenUsage -- an - // outbound advance touches only connectorState. `onReplySent` throws when no - // thread is active, which surfaces to the reply drain's failure callback - // rather than persisting a phantom advance. + // Same metadata-write pattern as runSeed; onReplySent throws when no thread + // is active so a phantom advance is never persisted. async function runReplySent(receipt: SendReceipt): Promise { connectorRouter.onReplySent(receipt); @@ -734,16 +476,7 @@ export interface DurableConversationRegistryOpts { signer: (payload: string) => Promise; } -/** - * Per-agent durable-conversation store registry (design §3c). One store - * per warm agent key, built lazily and reused across runs in the same - * child. The first `acquire` for a key builds the store and restores its - * prior conversation snapshot from the substrate -- the path that runs on - * the lazy first build AND on the respawn rebuild, so the warm agent - * resumes its conversation across child respawn. The registry is empty - * after a respawn (it lives in the child's address space); the substrate - * is the durable mirror that survives. - */ +/** Per-agent store registry, built lazily and reused across runs; empty after a respawn since the substrate is what survives. */ export interface DurableConversationRegistry { acquire(key: string): Promise; get(key: string): DurableConversationStore; @@ -782,13 +515,9 @@ export function createDurableConversationRegistry( principal: opts.principal, agentKey: key, }); - // Restore the prior conversation BEFORE the store is observable (and - // before the warm agent's reactor `load()` reads it). On a genuine - // first-ever run this is a no-op (no checkpoint/WAL yet); on a - // respawn rebuild it pulls the pre-respawn conversation back from the - // substrate (checkpoint + WAL replay). A restore failure surfaces -- - // a lost conversation on respawn is a correctness failure, not a - // silently-fresh start. + // Restore before the store is observable; a no-op on first-ever run, + // otherwise pulls the pre-respawn conversation back. A restore failure + // surfaces rather than silently starting fresh. await store.restoreFromSubstrate(); stores.set(key, store); building.delete(key); @@ -820,15 +549,7 @@ interface SnapshotMetadataValue { connectorState: ConnectorThreadState | null; } -/** - * The reconstructed conversation plus the bookkeeping the mirror path needs - * to resume appending. `totalTurns` is the full turn count (checkpoint + - * WAL). `boundaryCount` is the number of mirror boundaries durably - * committed (checkpoint's folded boundaries + replayed WAL entries) -- the - * next WAL entry uses this as its boundary seq. `checkpointBoundarySeq` is - * the boundary seq the checkpoint folded to (the first WAL boundary seq to - * expect), so the mirror knows the live WAL length and when to compact. - */ +/** Reconstructed conversation plus the boundary/turn bookkeeping the mirror path needs to resume appending. */ export interface ReconstructedConversation extends LoadedSnapshot { totalTurns: number; boundaryCount: number; @@ -836,20 +557,11 @@ export interface ReconstructedConversation extends LoadedSnapshot { } /** - * Reconstruct the warm agent's conversation from the two-tier on-disk - * layout under `agentStateDir` (`/agent-state//`): the - * compacted `checkpoint.json` turns followed by the replayed WAL tail. - * Pure read against the substrate working tree -- no inference, no commit. - * Returns `null` when neither a checkpoint nor any WAL exists (the genuine - * first-ever run). The latest metadata source wins (the last replayed WAL - * entry, or the checkpoint when the WAL is empty), mirroring how each - * mirror stamps the current metadata. Throws on any corrupt/unparseable - * blob or a WAL seq gap -- a damaged durable copy must surface, never - * silently start the agent fresh or drop a turn. - * - * Exported so a reader (durability test, recovery audit) reconstructs the - * conversation through the SAME code path the warm agent's restore uses, - * rather than re-deriving the WAL/checkpoint fold independently. + * Pure read: checkpoint turns + replayed WAL tail. Returns `null` on a + * genuine first-ever run; throws on any corrupt blob or WAL seq gap rather + * than silently starting fresh or dropping a turn. Exported so a reader + * (durability test, recovery audit) uses the same reconstruction path the + * warm agent's restore does. */ export async function reconstructDurableConversation( agentStateDir: string, @@ -950,14 +662,7 @@ async function readCheckpointFromDir( }; } -/** - * Read and seq-order the per-boundary WAL entries for boundary seqs >= - * `fromSeq` from `/wal//`. Throws on any unparseable - * or out-of-shape WAL blob -- a corrupt WAL must surface, never be skipped. - * Throws on a gap in the boundary seq sequence: a missing seq means a lost - * append, which would silently drop a boundary's turns + metadata from the - * reconstruction. - */ +/** Reads and seq-orders WAL entries >= fromSeq; throws on a corrupt blob or a seq gap rather than silently dropping a boundary. */ async function readWalTailFromDir( agentStateDir: string, agentKey: string, diff --git a/docs/sidecar-conversation-durability.md b/docs/sidecar-conversation-durability.md new file mode 100644 index 0000000000..c82406976a --- /dev/null +++ b/docs/sidecar-conversation-durability.md @@ -0,0 +1,44 @@ +# Warm-agent conversation durability + +`apps/sidecar/src/conversation-state.ts` makes a warm single-step agent's +multi-turn conversation durable across a child respawn by mirroring it into +the workflow-run substrate instead of the agent's per-run isogit store. + +## On-disk layout + +``` +agent-state// + checkpoint.json compacted full snapshot (turns + metadata) + checkpoint.meta.json { checkpointSeq, turnCount, tokenUsage, ... } + wal//.json one per-boundary delta blob +``` + +The WAL is keyed by mirror boundary, not by turn, so a turnless +metadata-only boundary still commits an entry — keying by turn dropped +metadata on those boundaries, regressing the metadata-equivalence +invariant. + +`bucket = floor(boundarySeq / WAL_BUCKET_SIZE)` bounds a directory's tree +size so no commit re-hashes a tree that grows with turn count. Compaction +every `CHECKPOINT_INTERVAL` boundaries folds the WAL into a fresh +checkpoint and truncates it, keeping per-boundary durable cost ~O(1) +amortized. Restore loads the checkpoint then replays the WAL tail, +concatenating turns and taking the latest entry's metadata. + +## Substrate-merge constraint + +`writeTreePreservingPrefix`'s `merge` callback receives only the direct +children of `preservePrefix`, and the substrate's `clearPrefix` recursively +removes the whole prefix subtree before writing the merge's return set. +WAL append uses `preservePrefix = agent-state//wal//` (no +isogit side-read needed); checkpoint write + WAL truncate uses +`preservePrefix = agent-state//` and simply omits `wal/...` paths from +the returned set to truncate atomically in the same commit. + +## Timing and failure semantics + +The mirror is still awaited synchronously at each run boundary — this is a +structure-only change to what the write does (O(1) append vs O(N) blob), +not when it happens. A restore that cannot parse/replay a checkpoint or WAL +throws rather than silently starting fresh; a mirror write failure +surfaces rather than leaving the next respawn to read a stale snapshot. From f0b6afad2be7bf3b4e246bcf5e5cd0dda5ce006b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:34:25 -0700 Subject: [PATCH 16/31] chore(tools): comments say why, never what (github, capability) --- tools/capability/src/client.ts | 15 +++--------- tools/capability/src/run-bearer.ts | 11 +++------ tools/capability/src/tool.ts | 39 ++++++------------------------ 3 files changed, 13 insertions(+), 52 deletions(-) diff --git a/tools/capability/src/client.ts b/tools/capability/src/client.ts index adac43a24c..f6139f37d6 100644 --- a/tools/capability/src/client.ts +++ b/tools/capability/src/client.ts @@ -1,15 +1,6 @@ -// A minimal client for the workflow-run-authenticated capabilities -// surface a running agent calls to add itself a tool package, skill, or -// model — authenticated via `createWorkflowRunAuthenticator` (sidecar -// bearer token + run address), never a human browser session. -// -// Intx gap: a run's own `kind: "workflow"` principal is never seeded a -// `workflow-definition: /update` grant, so the -// workflow-capability route skips a grant-store check for the narrow -// own-definition case, relying instead on `request_capability`'s -// `approval: "ask"` gate having already put a human in front of the -// call before this client is ever invoked — see the route's own -// file-level comment for the full authorization reasoning. +// A run's own workflow principal is never seeded a self-update grant, so +// this route skips the grant-store check and relies on request_capability's +// approval:"ask" gate having already put a human in front of the call. import { type } from "arktype"; import { runBearerHeaders, diff --git a/tools/capability/src/run-bearer.ts b/tools/capability/src/run-bearer.ts index 7a5800f123..7acf737c9e 100644 --- a/tools/capability/src/run-bearer.ts +++ b/tools/capability/src/run-bearer.ts @@ -1,11 +1,6 @@ -// Shared fetch plumbing for a run-authenticated tool client — every tool -// bundle that calls a hub route mounted behind a workflow-run -// authenticator (sidecar bearer token + run address, never a human -// session) needs the same two headers and the same -// `{ error: { code, userMessage } }` envelope parse. Kept local to this -// package (rather than imported from Workbench's own `@corbits/workflows`, -// which is private and carries a full server dependency graph) so this -// bundle installs standalone in any Interchange-backed hub. +// Kept local rather than imported from Workbench's private @corbits/workflows +// (full server dependency graph) so this bundle installs standalone in any +// Interchange-backed hub. import { type } from "arktype"; export interface RunBearerClientConfig { diff --git a/tools/capability/src/tool.ts b/tools/capability/src/tool.ts index a909fb18bd..727f151318 100644 --- a/tools/capability/src/tool.ts +++ b/tools/capability/src/tool.ts @@ -1,25 +1,9 @@ -// The `@corbits/capability-tools` bundle: `request_capability`, an -// agent's in-chat way to ask for a tool package, skill, or model it -// doesn't have yet. Declared `approval: "ask"` (`@intx/agent`'s native -// per-invocation gate, `@intx/agent/src/tool.ts`) — the reactor -// suspends the call as a pending approval BEFORE this bundle's `run` ever -// executes, renders it in-chat as an approve/deny card, and only resumes -// into `run` once a human allows it. This bundle's own code never sees -// or controls that gate; it only has to make the resulting card and -// result message read honestly. -// -// `definitionId` is threaded onto `env` by the sidecar's per-step env -// builder (`apps/sidecar/src/workflow-substrate-factory/step-env.ts`, -//), the same ground `@corbits/memory-tools`' `hubMemoryUrl`/ -// `sidecarToken`/`address` are threaded from — resolved at the -// substrate factory from the deploying definition's own id -// (`WORKFLOW_DEFINITION_REPO_ID`). `WorkflowCapabilityEnv` below -// declares `definitionId` as a required env key exactly the way -// `WorkflowMemoryEnv` declares its three. -// -// See `./client.ts` for the workflow-run-authenticated capabilities -// route (`@corbits/agent-directory`'s `createWorkflowCapabilityRoutes`, -// also) this bundle's execution calls. +// `request_capability` is declared `approval: "ask"`, so the reactor +// suspends it as a pending approval before `run` ever executes; this +// bundle never sees or controls that gate. `definitionId` is threaded onto +// `env` by the sidecar's per-step env builder, the same ground +// @corbits/memory-tools' env keys come from. See ./client.ts for the +// workflow-run-authenticated route this bundle's execution calls. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; @@ -142,16 +126,7 @@ async function runRequestCapability( } } -/** - * The `@corbits/capability-tools` bundle factory: one tool, - * `approval: "ask"`, four env keys — the self-service capability - * request path. `description` is written to read as an approval card - * headline (`@corbits/approvals`' `headlineFor` uses a tool's - * `description` verbatim, appending `arguments.title` in quotes when the - * model supplies one) — encouraging the model to fill `title` with a - * short human label makes the resulting card read naturally, e.g. - * " wants to add a capability: \"GitHub tools\" — Allow?". - */ +/** `description` is written to read as an approval-card headline, which appends `arguments.title` verbatim when supplied. */ export const capabilityTools = defineTool({ id: "@corbits/capability-tools/cap", requires: ["hubCapabilitiesUrl", "sidecarToken", "address", "definitionId"], From e3c559f8ee8fb67e2c549f41142071bd1f3daf45 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:35:09 -0700 Subject: [PATCH 17/31] chore(tools): comments say why, never what (workflow-authoring) --- tools/workflow-authoring/src/client.ts | 41 +++++++------------------- tools/workflow-authoring/src/tool.ts | 30 ++++--------------- 2 files changed, 16 insertions(+), 55 deletions(-) diff --git a/tools/workflow-authoring/src/client.ts b/tools/workflow-authoring/src/client.ts index 45408dfb88..7e09e3b913 100644 --- a/tools/workflow-authoring/src/client.ts +++ b/tools/workflow-authoring/src/client.ts @@ -1,23 +1,8 @@ -// Two surfaces, one credential. -// -// `deployWorkflow` speaks the STOCK `@intx/hub-api` routes: `GET -// /api/tenants/:tenantId/models` for the tenant's resolved inference -// catalog, then `POST /api/tenants/:tenantId/workflows/deployments`. The -// run bearer authenticates both — `apps/hub/src/workflow-run-tenant-auth.ts` -// resolves the sidecar token + run address into a principal and tenant -// across the whole tenant subtree. The tenant id is a path segment on -// every stock route, so it rides in the client config; it is never -// trusted as authority, since the hub sets the acting tenant from the -// authenticated run alone. -// -// `authorWorkflow` / `republishWorkflow` / `readWorkflowSource` / -// `previewDeployWorkflow` still call the Workbench-specific -// `/api/workflow-workflow-authoring` mount. Their stock equivalent is git -// smart-HTTP on the asset repo, which no run-bearer credential can reach -// today: `createGitTokenAuth` accepts only an `itx_pat_`/`itx_svc_` git -// token, and the stock mint route refuses a caller with no browser -// session. See for the upstream ask that would let these four -// follow `deploy` onto stock routes. +// deployWorkflow speaks stock @intx/hub-api routes; the tenant id rides in +// the client config as a path segment only, never trusted as authority +// (the hub sets the acting tenant from the authenticated run alone). The +// other four calls still hit the Workbench-specific mount because no +// run-bearer credential can reach stock git smart-HTTP today. import { type } from "arktype"; import { runBearerHeaders, runBearerFetch, type RunBearerClientConfig } from "./run-bearer"; @@ -171,17 +156,11 @@ async function throwForStockFailure(response: Response, operation: string): Prom } /** - * The ordered catalog offering chain a deploy hands the hub, rebuilt on - * the client from the tenant's resolved model catalog. - * - * The deleted `/api/workflow-workflow-authoring/:assetId/deploy` mirror - * resolved this server-side with `listVisibleOfferings` sorted by - * `priority`; the stock route takes it from the caller instead. The - * discovery route already applies the same inheritance, shadowing and - * disable cascade, and groups its offerings under each model, so the - * flattened list is re-sorted by priority here to restore the single - * global ordering the mirror produced. `offeringId` is deduplicated - * because the stock route rejects a chain with a repeat. + * The ordered catalog offering chain a deploy hands the hub. Rebuilt + * client-side (the stock route takes it from the caller, unlike the + * deleted server-side mirror), re-sorted by priority to restore the single + * global ordering, and deduplicated since the stock route rejects a + * repeat offeringId. */ export function orderedSourceOfferingIds( models: readonly { readonly offerings: readonly OfferingOrdering[] }[], diff --git a/tools/workflow-authoring/src/tool.ts b/tools/workflow-authoring/src/tool.ts index 967ed78393..d2ac58d0de 100644 --- a/tools/workflow-authoring/src/tool.ts +++ b/tools/workflow-authoring/src/tool.ts @@ -1,17 +1,8 @@ -// The `@corbits/workflow-authoring-tools` bundle: `workflow_author`, -// `workflow_republish`, `workflow_source_read`, and `workflow_deploy` — an -// agent's way to write a workflow code package into a `kind: "workflow"` -// hub asset, read it back, and deploy it through Interchange's native -// source pipeline. The first three carry no `approval: "ask"`: writing -// source is not a side effect (docs/workflow-model.md, "Authority -// boundaries"). `workflow_deploy` does — deploying is what makes a -// workflow selectable as a routine target, so a human sees the deploy -// intent and approves it before the tool call ever reaches the hub (// still owns showing the probed capability surface on that approval card; -// today's snapshot is the tool call's own arguments). -// -// A thrown error here is the honest result: `@intx/agent`'s tool runner -// converts a rejected `run` into `ToolResult { isError: true }` carrying -// the message, so the model sees exactly what the hub refused and why. +// The first three tools carry no approval: "ask" since writing source is not +// a side effect (docs/workflow-model.md, "Authority boundaries"); +// workflow_deploy does, since deploying makes a workflow selectable as a +// routine target. A thrown error surfaces as ToolResult{isError:true} so the +// model sees exactly what the hub refused. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; @@ -174,16 +165,7 @@ async function runDeploy(env: WorkflowAuthoringEnv, call: ToolCall): Promise:` is what goes on the - * provider wire, `@`, `/`, `:` and `-` each encode to three characters - * there (`@intx/inference`'s `encodeToolName`), and - * `@corbits/workflow-authoring-tools/:workflow_source_read` - * cannot fit OpenAI's 64-character cap. `defineTool` only requires the - * `@scope/pkg/name` shape; `apps/web/src/tools/registry-publish.test.ts`'s - * tool-name-limits test is what this id satisfies. - */ +/** id's middle segment is shortened (not the package name): `:` on the wire must fit OpenAI's 64-char cap. */ export const workflowAuthoringTools = defineTool({ id: "@corbits/workflow_authoring/wf", requires: ["hubWorkflowAuthoringUrl", "tenantId", "sidecarToken", "address"], From b61bf663afe9cd331007c85c8b79cb43f46cebe4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:35:39 -0700 Subject: [PATCH 18/31] chore(tools): comments say why, never what (agent-directory) --- tools/agent-directory/src/client.ts | 22 ++++------------------ tools/agent-directory/src/tool.ts | 27 +++++---------------------- 2 files changed, 9 insertions(+), 40 deletions(-) diff --git a/tools/agent-directory/src/client.ts b/tools/agent-directory/src/client.ts index 117d5d3483..16224edd8f 100644 --- a/tools/agent-directory/src/client.ts +++ b/tools/agent-directory/src/client.ts @@ -1,21 +1,7 @@ -// A minimal client for the two workflow-run-authenticated surfaces -// `create_agent`/`list_agents` reach: `@corbits/agent-directory`'s -// `createWorkflowAgentCreateRoutes` (`POST/GET .../definitions`) and -// `@corbits/chat`'s `createWorkflowParticipantRoutes` -// (`POST .../participants/invite`) — this bundle's execution half, -// mirroring `@corbits/capability-tools`' `client.ts` exactly: same -// auth-header shape, same error-handling, same arktype-response-parsing -// pattern. -// -// Two hub URLs, deliberately not one: `hubAgentDirectoryUrl` reaches -// `@corbits/agent-directory`'s workflow-create-routes, -// `hubChatUrl` reaches `@corbits/chat`'s workflow-participant-routes. -// They are very likely the same physical hub origin in every real -// deployment, but this bundle never assumes that — each surface -// declares its own env key, mirroring how every other tool bundle in -// this codebase (`@corbits/memory-tools`'s `hubMemoryUrl`, -// `@corbits/capability-tools`'s `hubCapabilitiesUrl`) declares its own -// `hubUrl` rather than a shared value. +// Two hub URLs, deliberately not one: the two surfaces are very likely the +// same physical hub origin in every real deployment, but this bundle never +// assumes that, mirroring how every other tool bundle here declares its +// own hubUrl rather than a shared value. import { type } from "arktype"; export interface AgentDirectoryToolClientConfig { diff --git a/tools/agent-directory/src/tool.ts b/tools/agent-directory/src/tool.ts index b5ba774a50..ca871b3fac 100644 --- a/tools/agent-directory/src/tool.ts +++ b/tools/agent-directory/src/tool.ts @@ -1,25 +1,8 @@ -// The `@corbits/agent-directory-tools` bundle: Myra's manager tools — -// `list_agents`, a plain read of the tenant's taskable agents, and -// `create_agent`, which materializes a brand-new specialist agent -// definition and, by default, opens that specialist its own `kind: chat` -// 1:1 (mint-dm) — never inviting into Myra's current DM. Neither tool -// carries an `approval` key: creation is free and the reactor never parks -// the call. Creation plus the default mint is tenant-internal — a new -// definition in the caller's own tenant, then (unless opted out) a fresh -// chat for that definition. `toolPackagePins` on that definition are pins -// the user asked Myra to set, not a capability grant that needs a -// per-invocation gate. State-changing MCP tools and run-now/execution -// stay gated on their own declarations. -// -// `hubAgentDirectoryUrl`/`hubChatUrl`/`sidecarToken`/`address` are -// threaded onto `env` by the sidecar's per-step env builder, the same -// ground `@corbits/memory-tools`'/`@corbits/capability-tools`' own env -// keys are threaded from. -// -// See `./client.ts` for the workflow-run-authenticated routes this -// bundle's execution calls: `@corbits/agent-directory`'s -// `createWorkflowAgentCreateRoutes` and `@corbits/chat`'s -// `createWorkflowParticipantRoutes` (mint-dm + invite). +// Neither tool carries an approval key: creation is tenant-internal and +// free (a new definition plus, by default, its own DM rather than an +// invite into Myra's current one) — toolPackagePins here are pins the user +// asked Myra to set, not a capability grant needing a per-invocation gate. +// See ./client.ts for the workflow-run-authenticated routes this calls. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; From 0e758b4246482dffafd8d42ab0ce8f0650b6343a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:36:13 -0700 Subject: [PATCH 19/31] chore(tools): comments say why, never what (access) --- tools/access/src/client.ts | 42 ++++++++++---------------------------- tools/access/src/tool.ts | 20 ++++-------------- 2 files changed, 15 insertions(+), 47 deletions(-) diff --git a/tools/access/src/client.ts b/tools/access/src/client.ts index 8e1a019722..e8d798c257 100644 --- a/tools/access/src/client.ts +++ b/tools/access/src/client.ts @@ -1,17 +1,7 @@ -// A minimal client for the STOCK `@intx/hub-api` principal and grant -// routes (`/api/tenants/:tenantId/principals`, `/grants`), spoken with the -// workflow run's own bearer credential: the sidecar token plus the run -// address, which the hub resolves to this run's principal and tenant before -// the stock handler's own `requireGrant` check runs. There is no -// Workbench-specific mirror of these routes any more. -// -// The tenant id is a path segment on every stock tenant route, so it rides -// in the client config; the sidecar threads it onto the step env from the -// hub's signed deploy frame. It is never trusted as authority — the hub sets -// the acting tenant from the authenticated run alone. -// -// Grants are NOT inherited across the tenant ancestor tree, so every read -// and write here is the run's own tenant and nothing above it. +// The tenant id rides in the client config as a path segment only, never +// trusted as authority (the hub sets the acting tenant from the +// authenticated run alone). Grants are not inherited across the tenant +// ancestor tree, so every read/write here is the run's own tenant only. import { type } from "arktype"; import { evaluateGrants, type GrantRule } from "@intx/authz"; @@ -250,23 +240,13 @@ function toGrantRule(row: GrantRow): GrantRule { export class DelegationCeilingError extends Error {} /** - * The delegation ceiling: a caller may only grant (or revoke) authority it - * already holds itself, or `grant:*`/`create` alone would let any principal - * escalate past its own reach. - * - * Stock `POST /grants` has no such check, so the tool enforces it before - * calling: it reads the caller's own grants in its own tenant and evaluates - * each requested pair with `@intx/authz`'s own `evaluateGrants`, the same - * engine the hub authorizes with. A parent tenant's grant is never consulted - * — grants are not inherited across the ancestor tree, unlike credentials - * and tool packages. - * - * Returns the first action outside the ceiling, or null when every pair is - * within it. - * - * Two fidelity gaps against a server-side check, both recorded as upstream - * asks on: role-derived grants are not visible through the stock - * principal-filtered listing, and the read-then-write is not atomic. + * A caller may only grant/revoke authority it already holds itself, or + * `grant:*`/`create` alone would let any principal escalate past its own + * reach. Stock `POST /grants` has no such check, so this enforces it + * client-side with the same `evaluateGrants` engine the hub authorizes + * with. Two known fidelity gaps against a true server-side check: + * role-derived grants aren't visible through the stock listing, and the + * read-then-write isn't atomic. */ export async function firstActionOutsideCeiling( config: AccessToolClientConfig, diff --git a/tools/access/src/tool.ts b/tools/access/src/tool.ts index 6f5ea12a91..55bf3bac17 100644 --- a/tools/access/src/tool.ts +++ b/tools/access/src/tool.ts @@ -1,19 +1,7 @@ -// The `@corbits/access-tools` bundle: `list_principals` and -// `list_grants` (plain reads, no approval) plus `grant_access` and -// `revoke_access` (`approval: "ask"` — `@intx/agent`'s native -// per-invocation gate suspends the call as a pending approval BEFORE -// this bundle's `run` ever executes, exactly like -// `@corbits/capability-tools`'s `request_capability`). Lets Myra create -// scoped grants for the specialist agents she stands up, gated behind a -// human approving each grant. -// -// `hubAccessUrl`/`sidecarToken`/`address` are threaded onto `env` by the -// sidecar's per-step env builder, the same ground every other manager- -// tools bundle's own env keys are threaded from. -// -// See `./client.ts` for the stock Interchange tenant routes this bundle's -// execution calls with the run bearer, and for the delegation ceiling it -// enforces before it grants anything. +// grant_access/revoke_access carry approval: "ask" (like request_capability), +// letting Myra create scoped grants for specialist agents gated behind a +// human approving each one. See ./client.ts for the delegation ceiling +// enforced before anything is granted. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; From c0bcf334d6edccb80e7095728910b49c3f595ab6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:36:37 -0700 Subject: [PATCH 20/31] chore(tools): comments say why, never what (catalog) --- tools/catalog/src/client.ts | 20 ++++---------------- tools/catalog/src/tool.ts | 25 ++++++------------------- 2 files changed, 10 insertions(+), 35 deletions(-) diff --git a/tools/catalog/src/client.ts b/tools/catalog/src/client.ts index 34286134bc..ea59864ac6 100644 --- a/tools/catalog/src/client.ts +++ b/tools/catalog/src/client.ts @@ -1,19 +1,7 @@ -// What this bundle reads, and from where. -// -// Both reads are STOCK Interchange tenant routes, addressed with the -// workflow run's own bearer credential (sidecar token plus run address), -// which the hub resolves to this run's principal and tenant: -// -// GET /api/tenants/:tenantId/models — the bench's resolved model -// catalog. The ancestor walk, shadowing, disable suppression and -// each offering's active price per currency are all the platform's -// own work; this package restates none of it. -// GET /api/tenants/:tenantId — the bench itself, for the model -// policy in its `config` blob under `corbits.modelPolicy`. -// -// There is no `/api/workflow-inference-catalog` mirror any more, and no -// hub-side chain resolution: the chain is computed here, in the workflow -// child, from those two reads. +// Both reads are stock Interchange tenant routes; no hub-side chain +// resolution or Workbench-specific mirror any more — the chain is computed +// here, in the workflow child, from the models list and the tenant's +// corbits.modelPolicy config blob. import { type } from "arktype"; import { DiscoveredModel } from "@corbits/inference-catalog/catalog"; import { readModelPolicy, type BenchModelPolicy } from "@corbits/inference-catalog/policy"; diff --git a/tools/catalog/src/tool.ts b/tools/catalog/src/tool.ts index b860c71756..92709ab12f 100644 --- a/tools/catalog/src/tool.ts +++ b/tools/catalog/src/tool.ts @@ -1,22 +1,9 @@ -// The `@corbits/catalog-tools` bundle: `list_model_concepts`, -// `pick_models`, and `estimate_run_cost`. -// -// The rule the whole bundle exists to enforce: an agent asks for a model by -// what the work needs — a kind of work, or the capabilities the work -// requires — and never by name. A model named from memory is a guess about -// a bench the agent cannot see; a concept is a question this bench can -// answer from its own connected providers, its own capability data, and its -// own prices. -// -// Every answer is an ordered chain, head first and fallbacks behind it, and -// every price is either real or reported as unknown. Nothing here invents a -// model, and nothing here reports a price of zero for something unpriced. -// -// All three tools read only, so none declares an `approval` key. -// -// The chain is resolved here, in the workflow child, from two stock tenant -// reads (see `client.ts`). Nothing about a kind of work, a price, or a -// bench's policy is computed by a Workbench-specific hub route. +// Enforces asking for a model by what the work needs, never by name — a +// named model is a guess about a bench the agent can't see. Every answer is +// an ordered chain with real or unknown prices, never an invented model or +// a zero price. All three tools are read-only (no approval key); the chain +// is resolved here from two stock tenant reads (see client.ts), not a +// Workbench-specific hub route. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; From 4172566b95d74d14b713dc09e5017dbe0d724b0f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:37:52 -0700 Subject: [PATCH 21/31] chore(tools): comments say why, never what (interaction) --- tools/interaction/src/client.ts | 29 ++++++----------------------- tools/interaction/src/tool.ts | 24 ++++-------------------- 2 files changed, 10 insertions(+), 43 deletions(-) diff --git a/tools/interaction/src/client.ts b/tools/interaction/src/client.ts index a9599b4a7a..732b89c098 100644 --- a/tools/interaction/src/client.ts +++ b/tools/interaction/src/client.ts @@ -62,34 +62,17 @@ export class NoOwnChannelError extends Error {} const PostedMessageResponse = type({ id: "string", createdAt: "string" }); -/** - * Derives the question card's `questionId` from a tool-call id. The call - * id is assigned by the runtime, not the model, so it is stable across a - * crash-retry of the same `ask_user` invocation — hashing it into the - * `q_` shape `postQuestion` otherwise mints keeps a retried call - * re-posting the same card without trusting model-supplied input. - */ +/** Derived from the runtime-assigned call id (never model-supplied) so a crash-retry re-posts the same card. */ export function questionIdForCall(callId: string): string { return `q_${createHash("sha256").update(callId).digest("hex").slice(0, 32)}`; } /** - * Posts a `question` block into the caller's own channel. Uses a caller- - * supplied `questionId` when present (never trusts the model to supply a - * stable, collision-free id — `ask_user` derives it from the tool-call id) - * and otherwise mints one here. Returns the id: the block-response route - * persists (and later relays, as an ordinary reply message) an answer keyed - * on this same id as `blockId` — a question block's `blockId` IS its - * `questionId`, the same way a poll's is its `pollId` - * (`packages/chat/src/schema.ts`'s `block_responses` table comment). - * `@intx/hub-common`'s `generateId` is a closed enum of platform id kinds - * (vendored, read-only source) with no "question" entry, so a minted id is - * `q_`-prefixed the same way `packages/chat/src/threads.ts`'s `thr_` ids are. - * - * Re-posting the same `questionId` into the same workbench is a no-op: - * the participants/messages route returns the existing card rather than - * inserting a second one, so a crash between this post and the tool call - * returning cannot orphan a duplicate on retry. + * Posts a `question` block; a caller-supplied `questionId` (never + * model-trusted) makes a block's `blockId` double as its `questionId`, the + * same way a poll's `blockId` is its `pollId`. Re-posting the same id is a + * no-op, so a crash between post and the tool call returning cannot orphan + * a duplicate on retry. */ export async function postQuestion( config: AskUserClientConfig, diff --git a/tools/interaction/src/tool.ts b/tools/interaction/src/tool.ts index a9b4c83598..0ab70472ca 100644 --- a/tools/interaction/src/tool.ts +++ b/tools/interaction/src/tool.ts @@ -1,11 +1,6 @@ -// The `ask_user` tool: poses an interview question in-thread as an -// interactive `question` block (`@corbits/chat`'s `blocks.ts`) instead of -// prose bullet options, then ends the turn. A Workbench agent is an -// unbounded interactive step — every inbound mail is its next turn — so -// "ask a person" is native as "post the question and stop," not a -// structural park: the person's answer, posted as an ordinary reply in the -// same channel (`packages/chat/src/routes.ts`'s block-response route, -// which sends it as a plain message), arrives as the next turn's own +// A Workbench agent is an unbounded interactive step — every inbound mail +// is its next turn — so "ask a person" is native as "post the question and +// stop," not a structural park: the answer arrives as the next turn's // inbound message, not as this call's result. import { defineTool } from "@intx/agent"; import type { BaseEnv } from "@intx/agent"; @@ -49,18 +44,7 @@ function clientConfig(env: AskUserEnv): AskUserClientConfig { }; } -/** - * `ask_user`'s `run`: posts the question card, then answers the call with a - * short instruction telling the model to end its turn — the person's answer - * is not this call's result, it is the next turn's inbound message. No gate, - * no correlation id, no park: a Workbench agent's next mail is already its - * next turn, so posting and stopping IS "asking a person" here. - * - * `postQuestion` stamps `questionIdForCall(call.id)` (derived from the - * tool-call id, not minted per attempt) on the outbound question card, so a - * crash-retry of the same call re-posts the same id and the write path - * returns the existing card rather than a duplicate. - */ +/** Posts the question then tells the model to end its turn; postQuestion keys the card on the call id so a crash-retry re-posts the same id, not a duplicate. */ async function runAskUser(env: AskUserEnv, call: ToolCall): Promise { const parsed = AskUserInput(call.arguments); if (parsed instanceof type.errors) { From 7327cf3c4e069e04dcb8d66a948ce33028e3c707 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:39:16 -0700 Subject: [PATCH 22/31] chore(tools): comments say why, never what (access, reddit follow-up) --- tools/access/src/client.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tools/access/src/client.ts b/tools/access/src/client.ts index e8d798c257..585fc795c4 100644 --- a/tools/access/src/client.ts +++ b/tools/access/src/client.ts @@ -239,15 +239,7 @@ function toGrantRule(row: GrantRow): GrantRule { * itself hold. */ export class DelegationCeilingError extends Error {} -/** - * A caller may only grant/revoke authority it already holds itself, or - * `grant:*`/`create` alone would let any principal escalate past its own - * reach. Stock `POST /grants` has no such check, so this enforces it - * client-side with the same `evaluateGrants` engine the hub authorizes - * with. Two known fidelity gaps against a true server-side check: - * role-derived grants aren't visible through the stock listing, and the - * read-then-write isn't atomic. - */ +/** Enforces client-side that a caller may only grant/revoke authority it already holds, since stock POST /grants has no such check. */ export async function firstActionOutsideCeiling( config: AccessToolClientConfig, resource: string, From fc88f4e6a157e8ee81d64eee80a3e3b92bd0d4f2 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:41:20 -0700 Subject: [PATCH 23/31] chore(sidecar): comments say why, never what (workflow-host-wiring, partial) --- apps/sidecar/src/workflow-host-wiring.ts | 165 ++++------------------- 1 file changed, 24 insertions(+), 141 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 8c1155c61b..e6ba0c8a6c 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1,9 +1,5 @@ -// Thin wiring module that constructs `createWorkflowSupervisor` with -// this sidecar's host-specific bindings: the existing mail-bus -// instance, the sidecar's Ed25519 signing keypair, the substrate -// RepoStore handle, and `Bun.spawn` as the subprocess spawner. Any -// logic that would benefit a future alternative-sidecar -// implementation lives inside `@intx/workflow-host`, not here. +// Thin wiring for createWorkflowSupervisor's sidecar-specific bindings; any +// logic reusable by a future alternative sidecar belongs in @intx/workflow-host, not here. import { rm, stat } from "node:fs/promises"; import { join as pathJoin } from "node:path"; @@ -100,14 +96,9 @@ const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); const ED25519_PUBLIC_KEY_BYTES = 32; /** - * Project an run address into the substrate-safe id of its - * workflow-run repo. Both deploy branches key `{ kind: "workflow-run", - * id }` by this slug, and the supervisor principal's `anchorRunId` - * must equal that id for the workflow-run kind handler's authz check to - * pass. The derivation is owned by `@intx/workflow-deploy` so the hub's - * read routes reconstruct the identical id; this thin delegator keeps - * the sidecar's call sites readable while the rationale and the - * substrate `SAFE_REPO_ID` contract live with the shared function. + * Projects a run address into its workflow-run repo's substrate-safe id. + * Owned by `@intx/workflow-deploy` so the hub's read routes reconstruct the + * identical id; this is a thin delegator for readability at the call sites. * * The name keeps "Deployment" deliberately: it derives the deploy-phase * routing slug (the workflow-run repo id an address projects to), not a @@ -117,14 +108,7 @@ export function deriveDeploymentId(agentAddress: string): string { return deriveWorkflowRunRepoId(agentAddress); } -/** - * The durable per-deployment store the sidecar checks a source-ref deployment's - * source assets out into. A SIBLING of the closure instance dir, not a child: - * `materializeDeploymentClosure` reclaims the closure dir on every apply and - * restore, but never this store, so the checked-out assets survive a restart - * and re-materialization needs no re-delivery. The store is reclaimed on - * redeploy (at the deploy call site) and on undeploy. - */ +/** A sibling of the closure instance dir, never reclaimed by apply/restore, so checked-out assets survive a restart. */ function deploymentSourceAssetRoot(dataDir: string, deploymentId: string): string { return pathJoin(dataDir, "workflow-definition-sources", deploymentId); } @@ -180,17 +164,7 @@ async function isExistingDir(dir: string): Promise { } } -/** - * Resolve the durable source-asset store root and `assetId -> mountPath` map a - * pinned deployment materializes its `kind:"asset"` closure entries from, - * asserting every referenced asset's mount directory is present on disk. This - * is a cheap early gate for a missing checkout; the loader still SRI-verifies - * each tarball's bytes at materialization. Derived purely from the pin, so the - * deploy path and the boot-time restore path (which has only the pin, no - * re-delivery) resolve the identical mounts. A missing mount is a broken - * deployment the hub must re-drive, so it fails loud rather than materializing - * against an absent store. - */ +/** Cheap early gate for a missing checkout (the loader still SRI-verifies bytes); a missing mount fails loud since the hub must re-drive it. */ export async function resolveDeploymentAssetMounts( dataDir: string, deploymentId: string, @@ -222,15 +196,7 @@ export async function resolveDeploymentAssetMounts( return { assetRoot, assetMounts, gitDirs }; } -/** - * Read a substrate-config byte cap (`SIDECAR_CACHE_MAX_BYTES` / - * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) from the multi-step substrate env and - * parse it to a positive finite number. The boot edge resolves these once and - * threads them through the substrate env; a source-ref deploy needs them to - * size the tarball cache and per-fetch cap when it materializes the frozen - * workflow closure. A missing or non-numeric value is a boot-edge wiring bug, - * so it fails loud rather than defaulting. - */ +/** A missing or non-numeric byte cap is a boot-edge wiring bug, so it fails loud rather than defaulting. */ function requireSubstrateByteCap(env: Record, key: string): number { const raw = env[key]; if (raw === undefined) { @@ -247,54 +213,20 @@ function requireSubstrateByteCap(env: Record, key: string): numb return parsed; } -/** - * Hub principal the deploy router presents when it writes a step's - * grants into the agent-state repo on the sidecar's substrate. The - * agent-state kind handler gates `writeTree` as hub-only; the deploy - * router is the local stand-in for the hub on the sidecar's disk, so it - * claims the hub principal for this single bookkeeping write. The child - * reads the same repo via the working-tree path (`getRepoDir`), which is - * not authorize-gated. - */ +/** The deploy router is the local stand-in for the hub on the sidecar's disk, so it claims the hub principal for this write since agent-state gates writeTree as hub-only. */ const GRANTS_WRITE_PRINCIPAL: Principal = { kind: "hub" }; -/** - * Per-deploy address/repo strategy. The single-step launched-agent - * deploy and the derived multi-step deploy disagree on how the per-step - * mail address and agent-state repo id are computed; both - * `deriveStepAddress` (consumed by the supervisor's credentialsSnapshot - * assembly for the step's mail address) and `deriveStepRepoId` (consumed - * by the same assembly to locate each step's grants) must agree on the - * choice, so they are minted together. - */ +/** Single-step and multi-step deploys disagree on how the address/repo id are computed, so the two derivations are minted together to stay in sync. */ type StepStrategy = { deriveStepAddress: DeriveStepAddress; deriveStepRepoId: DeriveStepRepoId; }; /** - * Decide the per-step address/repo strategy from the projection's step - * count. - * - * `stepOrder.length === 1` is the single-agent deploy: the sole - * step keeps the deploy's own mail address, so its grants live in the - * agent-state repo keyed by `parseAgentId(address)`. The spawned child - * reads the agent's grants from where the run's identity already - * lives, and the deploy frame's run address is preserved (the - * workflow-run repo stays keyed by `deriveWorkflowRunRepoId(address)`). - * - * Any other step count is a derived multi-step deploy: each step gets a - * derived `-` mail address (via the router's - * `multistepDeriveStepAddress`) and a derived agent-state repo under the - * default `-` convention. - * - * NOTE: the supervisor's `deriveStepAddress` feeds the credentials - * snapshot's per-step mail `address` and the grants-repo derivation. It - * does NOT feed the child's on-disk tool read (`stepDeployTreeDir` in - * `step-agent-tools.ts`), which re-derives the step address from the - * deployment mailbox address independently. The deploy tree must - * therefore be staged at the address `stepDeployTreeDir` computes, - * regardless of this strategy's address choice. + * A single-step deploy keeps the deploy's own mail address; any other step + * count derives a `-` address/repo per step. Note this does + * NOT feed step-agent-tools.ts's stepDeployTreeDir, which re-derives the + * step address independently — the deploy tree must still be staged there. */ function createStepStrategy(args: { legacyAddress: string; @@ -304,12 +236,7 @@ function createStepStrategy(args: { if (args.stepOrder.length === 1) { return { deriveStepAddress: () => args.legacyAddress, - // `parseAgentId` is deferred into the closure rather than computed - // eagerly: the supervisor only invokes `deriveStepRepoId` while - // assembling the credentialsSnapshot inside `spawn()`, so a - // malformed address surfaces at the same point the rest of the - // spawn path would fault rather than ahead of the deploy router's - // other boundary checks. + // Deferred into the closure so a malformed address surfaces at the same point the rest of spawn() would fault. deriveStepRepoId: () => ({ kind: "agent-state", id: parseAgentId(args.legacyAddress), @@ -412,23 +339,10 @@ export type AssembleRunCredentialsSnapshotOpts = { }; /** - * Resolve a run's credentials snapshot for the `onRunStart` grants - * barrier from its per-run grants file. - * - * Every legitimate run birth path writes `runs//grants.json` in - * the deployment's workflow-run repo before the run dispatches -- the - * external trigger route and the mail-triggered path both ship a - * `run.grants` frame the sidecar writes, and a spawned child inherits its - * parent's grants directly at spawn without reaching this barrier. The - * per-run file IS the run's snapshot: the run's single flat grant set is - * applied uniformly across every step, keyed on each step's address. - * - * A missing file is therefore not an internal run inheriting deploy-time - * grants -- it is a run that reached its barrier with no grants written, - * so it FAILS CLOSED here rather than running under-authorized. A file - * that exists but is malformed also throws (via `readRunGrants`), for the - * same reason: the file's presence implies a grants frame was delivered, - * so a structural failure is a boundary bug, not a default. + * Every legitimate run birth path writes `runs//grants.json` before + * the run dispatches, so a missing file means the run reached its barrier + * with no grants written; this fails closed rather than running + * under-authorized. A malformed file also throws for the same reason. */ export async function assembleRunCredentialsSnapshot( opts: AssembleRunCredentialsSnapshotOpts, @@ -456,49 +370,18 @@ export async function assembleRunCredentialsSnapshot( return { steps }; } -// The supervisor's `binaryPath` binding resolves to the sidecar's -// own `bin/workflow-child` script via `import.meta.resolve` against -// the `@intx/sidecar-app` package. The script lives next to this -// wiring module (`../bin/workflow-child`); resolving it statically -// at wiring-module load time keeps the production spawn surface -// independent of any runtime env override. Tests inject a sentinel -// path via the `binaryPath` opts override; production wiring -// closes over this constant. +// Resolved statically at module load so the production spawn surface is +// independent of any runtime env override; tests inject a sentinel path +// via the binaryPath opts override instead. const SIDECAR_WORKFLOW_CHILD_BINARY: string = (() => { const url = import.meta.resolve("../bin/workflow-child"); return fileURLToPath(url); })(); -/** - * Child fd the supervisor inherits the event-channel pipe on. The - * supervisor's spawn-time convention is: - * - * fd 0 stdin -- downstream control channel (supervisor -> child) - * fd 1 stdout -- upstream control channel (child -> supervisor) - * fd 2 stderr -- inherited so child diagnostics land on the - * sidecar's stderr - * fd 3 -- event-channel write side (child writes - * HMAC-authenticated InferenceEvent frames here; - * the supervisor reads the parent end as a - * `FrameReader`) - * - * The child opens fd 3 via `EVENT_CHANNEL_FD` in - * `@intx/workflow-host`'s `from-process-env`. The two ends of the - * pipe are provisioned by `Bun.spawn`'s `stdio` slot: setting - * `stdio[3] = "pipe"` makes Bun mint a pipe pair where the child - * inherits the write half at fd 3 and the parent receives the read - * half as a numeric fd at `proc.stdio[3]` in its own address space. - */ +/** fd 0/1 are the control channel, fd 2 is inherited stderr, fd 3 is the event-channel pipe Bun.spawn's stdio[3]="pipe" provisions. */ const CHILD_EVENT_CHANNEL_FD = 3; -/** - * Wrap a Bun `FileSink` as the supervisor's `NdjsonWriter`. The - * supervisor's control-channel sender writes one JSON line per - * frame (already including the trailing newline); the writer is - * responsible for passing the bytes through to the child's stdin - * without buffering across frames so each frame surfaces on the - * far side as soon as `write()` resolves. - */ +/** Passes each already-newline-terminated frame through without buffering, so it surfaces as soon as write() resolves. */ function ndjsonWriterFromFileSink(sink: Bun.FileSink): NdjsonWriter { return { async write(line: string): Promise { From f4d9e6cfb7970a4e649f41bd0b3e01233a2fdf95 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:42:36 -0700 Subject: [PATCH 24/31] chore(sidecar): comments say why, never what (workflow-host-wiring, more) --- apps/sidecar/src/workflow-host-wiring.ts | 85 +++--------------------- 1 file changed, 11 insertions(+), 74 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e6ba0c8a6c..4204462604 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -95,15 +95,7 @@ const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); // A raw Ed25519 public key is 32 bytes. const ED25519_PUBLIC_KEY_BYTES = 32; -/** - * Projects a run address into its workflow-run repo's substrate-safe id. - * Owned by `@intx/workflow-deploy` so the hub's read routes reconstruct the - * identical id; this is a thin delegator for readability at the call sites. - * - * The name keeps "Deployment" deliberately: it derives the deploy-phase - * routing slug (the workflow-run repo id an address projects to), not a - * run identity, so it survives the run-first identity sweep. - */ +/** Thin delegator to `@intx/workflow-deploy` so the hub's read routes reconstruct the identical id; named for the deploy-phase routing slug, not a run identity. */ export function deriveDeploymentId(agentAddress: string): string { return deriveWorkflowRunRepoId(agentAddress); } @@ -253,34 +245,10 @@ function createStepStrategy(args: { } /** - * Write a grant set into the sidecar's substrate as the canonical - * `{ grants: WireGrantRule[] }` envelope -- the shape - * `assembleCredentialsSnapshot` validates (`{ grants: unknown[] }`) and - * the child's `evaluateGrants` adapter narrows to `GrantRule[]`. - * - * Two destinations share this write machinery, selected by `runId`: - * - * - `runId` absent (deploy time): write every step's grants into its - * own `agent-state` repo at `STEP_GRANTS_PATH`, keyed by the same - * `deriveStepRepoId` the supervisor reads with, so read and write - * address the same repo. `stepOrder` here is the deployment's whole - * flat step-id namespace, not the bare top-level order, because the - * supervisor assembles a snapshot entry for every loop-body step too. - * The write is on the spawn critical path: a failure rejects the - * deploy (the caller's `finally` unwinds the partial state) rather - * than spawning a child that would fail every authorize closed - * against an empty grant set. - * - * - `runId` present (per-run delivery): write a single - * `runs//grants.json` into the deployment's `workflow-run` - * repo, sibling to that run's `runs//events/` subtree. The - * `stepOrder` / `deriveStepRepoId` fields are unused in this mode -- - * the destination is the one workflow-run repo keyed by - * `runId`, not a per-step fan-out. - * - * Both destinations use the same hub principal and `refs/heads/main` - * ref: the agent-state kind handler gates `writeTree` as hub-only, and - * the workflow-run repo pins its run subtree under the same moving ref. + * Two destinations selected by `runId`: absent means deploy time (writes + * every step's grants into its own agent-state repo, on the spawn critical + * path so a failure rejects the deploy); present means per-run delivery + * (writes one `runs//grants.json`, ignoring stepOrder/deriveStepRepoId). */ async function writeStepGrants(args: { repoStore: RepoStore; @@ -462,19 +430,9 @@ function frameReaderFromFd(fd: number): FrameReader { } /** - * Real `Bun.spawn`-backed subprocess spawner. Constructs a fresh env - * carrying exactly the trust anchors and substrate-config keys the - * supervisor passed in (no inheritance of the sidecar's process env); - * inherits stdio 0/1/2 as control + stderr; pipes fd 3 for the event - * channel and surfaces the parent-side read fd as the supervisor's - * `FrameReader`. - * - * Failure modes flow through the returned handle's `exited` promise. - * A `Bun.spawn` that fails to launch (binary missing, env malformed, - * `EXEC` error) settles `exited` with a non-zero code; the - * supervisor's `wireChild` races `exited` against `readyPromise` - * inside `spawn()` so a spawn-time crash surfaces as a rejected - * spawn rather than a wedged `starting` state. + * Constructs a fresh env with no inheritance of the sidecar's process env. + * A launch failure settles `exited` non-zero; the supervisor races that + * against `readyPromise` so a spawn-time crash rejects rather than wedging. */ export const defaultSubprocessSpawner: SubprocessSpawner = ({ binaryPath, @@ -496,13 +454,7 @@ export const defaultSubprocessSpawner: SubprocessSpawner = ({ controlReader: ndjsonReaderFromReadableStream(proc.stdout), eventReader: frameReaderFromFd(eventFd), kill(signal?: number | string): void { - // The supervisor's `SubprocessHandle.kill` widens the signal - // to `number | string`; Bun's `Subprocess.kill` accepts - // `number | NodeJS.Signals`. The supervisor's call sites pass - // `"SIGTERM"` / `"SIGKILL"` (recycle path) or no argument - // (shutdown path), which Bun handles directly. Cast at the - // boundary so the inner call matches Bun's narrower type - // without coercing valid input. + // Bun's kill() takes NodeJS.Signals, narrower than the supervisor's number|string; cast at the boundary. if (signal === undefined) { proc.kill(); return; @@ -543,28 +495,13 @@ export type CreateSidecarWorkflowSupervisorOpts = { * collapses onto the head for a single-step deployment. */ stepCount: number; - /** - * Every step id in the deployment's flat step-id namespace: its - * `stepOrder` plus the step ids of every `loop` body it carries. The - * `onRunStart` grants sink walks these to assemble the per-run - * credentialsSnapshot, so the sink needs the ids rather than the bare - * count -- and it needs the loop-body ids too, because a loop iteration - * inherits the parent run's env and authorizes against this same snapshot. - */ + /** Every step id (including loop-body ids) the onRunStart grants sink walks to assemble the per-run credentialsSnapshot. */ stepOrder: readonly string[]; /** Deployment's mail address. */ deploymentMailAddress: string; /** Per-step mail-address derivation. */ deriveStepAddress: DeriveStepAddress; - /** - * Optional override of the per-step `agent-state` repo identity the - * supervisor reads grants from while assembling the - * credentialsSnapshot. Defaults to the `-` - * convention; the single-step launched-agent deploy supplies a - * derivation that returns the legacy agent-state repo so the spawned - * child reads grants from the same repo the legacy agent identity - * keys. - */ + /** Overrides the default `-` repo id; single-step deploy returns the legacy agent-state repo instead. */ deriveStepRepoId?: DeriveStepRepoId; /** Substrate-config keys propagated to the child via spawn-time env. */ substrateEnv: Record; From 9007332881a04e81e78c4fe7244c46b2fc0ad36f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:51:12 -0700 Subject: [PATCH 25/31] chore(sidecar): comments say why, never what (workflow-host-wiring cont'd 1) --- apps/sidecar/src/workflow-host-wiring.ts | 184 +++-------------------- 1 file changed, 24 insertions(+), 160 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 4204462604..e8109d47a0 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -399,16 +399,7 @@ function ndjsonReaderFromReadableStream(stream: ReadableStream): Ndj }; } -/** - * Wrap the parent-side read fd of the event-channel pipe as the - * supervisor's `FrameReader`. The child publishes one HMAC- - * authenticated envelope per `FileSink.write()` and the supervisor's - * `receiveEventChannel` parses each yielded `Uint8Array` as one - * complete envelope. The pipe is a byte stream; this reader yields - * each raw chunk the kernel delivers and trusts the sender's - * one-write-per-envelope discipline. The buffer-overflow / framing - * discipline lives in `receiveEventChannel`'s parser. - */ +/** Yields each raw chunk the kernel delivers and trusts one-write-per-envelope; framing discipline lives in receiveEventChannel's parser. */ function frameReaderFromFd(fd: number): FrameReader { const stream = Bun.file(fd).stream(); return { @@ -483,17 +474,9 @@ export type CreateSidecarWorkflowSupervisorOpts = { workflowRunRef: string; /** Deployment id baked into principal claims and address derivation. */ runId: string; - /** - * Decrypted credential material for the deployment's tools (from the deploy - * frame). Delivered to the child on the pre-trigger barrier. Absent when the - * deployment binds no credentials. - */ + /** Decrypted credential material delivered to the child on the pre-trigger barrier; absent when the deployment binds no credentials. */ credentialDelivery?: CredentialDelivery; - /** - * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). - * Threaded into the child's spawn-time env so its deploy-tree read - * collapses onto the head for a single-step deployment. - */ + /** Threaded into the child's spawn-time env so its deploy-tree read collapses onto the head for a single-step deployment. */ stepCount: number; /** Every step id (including loop-body ids) the onRunStart grants sink walks to assemble the per-run credentialsSnapshot. */ stepOrder: readonly string[]; @@ -505,140 +488,48 @@ export type CreateSidecarWorkflowSupervisorOpts = { deriveStepRepoId?: DeriveStepRepoId; /** Substrate-config keys propagated to the child via spawn-time env. */ substrateEnv: Record; - /** - * Dynamic spawn-env fragment the supervisor recomputes on every spawn and - * recycle respawn (e.g. a live-rotated inference-source list). Its keys - * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. - */ + /** Recomputed on every spawn and recycle respawn (e.g. a live-rotated inference-source list); layers over substrateEnv. */ dynamicSpawnEnv: () => Record; - /** - * Override the subprocess spawner. Tests inject a deterministic - * mock; production defaults to the `Bun.spawn`-backed - * `defaultSubprocessSpawner`. - */ + /** Tests inject a deterministic mock; production defaults to defaultSubprocessSpawner. */ subprocessSpawner?: SubprocessSpawner; /** Override the `bin/workflow-child` path. */ binaryPath?: string; - /** - * Optional per-message dispatch-timing observer, forwarded verbatim to - * the supervisor's `onDispatchTiming` binding. Absent in production; - * the deploy router wires it (off a benchmark env gate) only for the - * Phase 4.7 latency gate, which needs the supervisor to emit the - * per-message infra round-trip from inside the sidecar subprocess. - */ + /** Absent in production; the deploy router wires this (off a benchmark env gate) only for the Phase 4.7 latency gate. */ onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** - * D2 §10c forced-repack A/B toggle, forwarded verbatim to the - * supervisor's `repackEveryMessages` binding. Absent in production; - * the deploy router wires it (off the same benchmark env gate) only - * for the D2 attribution run. - */ + /** Absent in production; the deploy router wires this (off the same benchmark env gate) only for the D2 attribution run. */ repackEveryMessages?: { everyMessages: number }; - /** - * Consumed-dedup retention horizon (ms), forwarded to the - * supervisor's `consumedRetentionMs` binding. The boot edge resolves - * the operator's `CONSUMED_RETENTION_MS` config; absent, the - * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). - */ + /** Boot edge resolves the operator's CONSUMED_RETENTION_MS; absent, the supervisor applies its own 24h default. */ consumedRetentionMs?: number; - /** - * Spawn ready-handshake timeout (ms), forwarded to the supervisor's - * `readyTimeoutMs` binding. The boot edge resolves the operator's - * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies - * `DEFAULT_READY_TIMEOUT_MS` (30s). - */ + /** Boot edge resolves the operator's CHILD_READY_TIMEOUT_MS; absent, the supervisor applies its own 30s default. */ readyTimeoutMs?: number; - /** - * Control-plane suspension sink forwarded to the supervisor's - * `onSuspensionRegister` binding. The supervisor stamps `runId` + - * `agentAddress` and invokes this when a workflow-process child reports a - * `park.notify`; production wiring routes it to the hub link so a - * `signal.correlation.register` frame reaches the hub. Absent means the - * deployment registers no suspensions. - */ + /** Invoked when a child reports park.notify; production wiring routes it to the hub link's signal.correlation.register. */ onSuspensionRegister?: (registration: SuspensionRegistration) => void; - /** - * Self-termination sink forwarded to the supervisor's `onSelfTerminate` - * binding. The supervisor invokes it when it drives itself to a terminal - * phase (crash-loop latch, channel crash, recycle failure); production wiring - * routes it to the deploy router's address reclaim. Absent means a - * self-termination is not surfaced to the host. - */ + /** Invoked on a terminal phase (crash-loop, channel crash, recycle failure); production routes it to the deploy router's address reclaim. */ onSelfTerminate?: (info: { phase: "stopped" | "crash-looping"; reason: string }) => void; - /** - * Predicate consulted at the `onRunStart` grants barrier: returns `true` - * for a runId whose `run.grants` write was attempted and FAILED, so the - * per-run grants file the run needs never landed. The barrier fails such a - * run loudly rather than starting it under-authorized. This is a - * fast-fail with a precise cause: an absent grants file already fails the - * barrier closed on its own (every run birth path writes the file before - * dispatch, so its absence is a defect), and this predicate lets a run - * whose write is KNOWN to have failed reject with that specific reason - * instead of a generic missing-file error. Absent means no run is - * poisoned. - */ + /** Lets a run whose grants write is KNOWN to have failed reject with that specific reason instead of a generic missing-file error. */ isRunPoisoned?: (runId: string) => boolean; }; export type SidecarWorkflowSupervisor = { supervisor: WorkflowSupervisor; - /** - * Hand a delivered inbound message off to the supervisor's mail - * subscription. The returned promise resolves once the message is durably - * accepted and rejects when it was not, so the hub-link can send a - * `mail.inbound.ack` only on resolution (resolve = ack, reject = withhold). - */ + /** Resolves once durably accepted (so the hub-link can send mail.inbound.ack only then) and rejects otherwise. */ routeInbound(message: Uint8Array): Promise; /** Snapshot accessor that proxies the supervisor's credentials view. */ getCredentialsSnapshot(): CredentialsSnapshot | null; - /** - * The per-run grants barrier the supervisor awaits before firing a run's - * trigger. Rejects for a poisoned run (its `run.grants` write failed) and - * otherwise resolves the run's credentials snapshot. Exposed so the barrier - * can be exercised without driving a full spawn. - */ + /** The per-run grants barrier; rejects for a poisoned run, exposed so it can be exercised without driving a full spawn. */ onRunStart(args: { runId: string; anchorRunId: string }): Promise; }; -/** - * Env key the multi-step branch uses to carry each step's ordered - * inference-source failover chain from `frame.workflow.sources` down to - * the workflow-process child. The substrate factory's `buildEnv` reads - * this and resolves a step's chain at step invocation, feeding it to the - * reactor for forward-only failover; the supervisor itself is opaque to - * the value (it is plumbed through `bindings.substrateEnv` verbatim). - * - * Listed here so the router and the future substrate-factory consumer - * spell the key the same way without a magic-string trip hazard. - */ +/** Listed here so the router and the substrate-factory consumer spell this env key the same way without a magic-string trip hazard. */ export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; -/** - * Spawn-env key carrying every spawned body's per-step inference-source pins as - * a JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` map. The sidecar - * decrypts the sealed body sources from the run record and serializes the - * plaintext here so the run child resolves a body's sources without holding the - * sidecar's cipher key. Mirrors `STEP_INFERENCE_SOURCES` for the top level. - */ +/** Carries every spawned body's per-step inference-source pins as plaintext JSON so the child resolves them without the sidecar's cipher key. */ export const WORKFLOW_BODY_SOURCES_ENV_KEY = "WORKFLOW_BODY_SOURCES"; -/** - * A single env value cannot exceed the OS argument-string ceiling - * (`MAX_ARG_STRLEN`, 128 KiB on Linux); an over-long value makes the child - * `execve` fail opaquely at spawn. `WORKFLOW_BODY_SOURCES` sums over every body - * at every depth, so the deploy path validates the serialized size against this - * bound and rejects an over-large deployment loudly at deploy time rather than - * letting a much-later spawn fail. Held below the ceiling with headroom for the - * rest of the env block. - */ +/** Below the OS argument-string ceiling (128 KiB on Linux) with headroom, so an over-large deployment is rejected loudly at deploy time, not at a much-later spawn's execve. */ const WORKFLOW_BODY_SOURCES_MAX_BYTES = 96 * 1024; -/** - * Build the record's `bodySources` map from the deploy frame's referenced body - * definitions (the flat set of onTrigger sections and childWorkflow children, - * each already source-pinned by the hub). `undefined` when the deployment - * references no bodies, so the record omits the field. - */ +/** `undefined` when the deployment references no bodies, so the record omits the field. */ function buildBodySourcesMap( referenced: NonNullable["referencedDefinitions"], ): WorkflowRunRecord["bodySources"] { @@ -653,39 +544,12 @@ function buildBodySourcesMap( } /** - * Validate the wire-projected workflow definition at the deploy-router - * boundary. The arktype `AgentDeployFrame` validator enforces the - * wire shape (`id` is non-empty, `stepOrder` is `string[]`, `steps` - * is an object, `sources` covers every `stepOrder` entry); this - * function takes `unknown`-typed inputs so it can also gate callers - * that bypass the wire boundary, and it enforces the invariants the - * router and the downstream supervisor rely on: - * - * - `definition.id` is a non-empty string. The arktype shape - * already enforces this on the wire; the re-check here protects - * bypass callers and keeps the failure shape consistent with the - * other invariants this function owns. - * - `definition.stepOrder` is non-empty. The wire shape admits - * `[]`; a zero-step workflow has no semantics here. - * - Every `stepOrder` entry matches `STEP_ID_PATTERN` so per-step - * mail-address derivation never needs escaping at the substrate - * boundary. - * - Every `stepOrder` entry has a corresponding `steps[id]` entry. - * The wire shape lets `steps[id]` be `unknown` and lets the - * entry be absent; presence is required so the workflow-process - * child can resolve each step's primitive at run time. - * - Every `stepOrder` entry has a corresponding `sources[id]` - * entry, and that entry is a non-empty array (the step's ordered - * failover chain). The arktype narrow already enforces both; the - * re-check here surfaces a structured router-side error instead of - * an arktype validation failure at the wire boundary, which keeps - * the failure shape consistent with the rest of the validations - * this function owns. An empty chain would leave the reactor with - * no initial source, so it is rejected here rather than deferred to - * a deep-stack child failure. - * - * A rejection here surfaces as a thrown `Error` the link's deploy - * frame caller converts into a structured failure reply. + * Takes `unknown`-typed inputs (unlike the arktype `AgentDeployFrame` + * validator) so it can also gate callers that bypass the wire boundary: + * non-empty id, non-empty stepOrder, every step id matches + * STEP_ID_PATTERN, every stepOrder entry has a steps[id] and a non-empty + * sources[id] chain. A rejection surfaces as a thrown Error the link's + * deploy frame caller converts into a structured failure reply. */ export function validateWorkflowProjection(projection: { definition: { id: unknown; stepOrder: unknown; steps: unknown }; From 6f2a9c78a526eab1d76a2eb1a043a90d6f44f20c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:53:32 -0700 Subject: [PATCH 26/31] chore(sidecar): comments say why, never what (workflow-host-wiring cont'd 2) --- apps/sidecar/src/workflow-host-wiring.ts | 399 +++-------------------- 1 file changed, 46 insertions(+), 353 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index e8109d47a0..0e5e2d2613 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -543,14 +543,7 @@ function buildBodySourcesMap( return map; } -/** - * Takes `unknown`-typed inputs (unlike the arktype `AgentDeployFrame` - * validator) so it can also gate callers that bypass the wire boundary: - * non-empty id, non-empty stepOrder, every step id matches - * STEP_ID_PATTERN, every stepOrder entry has a steps[id] and a non-empty - * sources[id] chain. A rejection surfaces as a thrown Error the link's - * deploy frame caller converts into a structured failure reply. - */ +/** Takes `unknown` inputs (unlike the arktype AgentDeployFrame validator) so it can also gate callers that bypass the wire boundary. */ export function validateWorkflowProjection(projection: { definition: { id: unknown; stepOrder: unknown; steps: unknown }; sources: unknown; @@ -603,177 +596,52 @@ export function validateWorkflowProjection(projection: { } } -/** - * Derive the supervisor's principal public key from the sidecar's - * Ed25519 signing seed. The supervisor signs every workflow-run event - * with this key; the multi-step branch surfaces it to the link so the - * hub records the verifying key for the deployment's signed events. - */ +/** The supervisor signs every workflow-run event with this key; the multi-step branch surfaces it so the hub records the verifying key. */ async function derivePrincipalPublicKeyHex(signingKeySeed: Uint8Array): Promise { return hexEncode(await derivePublicKeyBytes(signingKeySeed)); } -/** - * The sidecar's `DeployRouter` plus the boot-time restore driver. The link - * routes `agent.deploy`/`agent.undeploy` through the `DeployRouter` surface; - * the sidecar boot edge additionally calls `restoreWorkflowRuns` once, - * before connecting to the hub, to re-establish the deployments a prior - * process persisted. The extra method is sidecar-app-only, so it rides on the - * concrete router type rather than the shared `DeployRouter` contract. - */ +/** Adds the sidecar-app-only boot-time restore driver on top of the shared `DeployRouter` contract. */ export interface SidecarDeployRouter extends DeployRouter { - /** - * Re-establish every persisted workflow deployment on this sidecar's local - * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step - * head's mailbox/transport registration is live before the hub routes to it. - * Soft-fails per deployment: a record that cannot be restored (unbuildable - * provider, a source closure that cannot be re-materialized, spawn failure) - * is logged and left on disk for a later boot to retry -- it is never - * deleted here. - */ + /** Soft-fails per deployment (a record left on disk for a later boot to retry, never deleted) rather than failing the whole boot. */ restoreWorkflowRuns(): Promise; - /** - * The workflow-substrate deployment addresses (`run_@domain`) this router - * currently hosts a live supervisor for -- the set of addresses this - * sidecar can route mail to. The boot edge announces these to the hub on - * (re)connect so the hub re-registers them for routing: they are hub-minted - * and carry no per-address key; the allocation-authenticated announcement is - * what re-establishes their routes after a WS reconnect. Reflects - * `deploy`/`undeploy` - * and boot-time restore live, so a caller re-reads it per connect. - */ + /** Addresses this router currently hosts a live supervisor for; re-read per connect since deploy/undeploy/restore change it live. */ activeAddresses(): string[]; - /** - * Trigger B: re-register every correlation the deployment at `address` is - * currently parked on, by asking its live supervisor to re-emit them to the - * hub. The boot edge calls this per address the hub link re-routes on an - * authenticated reconnect (`onWorkflowAddressesRoutable`), so a register frame - * the hub dropped from its bounded send queue during the outage is recovered. - * The address-dispatch wrapper around the supervisor's own no-arg - * `reEmitParkedCorrelations`; fire-and-forget (the driver is best-effort and - * watchdog-bounded inside the supervisor). - */ + /** Re-registers parked correlations a long outage may have evicted from the hub's bounded send queue; fire-and-forget. */ reEmitParkedCorrelations(address: string): void; } export function createSidecarDeployRouter(deps: { sessions: SessionManager; keyStore: AgentKeyStore; - /** - * Cache of hub-vouched sender public keys. The grants handler writes each - * co-delivered `senderIdentities` entry here before the run's grants land, - * so a durable grant is never missing the key its recipient needs to verify - * the sender's inbound mail. - */ + /** Grants handler writes each co-delivered senderIdentities entry here before the run's grants land, so a durable grant always has its verify key. */ senderKeyCache: SenderKeyCache; transport: HubTransport; repoStore: RepoStore; signingKeySeed: Uint8Array; - /** - * The sidecar cipher that seals credential material at rest. Used to seal - * each run record's inference-source apiKeys on persist and unseal them on - * the boot scan, so the durable record carries ciphertext rather than - * plaintext secrets. - */ + /** Seals each run record's inference-source apiKeys on persist and unseals them on the boot scan. */ credentialCipher: CredentialCipher; - /** - * Per-agent crypto factory. Receives the agent's raw key pair and - * returns a `CryptoProvider` bound to it (production wires - * `@intx/crypto`'s `createEd25519Crypto`). The multi-step branch - * uses this to register the spawned single-step agent's signing key on - * the host transport before `spawn()`, so the supervisor's outbound - * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies - * with the AGENT's identity -- the OUTBOUND half of mailbox ownership - * (§3a). Without this registration the spawned agent's address has no - * `CryptoProvider` on the transport (nothing else registers one for - * it), and an outbound send would throw "address is not registered" - * rather than emit unsigned mail. - */ + /** Registers the spawned agent's signing key on the transport before spawn(), or an outbound send throws "address is not registered". */ createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; - /** - * Source-admission gate: throws if a step's pinned inference source - * names a provider this sidecar cannot build. The buildable-provider - * set is sidecar config (the boot edge's adapter registry), so this - * admission control lives at the sidecar -- the hub is a different - * process and cannot know a given sidecar's providers. Production wires - * the default harness builder's `canBuildSource` verbatim, so a rejected - * provider carries the same `"... is not registered"` message. - * - * Distinct from the orchestrator's operator-approval check - * (`pickStepInferenceSource`): that gates on whether the operator - * approved a `provider:model`; this gates on whether the provider is - * buildable at all. A source can be approved yet unbuildable. - */ + /** Distinct from the operator-approval check: this gates on whether a provider is buildable at all, not whether it was approved. */ assertSourceBuildable: (source: InferenceSource) => void; - /** - * Record a `(runId -> agentAddress)` mapping the boot edge's - * workflow-run pack push facade consults when it must address an - * outbound pack frame. Fires once per inbound `agent.deploy` frame - * before the deployment's supervisor spawns, so the first pack push - * the child triggers sees the mapping. Tests that do not exercise - * the pack push path may pass a no-op. - */ + /** Fires once per inbound agent.deploy before the supervisor spawns, so the first pack push sees the mapping. No-op-safe for tests. */ registerDeployment: (entry: { runId: string; agentAddress: string }) => void; - /** - * Symmetric removal hook for `registerDeployment`. Fires from the - * link's `agent.undeploy` path so the boot edge's - * `DeploymentAddressRegistry` drops the mapping when the deployment - * is torn down. A subsequent stale `writeTreePreservingPrefix` - * against the dead deployment's workflow-run ref surfaces - * structurally (`registry.resolve` returns `null`) rather than - * silently resolving to the prior address. Tests that do not - * exercise the pack push path may pass a no-op. - */ + /** Symmetric removal hook so a stale writeTreePreservingPrefix fails structurally instead of resolving to the prior address. */ unregisterDeployment: (entry: { runId: string; agentAddress: string }) => void; - /** - * Substrate-config env keys the multi-step branch propagates into - * the workflow-process child's spawn-time env (see - * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `workflow-substrate-factory.ts`). - * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step - * frame. Defaults to an empty record so a router built without - * substrate config (e.g. a test) needs no boot-edge threading. - */ + /** Defaults to empty so a router built without substrate config (e.g. a test) needs no boot-edge threading. */ multistepSubstrateEnv?: Record; - /** - * Subprocess spawner the multi-step branch hands to the supervisor. - * Defaults to the production `Bun.spawn`-backed - * `defaultSubprocessSpawner`; tests inject a deterministic mock. - */ + /** Defaults to the production Bun.spawn-backed spawner; tests inject a deterministic mock. */ multistepSubprocessSpawner?: SubprocessSpawner; - /** - * Optional override for the resolved `bin/workflow-child` path the - * multi-step branch hands to the supervisor. Production wiring uses - * the package-local default; tests inject a sentinel value so the - * mock spawner can assert on it. - */ + /** Production wiring uses the package-local default; tests inject a sentinel value the mock spawner can assert on. */ multistepBinaryPath?: string; - /** - * Callback the supervisor invokes for every verified InferenceEvent - * the workflow-process child publishes. The router threads the - * deployment's run address plus the deploy's session id through to - * the callback so a downstream fan-out can route each event to the - * hub timeline keyed to the right session. The `InferenceEvent` itself - * is sessionless; the session id rides alongside it, sourced from the - * deploy frame's `HarnessConfig.sessionId` per deployment. It is - * optional because a deploy frame need not carry a session id (a - * headless deployment with no hub-side session); the sink decides what - * an absent session id means. Defaults to a no-op; production wiring - * supplies the event publisher. - */ + /** Session id rides alongside the (sessionless) InferenceEvent since it's optional for a headless deployment; defaults to a no-op. */ publishWorkflowInferenceEvent?: ( agentAddress: string, event: InferenceEvent, sessionId: string | undefined, ) => void; - /** - * Callback the supervisor invokes for every control-plane suspension a - * workflow-process child reports (`park.notify`). The multi-step branch - * threads it into the supervisor's `onSuspensionRegister` binding so a - * parked run's correlation is registered at the hub (routing + approval - * rows). The supervisor stamps `runId` + `agentAddress` before - * invoking it. Defaults to a no-op; production wiring supplies the - * hub-link-backed publisher. - */ + /** Registers a parked run's correlation at the hub; defaults to a no-op. */ publishWorkflowSuspension?: (registration: { correlationId: string; runId: string; @@ -782,149 +650,29 @@ export function createSidecarDeployRouter(deps: { kind: SignalKind; approvalSnapshot?: ApprovalSnapshot; }) => void; - /** - * Optional override for the multi-step branch's per-step mail-address - * derivation. Defaults to `${runId}-${stepId}@` - * derived from the frame's run address. Tests inject a deterministic - * factory. - */ + /** Defaults to `${runId}-${stepId}@`; tests inject a deterministic factory. */ multistepDeriveStepAddress?: DeriveStepAddress; - /** - * Per-deployment-address mail handler registry the hub-link's - * `mail.inbound` path consults before falling back to the legacy - * session-routed delivery. The multi-step branch registers - * `wired.routeInbound` against the deployment's mail address once - * `supervisor.spawn` succeeds so inbound mail aimed at the - * deployment address flows into the supervisor's mail-bus - * subscription. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end mail loop can omit the binding; an absent registry - * simply means multi-step inbound mail cannot route through the - * hub-link until the wiring is plumbed. - */ + /** Registers wired.routeInbound once spawn succeeds; absent means multi-step inbound mail cannot route until wired. */ multistepMailRouter?: MultistepMailRouter; - /** - * Per-recipient-address registry of resolved inbound-mail admission - * policies the sidecar hub-link's `mail.inbound` seam enforces. The - * multi-step branch resolves this deployment's authored - * `inboundMailPolicy` into the registry once `supervisor.spawn` - * succeeds -- beside the mail-router registration -- and removes the - * entry in the same teardown, so a reused address never inherits a - * stale policy. - * - * Optional so tests that exercise the multi-step branch without the - * hub-link seam can omit it; an absent registry means an inbound frame - * for the address resolves to the fully-closed default and is rejected - * until the wiring is plumbed. - */ + /** Removed in the same teardown as the mail-router registration so a reused address never inherits a stale policy; absent means fully-closed default. */ inboundMailPolicyRegistry?: InboundMailPolicyRegistry; - /** - * Per-deployment-address signal handler registry the sidecar - * hub-link's `signal.deliver` path consults. The multi-step branch - * registers `wired.supervisor.deliverSignal` against the deployment's - * mail address once `supervisor.spawn` succeeds so a hub-side - * `signal.deliver` frame flows into the workflow-process child via - * the IPC's `signal.deliver` payload. The child commits the - * resulting `SignalReceived` event through its own substrate, - * preserving the workflow-run repo's single-writer invariant on the - * sidecar side. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end signal loop can omit the binding; an absent registry - * means hub-side signals cannot route through the hub-link until the - * wiring is plumbed. - */ + /** The child commits the resulting SignalReceived through its own substrate, preserving the single-writer invariant sidecar-side. */ multistepSignalRouter?: MultistepSignalRouter; - /** - * Per-deployment-address drain handler registry the sidecar - * hub-link's `drain.deliver` path consults. The multi-step branch - * registers `wired.supervisor.drain` against the deployment's mail - * address once `supervisor.spawn` succeeds so a hub-side - * `drain.deliver` frame flows into the workflow-process child via - * the IPC's `drain` payload and arms the supervisor's per-run - * `drainTimeout` accumulators. Cancel-mode in-flight steps abort on - * the child side; wait-mode steps continue. Accumulators commit a - * signed `CancelRequested{origin: "supervisor-drain"}` against the - * workflow-run repo when the deadline expires. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end drain loop can omit the binding; an absent registry - * means hub-side drain frames cannot route through the hub-link until - * the wiring is plumbed. - */ + /** Cancel-mode in-flight steps abort child-side; wait-mode steps continue until the drainTimeout deadline commits a CancelRequested. */ multistepDrainRouter?: MultistepDrainRouter; - /** - * Per-deployment-address grants handler registry the sidecar - * hub-link's `run.grants` path consults. The deploy router registers a - * handler against the deployment's mail address once `supervisor.spawn` - * succeeds, for single-step and multi-step deployments alike, so a - * hub-side `run.grants` frame writes the run's grants to - * `runs//grants.json` inside the deployment's `workflow-run` - * repo. The write is awaited in `tryRoute` so the frame's FIFO - * completion means the grants are durable on disk. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end grants loop can omit the binding; an absent registry - * means hub-side grants frames cannot route through the hub-link until - * the wiring is plumbed. - */ + /** The write is awaited in tryRoute so the frame's FIFO completion means grants are durable on disk. */ multistepGrantsRouter?: MultistepGrantsRouter; - /** - * Per-deployment-address sources-rotation handler registry. Only a - * single-step warm deployment registers a handler (against the - * deployment's mail address once `supervisor.spawn` succeeds) so a - * rotation resolved for its address flows into - * `wired.supervisor.deliverSources` and on to the child's warm agent. A - * multi-step deployment registers none -- it has no single warm agent to - * rotate -- so `tryRoute` reports its address as unrouted. - * - * Optional so tests that exercise deploys without a rotation loop can - * omit the binding; an absent registry means no rotation handler is - * installed for any deployment. - */ + /** Only a single-step warm deployment registers a handler; a multi-step deployment has no warm agent to rotate, so its address is unrouted. */ multistepSourcesRouter?: MultistepSourcesRouter; - /** - * Optional per-deployment credential-delivery handler registry. Every - * deployment with a supervisor registers a handler after `spawn` (not only - * warm single-step ones -- the material cell is per-child and read by every - * step's tool capabilities), and an inbound `credentials.update` for an - * unregistered (torn-down) address is unrouted. Optional so tests without a - * credential-delivery loop can omit the binding. - */ + /** Every deployment with a supervisor registers one after spawn (material cell is per-child); an unregistered address is unrouted. */ multistepCredentialsRouter?: MultistepCredentialsRouter; - /** - * Optional per-message dispatch-timing observer the multi-step branch - * forwards to each supervisor it constructs. Resolved at the sidecar - * boot edge from the Phase 4.7 latency-gate env gate; absent in - * ordinary production. The supervisor runs in this sidecar subprocess, - * so the observer sees both ends of the per-message IPC round-trip in - * one process and can emit a parseable timing line the benchmark - * harness reads off the subprocess's output stream. - */ + /** Resolved from the Phase 4.7 latency-gate env; absent in ordinary production. */ onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** - * D2 §10c forced-repack A/B toggle the multi-step branch forwards to - * each supervisor it constructs. Resolved at the sidecar boot edge from - * the same benchmark env gate; absent in ordinary production. - */ + /** Resolved from the same benchmark env gate; absent in ordinary production. */ repackEveryMessages?: { everyMessages: number }; - /** - * Consumed-dedup retention horizon (ms) forwarded to every supervisor - * the router constructs. The sidecar boot edge resolves the operator's - * `CONSUMED_RETENTION_MS` config; absent, the supervisor applies - * `DEFAULT_CONSUMED_RETENTION_MS` (24h). See the workflow-run kind - * handler for the operator-owned horizon invariant. - */ + /** Boot edge resolves the operator's CONSUMED_RETENTION_MS; absent, the supervisor applies its own 24h default. */ consumedRetentionMs?: number; - /** - * Spawn ready-handshake timeout (ms) forwarded to every supervisor the - * router constructs. The sidecar boot edge resolves the operator's - * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies - * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never - * signals ready is killed and its spawn rejected rather than hanging - * the deploy or boot-time restore. - */ + /** A child that spawns but never signals ready is killed and its spawn rejected rather than hanging the deploy or restore. */ readyTimeoutMs?: number; /** * Run-record writer, injectable so a test can block or fail the @@ -993,27 +741,14 @@ export function createSidecarDeployRouter(deps: { // with the deployment. const activeSupervisors = new Map(); - // Synchronous single-flight guard for the deploy path. The real supervisor - // does not exist until inside `spawnWorkflowRun`, so `deployMultiStep` - // cannot reserve its `activeSupervisors` slot up front; instead it records - // the address here synchronously, before its first await, and clears it in a - // finally once the deploy settles. `activeSupervisors` is populated only - // after `spawn` succeeds, so the has-check alone leaves a window in which two - // same-address frames both pass and the loser's unwind deletes the winner's - // live run record. This set closes that window: a second frame that - // arrives while the first is mid-deploy is rejected before it touches any - // durable state. Only the live deploy path reserves; the boot restore path - // is serial and relies on the `activeSupervisors` backstop instead. + // Closes the window before activeSupervisors is populated (only after spawn + // succeeds): a second same-address frame arriving mid-deploy is rejected + // here rather than deleting the winner's live run record on unwind. const reservingDeployAddresses = new Set(); - // Slug-collision tracking. `deriveDeploymentId` substitutes - // disallowed characters with `-`, which is deterministic but lossy: - // two distinct run addresses can collapse to the same slug, and - // a collision would let the second deploy silently overwrite the - // first deploy's workflow-run repo state (the slug IS the repoId). - // This map records the first-claimer; a subsequent deploy that - // produces the same slug from a different address is rejected at - // the router before any supervisor or repo state is touched. + // deriveDeploymentId's char substitution is lossy: two distinct addresses + // can collapse to the same slug, which IS the repoId, so a collision would + // let the second deploy silently overwrite the first's repo state. const slugClaims = new Map(); function claimSlug(runId: string, agentAddress: string): void { @@ -1023,10 +758,7 @@ export function createSidecarDeployRouter(deps: { `deriveDeploymentId collision: run addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to runId ${JSON.stringify(runId)}`, ); } - // A same-address re-claim is a defensive no-op: the `activeSupervisors` - // guard rejects a live re-deploy before claimSlug is re-invoked, and a - // failed or undeployed deploy releases the slug first, so in practice - // `existing` is only ever undefined or a different address here. + // Defensive no-op: activeSupervisors already rejects a live re-deploy, so existing is only ever undefined or a different address here. slugClaims.set(runId, agentAddress); } @@ -1035,47 +767,15 @@ export function createSidecarDeployRouter(deps: { if (existing === agentAddress) slugClaims.delete(runId); } - // Reclaim a deployment address whose supervisor drove ITSELF to a terminal - // phase (crash-loop latch, channel crash, recycle failure) without an - // operator undeploy. Drops the address's runtime/routing state so a redeploy - // of the same address succeeds without a manual undeploy: the `has`-guarded - // `activeSupervisors` entry is the redeploy gate, and `transport.register` - // throws on a double-register, so both must be released here. The routers and - // the slug are released too so a frame racing the reclaim is rejected at the - // boundary rather than dispatched into the dead supervisor, and the slug is - // free to re-claim. - // - // Three deliberate invariants: - // - This NEVER calls back into `wired.supervisor.*`. It runs as the - // supervisor's own `onSelfTerminate` sink, i.e. re-entrantly during the - // supervisor's terminal teardown; a call back in (e.g. `shutdown()`) - // would re-enter that teardown. The supervisor has already terminated, so - // there is nothing to shut down. - // - It does NOT delete the durable run record or on-disk state (unlike the - // `undeploy` hook). The run record mirrors the hub's deployment intent, - // and a self-termination is not a hub-initiated undeploy: the hub still - // believes the address is deployed. Leaving the record lets a boot restore - // re-spawn a fresh supervisor for the address, and a same-process redeploy - // overwrites the record destructively. This preserves the pre-existing - // boot-restore behavior exactly -- a self-terminated deployment already - // left its record on disk before this reclaim existed; the reclaim only - // drops in-memory routing state, never durable state. - // - It does NOT call `unregisterDeployment`, so the deployment address - // mapping (`deploymentAddressRegistry`) is RETAINED. That mapping is - // consulted only by the OUTBOUND workflow-run pack push - // (`registry.resolve`), which the supervisor itself drives; no inbound - // frame consults it (a frame racing the reclaim is rejected by the - // routers above, not routed through this mapping). The supervisor's own - // terminal `RunFailed` commit -- the crash-loop latch writes it AFTER its - // teardown fires this sink -- resolves this mapping, so dropping it here - // would strand that commit ("no run address registered for deployment") - // and leave the crash-loop with no durable failure tombstone. Retaining a - // stale entry never blocks a redeploy: `record` overwrites idempotently, - // and an operator `undeploy` or a process restart clears it. - // - // Fully synchronous with no `await` between the guard and the mutations, so it - // is idempotent and cannot interleave with a concurrent operator `undeploy` of - // the same address (both are "if present, drop"). + // Reclaims an address whose supervisor drove ITSELF to a terminal phase + // without an operator undeploy, so a redeploy of the same address succeeds. + // Three invariants: never calls back into wired.supervisor.* (this runs + // re-entrantly during the supervisor's own terminal teardown); never + // deletes the durable run record (a self-termination isn't a hub-initiated + // undeploy, so a boot restore can re-spawn from it); never calls + // unregisterDeployment, since the pack-push address mapping is still + // needed by the crash-loop latch's own terminal RunFailed commit. Fully + // synchronous so it can't interleave with a concurrent operator undeploy. function reclaimSelfTerminatedSupervisor(args: { runId: string; agentAddress: string }): void { if (!activeSupervisors.has(args.agentAddress)) return; // Drop racing frames at the router boundary first, then unwind the @@ -1092,14 +792,7 @@ export function createSidecarDeployRouter(deps: { releaseSlug(args.runId, args.agentAddress); } - /** - * Remove the legacy `${dataDir}/assets/workflow//` staging a - * pre-sealed-record deploy wrote for a body (a world-readable plaintext - * `sources.json`, INTR-310). Body sources now ride sealed in the run record and - * reach the child through the spawn env, so the staging is retired; a redeploy - * of an address that predates the change reaches here and reclaims its orphaned - * file. A never-staged body is a no-op (`force`). - */ + /** Reclaims a pre-sealed-record deploy's legacy plaintext body-source staging, now retired since sources ride sealed in the run record. */ async function sweepLegacyBodySources( sidecarDataDir: string | undefined, definitionId: string, From 690e04e571cad692b3038f6d8c6af46e297bf994 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:55:24 -0700 Subject: [PATCH 27/31] chore(sidecar): comments say why, never what (workflow-host-wiring cont'd 3) --- apps/sidecar/src/workflow-host-wiring.ts | 174 ++++------------------- 1 file changed, 27 insertions(+), 147 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 0e5e2d2613..d54fb27883 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -768,14 +768,9 @@ export function createSidecarDeployRouter(deps: { } // Reclaims an address whose supervisor drove ITSELF to a terminal phase - // without an operator undeploy, so a redeploy of the same address succeeds. - // Three invariants: never calls back into wired.supervisor.* (this runs - // re-entrantly during the supervisor's own terminal teardown); never - // deletes the durable run record (a self-termination isn't a hub-initiated - // undeploy, so a boot restore can re-spawn from it); never calls - // unregisterDeployment, since the pack-push address mapping is still - // needed by the crash-loop latch's own terminal RunFailed commit. Fully - // synchronous so it can't interleave with a concurrent operator undeploy. + // without an operator undeploy. Never deletes the durable run record (so a + // boot restore can re-spawn) and never calls unregisterDeployment (the + // pack-push mapping is still needed by the crash-loop latch's own commit). function reclaimSelfTerminatedSupervisor(args: { runId: string; agentAddress: string }): void { if (!activeSupervisors.has(args.agentAddress)) return; // Drop racing frames at the router boundary first, then unwind the @@ -814,82 +809,31 @@ export function createSidecarDeployRouter(deps: { */ interface WorkflowDeploySpec { agentAddress: string; - /** - * The runnable definition, projected to its inert wire shape. Source-ref is - * the only deploy lineage, so this is always the closure evaluation - * (`projectLiveToInert(applied.definition)`), NOT a frame-carried inline - * definition (the deploy frame carries none). Both the deploy path and the - * boot-time restore derive it the same way, from the materialized closure. - */ + /** Always the closure evaluation (source-ref is the only deploy lineage), never a frame-carried inline definition. */ definition: WorkflowProjectionDefinition; sources: NonNullable["sources"]; - /** - * Per spawned-body inference-source pins, keyed by the body's definition id - * (the flat set of onTrigger sections and childWorkflow children). Persisted - * in the record and delivered to the run child as plaintext through the spawn - * env, so the child resolves a body's sources without holding the sidecar's - * cipher key. `undefined` when the deployment spawns no bodies. - */ + /** Delivered to the run child as plaintext through the spawn env so it resolves sources without the sidecar's cipher key. */ bodySources: WorkflowRunRecord["bodySources"]; - /** - * The run's unified credential-material cell (inference + tool secrets, plus - * tool bindings). Persisted sealed in the record and delivered to the child - * on the pre-trigger barrier; the child resolves both its inference sources - * and its tools from it by `credentialId`. `undefined` when the deployment - * binds no credentials and its sources need none. - */ + /** Delivered to the child on the pre-trigger barrier; undefined when the deployment binds no credentials. */ credentials: CredentialDelivery | undefined; - /** - * The hub-approved wire hash the deploy frame carried - * (`AgentDeployWorkflow.approvedWireHash`). The child's `DEFINITION_HASH` - * is sourced from this hub authority, NOT a sidecar recompute. Undefined - * only for a frame that carried no approved hash on the wire; the shared - * spawn core fails closed rather than substitute a recompute. - */ + /** Sourced from hub authority, never a sidecar recompute; undefined only for a frame that carried no approved hash. */ approvedWireHash: string | undefined; /** Correlates the child's inference events to the deploy's session. */ sessionId: string | undefined; - /** - * Hub public key recorded at the head for deploy-pack verification and - * inbound hub-frame verification. Required for a single-step - * deployment (whose head IS the agent identity); undefined for a - * genuine multi-step deployment, which derives per-step addresses and - * records no head key. - */ + /** Required for a single-step deployment (head IS the agent identity); undefined for multi-step, which derives per-step addresses. */ hubPublicKey: string | undefined; - /** - * Sidecar-local directory of the materialized workflow-definition closure, - * set after the frozen closure is applied. The spawn core threads it into - * the child's spawn env so the run child evaluates the pinned code to a live - * definition. Sidecar-local: it never travels on the hub deploy frame and is - * not persisted to the deployment record. Always present -- source-ref is - * the only deploy lineage. - */ + /** Sidecar-local only: never travels on the hub deploy frame and isn't persisted to the record. Always present. */ closurePackageDir: string; - /** - * The source-ref pin, carried so `buildWorkflowRunRecord` can persist it and - * a boot-time restore can re-run `applyFrozenWorkflowClosure` to - * re-materialize the pinned code. Its `source` carries no secret (the - * registry token is resolved from env at apply time); its `closure` is - * frozen versions + SRIs. - */ + /** Carried so a boot-time restore can re-run applyFrozenWorkflowClosure; its source carries no secret, its closure is versions + SRIs. */ sourceRef: NonNullable["sourceRef"]; } - /** - * Build the durable run record from a spec and a source table. The - * table is a parameter (not `spec.sources`) so the deploy path writes the - * deploy-time sources while the rotation handler writes the live-rotated - * ones -- both through one shape, so a rotation persists the same record a - * boot-time restore reseeds from. - */ + /** Table is a parameter (not spec.sources) so both the deploy path and the rotation handler write through one shape. */ function buildWorkflowRunRecord( spec: WorkflowDeploySpec, sources: WorkflowRunRecord["sources"], ): WorkflowRunRecord { - // The record schema requires both for a source-ref record -- the only - // lineage -- so a spec missing either is a wiring defect. Fail loudly here - // rather than persist a record the boot scan would then reject as corrupt. + // A spec missing this is a wiring defect; fail loudly rather than persist a record the boot scan would reject as corrupt. if (spec.approvedWireHash === undefined) { throw new Error( `buildWorkflowRunRecord: a source-ref deployment (${spec.agentAddress}) must carry approvedWireHash`, @@ -914,22 +858,11 @@ export function createSidecarDeployRouter(deps: { } /** - * Materialize a source-ref deployment's frozen closure to its per-deployment - * instance dir and return the applied result. Owns the plumbing both the - * deploy path and the boot-time restore path share: the deterministic - * instance dir under `/workflow-definition-closures/`, - * the content-addressed cache root, the two substrate byte caps, and the - * registry table. The caller consumes the result: deploy validates - * `applied.definition` and takes `packageDir`; restore takes `packageDir` and - * re-carries the pin on its spec. - * - * The instance dir is force-reclaimed before the apply. `deploymentId` is - * deterministic per agent address, so a redeploy or a boot-restore reuses the - * same dir; a prior soft-failed deploy or a dead prior process can leave it - * half-materialized. The rm makes the apply write into a clean dir either - * way (a no-op on a never-deployed address). It is safe to reclaim only - * because no live reader holds the dir when this runs -- a precondition each - * caller establishes and notes at its call site. + * Shared plumbing for both the deploy path and boot-time restore. The + * instance dir is force-reclaimed before the apply since deploymentId is + * deterministic per address and a prior soft-failed deploy can leave it + * half-materialized; safe only because no live reader holds the dir when + * this runs, a precondition each caller establishes at its call site. */ async function materializeDeploymentClosure( dataDir: string, @@ -967,31 +900,15 @@ export function createSidecarDeployRouter(deps: { } /** - * The single owner of the workflow-deployment spawn sequence: construct - * the supervisor, register the single-step agent's outbound key + head - * repo + hub key, spawn the workflow-process child, then register the - * live deployment (supervisor, mail/signal/drain routers, address - * mapping). Its `try/finally` unwinds every piece of partial state if any - * step throws, so a failed spawn leaks nothing. Both the live deploy path - * and the boot-time restore path route through here so the two can never - * diverge on how a deployment is stood up. Callers materialize the - * deploy-only durable state (the source closure, step grants) before calling. + * The single owner of the workflow-deployment spawn sequence; its + * try/finally unwinds every piece of partial state if any step throws. + * Both the live deploy path and the boot-time restore path route through + * here so the two can never diverge on how a deployment is stood up. */ async function spawnWorkflowRun(spec: WorkflowDeploySpec): Promise { - // The run's credential material rides on `spec.credentials`: the sidecar now - // holds a cipher and seals it into the record, so both the live deploy and - // the boot-restore path carry it here (restore unseals it from the record), - // and it is delivered to the child on the pre-trigger barrier. + // Sealed into the record by the sidecar's cipher; restore unseals it from the record. const credentialDelivery = spec.credentials; - // Fail loud if this address already has a live supervisor. Both single- - // and multi-step now register on the transport, so both carry the - // `transport.register` duplicate-throw backstop; this `has()` check is the - // primary early guard that gives a clean error before that lower-level - // throw and before the `activeSupervisors.set` below could clobber the - // running deployment's handle. Both the deploy path and the boot restore - // path route through here, so this is the single transition guard against - // a double-spawn -- notably a boot restore racing a legacy restore for the - // same address (the B-reroute follow-up relies on it). + // The primary early guard, ahead of the lower-level transport.register duplicate-throw backstop; guards against a boot restore racing a legacy restore for the same address. if (activeSupervisors.has(spec.agentAddress)) { throw new Error( `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, @@ -999,39 +916,19 @@ export function createSidecarDeployRouter(deps: { } const runId = deriveDeploymentId(spec.agentAddress); - // Single-step launched-agent deploy vs. derived multi-step deploy. A - // one-step deployment keeps the deployment's own (legacy) mail address - // and its grants in the legacy agent-state repo keyed by the legacy - // instance id. A multi-step deployment derives `-` - // per step for both the mail address and the agent-state repo id. const stepStrategy = createStepStrategy({ legacyAddress: spec.agentAddress, stepOrder: spec.definition.stepOrder, multistepDeriveStepAddress, }); - // Every step id the deployment's credentials snapshot must cover. This is - // NOT `stepOrder`: a `loop` body runs in-process as a child run inheriting - // the parent's env, so a body step authorizes against the SAME snapshot the - // top-level steps do, keyed by its own plain step id. A snapshot built from - // `stepOrder` alone carries no entry for it and the child's authorize - // throws on the body's first tool call. The address/repo strategy above - // stays on `stepOrder`, because the head/step collapse is a property of the - // deployment's own step count, not of what a body can run. + // Not stepOrder: a loop body runs in-process inheriting the parent's env, so it must authorize against the same snapshot keyed by its own plain step id. const credentialStepIds = inertFlatNamespaceStepIds({ definition: spec.definition, context: "sidecar deploy router credentials snapshot: ", }); - // Unwind every piece of spawn state if any step in this block throws, - // so a failed spawn leaks no freshly-spawned workflow-process child, - // `activeSupervisors` entry, transport registration, or multistep - // router registration. (The deployment-address registration happens - // before spawn and is unwound by its own guard.) The ordering inside - // the finally is the reverse of the success-path registration order. - // The caller owns the deployment slug: it must - // claim the collision guard before any durable write and release it on - // failure, so the slug is not touched here. + // The caller owns the deployment slug and the deployment-address registration, so neither is touched in this unwind. let succeeded = false; let wiredForUnwind: SidecarWorkflowSupervisor | undefined; let supervisorRegistered = false; @@ -1040,15 +937,7 @@ export function createSidecarDeployRouter(deps: { let hubKeyRecorded = false; let deploymentRegistered = false; try { - // The child's `DEFINITION_HASH` is the HUB-APPROVED wire hash the deploy - // frame carried (`spec.approvedWireHash`) -- the hub is the authority, so - // the child re-verifies its own recompute against it. Both feeds into this - // core carry it: the production hub deploy builder always stamps it, and - // the boot restore re-attaches it from the persisted record. A missing - // hash here is therefore a wiring bug, not a legacy case to paper over: - // substituting a sidecar recompute would make the child re-verify against - // the sidecar's own hash rather than the hub authority -- a circular check - // that gives false assurance. Fail loud instead. + // A missing hash is a wiring bug, not a legacy case: substituting a sidecar recompute would collapse the child's re-verify to a self-check against its own hash. if (spec.approvedWireHash === undefined) { throw new Error( `workflow deploy spawn (${spec.agentAddress}): the deploy spec carries no approvedWireHash. The hub deploy builder must stamp the hub-approved wire hash and a restore must re-attach it from the persisted record; the sidecar will not recompute it, which would collapse the child's re-verify to a self-check.`, @@ -1056,16 +945,7 @@ export function createSidecarDeployRouter(deps: { } const definitionHash = spec.approvedWireHash; - // Per-deployment substrate-config keys the workflow-substrate-factory - // validator requires. The boot edge's `multistepSubstrateEnv` carries - // the boot-edge constants; the two workflow-run identity keys are - // derived per-deploy here. - // Spawned bodies' plaintext inference sources, serialized once for the - // deployment's lifetime (bodies do not live-rotate, unlike the top-level - // sources below). The child looks each body up by definition id and never - // holds the sidecar's cipher key. Guarded against the OS argument-string - // ceiling here, at deploy, so an over-large deployment fails loudly rather - // than at a later child `execve`. + // Bodies don't live-rotate, unlike top-level sources; guarded against the OS argument-string ceiling here so an over-large deployment fails at deploy, not at a later child execve. const bodySourcesEnv = JSON.stringify(spec.bodySources ?? {}); const bodySourcesBytes = Buffer.byteLength(bodySourcesEnv, "utf8"); if (bodySourcesBytes > WORKFLOW_BODY_SOURCES_MAX_BYTES) { From d3c7a8b08e93bc179573a8b0de44ca8a4ed8536c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:57:15 -0700 Subject: [PATCH 28/31] chore(sidecar): comments say why, never what (workflow-host-wiring cont'd 4) --- apps/sidecar/src/workflow-host-wiring.ts | 179 ++++------------------- 1 file changed, 25 insertions(+), 154 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index d54fb27883..6666a3c48b 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1034,39 +1034,18 @@ export function createSidecarDeployRouter(deps: { ...(deps.readyTimeoutMs !== undefined ? { readyTimeoutMs: deps.readyTimeoutMs } : {}), }); - // OUTBOUND half of mailbox ownership (§3a): register a signing key for - // the deployment mail address on the host transport so the supervisor - // signs the deployment's outbound mail. Every step -- single- or - // multi-step -- signs its outbound sends as `spec.agentAddress` (the - // one deployment mail address; no per-step sender reaches the host - // transport), so the transport MUST hold a `CryptoProvider` for it or - // `getTransportFor(senderAddress).send` throws "not registered". - // Registration happens before `spawn()` so the address is live the - // instant the first reply routes outbound. + // Every step signs outbound sends as spec.agentAddress, so the transport must hold a CryptoProvider for it before spawn() or getTransportFor throws "not registered". const { keyPair } = await deps.keyStore.loadOrGenerateKey(spec.agentAddress); deps.transport.register(spec.agentAddress, deps.createAgentCrypto(keyPair)); agentTransportRegistered = true; - // The deploy ack surfaces the deployment address's Ed25519 public key -- - // the key `loadOrGenerateKey` minted above and the workflow uses for - // signed content provenance. Every deployment acks it, single- and - // multi-step alike, so the Hub can publish the same identity. Reconnect - // ownership is established separately by the allocation credential. + // Every deployment acks this key, single- and multi-step alike, so the Hub can publish the same identity. const deploymentPublicKey = hexEncode(keyPair.publicKey); if (spec.definition.stepOrder.length === 1) { - // A single-step workflow stages its deploy tree at the head (the - // lone step IS the head). Initialize the head's on-disk deploy-tree - // repo (idempotent) so the hub's deploy-pack push has a repo to - // apply into. The narrow `initRepo` (not `provisionAgent`) is - // deliberate: the supervised child mints its own keypair and - // persists no hub-agent config. + // Idempotent init; the narrow initRepo (not provisionAgent) is deliberate since the supervised child mints its own keypair. await deps.sessions.initRepo(spec.agentAddress); - // Record the hub's public key at the head so the deploy-pack apply - // (and any inbound hub-signed frame) verifies against it. The - // verifier resolves the key from the in-memory key store's - // `recordHubKey` map, so a single-step deployment cannot stand up - // without it. + // Verifier resolves this from the in-memory key store, so a single-step deployment cannot stand up without it. if (spec.hubPublicKey === undefined) { throw new Error( "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", @@ -1076,20 +1055,14 @@ export function createSidecarDeployRouter(deps: { hubKeyRecorded = true; } - // Warm-keep is the single-step launched-agent deploy: the sole step - // IS the long-lived agent, so the child warm-keeps it across - // messages. A multi-step deploy keeps instantiate-send-teardown per - // step. The signal is carried explicitly down through the spawn env. + // The sole step IS the long-lived agent for single-step; multi-step keeps instantiate-send-teardown per step. const warmKeep = spec.definition.stepOrder.length === 1; const spawnOpts: SpawnOpts = { stepOrder: [...credentialStepIds], definitionHash, warmKeep, onInferenceEvent: (event) => { - // The event arrives HMAC-verified over the child's event channel. - // Re-narrow it to the hub's `InferenceEvent` union; a parse - // failure means upstream corruption, so drop it loudly rather - // than forwarding an unvalidated payload onto the hub timeline. + // A parse failure means upstream corruption, so drop loudly rather than forwarding an unvalidated payload onto the hub timeline. const validated = parseInferenceEvent(event); if (validated instanceof type.errors) { logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; @@ -1099,51 +1072,27 @@ export function createSidecarDeployRouter(deps: { }, }; - // Record the deployment-address mapping BEFORE `spawn`, because - // `spawn` kicks off `replayProcessingToInbox`, whose workflow-run - // substrate write routes through the boot-edge pack-pushing facade and - // resolves this mapping to address the outbound pack frame. Recording - // it after `spawn` (as the other registrations below are) loses the - // race: the replay's write throws "no run address registered" (a - // real defect masked as a swallowed best-effort warning in the - // supervisor's replay catch). Constraint ownership: the registry owns - // "address is resolvable"; the spawn path must satisfy that contract - // before the replay writes. The finally unwinds it on any failure - // between here and the end of the try. + // Recorded before spawn, since spawn's replayProcessingToInbox write resolves this mapping to address the outbound pack frame; recording after loses the race. deps.registerDeployment({ runId, agentAddress: spec.agentAddress, }); deploymentRegistered = true; - // Surface spawn-time errors structurally: a subprocess spawner that - // crashes immediately rejects here, and the caller converts the - // rejection into a structured failure frame. The supervisor is - // registered against the deployment address only after spawn succeeds, - // so a spawn-time rejection leaves the registry untouched. + // The supervisor is registered against the address only after spawn succeeds, so a spawn-time rejection leaves the registry untouched. await wired.supervisor.spawn(spawnOpts); wiredForUnwind = wired; activeSupervisors.set(spec.agentAddress, wired); supervisorRegistered = true; - // Bind the deployment's mail address to this supervisor's - // `routeInbound` so the hub-link dispatches inbound mail into the - // supervisor's mail-bus subscription. Registration happens after - // `spawn` succeeds so a spawn-time rejection leaves the registry - // untouched. deps.multistepMailRouter?.register(spec.agentAddress, (message) => wired.routeInbound(message), ); - // Resolve this deployment's authored inbound-mail policy into a total - // decision map ONCE and store it beside the mail-router registration, so - // the hub-link seam has the recipient's policy before any inbound frame - // for this address can route. + // Resolved once, beside the mail-router registration, so the hub-link seam has the recipient's policy before any inbound frame can route. deps.inboundMailPolicyRegistry?.register( spec.agentAddress, resolveInboundMailPolicy(spec.definition.inboundMailPolicy), ); - // Register the signal-delivery handler so a hub `signal.deliver` frame - // dispatches through the supervisor's `deliverSignal`. deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.deliverSignal({ runId: args.runId, @@ -1152,27 +1101,13 @@ export function createSidecarDeployRouter(deps: { payload: args.payload, }); }); - // Register the drain handler so a hub `drain.deliver` frame dispatches - // through the supervisor's `drain`. deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); }); - // Register the grants handler so a hub `run.grants` frame writes the - // run's grants to `runs//grants.json` in the deployment's - // workflow-run repo. The `runId` selects the per-run destination; the - // step-fan-out fields are inert in that mode but the shared write - // machinery still takes them. + // runId selects the per-run destination; the step-fan-out fields are inert here but the shared write machinery still takes them. deps.multistepGrantsRouter?.register(spec.agentAddress, async (args) => { try { - // Cache the co-delivered sender keys BEFORE the grants file lands, so - // "grant durable" implies "key durable": a cache-write fault falls - // into the catch below and poisons the run rather than starting it - // with a grant whose sender the recipient cannot verify. A malformed - // key is a hub-side defect that is keyless from here, so skip it and - // cache the valid ones (no weaker than the hub having omitted it) - // rather than wedge the run on every replay -- but log it at ERROR, - // naming the address and reason, so the hub bug surfaces loudly - // instead of being silently dropped. + // Cached before the grants file lands so "grant durable" implies "key durable"; a malformed key is skipped and logged at ERROR rather than wedging the run. for (const identity of args.senderIdentities ?? []) { let publicKey: Uint8Array; try { @@ -1197,66 +1132,30 @@ export function createSidecarDeployRouter(deps: { runId: args.runId, }); } catch (cause) { - // A sender-key or per-run grants write did not land. Poison the runId - // so the grants barrier fails the run instead of starting it under - // the deploy-time grant set, then re-throw so the hub-link logs the - // durable-write failure loudly. + // Poison the runId so the grants barrier fails the run instead of starting it under the deploy-time grant set. poisonedRunIds.add(args.runId); throw cause; } - // The per-run file now carries the (possibly overlaid) floor. Refresh a - // live child so a standing ("always") approval resolved while the run - // is running but not parked -- with no signal to piggyback the - // `deliverSignal` refresh on -- lowers its floor immediately. - // Best-effort and non-fatal: the same durable file governs the next - // barrier/respawn, so a skipped push never leaves the run under a stale - // floor for good. + // Best-effort and non-fatal: refreshes a live child so a standing approval lowers its floor immediately; a skipped push is healed by the next barrier/respawn. await wired.supervisor.deliverGrants(args.runId); }); - // Register the sources-rotation handler ONLY for a single-step warm - // deployment: it has one long-lived agent whose sources can be - // swapped in place. A multi-step deployment has no single warm agent, - // so it registers no handler and `tryRoute` reports its address as - // unrouted. + // Only a single-step warm deployment (one long-lived agent to swap sources on); multi-step registers none, so tryRoute reports its address as unrouted. if (warmKeep) { - // A single-step deployment's source table has exactly one entry, - // keyed by the head step. Derive that key once here (the layer that - // owns the single-key invariant); `deliverSources` stays flat and - // stepId-agnostic. + // The layer that owns the single-key invariant; deliverSources stays flat and stepId-agnostic. const rotationStepId = spec.definition.stepOrder[0]; if (rotationStepId === undefined) { throw new Error("single-step deploy has no step id for sources rotation"); } deps.multistepSourcesRouter?.register(spec.agentAddress, async (args) => { const rotated = { [rotationStepId]: args.sources }; - // Swap `currentSources` synchronously BEFORE the durable persist. - // `currentSources` is the process-local respawn hint the - // supervisor reads synchronously through `dynamicSpawnEnv`, so a - // recycle that interleaves the persist `await` must respawn the - // child on the SAME sources being persisted, not the stale prior - // table. The obvious inverse -- persist first, then swap -- is - // rejected: it leaves the child on the OLD sources during the - // persist window while the record has already moved to NEW, so a - // recycle there respawns stale and a restart would "correct" it, - // i.e. the running child contradicts durable intent. Swapping - // first makes the only residual disagreement child-ahead-of- - // durable on a failed persist, which the next recycle heals down - // to the rolled-back durable truth -- the benign direction. The - // wire boundary guarantees `args.sources[0]` is the default, - // which the recycle env form pins as the active source. + // Swapped before the durable persist so a recycle interleaving the + // persist await respawns on the SAME sources being persisted, not + // the stale prior table; a failed persist rolls the hint back so + // the only residual disagreement is child-ahead-of-durable, which + // the next recycle heals down to the rolled-back durable truth. const prevSources = currentSources; currentSources = rotated; - // The durable write still precedes the LIVE swap - // (`deliverSources`), preserving persist-before-externally-visible - // for state that outlives the process; only the process-local - // respawn hint moves ahead. On a failed persist, roll the hint - // back so `currentSources` and the record stay in agreement in the - // common (no interleaved recycle) failure case -- the invariant - // restart consistency depends on. Persistence lets the rotation - // survive a full sidecar restart, not just a recycle: the boot - // scan reseeds spec.sources from record.sources. Overwrites the - // deploy-time record in place. Skipped when no data dir was wired - // (a test router that never persists), matching the restore guard. + // Skipped when no data dir was wired (a test router that never persists), matching the restore guard. if (stepStateDataDir !== undefined) { try { await persistWorkflowRunRecord( @@ -1266,16 +1165,7 @@ export function createSidecarDeployRouter(deps: { deps.credentialCipher, ); } catch (cause) { - // Restoring unconditionally is safe because rotations for one - // deployment are serialized by the sidecar's per-connection - // inbound-frame queue: each hub frame, sources.update - // included, runs its handler to completion on that queue - // before the next frame's handler starts, so no second - // rotation is in flight whose committed table this rollback - // could clobber. This does NOT rely on the hub pacing its - // sends -- the hub dispatches sources.update fire-and-forget; - // the sidecar frame queue is the sole serializer. Parallelizing - // inbound-frame dispatch would break this rollback. + // Safe because the sidecar's per-connection inbound-frame queue serializes rotations for one deployment, so no second rotation is in flight. currentSources = prevSources; throw cause; } @@ -1287,12 +1177,7 @@ export function createSidecarDeployRouter(deps: { }); } - // Register the credential-delivery handler for EVERY deployment (not only - // warm single-step ones): the material cell is per-child and read by - // every step's tool capabilities. The handler hands the delivery to the - // supervisor's `deliverCredentials`, which sends a `credentials-updated` - // control frame to the child where the material cell is swapped. No - // durable persist -- credential material never touches disk. + // For EVERY deployment (not only warm single-step): the material cell is per-child. No durable persist; credential material never touches disk. deps.multistepCredentialsRouter?.register(spec.agentAddress, async (args) => { await wired.supervisor.deliverCredentials({ delivery: args.delivery, @@ -1313,10 +1198,7 @@ export function createSidecarDeployRouter(deps: { deps.multistepSignalRouter?.unregister(spec.agentAddress); deps.multistepDrainRouter?.unregister(spec.agentAddress); deps.multistepGrantsRouter?.unregister(spec.agentAddress); - // Unregister unconditionally: the sources handler was registered - // only for a single-step deploy, but `unregister` is a no-op for - // an address that never registered one, so a multi-step unwind - // safely calls it too. + // unregister is a no-op for an address that never registered, so a multi-step unwind safely calls this too. deps.multistepSourcesRouter?.unregister(spec.agentAddress); deps.multistepCredentialsRouter?.unregister(spec.agentAddress); } @@ -1335,18 +1217,7 @@ export function createSidecarDeployRouter(deps: { deps.transport.unregister(spec.agentAddress); } if (hubKeyRecorded) { - // Reverse the single-step head's `recordHubKey` so a failed deploy - // leaves no in-memory hub key behind. `forgetAgent` also drops the - // agent keypair cache `loadOrGenerateKey` populated, which is safe: - // the transport registration is already unwound above, nothing reads - // that cache after unwind, and a redeploy reloads the keypair from - // disk. The on-disk deploy-tree repo `initRepo` created is - // deliberately NOT reversed. It is idempotent and the hub re-pushes - // the deploy pack on every redeploy, so it is benign residue; and - // decisively, the durable Ed25519 identity keypair lives inside that - // same directory (`keys/` nests under the agent repo dir), so - // removing the repo would destroy an identity a rerouted head must - // keep across a failed redeploy. + // forgetAgent also drops the keypair cache, safe since a redeploy reloads it from disk. The on-disk deploy-tree repo is deliberately NOT reversed: it holds the durable identity keypair a rerouted head must keep. deps.keyStore.forgetAgent(spec.agentAddress); } if (deploymentRegistered) { From 9bb39759dc779d3850580e52709c532c293f3cb1 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 18:58:19 -0700 Subject: [PATCH 29/31] chore(sidecar): comments say why, never what (workflow-host-wiring cont'd 5) --- apps/sidecar/src/workflow-host-wiring.ts | 88 ++++-------------------- 1 file changed, 15 insertions(+), 73 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 6666a3c48b..8734c6bc5d 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1237,20 +1237,10 @@ export function createSidecarDeployRouter(deps: { } /** - * Provision one step of a multi-step deploy WITHOUT spawning. The hub - * stages each step's deploy tree before firing the deployment-level - * workflow frame; a full-closure deploy pack still needs an initialized - * agent-state repo to apply into and the hub key recorded to verify the - * pack commit signature. This does exactly those two things -- the same - * harness-free `initRepo` + `recordHubKey` seam the single-step head uses - * -- and constructs no supervisor or child. The deployment-level workflow - * frame (fired once after every step is provisioned) spawns the child, - * which reads each step's staged deploy tree from disk. - * - * Returns the sidecar's principal public key so the link's - * `agent.deploy.ack` carries a key, matching the multi-step ack. A - * per-step address is workflow-derived and records no `agent_instance` - * key, so the hub discards this value. + * Provisions one step of a multi-step deploy WITHOUT spawning (same + * initRepo + recordHubKey seam the single-step head uses), since a + * full-closure deploy pack needs an initialized repo before the + * deployment-level workflow frame spawns the child. */ async function provisionStep(frame: AgentDeployFrame): Promise { await deps.sessions.initRepo(frame.agentAddress); @@ -1264,19 +1254,10 @@ export function createSidecarDeployRouter(deps: { frame: AgentDeployFrame, projection: NonNullable, ): Promise { - // Reject a re-deploy of an address already live OR mid-deploy in this - // process BEFORE touching any durable state. The durable writes below (the - // run record, the materialized closure, step grants) are destructive - // overwrites of state owned by whatever deployment currently holds the - // address; overwriting is only legal when this deploy owns the address. - // `activeSupervisors` catches an address whose deploy has completed; - // `reservingDeployAddresses` catches one whose deploy is still in flight. - // The map is populated only after `spawn` succeeds, so the has-check alone - // leaves a window in which two frames both pass and the loser's catch below - // deletes the winner's live record; the reservation set closes it. A - // re-deploy after `undeploy` passes: `undeploy` drops the - // `activeSupervisors` entry, and a failed or completed deploy has already - // cleared its reservation. + // activeSupervisors catches a completed deploy; reservingDeployAddresses + // catches one still in flight, closing the window before spawn populates + // the map, where two frames could both pass and the loser deletes the + // winner's live record. if ( activeSupervisors.has(frame.agentAddress) || reservingDeployAddresses.has(frame.agentAddress) @@ -1288,9 +1269,7 @@ export function createSidecarDeployRouter(deps: { const runId = deriveDeploymentId(frame.agentAddress); - // Resolve the sidecar data dir once: the run record, the materialized - // closure, and the per-step scratch all root under it. Required for any - // deployment that spawns a child. + // The run record, materialized closure, and per-step scratch all root under this. const dataDir = stepStateDataDir; if (typeof dataDir !== "string" || dataDir.length === 0) { throw new Error( @@ -1298,39 +1277,12 @@ export function createSidecarDeployRouter(deps: { ); } - // Claim the deployment slug BEFORE any durable write so a colliding runId - // (two distinct addresses projecting to the same slug) is rejected before - // the closure, the step grants, or the supervisor touch disk -- the - // router's "no repo state touched before rejection" guarantee. The claim is - // released on any failure below; a successful deploy keeps it (the undeploy - // hook releases it at teardown). The spawn core owns unwinding the - // supervisor and registrations it stands up; the slug is the caller's. + // Claimed before any durable write so a colliding runId is rejected before disk is touched; released on failure, kept on success (undeploy releases it). claimSlug(runId, frame.agentAddress); - // Hold the single-flight reservation across the async body below and clear - // it in the finally. Everything above is synchronous and throws before any - // durable write, so the reservation is only needed from the first await - // here onward; the top-of-method guard already consults this set for a - // concurrent frame, and claimSlug/runId derivation above cannot yield - // control before this point. reservingDeployAddresses.add(frame.agentAddress); try { - // Source-ref apply -- the only deploy lineage. Materialize EXACTLY the - // hub's frozen dependency `closure` and evaluate the PINNED CODE to the - // workflow definition; the deploy frame carries no inline definition to - // trust. The closure is applied byte-for-byte (concrete versions + - // integrity SRIs); the sidecar never re-resolves the pin at apply time. - // The re-evaluated projection IS the runnable definition, and the child's - // load-boundary re-verify recomputes the wire hash over the closure and - // fails closed if it diverges from the hub-approved hash, so a closure - // that no longer projects to the approved content cannot deploy. - // - // Check the frame's inline source assets out into the durable - // per-deployment store the closure materializes from. Reclaim the store - // first so a redeploy drops assets no longer referenced. This runs only on - // the DEPLOY path -- restore re-reads the store the deploy persisted, with - // no re-delivery -- so the checkout lives here, not in - // `materializeDeploymentClosure` (which also runs on restore). A - // registry-sourced pin delivers no assets and only clears the store. + // Materializes the hub's frozen closure byte-for-byte (never re-resolves the pin); the child's load-boundary re-verify fails closed if it diverges from the hub-approved hash. + // Reclaims the store first so a redeploy drops unreferenced assets; runs only on the deploy path since restore re-reads what deploy persisted, with no re-delivery. const assetStore = deploymentSourceAssetRoot(dataDir, runId); const gitStore = deploymentSourceGitRoot(dataDir, runId); await rm(assetStore, { recursive: true, force: true }); @@ -1344,9 +1296,7 @@ export function createSidecarDeployRouter(deps: { maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, }); } - // Safe to reclaim the instance dir inside the helper: this deploy is - // single-flight-guarded (the reservation above) and the child is not yet - // spawned, so no live reader holds it. + // Safe: this deploy is single-flight-guarded and the child is not yet spawned, so no live reader holds the instance dir. const applied = await materializeDeploymentClosure(dataDir, runId, projection.sourceRef); const validatedDefinition = WorkflowProjectionDefinition( projectLiveToInert(applied.definition), @@ -1358,21 +1308,13 @@ export function createSidecarDeployRouter(deps: { } const effectiveDefinition = validatedDefinition; - // Structural invariants the wire arktype does not cover (non-empty - // stepOrder, every stepOrder entry backed by a `steps` entry AND a - // `sources` entry), checked against the closure-derived definition -- the - // frame carries none to cover. Mirrors the restore path. + // Checked against the closure-derived definition since the wire arktype doesn't cover it and the frame carries none. Mirrors the restore path. validateWorkflowProjection({ definition: effectiveDefinition, sources: projection.sources, }); - // Source-admission gate: reject a deploy where any step pins an inference - // provider this sidecar cannot build. Every source in a step's failover - // chain must be buildable -- a chain with an unbuildable tail would fail - // only after the reactor failed over onto it -- so this iterates the whole - // list. The throw propagates back through the deploy frame so the hub's - // `deployWorkflow` rejects synchronously at deploy time. + // Every source in a step's failover chain must be buildable, since an unbuildable tail would fail only after the reactor failed over onto it. for (const stepId of effectiveDefinition.stepOrder) { const chain = projection.sources[stepId]; if (chain !== undefined) { From 6c855a7e68a069c667fb387fbe54216af7e678cc Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 19:00:06 -0700 Subject: [PATCH 30/31] chore(sidecar): comments say why, never what (workflow-host-wiring, finish) --- apps/sidecar/src/workflow-host-wiring.ts | 156 +++-------------------- 1 file changed, 17 insertions(+), 139 deletions(-) diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts index 8734c6bc5d..2202f747ba 100644 --- a/apps/sidecar/src/workflow-host-wiring.ts +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -1374,18 +1374,7 @@ export function createSidecarDeployRouter(deps: { await sweepLegacyBodySources(dataDir, referenced.definition.id); } - // Grants bridge: the spawned child does not see the frame; it reads - // each step's grants out of `state/grants.json` in the step's - // agent-state repo while the supervisor assembles the - // credentialsSnapshot. Write the operator-approved - // `frame.config.grants` to the same repo the supervisor reads via - // `deriveStepRepoId`, before the spawn core, so the read sees them. - // - // The write covers the deployment's whole flat step-id namespace, loop - // body steps included: the supervisor assembles a snapshot entry for each - // of them, and an entry read out of a repo that was never written carries - // an empty grant set, which denies the body every resource the deploy - // approved. + // The spawned child doesn't see the frame, so operator-approved grants are written to the same repo the supervisor reads, before the spawn core. Covers loop body steps too, or an unwritten repo denies the body every resource. await writeStepGrants({ repoStore: deps.repoStore, anchorRunId: runId, @@ -1400,12 +1389,7 @@ export function createSidecarDeployRouter(deps: { // Hand off to the shared spawn core. return await spawnWorkflowRun(spec); } catch (cause) { - // Soft failure (this process survived, the deploy threw): drop the - // record and release the slug so the failed deploy is neither restored - // nor leaks its slug. The record delete must not mask the real deploy - // error or skip releasing the slug: a rejecting delete is logged (the - // orphaned record is a durable-state leak the next boot scan re-drives) - // but `cause` is still what propagates and the slug is still released. + // A rejecting delete is logged, not thrown, so `cause` still propagates and the slug is still released. try { await deleteWorkflowRunRecord(dataDir, runId); } catch (cleanupError) { @@ -1415,9 +1399,7 @@ export function createSidecarDeployRouter(deps: { releaseSlug(runId, frame.agentAddress); throw cause; } finally { - // Release the single-flight reservation whether the deploy succeeded or - // threw. On success the address is now in `activeSupervisors`, which the - // guard also consults, so a later re-deploy is still rejected. + // On success the address is now in activeSupervisors, which the guard also consults, so a later re-deploy is still rejected. reservingDeployAddresses.delete(frame.agentAddress); } } @@ -1430,66 +1412,28 @@ export function createSidecarDeployRouter(deps: { if (frame.workflow !== undefined) { return await deployMultiStep(frame, frame.workflow); } - // Every deploy stages through the workflow-run substrate: a - // provision-step frame primes the per-step repo, and a workflow - // frame spawns the supervised child. A frame carrying neither is - // an unsupported shape -- there is no in-process fall-through. + // A frame carrying neither provisionStep nor workflow is an unsupported shape; there is no in-process fall-through. throw new Error( `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, ); }, async undeploy(frame): Promise { - // Symmetric teardown for `deploy`: release the per-deployment - // routing state both branches install so a stale `signal.deliver` - // / `drain.deliver` / `mail.inbound` aimed at the dead deployment - // address is rejected by the router rather than dispatched into - // an orphan supervisor handler. The unregister calls are - // idempotent -- they are no-ops when no handler is registered. - // - // Routers come down BEFORE the supervisor's `shutdown()` so any - // hub-side frame racing the undeploy is dropped at the router - // boundary rather than dispatched into a supervisor that is in - // the middle of tearing its child down. The pattern is: drop - // racing frames first, then unwind the underlying resource. + // Routers come down before shutdown() so a hub-side frame racing the undeploy is dropped at the boundary rather than dispatched into a supervisor mid-teardown. unregister is idempotent. const runId = deriveDeploymentId(frame.agentAddress); deps.multistepMailRouter?.unregister(frame.agentAddress); deps.inboundMailPolicyRegistry?.unregister(frame.agentAddress); deps.multistepSignalRouter?.unregister(frame.agentAddress); deps.multistepDrainRouter?.unregister(frame.agentAddress); deps.multistepGrantsRouter?.unregister(frame.agentAddress); - // Unregister unconditionally (a no-op for a multi-step address that - // registered no sources handler), matching the sibling routers. deps.multistepSourcesRouter?.unregister(frame.agentAddress); deps.multistepCredentialsRouter?.unregister(frame.agentAddress); - // Shut the per-deployment supervisor down so the workflow-process - // child, its IPC pipes, and its event-channel fd are released. - // The supervisor's `shutdown()` is idempotent (returns early when - // the supervisor is already in `idle`/`stopped`) and handles the - // kill + `exited` await internally. The map entry is removed - // before the await so a subsequent re-deploy on the same address - // cannot observe a stale handle even if `shutdown()` rejects. + // Map entry removed before the await so a subsequent re-deploy cannot observe a stale handle even if shutdown() rejects. const wired = activeSupervisors.get(frame.agentAddress); if (wired !== undefined) { activeSupervisors.delete(frame.agentAddress); await wired.supervisor.shutdown(); - // Drop the deployment address's transport registration installed at - // spawn (OUTBOUND half of mailbox ownership, §3a). Both single- and - // multi-step register the deployment address for outbound signing, so - // this tears down a real registration for either; `unregister` is a - // no-op only if the spawn failed before registering, so it is safe to - // call unconditionally for any spawned deployment. deps.transport.unregister(frame.agentAddress); - // Reclaim the deployment's per-step local-disk scratch now that - // its supervisor + workflow-process child are torn down. The - // whole `workflow-step-state//` subtree goes: the - // warm single-step agent's stable workspace under `warm/` (the - // dir bounded keying parks per agent) AND any cold `runs//` - // subtrees a multi-step deploy's per-run cleanup did not already - // drop. Awaiting `shutdown()` above guarantees no child still - // holds the scratch, so this is a safe `rm -rf`. The durable - // conversation under `agent-conversation-state/` is a DIFFERENT - // root and is deliberately NOT touched here -- a re-deploy on the - // same address must restore the prior conversation from it. + // Whole workflow-step-state// subtree goes; agent-conversation-state/ is a different root, deliberately not touched, so a re-deploy restores the prior conversation. if (stepStateDataDir !== undefined) { await rm(pathJoin(stepStateDataDir, "workflow-step-state", runId), { recursive: true, @@ -1497,14 +1441,7 @@ export function createSidecarDeployRouter(deps: { }); } } - // Drop the run record so a boot-time restore does not re-spawn a - // torn-down deployment, and reclaim a source-ref deployment's - // materialized closure tree AND its durable source-asset store. All run - // on every undeploy -- not only when a supervisor was active -- so state - // left behind by a crash-interrupted deploy, or by a source-ref restore - // that materialized the closure and then failed to spawn (registry down), - // is reclaimed too. A registry-sourced deployment never creates the source - // store, so its `force` remove is a no-op there. + // Runs on every undeploy, not only when a supervisor was active, so a crash-interrupted deploy's leftover state is reclaimed too. if (stepStateDataDir !== undefined) { await deleteWorkflowRunRecord(stepStateDataDir, runId); await rm(pathJoin(stepStateDataDir, "workflow-definition-closures", runId), { @@ -1556,24 +1493,7 @@ export function createSidecarDeployRouter(deps: { continue; } - // Reconstruct this deployment's runnable definition. Source-ref is - // the only lineage: re-materialize the pinned closure and evaluate the - // pinned code to the live definition, then project it to the inert - // wire shape -- the SAME computation the deploy path applies - // (`WorkflowProjectionDefinition(projectLiveToInert(...))`). The - // closure IS the source of truth; no on-disk definition is read. The - // helper reclaims the instance dir first, which is safe here because - // the prior process (the only reader) is dead and restore is serial - // before `hubLink.connect()`, so no concurrent reader holds it. - // Registry-sourced entries fetch from the content-addressed closure - // cache (a hit populated on the original deploy, surviving restart); - // asset-sourced entries read from the durable source store the - // original deploy checked out (`materializeDeploymentClosure` derives - // the mounts from the pin, so no re-delivery is needed). Both are - // SRI-verified. A cache/store miss soft-fails the record (kept for the - // next boot), matching the `assertSourceBuildable` retry-on-later-boot - // behavior below. The schema guarantees a source-ref record carries a - // `sourceRef` pin, so no undefined-check is needed. + // Same computation the deploy path applies; safe to reclaim the instance dir first since the prior process is dead and restore is serial before hubLink.connect(). A cache/store miss soft-fails the record, kept for the next boot. const applied = await materializeDeploymentClosure(dataDir, runId, record.sourceRef); const validatedDefinition = WorkflowProjectionDefinition( projectLiveToInert(applied.definition), @@ -1585,18 +1505,10 @@ export function createSidecarDeployRouter(deps: { const definition: WorkflowProjectionDefinition = validatedDefinition; const closurePackageDir = applied.packageDir; - // Structural invariants the wire arktype does not cover (non-empty - // stepOrder, every stepOrder entry backed by a `steps` entry AND a - // `sources` entry). The closure eval skips the deploy frame's coverage - // narrow, so this is where its definition-vs-sources coverage is - // checked. + // The closure eval skips the deploy frame's coverage narrow, so this is where it's checked. validateWorkflowProjection({ definition, sources: record.sources }); - // Re-run the source-admission gate: refuse to restore a deployment - // whose pinned provider this sidecar can no longer build. Every - // source in a step's failover chain must be buildable, so this - // iterates the whole list. The record is KEPT (not deleted) so a - // later boot with the provider restored retries it. + // Refuses to restore a deployment whose pinned provider this sidecar can no longer build; the record is KEPT so a later boot with the provider restored retries it. for (const stepId of definition.stepOrder) { const chain = record.sources[stepId]; if (chain !== undefined) { @@ -1608,47 +1520,21 @@ export function createSidecarDeployRouter(deps: { agentAddress: record.agentAddress, definition, sources: record.sources, - // The record's unsealed body sources, re-delivered to the run child - // through the spawn env. `undefined` for a legacy record written - // before body sources were persisted here; the child then falls back - // to that deployment's on-disk plaintext file for the missing bodies. + // undefined for a legacy record written before body sources were persisted; the child falls back to the on-disk plaintext file. bodySources: record.bodySources, - // The record's credential-material cell, unsealed by the boot scan - // and re-delivered to the child on the pre-trigger barrier so an - // offline restart restores both inference and tool credentials - // without the hub. `undefined` when the deployment bound none. + // Unsealed by the boot scan so an offline restart restores credentials without the hub. credentials: record.credentials, - // The hub-approved wire hash the original deploy persisted, so the - // restore re-spawn carries the same `DEFINITION_HASH` rather than a - // recompute. Always present -- the record schema requires it. approvedWireHash: record.approvedWireHash, sessionId: record.sessionId, hubPublicKey: record.hubPublicKey, - // The sidecar-local dir of the just-materialized closure the spawn - // core threads into the child's env so it re-evaluates the pinned - // code. closurePackageDir, - // Carry the source-ref pin so a post-restore source rotation -- - // which rebuilds the record from the spec -- re-persists it; without - // this a rotation would silently drop it and wedge the NEXT restart. + // Carried so a post-restore rotation re-persists it; otherwise a rotation would silently drop it and wedge the next restart. sourceRef: record.sourceRef, }; - // The slug is the caller's, matching `deployMultiStep`: claim before - // the spawn, release on failure. Unlike deploy's soft-fail, restore - // does NOT delete the record and does NOT re-materialize the step - // grants or the onTrigger body sources -- both are already on disk - // from the original deploy. A failed restore just warns and - // leaves the record for the next boot; there is deliberately no GC - // of a permanently-unrestorable record here (an operator reclaims it - // by undeploying the address). + // Unlike deploy's soft-fail, restore does NOT delete the record: a failed restore just warns and leaves it for the next boot (an operator reclaims via undeploy). // - // Release only a slug THIS pass newly claimed: if the address is - // already live (its slug still held by the running deployment), the - // core's double-spawn guard throws, and freeing the slug then would - // strand a live deployment's collision guard. `claimSlug` is a - // no-op for an already-held (runId, address) pair, so the - // pre-claim check distinguishes the two. + // Release only a slug THIS pass newly claimed, or freeing it would strand a live deployment's collision guard. const slugNewlyClaimed = slugClaims.get(runId) !== record.agentAddress; claimSlug(runId, record.agentAddress); try { @@ -1697,15 +1583,7 @@ export function createSidecarDeployRouter(deps: { }; } -/** - * Logical mail-audit reference the supervisor stamps onto every - * inbox/processing/consumed envelope for sidecar-hosted deployments. - * The substrate does not dereference the value; it is a host-side - * pointer the audit consumer joins on. The mail audit is keyed by the - * deployment id plus the parsed messageId, which is unique per inbound - * message and stable across the FIFO pipeline's - * enqueue/dequeue/markConsumed transitions. - */ +/** The substrate never dereferences this; it's a host-side pointer the audit consumer joins on, unique per inbound message. */ export function deriveSidecarMailAuditRef(runId: string): ( messageId: string, rawMessage: Uint8Array, From 93792c997d2dc37ad604fe6d2dd0bca4c2352402 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 19:02:24 -0700 Subject: [PATCH 31/31] chore: refresh bun.lock after rebase (stale package subtree entries) --- bun.lock | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/bun.lock b/bun.lock index 2a531291d8..9afa22c250 100644 --- a/bun.lock +++ b/bun.lock @@ -2015,8 +2015,6 @@ "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - "@corbits/inference-catalog/@corbits/ollama-adapter": ["@corbits/ollama-adapter@github:corbitsdev/corbits-ollama-adapter#0387eb1", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-ollama-adapter-0387eb1", "sha512-O1dl5lSwWzNGJibQLlpYDFKJZWpstgHX519bl8oe1ALL/KHszT/tBDlpsD9jN3hmQD3jV2L1jNwvOwLOBLtpyw=="], - "@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], @@ -2035,16 +2033,8 @@ "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "@workbench/hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#5309a15", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-5309a15", "sha512-zGHkq++juclUrOd0O8/Z5ziTldmJZ6gqyH/c9cs5S10UPU4gqn8CHwq174SiJXY0ujJ3cNpJXnrr+0fcqlfQ1w=="], - - "@workbench/hub/@corbits/codex-provider": ["@corbits/codex-provider@github:corbitsdev/corbits-codex-provider#a1f475f", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-codex-provider-a1f475f", "sha512-D6d0lblWx/4KudfLwQoQGrLB8GKRLP/HVp1cx08L2/hOGZAgH6txLPpeEtBqjl+R520qXSSPfM0kDC+3XNaAfw=="], - - "@workbench/hub/@corbits/mailbox": ["@corbits/mailbox@github:corbitsdev/corbits-mailbox#e85b81e", { "dependencies": { "@hono/standard-validator": "0.2.3", "@intx/crypto": "0.3.0", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", "arktype": "2.1.29", "hono-openapi": "1.3.1" }, "peerDependencies": { "@intx/log": "^0.2.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.0", "postgres": "^3.4.0" } }, "corbitsdev-corbits-mailbox-e85b81e", "sha512-ATPP/pll1sYG1DLKV1HvtboK8q1tFkqayQqRXdUmRUIsFJx5wqzd8YA1XELjXzLu1P2f2uCMe2CjsJHGv6E90Q=="], - "@workbench/hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#e74da20", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-e74da20", "sha512-/uA9xTcF7XwMXgnR8OZamierzkLG4g4mhw3u9Fl3Uj6ait8pH3wcoQ6V32on1nMdaqtPRdYVFnxof5O+4ZacGA=="], - "@workbench/hub/@corbits/xai-provider": ["@corbits/xai-provider@github:corbitsdev/corbits-xai-provider#9f2d4ba", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-xai-provider-9f2d4ba", "sha512-ZjvKxGcp+QzCh8zeXHg7kNbtkpR/itZkmQAcdG1aOAOjHaNwfGVpkPc/c/XtwPNsQyaDjF8W9CHb4rH1xI2uCA=="], - "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], @@ -2135,16 +2125,6 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], - "@workbench/hub/@corbits/codex-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@workbench/hub/@corbits/codex-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - - "@workbench/hub/@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - - "@workbench/hub/@corbits/xai-provider/@corbits/oauth-core": ["@corbits/oauth-core@github:corbitsdev/corbits-oauth-core#1b7f9fb", { "dependencies": { "arktype": "2.2.3" } }, "corbitsdev-corbits-oauth-core-1b7f9fb", "sha512-PscyiDAl46umTM44NQPN1q5+PsGUV53Rxr4Wz/Ofz+k9LOOz6FBKamRFtWR8YWdKJt9I8sPOLOyEcMNNN2i+YQ=="], - - "@workbench/hub/@corbits/xai-provider/@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#7efe000", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-7efe000", "sha512-pH6C8T1WDJjqM+YbnQpofd7HgP5krEj48JCCQK16KI98a6kaz+gffpgdOdQpU3aNVZnFlyADXod1Z4tqUG/3ww=="], - "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], "minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],