diff --git a/VENDORED.md b/VENDORED.md index 6d5e167a7c..e4491bcb48 100644 --- a/VENDORED.md +++ b/VENDORED.md @@ -24,7 +24,7 @@ never a convenience. | Vendored path | What was copied | Upstream repo @ commit | Why not a published package | Owner | Kill date | Kill-date test | | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ---------- | ----------------- | -| `apps/sidecar` | Derived from upstream's own `apps/sidecar`: six modules byte-identical (`default-harness.ts`, `signing-keypair.ts`, `source-asset-delivery.ts`, `workflow-closure-apply.ts`, `workflow-probe-handler.ts`, `workflow-run-pack-restore.ts`), three near-verbatim (`atomic-write.ts` differs only by the repo-wide logger namespace), the rest (`index.ts`, `config.ts`, `tool-materialization.ts`, `step-agent-tools.ts`, `workflow-host-wiring/`, `workflow-substrate-factory/`, …) substantially rewritten, plus workbench-only modules. A living fork, not a frozen copy, so this row carries no tree hash. | [faremeter/interchange](https://github.com/faremeter/interchange) @ `79adc433` (origin/main, 2026-09-17) | An app is never npm-published, so no publish can cover the execution host; retired by consuming an upstream-published host, or by renewing this row deliberately | sawyer | 2026-11-03 | `check:killdates` | +| `apps/sidecar` | Byte-identical to upstream `apps/sidecar` at this pin — no local delta; every Workbench-only module (`oauth-login.ts`, `originating-workbench.ts`, `hibernated-agent-identity-vault.ts`, `hub-link-watchdog.ts`, `workflow-deployment-record.ts`, `shutdown.ts`, `concurrency.ts`, `boot-restore-push-hold.ts`, the `workflow-host-wiring/`/`workflow-substrate-factory/` split directories, `step-strategy.ts`, `workbench-director.ts`, and their tests) deleted outright. | [faremeter/interchange](https://github.com/faremeter/interchange) @ `79adc433` (origin/main, 2026-09-17) | An app is never npm-published, so no publish can cover the execution host; retired by an upstream `@intx/sidecar` publish, or by renewing this row deliberately | sawyer | 2026-11-03 | none — no check scripts | | `vendor/intx/agent` | `@intx/agent` source (`src/`, manifest, tsconfig) | [faremeter/interchange](https://github.com/faremeter/interchange) @ `79adc433` (origin/main, 2026-09-17) | npm 0.3.0 predates the operator-configurable doom-loop threshold (`afd0c82b`, `c421c092`) the re-vendored `workflow-host` configures; the CL-7190 `beforeToolExtension` delta is absorbed upstream at this pin — no local delta; retired by the next `@intx/agent` publish | sawyer | 2026-11-03 | `check:killdates` | | `vendor/intx/db` | `@intx/db` source (`src/`, `migrations/`, drizzle config, manifest, tsconfigs) | [faremeter/interchange](https://github.com/faremeter/interchange) @ `79adc433` (origin/main, 2026-09-17) | npm 0.3.0 predates this pin's schema; the CL-6324 `wire_projection` column/loader and the CL-6452 `workflow_definition.origin` column are GONE upstream at this pin (CL-8206) — every hub-side reader of either is a hard cutover in this change, not a port — and the unpublished per-principal signing-key series (`principal_key`) this pin previously re-vendored is now upstream-native, so that delta is dropped; the pin keeps a CL-7505 serving-time refresh delta (`model-source-resolution.ts`: `buildSource`/`resolveModelSources`/`resolveInstanceModelSources` accept an optional `servingRefresh` hook, with a `credential_needs_reauth` skip so an unrefreshable `oauth_token` credential is never served) and a CL-7510 delta (`"openai-responses"` in `schema/catalog.ts`'s `modelProvider.plugin` enum, and an adapter registry-key dispatch rewrite in `model-source-resolution.ts`'s `buildSource` so `ollama`/`codex`/`xai-oauth` resolve their own registry key while the catalog `plugin` column keeps the wire-format id); retired when upstream absorbs the deltas | sawyer | 2026-11-03 | `check:killdates` | | `vendor/intx/harness` | `@intx/harness` source (`src/`, manifest, tsconfig) | [faremeter/interchange](https://github.com/faremeter/interchange) @ `79adc433` (origin/main, 2026-09-17) | npm 0.3.0 predates the connector reply drain (`driveConnectorReplies`, `ConnectorReplyDrain`, `AgentEventStream`; `11590e66`) the sidecar's warm mail loop drives; no local delta; retired by the next `@intx/harness` publish | sawyer | 2026-11-03 | `check:killdates` | diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index a16a1a5473..31b9a80fe0 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -110,7 +110,6 @@ import { createMcpOAuthRoutes, createMcpServerRoutes, createOAuthConnectRoutes, - createOAuthLoopbackRoutes, createTenantConnectCredential, createWorkflowConnectionRoutes, DEFAULT_RETURN_PATH_ALLOWLIST, @@ -516,26 +515,20 @@ export async function createHub(config: HubConfig) { // process-provisioned sidecar reaches easily, since a child that // outlives its allocation keeps reconnecting to the same hub. const sidecarCredentials = createSidecarCredentialResolver({ db }); + // CL-7508's loopback login (`oauthLogin` gate + `requestOAuthLogin`) is + // native to `@intx/hub-sessions`'s sidecar router, but it only works when + // the sidecar side answers the hub's `oauth.login.start` frame with an + // `oauthLoginExecutor` wired into `createSidecarOrchestrator`'s hub link + // (`@intx/hub-agent`). Absent that, the link falls back to a fail-closed + // placeholder executor that errors every `oauth.login.start` it receives. + // Upstream's `apps/sidecar` (the byte-for-byte copy this repo now runs) + // wires no such executor, so the gate is dead on arrival. Left unset here + // rather than configured against a sidecar that can never answer it. const sidecarRouter = createSidecarRouter({ hubPublicKey, authenticateSidecar: async ({ token }) => sidecarCredentials.resolve(token), validateSidecarIdentity: sidecarCredentials.isCurrent, lookups, - // CL-7508: a loopback login may only run on a sidecar whose allocation - // was provisioned by the `process` backend — the only one that runs on - // this host, where the user's browser can reach the pinned ports - // (1455/1456). A docker/e2b-provisioned sidecar's localhost is the - // container or the sandbox, never this machine, so it fails the gate - // and the connect request gets the typed gate outcome. - oauthLogin: { - isLocalSidecar: async (identity) => { - const allocation = await db.query.sidecarAllocation.findFirst({ - where: (allocation, { eq: equals }) => equals(allocation.id, identity.allocationId), - columns: { provisionerId: true }, - }); - return allocation?.provisionerId === "process"; - }, - }, }); const isSidecarRoutable = (address: string) => sidecarRouter.getRoutableAddresses().includes(address); @@ -1119,21 +1112,6 @@ export async function createHub(config: HubConfig) { returnPathAllowlist: [...DEFAULT_RETURN_PATH_ALLOWLIST, "/plugins", "/w/"], }), ); - // Loopback OAuth connect (CL-7508): codex/xai-oauth connect through a - // sidecar-hosted pinned-port login, not a hub redirect. The gate (a local - // sidecar) lives on the router; this mount only ships the authorize URL - // back to the caller and lets the sidecar's terminal result frame persist - // through the shared connect sequence. - app.route( - `${TENANT_PREFIX}/connections/oauth`, - createOAuthLoopbackRoutes({ - hubUrl: config.baseUrl, - log: (line) => log.info`${line}`, - registry: CONNECTOR_REGISTRY, - requestOAuthLogin: (args) => sidecarRouter.requestOAuthLogin(args), - providerHealth: providerHealthStore, - }), - ); // MCP servers: the tenant-scoped connect/list/disconnect surface // Plugins drives (CL-6142), mirroring `connections` above but for // tenant-minted `mcp:` connectors rather than diff --git a/apps/sidecar/README.md b/apps/sidecar/README.md index ab5920d1a8..b5725a09d8 100644 --- a/apps/sidecar/README.md +++ b/apps/sidecar/README.md @@ -1,58 +1,28 @@ -# @workbench/sidecar - -Generic execution host that dials in to a hub over WebSocket and hosts -whatever it is deployed. Apps stay generic (see -[AGENTS.md](../../AGENTS.md)): the sidecar knows no deployment by name -and holds no policy about what runs on it — the hub tells it everything -over the wire, so hosted and local topologies are identical and any -number of sidecars can dial the same hub unchanged. - -Boot order matters: orphaned tarball-cache staging is swept before any -apply work is accepted, the process identity keypair loads before -anything touches the data dir's substrate, the deploy router is captured -during orchestrator construction, restored deployments re-establish -before the hub connection opens (mailbox registrations must be live -before the hub routes to them), the watchdog arms for the first connect -attempt, and only then does the link dial out. - -## What it wires - -- `@intx/hub-agent`'s `createSidecarOrchestrator` and `HubLink` for the - WebSocket connection to the hub; workflow deployments arrive as - `agent.deploy` frames and run in supervised workflow-process children - via `@intx/workflow-host`. -- `src/workflow-host-wiring/` binds `createWorkflowSupervisor` to this - host's specifics: the mail-bus instance, the sidecar's Ed25519 signing - keypair, the substrate `RepoStore` handle, and `Bun.spawn` as the - subprocess spawner. Logic that would benefit an alternative sidecar - implementation lives in `@intx/workflow-host`, not here. -- `@intx/hub-sessions`' `createAgentRepoStore`, `@intx/inference`'s - provider adapter registry, and `@intx/tool-packaging`'s tarball cache. -- `signing-keypair.ts`, `hub-link-watchdog.ts`, `shutdown.ts` — process - identity, reconnect supervision, and graceful drain on exit. - -## Key modules - -- `index.ts` — composition root; owns the boot order described above. -- `config.ts` — env schema for hub location and process identity. -- `workflow-host-wiring/` — `supervisor.ts`, `transport.ts`, - `step-strategy.ts`, `asset-materialization.ts`, `wire-validation.ts`. -- `workflow-substrate-factory/` — per-run substrate construction: - `child-runtime.ts`, `step-env.ts`, `storage-paths.ts`. -- `run-grants.ts`, `step-agent-tools.ts` — grant resolution and tool - materialization for a running step. -- `bin/workflow-child` — the subprocess entry point workflow steps spawn - into. - -## Running - -``` -bun run dev # apps/sidecar/package.json -bun run start -``` - -## Tests - -``` -cd apps/sidecar && bun test -``` +# @intx/sidecar-app + +The canonical host process for hub-orchestrated agents. Turns a +single Bun process into a fleet of agent runtimes driven by the +hub. + +`src/index.ts` resolves the tarball cache configuration, sweeps any +orphaned staging directories left by a crashed apply, then starts a +`@intx/hub-agent` sidecar orchestrator. The orchestrator is wired +with an in-memory mail transport (`@intx/mail-memory`), the Web +Crypto provider (`@intx/crypto`), and a deploy router; on each +inbound deploy the router creates an `@intx/workflow-host` supervisor +for the deployment and spawns a supervised workflow-process child. +The child assembles the step's runtime across two seams: +`src/step-agent-tools.ts` materializes the pinned tool-package +closure and composes the POSIX and LSP plugin chain, and +`src/workflow-substrate-factory.ts` builds the per-step environment — +the isomorphic-git context store and the supervisor-backed mail +transport the tools bind to. `src/default-harness.ts` provides only +the `HarnessBuilder` source-admission check (`canBuildSource`) the +deploy router uses to reject an unbuildable inference source before +spawning. + +Run it under Bun. Startup requires `SIDECAR_DATA_DIR`, `HUB_WS_URL`, +`SIDECAR_ID`, and `SIDECAR_TOKEN`. Optional `SIDECAR_CACHE_DIR` +relocates the tarball cache (defaults to a subdirectory of the data +dir), and the cache and registry size caps are read through +`src/config.ts`. diff --git a/apps/sidecar/bin/workflow-child b/apps/sidecar/bin/workflow-child index edc8ab7927..4a49a52075 100755 --- a/apps/sidecar/bin/workflow-child +++ b/apps/sidecar/bin/workflow-child @@ -1,9 +1,8 @@ -#!/usr/bin/env bun -// The workflow-process child this host spawns, one per workflow -// deployment. The supervisor spawns it by path (`Bun.spawn([binaryPath])`) -// with a fresh env carrying only the typed substrate-config keys; the -// published `runWorkflowChildFromProcessEnv` owns the env crossing and -// hands the validated config to this host's substrate factory. +#!/usr/bin/env -S bun --conditions=intx-src +// The intx-src condition resolves @intx/* to TypeScript source. The +// sidecar spawns this child by path (`Bun.spawn([binaryPath])`) and +// `--conditions` is not inherited, so it is set on the shebang. The dev +// loop builds no dist; a bundled deployment does not run this file. import { runWorkflowChildFromProcessEnv } from "@intx/workflow-host"; import { SIDECAR_SUBSTRATE_CONFIG_KEYS, diff --git a/apps/sidecar/bin/workflow-probe-child b/apps/sidecar/bin/workflow-probe-child index 451b215412..490fc322fd 100755 --- a/apps/sidecar/bin/workflow-probe-child +++ b/apps/sidecar/bin/workflow-probe-child @@ -1,6 +1,8 @@ -#!/usr/bin/env bun +#!/usr/bin/env -S bun --conditions=intx-src // One-shot workflow-probe child. The sidecar host spawns this by path -// (`Bun.spawn([binaryPath])`). +// (`Bun.spawn([binaryPath])`); `--conditions` is not inherited, so it is +// set on the shebang to resolve @intx/* to TypeScript source in the dev +// loop. A bundled deployment does not run this file. // // The child reads the materialized package dir and IPC anchors from its // fresh env, evaluates the workflow entry behind the airlock, and ships diff --git a/apps/sidecar/package.json b/apps/sidecar/package.json index 7b66a1a006..a663a96f38 100644 --- a/apps/sidecar/package.json +++ b/apps/sidecar/package.json @@ -1,8 +1,8 @@ { "name": "@workbench/sidecar", - "version": "0.0.1", + "version": "0.3.0", "private": true, - "description": "Generic execution host that dials in to a hub over WebSocket and hosts whatever it is deployed", + "description": "Host process running a fleet of hub-orchestrated agent runtimes in one Bun process", "license": "SEE LICENSE IN ../../LICENSE.md", "bin": { "workflow-child": "./bin/workflow-child" @@ -15,12 +15,6 @@ "test": "bun test" }, "dependencies": { - "@corbits/codex-provider": "workspace:*", - "@corbits/credential-providers": "workspace:*", - "@corbits/error-sink": "workspace:*", - "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core#v0.1.1", - "@corbits/ollama-adapter": "workspace:*", - "@corbits/xai-provider": "workspace:*", "@intx/agent": "workspace:*", "@intx/authz": "0.3.0", "@intx/crypto": "0.3.0", @@ -30,9 +24,12 @@ "@intx/inference": "workspace:*", "@intx/log": "0.3.0", "@intx/mail-memory": "workspace:*", - "@intx/mime": "workspace:*", + "@intx/pack-transport": "0.3.0", "@intx/storage-isogit": "0.3.0", "@intx/tool-packaging": "0.3.0", + "@intx/tools-lsp": "0.3.0", + "@intx/tools-mail": "0.3.0", + "@intx/tools-posix": "0.3.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -41,6 +38,7 @@ }, "devDependencies": { "@types/bun": "catalog:", + "isomorphic-git": "catalog:", "typescript": "catalog:" } } diff --git a/apps/sidecar/src/agent-key-registration-lifecycle.test.ts b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts new file mode 100644 index 0000000000..e463a0b8b8 --- /dev/null +++ b/apps/sidecar/src/agent-key-registration-lifecycle.test.ts @@ -0,0 +1,394 @@ +// Pins the security-critical registration lifecycle of the launched-agent +// signing key on the host transport: the single-step deploy registers the +// agent's CryptoProvider on the host transport at spawn (sub-step 4.3's +// outbound-signing registration), and undeploy UNREGISTERS it so no signing +// key leaks for a torn-down agent. The undeploy-supervisor wiring test drives +// the same harness but never asserts the transport registration state. +// +// Probe: `getTransportFor(address)` throws "is not registered" for an address +// with no CryptoProvider; it succeeds once registered. We assert (a) it +// throws BEFORE deploy, (b) succeeds AFTER deploy, (c) throws again AFTER +// undeploy. + +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { createEd25519Crypto, createNoopCredentialCipher, generateKeyPair } from "@intx/crypto"; +import { hexEncode } from "@intx/types"; +import { createInMemoryTransport } from "@intx/mail-memory"; +import type { RepoId, RepoStore } from "@intx/hub-sessions"; +import { + createControlChannelSender, + type FrameReader, + type NdjsonReader, + type NdjsonWriter, + type SubprocessHandle, + type SubprocessSpawner, +} from "@intx/workflow-host"; +import type { AgentDeployFrame } from "@intx/types/sidecar"; +import type { WorkflowDefinition } from "@intx/workflow"; + +import { type } from "arktype"; + +import { createSidecarDeployRouter, deriveDeploymentId } from "./workflow-host-wiring"; +import { WorkflowRunRecord } from "./workflow-run-record"; +import { + createMultistepDrainRouter, + createMultistepMailRouter, + createMultistepSignalRouter, +} from "./workflow-run-pack-client"; + +function createMemoryNdjsonStream() { + const buffer: string[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: NdjsonReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) throw new Error("buffer shift undefined"); + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + const writer: NdjsonWriter = { + write(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + return Promise.resolve(); + }, + }; + return { + writer, + reader, + inject(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + }, + close() { + done = true; + wake(); + }, + }; +} + +function createMemoryFrameStream() { + const buffer: Uint8Array[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: FrameReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) throw new Error("frame shift undefined"); + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + return { + reader, + close() { + done = true; + wake(); + }, + }; +} + +function createSpawnTestRepoStore(tempBase: string): RepoStore { + const stub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(tempBase, repoId.kind, repoId.id); + }, + async writeTreePreservingPrefix(_p, _id, _ref, args) { + await args.merge(new Map()); + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + async writeTree(_p, repoId, _ref, content) { + const dir = path.join(tempBase, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = path.join(dir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, contents); + } + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented`); + }; + }, + }); +} + +const AGENT_ADDRESS = "run_keylifecycle@example.com"; + +function isRegistered(transport: ReturnType): boolean { + try { + transport.getTransportFor(AGENT_ADDRESS); + return true; + } catch { + return false; + } +} + +describe("agent signing-key registration lifecycle on the host transport", () => { + test("single-step deploy registers the agent crypto; undeploy unregisters it", async () => { + type SpawnEntry = { + env: Record; + childToSupervisor: ReturnType; + handle: SubprocessHandle; + killed: boolean; + resolveExited: (code: number) => void; + }; + const spawns: SpawnEntry[] = []; + const spawner: SubprocessSpawner = ({ env }) => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExited: (code: number) => void = () => undefined; + const exited = new Promise((resolve) => { + resolveExited = resolve; + }); + const entry: SpawnEntry = { + env, + childToSupervisor, + killed: false, + resolveExited, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- assigned below + handle: undefined as unknown as SubprocessHandle, + }; + const handle: SubprocessHandle = { + pid: 6100 + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + entry.killed = true; + childToSupervisor.close(); + eventChildToSupervisor.close(); + entry.resolveExited(0); + }, + exited, + }; + entry.handle = handle; + spawns.push(entry); + return handle; + }; + + const transport = createInMemoryTransport(); + const keyPair = await generateKeyPair(); + const tempBase = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-keylifecycle-")); + const dataDir = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-keylifecycle-data-")); + const repoStore = createSpawnTestRepoStore(tempBase); + + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) + sessions: { + provisionAgent: async () => { + throw new Error("must not invoke provisionAgent"); + }, + persistHubPublicKey: async () => { + throw new Error("must not invoke persistHubPublicKey"); + }, + initRepo: async () => undefined, + } as unknown as Parameters[0]["sessions"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn + keyStore: { + recordHubKey: () => undefined, + loadOrGenerateKey: async () => ({ + keyPair: await generateKeyPair(), + isNew: false, + }), + } as unknown as Parameters[0]["keyStore"], + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: () => undefined, + unregisterDeployment: () => undefined, + multistepSubprocessSpawner: spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: dataDir, + // Source-ref materialization reads both byte caps from the substrate env. + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }, + // Source-ref is the only deploy lineage: the router derives the runnable + // definition by materializing the pin's closure through this dependency. + // The stub returns a valid single-step `step-1` live definition (its step + // carries an agent so it survives `projectLiveToInert`) so the deploy + // reaches the key-registration lifecycle this test exercises. + applyFrozenWorkflowClosure: (applyArgs) => + Promise.resolve({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a hand-built live definition cannot satisfy the full WorkflowDefinition nominal type; it stands in for a real closure evaluation + definition: { + id: "wf-keylifecycle", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { + "step-1": { + kind: "step", + id: "step-1", + agent: { + id: "agent-step-1", + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }, + }, + } as unknown as WorkflowDefinition, + packageDir: path.join(applyArgs.instanceDir, "package"), + deployDir: path.join(applyArgs.instanceDir, "deploy"), + }), + multistepMailRouter: createMultistepMailRouter(), + multistepSignalRouter: createMultistepSignalRouter(), + multistepDrainRouter: createMultistepDrainRouter(), + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + agentAddress: AGENT_ADDRESS, + agentId: "keylifecycle-agent", + hubPublicKey: "hub-pk", + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch does not read config + config: {} as AgentDeployFrame["config"], + workflow: { + // Placeholder hub-approved wire hash so the deploy path's fail-loud + // guard passes; production always stamps it. + approvedWireHash: "a".repeat(64), + // Source-ref is the only deploy lineage: the frame carries no inline + // definition, only the pin the sidecar re-materializes (the injected + // closure stub above evaluates it to a single-step `step-1` definition). + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + sources: { + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-step-1", + model: "claude-3-5", + }, + ], + }, + }, + }; + + // (a) Not registered before deploy. + expect(isRegistered(transport)).toBe(false); + + const deployPromise = router.deploy(frame); + while (spawns.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const spawn = spawns[0]; + if (spawn === undefined) throw new Error("unreachable"); + const channelId = spawn.env.IPC_CHANNEL_ID; + if (channelId === undefined) throw new Error("IPC_CHANNEL_ID missing"); + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + spawn.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: spawn.handle.pid, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + await deployPromise; + + // (b) Registered after deploy -- the supervisor can now sign agent mail. + expect(isRegistered(transport)).toBe(true); + + // (b') The deploy persisted a schema-valid restore record for the + // deployment, carrying the head address so a boot-time restore can + // re-establish it. + const anchorRunId = deriveDeploymentId(AGENT_ADDRESS); + const recordFile = path.join(dataDir, "workflow-runs", anchorRunId, "deployment.json"); + const parsedRecord = WorkflowRunRecord(JSON.parse(await fs.readFile(recordFile, "utf8"))); + if (parsedRecord instanceof type.errors) { + throw new Error(`run record failed validation: ${parsedRecord.summary}`); + } + expect(parsedRecord.agentAddress).toBe(AGENT_ADDRESS); + + const undeploy = router.undeploy; + if (undeploy === undefined) throw new Error("router.undeploy undefined"); + await undeploy({ + type: "agent.undeploy", + agentAddress: AGENT_ADDRESS, + reason: "key-lifecycle undeploy", + }); + + // (c) Unregistered after undeploy -- no leaked key for a torn-down agent. + expect(isRegistered(transport)).toBe(false); + + // (c') Undeploy dropped the restore record so a boot-time restore will + // not re-spawn the torn-down deployment. + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + + await fs.rm(tempBase, { recursive: true, force: true }); + await fs.rm(dataDir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/atomic-write.test.ts b/apps/sidecar/src/atomic-write.test.ts new file mode 100644 index 0000000000..34fcfc8d08 --- /dev/null +++ b/apps/sidecar/src/atomic-write.test.ts @@ -0,0 +1,137 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { removeFileAtomicDurable, writeFileAtomicDurable } from "./atomic-write"; + +async function makeDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "atomic-write-")); +} + +async function listNames(dir: string): Promise { + return (await fs.readdir(dir)).sort(); +} + +describe("writeFileAtomicDurable", () => { + test("round-trips contents to the target path", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, '{"a":1}', { mode: 0o600 }); + expect(await fs.readFile(file, "utf8")).toBe('{"a":1}'); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("applies mode on the created file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("re-applies mode when overwriting an existing file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Seed a pre-existing file with a laxer mode. A plain in-place + // overwrite keeps the original mode; the temp+rename path creates a + // fresh file every write, so the requested mode actually lands. + await fs.writeFile(file, "old", { mode: 0o644 }); + await writeFileAtomicDurable(file, "new", { mode: 0o600 }); + + expect(await fs.readFile(file, "utf8")).toBe("new"); + const stat = await fs.stat(file); + expect(stat.mode & 0o777).toBe(0o600); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("leaves no temp orphan after a successful write", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + await writeFileAtomicDurable(file, "x", { mode: 0o600 }); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("a write that fails before staging leaves the prior file intact", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + await fs.writeFile(file, "old", { mode: 0o600 }); + + // Deny writes to the directory so the temp-file creation fails + // outright. The prior complete record must survive untouched -- an + // interrupted rotation must never corrupt the sole restore source. + await fs.chmod(dir, 0o500); + try { + await expect(writeFileAtomicDurable(file, "new", { mode: 0o600 })).rejects.toThrow(); + expect(await fs.readFile(file, "utf8")).toBe("old"); + expect(await listNames(dir)).toEqual(["record.json"]); + } finally { + await fs.chmod(dir, 0o700); + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + test("a write that fails at the rename unlinks its staged temp", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + + // Occupy the target path with a directory so the rename fails after + // the temp file has already been staged and fsynced. This exercises + // the error-path unlink: the staged temp must be removed rather than + // stranded as an orphan. + await fs.mkdir(file); + await expect(writeFileAtomicDurable(file, "new", { mode: 0o600 })).rejects.toThrow(); + expect(await listNames(dir)).toEqual(["record.json"]); + + await fs.rm(dir, { recursive: true, force: true }); + }); +}); + +describe("removeFileAtomicDurable", () => { + test("removes an existing file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "record.json"); + await fs.writeFile(file, "x"); + + await removeFileAtomicDurable(file); + expect(await listNames(dir)).toEqual([]); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("is a no-op for an already-absent file", async () => { + const dir = await makeDir(); + const file = path.join(dir, "never-existed.json"); + + // Idempotent: removing an absent file completes without throwing, so a + // re-driven eviction of an already-evicted key is not a fault. + await removeFileAtomicDurable(file); + expect(await listNames(dir)).toEqual([]); + + await fs.rm(dir, { recursive: true, force: true }); + }); + + test("leaves sibling files untouched", async () => { + const dir = await makeDir(); + const target = path.join(dir, "target.json"); + const sibling = path.join(dir, "keep.json"); + await fs.writeFile(target, "gone"); + await fs.writeFile(sibling, "stay"); + + await removeFileAtomicDurable(target); + expect(await listNames(dir)).toEqual(["keep.json"]); + expect(await fs.readFile(sibling, "utf8")).toBe("stay"); + + await fs.rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/apps/sidecar/src/atomic-write.ts b/apps/sidecar/src/atomic-write.ts index f700c48c07..d0a32dda2e 100644 --- a/apps/sidecar/src/atomic-write.ts +++ b/apps/sidecar/src/atomic-write.ts @@ -7,13 +7,12 @@ // exposing a torn record. import { open, rename, unlink } from "node:fs/promises"; -import { existsSync } from "node:fs"; import { dirname } from "node:path"; import { getLogger } from "@intx/log"; -import { hexEncode } from "@intx/types"; +import { hasCode, hexEncode } from "@intx/types"; -const logger = getLogger(["sidecar", "atomic-write"]); +const logger = getLogger(["interchange", "sidecar", "atomic-write"]); export interface AtomicWriteOptions { /** Permission mode applied when the temp file is created. */ @@ -80,15 +79,26 @@ export async function writeFileAtomicDurable( } /** - * Durably remove `path`: unlink, then fsync the parent directory so the - * removal survives a power loss (an unlink with no directory fsync can be - * resurrected after a crash, re-staling a cache the removal was meant to - * invalidate). A missing file is a no-op, not an error -- the caller may be - * evicting an entry that never made it to disk. + * Remove `path` durably: unlink it, then fsync the parent directory so the + * removal survives a power loss. A raw `unlink` leaves the directory-entry + * removal in the OS's delayed-metadata window, so a power loss can resurrect + * the file -- for a cache whose on-disk entry is the restore source, a + * resurrected entry re-stales the cache. Idempotent: a file already absent is a + * completed removal, so ENOENT on the unlink is success. + * + * The parent-directory fsync mirrors `writeFileAtomicDurable`, including its + * degrade: a filesystem that rejects directory fsync (FAT/exFAT, some network + * mounts) only weakens durability -- the file is already unlinked -- so that + * failure is logged, not thrown. It runs unconditionally so a prior removal + * that unlinked but died before the fsync is made durable on the next attempt. */ export async function removeFileAtomicDurable(path: string): Promise { - if (!existsSync(path)) return; - await unlink(path).catch(() => undefined); + try { + await unlink(path); + } catch (err) { + if (!(hasCode(err) && err.code === "ENOENT")) throw err; + } + try { const dirHandle = await open(dirname(path), "r"); try { @@ -97,9 +107,6 @@ export async function removeFileAtomicDurable(path: string): Promise { await dirHandle.close(); } } catch (err) { - // report-error-ignore: mirrors writeFileAtomicDurable's own degraded-durability - // warn above — the file is already unlinked, so this is a best-effort fsync, - // not a failed operation, and the caller has nothing actionable to catch. - logger.warn`parent-dir fsync failed removing ${path}; durability is degraded but the file is unlinked — ${err instanceof Error ? err.message : String(err)}`; + logger.warn`parent-dir fsync failed for ${path}; durability is degraded but the file is unlinked — ${err instanceof Error ? err.message : String(err)}`; } } diff --git a/apps/sidecar/src/boot-restore-push-hold.test.ts b/apps/sidecar/src/boot-restore-push-hold.test.ts deleted file mode 100644 index 0894be3652..0000000000 --- a/apps/sidecar/src/boot-restore-push-hold.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { createBootRestorePushHold } from "./boot-restore-push-hold"; - -function recordingStore() { - const held: string[] = []; - return { - held, - markAddressUnroutable(agentAddress: string) { - held.push(agentAddress); - }, - }; -} - -describe("createBootRestorePushHold", () => { - test("holds every address a boot restore registers", () => { - const store = recordingStore(); - const hold = createBootRestorePushHold(store); - - hold.begin(); - hold.onDeploymentRegistered("run_a@bench.localhost"); - hold.onDeploymentRegistered("run_b@bench.localhost"); - hold.end(); - - expect(store.held).toEqual(["run_a@bench.localhost", "run_b@bench.localhost"]); - }); - - test("does not hold a deployment that arrives over a live link", () => { - const store = recordingStore(); - const hold = createBootRestorePushHold(store); - - hold.begin(); - hold.end(); - hold.onDeploymentRegistered("run_c@bench.localhost"); - - expect(store.held).toEqual([]); - }); - - test("holds nothing before the restore is armed", () => { - const store = recordingStore(); - const hold = createBootRestorePushHold(store); - - hold.onDeploymentRegistered("run_d@bench.localhost"); - - expect(store.held).toEqual([]); - }); -}); diff --git a/apps/sidecar/src/boot-restore-push-hold.ts b/apps/sidecar/src/boot-restore-push-hold.ts deleted file mode 100644 index f2d26e3bc2..0000000000 --- a/apps/sidecar/src/boot-restore-push-hold.ts +++ /dev/null @@ -1,43 +0,0 @@ -// Boot restore spawns each restored deployment's supervisor BEFORE -// `hubLink.connect()`, and a supervisor that finds a run whose step died -// mid-invocation commits that run's `StepFailed`/`RunFailed` immediately. -// Those commits schedule a workflow-run pack push into a link that does not -// exist yet, so the push fails "Connection lost" and its only recovery is the -// post-challenge re-drive, which re-ships a slot whose error has already -// latched. When the rejection lands AFTER the challenge fired, nothing -// re-arms the slot: the terminal event stays on sidecar disk, the hub's copy -// of the durable log stops at the last pre-crash event, and -// `workflow_run.status` stays "running" for a run that will never accept mail -// again. -// -// Holding every address registered during boot restore until the reconnect -// challenge proves it routable removes the race rather than recovering from -// it — the same block the link already applies across a mid-life disconnect. -import type { WorkflowRunPackPushingRepoStore } from "./workflow-run-pack-client"; - -export interface BootRestorePushHold { - /** Arm the hold for the duration of `restoreWorkflowDeployments`. */ - begin(): void; - /** Disarm it; deployments registered afterwards arrive over a live link. */ - end(): void; - /** Called for every deployment registration the deploy router makes. */ - onDeploymentRegistered(agentAddress: string): void; -} - -export function createBootRestorePushHold( - store: Pick, -): BootRestorePushHold { - let restoring = false; - return { - begin() { - restoring = true; - }, - end() { - restoring = false; - }, - onDeploymentRegistered(agentAddress) { - if (!restoring) return; - store.markAddressUnroutable(agentAddress); - }, - }; -} diff --git a/apps/sidecar/src/child-grant-filter.test.ts b/apps/sidecar/src/child-grant-filter.test.ts index e1ec60f3dd..c573e06845 100644 --- a/apps/sidecar/src/child-grant-filter.test.ts +++ b/apps/sidecar/src/child-grant-filter.test.ts @@ -92,6 +92,7 @@ describe("filterGrantsToDeclaredResources", () => { declared, ); const decision = await evaluateGrants( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the filter returns unknown[]; the seeded rows are GrantRule by construction here [...(filtered as readonly GrantRule[])], "tool:foo", "invoke", diff --git a/apps/sidecar/src/concurrency.test.ts b/apps/sidecar/src/concurrency.test.ts deleted file mode 100644 index 3eeeeed74e..0000000000 --- a/apps/sidecar/src/concurrency.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { describe, test, expect } from "bun:test"; - -import { runWithConcurrency } from "./concurrency"; - -describe("runWithConcurrency", () => { - test("runs every item and never exceeds the concurrency limit", async () => { - let inFlight = 0; - let maxInFlight = 0; - const seen: number[] = []; - - await runWithConcurrency([1, 2, 3, 4, 5, 6, 7], 3, async (item) => { - inFlight += 1; - maxInFlight = Math.max(maxInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 5)); - seen.push(item); - inFlight -= 1; - }); - - expect(seen.sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5, 6, 7]); - expect(maxInFlight).toBeLessThanOrEqual(3); - }); - - test("isolates a failing item: the rest still complete and the failure is reported per item", async () => { - const completed: number[] = []; - const failures = await runWithConcurrency([1, 2, 3], 2, async (item) => { - if (item === 2) throw new Error("boom"); - completed.push(item); - }); - - expect(completed.sort()).toEqual([1, 3]); - expect(failures).toHaveLength(1); - expect(failures[0]?.item).toBe(2); - expect((failures[0]?.error as Error).message).toBe("boom"); - }); - - test("an empty list resolves immediately with no failures", async () => { - const failures = await runWithConcurrency([] as number[], 4, async () => {}); - expect(failures).toEqual([]); - }); - - test("a limit larger than the item count still runs everything exactly once", async () => { - const seen: number[] = []; - await runWithConcurrency([1, 2], 10, async (item) => { - seen.push(item); - }); - expect(seen.sort()).toEqual([1, 2]); - }); -}); diff --git a/apps/sidecar/src/concurrency.ts b/apps/sidecar/src/concurrency.ts deleted file mode 100644 index 88b5ddc79c..0000000000 --- a/apps/sidecar/src/concurrency.ts +++ /dev/null @@ -1,38 +0,0 @@ -export interface ConcurrencyFailure { - item: T; - error: unknown; -} - -/** - * Runs `fn` over every item with at most `limit` in flight at once. Each - * item's failure is caught and returned rather than thrown, so one bad - * item never stops the rest of the batch from running -- the same - * per-item isolation a serial `for` loop with a `try`/`catch` gives, just - * bounded-parallel instead of one-at-a-time. - */ -export async function runWithConcurrency( - items: readonly T[], - limit: number, - fn: (item: T) => Promise, -): Promise[]> { - const failures: ConcurrencyFailure[] = []; - let nextIndex = 0; - - async function worker(): Promise { - while (nextIndex < items.length) { - const index = nextIndex; - nextIndex += 1; - const item = items[index] as T; - try { - await fn(item); - } catch (error) { - failures.push({ item, error }); - } - } - } - - const workerCount = Math.max(1, Math.min(limit, items.length)); - await Promise.all(Array.from({ length: workerCount }, () => worker())); - - return failures; -} diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index 0ce26eece1..bf3b4651f0 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -1,221 +1,83 @@ -// The one boundary that reads the process environment. Everything the -// host needs from the outside world is an irreducible process fact: -// where to keep durable state, where the hub is, who this sidecar is to -// that hub, and the OS facts a spawned workflow-process child cannot -// function without. Anything else the host learns is data it is told -// over the wire, never configuration. - -import { type } from "arktype"; +// Boundary readers for the sidecar's env-config inputs. +// +// Centralizing the readers here keeps env-validation rules in one +// place — `SIDECAR_CACHE_MAX_BYTES` must be a positive finite +// number, with a 10 GiB default — even though the sidecar only +// invokes the reader at one site today (the orchestrator boot in +// `apps/sidecar/src/index.ts`). The harness builder receives the +// resolved value through `DefaultHarnessBuilderConfig` rather than +// re-reading env, so the boundary stays at the boot edge. import { AdapterManifest } from "@intx/inference"; +import { hexDecode } from "@intx/types"; -import { parseToolRegistries } from "./tool-materialization"; - -/** - * The shipped default manifest: registers `@corbits/ollama-adapter`'s - * `createOllamaAdapter` for the `"ollama"` provider key so a seeded - * Ollama deployment's `quirks.numCtx` (see `@corbits/inference-catalog`'s - * `quirksForDeployment` and `./workflow-substrate-factory/context-budget`) actually - * reaches Ollama's `options.num_ctx` instead of silently falling back - * to the built-in adapter's defaults, and the CL-7510 loopback-OAuth - * Responses adapters (`codex`, `xai-oauth`) so a launched source — - * rewritten to the provider's own registry key at - * `@intx/db`'s `buildSource` — serves through its own adapter rather - * than the shared wire-format one. An operator who sets - * `SIDECAR_ADAPTER_MANIFEST` explicitly gets exactly what they wrote -- - * this default never merges with an operator value, only replaces the - * unset case. - */ -const DEFAULT_ADAPTER_MANIFEST: AdapterManifest = [ - { - provider: "ollama", - specifier: "@corbits/ollama-adapter", - export: "createOllamaAdapter", - }, - { - provider: "codex", - specifier: "@corbits/codex-provider", - export: "createCodexResponsesAdapter", - }, - { - provider: "xai-oauth", - specifier: "@corbits/xai-provider", - export: "createXaiResponsesAdapter", - }, -]; +const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; -const WsURL = type("string").narrow((url, ctx) => { - if (!url.startsWith("ws://") && !url.startsWith("wss://")) { - return ctx.mustBe("a ws:// or wss:// URL"); +// The operator key the sidecar seals its at-rest credential material under +// (inference-source apiKeys, and the tool credential material store). A +// SEPARATE key from the hub's `CREDENTIAL_ENCRYPTION_KEY`: the sidecar seals a +// local store on a host it does not control as tightly, so a sidecar +// disk-plus-key compromise must not also decrypt the hub's credential +// database, and the two keys rotate independently. Required at boot: a missing +// key fails loudly here rather than letting the sidecar run and persist secrets +// it cannot protect. 32 bytes, hex -- e.g. `openssl rand -hex 32`, the same +// shape as the hub's key. +export function readCredentialEncryptionKey(): Uint8Array { + const raw = process.env["SIDECAR_CREDENTIAL_ENCRYPTION_KEY"]; + if (raw === undefined || raw.trim() === "") { + throw new Error("SIDECAR_CREDENTIAL_ENCRYPTION_KEY environment variable is required"); } - if (!URL.canParse(url)) { - return ctx.mustBe("a parseable ws:// or wss:// URL"); + return hexDecode(raw); +} + +export function readCacheMaxBytes(): number { + const raw = process.env["SIDECAR_CACHE_MAX_BYTES"]; + if (raw === undefined || raw.trim() === "") return DEFAULT_CACHE_MAX_BYTES; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + throw new Error(`SIDECAR_CACHE_MAX_BYTES must be a positive number; got ${raw}`); } - return true; -}); + return n; +} -const SidecarEnv = type({ - SIDECAR_DATA_DIR: type("string > 0").describe("a non-empty filesystem path"), - HUB_WS_URL: WsURL, - SIDECAR_ID: type("string > 0").describe("a non-empty sidecar identifier"), - SIDECAR_TOKEN: type("string > 0").describe("a non-empty hub-issued token"), - // OS facts forwarded into each workflow-process child's fresh spawn - // env (nothing else is inherited): PATH so the child's `bun` shebang - // resolves, HOME/TMPDIR so agent code finds a writable home and the - // same temp root the host uses. - PATH: type("string > 0").describe("the executable search path"), - "HOME?": "string", - "TMPDIR?": "string", - // Optional JSON array of tool-package registries, - // `[{"name":"...","url":"...","auth?":{...}}]`. Unset means the - // public npmjs registry. The value is validated here so a malformed - // registry pin kills the boot, and threaded into every - // workflow-process child's spawn env so per-step tool - // materialization resolves the exact registries the operator pinned. - "SIDECAR_TOOL_REGISTRIES?": "string", - // Optional JSON-encoded custom inference adapter manifest override - // (`AdapterManifestEntry[]`, `[{"provider","specifier","export"}]`). - // Unset resolves to `DEFAULT_ADAPTER_MANIFEST` (the shipped Ollama - // adapter); set, it replaces that default entirely rather than merging - // with it. Validated here so a malformed manifest kills the boot with - // the variable named, and threaded (as its parsed form) into both this - // process's own adapter registry and every workflow-process child's - // `SIDECAR_ADAPTER_MANIFEST` substrate-config entry, so a child - // resolves the exact adapters this boot edge resolved. - "SIDECAR_ADAPTER_MANIFEST?": "string", - // Operator overrides for two workflow-supervisor timing bindings, - // threaded verbatim to every deployment's supervisor - // (`createSidecarWorkflowSupervisor`'s `consumedRetentionMs` / - // `readyTimeoutMs`). Unset leaves each supervisor on the vendor's own - // default (`DEFAULT_CONSUMED_RETENTION_MS` / `DEFAULT_READY_TIMEOUT_MS`). - // Validated as positive-finite-number strings, matching the - // `SIDECAR_CACHE_MAX_BYTES`-style numeric env keys. - "CONSUMED_RETENTION_MS?": "string", - "CHILD_READY_TIMEOUT_MS?": "string", - // CL-8187: how long a deployment may sit with no observed inbound mail - // before this sidecar's own idle-hibernate sweep tears it down with a - // state-preserving undeploy (deployment record, step-state, and slug - // survive so a later wake resumes the same run). Unset disables the - // sweep entirely -- nothing is tracked, no interval runs -- matching - // the behavior before this knob existed. - "CHAT_IDLE_HIBERNATE_MS?": "string", -}); +// Mirrors the hub's `DEFAULT_HUB_MAX_TARBALL_BYTES`. The sidecar's +// HTTP-registry fetcher enforces this cap on every upstream registry +// tarball pull. An operator pointing the sidecar at a third-party +// registry whose curated tarballs run larger should raise the cap +// explicitly via `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`. +const DEFAULT_REGISTRY_MAX_TARBALL_BYTES = 10 * 1024 * 1024; -/** - * Parse an optional positive-finite-number env value. Returns `undefined` - * for an unset key so the caller's binding falls back to the vendor's own - * default rather than this boundary inventing one. - */ -function parsePositiveMsEnv(raw: string | undefined, name: string): number | undefined { - if (raw === undefined) return undefined; +export function readRegistryMaxTarballBytes(): number { + const raw = process.env["SIDECAR_REGISTRY_MAX_TARBALL_BYTES"]; + if (raw === undefined || raw.trim() === "") return DEFAULT_REGISTRY_MAX_TARBALL_BYTES; const n = Number(raw); if (!Number.isFinite(n) || n <= 0) { - throw new Error( - `invalid sidecar environment: ${name} must be a positive finite number; got ${JSON.stringify(raw)}`, - ); + throw new Error(`SIDECAR_REGISTRY_MAX_TARBALL_BYTES must be a positive number; got ${raw}`); } return n; } -export type SidecarConfig = { - readonly dataDir: string; - readonly hubURL: string; - readonly sidecarId: string; - readonly token: string; - readonly path: string; - readonly home: string | undefined; - readonly tmpdir: string | undefined; - /** - * The operator's tool-registry pin, carried as the validated raw JSON - * so the boot edge can thread it verbatim into each workflow-process - * child's spawn env. `undefined` means no pin (the public npmjs - * default). - */ - readonly toolRegistries: string | undefined; - /** - * The inference adapter manifest, already validated against - * {@link AdapterManifest}: {@link DEFAULT_ADAPTER_MANIFEST} unless the - * operator set `SIDECAR_ADAPTER_MANIFEST`, in which case it is exactly - * (and only) what the operator wrote. - */ - readonly adapterManifest: AdapterManifest; - /** - * Consumed-dedup retention horizon (ms), forwarded verbatim to every - * deployment's supervisor. `undefined` means the operator did not - * override it; the supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` - * (24h). - */ - readonly consumedRetentionMs: number | undefined; - /** - * Spawn ready-handshake timeout (ms), forwarded verbatim to every - * deployment's supervisor. `undefined` means the operator did not - * override it; the supervisor applies `DEFAULT_READY_TIMEOUT_MS` (30s). - */ - readonly readyTimeoutMs: number | undefined; - /** - * How long a deployment may sit idle before this sidecar's own - * idle-hibernate sweep tears it down. `undefined` means the operator - * did not set `CHAT_IDLE_HIBERNATE_MS`; the sweep does not run at all. - */ - readonly idleHibernateMs: number | undefined; -}; - -/** - * Parse the optional `SIDECAR_ADAPTER_MANIFEST` env value into a validated - * {@link AdapterManifest}. Unset resolves to {@link DEFAULT_ADAPTER_MANIFEST} - * (the shipped Ollama adapter, so `num_ctx` reaches Ollama without operator - * configuration); a malformed value dies at boot with the variable named, - * rather than surfacing as a deep-stack `loadAdapterRegistry` import - * failure. - */ -export function parseSidecarAdapterManifest(raw: string | undefined): AdapterManifest { - if (raw === undefined) return DEFAULT_ADAPTER_MANIFEST; +// Operator-configured custom inference adapter manifest. The value is +// TRUSTED operator input read only from this process's environment; +// `import(specifier)` is arbitrary code execution, so a specifier must +// never originate from deploy or tenant data — the agent deploy tree +// carries only a `provider` key, never a specifier. The shape is +// arktype-validated here (and re-validated at the workflow-child spawn +// boundary as defense in depth), but the value itself is trusted. +// +// Unset or whitespace-only means "no custom adapters", a valid +// configuration — the sidecar then resolves only the statically-linked +// built-ins. A present-but-malformed value fails loud at boot: an +// opaque `JSON.parse` SyntaxError is rethrown naming the env key, and +// the parsed value is asserted against `AdapterManifest`. +export function readAdapterManifest(): AdapterManifest { + const raw = process.env["SIDECAR_ADAPTER_MANIFEST"]; + if (raw === undefined || raw.trim() === "") return []; let parsed: unknown; try { parsed = JSON.parse(raw); } catch (cause) { - throw new Error("invalid sidecar environment: SIDECAR_ADAPTER_MANIFEST is not valid JSON", { - cause, - }); - } - const validated = AdapterManifest(parsed); - if (validated instanceof type.errors) { - throw new Error( - `invalid sidecar environment: SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** - * Parse the sidecar's configuration out of an environment map. Throws at - * the call site when any variable is missing or malformed, naming the - * variable and the shape it must have, so a misconfigured process dies - * at boot instead of failing at first use. - */ -export function readSidecarConfig(env: Record): SidecarConfig { - const parsed = SidecarEnv(env); - if (parsed instanceof type.errors) { - throw new Error(`invalid sidecar environment: ${parsed.summary}`); - } - // Validate the registries JSON now so a malformed pin dies at boot - // with the variable named, not at the first tool materialization - // inside a workflow-process child. - if (parsed.SIDECAR_TOOL_REGISTRIES !== undefined) { - parseToolRegistries(parsed.SIDECAR_TOOL_REGISTRIES); + throw new Error("SIDECAR_ADAPTER_MANIFEST is not valid JSON", { cause }); } - return { - dataDir: parsed.SIDECAR_DATA_DIR, - hubURL: parsed.HUB_WS_URL, - sidecarId: parsed.SIDECAR_ID, - token: parsed.SIDECAR_TOKEN, - path: parsed.PATH, - home: parsed.HOME, - tmpdir: parsed.TMPDIR, - toolRegistries: parsed.SIDECAR_TOOL_REGISTRIES, - adapterManifest: parseSidecarAdapterManifest(parsed.SIDECAR_ADAPTER_MANIFEST), - consumedRetentionMs: parsePositiveMsEnv(parsed.CONSUMED_RETENTION_MS, "CONSUMED_RETENTION_MS"), - readyTimeoutMs: parsePositiveMsEnv(parsed.CHILD_READY_TIMEOUT_MS, "CHILD_READY_TIMEOUT_MS"), - idleHibernateMs: parsePositiveMsEnv(parsed.CHAT_IDLE_HIBERNATE_MS, "CHAT_IDLE_HIBERNATE_MS"), - }; + return AdapterManifest.assert(parsed); } diff --git a/apps/sidecar/src/conversation-state.test.ts b/apps/sidecar/src/conversation-state.test.ts deleted file mode 100644 index b4c9422c89..0000000000 --- a/apps/sidecar/src/conversation-state.test.ts +++ /dev/null @@ -1,333 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, expect, spyOn, test } from "bun:test"; - -import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions/substrate"; -import type { ConversationTurn, PendingOperation } from "@intx/types/runtime"; - -import { - createDurableConversationStore, - durableConversationAgentStateDir, - durableConversationAgentStatePrefix, - prepareConversationForOriginatingWorkbench, - reconstructDurableConversation, -} from "./conversation-state"; -import { originatingWorkbenchIdFromRequest } from "./originating-workbench"; - -const tmpDirs: string[] = []; - -afterEach(async () => { - await Promise.all( - tmpDirs.splice(0).map((dir) => fs.promises.rm(dir, { recursive: true, force: true })), - ); -}); - -function tmpDir(prefix: string): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); - tmpDirs.push(dir); - return dir; -} - -const WORKFLOW_RUN_REPO_ID: RepoId = { kind: "workflow-run", id: "wfr_iso" }; - -function userTurn(text: string): ConversationTurn { - return { - role: "user", - content: [{ type: "text", text }], - } as ConversationTurn; -} - -function peekTurns(store: { storage: unknown }): ConversationTurn[] { - const storage = store.storage as { peekTurns: () => ConversationTurn[] }; - return storage.peekTurns(); -} - -function fsBackedSubstrate(repoDir: string): RepoStore { - return { - getRepoDir: () => repoDir, - writeTreePreservingPrefix: async ( - _principal: Principal, - _repoId: RepoId, - _ref: string, - args: Parameters[3], - ) => { - const prefixDir = path.join( - repoDir, - ...args.preservePrefix.split("/").filter((seg: string) => seg.length > 0), - ); - const existing = new Map(); - let names: string[] = []; - try { - names = await fs.promises.readdir(prefixDir); - } catch (cause) { - if ( - !( - typeof cause === "object" && - cause !== null && - "code" in cause && - cause.code === "ENOENT" - ) - ) { - throw cause; - } - } - for (const name of names) { - const full = path.join(prefixDir, name); - const st = await fs.promises.stat(full); - if (st.isFile()) { - existing.set(`${args.preservePrefix}${name}`, await fs.promises.readFile(full)); - } - } - const files = await args.merge(existing); - await fs.promises.rm(prefixDir, { recursive: true, force: true }); - for (const [rel, content] of Object.entries(files)) { - const dest = path.join(repoDir, ...rel.split("/").filter((seg: string) => seg.length > 0)); - await fs.promises.mkdir(path.dirname(dest), { recursive: true }); - await fs.promises.writeFile(dest, content); - } - return { commitSha: "test" }; - }, - } as unknown as RepoStore; -} - -async function makeStore(root: string) { - const repoDir = path.join(root, "repo"); - await fs.promises.mkdir(repoDir, { recursive: true }); - return createDurableConversationStore({ - localStoreDir: path.join(root, "local"), - signer: () => Promise.resolve("sig"), - substrate: fsBackedSubstrate(repoDir), - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: "refs/heads/main", - principal: { kind: "workflow-process", anchorRunId: "run_1" } as Principal, - agentKey: "default", - }).then((store) => ({ store, repoDir })); -} - -test("durable conversation prefix nests workbench under agentKey", () => { - expect(durableConversationAgentStatePrefix("default", "chan_a")).toBe( - "agent-state/default/chan_a/", - ); - expect(durableConversationAgentStatePrefix("step_1", "chan_a")).not.toBe( - durableConversationAgentStatePrefix("step_2", "chan_a"), - ); -}); - -test("two rooms of one agent keep isolated turns; a new room starts empty", async () => { - const root = tmpDir("conv-iso-"); - const { store, repoDir } = await makeStore(root); - - expect(await store.bindOriginatingWorkbench("chan_a")).toBe(true); - await store.storage.writeTurns([userTurn("room A tool result")]); - await store.mirrorToSubstrate(); - - const roomADir = durableConversationAgentStateDir(repoDir, "default", "chan_a"); - const reconstructedA = await reconstructDurableConversation(roomADir, "default/chan_a"); - expect(reconstructedA?.turns).toHaveLength(1); - - expect(await store.bindOriginatingWorkbench("chan_b")).toBe(true); - expect(peekTurns(store)).toEqual([]); - - const mixedLegacy = path.join(repoDir, "agent-state", "default", "checkpoint.json"); - expect(fs.existsSync(mixedLegacy)).toBe(false); - - await store.storage.writeTurns([userTurn("room B first infer")]); - await store.mirrorToSubstrate(); - - expect(await store.bindOriginatingWorkbench("chan_a")).toBe(true); - expect(peekTurns(store)).toEqual([userTurn("room A tool result")]); - - expect(await store.bindOriginatingWorkbench("chan_b")).toBe(true); - expect(peekTurns(store)).toEqual([userTurn("room B first infer")]); - - expect(await store.bindOriginatingWorkbench("chan_b")).toBe(false); -}); - -const EMPTY_TOKEN_USAGE = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - thinking: 0, -}; - -test("a retired signal-kind pending op is dropped on restore, not thrown on", async () => { - const root = tmpDir("conv-retired-kind-"); - const agentStateDir = path.join(root, "agent-state", "default", "chan_a"); - await fs.promises.mkdir(agentStateDir, { recursive: true }); - - const survivingOp: PendingOperation = { - correlationId: "corr-approval", - kind: "approval", - registeredAt: 0, - gateId: "gate-approval", - }; - const retiredOp = { - correlationId: "corr-retired", - kind: "message_response", - registeredAt: 0, - gateId: "gate-retired", - }; - - await fs.promises.writeFile( - path.join(agentStateDir, "checkpoint.json"), - JSON.stringify({ - turns: [], - pendingOperations: [survivingOp, retiredOp], - tokenUsage: EMPTY_TOKEN_USAGE, - connectorState: null, - }), - ); - await fs.promises.writeFile( - path.join(agentStateDir, "checkpoint.meta.json"), - JSON.stringify({ - checkpointSeq: 0, - turnCount: 0, - pendingOperations: [survivingOp, retiredOp], - tokenUsage: EMPTY_TOKEN_USAGE, - connectorState: null, - }), - ); - - const reconstructed = await reconstructDurableConversation(agentStateDir, "default/chan_a"); - - expect(reconstructed?.pendingOperations).toEqual([survivingOp]); -}); - -test("two queued mails bind and mirror under each message's From not a later origin", async () => { - const root = tmpDir("conv-queued-"); - const { store } = await makeStore(root); - const registry = { - acquire: () => Promise.resolve(store), - get: () => store, - peek: () => store, - }; - const mailRequest = (from: string) => ({ - input: { - headers: { from, to: ["myra@alice.localhost"] }, - rawHeaders: {}, - parts: [], - }, - }); - // Both inbound mails exist (enqueued) before the first invokeStep. A - // latest-wins origin file would now name chan_b for both binds. - const fromA = originatingWorkbenchIdFromRequest(mailRequest("chan_a@alice.localhost")); - const fromB = originatingWorkbenchIdFromRequest(mailRequest("chan_b@alice.localhost")); - expect(fromA).toBe("chan_a"); - expect(fromB).toBe("chan_b"); - - await prepareConversationForOriginatingWorkbench({ - registry: registry as never, - agentKey: "default", - originatingWorkbenchId: fromA, - }); - await store.storage.writeTurns([userTurn("from A")]); - await store.mirrorToSubstrate(); - - await prepareConversationForOriginatingWorkbench({ - registry: registry as never, - agentKey: "default", - originatingWorkbenchId: fromB, - }); - expect(peekTurns(store)).toEqual([]); - await store.storage.writeTurns([userTurn("from B")]); - await store.mirrorToSubstrate(); - - await prepareConversationForOriginatingWorkbench({ - registry: registry as never, - agentKey: "default", - originatingWorkbenchId: fromA, - }); - expect(peekTurns(store)).toEqual([userTurn("from A")]); -}); - -test("prepareConversationForOriginatingWorkbench evicts the warm agent on room change only", async () => { - const evictions: string[] = []; - const store = { - bindOriginatingWorkbench: (id: string) => Promise.resolve(id === "chan_b"), - boundOriginatingWorkbenchId: () => null, - }; - const registry = { - acquire: () => Promise.resolve(store), - get: () => store, - peek: () => store, - }; - const warmCache = { - evictAll: (reason: string) => { - evictions.push(reason); - return Promise.resolve(); - }, - }; - - const swappedA = await prepareConversationForOriginatingWorkbench({ - registry: registry as never, - agentKey: "default", - originatingWorkbenchId: "chan_a", - warmCache, - }); - expect(swappedA).toBe(false); - expect(evictions).toEqual([]); - - const swappedB = await prepareConversationForOriginatingWorkbench({ - registry: registry as never, - agentKey: "default", - originatingWorkbenchId: "chan_b", - warmCache, - }); - expect(swappedB).toBe(true); - expect(evictions).toHaveLength(1); - expect(evictions[0]).toContain("chan_b"); -}); - -test("reactor startup waits for an in-flight connector metadata write", async () => { - const root = tmpDir("conv-startup-metadata-"); - const { store } = await makeStore(root); - await store.bindOriginatingWorkbench("chan_a"); - const metadataPath = path.join(root, "local", "metadata.json"); - const truncated = Promise.withResolvers(); - const resumeWrite = Promise.withResolvers(); - const writeFile = fs.promises.writeFile.bind(fs.promises); - let paused = false; - const writeSpy = spyOn(fs.promises, "writeFile").mockImplementation( - async (file, data, options) => { - if (file === metadataPath && !paused) { - paused = true; - await writeFile(file, "", options); - truncated.resolve(); - await resumeWrite.promise; - } - await writeFile(file, data, options); - }, - ); - try { - const seed = store.seedInbound({ - ref: { uid: 1, mailbox: "INBOX" }, - headers: { - from: "human@example.test", - to: ["agent@example.test"], - date: "2026-09-08T00:00:00.000Z", - messageId: "", - interchangeType: "conversation.message", - }, - flags: [], - content: "heartbeat", - signatureStatus: "valid", - }); - await truncated.promise; - const loaded = store.storage.load(); - const outcome = loaded.then( - () => "loaded", - () => "invalid metadata", - ); - await Bun.sleep(20); - resumeWrite.resolve(); - await seed; - expect(await outcome).toBe("loaded"); - await store.mirrorToSubstrate(); - } finally { - resumeWrite.resolve(); - writeSpy.mockRestore(); - } -}); diff --git a/apps/sidecar/src/conversation-state.ts b/apps/sidecar/src/conversation-state.ts index b064174cb9..164dfd6b1e 100644 --- a/apps/sidecar/src/conversation-state.ts +++ b/apps/sidecar/src/conversation-state.ts @@ -1,4 +1,5 @@ -// Durable conversation state for the warm single-step agent. +// Durable conversation state for the warm single-step agent (design §3c, +// §4 Phase D1). // // A long-lived single-step agent holds its multi-turn conversation in // the reactor's in-memory turn buffer, backed by a per-step isogit @@ -10,24 +11,21 @@ // This module makes the warm agent's conversation DURABLE in the // workflow-run substrate (the single-writer proxy `RepoStore`, written // through the supervisor). The durable copy lives under the workflow-run -// repo at a per-agent, per-originating-workbench path -// (`agent-state///...`), sibling to the per-run -// event log under `runs//...` and NOT confused with it. `agentKey` -// stays the warm stepId; `workbenchId` is the inbound mail From local-part -// so two rooms sharing one principal do not share turns. On a new run (and -// after a child respawn, once the warm agent is rebuilt lazily) the -// conversation is restored from the substrate into +// repo at a stable per-agent path (`agent-state//...`), sibling +// to the per-run event log under `runs//...` and NOT confused with +// it. On a new run (and after a child respawn, once the warm agent is +// rebuilt lazily) the conversation is restored from the substrate into // the agent's local store BEFORE the agent's reactor loads, so multi-turn // continuity holds across runs and across respawn. // -// Two-tier on-disk layout. The prior design wrote +// Two-tier on-disk layout (design §4, Phase D1). The prior design wrote // the WHOLE conversation as a single `conversation.json` blob on every // message. That re-serialized and re-hashed every prior turn per message, // so the per-message durable cost grew O(N) in the turn count -- O(N^2) -// over a conversation. It is replaced here with an append-only, -// bucket-sharded write-ahead log plus a periodic compacted checkpoint: +// over a conversation. D1 replaces it with an append-only, bucket-sharded +// write-ahead log plus a periodic compacted checkpoint: // -// agent-state/// +// agent-state// // checkpoint.json compacted full snapshot (turns + metadata) // checkpoint.meta.json { checkpointSeq: , turnCount, // tokenUsage, pendingOperations, @@ -53,7 +51,7 @@ // `checkpoint.json` (capturing the freshest metadata in checkpoint.meta) and // truncates the WAL, so between checkpoints the WAL holds at most K entries // and per-boundary durable cost is ~O(1) amortized. K and B are constants -// here, flagged as measurement-tunable. +// here; the design flags them as measurement-tunable. // // Restore = load `checkpoint.json` (folded turns + its metadata) then replay // the WAL tail in boundary-seq order, concatenating each boundary's turns @@ -63,28 +61,28 @@ // rebuilds the EXACT turn list + metadata the old whole-blob mirror would // have restored. // -// Substrate-merge constraint (load-bearing). -// `writeTreePreservingPrefix`'s `merge` callback receives only +// Substrate-merge constraint (load-bearing, design §4 "Substrate-merge +// note"). `writeTreePreservingPrefix`'s `merge` callback receives only // the DIRECT CHILDREN of `preservePrefix`, and the substrate's // `clearPrefix` step recursively removes the whole `preservePrefix` // subtree before writing the merge's returned set (paths outside the // prefix pass through untouched). A WAL blob is two levels below // `agent-state//`, so: // -// - WAL append uses `preservePrefix = agent-state///wal//`. +// - WAL append uses `preservePrefix = agent-state//wal//`. // The bucket's existing blobs ARE direct children, so the merge // pre-image is exactly that bucket and the append adds one entry -- -// no isogit side-read, and the checkpoint / other buckets / sibling -// rooms are untouched (outside the prefix). +// no isogit side-read, and the checkpoint / other buckets are +// untouched (outside the prefix). // - Checkpoint write + WAL truncate uses `preservePrefix = -// agent-state///`. The top-level checkpoint files are -// direct children; the merge returns ONLY those files and NO `wal/...` -// paths, so the recursive `clearPrefix` at the workbench dir drops -// that room's WAL subtree in the same atomic commit without touching -// sibling rooms. The truncate needs no nested read: omitting the WAL -// paths from the returned set IS the truncate. +// agent-state//`. The top-level checkpoint files are direct +// children; the merge returns ONLY those files and NO `wal/...` +// paths, so the recursive `clearPrefix` at `agent-state//` drops +// the entire WAL subtree in the same atomic commit. The truncate +// needs no nested read: omitting the WAL paths from the returned set +// IS the truncate. // -// Persistence sink (the riskiest part). The connector router's +// Persistence sink (the riskiest part, design §6). The connector router's // `snapshot()` / `restore()` surface and the harness's // `createWrappedStorageOverrides` are reused, but the persistence sink is // repointed from the agent's local isogit store to the workflow-run @@ -97,20 +95,20 @@ // single writer, and the preserve-prefix merge leaves every other subtree // byte-for-byte intact. // -// Timing. This is a STRUCTURE-only +// Timing (design §4 Phase D1, invariant 1). This is a STRUCTURE-only // change. The mirror is still `await`ed synchronously at the same run // boundary (`onRunBoundary` -> `mirrorToSubstrate`), so every turn is -// still durably committed before the next message is processed. The -// change alters WHAT the write does (O(1) append instead of O(N) -// whole-blob), not WHEN it happens. The run log is NOT yet a durable -// backstop for the turn (it carries a constant ref), so the +// still durably committed before the next message is processed. D1 +// changes WHAT the write does (O(1) append instead of O(N) whole-blob), +// not WHEN it happens. The run log is NOT yet a durable backstop for the +// turn (it carries a constant ref, design rev-2 FACT 1), so the // conversation copy here remains the sole durable copy of the agent's -// per-turn output -- which is exactly why the write stays synchronous. +// per-turn output -- which is exactly why D1 keeps the write synchronous. // The async flusher, run-log enrichment, and crash reconciliation are -// later, conditional work, not done here. +// later, conditional phases (D2-D4), not done here. // -// Commit granularity. The design calls for connector-state-change-driven -// commits via the router's +// Commit granularity (greybeard's pick, design §3c open question). The +// design calls for connector-state-change-driven commits via the router's // `onStateChanged` hook. The warm-agent path drives the connector router // through `seedInbound`: each mail-derived inbound message routes and // commits its thread state before the agent's send, so `onStateChanged` @@ -131,15 +129,11 @@ import path from "node:path"; import { type } from "arktype"; import { getLogger } from "@intx/log"; -import { reportError } from "@corbits/error-sink"; import { createConnectorRouter } from "@intx/harness"; import type { ConnectorReplyParts, RouteDecision } from "@intx/harness"; -import { createIsogitStorage, createNodeIsogitRuntime } from "@intx/storage-isogit/node"; +import { createIsogitStore } from "@intx/storage-isogit/node"; import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions/substrate"; import { WORKFLOW_RUN_AGENT_STATE_PREFIX } from "@intx/hub-sessions/substrate"; -import { SignalKind } from "@intx/types"; - -import { UNSCOPED_ORIGINATING_WORKBENCH_ID } from "./originating-workbench"; import { ConnectorThreadState, TokenUsage, @@ -153,50 +147,15 @@ import { const logger = getLogger(["sidecar", "workflow-child", "conversation-state"]); -const isogitStorage = createIsogitStorage(createNodeIsogitRuntime()); - const CHECKPOINT_FILE = "checkpoint.json"; const CHECKPOINT_META_FILE = "checkpoint.meta.json"; const WAL_DIR = "wal"; -const EMPTY_TOKEN_USAGE: TokenUsage = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - thinking: 0, -}; - -/** - * Substrate prefix for one agent's conversation in one originating - * workbench. Nested under `agent-state//` so two agents in the - * same room cannot collide, and two rooms of one agent cannot share turns. - */ -export function durableConversationAgentStatePrefix( - agentKey: string, - originatingWorkbenchId: string, -): string { - return `${WORKFLOW_RUN_AGENT_STATE_PREFIX}/${encodeURIComponent(agentKey)}/${encodeURIComponent(originatingWorkbenchId)}/`; -} - -export function durableConversationAgentStateDir( - repoDir: string, - agentKey: string, - originatingWorkbenchId: string, -): string { - return path.join( - repoDir, - WORKFLOW_RUN_AGENT_STATE_PREFIX, - encodeURIComponent(agentKey), - encodeURIComponent(originatingWorkbenchId), - ); -} - /** * Compaction interval: fold the WAL into a fresh checkpoint once it holds * this many turns since the last checkpoint. Bounds the WAL tail (and so - * the restore-replay length) between checkpoints. Measurement-tunable; - * fixed here at 64. + * the restore-replay length) between checkpoints. Measurement-tunable + * (design §6, open question 4); D1 fixes it at 64. */ const CHECKPOINT_INTERVAL = 64; @@ -204,7 +163,8 @@ const CHECKPOINT_INTERVAL = 64; * WAL directory fan-out bound: turn `seq` lives in bucket * `floor(seq / WAL_BUCKET_SIZE)`. Caps any single `wal//` tree at * this many entries so no commit re-hashes a tree that grows with the - * conversation length. Measurement-tunable; fixed here at 128. + * conversation length. Measurement-tunable (design §6, open question 4); + * D1 fixes it at 128. */ const WAL_BUCKET_SIZE = 128; @@ -226,8 +186,8 @@ const SnapshotMetadata = type({ /** * On-disk shape of the compacted checkpoint blob committed at - * `agent-state///checkpoint.json`. Carries the folded - * turn history (turns 0..checkpointSeq-1) plus the non-turn reactor metadata. + * `agent-state//checkpoint.json`. Carries the folded turn + * history (turns 0..checkpointSeq-1) plus the non-turn reactor metadata. * Validated on read because it crosses back into the program from the * substrate working tree -- a corrupt or partially-written checkpoint must * surface at the boundary, never be half-applied into the agent. @@ -240,9 +200,8 @@ const CheckpointSnapshot = type({ }); /** - * On-disk shape of - * `agent-state///checkpoint.meta.json`. The checkpoint - * pointer the restore path reads first to learn the boundary seq + * On-disk shape of `agent-state//checkpoint.meta.json`. The + * checkpoint pointer the restore path reads first to learn the boundary seq * the checkpoint folded to (`checkpointSeq`) -- and therefore which WAL * boundary seqs remain to replay -- plus the folded turn count * (`turnCount`, = `checkpoint.json`'s turn array length) and the freshest @@ -261,8 +220,8 @@ const CheckpointMeta = type({ /** * On-disk shape of one WAL entry blob at - * `agent-state///wal//.json`. One entry - * per MIRROR BOUNDARY (keyed by boundary `seq`, not turn index). Records the 0-or-more + * `agent-state//wal//.json`. One entry per MIRROR + * BOUNDARY (keyed by boundary `seq`, not turn index). Records the 0-or-more * new turns that boundary added (the O(1) append payload -- it never * carries prior turns) plus the latest non-turn metadata snapshot. The * append is UNCONDITIONAL: a turnless boundary still writes one entry with @@ -306,10 +265,9 @@ export interface DurableConversationStoreOpts { principal: Principal; /** * Stable per-agent key the snapshot is filed under - * (`agent-state///`). The warm single-step agent's - * stepId is the natural agentKey: it is stable across that agent's whole - * lifetime and disjoint from any runId. Originating workbench is bound - * separately so one warm agent can swap rooms without cloning. + * (`agent-state//`). The warm single-step agent's stepId is + * the natural key: it is stable across that agent's whole lifetime and + * disjoint from any runId. */ agentKey: string; } @@ -344,8 +302,7 @@ export interface DurableConversationStore { * Commit the local store's new turn(s) to the substrate as O(1) WAL * appends, folding into a fresh checkpoint when the WAL reaches the * compaction interval. Called synchronously at the run boundary (after - * the agent's send settles). A write failure surfaces. Flushes the - * currently bound originating-workbench prefix (the composite key). + * the agent's send settles). A write failure surfaces. */ mirrorToSubstrate(): Promise; /** @@ -380,25 +337,13 @@ export interface DurableConversationStore { * seeded thread. A metadata write failure surfaces. */ onReplySent(receipt: SendReceipt): Promise; - /** - * Point this store at `originatingWorkbenchId`'s nested substrate - * snapshot. Same room is a no-op. On a change: mirror the current room, - * retarget `agent-state///`, restore that - * snapshot. Missing snapshot starts empty -- the prior mixed - * `agent-state//` blob is never migrated. Returns true when - * the bound room changed (caller must rebuild the warm agent so - * `reactor.start()` loads the restored turns). - */ - bindOriginatingWorkbench(originatingWorkbenchId: string): Promise; - /** Room this store is bound to, or null before the first bind. */ - boundOriginatingWorkbenchId(): string | null; } export async function createDurableConversationStore( opts: DurableConversationStoreOpts, ): Promise { await fs.promises.mkdir(opts.localStoreDir, { recursive: true }); - const baseStorage = await isogitStorage.createIsogitStore(opts.localStoreDir, opts.signer); + const baseStorage = await createIsogitStore(opts.localStoreDir, opts.signer); // Reuse the connector router + the harness storage-override seam. The // router's `onStateChanged` is the change-driven commit hook the design @@ -415,24 +360,8 @@ export async function createDurableConversationStore( }, }); - // Nested substrate prefix is retargeted by bindOriginatingWorkbench. - // Unbound until the first inbound origin is known -- mirror/restore - // require a bound room so they cannot write the mixed agent-state// - // blob. - let originatingWorkbenchId: string | null = null; + const agentStatePrefix = `${WORKFLOW_RUN_AGENT_STATE_PREFIX}/${encodeURIComponent(opts.agentKey)}/`; - function requireBoundWorkbenchId(): string { - if (originatingWorkbenchId === null) { - throw new Error( - `sidecar conversation-state: originating workbench is not bound for ${opts.agentKey}; bindOriginatingWorkbench must run before restore or mirror`, - ); - } - return originatingWorkbenchId; - } - - function agentStatePrefix(): string { - return durableConversationAgentStatePrefix(opts.agentKey, requireBoundWorkbenchId()); - } // The number of mirror boundaries already durably committed (the // checkpoint's folded boundaries plus every appended WAL entry). It is // the seq of the NEXT WAL entry. `null` until learned -- lazily from the @@ -461,8 +390,7 @@ export async function createDurableConversationStore( // does not poison the chain, while each caller still observes its own op's // result (or rejection) through the returned promise. // - // Reactor startup reads and metadata writes join this same queue, so - // startup cannot read a connector metadata file mid-write. It does NOT + // This serializes mirror-vs-mirror and mirror-vs-restore only. It does NOT // address the reactor-vs-mirror peek-snapshot window documented on // `runMirror` below (nothing must append to the reactor's turn array // between its last writeTurns and the mirror's peekTurns) -- that is a @@ -498,11 +426,8 @@ export async function createDurableConversationStore( } function substrateAgentStateFsDir(): string { - return durableConversationAgentStateDir( - opts.substrate.getRepoDir(opts.workflowRunRepoId), - opts.agentKey, - requireBoundWorkbenchId(), - ); + const repoDir = opts.substrate.getRepoDir(opts.workflowRunRepoId); + return path.join(repoDir, WORKFLOW_RUN_AGENT_STATE_PREFIX, encodeURIComponent(opts.agentKey)); } function bucketOf(seq: number): number { @@ -510,7 +435,7 @@ export async function createDurableConversationStore( } function walBucketPrefix(bucket: number): string { - return `${agentStatePrefix()}${WAL_DIR}/${String(bucket)}/`; + return `${agentStatePrefix}${WAL_DIR}/${String(bucket)}/`; } function walEntryPath(seq: number): string { @@ -518,38 +443,25 @@ export async function createDurableConversationStore( } function checkpointPath(): string { - return `${agentStatePrefix()}${CHECKPOINT_FILE}`; + return `${agentStatePrefix}${CHECKPOINT_FILE}`; } function checkpointMetaPath(): string { - return `${agentStatePrefix()}${CHECKPOINT_META_FILE}`; - } - - async function applyEmptyLocalConversation(reason: string): Promise { - await baseStorage.writeTurns([]); - baseStorage.setConnectorState(null); - connectorRouter.restore(null); - await baseStorage.writeMetadata({ - pendingOperations: [], - tokenUsage: EMPTY_TOKEN_USAGE, - }); - await baseStorage.commit({ message: reason }); - mirroredBoundaryCount = 0; - mirroredTurnCount = 0; - checkpointBoundarySeq = 0; + return `${agentStatePrefix}${CHECKPOINT_META_FILE}`; } async function runRestore(): Promise { const reconstructed = await reconstructDurableConversation( substrateAgentStateFsDir(), - `${opts.agentKey}/${requireBoundWorkbenchId()}`, + opts.agentKey, ); if (reconstructed === null) { - // No snapshot for this room: start empty. Do not migrate a mixed - // agent-state// blob from before per-room nesting. - await applyEmptyLocalConversation( - `reset conversation for ${opts.agentKey} workbench ${requireBoundWorkbenchId()} (no prior snapshot)`, - ); + // No durable state yet: the next mirror starts the WAL from an empty + // checkpoint. Record the (empty) committed counts so the first + // mirror appends from boundary seq 0. + mirroredBoundaryCount = 0; + mirroredTurnCount = 0; + checkpointBoundarySeq = 0; return false; } // Write the reconstructed turns + metadata into the local store's @@ -622,12 +534,12 @@ export async function createDurableConversationStore( } /** - * Fold the full conversation into a fresh checkpoint and truncate the WAL in - * one atomic commit at `preservePrefix = agent-state///`. - * The merge returns ONLY the two checkpoint files and NO `wal/...` paths; - * because the substrate's `clearPrefix` recursively removes the whole room - * subtree before writing the returned set, omitting the WAL paths IS the - * truncate. + * Fold the full conversation into a fresh checkpoint and truncate the + * WAL in one atomic commit at `preservePrefix = agent-state//`. The + * merge returns ONLY the two checkpoint files and NO `wal/...` paths; + * because the substrate's `clearPrefix` recursively removes the whole + * `agent-state//` subtree before writing the returned set, omitting + * the WAL paths IS the truncate. */ async function writeCheckpoint( boundarySeq: number, @@ -661,7 +573,7 @@ export async function createDurableConversationStore( opts.workflowRunRepoId, opts.workflowRunRef, { - preservePrefix: agentStatePrefix(), + preservePrefix: agentStatePrefix, merge: async () => ({ [checkpointPath()]: JSON.stringify(snapshot), [checkpointMetaPath()]: JSON.stringify(meta), @@ -694,7 +606,7 @@ export async function createDurableConversationStore( if (mirroredBoundaryCount === null) { const reconstructed = await reconstructDurableConversation( substrateAgentStateFsDir(), - `${opts.agentKey}/${requireBoundWorkbenchId()}`, + opts.agentKey, ); checkpointBoundarySeq = reconstructed?.checkpointBoundarySeq ?? 0; mirroredBoundaryCount = reconstructed?.boundaryCount ?? 0; @@ -797,46 +709,13 @@ export async function createDurableConversationStore( }); } - function bindOriginatingWorkbench(nextWorkbenchId: string): Promise { - return serializeStateOp(async () => { - if (nextWorkbenchId.length === 0) { - throw new Error( - `sidecar conversation-state: originating workbench id must be non-empty for ${opts.agentKey}`, - ); - } - if (originatingWorkbenchId === nextWorkbenchId) return false; - if (originatingWorkbenchId !== null) { - await runMirror(); - } - originatingWorkbenchId = nextWorkbenchId; - await runRestore(); - return true; - }); - } - - const storageOverrides: Pick = { - load: (signal) => serializeStateOp(() => baseStorage.load(signal)), - writeMetadata: (metadata, signal) => - serializeStateOp(() => baseStorage.writeMetadata(metadata, signal)), - }; - const storage = new Proxy(baseStorage, { - get(target, prop) { - if (prop === "load") return storageOverrides.load; - if (prop === "writeMetadata") return storageOverrides.writeMetadata; - const value = Reflect.get(target, prop, target); - return typeof value === "function" ? value.bind(target) : value; - }, - }); - return { - storage, + storage: baseStorage, restoreFromSubstrate, mirrorToSubstrate, seedInbound, composeReply, onReplySent, - bindOriginatingWorkbench, - boundOriginatingWorkbenchId: () => originatingWorkbenchId, }; } @@ -856,23 +735,18 @@ export interface DurableConversationRegistryOpts { } /** - * Per-agent durable-conversation store registry. One store - * per warm agent key (stepId), built lazily and reused across runs in the - * same child. Originating-workbench snapshots nest under that key; bind - * retargets the live store before each send. The registry is empty + * Per-agent durable-conversation store registry (design §3c). One store + * per warm agent key, built lazily and reused across runs in the same + * child. The first `acquire` for a key builds the store and restores its + * prior conversation snapshot from the substrate -- the path that runs on + * the lazy first build AND on the respawn rebuild, so the warm agent + * resumes its conversation across child respawn. The registry is empty * after a respawn (it lives in the child's address space); the substrate * is the durable mirror that survives. */ export interface DurableConversationRegistry { acquire(key: string): Promise; get(key: string): DurableConversationStore; - /** - * The store for `key` if one has been acquired, else `undefined`. - * The body-turn mirror (CL-6448) runs in a `finally` that must not - * mask a build failure with `get`'s throw when the env builder never - * reached its acquire. - */ - peek(key: string): DurableConversationStore | undefined; } export function createDurableConversationRegistry( @@ -908,9 +782,14 @@ export function createDurableConversationRegistry( principal: opts.principal, agentKey: key, }); - // Restore happens in bindOriginatingWorkbench once the inbound - // From local-part is known. Restoring here would load a mixed - // agent-state// blob (or throw unbound). + // Restore the prior conversation BEFORE the store is observable (and + // before the warm agent's reactor `load()` reads it). On a genuine + // first-ever run this is a no-op (no checkpoint/WAL yet); on a + // respawn rebuild it pulls the pre-respawn conversation back from the + // substrate (checkpoint + WAL replay). A restore failure surfaces -- + // a lost conversation on respawn is a correctness failure, not a + // silently-fresh start. + await store.restoreFromSubstrate(); stores.set(key, store); building.delete(key); return store; @@ -922,10 +801,6 @@ export function createDurableConversationRegistry( return promise; } - function peek(key: string): DurableConversationStore | undefined { - return stores.get(key); - } - function get(key: string): DurableConversationStore { const store = stores.get(key); if (store === undefined) { @@ -936,31 +811,7 @@ export function createDurableConversationRegistry( return store; } - return { acquire, get, peek }; -} - -/** - * Bind the warm agent's live store to the originating workbench and, when - * the room changed, evict the cached agent so the next send rebuilds - * against the restored snapshot. The cache stays keyed by stepId -- one - * warm agent, not one per room. - */ -export async function prepareConversationForOriginatingWorkbench(args: { - registry: DurableConversationRegistry; - agentKey: string; - /** Room named by this request's mail; undefined keeps the current binding. */ - originatingWorkbenchId: string | undefined; - warmCache?: { evictAll: (reason: string) => Promise }; -}): Promise { - const store = await args.registry.acquire(args.agentKey); - const requested = args.originatingWorkbenchId; - const current = store.boundOriginatingWorkbenchId(); - const target = requested ?? current ?? UNSCOPED_ORIGINATING_WORKBENCH_ID; - const swapped = await store.bindOriginatingWorkbench(target); - if (swapped && args.warmCache !== undefined) { - await args.warmCache.evictAll(`originating workbench changed to ${target}`); - } - return swapped; + return { acquire, get }; } interface SnapshotMetadataValue { @@ -984,64 +835,10 @@ export interface ReconstructedConversation extends LoadedSnapshot { checkpointBoundarySeq: number; } -/** - * The reactor re-narrows turn/operation elements on load; the read helpers - * enforce only the structural envelope, matching the boundary the - * whole-blob mirror used. This is the single boundary cast for that - * element-level narrowing -- it cannot be replaced by a runtime check here - * because the element validators live in the reactor, not this module. - */ -function castValidatedEnvelope(items: unknown[]): T[] { - return items as T[]; -} - -/** - * A pending operation's `kind` envelope: just enough structure to read - * `kind` back off an otherwise-unvalidated `PendingOperation` (the element - * validator itself lives in the reactor, per `castValidatedEnvelope`'s - * doc comment above). Used only to detect a `kind` the running build's - * `SignalKind` no longer recognizes -- e.g. CL-7443 retired - * `message_response` -- before the reactor's `rehydrateGates` reaches it, - * since an unclassified kind there throws and blocks every future respawn. - */ -const PendingOperationKindEnvelope = type({ kind: "unknown" }); - -/** - * Drop any restored pending operation whose `kind` is not a `SignalKind` - * this build still classifies, reporting each drop through `reportError` - * rather than letting the reactor's `rehydrateGates` throw on it. A retired - * signal kind (CL-7443's `message_response`) can still be sitting in a warm - * agent's on-disk checkpoint/WAL from before the retirement; without this - * filter, every respawn of that agent would throw - * "Unclassified signal kind" and never start. The person's in-flight - * question is simply lost here -- their eventual reply just becomes the - * next ordinary turn (docs/CHAT.md). - */ -function dropUnclassifiedPendingOperations(items: unknown[], agentKey: string): unknown[] { - return items.filter((item) => { - const envelope = PendingOperationKindEnvelope(item); - const kind = envelope instanceof type.errors ? undefined : envelope.kind; - if (SignalKind(kind) instanceof type.errors) { - reportError( - new Error( - `pending operation with unclassified signal kind ${JSON.stringify(kind)} dropped on restore for agent ${agentKey}`, - ), - { - operation: "pending_op_dropped_unknown_kind", - agentId: agentKey, - }, - ); - return false; - } - return true; - }); -} - /** * Reconstruct the warm agent's conversation from the two-tier on-disk - * layout under `agentStateDir` - * (`/agent-state///`): the compacted - * `checkpoint.json` turns followed by the replayed WAL tail. + * layout under `agentStateDir` (`/agent-state//`): the + * compacted `checkpoint.json` turns followed by the replayed WAL tail. * Pure read against the substrate working tree -- no inference, no commit. * Returns `null` when neither a checkpoint nor any WAL exists (the genuine * first-ever run). The latest metadata source wins (the last replayed WAL @@ -1086,10 +883,13 @@ export async function reconstructDurableConversation( metadata = entry.metadata; } return { - turns: castValidatedEnvelope(turns), - pendingOperations: castValidatedEnvelope( - dropUnclassifiedPendingOperations(metadata.pendingOperations, agentKey), - ), + // The reactor re-narrows turn/operation elements on load; the + // validators below enforce only the structural envelope, matching the + // boundary the whole-blob mirror used. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- envelope validated in the read helpers; turn element narrows live in the reactor on load + turns: turns as ConversationTurn[], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- envelope validated in the read helpers; pending-operation element narrows live in the reactor on load + pendingOperations: metadata.pendingOperations as PendingOperation[], tokenUsage: metadata.tokenUsage, connectorState: metadata.connectorState, totalTurns: turns.length, @@ -1230,5 +1030,7 @@ function parseJsonOrThrow(raw: string, label: string): unknown { } export function isErrnoNotFound(cause: unknown): boolean { - return typeof cause === "object" && cause !== null && "code" in cause && cause.code === "ENOENT"; + if (cause === null || typeof cause !== "object") return false; + const code = (cause as { code?: unknown }).code; + return code === "ENOENT"; } diff --git a/apps/sidecar/src/hibernated-agent-identity-vault.test.ts b/apps/sidecar/src/hibernated-agent-identity-vault.test.ts deleted file mode 100644 index ea67f39090..0000000000 --- a/apps/sidecar/src/hibernated-agent-identity-vault.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -// Exercises `hibernated-agent-identity-vault.ts` against the real, -// unmodified `@intx/hub-agent` key/repo stores -- the same technique -// `apps/sidecar/test/suspend-key-preservation.poc.test.ts` (PR #291) -// proved works, now against the actual snapshot/restore/reap module this -// repo ships. -import { describe, test, expect, mock } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { createAgentKeyStore, createAgentRepoStore, agentDir } from "@intx/hub-agent"; -import { generateKeyPair, signEd25519, verifySSHSignature } from "@intx/crypto"; - -import { - snapshotAgentIdentity, - restoreAgentIdentity, - reapExpiredHibernationSnapshots, -} from "./hibernated-agent-identity-vault"; - -const cryptoOps = { - generateKeyPair, - signEd25519, - verifySSHSig: verifySSHSignature, -}; - -async function makeTmpDataDir(): Promise { - return fs.mkdtemp(path.join(os.tmpdir(), "hibernated-agent-vault-")); -} - -async function modeOf(target: string): Promise { - const stat = await fs.stat(target); - return stat.mode & 0o777; -} - -describe("hibernated-agent-identity-vault", () => { - test("snapshot then restore preserves the real agent keypair with isNew: false", async () => { - const dataDir = await makeTmpDataDir(); - const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); - const repoStore = createAgentRepoStore({ dataDir }); - const address = "run_hibernate-vault@example.com"; - - const { keyPair: original } = await keyStore.loadOrGenerateKey(address); - - const snapshot = await snapshotAgentIdentity(dataDir, address); - expect(snapshot.snapshotted).toBe(true); - - // The real, unmodified destructive call `hub-link.js` makes on every - // undeploy -- exercised here exactly as the PR #291 spike exercised it. - await repoStore.remove(address); - expect(await fs.readdir(dataDir)).not.toContain(path.basename(agentDir(dataDir, address))); - - const restore = await restoreAgentIdentity(dataDir, address); - expect(restore.restored).toBe(true); - - const { keyPair: restored, isNew } = await keyStore.loadOrGenerateKey(address); - expect(isNew).toBe(false); - expect(restored.privateKey).toEqual(original.privateKey); - expect(restored.publicKey).toEqual(original.publicKey); - }); - - test("no snapshot means the destructive remove wins -- a fresh keypair comes back", async () => { - const dataDir = await makeTmpDataDir(); - const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); - const repoStore = createAgentRepoStore({ dataDir }); - const address = "run_no-snapshot@example.com"; - - const { keyPair: original } = await keyStore.loadOrGenerateKey(address); - - // No snapshotAgentIdentity call: models the reclaimDirs: true (real - // undeploy) path, which never protects the identity directory. - await repoStore.remove(address); - - const restore = await restoreAgentIdentity(dataDir, address); - expect(restore.restored).toBe(false); - - const { keyPair: regenerated, isNew } = await keyStore.loadOrGenerateKey(address); - expect(isNew).toBe(true); - expect(regenerated.privateKey).not.toEqual(original.privateKey); - }); - - test("the vault snapshot is hardened to owner-only permissions", async () => { - const dataDir = await makeTmpDataDir(); - const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); - const address = "run_permissions@example.com"; - await keyStore.loadOrGenerateKey(address); - - await snapshotAgentIdentity(dataDir, address); - - const vaultEntry = path.join( - dataDir, - "hibernated-agent-identity", - path.basename(agentDir(dataDir, address)), - ); - expect(await modeOf(vaultEntry)).toBe(0o700); - const keysDir = path.join(vaultEntry, "keys"); - expect(await modeOf(keysDir)).toBe(0o700); - const privateKeyFile = path.join(keysDir, "id_ed25519"); - expect(await modeOf(privateKeyFile)).toBe(0o600); - }); - - test("a snapshot taken after the identity directory is already gone reports loudly and returns snapshotted: false", async () => { - const dataDir = await makeTmpDataDir(); - const address = "run_already-gone@example.com"; - - const reportErrorMock = mock((_error: unknown, _context: unknown) => "ref-test"); - mock.module("@corbits/error-sink", () => ({ - reportError: reportErrorMock, - })); - const { snapshotAgentIdentity: freshSnapshot } = - await import("./hibernated-agent-identity-vault"); - - const result = await freshSnapshot(dataDir, address); - - expect(result.snapshotted).toBe(false); - expect(reportErrorMock).toHaveBeenCalledTimes(1); - const context = reportErrorMock.mock.calls[0]?.[1]; - expect(context).toMatchObject({ - operation: "hibernated-agent-identity-vault.snapshot", - agentId: address, - }); - - mock.restore(); - }); - - test("reapExpiredHibernationSnapshots deletes only entries past retention", async () => { - const dataDir = await makeTmpDataDir(); - const keyStoreOld = createAgentKeyStore({ dataDir, ...cryptoOps }); - const keyStoreFresh = createAgentKeyStore({ dataDir, ...cryptoOps }); - const oldAddress = "run_old-orphan@example.com"; - const freshAddress = "run_fresh-hibernate@example.com"; - const retentionMs = 1_000; - - await keyStoreOld.loadOrGenerateKey(oldAddress); - await snapshotAgentIdentity(dataDir, oldAddress); - // Backdate the orphan's marker past retention; a real orphan is one - // nothing has redeployed (and thus restored) since it hibernated. - const oldVaultEntry = path.join( - dataDir, - "hibernated-agent-identity", - path.basename(agentDir(dataDir, oldAddress)), - ); - await fs.writeFile( - path.join(oldVaultEntry, ".snapshotted-at"), - new Date(Date.now() - retentionMs - 1).toISOString(), - { mode: 0o600 }, - ); - - await keyStoreFresh.loadOrGenerateKey(freshAddress); - await snapshotAgentIdentity(dataDir, freshAddress); - - const result = await reapExpiredHibernationSnapshots(dataDir, { - retentionMs, - }); - - expect(result.reapedEntries).toEqual([path.basename(agentDir(dataDir, oldAddress))]); - await expect(fs.stat(oldVaultEntry)).rejects.toThrow(); - const freshVaultEntry = path.join( - dataDir, - "hibernated-agent-identity", - path.basename(agentDir(dataDir, freshAddress)), - ); - await expect(fs.stat(freshVaultEntry)).resolves.toBeDefined(); - }); - - test("reapExpiredHibernationSnapshots on a vault-less data dir is a no-op", async () => { - const dataDir = await makeTmpDataDir(); - const result = await reapExpiredHibernationSnapshots(dataDir); - expect(result.reapedEntries).toEqual([]); - }); -}); diff --git a/apps/sidecar/src/hibernated-agent-identity-vault.ts b/apps/sidecar/src/hibernated-agent-identity-vault.ts deleted file mode 100644 index 05ac0d92dc..0000000000 --- a/apps/sidecar/src/hibernated-agent-identity-vault.ts +++ /dev/null @@ -1,214 +0,0 @@ -// Snapshot/restore for an agent's on-disk identity directory across a -// state-preserving "hibernate" teardown (CL-6581, compensating for -// CL-6239's still-open upstream ask). -// -// WHY THIS EXISTS: the published `@intx/hub-agent` package's -// `handleAgentUndeploy` (`ws/hub-link.js`) unconditionally destroys -// `agentDir(dataDir, address)` -- which holds the agent's -// reconnect-challenge Ed25519 keypair under its `keys/` subdirectory -- -// on EVERY `sendAgentUndeploy`, hibernate or not, and that call happens -// AFTER this sidecar's `undeploy` hook returns. Losing that keypair is the -// documented root cause of CL-6203/CL-6044: a woken agent mints a fresh -// identity, fails the hub's reconnect challenge, and the conversation goes -// silent. `agentDir` is `@intx/hub-agent`'s only stable public export for -// this path (its `keys/` subdirectory name is a documented internal -// implementation detail, not exported), so this module snapshots the whole -// directory before the destructive delete and restores it before the next -// deploy -- a workaround that depends on that call-ordering, not a fix. -// CL-6239 stays open for the real one: a non-destructive upstream undeploy. -import type { Dirent } from "node:fs"; -import fsp from "node:fs/promises"; -import path from "node:path"; - -import { agentDir } from "@intx/hub-agent"; -import { reportError } from "@corbits/error-sink"; - -import { isErrnoNotFound } from "./conversation-state"; - -const VAULT_DIRNAME = "hibernated-agent-identity"; -const MARKER_FILENAME = ".snapshotted-at"; - -/** - * How long an untouched snapshot may sit in the vault before - * `reapExpiredHibernationSnapshots` reclaims it. Generous relative to any - * expected idle-sleep duration (hours to low days): an agent that is - * legitimately woken always redeploys and restores its snapshot well - * inside this window. An agent hibernated and then never redeployed - * (channel deleted, member removed, workspace archived, ...) instead - * leaves an orphaned snapshot on disk -- this bounds that orphan's - * lifetime rather than letting it accumulate forever, per the same class - * of leak that left 62 dead deployment records unreaped. - */ -export const HIBERNATION_SNAPSHOT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; - -function vaultRoot(dataDir: string): string { - return path.join(dataDir, VAULT_DIRNAME); -} - -function vaultEntryDir(dataDir: string, agentAddress: string): string { - return path.join(vaultRoot(dataDir), path.basename(agentDir(dataDir, agentAddress))); -} - -async function pathExists(target: string): Promise { - try { - await fsp.access(target); - return true; - } catch (cause) { - if (isErrnoNotFound(cause)) return false; - throw cause; - } -} - -/** - * Recursively chmods every file 0600 and every directory 0700, key - * material's minimum-privilege mode -- matching the precedent in - * `workflow-deployment-record.ts`'s `writeWorkflowDeploymentRecord`. Run - * after a plain `fsp.cp`, which does not otherwise guarantee the copy's - * permission bits regardless of the source's. - */ -async function hardenPermissionsRecursive(root: string): Promise { - const entries = await fsp.readdir(root, { withFileTypes: true }); - for (const entry of entries) { - const entryPath = path.join(root, entry.name); - if (entry.isDirectory()) { - await hardenPermissionsRecursive(entryPath); - await fsp.chmod(entryPath, 0o700); - } else { - await fsp.chmod(entryPath, 0o600); - } - } -} - -export type SnapshotAgentIdentityResult = { snapshotted: boolean }; - -/** - * Copy `agentDir(dataDir, agentAddress)` into this sidecar's own vault - * directory, hardened to owner-only permissions, before the published - * package's destructive undeploy delete reaches it. Must be called and - * awaited from inside the `undeploy` hook, before it returns -- the - * published `handleAgentUndeploy`'s delete runs only after that hook - * settles, and that ordering is this technique's entire foundation. - * - * A missing source directory at this point means that ordering has - * already broken (a future `@intx/hub-agent` release deletes before - * calling this hook), so every wake from here on would silently mint a - * fresh identity -- the exact CL-6203 bug class. That is reported loudly - * through `reportError` rather than left to fail silent; the teardown - * itself still proceeds either way. - */ -export async function snapshotAgentIdentity( - dataDir: string, - agentAddress: string, -): Promise { - const source = agentDir(dataDir, agentAddress); - const dest = vaultEntryDir(dataDir, agentAddress); - - if (!(await pathExists(source))) { - reportError( - new Error( - "hibernate snapshot found no agent identity directory to preserve; " + - "@intx/hub-agent's undeploy call ordering may have changed, which " + - "would make every subsequent wake for this address mint a fresh " + - "identity and fail the hub's reconnect challenge", - ), - { - operation: "hibernated-agent-identity-vault.snapshot", - agentId: agentAddress, - }, - ); - return { snapshotted: false }; - } - - await fsp.rm(dest, { recursive: true, force: true }); - await fsp.mkdir(vaultRoot(dataDir), { recursive: true, mode: 0o700 }); - await fsp.cp(source, dest, { recursive: true }); - await hardenPermissionsRecursive(dest); - await fsp.chmod(dest, 0o700); - await fsp.writeFile(path.join(dest, MARKER_FILENAME), new Date().toISOString(), { - mode: 0o600, - }); - return { snapshotted: true }; -} - -export type RestoreAgentIdentityResult = { restored: boolean }; - -/** - * Restore a previously vaulted identity directory back to - * `agentDir(dataDir, agentAddress)`, if one exists, and remove it from the - * vault. A no-op returning `{ restored: false }` for an address that was - * never snapshotted (an ordinary fresh deploy) -- the caller's own - * `loadOrGenerateKey` call then mints a fresh identity exactly as it - * would with no vault involved. Must run BEFORE that `loadOrGenerateKey` - * call so it observes the restored files rather than minting a fresh key - * over an empty directory. - */ -export async function restoreAgentIdentity( - dataDir: string, - agentAddress: string, -): Promise { - const source = vaultEntryDir(dataDir, agentAddress); - if (!(await pathExists(source))) { - return { restored: false }; - } - - const dest = agentDir(dataDir, agentAddress); - await fsp.rm(path.join(source, MARKER_FILENAME), { force: true }); - await fsp.rm(dest, { recursive: true, force: true }); - await fsp.cp(source, dest, { recursive: true }); - await fsp.rm(source, { recursive: true, force: true }); - return { restored: true }; -} - -export type ReapExpiredHibernationSnapshotsResult = { - /** Opaque sanitized-address directory names reaped, for observability -- never the raw address. */ - reapedEntries: string[]; -}; - -/** - * Sweep the vault for entries older than `retentionMs` - * (`HIBERNATION_SNAPSHOT_RETENTION_MS` by default) and delete them. An - * entry with no marker (an interrupted snapshot write) is treated as - * immediately expired rather than kept forever with no way to age it. - * Intended to run once at boot, independent of and before/after the - * unrelated deployment-record boot-restore scan -- this function does not - * touch that scan or its concurrency. - */ -export async function reapExpiredHibernationSnapshots( - dataDir: string, - opts: { retentionMs?: number; nowMs?: number } = {}, -): Promise { - const retentionMs = opts.retentionMs ?? HIBERNATION_SNAPSHOT_RETENTION_MS; - const nowMs = opts.nowMs ?? Date.now(); - const root = vaultRoot(dataDir); - - let entries: Dirent[]; - try { - entries = await fsp.readdir(root, { withFileTypes: true }); - } catch (cause) { - if (isErrnoNotFound(cause)) return { reapedEntries: [] }; - throw cause; - } - - const reapedEntries: string[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const entryPath = path.join(root, entry.name); - const markerPath = path.join(entryPath, MARKER_FILENAME); - - let snapshottedAtIso: string | undefined; - try { - snapshottedAtIso = await fsp.readFile(markerPath, "utf8"); - } catch (cause) { - if (!isErrnoNotFound(cause)) throw cause; - } - - const snapshottedAtMs = snapshottedAtIso === undefined ? NaN : Date.parse(snapshottedAtIso); - const expired = Number.isNaN(snapshottedAtMs) ? true : nowMs - snapshottedAtMs >= retentionMs; - - if (expired) { - await fsp.rm(entryPath, { recursive: true, force: true }); - reapedEntries.push(entry.name); - } - } - return { reapedEntries }; -} diff --git a/apps/sidecar/src/hub-link-watchdog.ts b/apps/sidecar/src/hub-link-watchdog.ts deleted file mode 100644 index a484fe3f5e..0000000000 --- a/apps/sidecar/src/hub-link-watchdog.ts +++ /dev/null @@ -1,61 +0,0 @@ -// A hub connect attempt that lands while the hub is being replaced can -// dangle forever: no open, no close, so the link never re-schedules a -// reconnect and the sidecar sits disconnected. The watchdog arms a stall -// deadline around every connect attempt and fires `onStall` when neither -// outcome arrives in time; the host exits non-zero so the platform's -// restart policy re-runs the idempotent boot path. - -import type { ReconnectScheduler } from "@intx/hub-agent"; - -export interface HubLinkWatchdog { - /** - * Wrap around the link's reconnect scheduling so every scheduled - * attempt re-arms the stall deadline just before it runs. - */ - scheduleReconnect: ReconnectScheduler; - /** - * Arm the deadline for the first connect attempt, which bypasses the - * reconnect scheduler. - */ - armForBoot(): void; - /** - * Disarm the deadline. Call from any signal that proves the link made - * progress (an open handler firing, a close that re-schedules). - */ - markAlive(): void; -} - -export type CreateHubLinkWatchdogOpts = { - stallDeadlineMs: number; - onStall: () => void; -}; - -export function createHubLinkWatchdog(opts: CreateHubLinkWatchdogOpts): HubLinkWatchdog { - let deadline: ReturnType | undefined; - - function arm(): void { - disarm(); - deadline = setTimeout(opts.onStall, opts.stallDeadlineMs); - } - - function disarm(): void { - if (deadline !== undefined) { - clearTimeout(deadline); - deadline = undefined; - } - } - - return { - armForBoot: arm, - markAlive: disarm, - scheduleReconnect: (callback, delayMs) => { - const handle = setTimeout(() => { - arm(); - callback(); - }, delayMs); - return () => { - clearTimeout(handle); - }; - }, - }; -} diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index ba1baa0f8e..8ba4b9cd35 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -1,128 +1,351 @@ -// Composition root for the sidecar host. The host is deliberately -// generic: it reads its identity and hub location from the environment, -// dials in to the hub over a WebSocket, and runs whatever it is given -// through the deploy-router seam -- workflow deployments arrive as -// `agent.deploy` frames and are executed in supervised workflow-process -// children. It knows no deployment by name and holds no policy about -// what runs on it; the hub tells it everything over the wire. Because -// the sidecar dials the hub (never the reverse), hosted and local -// topologies are identical, and any number of sidecars can dial the -// same hub without this file changing. -// -// Boot order matters: orphaned tarball-cache staging is swept before -// any apply work is accepted, the process identity keypair loads before -// anything touches the data dir's substrate, the deploy router is -// captured during orchestrator construction, restored deployments are -// re-established BEFORE the hub connection opens (mailbox registrations -// must be live before the hub routes to them), the watchdog is armed -// for the first connect attempt, and only then does the link dial out. - +import { appendFileSync } from "node:fs"; import path from "node:path"; -import { createEd25519Crypto, generateKeyPair, verifySSHSignature } from "@intx/crypto"; +import { setup } from "@intx/log"; +import { createInMemoryTransport } from "@intx/mail-memory"; +import { + createEd25519Crypto, + createEnvKeyCredentialCipher, + generateKeyPair, + verifySSHSignature, +} from "@intx/crypto"; import { - createSidecarOrchestrator, createSenderKeyCache, createSenderCryptoResolver, createInboundMailPolicyRegistry, createInboundMailPolicyLookup, + createSidecarOrchestrator, type HubLink, } from "@intx/hub-agent"; +import { hexDecode, hexEncode } from "@intx/types"; import { createAgentRepoStore } from "@intx/hub-sessions"; -import { loadAdapterRegistry } from "@intx/inference/providers"; -import { getLogger, setup } from "@intx/log"; -import { createInMemoryTransport } from "@intx/mail-memory"; import { createTarballCache } from "@intx/tool-packaging"; -import { hexDecode, hexEncode } from "@intx/types"; -import { reportError } from "@corbits/error-sink"; -import { removeFileAtomicDurable, writeFileAtomicDurable } from "./atomic-write"; -import { readSidecarConfig } from "./config"; -import { DEFAULT_TOOL_REGISTRIES_JSON, parseToolRegistries } from "./tool-materialization"; -import { createWorkflowProbeExecutor } from "./workflow-probe-handler"; -import { createWorkflowClosureMaterializer } from "./workflow-closure-materialization"; -import { MAX_INLINE_ASSET_PAYLOAD_BYTES } from "./source-asset-delivery"; +import { loadAdapterRegistry } from "@intx/inference/providers"; + +import { + readAdapterManifest, + readCacheMaxBytes, + readCredentialEncryptionKey, + readRegistryMaxTarballBytes, +} from "./config"; import { createDefaultHarnessBuilder } from "./default-harness"; -import { createHubLinkWatchdog } from "./hub-link-watchdog"; -import { createOAuthLoopbackLoginService } from "./oauth-login"; -import { attachShutdownRejectionHandler, runSidecarShutdown } from "./shutdown"; -import { loadOrMintSidecarKeypair } from "./signing-keypair"; +// `createSidecarDeployRouter` is the production routing the +// orchestrator hands to the link's `agent.deploy` handler; every +// inbound frame stages through the workflow-run substrate, spawning a +// supervised workflow-process child for a workflow deploy. +import type { DispatchTimingMark } from "@intx/workflow-host"; + import { createSidecarDeployRouter, type SidecarDeployRouter } from "./workflow-host-wiring"; import { createDeploymentAddressRegistry, - createMultistepCredentialsRouter, createMultistepDrainRouter, createMultistepGrantsRouter, createMultistepMailRouter, createMultistepSignalRouter, createMultistepSourcesRouter, + createMultistepCredentialsRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, } from "./workflow-run-pack-client"; import { createWorkflowRunPackRestorer } from "./workflow-run-pack-restore"; -import { createBootRestorePushHold } from "./boot-restore-push-hold"; +import { readRegistries } from "./sidecar-materialization-config"; +import { createWorkflowClosureMaterializer } from "./workflow-closure-materialization"; +import { MAX_INLINE_ASSET_PAYLOAD_BYTES } from "./source-asset-delivery"; +import { createWorkflowProbeExecutor } from "./workflow-probe-handler"; +import { loadOrMintSidecarKeypair } from "./signing-keypair"; +import { removeFileAtomicDurable, writeFileAtomicDurable } from "./atomic-write"; await setup(); -const config = readSidecarConfig(process.env); - -// Host policy constants, not configuration: the tarball cache lives -// inside the data dir, and the two byte caps bound per-step tool -// materialization. They are data the host owns, so they are pinned here -// rather than surfaced as environment knobs. -const CACHE_ROOT = path.join(config.dataDir, "cache", "tarballs"); -const CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; -const REGISTRY_MAX_TARBALL_BYTES = 10 * 1024 * 1024; - -// Built-in adapters merged with any operator-configured custom adapters -// named in `SIDECAR_ADAPTER_MANIFEST`. Installing a custom adapter still -// means installing its package into this workspace -- the manifest only -// names an already-installed module's specifier and export, it never -// carries code of its own. -const adapters = await loadAdapterRegistry(config.adapterManifest); - -// Sweep any tmp staging directories left behind by a tarball put or -// extract that crashed between staging and the final rename on a -// previous boot, before the orchestrator starts accepting apply work. +function requireEnv(name: string): string { + const value = process.env[name]; + if (value === undefined) { + throw new Error(`${name} environment variable is required`); + } + return value; +} + +const dataDir = requireEnv("SIDECAR_DATA_DIR"); + +// The sidecar seals its at-rest credential material (run-record apiKeys, and +// the tool credential store) under this operator key. Constructed at the boot +// edge so a missing or malformed key fails loudly here rather than when the +// first record is persisted. +const credentialCipher = createEnvKeyCredentialCipher(readCredentialEncryptionKey()); + +// Resolve cache configuration at the boot edge so the workflow-child's +// per-apply loader receives a concrete path and cap through its spawn +// env rather than re-reading env at non-boundary call sites. +const sidecarCacheDir = process.env["SIDECAR_CACHE_DIR"]; +const cacheRoot = + sidecarCacheDir !== undefined && sidecarCacheDir.trim() !== "" + ? sidecarCacheDir + : path.join(dataDir, "cache", "tarballs"); +const cacheMaxBytes = readCacheMaxBytes(); +const registryMaxTarballBytes = readRegistryMaxTarballBytes(); + +// Operator-configured custom inference adapters, resolved once at the +// boot edge. `loadAdapterRegistry` merges the statically-linked +// built-ins with any custom adapters the manifest names, importing each +// custom module eagerly here so a bad specifier fails the sidecar at +// boot rather than at first inference. The SAME registry backs the +// deploy router's source-admission check below; the workflow child +// cannot receive this object across the fork, so the validated +// manifest is serialized into the child's spawn env (see +// `multistepSubstrateEnv`) and the child rebuilds an equivalent +// registry from it. Specifiers are operator-config-only — the agent +// deploy tree never contributes one. +const adapterManifest = readAdapterManifest(); +const adapters = await loadAdapterRegistry(adapterManifest); + +// Phase 4.7 latency-gate hook. When `SIDECAR_LATENCY_BENCH_FILE` names a +// path, the deploy router wires the supervisor's `onDispatchTiming` +// observer to append a parseable line per per-message dispatch boundary +// to that file, which the benchmark harness reads after the run. A file +// (rather than stdout) is the channel because the spawn fixture caps its +// stdout drain buffer, and a few-hundred-message run emits more lines +// than that cap holds. Resolved here at the boot edge (the one layer +// that reads env) so no non-boundary site re-decides what the absent +// value means; an unset path leaves the observer unwired and the +// supervisor's dispatch path untouched. This is observability-only -- no +// control-flow effect -- and is the single benchmark-side instrumentation +// hook the 4.7 gate adds. The synchronous append (not the logger) is the +// raw measurement channel: it is not application logging and must not be +// formatted or level-gated. A failed append surfaces (it is not +// swallowed) so a broken benchmark channel does not silently yield an +// empty result set. +// Two line shapes share the channel, discriminated by `mark.kind`. Both +// lead with the per-message id (the mail's Message-ID), the key the +// per-message fits group on: +// roundtrip: ` ` (4.7 latency gate) +// leg: ` leg [runsFanOut consumedFanOut looseObjects gitBytes]` +// (D2 per-leg attribution; the trailing four counters are +// present only on the `end` phase). The leg shape is a +// strict superset prefixed with the literal `leg` token, so +// a reader can split on whitespace and branch on field 2. +const latencyBenchFile = process.env["SIDECAR_LATENCY_BENCH_FILE"]; +const onDispatchTiming: ((mark: DispatchTimingMark) => void) | undefined = + latencyBenchFile !== undefined && latencyBenchFile.trim() !== "" + ? (mark) => { + let line: string; + if (mark.kind === "roundtrip") { + line = `${mark.messageId} ${mark.marker} ${mark.atMs.toFixed(3)}\n`; + } else { + const counters = + mark.counters !== undefined + ? ` ${String(mark.counters.runsFanOut)} ${String(mark.counters.consumedFanOut)} ${String(mark.counters.looseObjects)} ${String(mark.counters.gitBytes)}` + : ""; + line = `${mark.messageId} leg ${mark.leg} ${mark.phase} ${mark.atMs.toFixed(3)}${counters}\n`; + } + appendFileSync(latencyBenchFile, line); + } + : undefined; + +// D2 §10c forced-repack A/B toggle. When `SIDECAR_REPACK_EVERY_MESSAGES` +// names a positive integer, the supervisor forces a `git gc`/repack of +// the workflow-run repo every Nth dispatched message (under the +// single-writer discipline). Resolved here at the boot edge alongside the +// timing channel; absent or non-positive => the supervisor forks no +// `git gc` and the dispatch path is untouched. Measurement-only: this +// exists to discriminate pack-growth from tree-fan-out, never to run in +// production. +const repackEveryRaw = process.env["SIDECAR_REPACK_EVERY_MESSAGES"]; +let repackEveryMessages: { everyMessages: number } | undefined; +if (repackEveryRaw !== undefined && repackEveryRaw.trim() !== "") { + const parsed = Number.parseInt(repackEveryRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `SIDECAR_REPACK_EVERY_MESSAGES must be a positive integer, got ${repackEveryRaw}`, + ); + } + repackEveryMessages = { everyMessages: parsed }; +} + +// Consumed-dedup retention horizon (ms). Resolved here at the boot edge +// -- the single layer that owns operator config -- and threaded into +// every supervisor the deploy router constructs. Absent or empty => +// the supervisor applies its 24h default. This is the OPERATOR-policy +// value: the longest window in which the same message could +// legitimately be re-submitted and still must be caught as a duplicate +// (the consumed/ dedup index retains at least this long before the +// retention watermark prunes it). It must be >= the maximum redelivery +// window of any at-least-once mail source if one is ever added. +const consumedRetentionRaw = process.env["CONSUMED_RETENTION_MS"]; +let consumedRetentionMs: number | undefined; +if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { + const parsed = Number.parseInt(consumedRetentionRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `CONSUMED_RETENTION_MS must be a positive integer (milliseconds), got ${consumedRetentionRaw}`, + ); + } + consumedRetentionMs = parsed; +} + +// Bound on the child's spawn-time `ready` handshake. Threaded to every +// per-deployment supervisor; on expiry the supervisor kills the child and +// rejects the spawn, so a child that spawns but never signals ready fails +// the deploy (or is skipped by boot-time restore) instead of hanging it. +// Absent, the supervisor applies its 30s default. +const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; +let readyTimeoutMs: number | undefined; +if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { + const parsed = Number.parseInt(readyTimeoutRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `CHILD_READY_TIMEOUT_MS must be a positive integer (milliseconds), got ${readyTimeoutRaw}`, + ); + } + readyTimeoutMs = parsed; +} + +// Hub-link reconnect backoff (ms). Resolved here at the boot edge -- the +// single layer that owns operator config -- and forwarded to the +// orchestrator's hub link. Absent or empty => the link's 3s +// `DEFAULT_RECONNECT_DELAY_MS`. Production never sets this; the deploy-flow +// test harness sets a short value so reconnect-survival tests that do not +// assert the delay itself (they assert the reconnect's recovery semantics, +// not the backoff duration) do not burn 3s of wall clock per dropped link. +const reconnectDelayRaw = process.env["SIDECAR_RECONNECT_DELAY_MS"]; +let reconnectDelayMs: number | undefined; +if (reconnectDelayRaw !== undefined && reconnectDelayRaw.trim() !== "") { + const parsed = Number.parseInt(reconnectDelayRaw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `SIDECAR_RECONNECT_DELAY_MS must be a positive integer (milliseconds), got ${reconnectDelayRaw}`, + ); + } + reconnectDelayMs = parsed; +} + +// Sweep any tmp staging directories left behind by a `put` or +// `extractTarball` that crashed between staging and the final rename +// on a previous boot. Running here, before the orchestrator starts +// accepting apply work, keeps the cache root from accumulating +// orphans for the lifetime of the sidecar's data directory. await createTarballCache({ - rootDir: CACHE_ROOT, - maxBytes: CACHE_MAX_BYTES, + rootDir: cacheRoot, + maxBytes: cacheMaxBytes, }).sweepOrphans(); -// Load or mint the host's persisted Ed25519 identity before anything -// else touches the data directory. One key, one identity for this -// process: the supervisor principal signs every workflow-run commit -// with it, the substrate's signing callback signs every SSH-signed -// commit with it, and each workflow-process child re-derives it from -// its spawn-time env. The public key is logged so an operator can pin -// which identity a given process advertises. -const signingKey = await loadOrMintSidecarKeypair(path.join(config.dataDir, ".sidecar-signing")); -getLogger(["sidecar", "boot"]).info`Sidecar identity ${hexEncode(signingKey.publicKey)}`; - -// The substrate-backed RepoStore the supervisors read and write through. -// Wrapped below with the pack-pushing facade so a successful -// workflow-run write ships to the hub before its Promise resolves. +// Load or mint the sidecar's local Ed25519 keypair. The supervisor +// principal signs every workflow-run commit with this key; the +// substrate's `signingCallback` signs every SSH-signed commit with +// it; the workflow-host child's substrate factory re-uses it via the +// `SIDECAR_SIGNING_*` spawn-time env vars. One key, one identity for +// the sidecar process. +const SIDECAR_SIGNING_DIR = path.join(dataDir, ".sidecar-signing"); + +const sidecarSigningKey = await loadOrMintSidecarKeypair(SIDECAR_SIGNING_DIR); + +// Construct the substrate-backed RepoStore at the boot edge. The +// supervisor consumes this through the deploy router; the supervisor's +// conversation-state writes reach `writeTreePreservingPrefix` on this +// store. The boot-edge facade below wraps the store so a successful +// workflow-run write fires the pack push hook before its Promise +// resolves. const agentRepoStore = createAgentRepoStore({ - dataDir: config.dataDir, - signingKey, + dataDir, + signingKey: sidecarSigningKey, }); -// Per-deployment registries the hub link consults on inbound frames and -// the pack-push facade consults when addressing outbound frames. +// Cache of the hub-vouched public keys of senders this sidecar's deployments +// are authorized to receive mail from. The grants handler writes each key the +// hub co-delivers on a `run.grants` frame; the recipient's inbound-mail verify +// reads it back. The keyring is loaded here at boot so a restart keeps every +// previously-cached key. The durable-write primitive is injected so the cache +// write is atomic and fsynced -- at least as durable as the run-grants write it +// gates -- while the cache itself stays in @intx/hub-agent. +const senderKeyCache = await createSenderKeyCache({ + dataDir, + writeFileDurable: (filePath, contents) => + writeFileAtomicDurable(filePath, contents, { mode: 0o600 }), + removeFileDurable: (filePath) => removeFileAtomicDurable(filePath), +}); + +// The read side of the same cache: the inbound signature verify resolves a +// sender address to the crypto that verifies its mail. Built here at the edge +// so the hub link stays source-opaque -- it resolves address -> crypto without +// holding the cache or knowing where the key came from. +const resolveSenderCrypto = createSenderCryptoResolver(senderKeyCache); + +// Per-recipient-address registry of resolved inbound-mail admission policies. +// The workflow-host wiring resolves each hydrated deployment's authored +// `inboundMailPolicy` into this registry beside its mail-router registration +// and removes it on teardown; the read side below hands the hub-link's +// `mail.inbound` seam a total policy for every address. Built here at the edge +// so the hub link stays policy-source-opaque, mirroring `resolveSenderCrypto`. +const inboundMailPolicyRegistry = createInboundMailPolicyRegistry(); +const lookupInboundMailPolicy = createInboundMailPolicyLookup(inboundMailPolicyRegistry); + +// The deploy router records `(runId -> agentAddress)` here on +// every inbound `agent.deploy`; the facade resolves the mapping when +// firing the pack push so the outbound frames carry the right +// agentAddress for hub-side routing. const deploymentAddressRegistry = createDeploymentAddressRegistry(); + +// Per-deployment-address mail handler registry the hub-link consults on +// every inbound `mail.inbound` frame. The deploy router's multi-step +// branch registers `wired.routeInbound` against the deployment's mail +// address once its supervisor spawns; the hub-link's `mail.inbound` +// handler calls `tryRoute` so an inbound deployment-address message +// lands on the supervisor's mail-bus subscription. Mail for an address +// with no registered handler has no receiver and is logged-and-dropped. const multistepMailRouter = createMultistepMailRouter(); + +// Per-deployment-address signal handler registry the hub-link consults +// on every inbound `signal.deliver` frame. The multi-step deploy +// router registers a handler against the deployment's mail address +// once its supervisor spawns; the handler forwards the signal into the +// supervisor's `deliverSignal`, which sends a `signal.deliver` control +// IPC frame to the workflow-process child. The child commits the +// resulting `SignalReceived` event through its own substrate -- the +// single writer of the workflow-run repo on the sidecar side. const multistepSignalRouter = createMultistepSignalRouter(); + +// Per-deployment-address drain handler registry the hub-link consults +// on every inbound `drain.deliver` frame. The multi-step deploy router +// registers a handler against the deployment's mail address once its +// supervisor spawns; the handler forwards the drain into the +// supervisor's `drain`, which sends a `drain` control IPC frame to the +// workflow-process child and arms one drainTimeout accumulator per +// in-flight run. Cancel-mode in-flight steps abort on the child side; +// wait-mode steps continue. Accumulators commit a signed +// `CancelRequested{origin: "supervisor-drain"}` against the +// workflow-run repo when the deadline expires. const multistepDrainRouter = createMultistepDrainRouter(); + +// Per-deployment-address grants handler registry the hub-link consults +// on every inbound `run.grants` frame. The deploy router registers a +// handler against the deployment's mail address once its supervisor +// spawns; the handler writes the run's grants to `runs//grants.json` +// inside the deployment's workflow-run repo, sibling to that run's +// `runs//events/` subtree. The write is awaited so the frame's +// FIFO completion means the grants are durable on disk. const multistepGrantsRouter = createMultistepGrantsRouter(); + +// Per-deployment-address sources-rotation handler registry. Only a +// single-step warm deployment registers a handler once its supervisor +// spawns; the handler forwards the rotated list into the supervisor's +// `deliverSources`, which sends a `sources-updated` control IPC frame to +// the child, where the warm agent's live sources are swapped in place. A +// multi-step deployment registers none, so a rotation resolved against +// its address is unrouted. const multistepSourcesRouter = createMultistepSourcesRouter(); +// Per-deployment credential-delivery handler registry. A deployment registers +// its handler after `spawn` (any deployment with a supervisor, not only warm +// single-step ones); a `credentials.update` for an unregistered address is +// unrouted. const multistepCredentialsRouter = createMultistepCredentialsRouter(); const transport = createInMemoryTransport(); -// The pack-push client closes over the substrate (for pack creation) -// and a lazy hub-link binding: `createSidecarOrchestrator` invokes -// `createDeployRouter` during construction, before the orchestrator -// handle exists, so the link reference is bound once construction -// returns and consulted lazily here. +// The pack-push client closes over the substrate (for `createPack`) +// and a lazy hub-link binding (for `pushWorkflowRunPack`). The link +// reference is set once the orchestrator is constructed below; the +// closure here is consulted lazily because the +// `createSidecarOrchestrator` factory calls `createDeployRouter` +// during its constructor, before the orchestrator handle is bound. let resolvedHubLink: HubLink | null = null; const workflowRunPackClient = createWorkflowRunPackClient({ substrate: agentRepoStore.repoStore, @@ -137,7 +360,6 @@ const workflowRunPackClient = createWorkflowRunPackClient({ }, }, }); - const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ // Restore into the unwrapped substrate. Running Hub-authored history // through the push facade would echo the same pack straight back to the @@ -146,193 +368,230 @@ const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ markRestored: workflowRunPackClient.markRestored, }); +// Wrap the substrate's RepoStore with the boot-edge facade so a +// successful supervisor write against a workflow-run repo fires the +// pack push hook before its Promise resolves. Non-workflow-run writes +// (today, the agent-state deploy-applier path) flow through +// unchanged. const wrappedRepoStore = createWorkflowRunPackPushingRepoStore({ underlying: agentRepoStore.repoStore, packClient: workflowRunPackClient, registry: deploymentAddressRegistry, }); -const bootRestorePushHold = createBootRestorePushHold(wrappedRepoStore); - -// Substrate-config keys threaded into every workflow-process child's -// fresh spawn env (nothing is inherited from this process). PATH lets -// the child's `bun` shebang resolve; HOME/TMPDIR give agent code a -// writable home and the host's temp root. The signing keys let the -// child's substrate factory re-derive the host identity. +const hubWsUrl = requireEnv("HUB_WS_URL"); +const sidecarId = requireEnv("SIDECAR_ID"); +const sidecarToken = requireEnv("SIDECAR_TOKEN"); + +// Multi-step substrate-config the deploy router threads into the +// workflow-process child's spawn-time env. The child's substrate +// factory consumes these via the typed `SubstrateConfig` validator so +// the per-step pack-push wrap can identify the deployment's hub-side +// trust anchors. Today the IPC bridge in `pack.push.request` carries +// the pack; the WebSocket-connection keys are reserved for a future +// child-local hub link without redoing the boot-edge wiring. +// +// `PATH`, `HOME`, and `TMPDIR` are propagated from the boot edge's own +// environment so the child's `#!/usr/bin/env bun` shebang can resolve +// `bun`, agent code can find a writable home, and tmp-file APIs land +// on the same temp root the host uses. The substrate-config validator +// ignores undeclared keys, so these are visible to the OS for binary +// lookup but invisible to the typed `SubstrateConfig` shape. const multistepSubstrateEnv: Record = { - SIDECAR_DATA_DIR: config.dataDir, - SIDECAR_SIGNING_PUBLIC_KEY: hexEncode(signingKey.publicKey), - SIDECAR_SIGNING_PRIVATE_KEY: hexEncode(signingKey.privateKey), - HUB_WS_URL: config.hubURL, - SIDECAR_ID: config.sidecarId, - SIDECAR_TOKEN: config.token, - PATH: config.path, - SIDECAR_CACHE_MAX_BYTES: String(CACHE_MAX_BYTES), - SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(REGISTRY_MAX_TARBALL_BYTES), - // Threaded verbatim from this boot edge's own resolved manifest so a - // workflow-process child resolves the exact custom adapters this - // process resolved -- never a default of its own. - SIDECAR_ADAPTER_MANIFEST: JSON.stringify(config.adapterManifest), - // Always serialized, defaulting to the public npmjs registry when the - // operator pinned none, so the child's per-step tool materialization - // resolves the exact registries this boot edge resolved — a child - // never falls back to a default of its own. - SIDECAR_TOOL_REGISTRIES: config.toolRegistries ?? DEFAULT_TOOL_REGISTRIES_JSON, + SIDECAR_DATA_DIR: dataDir, + SIDECAR_SIGNING_PUBLIC_KEY: hexEncode(sidecarSigningKey.publicKey), + SIDECAR_SIGNING_PRIVATE_KEY: hexEncode(sidecarSigningKey.privateKey), + HUB_WS_URL: hubWsUrl, + SIDECAR_ID: sidecarId, + SIDECAR_TOKEN: sidecarToken, + PATH: requireEnv("PATH"), + // Tool-loader caps the child's per-step tool materialization uses. + // Resolved once at the boot edge and threaded into the child through + // the substrate config so the child does not re-read env at a + // non-boundary site. + SIDECAR_CACHE_MAX_BYTES: String(cacheMaxBytes), + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(registryMaxTarballBytes), + // Serialize the parent's ALREADY-VALIDATED manifest (the object + // `readAdapterManifest` returned), not the raw env string, so the + // child rebuilds the same custom-adapter set. Always present (defaults + // to "[]" when no custom adapters are configured); the child treats a + // missing key as a serialization bug and fails loud. The child + // re-validates the shape before importing any module — defense in + // depth at the deserialization boundary, since the child env is + // operator-controlled via Bun.spawn. + SIDECAR_ADAPTER_MANIFEST: JSON.stringify(adapterManifest), }; -if (config.home !== undefined) { - multistepSubstrateEnv["HOME"] = config.home; +const hostHome = process.env["HOME"]; +if (hostHome !== undefined) { + multistepSubstrateEnv["HOME"] = hostHome; } -if (config.tmpdir !== undefined) { - multistepSubstrateEnv["TMPDIR"] = config.tmpdir; +const hostTmpdir = process.env["TMPDIR"]; +if (hostTmpdir !== undefined) { + multistepSubstrateEnv["TMPDIR"] = hostTmpdir; } // The deploy router's source-admission gate reuses this exact -// `canBuildSource` predicate against the one adapter registry. +// `canBuildSource` predicate, against the one adapter registry, rather +// than a second copy of the check. const buildHarness = createDefaultHarnessBuilder({ adapters }); -// Airlocked workflow-probe executor, assembled here and injected through the -// orchestrator so the sidecar answers `workflow.probe.request` with a real -// inert projection and its wire hash instead of the hub-link's rejecting -// placeholder. The materializer lays a probe frame's frozen closure out under -// a per-probe scratch dir (rooted in the sidecar data dir so it shares that -// dir's lifecycle); the executor spawns the one-shot child that evaluates the -// workflow entry against it. A probe delivers its source assets inline in one -// frame, capped by the shared inline-payload bound. +// Airlocked workflow-probe executor, assembled at the boot edge and +// injected through the orchestrator so the live sidecar answers +// `workflow.probe.request` with a real inert projection instead of the +// hub-link's rejecting placeholder. The host-side `MaterializeWorkflowClosure` +// lays out a probe frame's frozen closure under a per-probe scratch dir +// (rooted here so it shares the sidecar data dir's lifecycle); the executor +// spawns the one-shot child that evaluates the workflow entry against it. +// The cache/registry caps are the same boot-edge-resolved values the tool +// loader uses. +// The probe delivers source assets inline in one frame (see +// `WorkflowProbeRequestFrame`). A git-sourced asset that grows past the shared +// inline-payload cap is the signal to move the probe's asset delivery to the +// streamed transfer the deploy path uses. const workflowProbeExecutor = createWorkflowProbeExecutor({ materialize: createWorkflowClosureMaterializer({ - cacheRoot: CACHE_ROOT, - cacheMaxBytes: CACHE_MAX_BYTES, - registryMaxTarballBytes: REGISTRY_MAX_TARBALL_BYTES, + cacheRoot, + cacheMaxBytes, + registryMaxTarballBytes, maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, - registries: parseToolRegistries(config.toolRegistries ?? DEFAULT_TOOL_REGISTRIES_JSON), - scratchRoot: path.join(config.dataDir, "workflow-probe", "closures"), + registries: readRegistries(), + scratchRoot: path.join(dataDir, "workflow-probe", "closures"), }), }); -// Sidecar-hosted loopback OAuth logins (CL-7508): stages the pinned-port -// PKCE login for codex/xai-oauth on this machine when the hub requests it. -const oauthLoopbackLogin = createOAuthLoopbackLoginService(); - -// Sender-key cache backing inbound-mail signature verification: durably -// persists the hub-vouched public key per sender address so a later inbound -// frame can be checked against a locally cached key, not only the key the -// hub stamped on the frame. The registry starts empty; a deployment -// registers its resolved inbound-mail policy when it hydrates and -// unregisters it on teardown, so an unregistered (or torn-down) address -// falls through to the fully-closed default via `createInboundMailPolicyLookup`. -const senderKeyCache = await createSenderKeyCache({ - dataDir: config.dataDir, - writeFileDurable: (writePath, contents) => - writeFileAtomicDurable(writePath, contents, { mode: 0o600 }), - removeFileDurable: (removePath) => removeFileAtomicDurable(removePath), -}); -const inboundMailPolicyRegistry = createInboundMailPolicyRegistry(); - -const watchdogLog = getLogger(["sidecar", "hub-link-watchdog"]); -const watchdog = createHubLinkWatchdog({ - stallDeadlineMs: 60_000, - onStall: () => { - watchdogLog.error`Hub link stalled: connect attempt got neither open nor close within the deadline; exiting for a clean restart`; - process.exit(1); - }, -}); - -// Captured by the createDeployRouter callback, which the orchestrator -// invokes synchronously during construction; asserted below so a wiring -// regression fails loud at boot instead of silently skipping restore. -let capturedRouter: SidecarDeployRouter | undefined; +// Set by the `createDeployRouter` callback below (invoked synchronously +// during construction) so the boot edge can drive the router's restore pass +// before `orchestrator.start()` connects to the hub. +let sidecarDeployRouter: SidecarDeployRouter | undefined; const orchestrator = createSidecarOrchestrator({ - hubURL: config.hubURL, - sidecarId: config.sidecarId, - token: config.token, - dataDir: config.dataDir, + hubURL: hubWsUrl, + sidecarId, + token: sidecarToken, + dataDir, transport, cryptoOps: { generateKeyPair, verifySSHSig: verifySSHSignature, }, - scheduleReconnect: watchdog.scheduleReconnect, + resolveSenderCrypto, + lookupInboundMailPolicy, + // Write peer of `resolveSenderCrypto`: an inbound `sender.key.refresh` frame + // re-pushes a rotated sender key here. Decode the hex and persist through the + // same cache the read side serves from; `put` owns the 32-byte length check + // and `hexDecode` owns hex validity, so both faults surface to the link's + // handler rather than being masked here. + cacheSenderKey: (address, publicKey) => senderKeyCache.put(address, hexDecode(publicKey)), + // Evicting peer of `cacheSenderKey`: an inbound `sender.key.evict` frame + // durably removes a revoked sender's cached key through the same cache. + evictSenderKey: (address) => senderKeyCache.evict(address), mailInboundRouter: multistepMailRouter, signalInboundRouter: multistepSignalRouter, drainInboundRouter: multistepDrainRouter, grantsInboundRouter: multistepGrantsRouter, sourcesInboundRouter: multistepSourcesRouter, credentialsInboundRouter: multistepCredentialsRouter, - // Install Hub-authoritative workflow-run history before a replacement - // supervisor spawns, against the unwrapped substrate so the restore is - // never echoed back to the Hub as a new sidecar-authored update. applyWorkflowRunPack: restoreWorkflowRunPack, workflowProbeExecutor, - oauthLoginExecutor: oauthLoopbackLogin.start, - resolveSenderCrypto: createSenderCryptoResolver(senderKeyCache), - lookupInboundMailPolicy: createInboundMailPolicyLookup(inboundMailPolicyRegistry), - cacheSenderKey: async (address, publicKey) => { - await senderKeyCache.put(address, hexDecode(publicKey)); - }, - evictSenderKey: async (address) => { - await senderKeyCache.evict(address); - }, - // Called from every connection's open handler -- the watchdog's - // aliveness signal -- and from the close path, which immediately - // re-schedules a reconnect that re-arms the deadline. + // Test-only override of the hub-link reconnect backoff; unset in + // production, where the link applies its 3s default. + ...(reconnectDelayMs !== undefined ? { reconnectDelayMs } : {}), + // The hub link calls this on every (re)connect to announce the workflow + // deployments this sidecar hosts so the hub re-registers their routes. + // `createDeployRouter` runs synchronously during construction (below), so + // the router is captured before the link ever connects; assert rather than + // optional-chain so a wiring regression fails loud instead of silently + // announcing no deployments. getWorkflowAddresses: () => { - watchdog.markAlive(); - if (capturedRouter === undefined) { + if (sidecarDeployRouter === undefined) { throw new Error( - "sidecar boot: deploy router was not constructed before the hub link requested deployment addresses", + "sidecar boot: deploy router was not constructed before the hub link requested workflow addresses", ); } - return capturedRouter.activeAddresses(); + return sidecarDeployRouter.activeAddresses(); }, - // After the link re-answers a reconnect challenge for a deployment - // address, re-drive any workflow-run pack the disconnect cancelled. + // Report the sidecar's cached rotatable senders on every (re)connect so the + // hub re-resolves each current key and re-pushes it, catching a rotation that + // landed while the sidecar was disconnected. The cache owns the "only user + // senders rotate" filter (a run sender's key is the immutable + // workflow_run.public_key); the link stays source-opaque and reports whatever + // this returns. + getCachedSenderAddresses: () => senderKeyCache.rotatableAddresses(), + // When the hub-link re-announces a deployment address in an authenticated + // reconnect, re-drive any workflow-run pack the disconnect cancelled. The + // link fires this AFTER sending the reconnect frame, so the hub routes the + // address before it sees the re-shipped pack (both frame families queue on + // the hub's per-connection chain). This is the liveness half of + // reconnect recovery: without it, a synchronous single-step run whose only + // pack was interrupted mid-transfer never re-ships, because it has no later + // local write to re-arm the coalescing loop. onWorkflowAddressesRoutable: (addresses) => { + // The deploy router is captured synchronously during orchestrator + // construction, before the link ever connects, so a routable callback can + // only fire once it exists; assert (like `getWorkflowAddresses`) rather + // than optional-chain so a wiring regression fails loud. + if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before a reconnect made workflow addresses routable", + ); + } for (const address of addresses) { wrappedRepoStore.notifyAddressRoutable(address); + // Trigger B: re-register the deployment's parked correlations. A long hub + // outage can evict a `signal.correlation.register` frame from the link's + // bounded send queue; re-driving it on reconnect recovers the parked + // run's approvability. Two independent facts make this safe: + // - The hub-side co-write's `registerSignalCorrelation` throws only when + // the deployment has no `status = "deployed"` row. That row is written + // at DEPLOY time and persists across the outage -- the reconnect + // re-routes the address in the hub's in-memory index but writes no DB + // status, so the lookup already resolves. (The frame also lands after + // the address is wire-routable: the link fires this after the + // reconnect frame, and both frames queue on the hub's per-connection + // chain -- see the pack re-ship note above.) + // - A sidecar restart re-emits the parked set twice (child + // re-establishment fires Trigger A too), but the co-write dedups on + // the `correlationId` PK/unique constraints via `onConflictDoNothing`, + // so the duplicate is a no-op. The pre-existing non-transactional + // status-check window in `registerSignalCorrelation` that this + // widened concurrency exercises is tracked in INTR-338, out of scope + // here. + sidecarDeployRouter.reEmitParkedCorrelations(address); } }, - // On disconnect, block the addresses' workflow-run pushes until the - // reconnect challenge re-routes them; a push shipped on the fresh, - // not-yet-challenged connection would be dropped by the hub. + // On disconnect, block the deployment addresses' workflow-run pushes until + // the authenticated reconnect above re-routes them. Without the block, the + // coalescing pusher re-ships onto the fresh, not-yet-registered connection + // and the hub drops the frames as "unrouted". onWorkflowAddressesUnroutable: (addresses) => { for (const address of addresses) { wrappedRepoStore.markAddressUnroutable(address); } }, - createDeployRouter: ({ sessions, keyStore, publishWorkflowInferenceEvent }) => { - const deployRouterConfigBase: Parameters[0] = { + createDeployRouter: ({ + sessions, + keyStore, + publishWorkflowInferenceEvent, + publishWorkflowSuspension, + }) => { + const router = createSidecarDeployRouter({ sessions, keyStore, + senderKeyCache, transport, repoStore: wrappedRepoStore, - signingKeySeed: signingKey.privateKey, + signingKeySeed: sidecarSigningKey.privateKey, + credentialCipher, createAgentCrypto: createEd25519Crypto, assertSourceBuildable: buildHarness.canBuildSource, - registerDeployment: ({ deploymentId, agentAddress }) => { - deploymentAddressRegistry.record(deploymentId, agentAddress); - bootRestorePushHold.onDeploymentRegistered(agentAddress); - }, - unregisterDeployment: ({ deploymentId, agentAddress }) => { - deploymentAddressRegistry.unregister(deploymentId); - wrappedRepoStore.reclaimPushState({ deploymentId, agentAddress }); + registerDeployment: ({ runId, agentAddress }) => { + deploymentAddressRegistry.record(runId, agentAddress); }, - // Lazy-bound the same way as `workflowRunPackClient.hubLink` above: - // `createDeployRouter` runs synchronously during `createSidecarOrchestrator` - // construction, before `orchestrator.hubLink` exists, so `resolvedHubLink` - // is consulted at call time rather than captured now. Without this, a - // workflow-child's ask-rail suspension never reaches the hub as a - // `signal.correlation.register` frame and its approval is never - // registered. - registerSuspension: (registration) => { - if (resolvedHubLink === null) { - throw new Error( - "sidecar boot: suspension register attempted before hub link was constructed", - ); - } - resolvedHubLink.sendSignalCorrelationRegister(registration); + unregisterDeployment: ({ runId }) => { + deploymentAddressRegistry.unregister(runId); }, multistepMailRouter, + inboundMailPolicyRegistry, multistepSignalRouter, multistepDrainRouter, multistepGrantsRouter, @@ -340,93 +599,35 @@ const orchestrator = createSidecarOrchestrator({ multistepCredentialsRouter, multistepSubstrateEnv, publishWorkflowInferenceEvent, - }; - const deployRouterConfigWithConsumedRetentionMs = - config.consumedRetentionMs !== undefined - ? { - ...deployRouterConfigBase, - consumedRetentionMs: config.consumedRetentionMs, - } - : deployRouterConfigBase; - const deployRouterConfigWithReadyTimeoutMs = - config.readyTimeoutMs !== undefined - ? { - ...deployRouterConfigWithConsumedRetentionMs, - readyTimeoutMs: config.readyTimeoutMs, - } - : deployRouterConfigWithConsumedRetentionMs; - const deployRouterConfigWithIdleHibernateMs = - config.idleHibernateMs !== undefined - ? { - ...deployRouterConfigWithReadyTimeoutMs, - idleHibernateMs: config.idleHibernateMs, - } - : deployRouterConfigWithReadyTimeoutMs; - const router = createSidecarDeployRouter(deployRouterConfigWithIdleHibernateMs); - capturedRouter = router; + publishWorkflowSuspension, + ...(onDispatchTiming !== undefined ? { onDispatchTiming } : {}), + ...(repackEveryMessages !== undefined ? { repackEveryMessages } : {}), + ...(consumedRetentionMs !== undefined ? { consumedRetentionMs } : {}), + ...(readyTimeoutMs !== undefined ? { readyTimeoutMs } : {}), + }); + // Capture the router so the boot edge can drive its restore pass before + // connecting. `createDeployRouter` runs synchronously during + // `createSidecarOrchestrator` construction (exactly once, before the + // handle returns), so `sidecarDeployRouter` is populated by the time the + // restore call below runs. + sidecarDeployRouter = router; return router; }, }); resolvedHubLink = orchestrator.hubLink; -if (capturedRouter === undefined) { - throw new Error("sidecar boot: deploy router was not constructed before deployment restore"); +// Re-establish the workflow deployments a prior sidecar process persisted, +// BEFORE opening the hub connection: each single-step head must have its +// mailbox/transport registration live before the hub can route to it. +// Assert the router was captured rather than optional-chaining it, so a +// future refactor that made `createDeployRouter` fire lazily would fail loud +// here instead of silently skipping restore. +if (sidecarDeployRouter === undefined) { + throw new Error( + "sidecar boot: deploy router was not constructed before workflow-deployment restore", + ); } -const deployRouter: SidecarDeployRouter = capturedRouter; - -// Re-establish the deployments a prior process persisted BEFORE the -// connection opens: each deployment's mailbox/transport registration -// must be live before the hub can route to it, and the first register -// frame must announce every restored address. -bootRestorePushHold.begin(); -try { - await deployRouter.restoreWorkflowDeployments(); -} finally { - bootRestorePushHold.end(); -} - -// Independent of the deployment restore above: reclaims any -// hibernated-agent-identity snapshot (see -// `hibernated-agent-identity-vault.ts`) whose address hibernated and was -// never redeployed within the retention window, so a permanently -// abandoned hibernate does not leak disk forever. -await deployRouter.reapExpiredHibernationSnapshots(); +await sidecarDeployRouter.restoreWorkflowRuns(); -// The first connect bypasses the reconnect scheduler, so arm the stall -// deadline by hand; the open path's getWorkflowAddresses disarms it. -watchdog.armForBoot(); orchestrator.start(); - -const SHUTDOWN_DRAIN_MS = 8_000; -let shuttingDown = false; - -async function shutdown(signal: string): Promise { - if (shuttingDown) return; - shuttingDown = true; - const log = getLogger(["sidecar", "shutdown"]); - await runSidecarShutdown({ - signal, - close: () => { - orchestrator.close(); - }, - drain: () => deployRouter.shutdownAll(), - drainTimeoutMs: SHUTDOWN_DRAIN_MS, - exit: (code) => { - process.exit(code); - }, - log, - }); -} - -function onShutdownRejection(error: unknown): void { - reportError(error, { operation: "sidecar.shutdown.signal" }); - process.exit(1); -} - -process.on("SIGTERM", () => { - attachShutdownRejectionHandler(shutdown("SIGTERM"), onShutdownRejection); -}); -process.on("SIGINT", () => { - attachShutdownRejectionHandler(shutdown("SIGINT"), onShutdownRejection); -}); diff --git a/apps/sidecar/src/multistep-credentials-router.test.ts b/apps/sidecar/src/multistep-credentials-router.test.ts deleted file mode 100644 index 5108961e05..0000000000 --- a/apps/sidecar/src/multistep-credentials-router.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { expect, test } from "bun:test"; - -import { createMultistepCredentialsRouter } from "./workflow-run-pack-client"; - -function makeFrame(agentAddress: string) { - return { - type: "credentials.update" as const, - agentAddress, - delivery: { - bindings: [ - { - handle: "mcp.exa", - credentialId: "cred_1", - consumer: "tool:@corbits/mcp-tools", - }, - ], - materials: [ - { - credentialId: "cred_1", - providerKey: "http", - origin: "https://mcp.exa.ai/mcp", - secret: "rotated-secret", - }, - ], - }, - }; -} - -test("tryRoute reports false for an address with no registered handler", async () => { - const router = createMultistepCredentialsRouter(); - expect(await router.tryRoute(makeFrame("ins_dep_1@example.com"))).toBe(false); -}); - -test("tryRoute dispatches the frame's delivery to the handler", async () => { - const router = createMultistepCredentialsRouter(); - const seen: { delivery: unknown }[] = []; - router.register("ins_dep_1@example.com", async (args) => { - seen.push(args); - }); - const frame = makeFrame("ins_dep_1@example.com"); - expect(await router.tryRoute(frame)).toBe(true); - expect(seen).toEqual([{ delivery: frame.delivery }]); -}); - -test("a handler rejection propagates to the caller", async () => { - const router = createMultistepCredentialsRouter(); - router.register("ins_dep_1@example.com", () => - Promise.reject(new Error("child rejected credentials-updated")), - ); - await expect(router.tryRoute(makeFrame("ins_dep_1@example.com"))).rejects.toThrow( - /child rejected credentials-updated/, - ); -}); - -test("unregister stops routing for the address", async () => { - const router = createMultistepCredentialsRouter(); - router.register("ins_dep_1@example.com", async () => undefined); - router.unregister("ins_dep_1@example.com"); - expect(await router.tryRoute(makeFrame("ins_dep_1@example.com"))).toBe(false); -}); diff --git a/apps/sidecar/src/multistep-grants-router.test.ts b/apps/sidecar/src/multistep-grants-router.test.ts deleted file mode 100644 index 1631b2778d..0000000000 --- a/apps/sidecar/src/multistep-grants-router.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { expect, test } from "bun:test"; - -import { createMultistepGrantsRouter } from "./workflow-run-pack-client"; - -function makeFrame(agentAddress: string) { - return { - type: "run.grants" as const, - agentAddress, - runId: "run-1", - stepGrants: [{ resource: "tool:echo", action: "invoke" }], - }; -} - -test("tryRoute reports false for an address with no registered handler", async () => { - const router = createMultistepGrantsRouter(); - expect(await router.tryRoute(makeFrame("ins_dep_1@example.com"))).toBe(false); -}); - -test("tryRoute dispatches the frame's runId and stepGrants to the handler", async () => { - const router = createMultistepGrantsRouter(); - const seen: { runId: string; stepGrants: readonly unknown[] }[] = []; - router.register("ins_dep_1@example.com", async (args) => { - seen.push(args); - }); - expect(await router.tryRoute(makeFrame("ins_dep_1@example.com"))).toBe(true); - expect(seen).toEqual([ - { - runId: "run-1", - stepGrants: [{ resource: "tool:echo", action: "invoke" }], - }, - ]); -}); - -test("a handler rejection propagates to the caller", async () => { - const router = createMultistepGrantsRouter(); - router.register("ins_dep_1@example.com", () => Promise.reject(new Error("durable write failed"))); - await expect(router.tryRoute(makeFrame("ins_dep_1@example.com"))).rejects.toThrow( - /durable write failed/, - ); -}); - -test("unregister stops routing for the address", async () => { - const router = createMultistepGrantsRouter(); - router.register("ins_dep_1@example.com", async () => undefined); - router.unregister("ins_dep_1@example.com"); - expect(await router.tryRoute(makeFrame("ins_dep_1@example.com"))).toBe(false); -}); diff --git a/apps/sidecar/src/oauth-login.test.ts b/apps/sidecar/src/oauth-login.test.ts deleted file mode 100644 index 57da8034da..0000000000 --- a/apps/sidecar/src/oauth-login.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -// CL-7508: the sidecar-hosted loopback login service binds exactly the -// pinned provider ports, surfaces a busy port as the typed -// `OAuthCallbackPortInUseError`, and never opens a browser sidecar-side. -import { afterAll, describe, expect, test } from "bun:test"; -import { createServer, type Server } from "node:http"; -import { CODEX_AUTHORIZE_URL, CODEX_REDIRECT_URI } from "@corbits/codex-provider/constants"; -import { XAI_REDIRECT_URI } from "@corbits/xai-provider"; -import { OAuthCallbackPortInUseError } from "@corbits/oauth-core"; -import { createOAuthLoopbackLoginService } from "./oauth-login"; - -const occupied: Server[] = []; -afterAll(() => { - for (const server of occupied) server.close(); -}); - -/** Occupies a pinned port so a login attempt must fail with the typed - * port-in-use error. Resolves false when the port was already busy (the - * assertion we are about to make is then trivially proven by whatever - * holds the port). */ -function occupy(port: number): Promise { - return new Promise((resolve) => { - const server = createServer(); - server.once("error", () => resolve(false)); - server.listen(port, "127.0.0.1", () => { - occupied.push(server); - resolve(true); - }); - }); -} - -function redirectOf(authorizeUrl: string): string { - return new URL(authorizeUrl).searchParams.get("redirect_uri") ?? ""; -} - -describe("oauth loopback login service", () => { - test("codex pins localhost:1455 in the authorize URL it returns", async () => { - const service = createOAuthLoopbackLoginService(); - const handle = await service.start("codex"); - try { - expect(handle.authorizeUrl.startsWith(CODEX_AUTHORIZE_URL)).toBe(true); - expect(redirectOf(handle.authorizeUrl)).toBe(CODEX_REDIRECT_URI); - expect(new URL(CODEX_REDIRECT_URI).port).toBe("1455"); - } finally { - handle.cancel(); - } - }); - - test("xai-oauth pins 127.0.0.1:1456 in the authorize URL it returns", async () => { - const service = createOAuthLoopbackLoginService(); - const handle = await service.start("xai-oauth"); - try { - expect(redirectOf(handle.authorizeUrl)).toBe(XAI_REDIRECT_URI); - expect(new URL(XAI_REDIRECT_URI).hostname).toBe("127.0.0.1"); - expect(new URL(XAI_REDIRECT_URI).port).toBe("1456"); - } finally { - handle.cancel(); - } - }); - - test("a bound pinned port surfaces OAuthCallbackPortInUseError, never a fallback port", async () => { - // The xai pin shares this machine's port space, so holding 1456 forces - // the bind failure path for whichever login reaches it. - const held = await occupy(1456); - if (!held) return; // something else already proves the port is busy - const service = createOAuthLoopbackLoginService(); - // The bind happens while the handle is being staged, so `start` itself - // is what rejects. - const outcome = await service.start("xai-oauth").then( - () => "completed", - (cause: unknown) => cause, - ); - expect(outcome).toBeInstanceOf(OAuthCallbackPortInUseError); - expect((outcome as OAuthCallbackPortInUseError).port).toBe(1456); - }); - - test("a cancelled login frees the pinned port for the next one", async () => { - const service = createOAuthLoopbackLoginService(); - const first = await service.start("codex"); - first.cancel(); - // If cancel did not close 1455's listener, this rebind would fail with - // the typed port-in-use error instead of staging a fresh login. - const second = await service.start("codex"); - expect(redirectOf(second.authorizeUrl)).toBe(CODEX_REDIRECT_URI); - second.cancel(); - }); -}); diff --git a/apps/sidecar/src/oauth-login.ts b/apps/sidecar/src/oauth-login.ts deleted file mode 100644 index 53e7ab5cef..0000000000 --- a/apps/sidecar/src/oauth-login.ts +++ /dev/null @@ -1,138 +0,0 @@ -// Sidecar-hosted loopback OAuth login for the pinned-port providers -// (CL-7508). The authorization servers for Codex ("Login with ChatGPT") and -// xai-oauth (Grok CLI) only accept the fixed loopback redirect URIs their -// own CLIs register — `http://localhost:1455/auth/callback` and -// `http://127.0.0.1:1456/callback` — so the callback listener must bind -// exactly those ports on the machine the user is browsing from. That -// machine is the sidecar's host, which is why the login runs here, not in -// the hub: the hub threads an `oauth.login.start` frame over the ws -// channel, this service stages the login (PKCE pair, callback server, -// code exchange) sidecar-side, and only the finished tokens cross the -// wire — the PKCE verifier never leaves this process, and the browser is -// never opened sidecar-side (the hub ships the authorize URL to the web -// UI for navigation). -import { - buildAuthorizeUrl, - startCallbackServer, - startOAuthLogin, - type BaseTokens, - type CallbackServerConfig, - type FetchLike, - type OAuthClientConfig, -} from "@corbits/oauth-core"; -import { codexOAuthConfig, exchangeCodexCode, CODEX_REDIRECT_URI } from "@corbits/codex-provider"; -import { exchangeXaiCode, xaiOAuthConfig, XAI_REDIRECT_URI } from "@corbits/xai-provider"; - -/** The sidecar-hosted login outcome: the shape `@intx/types`'s - * `OAuthLoginTokens` wire arm carries. */ -export type StagedLoginTokens = BaseTokens & { - /** The issuer's id_token, when it issues one (xai-oauth). */ - idToken?: string; - /** id_token-derived account label (codex: `chatgpt_account_id`). */ - accountId?: string; -}; - -export type OAuthLoopbackLoginService = { - start: (connectorId: "codex" | "xai-oauth") => Promise<{ - authorizeUrl: string; - completed: Promise; - /** Closes the callback server and abandons the staged login. */ - cancel: () => void; - }>; -}; - -// The pins this service exists to enforce. A redirect URI whose port is not -// one of these is a manifest drift, not a cue to rebind dynamically. -const CODEX_PINNED_PORT = 1455; -const XAI_PINNED_PORT = 1456; - -function isLoopbackHost(host: string): boolean { - return host === "localhost" || host === "127.0.0.1" || host === "::1"; -} - -/** Builds the callback-server config from a provider's registered redirect - * URI, asserting the loopback + pinned-port contract up front: a bind - * failure later is genuinely "port already in use", never "we drifted onto - * some other port". */ -function callbackConfig(redirectUri: string, pinnedPort: number): CallbackServerConfig { - const url = new URL(redirectUri); - if (url.protocol !== "http:" || !isLoopbackHost(url.hostname)) { - throw new Error( - `${redirectUri} is not a loopback http redirect; refusing to host a login for it`, - ); - } - const port = Number(url.port); - if (port !== pinnedPort) { - throw new Error( - `${redirectUri} is not the pinned port ${String(pinnedPort)}; the authorization server only accepts the registered redirect`, - ); - } - return { - port, - host: url.hostname, - path: url.pathname, - doneHtml: - "Connected

Login complete — you can close this window.

", - failedHtml: (reason) => - `Login failed

Login failed: ${reason}

`, - }; -} - -export function createOAuthLoopbackLoginService(deps?: { - /** Epoch-ms clock; defaults to `Date.now`. Injectable for tests. */ - now?: () => number; - /** HTTP client for the token exchange; defaults to `fetch`. */ - fetchImpl?: FetchLike; -}): OAuthLoopbackLoginService { - const now = deps?.now ?? Date.now; - const fetchImpl = deps?.fetchImpl ?? fetch; - - // Browser-open suppression is deliberate: the sidecar runs on the user's - // machine but the hub owns navigation — it ships `authorizeUrl` to the web - // UI, and a sidecar-side `open` would race it with a second consent page. - const suppressedOpen = () => undefined; - - const connectors = { - codex: { - callback: callbackConfig(CODEX_REDIRECT_URI, CODEX_PINNED_PORT), - oauthConfig: codexOAuthConfig satisfies OAuthClientConfig, - exchange: (code: string, verifier: string, at: number) => - exchangeCodexCode(code, verifier, at, fetchImpl), - }, - "xai-oauth": { - callback: callbackConfig(XAI_REDIRECT_URI, XAI_PINNED_PORT), - oauthConfig: xaiOAuthConfig, - exchange: (code: string, verifier: string, at: number) => - exchangeXaiCode(code, verifier, at, fetchImpl), - }, - } as const; - - return { - async start(connectorId) { - const connector = connectors[connectorId]; - const handle = await startOAuthLogin( - { profile: connectorId, signal: new AbortController().signal, now }, - { - startCallbackServer: (state) => startCallbackServer(state, connector.callback), - buildAuthorizeUrl: (pkce, state) => buildAuthorizeUrl(connector.oauthConfig, pkce, state), - exchangeCode: (code, verifier, at) => connector.exchange(code, verifier, at), - // Persistence is the hub's job: the connect pipeline stores the - // credential row, so the staged `commit` here is a no-op that - // exists only to satisfy the oauth-core login contract. - saveProfile: async () => undefined, - // A no-op keeps navigation authority with the hub: the web UI - // opens the authorize URL the hub ships it, and a sidecar-side - // opener would race it with a second consent page. - openInBrowser: suppressedOpen, - }, - ); - return { - authorizeUrl: handle.authorizeUrl, - completed: handle.completed.then((staged) => staged.profile.tokens), - cancel: () => { - handle.cancel(); - }, - }; - }, - }; -} diff --git a/apps/sidecar/src/originating-workbench.test.ts b/apps/sidecar/src/originating-workbench.test.ts deleted file mode 100644 index c74d3fd52b..0000000000 --- a/apps/sidecar/src/originating-workbench.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { expect, test } from "bun:test"; - -import { - extractOriginatingWorkbenchId, - originatingWorkbenchIdFromRequest, - resolveOriginatingWorkbenchId, - UNSCOPED_ORIGINATING_WORKBENCH_ID, -} from "./originating-workbench"; - -function mailRequest(from: string) { - return { - input: { - headers: { from, to: ["myra@alice.localhost"] }, - rawHeaders: {}, - parts: [], - }, - }; -} - -test("extractOriginatingWorkbenchId reads the From local-part", () => { - expect(extractOriginatingWorkbenchId("chan_room_a@alice.localhost")).toBe("chan_room_a"); - expect(extractOriginatingWorkbenchId("Workbench ")).toBe( - "ins_workbench1", - ); -}); - -test("extractOriginatingWorkbenchId is undefined for an empty or bare From", () => { - expect(extractOriginatingWorkbenchId("")).toBeUndefined(); - expect(extractOriginatingWorkbenchId("@alice.localhost")).toBeUndefined(); -}); - -test("resolveOriginatingWorkbenchId uses the unscoped sentinel when From is missing", () => { - expect(resolveOriginatingWorkbenchId(undefined)).toBe(UNSCOPED_ORIGINATING_WORKBENCH_ID); - expect(resolveOriginatingWorkbenchId("chan_a")).toBe("chan_a"); -}); - -test("originatingWorkbenchIdFromRequest is a function of this request's mail", () => { - expect(originatingWorkbenchIdFromRequest(mailRequest("chan_a@alice.localhost"))).toBe("chan_a"); - expect(originatingWorkbenchIdFromRequest(mailRequest("chan_b@alice.localhost"))).toBe("chan_b"); -}); - -test("a resumed input delivers the decision's mail, not the original input", () => { - expect( - originatingWorkbenchIdFromRequest({ - ...mailRequest("chan_a@alice.localhost"), - resume: { - correlationId: "c1", - kind: "input", - decision: mailRequest("chan_b@alice.localhost").input, - }, - }), - ).toBe("chan_b"); -}); - -test("a request without mail names no room", () => { - expect(originatingWorkbenchIdFromRequest({ input: "plain text" })).toBe(undefined); - expect( - originatingWorkbenchIdFromRequest({ - input: mailRequest("chan_a@alice.localhost").input, - resume: { correlationId: "c1", kind: "approval", decision: "approved" }, - }), - ).toBe(undefined); -}); diff --git a/apps/sidecar/src/originating-workbench.ts b/apps/sidecar/src/originating-workbench.ts deleted file mode 100644 index 8754d5a573..0000000000 --- a/apps/sidecar/src/originating-workbench.ts +++ /dev/null @@ -1,52 +0,0 @@ -// Originating workbench for the warm agent's durable conversation. -// -// Chat mail names the room it speaks for in the RFC 2822 From local-part -// (`fromWorkbenchId`). The step request carries THIS message's Mail as its -// input (the same object the workflow-host step invoker sends to the agent), -// so the room is read from the request itself -- never from a per-mailbox -// side-channel, which binds an earlier mail under a later room when two -// inbound messages enqueue before the first invokeStep. A request that -// carries no mail (an approval resume, a synthetic input) names no room and -// leaves the current binding alone; a mail without a usable From is -// `_unscoped` so a later room never inherits a mixed blob. - -import { extractAddrSpec } from "@intx/mime"; -import { isMail } from "@intx/types/runtime"; -import type { StepInvokeRequest } from "@intx/workflow"; - -export const UNSCOPED_ORIGINATING_WORKBENCH_ID = "_unscoped"; - -/** - * Local-part of a From header value. Chat encodes the originating workbench - * id there; anything else (empty, unparseable) is undefined. - */ -export function extractOriginatingWorkbenchId(from: string): string | undefined { - if (from.trim() === "") return undefined; - try { - const spec = extractAddrSpec(from); - const at = spec.lastIndexOf("@"); - if (at <= 0) return undefined; - return spec.slice(0, at); - } catch { - return undefined; - } -} - -export function resolveOriginatingWorkbenchId(fromWorkbenchId: string | undefined): string { - return fromWorkbenchId !== undefined && fromWorkbenchId.length > 0 - ? fromWorkbenchId - : UNSCOPED_ORIGINATING_WORKBENCH_ID; -} - -/** - * Room named by the mail this step request delivers, or undefined when the - * request carries no mail. Mirrors the step invoker's own input selection: - * a resume delivers its decision, anything else delivers `input`. - */ -export function originatingWorkbenchIdFromRequest( - req: Pick, -): string | undefined { - const delivered = req.resume === undefined ? req.input : req.resume.decision; - if (!isMail(delivered)) return undefined; - return resolveOriginatingWorkbenchId(extractOriginatingWorkbenchId(delivered.headers.from)); -} diff --git a/apps/sidecar/src/run-grants.test.ts b/apps/sidecar/src/run-grants.test.ts deleted file mode 100644 index f20774aa9f..0000000000 --- a/apps/sidecar/src/run-grants.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import type { RepoStore } from "@intx/hub-sessions"; - -import { readRunGrants, runGrantsPath } from "./run-grants"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeRepoStore(): Promise<{ store: RepoStore; dir: string }> { - const dir = await mkdtemp(path.join(tmpdir(), "run-grants-")); - tempDirs.push(dir); - const store = { - getRepoDir: () => dir, - } as unknown as RepoStore; - return { store, dir }; -} - -test("runGrantsPath keys the file under the run's own subtree", () => { - expect(runGrantsPath("run-1")).toBe("runs/run-1/grants.json"); -}); - -test("readRunGrants returns undefined when the grants file is absent", async () => { - const { store } = await makeRepoStore(); - const grants = await readRunGrants({ - repoStore: store, - deploymentId: "dep-1", - runId: "run-1", - }); - expect(grants).toBeUndefined(); -}); - -test("readRunGrants returns the grants array from a valid file", async () => { - const { store, dir } = await makeRepoStore(); - const filePath = path.join(dir, runGrantsPath("run-1")); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile( - filePath, - JSON.stringify({ grants: [{ resource: "tool:echo", action: "invoke" }] }), - ); - const grants = await readRunGrants({ - repoStore: store, - deploymentId: "dep-1", - runId: "run-1", - }); - expect(grants).toEqual([{ resource: "tool:echo", action: "invoke" }]); -}); - -test("readRunGrants throws on a file that is not valid JSON", async () => { - const { store, dir } = await makeRepoStore(); - const filePath = path.join(dir, runGrantsPath("run-1")); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, "not json"); - await expect( - readRunGrants({ repoStore: store, deploymentId: "dep-1", runId: "run-1" }), - ).rejects.toThrow(/not valid JSON/); -}); - -test("readRunGrants throws on a file missing the grants envelope", async () => { - const { store, dir } = await makeRepoStore(); - const filePath = path.join(dir, runGrantsPath("run-1")); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify({ notGrants: [] })); - await expect( - readRunGrants({ repoStore: store, deploymentId: "dep-1", runId: "run-1" }), - ).rejects.toThrow(/failed validation/); -}); diff --git a/apps/sidecar/src/run-grants.ts b/apps/sidecar/src/run-grants.ts index 30a13633c1..27e3ac25f4 100644 --- a/apps/sidecar/src/run-grants.ts +++ b/apps/sidecar/src/run-grants.ts @@ -1,7 +1,17 @@ -// Per-run grants read primitives, ported from upstream Interchange's -// sidecar (apps/sidecar/src/run-grants.ts at the vendored pin). The -// hub's `run.grants` frame writes `runs//grants.json` into a -// deployment's `workflow-run` repo; this module reads it back. +// Per-run grants read primitives shared by the two sidecar modules that +// resolve a run's authorization from disk: the supervisor wiring +// (`workflow-host-wiring.ts`), which reads a run's grants at the +// `onRunStart` barrier, and the in-process child runtime +// (`workflow-substrate-factory.ts`), which reads the PARENT run's grants +// to bind a spawned child's authorize. Both destinations read the same +// `runs//grants.json` file in a deployment's `workflow-run` repo, +// so the read lives in one place. +// +// The module's dependency surface is deliberately narrow -- node fs/path, +// arktype, and the substrate `RepoStore` type -- so importing it into the +// child subprocess module does not drag the hub-agent deploy-router +// surface `workflow-host-wiring.ts` also depends on into the child +// binary. import { readFile } from "node:fs/promises"; import { join as pathJoin } from "node:path"; @@ -14,8 +24,8 @@ import { isErrnoNotFound } from "@intx/workflow-host"; /** * Path inside a deployment's `workflow-run` repo that carries a single * run's grants. It sits under the run's own `runs//` subtree -- - * sibling to that run's `events/` blobs -- so a run's grants live and - * are reclaimed with the rest of the run's state. + * sibling to that run's `events/` blobs -- so a run's grants live and are + * reclaimed with the rest of the run's state. */ export function runGrantsPath(runId: string): string { return `runs/${runId}/grants.json`; @@ -23,9 +33,10 @@ export function runGrantsPath(runId: string): string { /** * Envelope the per-run grants file carries: the canonical `{ grants: [] }` - * shape the grants writer produces. The inner entries stay `unknown` -- - * the child's authorize layer narrows each against its own grant-rule - * validator. + * shape the deploy-time snapshot writes and validates. The inner entries + * stay `unknown` -- the child's authorize layer narrows each against its + * own grant-rule validator, exactly as the deploy-time snapshot's grants + * are surfaced untyped. */ export const RunGrantsFile = type({ grants: "unknown[]", @@ -33,23 +44,26 @@ export const RunGrantsFile = type({ /** * Read a single run's grants from `runs//grants.json` inside the - * deployment's `workflow-run` repo, via the working tree (`getRepoDir`). + * deployment's `workflow-run` repo. The read is a working-tree read via + * `getRepoDir` (the same substrate access pattern the deploy-time + * per-step read uses), so it observes the tip the grants handler wrote. * - * Returns `undefined` -- distinct from an empty grants array -- when the - * file is absent, so the caller can distinguish "this run got no per-run + * Returns the run's grants when the file exists. Returns `undefined` + * -- distinct from an empty grants array -- when the file is ABSENT + * (ENOENT), so the caller can distinguish "this run got no per-run * grants file" from "this run's grants are the empty set". A file that - * exists but is malformed THROWS: its presence implies a grants frame - * was delivered, so a structural failure is a boundary bug, not a - * default. + * exists but is malformed (invalid JSON or a shape the envelope rejects) + * THROWS: the file's presence implies a grants frame was delivered, and a + * structural failure is a boundary bug, not a default. */ export async function readRunGrants(args: { repoStore: RepoStore; - deploymentId: string; + anchorRunId: string; runId: string; }): Promise { const dir = args.repoStore.getRepoDir({ kind: "workflow-run", - id: args.deploymentId, + id: args.anchorRunId, }); const filePath = pathJoin(dir, runGrantsPath(args.runId)); let raw: string; @@ -64,14 +78,14 @@ export async function readRunGrants(args: { parsed = JSON.parse(raw); } catch (cause) { throw new Error( - `workflow-run/${args.deploymentId}:${runGrantsPath(args.runId)} is not valid JSON`, + `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} is not valid JSON`, { cause }, ); } const validated = RunGrantsFile(parsed); if (validated instanceof type.errors) { throw new Error( - `workflow-run/${args.deploymentId}:${runGrantsPath(args.runId)} failed validation: ${validated.summary}`, + `workflow-run/${args.anchorRunId}:${runGrantsPath(args.runId)} failed validation: ${validated.summary}`, ); } return validated.grants; diff --git a/apps/sidecar/src/shutdown.ts b/apps/sidecar/src/shutdown.ts deleted file mode 100644 index feb872bfe3..0000000000 --- a/apps/sidecar/src/shutdown.ts +++ /dev/null @@ -1,105 +0,0 @@ -// Bounded drain for process shutdown. The platform sends SIGTERM and -// expects a prompt exit; a drain that hangs would turn every deploy into -// an apparent crash, so the bound cuts it off. Timing out is not a -// fault -- the process is exiting either way -- but a drain that throws -// is, and the two outcomes are kept distinct so the caller can exit -// non-zero only for the genuine failure. -// -// `runSidecarShutdown` is the SIGTERM/SIGINT body: close, then drain. -// Throws from either (including a sync throw from orchestrator.close) -// become process.exit(1) rather than an unhandled rejection. -// `attachShutdownRejectionHandler` is the belt on the signal listener -// itself so a rejection that still escapes that body cannot become an -// unhandledRejection. - -import { reportError } from "@corbits/error-sink"; - -export type DrainOutcome = - | { kind: "drained" } - | { kind: "timed-out" } - | { kind: "failed"; error: unknown }; - -export async function drainWithTimeout( - drain: () => Promise, - timeoutMs: number, -): Promise { - let timer: ReturnType | undefined; - const timedOut = new Promise((resolve) => { - timer = setTimeout(() => { - resolve({ kind: "timed-out" }); - }, timeoutMs); - }); - const drained = (async (): Promise => { - try { - await drain(); - return { kind: "drained" }; - } catch (error) { - return { kind: "failed", error }; - } - })(); - const outcome = await Promise.race([drained, timedOut]); - clearTimeout(timer); - return outcome; -} - -export type SidecarShutdownLog = { - info(strings: TemplateStringsArray, ...values: unknown[]): void; - error(strings: TemplateStringsArray, ...values: unknown[]): void; -}; - -export type RunSidecarShutdownArgs = { - signal: string; - close: () => void; - drain: () => Promise; - drainTimeoutMs: number; - exit: (code: number) => void; - log: SidecarShutdownLog; - report?: typeof reportError; -}; - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -export async function runSidecarShutdown({ - signal, - close, - drain, - drainTimeoutMs, - exit, - log, - report = reportError, -}: RunSidecarShutdownArgs): Promise { - log.info`Received ${signal}; draining before exit`; - try { - close(); - const outcome = await drainWithTimeout(drain, drainTimeoutMs); - // Drained and timed-out both exit 0: the process is going down either - // way and a bound cutting a slow drain short is not a crash. A drain - // that threw is a genuine fault and exits non-zero. - if (outcome.kind === "failed") { - // report-error-ignore: CL-7221 — `report` defaults to reportError - // from @corbits/error-sink (see RunSidecarShutdownArgs above); it's - // a parameter, not an import binding, so check:report-error's - // static import-binding match can't see through the default. - report(outcome.error, { operation: "sidecar.shutdown" }); - log.error`Drain threw during shutdown; exiting non-zero: ${errorMessage(outcome.error)}`; - exit(1); - return; - } - exit(0); - } catch (error) { - // report-error-ignore: CL-7221 — same injected-reporter default as - // the catch above. - report(error, { operation: "sidecar.shutdown" }); - log.error`Shutdown threw; exiting non-zero: ${errorMessage(error)}`; - exit(1); - } -} - -export function attachShutdownRejectionHandler( - work: Promise, - onRejection: (error: unknown) => void, -): void { - void work.catch(onRejection); -} diff --git a/apps/sidecar/src/sidecar-materialization-config.ts b/apps/sidecar/src/sidecar-materialization-config.ts index 2d26cba8ea..9ff1de0e98 100644 --- a/apps/sidecar/src/sidecar-materialization-config.ts +++ b/apps/sidecar/src/sidecar-materialization-config.ts @@ -49,7 +49,6 @@ export function readRegistries(): ReadonlyMap { } catch (err) { throw new Error( `SIDECAR_TOOL_REGISTRIES is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, - { cause: err }, ); } const validated = RegistryConfigEnvArray(parsed); diff --git a/apps/sidecar/src/signing-keypair.test.ts b/apps/sidecar/src/signing-keypair.test.ts new file mode 100644 index 0000000000..fa9527c38a --- /dev/null +++ b/apps/sidecar/src/signing-keypair.test.ts @@ -0,0 +1,133 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { loadOrMintSidecarKeypair } from "./signing-keypair"; + +async function makeTempDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "sidecar-signing-keypair-")); +} + +const PRIVATE_KEY_PATH = (dir: string) => path.join(dir, "ed25519.private"); +const PUBLIC_KEY_PATH = (dir: string) => path.join(dir, "ed25519.public"); + +describe("loadOrMintSidecarKeypair", () => { + test("mints a fresh keypair on first boot and persists both files", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + const minted = await loadOrMintSidecarKeypair(signingDir); + + expect(minted.privateKey).toHaveLength(32); + expect(minted.publicKey).toHaveLength(32); + expect(Array.from(await fs.readFile(PRIVATE_KEY_PATH(signingDir)))).toEqual( + Array.from(minted.privateKey), + ); + expect(Array.from(await fs.readFile(PUBLIC_KEY_PATH(signingDir)))).toEqual( + Array.from(minted.publicKey), + ); + }); + + test("reloads a consistent keypair and derives the public key from the seed", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + const minted = await loadOrMintSidecarKeypair(signingDir); + const reloaded = await loadOrMintSidecarKeypair(signingDir); + + expect(reloaded.privateKey).toEqual(minted.privateKey); + expect(reloaded.publicKey).toEqual(minted.publicKey); + }); + + test("rejects a public key file that does not match the seed", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + const minted = await loadOrMintSidecarKeypair(signingDir); + + const tampered = new Uint8Array(minted.publicKey); + const firstByte = tampered[0]; + if (firstByte === undefined) { + throw new Error("expected a 32-byte public key"); + } + tampered[0] = firstByte ^ 0xff; + await fs.writeFile(PUBLIC_KEY_PATH(signingDir), tampered); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( + /does not match the key derived from the seed/, + ); + }); + + test("rejects a truncated public key file", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + await fs.writeFile(PUBLIC_KEY_PATH(signingDir), new Uint8Array(5)); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( + /does not match the key derived from the seed/, + ); + }); + + test("rejects an empty public key file", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + await fs.writeFile(PUBLIC_KEY_PATH(signingDir), new Uint8Array(0)); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( + /does not match the key derived from the seed/, + ); + }); + + test("rejects a seed that is not a valid Ed25519 seed", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + + await fs.writeFile(PRIVATE_KEY_PATH(signingDir), new Uint8Array(16)); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( + /is not a valid Ed25519 seed/, + ); + }); + + test("rejects a length-valid seed that derives to a different public key", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + + const otherDir = path.join(await makeTempDir(), ".sidecar-signing"); + const other = await loadOrMintSidecarKeypair(otherDir); + await fs.writeFile(PRIVATE_KEY_PATH(signingDir), other.privateKey); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow( + /does not match the key derived from the seed/, + ); + }); + + test("rejects a partial keypair where only the private key is present", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + await fs.rm(PUBLIC_KEY_PATH(signingDir)); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/is partial/); + }); + + test("rejects a partial keypair where only the public key is present", async () => { + const dataDir = await makeTempDir(); + const signingDir = path.join(dataDir, ".sidecar-signing"); + + await loadOrMintSidecarKeypair(signingDir); + await fs.rm(PRIVATE_KEY_PATH(signingDir)); + + await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/is partial/); + }); +}); diff --git a/apps/sidecar/src/source-asset-delivery.test.ts b/apps/sidecar/src/source-asset-delivery.test.ts new file mode 100644 index 0000000000..99a439f840 --- /dev/null +++ b/apps/sidecar/src/source-asset-delivery.test.ts @@ -0,0 +1,204 @@ +import { describe, test, expect, afterEach } from "bun:test"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; + +import type { ToolPackageManifest } from "@intx/types/tool-packages"; +import { collectReachableObjects } from "@intx/storage-isogit/node"; + +import { + assetReferenceFormats, + indexAssetPackIntoGitDir, + sourceAssetGitDir, +} from "./source-asset-delivery"; + +// The subtree-checkout path (`materializeWorkflowAssets`) is exercised +// end-to-end by the source-workflow e2e; these cover the pure classification and +// path helpers, plus `indexAssetPackIntoGitDir`'s atomic-publish semantics, in +// isolation. + +function manifest(entries: ToolPackageManifest["entries"]): ToolPackageManifest { + return { schemaVersion: "1", topLevel: [], entries }; +} + +describe("assetReferenceFormats", () => { + test("classifies each asset by the formats it is referenced with", () => { + const formats = assetReferenceFormats( + manifest([ + { + name: "@x/a", + version: "1.0.0", + source: { + kind: "asset", + assetId: "asset_src", + package: { + format: "source", + commitSha: "c1", + packageDir: ".", + treeOid: "t1", + }, + }, + }, + { + name: "@x/b", + version: "1.0.0", + source: { + kind: "asset", + assetId: "asset_src", + package: { + format: "source", + commitSha: "c1", + packageDir: "packages/b", + treeOid: "t2", + }, + }, + }, + { + name: "@x/c", + version: "1.0.0", + source: { + kind: "asset", + assetId: "asset_tar", + package: { + format: "tarball", + path: "tarballs/c.tgz", + integrity: "sha512-c", + }, + }, + }, + { + name: "left-pad", + version: "1.3.0", + source: { kind: "registry", registry: "npm", integrity: "sha512-lp" }, + }, + ]), + ); + + expect(formats.get("asset_src")).toEqual({ tarball: false, source: true }); + expect(formats.get("asset_tar")).toEqual({ tarball: true, source: false }); + // A registry entry backs no asset. + expect(formats.has("npm")).toBe(false); + expect(formats.size).toBe(2); + }); + + test("marks an asset referenced both ways as needing both checkouts", () => { + const formats = assetReferenceFormats( + manifest([ + { + name: "@x/a", + version: "1.0.0", + source: { + kind: "asset", + assetId: "asset_dual", + package: { + format: "source", + commitSha: "c1", + packageDir: ".", + treeOid: "t1", + }, + }, + }, + { + name: "@x/b", + version: "1.0.0", + source: { + kind: "asset", + assetId: "asset_dual", + package: { + format: "tarball", + path: "tarballs/b.tgz", + integrity: "sha512-b", + }, + }, + }, + ]), + ); + + expect(formats.get("asset_dual")).toEqual({ tarball: true, source: true }); + }); +}); + +describe("sourceAssetGitDir", () => { + test("joins a safe assetId under the gitDir root", () => { + expect(sourceAssetGitDir("/root/gits", "ast_wf-1.2")).toBe( + path.join("/root/gits", "ast_wf-1.2"), + ); + }); + + test.each([ + ["a/b", "slash"], + ["../escape", "parent traversal"], + ["with space", "space"], + ["", "empty"], + // All-dots ids satisfy SAFE_ASSET_ID (which permits ".") but escape or + // collapse the per-asset dir, so they must be rejected explicitly. + ["..", "bare parent"], + [".", "current dir"], + ["...", "triple dot"], + ])("rejects an unsafe assetId (%s)", (assetId) => { + expect(() => sourceAssetGitDir("/root/gits", assetId)).toThrow(/unsafe assetId/); + }); +}); + +describe("indexAssetPackIntoGitDir", () => { + const tempDirs: string[] = []; + + afterEach(async () => { + const dirs = tempDirs.splice(0); + await Promise.all(dirs.map((d) => fsp.rm(d, { recursive: true, force: true }))); + }); + + async function tempDir(): Promise { + const d = await fsp.mkdtemp(path.join(os.tmpdir(), "source-delivery-test-")); + tempDirs.push(d); + return d; + } + + // Pack a one-commit repo and return its commit sha plus a pack of every + // object reachable from it. + async function buildPack(): Promise<{ commitSha: string; pack: Uint8Array }> { + const dir = await tempDir(); + await git.init({ fs, dir, defaultBranch: "main" }); + await fsp.writeFile(path.join(dir, "package.json"), '{"name":"x"}\n'); + await git.add({ fs, dir, filepath: "package.json" }); + const commitSha = await git.commit({ + fs, + dir, + message: "t", + author: { name: "t", email: "t@t.dev" }, + }); + const oids = await collectReachableObjects(dir, commitSha); + const { packfile } = await git.packObjects({ fs, dir, oids }); + if (packfile === undefined) { + throw new Error("source-delivery test: packObjects returned no packfile"); + } + return { commitSha, pack: packfile }; + } + + test("retains the durable gitDir on success", async () => { + const root = await tempDir(); + const gitDir = path.join(root, "asset-ok"); + const { commitSha, pack } = await buildPack(); + + await indexAssetPackIntoGitDir({ pack, commitSha, gitDir }); + + expect(fs.existsSync(path.join(gitDir, ".git"))).toBe(true); + }); + + test("leaves no durable gitDir when indexing fails", async () => { + // The pin is absent from the pack, so indexing throws. The atomic + // temp+rename build means the final gitDir is never created -- restore's + // dir-exists gate must not find a partial store. + const root = await tempDir(); + const gitDir = path.join(root, "asset-bad"); + const { pack } = await buildPack(); + + await expect( + indexAssetPackIntoGitDir({ pack, commitSha: "0".repeat(40), gitDir }), + ).rejects.toThrow(/not found in the pack/); + + expect(fs.existsSync(gitDir)).toBe(false); + }); +}); diff --git a/apps/sidecar/src/step-agent-tools.test.ts b/apps/sidecar/src/step-agent-tools.test.ts index 419784e005..45a33f3db0 100644 --- a/apps/sidecar/src/step-agent-tools.test.ts +++ b/apps/sidecar/src/step-agent-tools.test.ts @@ -1,193 +1,683 @@ -// Unit tests for the credential half of the tool-bearing step wiring -// (CL-6032): `consumerBindings` derives a consumer-scoped -// `ResolvedCredentialBinding` map from a step's live `CredentialWiring`, -// and composes with the platform's own `createCredentialCapability` -// (`@intx/harness/src/credential-capability.ts`) exactly as -// `createToolBearingAgentFactory` does inside its `credentialCapabilityFor` -// closure. These tests exercise that composition directly -- a fake -// `CredentialWiring` in, a shaped `credentials` capability out -- without -// standing up a full `Agent`, which `tools/granola` and -// `tools/linear`'s own `tool.test.ts` suites already cover from -// the tool side (a `CredentialCapability` in, a `ToolResult` out). -import { describe, expect, test } from "bun:test"; +// LSP-lifecycle seam test for the workflow-process child's tool-bearing +// agent factory. +// +// What this asserts, precisely: +// - The agent factory built by `createToolBearingAgentFactory` runs +// each materialized plugin factory when it builds the step's agent +// (the plugin chain mirrors `default-harness.ts`). +// - The plugin's `dispose` runs when `agent.close()` is called, and +// `agent.close()` is what the step-invoker adapter calls in its +// `finally` on every exit path. +// +// What this does NOT assert: a real language server protocol exchange. +// The real LSP plugin (`@intx/tools-lsp` `createLSPPlugin`) spawns its +// server subprocess LAZILY -- only when a tool touches a file -- and its +// `dispose` chains to `lsp.dispose()`, which terminates whatever server +// subprocesses were spawned. This test stands in a plugin whose factory +// spawns a REAL subprocess eagerly and whose `dispose` kills it, so the +// load-bearing seam under test -- "the child's agent.close() tears down +// the plugin's subprocess" -- is exercised against a real OS process +// without depending on a language-server binary being present in CI. +// The LSP-specific lazy-spawn behavior is covered by the `tools-lsp` +// package's own tests; what is sidecar-specific (and new in Phase 2) is +// the close -> plugin-dispose wiring proven here. + +import { describe, test, expect, afterEach } from "bun:test"; + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { - createCredentialCapability, - createCredentialProviderRegistry, - createHttpCredentialProvider, -} from "@intx/harness"; -import { toolConsumer } from "@intx/authz"; -import type { GrantRule } from "@intx/authz"; -import type { CredentialDelivery } from "@intx/types/sidecar"; -import type { CredentialWiring } from "@intx/workflow-host"; - -import { consumerBindings, type StepCredentialContext } from "./step-agent-tools"; - -const CONSUMER = toolConsumer("@corbits/granola-tools"); -const OTHER_CONSUMER = toolConsumer("@corbits/linear-tools"); -const CREDENTIAL_ID = "cred_1"; -const STEP_ID = "step_1"; + definePlugin, + defineTool, + type AnnotatedToolFactory, + type BaseEnv, + type ToolBundle, + type ToolDeclaration, +} from "@intx/agent"; +import { createDefaultDirectorRegistry } from "@intx/agent"; +import { evaluateGrants } from "@intx/authz"; +import type { HostCredentialCapability } from "@intx/harness"; +import { createIsogitStore } from "@intx/storage-isogit/node"; +import { noopAuditStore } from "@intx/agent/testing"; +import type { GrantRule } from "@intx/types/authz"; +import type { InferenceSource } from "@intx/types/runtime"; +import { + createRuntimeCapabilities, + type RuntimeCapabilities, +} from "@intx/types/runtime-capabilities"; +import type { LoadedToolFactory } from "@intx/tool-packaging"; -/** - * Provider registry stubbed with a captured-header fetch, mirroring the - * `createSidecarSubstrateFactory` production wiring - * (`createCredentialProviderRegistry(builtinCredentialProviders())`) but - * with the network swapped for a spy so a test can assert on the header - * the mediated `fetch` injected without a real HTTP call. - */ -function providersWithCapturedAuth(): { - providers: ReturnType; - lastAuthHeader: () => string | null; -} { - let captured: string | null = null; - const providers = createCredentialProviderRegistry([ - createHttpCredentialProvider({ - fetch: async (_input, init) => { - captured = (init?.headers as Headers | undefined)?.get("authorization") ?? null; - return new Response("{}", { status: 200 }); - }, - }), - ]); - return { providers, lastAuthHeader: () => captured }; -} +import { + attachStepTools, + createToolBearingAgentFactory, + deriveToolMarkFloorGrants, + rewrapStepToolFactory, + stepDeployTreeDir, + type StepToolMaterialization, +} from "./step-agent-tools"; -function delivery(secret: string): CredentialDelivery { - return { - bindings: [{ handle: "granola", credentialId: CREDENTIAL_ID, consumer: CONSUMER }], - materials: [ - { - credentialId: CREDENTIAL_ID, - providerKey: "http", - origin: "https://api.granola.ai", - secret, - }, - ], - }; +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((d) => fs.promises.rm(d, { recursive: true, force: true })), + ); +}); + +async function tempDir(): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "step-agent-tools-test-")); + tempDirs.push(d); + return d; } -function useGrant(consumer: string): GrantRule { +const SOURCE: InferenceSource = { + id: "anthropic:mock-model", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-test", + model: "mock-model", +}; + +async function buildStepEnv(): Promise { + const dir = await tempDir(); + const workdir = path.join(dir, "workspace"); + await fs.promises.mkdir(workdir, { recursive: true }); + const storage = await createIsogitStore(dir); return { - id: "grant_1", - resource: `credential:${CREDENTIAL_ID}`, - action: "use", - effect: "allow", - origin: "system", - conditions: { tool: consumer }, - expiresAt: null, - roleId: null, - principalId: null, + sources: [SOURCE], + defaultSource: SOURCE.id, + storage, + workdir, + audit: noopAuditStore(), + authorize: async () => ({ + effect: "allow", + matchingGrants: [], + resolvedBy: null, + }), + directors: createDefaultDirectorRegistry(), }; } -function wiring( - current: CredentialDelivery | null, - grants: readonly GrantRule[] = [useGrant(CONSUMER)], -): CredentialWiring { - return { - materialRef: { current }, - resolveStepGrants: () => grants, - }; +/** + * Returns `true` if a process with the given pid is alive. `kill(pid, 0)` + * throws ESRCH when the process does not exist and EPERM when it exists + * but is not signalable by this user; either non-throw / EPERM means + * "alive", ESRCH means "gone". + */ +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + if (err instanceof Error && "code" in err && err.code === "EPERM") { + return true; + } + return false; + } } -describe("consumerBindings", () => { - test("derives the consumer's bound handle from a fake CredentialWiring's live delivery", () => { - const context: StepCredentialContext = { - wiring: wiring(delivery("s3cr3t")), - stepId: STEP_ID, +describe("createToolBearingAgentFactory plugin/LSP lifecycle", () => { + test("agent.close() runs the plugin disposer and tears down its subprocess", async () => { + // A plugin standing in for the LSP plugin: its factory spawns a REAL + // subprocess (a sleeping shell) and its `dispose` kills it. This is + // the same shape `createLSPPlugin` produces -- a `ToolPlugin` whose + // `dispose` terminates a server subprocess -- minus the lazy spawn. + let spawnedPid: number | undefined; + let disposeCalls = 0; + const lspLikePlugin = definePlugin({ + id: "@intx/tools-lsp-fake/sidecar-bundle", + factory: () => { + const proc = Bun.spawn(["sleep", "120"], { + stdout: "ignore", + stderr: "ignore", + }); + spawnedPid = proc.pid; + return { + tools: [], + dispose: () => { + disposeCalls += 1; + proc.kill(); + }, + }; + }, + }); + + // A trivial tool factory so the agent has at least one tool bundle. + const noopTool = defineTool({ + id: "@intx/test-tool/sidecar-bundle", + requires: [], + definitions: [], + factory: (): ToolBundle => ({ + definitions: [], + run: (call) => Promise.resolve({ callId: call.id, content: "" }), + }), + }); + + const materialization: StepToolMaterialization = { + factories: [ + { + packageName: "@intx/test-tool", + declaredCredentials: [], + factory: noopTool, + }, + ], + pluginFactories: [lspLikePlugin], }; - const bindings = consumerBindings(context, CONSUMER); - expect([...bindings.keys()]).toEqual(["granola"]); - expect(bindings.get("granola")?.credentialId).toBe(CREDENTIAL_ID); - expect(bindings.get("granola")?.providerKey).toBe("http"); - expect(bindings.get("granola")?.origin).toBe("https://api.granola.ai"); + + const env = await buildStepEnv(); + attachStepTools(env, materialization); + // The step-invoker adapter spreads the env (`{ ...envBase, authorize }`) + // before handing it to the agent factory; replicate that spread so the + // test exercises the symbol-slot-survives-spread path. + const spreadEnv: BaseEnv = { ...env }; + + const agentFactory = createToolBearingAgentFactory(); + const def = { + id: "agent-lsp-lifecycle", + systemPrompt: "lsp lifecycle test", + toolFactories: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, + } as const; + + const agent = await agentFactory(def, spreadEnv); + + // The plugin factory ran during agent build: its subprocess is live. + if (spawnedPid === undefined) { + throw new Error("plugin factory did not spawn a subprocess"); + } + expect(isAlive(spawnedPid)).toBe(true); + expect(disposeCalls).toBe(0); + + // Closing the agent must run the plugin disposer, which kills the + // subprocess. This is the exact call the step-invoker adapter makes + // in its `finally`. + await agent.close(); + + expect(disposeCalls).toBe(1); + // The process is reaped; allow a brief moment for the OS to reflect + // the kill. + let alive = isAlive(spawnedPid); + for (let i = 0; i < 50 && alive; i += 1) { + await new Promise((r) => setTimeout(r, 20)); + alive = isAlive(spawnedPid); + } + expect(alive).toBe(false); }); - test("filters out a binding delivered for a different consumer", () => { - const context: StepCredentialContext = { - wiring: wiring(delivery("s3cr3t")), - stepId: STEP_ID, + test("a plugin factory that throws mid-chain disposes the already-built plugins", async () => { + // Mirrors `default-harness.ts`'s plugin-construction rollback: if a + // later plugin factory throws, every earlier plugin instance must be + // disposed so a partial-success chain does not leak (the LSP server + // subprocess being the resource that would leak in production). + let spawnedPid: number | undefined; + let disposed = false; + const firstPlugin = definePlugin({ + id: "@intx/first-plugin/sidecar-bundle", + factory: () => { + const proc = Bun.spawn(["sleep", "120"], { + stdout: "ignore", + stderr: "ignore", + }); + spawnedPid = proc.pid; + return { + tools: [], + dispose: () => { + disposed = true; + proc.kill(); + }, + }; + }, + }); + const throwingPlugin = definePlugin({ + id: "@intx/throwing-plugin/sidecar-bundle", + factory: () => { + throw new Error("plugin construction failure"); + }, + }); + + const materialization: StepToolMaterialization = { + factories: [], + pluginFactories: [firstPlugin, throwingPlugin], }; - expect(consumerBindings(context, OTHER_CONSUMER).size).toBe(0); + + const env = await buildStepEnv(); + attachStepTools(env, materialization); + + const agentFactory = createToolBearingAgentFactory(); + const def = { + id: "agent-plugin-rollback", + systemPrompt: "plugin rollback test", + toolFactories: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, + } as const; + + await expect(agentFactory(def, { ...env })).rejects.toThrow(/plugin construction failure/); + + expect(disposed).toBe(true); + if (spawnedPid === undefined) { + throw new Error("first plugin factory did not spawn a subprocess"); + } + let alive = isAlive(spawnedPid); + for (let i = 0; i < 50 && alive; i += 1) { + await new Promise((r) => setTimeout(r, 20)); + alive = isAlive(spawnedPid); + } + expect(alive).toBe(false); }); - test("is empty when the run carries no credential delivery at all", () => { - const context: StepCredentialContext = { - wiring: wiring(null), - stepId: STEP_ID, + test("agent.close() rejects with an AggregateError when a disposer fails, and still runs the rest", async () => { + // The underlying agent close succeeds; a plugin disposer (standing in + // for the LSP subprocess kill) then throws. That failure must surface + // through close() rather than be swallowed -- a leaked LSP subprocess + // is otherwise invisible -- and one failing disposer must not strand + // the others. + const disposeError = new Error("lsp dispose boom"); + let survivorDisposed = 0; + const throwingPlugin = definePlugin({ + id: "@intx/throwing-disposer/sidecar-bundle", + factory: () => ({ + tools: [], + dispose: () => { + throw disposeError; + }, + }), + }); + const survivorPlugin = definePlugin({ + id: "@intx/survivor-disposer/sidecar-bundle", + factory: () => ({ + tools: [], + dispose: () => { + survivorDisposed += 1; + }, + }), + }); + + const materialization: StepToolMaterialization = { + factories: [], + pluginFactories: [throwingPlugin, survivorPlugin], }; - expect(consumerBindings(context, CONSUMER).size).toBe(0); - }); - test("is empty when the step's env carries no credential wiring at all", () => { - expect(consumerBindings(undefined, CONSUMER).size).toBe(0); + const env = await buildStepEnv(); + attachStepTools(env, materialization); + + const agentFactory = createToolBearingAgentFactory(); + const def = { + id: "agent-disposer-aggregate", + systemPrompt: "disposer aggregate test", + toolFactories: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, + } as const; + + const agent = await agentFactory(def, { ...env }); + + let thrown: unknown; + try { + await agent.close(); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(AggregateError); + if (!(thrown instanceof AggregateError)) { + throw new Error("expected an AggregateError from a failing teardown"); + } + expect(thrown.errors).toContain(disposeError); + // The loop kept going after the first disposer threw: the other ran. + expect(survivorDisposed).toBe(1); }); - test("readCurrentMaterial re-reads the live materialRef, reflecting a rotation", () => { - const ref: { current: CredentialDelivery | null } = { - current: delivery("original-secret"), - }; - const context: StepCredentialContext = { - wiring: { - materialRef: ref, - resolveStepGrants: () => [useGrant(CONSUMER)], + test("a disposer that throws during construction rollback does not mask the construction error", async () => { + // On a construction-failure rollback the pending construction error is + // the one worth surfacing; a disposer failure during that rollback is + // logged but must never replace it. + const firstPlugin = definePlugin({ + id: "@intx/rollback-throwing-disposer/sidecar-bundle", + factory: () => ({ + tools: [], + dispose: () => { + throw new Error("rollback dispose boom"); + }, + }), + }); + const throwingPlugin = definePlugin({ + id: "@intx/rollback-construction-throw/sidecar-bundle", + factory: () => { + throw new Error("plugin construction failure"); }, - stepId: STEP_ID, + }); + + const materialization: StepToolMaterialization = { + factories: [], + pluginFactories: [firstPlugin, throwingPlugin], }; - const binding = consumerBindings(context, CONSUMER).get("granola"); - if (binding === undefined) throw new Error("expected a granola binding"); - expect(binding.readCurrentMaterial().secret).toBe("original-secret"); - ref.current = delivery("rotated-secret"); - expect(binding.readCurrentMaterial().secret).toBe("rotated-secret"); + const env = await buildStepEnv(); + attachStepTools(env, materialization); + + const agentFactory = createToolBearingAgentFactory(); + const def = { + id: "agent-rollback-dispose-mask", + systemPrompt: "rollback dispose mask test", + toolFactories: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, + } as const; + + await expect(agentFactory(def, { ...env })).rejects.toThrow(/plugin construction failure/); }); -}); -describe("createCredentialCapability composed over a fake CredentialWiring", () => { - test("a bound handle with a matching grant resolves a mediated credential the tool can use to reach the secret", async () => { - const context: StepCredentialContext = { - wiring: wiring(delivery("s3cr3t")), - stepId: STEP_ID, + test("a second close() after a failed teardown neither re-runs disposers nor rethrows", async () => { + let disposeCalls = 0; + const throwingPlugin = definePlugin({ + id: "@intx/double-close-disposer/sidecar-bundle", + factory: () => ({ + tools: [], + dispose: () => { + disposeCalls += 1; + throw new Error("dispose boom"); + }, + }), + }); + + const materialization: StepToolMaterialization = { + factories: [], + pluginFactories: [throwingPlugin], }; - const { providers, lastAuthHeader } = providersWithCapturedAuth(); - const capability = createCredentialCapability({ - consumer: CONSUMER, - bindings: consumerBindings(context, CONSUMER), - providers, - grants: [...context.wiring.resolveStepGrants(STEP_ID)] as GrantRule[], - }); - - const mediated = await capability.resolve("granola"); - expect(mediated.kind).toBe("http"); - await mediated.fetch("https://api.granola.ai/v1/notes"); - expect(lastAuthHeader()).toBe("Bearer s3cr3t"); - await capability.dispose(); + + const env = await buildStepEnv(); + attachStepTools(env, materialization); + + const agentFactory = createToolBearingAgentFactory(); + const def = { + id: "agent-double-close", + systemPrompt: "double close test", + toolFactories: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "mock-model" }] }, + } as const; + + const agent = await agentFactory(def, { ...env }); + + await expect(agent.close()).rejects.toBeInstanceOf(AggregateError); + expect(disposeCalls).toBe(1); + // wrapAgentClose guards teardown behind `tornDown`, so a second close + // is a no-op: it neither re-runs the disposer nor re-surfaces the + // failure. + await agent.close(); + expect(disposeCalls).toBe(1); + }); +}); + +describe("stepDeployTreeDir base-step resolution", () => { + const dataDir = "/data"; + const mailboxAddress = "run_dep-map@example.com"; + + test("a map iteration resolves the base step's deploy tree", () => { + // Deploy stages one deploy tree per base step; every map iteration + // `[]` must read that one tree, not a per-iteration address + // that was never staged. + const base = stepDeployTreeDir({ + dataDir, + mailboxAddress, + stepId: "summarize", + stepCount: 2, + }); + const iter0 = stepDeployTreeDir({ + dataDir, + mailboxAddress, + stepId: "summarize[0]", + stepCount: 2, + }); + const iter1 = stepDeployTreeDir({ + dataDir, + mailboxAddress, + stepId: "summarize[1]", + stepCount: 2, + }); + expect(iter0).toBe(base); + expect(iter1).toBe(base); }); - test("an unbound handle -- absent CredentialWiring -- rejects rather than silently resolving", async () => { - const { providers } = providersWithCapturedAuth(); - const capability = createCredentialCapability({ - consumer: CONSUMER, - bindings: consumerBindings(undefined, CONSUMER), - providers, - grants: [], + test("distinct base steps still resolve distinct deploy trees", () => { + const a = stepDeployTreeDir({ + dataDir, + mailboxAddress, + stepId: "alpha[0]", + stepCount: 2, + }); + const b = stepDeployTreeDir({ + dataDir, + mailboxAddress, + stepId: "beta[0]", + stepCount: 2, }); + expect(a).not.toBe(b); + }); +}); - await expect(capability.resolve("granola")).rejects.toThrow(/no credential is bound to handle/); +describe("rewrapStepToolFactory", () => { + test("preserves the source factory's static definitions on the re-wrap", () => { + const source = defineTool({ + id: "@intx/test-tool/sidecar-bundle", + requires: ["transport"], + definitions: [{ name: "alpha" }, { name: "beta" }], + factory: (): ToolBundle => ({ + definitions: [], + run: (call) => Promise.resolve({ callId: call.id, content: "" }), + }), + }); + + // This test only inspects the re-wrapped factory's static metadata; + // it never invokes the factory, so the disposer-capture callback + // must not fire. + const rewrapped = rewrapStepToolFactory( + source, + () => { + throw new Error("onDispose must not be called: factory is not invoked"); + }, + undefined, + ); + + expect(rewrapped.definitions).toEqual(source.definitions); + expect(rewrapped.id).toBe(source.id); + expect(rewrapped.requires).toEqual(source.requires); }); - test("a bound handle with no matching grant fails closed even though the binding exists", async () => { - const context: StepCredentialContext = { - wiring: wiring(delivery("s3cr3t"), [useGrant(OTHER_CONSUMER)]), - stepId: STEP_ID, + test("layers the given credentials capability onto the bundle's env", async () => { + let seenEnv: BaseEnv | undefined; + const source = defineTool({ + id: "@intx/test-tool/sidecar-bundle", + requires: ["capabilities"], + definitions: [], + factory: (factoryEnv): ToolBundle => { + seenEnv = factoryEnv; + return { + definitions: [], + run: (call) => Promise.resolve({ callId: call.id, content: "" }), + }; + }, + }); + const capability: HostCredentialCapability = { + resolve: () => Promise.reject(new Error("resolve unused in this test")), + dispose: () => Promise.resolve(), }; - const { providers } = providersWithCapturedAuth(); - const capability = createCredentialCapability({ - consumer: CONSUMER, - bindings: consumerBindings(context, CONSUMER), - providers, - grants: [...context.wiring.resolveStepGrants(STEP_ID)] as GrantRule[], + + const rewrapped = rewrapStepToolFactory( + source, + () => { + /* the bundle returns no disposer, so this never fires */ + }, + capability, + ); + + const env = await buildStepEnv(); + // The base bag buildEnv would set; an empty one suffices to prove the + // credentials key is layered on top of it for this bundle. + Reflect.set(env, "capabilities", createRuntimeCapabilities({})); + rewrapped(env); + + if (seenEnv === undefined) { + throw new Error("the bundle factory was not invoked"); + } + // The bundle sees a layered bag resolving THIS package's capability -- the + // seam that must never hand a bundle another package's capability. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the slot is the RuntimeCapabilities resolver this test set + const bag = Reflect.get(seenEnv, "capabilities") as RuntimeCapabilities; + expect(bag.resolve("credentials")).toBe(capability); + }); + + test("passes the base env through unchanged when no capability is layered", async () => { + let seenEnv: BaseEnv | undefined; + const source = defineTool({ + id: "@intx/test-tool/sidecar-bundle", + requires: ["capabilities"], + definitions: [], + factory: (factoryEnv): ToolBundle => { + seenEnv = factoryEnv; + return { + definitions: [], + run: (call) => Promise.resolve({ callId: call.id, content: "" }), + }; + }, }); - await expect(capability.resolve("granola")).rejects.toThrow(/not authorized to use credential/); + const rewrapped = rewrapStepToolFactory( + source, + () => { + /* no disposer captured in this test */ + }, + undefined, + ); + + const env = await buildStepEnv(); + Reflect.set(env, "capabilities", createRuntimeCapabilities({})); + rewrapped(env); + + if (seenEnv === undefined) { + throw new Error("the bundle factory was not invoked"); + } + // No capability -> the exact base env is passed through, no credentials key + // layered on, so a resolve fails closed as not-provided. + expect(seenEnv).toBe(env); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the slot is the RuntimeCapabilities resolver this test set + const bag = Reflect.get(seenEnv, "capabilities") as RuntimeCapabilities; + expect(() => bag.resolve("credentials")).toThrow(); + }); +}); + +describe("deriveToolMarkFloorGrants", () => { + // A loaded tool factory carrying only the static metadata the floor + // deriver reads (`definitions`), shaped like the pinned factories the + // child's loader hands back. Never invoked here. + function loadedFactory(id: string, definitions: readonly ToolDeclaration[]): LoadedToolFactory { + const factory: AnnotatedToolFactory = Object.assign( + (_env: BaseEnv) => ({ + definitions: [], + run: () => Promise.resolve({ callId: "", content: "", isError: false as const }), + }), + { id, requires: [] as readonly string[], definitions }, + ); + return factory; + } + + // Mirror the sidecar's grant evaluator merge: the credentials snapshot's + // grants plus the derived floor, resolved via `evaluateGrants`. This is + // the exact composition the `evaluateGrantsAdapter` performs. + async function resolveWithFloor( + toolName: string, + factories: readonly LoadedToolFactory[], + snapshotGrants: readonly GrantRule[], + ): Promise<"allow" | "deny" | "ask" | null> { + const floor = deriveToolMarkFloorGrants(factories); + const result = await evaluateGrants( + [...snapshotGrants, ...floor], + `tool:${toolName}`, + "invoke", + ); + return result.effect; + } + + test("a pinned ask-marked tool resolves to ask on its floor alone", async () => { + const factories = [ + loadedFactory("@intx/tools-posix/sidecar-bundle", [{ name: "run_shell", approval: "ask" }]), + ]; + expect(await resolveWithFloor("run_shell", factories, [])).toBe("ask"); + }); + + test("a pinned unmarked tool resolves to allow on its floor alone", async () => { + const factories = [loadedFactory("@intx/tools-mail/sidecar-bundle", [{ name: "mail_send" }])]; + expect(await resolveWithFloor("mail_send", factories, [])).toBe("allow"); + }); + + test("a declared deny still beats the derived floor", async () => { + const factories = [ + loadedFactory("@intx/tools-posix/sidecar-bundle", [{ name: "run_shell", approval: "ask" }]), + ]; + // A credentials-snapshot grant explicitly denying the tool at equal + // specificity. `deny` (priority 2) outranks the derived `ask`, so the + // floor never overrides an explicit denial. + const declaredDeny: GrantRule = { + id: "declared-deny", + resource: "tool:run_shell", + action: "invoke", + effect: "deny", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; + expect(await resolveWithFloor("run_shell", factories, [declaredDeny])).toBe("deny"); + }); + + test("an allow grant at equal specificity does not drop the ask floor", async () => { + const factories = [ + loadedFactory("@intx/tools-posix/sidecar-bundle", [{ name: "run_shell", approval: "ask" }]), + ]; + // A run grant explicitly ALLOWING the tool at the SAME specificity as the + // derived floor (both `tool:run_shell`/`invoke`, exact match). `ask` + // (priority 1) outranks `allow` (priority 0) at equal specificity, so the + // floor holds -- a workflow cannot declare its way below a tool's + // approval gate. This is the `ask > allow` half of the effect ordering in + // packages/authz/src/evaluate.ts. + const declaredAllow: GrantRule = { + id: "declared-allow", + resource: "tool:run_shell", + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; + expect(await resolveWithFloor("run_shell", factories, [declaredAllow])).toBe("ask"); + }); + + test("a broader allow glob loses to the exact ask floor on specificity", async () => { + const factories = [ + loadedFactory("@intx/tools-posix/sidecar-bundle", [{ name: "run_shell", approval: "ask" }]), + ]; + // A run grant allowing EVERY tool via a `tool:*` glob. The glob is far + // less specific than the exact `tool:run_shell` floor, so specificity -- + // ranked before effect -- resolves to the exact floor regardless of + // effect. The exact `ask` wins; the broad `allow` never enters the effect + // tie-break. + const broadAllow: GrantRule = { + id: "broad-allow", + resource: "tool:*", + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; + expect(await resolveWithFloor("run_shell", factories, [broadAllow])).toBe("ask"); }); }); diff --git a/apps/sidecar/src/step-agent-tools.ts b/apps/sidecar/src/step-agent-tools.ts index ab1806e050..e61b3b3aea 100644 --- a/apps/sidecar/src/step-agent-tools.ts +++ b/apps/sidecar/src/step-agent-tools.ts @@ -11,7 +11,7 @@ // `AgentDefinition`, and returns an `Agent` whose `close()` tears every // plugin and tool bundle down. // -// LSP lifecycle (the riskiest sub-item): the LSP plugin +// LSP lifecycle (the riskiest sub-item, design §6): the LSP plugin // factory's `dispose` terminates the LSP subprocess. The // `createWorkflowStepInvoker` adapter calls `agent.close()` in a // `finally` on EVERY exit path (clean completion, abort, rejection), @@ -20,7 +20,7 @@ // no leak across steps, on abort, or on child recycle/drain (recycle // kills the child process, which kills the LSP grandchild regardless). // -// Layering: everything here lives in `apps/sidecar`. The +// Layering (design §3d): everything here lives in `apps/sidecar`. The // portable `@intx/workflow-host` package never gains a dependency on // the tool runtime; it only sees the `agentFactory` callback this // module produces. @@ -31,28 +31,32 @@ import { createAgent, defineAgent, defineTool, + toolApprovalEffect, type Agent, type AgentDefinition, type AnnotatedPluginFactory, + type AnnotatedToolFactory, type BaseEnv, type ToolBundle, } from "@intx/agent"; -import { toolConsumer, type GrantRule } from "@intx/authz"; -import { - createCredentialCapability, - type CredentialProviderRegistry, - type HostCredentialCapability, - type ResolvedCredentialBinding, -} from "@intx/harness"; import { readDeployTree, agentDir } from "@intx/hub-agent/paths"; import { getLogger } from "@intx/log"; -import type { LoadedToolFactory, RegistryConfig } from "@intx/tool-packaging"; +import type { HostCredentialCapability } from "@intx/harness"; +import type { LoadedToolFactory } from "@intx/tool-packaging"; import { resolveStepAddress } from "@intx/workflow-deploy"; import { parseRunAddress } from "@intx/types"; -import type { CredentialMaterialResolver } from "@intx/types"; -import type { CredentialWiring } from "@intx/workflow-host"; +import type { GrantRule } from "@intx/types/authz"; +import { + layerRuntimeCapabilities, + type RuntimeCapabilities, +} from "@intx/types/runtime-capabilities"; +import { baseStepId } from "@intx/workflow"; -import { materializeToolPackages } from "./tool-materialization"; +import { + buildCredentialCapabilities, + type StepCredentialWiring, +} from "./step-credential-capabilities"; +import { materializeToolPackages, type StepToolFactory } from "./tool-materialization"; const logger = getLogger(["sidecar", "workflow-child", "step-tools"]); @@ -76,10 +80,56 @@ export interface StepToolCacheConfig { * can cooperate without widening the portable adapter's surface. */ export interface StepToolMaterialization { - readonly factories: readonly LoadedToolFactory[]; + readonly factories: readonly StepToolFactory[]; readonly pluginFactories: readonly AnnotatedPluginFactory[]; } +/** + * Derive the per-step tool-mark floor grants from a step's materialized + * tool factories. Each loaded factory carries its static + * `definitions` (name + optional `approval` mark) verbatim from the tool + * package; every declared tool contributes a `tool:` / `invoke` + * grant whose effect is the tool's floor (`ask` for an approval-gated + * tool, `allow` otherwise), computed through the same + * `toolApprovalEffect` mapping the deploy-time capability walk uses so a + * pinned tool floors identically hub-side and sidecar-side. + * + * The hub's capability walk reads only INLINE `agent.toolFactories`, so a + * tool that ships as a pinned package loads in the child and never + * produces a `tool:` grant on the run principal. These derived rows + * supply that missing floor: they join the per-step grants the authz path + * evaluates as ADDITIONAL rows, so `evaluateGrants` precedence still + * resolves an explicit `deny` (priority 2) over the derived `ask`/`allow` + * -- the floor only raises the minimum authority to the tool's mark, it + * never overrides a declared denial. + * + * The grant `id` is deterministic (`floor:tool:`) rather than + * random: `evaluateGrants` never dedupes or joins on `id` (it ranks by + * specificity then effect), so a stable id keeps the rows reproducible + * without a generator, and a floor row that coincides with a + * hub-supplied `tool:` row resolves by effect precedence regardless + * of the ids. + */ +export function deriveToolMarkFloorGrants(factories: readonly LoadedToolFactory[]): GrantRule[] { + const rows: GrantRule[] = []; + for (const factory of factories) { + for (const definition of factory.definitions) { + rows.push({ + id: `floor:tool:${definition.name}`, + resource: `tool:${definition.name}`, + action: "invoke", + effect: toolApprovalEffect(definition), + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }); + } + } + return rows; +} + /** * Symbol-keyed slot the `buildEnv` callback sets on the env it returns * and the `agentFactory` reads. Object spread (`{ ...envBase, @@ -121,89 +171,78 @@ function isStepToolMaterialization(value: unknown): value is StepToolMaterializa } /** - * The per-step credential inputs `buildEnv` carries to the tool-bearing - * `agentFactory` via a second symbol-keyed slot, mirroring `STEP_TOOLS`: - * the live `CredentialWiring` the child's runtime built (materialRef + - * grant resolver) plus this invocation's `stepId`, which - * `resolveStepGrants` is keyed on. `attachStepCredentials` is called from - * `createSidecarStepBuildEnv` for a tool-bearing step; a toolless step - * (an onTrigger body) never calls it, so the slot is absent there and no - * tool ever asks for a credential. + * Symbol-keyed slot carrying the per-step credential wiring the + * `agentFactory` assembles each bundle's consumer-scoped `credentials` + * capability from: the live material cell, the step's grants, and the + * provider registry. Set by `buildEnv` alongside the tool slot, read by + * `createToolBearingAgentFactory`. Absent for a build that threaded no + * credential context (a unit test using the bare factory) -- then no + * credentials capability is assembled and bundles keep the base capabilities + * bag. */ -export interface StepCredentialContext { - readonly wiring: CredentialWiring; - readonly stepId: string; -} - -const STEP_CREDENTIALS = Symbol("intx.sidecar.step-credentials"); +const STEP_CREDENTIAL_WIRING = Symbol("intx.sidecar.step-credential-wiring"); -function isStepCredentialContext(value: unknown): value is StepCredentialContext { - return typeof value === "object" && value !== null && "wiring" in value && "stepId" in value; +function setStepCredentialWiring(env: object, wiring: StepCredentialWiring): void { + Reflect.set(env, STEP_CREDENTIAL_WIRING, wiring); } -/** - * Attach this step's credential wiring to the per-step env so the - * tool-bearing `agentFactory` can shape a consumer-scoped `credentials` - * capability for each tool package that declares one. Called by - * `buildEnv` once the `CredentialWiring` the child's runtime carries is - * known. - */ -export function attachStepCredentials(env: object, context: StepCredentialContext): void { - Reflect.set(env, STEP_CREDENTIALS, context); -} - -/** - * Read back the credential context `attachStepCredentials` set on the - * per-step env. - */ -export function getStepCredentialContext(env: object): StepCredentialContext | undefined { - const value: unknown = Reflect.get(env, STEP_CREDENTIALS); +function getStepCredentialWiring(env: object): StepCredentialWiring | undefined { + const value: unknown = Reflect.get(env, STEP_CREDENTIAL_WIRING); if (value === undefined) return undefined; - if (!isStepCredentialContext(value)) { + if (!isStepCredentialWiring(value)) { throw new Error( - "sidecar workflow-child step tools: the per-step env's credentials slot is not a StepCredentialContext; the slot is private to this module and must only be set by attachStepCredentials", + "sidecar workflow-child step tools: the per-step env's credential-wiring slot is not a StepCredentialWiring; the slot is private to this module and must only be set by attachStepCredentialWiring", ); } return value; } +function isStepCredentialWiring(value: unknown): value is StepCredentialWiring { + return ( + typeof value === "object" && + value !== null && + "materialCell" in value && + "resolveGrants" in value && + "providers" in value + ); +} + /** - * The package portion of a package-namespaced tool id - * (`@vendor/pkg/name` or `pkg/name`), i.e. the id with its trailing - * `/name` segment removed. This is the same string a tool package's - * `package.json` `name` field carries, which is what launch-time - * credential binding (`toolConsumer`, `vendor/intx/db/src/ - * credential-resolution.ts`) keys the consumer identity on -- so a tool - * factory's `id` and its declared credential handles resolve to the - * same consumer here without a second source of truth. - * - * Trust boundary: this derives the consumer identity from the tool - * bundle's OWN self-declared `id` string (`defineTool({ id, ... })`, - * `@intx/agent/src/tool.ts`) -- nothing in the loader checks that a - * factory whose `id` claims package X actually shipped inside package - * X's tarball. A factory could declare `id: "@corbits/granola-tools/ - * granola"` from inside a different package's code and this function - * would hand it that package's `credentials` capability, resolving - * whatever the launch bound to the real `@corbits/granola-tools` - * consumer. This is acceptable ONLY because tool packages are - * operator-installed, root-bucket-trusted code (AGENTS.md: "Root-bucket - * modules are operator-installed... sandboxed installables use - * Interchange's native contracts and never get root-bucket powers") -- - * the same trust level a root-bucket module already holds to declare - * routes, migrations, and grants. It is NOT a boundary that holds against - * a hostile or sandboxed tool package, which this substrate never loads. - * Binding the loader's package-provenance to a factory's declared `id` is - * a real gap for a future untrusted-tool-package story; it is filed and - * tracked separately from this wiring, not solved here. + * Attach a step's credential wiring to the per-step env so the tool-bearing + * `agentFactory` can assemble each bundle's `credentials` capability. Called + * by `buildEnv` for a step whose build threaded a credential context; omitted + * otherwise, which leaves the slot unset and the credentials capability + * unassembled. */ -export function packageFromToolId(id: string): string { - const lastSlash = id.lastIndexOf("/"); - if (lastSlash <= 0) { +export function attachStepCredentialWiring( + env: Omit, + wiring: StepCredentialWiring, +): void { + setStepCredentialWiring(env, wiring); +} + +/** + * Read the base `RuntimeCapabilities` bag `buildEnv` set on the per-step env + * (`env.capabilities`, currently `mail.transport`). `BaseEnv` does not type + * the key -- it is a runtime-only widening the step env carries -- so it is + * read reflectively and validated. Throws when a bundle has a credentials + * capability to layer but the env carries no base bag: that is a wiring + * inconsistency (the same `buildEnv` sets both), not a condition to paper over. + */ +function requireCapabilitiesBag(env: BaseEnv): RuntimeCapabilities { + const value: unknown = Reflect.get(env, "capabilities"); + if ( + typeof value !== "object" || + value === null || + !("resolve" in value) || + typeof (value as { resolve: unknown }).resolve !== "function" + ) { throw new Error( - `sidecar workflow-child step tools: tool id ${JSON.stringify(id)} is not package-namespaced`, + "sidecar workflow-child step tools: a credentials capability was assembled for this bundle but the per-step env carries no base capabilities bag to layer it onto; buildEnv must set env.capabilities before assembling credentials", ); } - return id.slice(0, lastSlash); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated structurally above: an object whose `resolve` is callable is the RuntimeCapabilities resolver buildEnv set + return value as RuntimeCapabilities; } /** @@ -213,19 +252,19 @@ export function packageFromToolId(id: string): string { * `deploy/asset-mounts.json`) is shipped to the sidecar per step by the * hub's `launchSession` deploy-pack push, which lands it in the LEGACY * per-agent directory keyed by the step's sanitized mail address (see - * `@intx/hub-agent` `agentDir`). It is NOT in the - * substrate's `agent-state/` layout -- the multi-step deploy path - * never pushes step `agent-state` packs to the child's substrate. + * `@intx/hub-agent` `agentDir`). It is NOT in the substrate's + * `agent-state/` layout -- the multi-step deploy path never pushes + * step `agent-state` packs to the child's substrate. * * The step's mail address is `resolveStepAddress(...)`, the single owner * of the head/step collapse: for a single-step deployment the lone step * IS the head (the deployment mailbox itself), so the tree is read at the - * head; for multi-step it is the derived `-@` step - * address. The anchor `runId`/`domain` are recovered from the deployment - * mailbox address the supervisor threaded into the child as - * `MAILBOX_ADDRESS` (`@`). `stepCount` is sourced from - * the host (via `substrateEnv`) so producer and consumer never derive - * divergent addresses. + * head; for multi-step it is `deriveStepAddress(runId, stepId, + * domain)`. The `runId`/`domain` are recovered from the deployment mailbox + * address the supervisor threaded into the child as `MAILBOX_ADDRESS` + * (`@`): the local part is the run id and the address domain + * is the deployment domain. `stepCount` is sourced from the host (via + * `substrateEnv`) so producer and consumer never derive divergent addresses. */ export function stepDeployTreeDir(args: { dataDir: string; @@ -239,9 +278,14 @@ export function stepDeployTreeDir(args: { `sidecar workflow-child step tools: deployment mailbox address ${JSON.stringify(args.mailboxAddress)} is not a parseable run address; cannot locate the step's deploy tree`, ); } + // A `map` iteration runs under a scoped step id `[]`, but + // deploy stages one deploy tree per base step, so the scoped id resolves + // to its base address -- every iteration reads the base step's tree. + // `baseStepId` is the identity on an unscoped id, so a plain step is + // unaffected. const stepAddress = resolveStepAddress({ runId: parsed.runId, - stepId: args.stepId, + stepId: baseStepId(args.stepId), domain: parsed.domain, stepCount: args.stepCount, }); @@ -251,7 +295,7 @@ export function stepDeployTreeDir(args: { /** * Read a step's deploy tree and materialize its pinned tool-package * closure. The tarball cache and the tool instance dir are rooted under - * the supplied per-step `storeDir` (the per-step state root) so + * the supplied per-step `storeDir` (the Phase-1 per-step state root) so * concurrent steps/agents in one child never collide on cache or * apply-state paths. * @@ -269,13 +313,6 @@ export async function materializeStepTools(args: { /** Per-step state root; cache + instance dir + workspace live under it. */ storeDir: string; cache: StepToolCacheConfig; - /** - * Tool-package registries the loader resolves against, parsed from - * the boot-edge-threaded `SIDECAR_TOOL_REGISTRIES` substrate-config - * entry so a child materializes against the registries the operator - * pinned, never a default of its own. - */ - registries: ReadonlyMap; }): Promise { const deployTreeDir = stepDeployTreeDir({ dataDir: args.dataDir, @@ -288,7 +325,7 @@ export async function materializeStepTools(args: { // Root the tarball cache per step so concurrent steps in one child // do not race on the content-addressable cache root. The cache is // content-addressed and therefore safe to share globally, but the - // design calls for a per-step cacheRoot so a wedged + // design (§3d, point 3) calls for a per-step cacheRoot so a wedged // or partially-written apply in one step cannot corrupt another's // view. const cacheRoot = path.join(args.storeDir, "tarball-cache"); @@ -298,6 +335,12 @@ export async function materializeStepTools(args: { // tree lives in), not under the per-step store dir. Point the loader's // asset resolution there while keeping the apply-state + cache rooted // per step under `storeDir`. + // + // This `/workspace` (read-only staged assets, keyed by the + // BASE step) and the agent's read-write workdir `/workspace` + // (keyed by the SCOPED step) share a leaf name but are deliberately + // different roots -- a map iteration reads one shared deploy tree while + // each iteration writes its own scratch. Do not unify them. const assetRoot = path.join(deployTreeDir, "workspace"); const materialized = await materializeToolPackages({ @@ -309,7 +352,6 @@ export async function materializeStepTools(args: { cacheRoot, cacheMaxBytes: args.cache.cacheMaxBytes, registryMaxTarballBytes: args.cache.registryMaxTarballBytes, - registries: args.registries, }); return { factories: materialized.factories, @@ -334,6 +376,68 @@ export function attachStepTools( setStepToolSlot(env, materialization); } +/** + * Re-wrap a loaded tool factory so its bundle's `dispose` (when present) + * is captured via `onDispose`, forwarding the loader's `id`, `requires`, + * and `definitions` so the result is a real `AnnotatedToolFactory`, not a + * hand-shaped lookalike. The static `definitions` declaration is + * forwarded verbatim: this wrapper does not rename tools, so the names + * the deploy-time walk enumerates must survive the re-wrap unchanged. + * + * When `credentials` is supplied, the bundle's factory sees a per-bundle + * capabilities bag: the step's base bag layered with THIS package's + * consumer-scoped `credentials` capability, so the bundle resolves only the + * handles its own package is authorized for. When it is absent (the package + * declares and binds no credential), the base bag is passed through unchanged. + * + * Exported so the definitions-preservation contract is testable in + * isolation; the production path calls it from + * `createToolBearingAgentFactory`. + */ +export function rewrapStepToolFactory( + annotated: AnnotatedToolFactory, + onDispose: (dispose: () => unknown) => void, + credentials: HostCredentialCapability | undefined, +): AnnotatedToolFactory { + return defineTool({ + id: annotated.id, + requires: annotated.requires, + definitions: annotated.definitions, + factory: (factoryEnv: BaseEnv): ToolBundle => { + const env = + credentials === undefined ? factoryEnv : layerCredentialsOntoEnv(factoryEnv, credentials); + const bundle = annotated(env); + if (bundle.dispose !== undefined) { + onDispose(bundle.dispose); + } + return bundle; + }, + }); +} + +/** + * Produce a copy of `factoryEnv` whose `capabilities` bag is the base bag + * layered with this bundle's consumer-scoped `credentials` capability. Object + * spread copies own enumerable properties -- including the runtime-only + * `capabilities` key `BaseEnv` does not type and the private tool slot -- so + * every other env surface survives; only `capabilities` is replaced. + */ +function layerCredentialsOntoEnv( + factoryEnv: BaseEnv, + credentials: HostCredentialCapability, +): BaseEnv { + const base = requireCapabilitiesBag(factoryEnv); + const layered = layerRuntimeCapabilities(base, { credentials }); + // Copy every env surface (spread carries own enumerable string AND symbol + // keys, so `transport`, `address`, and the private tool/credential slots + // survive), then replace the runtime-only `capabilities` key via Reflect -- + // a `{ ...factoryEnv, capabilities }` literal trips the excess-property + // check because BaseEnv does not type the key. + const layeredEnv: BaseEnv = { ...factoryEnv }; + Reflect.set(layeredEnv, "capabilities", layered); + return layeredEnv; +} + /** * Build the `agentFactory` the workflow step-invoker uses. The returned * factory reads the materialized tool runtime off the env (set by @@ -347,17 +451,11 @@ export function attachStepTools( * When the env carries no materialized tools (the `buildEnv` did not * run materialization, e.g. a unit test using the bare factory), the * factory falls back to `createAgent(def, env)` unchanged. - * - * `deps.providers` is the fixed `CredentialProviderRegistry` the child - * builds once at boot (`builtinCredentialProviders()`, composed via - * `@intx/harness`'s `createCredentialProviderRegistry`). It is the only - * credential dependency that is NOT per-step: the wiring (materials + - * grants) varies per step and rides the env's credential slot instead - * (see `attachStepCredentials`). */ -export function createToolBearingAgentFactory(deps: { - providers: CredentialProviderRegistry; -}): (def: AgentDefinition, env: EnvReq) => Promise { +export function createToolBearingAgentFactory(): ( + def: AgentDefinition, + env: EnvReq, +) => Promise { return async ( def: AgentDefinition, env: EnvReq, @@ -367,75 +465,58 @@ export function createToolBearingAgentFactory(deps: { return createAgent(def, env); } - // Consumer-scoped `credentials` capabilities for this step's tool - // packages, built lazily (only for a tool factory that actually - // declares `requires: ["credentials"]`) and memoized by consumer so - // two bundles from the same package share one capability -- and one - // `dispose()` -- rather than shaping the same handle twice. - // - // Always returns a capability, never `undefined`: `requires: - // ["credentials"]` makes the runtime's presence-only `validateEnv` - // demand a non-nullish `env.credentials` at agent construction, so - // the "not connected" signal has to live in `resolve()` throwing, - // not in the field's presence. A step with no credential wiring at - // all (`getStepCredentialContext` returns `undefined` -- a toolless - // body step never calls `attachStepCredentials`) or a consumer with - // no bound handle both yield a capability with an empty binding map, - // so every `resolve(handle)` call throws "no credential is bound" - // and the tool reports the same honest not-connected result either - // way. - const credentialContext = getStepCredentialContext(env); - const credentialCapabilities = new Map(); - function credentialCapabilityFor(consumer: string): HostCredentialCapability { - const existing = credentialCapabilities.get(consumer); - if (existing !== undefined) return existing; - const bindings = consumerBindings(credentialContext, consumer); - const capability = createCredentialCapability({ - consumer, - bindings, - providers: deps.providers, - grants: - bindings.size === 0 || credentialContext === undefined - ? [] - : [ - ...(credentialContext.wiring.resolveStepGrants( - credentialContext.stepId, - ) as readonly GrantRule[]), - ], - }); - credentialCapabilities.set(consumer, capability); - return capability; - } + // Assemble each package's consumer-scoped `credentials` capability once, + // when the step carries credential wiring (a bare test build carries none, + // yielding an empty map). Every factory in a package shares the one + // capability; a package that declares a handle no binding resolves fails + // the launch here, loudly, rather than at the tool's first resolve. This + // can throw (reconcile fail-closed, malformed delivery) before any handle + // is shaped, so nothing is orphaned on that path. + const credentialWiring = getStepCredentialWiring(env); + const credentialCapabilities = + credentialWiring === undefined + ? new Map() + : buildCredentialCapabilities(materialization.factories, credentialWiring); // Wrap each loaded tool factory so its bundle's `dispose` (when // present) is captured. Dedupe by closure identity: a factory whose // bundle returns the same `dispose` on every invocation must not be - // torn down once per - // push. `defineTool` re-annotates the wrapper with the loader's - // `id`/`requires` so the resulting factory is a real - // `AnnotatedToolFactory`, not a hand-shaped lookalike. + // torn down once per push. Each package's credentials capability joins + // the same teardown set so its shaped handles are released with the agent. const capturedDisposers = new Set<() => unknown>(); - const factoriesWithCapture = materialization.factories.map((annotated) => { - const consumer = toolConsumer(packageFromToolId(annotated.id)); - return defineTool({ - id: annotated.id, - requires: annotated.requires, - definitions: annotated.definitions, - factory: (factoryEnv: BaseEnv): ToolBundle => { - const credentials = annotated.requires.includes("credentials") - ? credentialCapabilityFor(consumer) - : undefined; - const scopedEnv: BaseEnv & { - credentials?: HostCredentialCapability; - } = credentials !== undefined ? { ...factoryEnv, credentials } : factoryEnv; - const bundle = annotated(scopedEnv); - if (bundle.dispose !== undefined) { - capturedDisposers.add(bundle.dispose); - } - return bundle; + for (const capability of credentialCapabilities.values()) { + capturedDisposers.add(() => capability.dispose()); + } + const factoriesWithCapture = materialization.factories.map((stf) => + rewrapStepToolFactory( + stf.factory, + (dispose) => { + capturedDisposers.add(dispose); }, - }); - }); + credentialCapabilities.get(stf.packageName), + ), + ); + + // Run every captured disposer -- each credentials capability and each tool + // bundle -- guarding each so one failure does not strand the rest. Used on + // the success teardown AND on the construction-failure rollbacks below: the + // credentials capabilities are built before the plugin chain, so a plugin + // or agent build that throws must still release them (an http handle holds + // nothing, but a future key-file/socket handle would leak otherwise). + // Bundle disposers are idempotent, so re-running one `createAgent` already + // disposed on its own failure path is safe. + const runCapturedDisposers = async (): Promise => { + const failures: unknown[] = []; + for (const dispose of capturedDisposers) { + try { + await dispose(); + } catch (cause) { + logger.error`step tool bundle dispose failed: ${cause instanceof Error ? cause.message : String(cause)}`; + failures.push(cause); + } + } + return failures; + }; // Rebuild the def with the materialized tool factories. The // serialized `def.toolFactories` carry only `{ id, requires }` @@ -444,26 +525,16 @@ export function createToolBearingAgentFactory(deps: { // incoming def. `defineAgent` owns the contravariance escape for // the `BaseEnv`-typed loader factories (see its `EnvRequiredByAll` // machinery). - const toolDefBaseConfig = { + const toolDef = defineAgent({ id: def.id, systemPrompt: def.systemPrompt, tools: factoriesWithCapture, capabilities: [...def.capabilities], inference: { sources: [...def.inference.sources] }, - }; - const toolDefConfigWithDescription = - def.description !== undefined - ? { ...toolDefBaseConfig, description: def.description } - : toolDefBaseConfig; - const toolDefConfigWithDirector = - def.director !== undefined - ? { ...toolDefConfigWithDescription, director: def.director } - : toolDefConfigWithDescription; - const toolDefConfig = - def.tags !== undefined - ? { ...toolDefConfigWithDirector, tags: def.tags } - : toolDefConfigWithDirector; - const toolDef = defineAgent(toolDefConfig); + ...(def.description !== undefined ? { description: def.description } : {}), + ...(def.director !== undefined ? { director: def.director } : {}), + ...(def.tags !== undefined ? { tags: def.tags } : {}), + }); // Instantiate plugin factories one at a time so each successive // factory sees the prior plugins' instances on `env.plugins`: @@ -476,7 +547,7 @@ export function createToolBearingAgentFactory(deps: { // subprocess) before the construction error propagates, so a // partial-success chain never leaks an LSP subprocess. const pluginInstances: unknown[] = []; - let chainEnv: BaseEnv & { credentials?: HostCredentialCapability } = env; + let chainEnv: BaseEnv = env; try { for (const factory of materialization.pluginFactories) { const instance = factory(chainEnv); @@ -487,74 +558,42 @@ export function createToolBearingAgentFactory(deps: { }; } } catch (err) { + // Release the credentials capabilities (built above, before the plugin + // chain) and any bundle disposers captured so far, then the plugin + // instances this module owns. + await runCapturedDisposers(); await disposeAll(pluginInstances, "plugin construction rollback"); - await disposeCredentialCapabilities(credentialCapabilities); throw err; } - // `createAgent`'s presence-only `validateEnv` checks every tool - // contributor's declared `requires` keys against THIS shared - // `chainEnv`, not against the per-tool `scopedEnv` each wrapped - // factory above builds for itself when it actually runs -- so a - // `requires: ["credentials"]` tool (e.g. `@corbits/mcp-tools`) - // needs `chainEnv.credentials` to already be non-nullish here, even - // though every such factory's own wrapper unconditionally replaces - // the value with its consumer-scoped capability before building its - // bundle. Without this, `validateEnv` throws `AgentEnvError` before - // any factory ever runs. Which consumer's capability lands on the - // shared env does not matter for correctness -- it is never read - // directly, only overridden -- so the first credential-requiring - // factory's capability satisfies the presence check for all of - // them. - const credentialRequiringFactory = materialization.factories.find((factory) => - factory.requires.includes("credentials"), - ); - if (credentialRequiringFactory !== undefined) { - chainEnv = { - ...chainEnv, - credentials: credentialCapabilityFor( - toolConsumer(packageFromToolId(credentialRequiringFactory.id)), - ), - }; - } - let agent: Agent; try { agent = await createAgent(toolDef, chainEnv); } catch (err) { // `createAgent` disposes the tool bundles it constructed on its - // own failure path, but the plugin instances and credential - // capabilities are this module's to own -- tear them down so a - // failed agent build does not leak the LSP subprocess or a - // shaped credential handle. + // own failure path, but the credentials capabilities and the plugin + // instances are this module's to own -- tear them down so a failed + // agent build does not leak a shaped handle or the LSP subprocess. + await runCapturedDisposers(); await disposeAll(pluginInstances, "agent construction failure"); - await disposeCredentialCapabilities(credentialCapabilities); throw err; } return wrapAgentClose(agent, async () => { - // Tool bundle disposers first (posix's bundle dispose chains - // through to the LSP plugin's `dispose`), then the plugin - // instances directly. Disposing the LSP plugin twice is safe: - // `lsp.dispose()` clears its client set and the posix bundle's - // dispose is idempotent. Running both guarantees the LSP - // subprocess is torn down even for a plugin no tool bundle - // consumed. - // Every disposer runs and failures are collected rather than thrown - // mid-loop, so one failing disposer never strands the rest. A leaked - // or failing LSP subprocess must surface, not be swallowed: any - // collected failure fails the close, so the caller sees it. - const failures: unknown[] = []; - for (const dispose of capturedDisposers) { - try { - await dispose(); - } catch (cause) { - logger.error`step tool bundle dispose failed: ${cause instanceof Error ? cause.message : String(cause)}`; - failures.push(cause); - } - } - failures.push(...(await disposeAll(pluginInstances, "step teardown"))); - await disposeCredentialCapabilities(credentialCapabilities); + // Captured disposers first (each credentials capability, then the tool + // bundles -- posix's bundle dispose chains through to the LSP plugin's + // `dispose`), then the plugin instances directly. Disposing the LSP + // plugin twice is safe: `lsp.dispose()` clears its client set and the + // posix bundle's dispose is idempotent. Running both guarantees the LSP + // subprocess is torn down even for a plugin no tool bundle consumed. + // Both loops run every disposer and collect failures rather than + // throwing mid-loop, so one failing disposer never strands the rest. + // A leaked or failing LSP subprocess must surface, not be swallowed: + // any collected failure fails the close, so the caller sees it. + const failures = [ + ...(await runCapturedDisposers()), + ...(await disposeAll(pluginInstances, "step teardown")), + ]; if (failures.length > 0) { throw new AggregateError( failures, @@ -565,104 +604,6 @@ export function createToolBearingAgentFactory(deps: { }; } -/** - * Release every consumer-scoped credential capability this step shaped. - * Mirrors `disposeAll`'s best-effort policy: one capability's disposal - * failure is logged and swallowed rather than blocking the others or the - * step's own teardown. - */ -async function disposeCredentialCapabilities( - capabilities: ReadonlyMap, -): Promise { - for (const capability of capabilities.values()) { - try { - await capability.dispose(); - } catch (cause) { - logger.warn`step credential capability dispose failed: ${cause instanceof Error ? cause.message : String(cause)}`; - } - } -} - -/** - * Resolve one consumer's bound credential handles from the step's live - * `CredentialDelivery` snapshot. Reads `materialRef.current` fresh (not - * captured at closure-build time) so a `credentials-updated` control - * frame that lands before this step's tools resolve their first handle - * is reflected. `readCurrentMaterial` on each binding likewise re-reads - * the ref per call -- the rotation indirection `createCredentialCapability` - * documents. - * - * A binding whose `credentialId` has no matching `materials` entry is a - * delivery-payload integrity fault (`buildCredentialDelivery` always - * pairs a binding with its material) and throws rather than silently - * dropping the handle. - */ -export function consumerBindings( - context: StepCredentialContext | undefined, - consumer: string, -): ReadonlyMap { - const bindings = new Map(); - if (context === undefined) return bindings; - const delivery = context.wiring.materialRef.current; - if (delivery === null) return bindings; - for (const binding of delivery.bindings) { - if (binding.consumer !== consumer) continue; - const material = delivery.materials.find( - (entry) => entry.credentialId === binding.credentialId, - ); - if (material === undefined) { - throw new Error( - `sidecar workflow-child step tools: credential delivery binds handle ${JSON.stringify(binding.handle)} to credential ${binding.credentialId} with no matching material entry`, - ); - } - bindings.set(binding.handle, { - credentialId: binding.credentialId, - providerKey: material.providerKey, - origin: material.origin, - readCurrentMaterial: () => { - const current = context.wiring.materialRef.current; - const currentMaterial = current?.materials.find( - (entry) => entry.credentialId === binding.credentialId, - ); - if (currentMaterial === undefined) { - throw new Error( - `sidecar workflow-child step tools: credential ${binding.credentialId} material is no longer available (rotated or revoked mid-step)`, - ); - } - return { secret: currentMaterial.secret }; - }, - }); - } - return bindings; -} - -/** - * Resolve an inference source's secret BY `credentialId` from the step's - * live credential cell, the resolver `BaseEnv.readCurrentMaterial` expects. - * - * Inference stopped carrying an inline `apiKey` upstream: an - * `InferenceSource` now names a `credentialId` and the reactor fills the - * secret at send time through this seam, off the same cell tool credentials - * resolve from. Reads `materialRef.current` fresh per call, so a - * `credentials-updated` frame mid-step is picked up, and fails closed when - * the credential was revoked or never delivered rather than dialing out - * unauthenticated. - */ -export function stepInferenceMaterialResolver( - context: StepCredentialContext, -): CredentialMaterialResolver { - return (credentialId) => { - const current = context.wiring.materialRef.current; - const material = current?.materials.find((entry) => entry.credentialId === credentialId); - if (material === undefined) { - throw new Error( - `sidecar workflow-child step ${JSON.stringify(context.stepId)}: inference source credential ${credentialId} has no delivered material (revoked, rotated away, or never delivered)`, - ); - } - return { secret: material.secret }; - }; -} - /** * Return an `Agent` whose `close()` runs the original close and then * the supplied teardown. The teardown runs AFTER the agent's own close diff --git a/apps/sidecar/src/step-credential-capabilities.ts b/apps/sidecar/src/step-credential-capabilities.ts index 0c122b47e3..354aae5031 100644 --- a/apps/sidecar/src/step-credential-capabilities.ts +++ b/apps/sidecar/src/step-credential-capabilities.ts @@ -30,7 +30,11 @@ import { type HostCredentialCapability, type ResolvedCredentialBinding, } from "@intx/harness"; -import type { CredentialMaterialSource } from "@intx/types"; +import type { + CredentialMaterial, + CredentialMaterialResolver, + CredentialMaterialSource, +} from "@intx/types"; import type { GrantRule } from "@intx/types/authz"; import type { CredentialDelivery } from "@intx/types/sidecar"; @@ -206,3 +210,33 @@ function makeReadCurrentMaterial(args: { return { secret: material.secret }; }; } + +/** + * A generic inference credential resolver over the live cell: resolves a + * credential's current secret by `credentialId`, failing closed when the cell is + * empty or the credential is absent (a re-push dropped it -- rotated away or + * revoked). Unlike `makeReadCurrentMaterial` it is keyed by `credentialId` at + * call time -- an inference source's forward-only failover chain carries a + * distinct credential per entry -- and pins no provider/origin: an inference + * request authenticates to the source's own `baseURL` (resolved separately), + * so there is no shaped handle to protect against origin drift. + */ +export function createInferenceCredentialResolver( + cell: CredentialMaterialCell, +): CredentialMaterialResolver { + return (credentialId: string): CredentialMaterial => { + const delivery = cell.current; + if (delivery === null) { + throw new Error( + `inference credential material for ${credentialId} is not available: the delivery cell is empty`, + ); + } + const material = delivery.materials.find((entry) => entry.credentialId === credentialId); + if (material === undefined) { + throw new Error( + `inference credential material for ${credentialId} is no longer delivered: a re-push dropped it (rotated away or revoked)`, + ); + } + return { secret: material.secret }; + }; +} diff --git a/apps/sidecar/src/tool-materialization.test.ts b/apps/sidecar/src/tool-materialization.test.ts index d3facada1a..15ee9aac6e 100644 --- a/apps/sidecar/src/tool-materialization.test.ts +++ b/apps/sidecar/src/tool-materialization.test.ts @@ -34,7 +34,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map(), }); expect(result.factories).toEqual([]); expect(result.pluginFactories).toEqual([]); @@ -53,7 +52,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map(), }); } catch (err) { caught = err; @@ -103,7 +101,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map(), }); } catch (err) { caught = err; @@ -173,7 +170,7 @@ describe("materializeToolPackages — manifest.invalid gate", () => { // The dirty marker must be cleared so the next boot reads the // freshly-recorded id directly, not the stale marker that // pre-dated the successful persist. - let dirtyExists: boolean; + let dirtyExists = false; try { await fs.promises.access(dirtyFile); dirtyExists = true; @@ -200,7 +197,7 @@ describe("materializeToolPackages — manifest.invalid gate", () => { const dirtyFile = `${activeIdFile}.dirty`; await fs.promises.writeFile(dirtyFile, "v1:stale-from-prior-apply"); await clearDirtyMarker(activeIdFile, "test"); - let dirtyExists: boolean; + let dirtyExists = false; try { await fs.promises.access(dirtyFile); dirtyExists = true; @@ -267,7 +264,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map(), }); } catch (err) { caught = err; @@ -299,7 +295,6 @@ describe("materializeToolPackages — manifest.invalid gate", () => { cacheRoot: "/unused-by-manifest-invalid-gate", cacheMaxBytes: 1024 * 1024, registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map(), }); } catch (err) { caught = err; diff --git a/apps/sidecar/src/tool-materialization.ts b/apps/sidecar/src/tool-materialization.ts index d92c04844f..23efead255 100644 --- a/apps/sidecar/src/tool-materialization.ts +++ b/apps/sidecar/src/tool-materialization.ts @@ -21,10 +21,8 @@ import { type } from "arktype"; import { type AnnotatedPluginFactory } from "@intx/agent"; import { getLogger } from "@intx/log"; import { - type HostPlatform, type LoadedToolFactory, type LoadedToolPackage, - type RegistryConfig, applyAtomic, createTarballCache, createToolLoader, @@ -33,164 +31,10 @@ import { hasCode } from "@intx/types"; import type { ToolCredentialDeclaration } from "@intx/types/package-json"; import { ToolPackageManifest } from "@intx/types/tool-packages"; -/** - * A loaded tool factory paired with the identity of the package it came - * from. `buildCredentialCapabilities` keys Gate 2 on `packageName`. - */ -export interface StepToolFactory { - readonly packageName: string; - readonly declaredCredentials: readonly ToolCredentialDeclaration[]; - readonly factory: LoadedToolFactory; -} - -// Boundary validator for the tool-registries JSON the sidecar's config -// (`SIDECAR_TOOL_REGISTRIES`) resolves at boot and threads into every -// workflow-process child's spawn env. The wire shape carries `name` -// alongside the registry config so an operator can author the JSON as -// a flat array; the boundary collapses the array into a Map keyed by -// name before handing it to the loader. -const RegistryConfigEnvEntry = type({ - name: "string", - url: "string", - "auth?": type({ - "token?": "string", - "basic?": type({ user: "string", pass: "string" }), - }), -}); -const RegistryConfigEnvArray = RegistryConfigEnvEntry.array(); - -/** - * The registries serialized into the child spawn env when the operator - * pins none: the public npmjs registry. The default is decided once at - * the sidecar's boot edge — materialization itself always receives an - * explicit registry map and never falls back on its own, so a value - * lost in threading fails loud instead of silently routing tool - * packages through public npm. - */ -export const DEFAULT_TOOL_REGISTRIES_JSON = JSON.stringify([ - { name: "npmjs", url: "https://registry.npmjs.org" }, -]); - -/** - * Parse and validate a `SIDECAR_TOOL_REGISTRIES` JSON payload into the - * registry map the tool loader consumes. Throws, naming the variable, - * on an empty value (a set-but-empty variable almost always indicates - * misconfig — CI secret expansion failed, a templater dropped the - * value; the recovery is `unset SIDECAR_TOOL_REGISTRIES`, not `=""`), - * on malformed JSON, on a wrong-shape payload, and on duplicate - * registry names. - */ -export function parseToolRegistries(raw: string): ReadonlyMap { - if (raw.trim() === "") { - throw new Error( - "SIDECAR_TOOL_REGISTRIES is set but empty — unset the variable to use the default npmjs registry", - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err) { - throw new Error( - `SIDECAR_TOOL_REGISTRIES is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, - { cause: err }, - ); - } - const validated = RegistryConfigEnvArray(parsed); - if (validated instanceof type.errors) { - throw new Error(`SIDECAR_TOOL_REGISTRIES failed validation: ${validated.summary}`); - } - const out = new Map(); - for (const entry of validated) { - if (out.has(entry.name)) { - throw new Error( - `SIDECAR_TOOL_REGISTRIES has duplicate registry name ${JSON.stringify(entry.name)}`, - ); - } - const config: RegistryConfig = - entry.auth !== undefined ? { url: entry.url, auth: entry.auth } : { url: entry.url }; - out.set(entry.name, config); - } - return out; -} +import { readRegistries, resolveHostPlatform } from "./sidecar-materialization-config"; const logger = getLogger(["sidecar", "harness-builder"]); -// npm's `os` token namespace, mirrored from Node's `process.platform` -// enum. Any value outside this set means the host is running a Node -// build the loader's platform filter would silently mis-route — a -// pinned package whose `os` list excludes the host would not be -// excluded if `process.platform` is a token npm has never heard of. -// Validate at the boundary so an unknown platform fails the boot -// instead of producing a quiet, host-shaped mis-resolution at apply -// time. -// -// UPGRADE TAX: Node periodically adds platforms (and Bun ships -// extensions of its own). A Node/Bun major bump that lands a new -// `process.platform` value will fail boot until this allowlist is -// refreshed against the upstream enum. Sidecar operators upgrading -// the runtime should expect this as part of the cutover, not as a -// surprise regression. -const KNOWN_PROCESS_PLATFORMS = new Set([ - "aix", - "android", - "darwin", - "freebsd", - "haiku", - "linux", - "openbsd", - "sunos", - "win32", - "cygwin", - "netbsd", -]); - -// npm's `cpu` token namespace, mirrored from Node's `process.arch` -// enum. Same rationale as KNOWN_PROCESS_PLATFORMS — an unknown arch -// would mis-route the loader's filter without surfacing the gap. -// Same upgrade tax applies: Node has added `loong64` and `riscv64` -// in recent releases, and future arch additions will need to be -// added here when the sidecar is rebuilt against them. -const KNOWN_PROCESS_ARCHS = new Set([ - "arm", - "arm64", - "ia32", - "loong64", - "mips", - "mipsel", - "ppc64", - "riscv64", - "s390x", - "x64", -]); - -function assertKnownHostPlatform(platform: NodeJS.Platform): void { - if (!KNOWN_PROCESS_PLATFORMS.has(platform)) { - throw new Error( - `sidecar boot: process.platform ${JSON.stringify(platform)} is not a recognized npm \`os\` token; tool-package platform filtering would be unreliable`, - ); - } -} - -function assertKnownHostArch(arch: NodeJS.Architecture): void { - if (!KNOWN_PROCESS_ARCHS.has(arch)) { - throw new Error( - `sidecar boot: process.arch ${JSON.stringify(arch)} is not a recognized npm \`cpu\` token; tool-package platform filtering would be unreliable`, - ); - } -} - -/** - * The host platform token pair the `@intx/tool-packaging` loader filters - * manifest entries against, asserted against npm's `os`/`cpu` namespaces - * first. Shared by the per-step tool apply below and the workflow-definition - * closure materializer so both filter against one resolution. - */ -export function resolveHostPlatform(): HostPlatform { - assertKnownHostPlatform(process.platform); - assertKnownHostArch(process.arch); - return { os: process.platform, cpu: process.arch }; -} - // Sentinel `previousDeployId` for an instance that has never applied // a deploy successfully. Encoded as a literal string so the value // travels through `applyAtomic`'s `ApplyAtomicFailure.previousDeployId` @@ -198,8 +42,25 @@ export function resolveHostPlatform(): HostPlatform { // than as a real id. const NO_PRIOR_DEPLOY_ID = "none"; +/** + * A loaded tool factory paired with the identity of the package it came + * from. `collectFactories` flattens every package's factories into one list + * but preserves this association per factory, because the per-bundle + * credential assembly downstream needs two things the bare factory (which + * carries only its bundle id) cannot supply: the package name, to derive the + * consumer identity (`toolConsumer(packageName)`) that Gate 2 checks, and the + * package's declared credential handles, to reconcile against the delivered + * bindings. `declaredCredentials` is the package-level set, repeated on each + * of that package's factories. + */ +export interface StepToolFactory { + readonly packageName: string; + readonly declaredCredentials: readonly ToolCredentialDeclaration[]; + readonly factory: LoadedToolFactory; +} + interface MaterializedToolPackages { - readonly factories: readonly LoadedToolFactory[]; + readonly factories: readonly StepToolFactory[]; readonly pluginFactories: readonly AnnotatedPluginFactory[]; } @@ -233,14 +94,6 @@ export async function materializeToolPackages(args: { cacheRoot: string; cacheMaxBytes: number; registryMaxTarballBytes: number; - /** - * Registries the tool loader resolves registry-kind manifest entries - * against. Always supplied explicitly — the sidecar's config resolves - * `SIDECAR_TOOL_REGISTRIES` (or the npmjs default) at boot and - * threads it here — so there is no in-process env fallback to mask a - * value lost in threading. - */ - registries: ReadonlyMap; }): Promise { if (args.rawManifestBytes === undefined) { return { factories: [], pluginFactories: [] }; @@ -342,7 +195,7 @@ export async function materializeToolPackages(args: { }); const loader = createToolLoader({ cache, - registries: args.registries, + registries: readRegistries(), host: resolveHostPlatform(), maxRegistryTarballBytes: args.registryMaxTarballBytes, }); @@ -673,10 +526,16 @@ async function fsyncWriteFile(filePath: string, contents: string): Promise } } -function collectFactories(loaded: readonly LoadedToolPackage[]): readonly LoadedToolFactory[] { - const out: LoadedToolFactory[] = []; +function collectFactories(loaded: readonly LoadedToolPackage[]): readonly StepToolFactory[] { + const out: StepToolFactory[] = []; for (const pkg of loaded) { - for (const f of pkg.factories) out.push(f); + for (const f of pkg.factories) { + out.push({ + packageName: pkg.name, + declaredCredentials: pkg.credentials, + factory: f, + }); + } } return out; } diff --git a/apps/sidecar/src/workflow-child-boot-graph.test.ts b/apps/sidecar/src/workflow-child-boot-graph.test.ts new file mode 100644 index 0000000000..d927b6a9d3 --- /dev/null +++ b/apps/sidecar/src/workflow-child-boot-graph.test.ts @@ -0,0 +1,175 @@ +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +// Boot-graph guard for the spawned workflow-child. +// +// The workflow-child evaluates its whole module graph on every cold start. It +// must never pull the control-plane database layer (`@intx/db` / `drizzle`) or +// the sidecar orchestrator (`@intx/hub-agent`) into that graph: the child +// never executes either, the database dependency is a layering violation, and +// importing the orchestrator that spawns the child is a backwards dependency. +// The db-free substrate is reachable via `@intx/hub-sessions/substrate` and +// the path helpers via `@intx/hub-agent/paths`; the package barrels are not. +// +// This walks the child's VALUE-import graph and fails if any forbidden module +// is reachable. `Bun.build` erases `import type` and type-only specifiers +// before resolution (the project's `verbatimModuleSyntax` + `isolatedModules` +// guarantee every value-syntax import that survives is a real runtime import), +// so the captured graph is exactly what the child evaluates at runtime. +// Dynamic `import()` edges are included too, so even a lazy import of a +// forbidden module is caught. npm and `node:` packages are recorded but not +// traversed; workspace (`@intx/*`) packages are resolved to their real source +// (honoring `exports` subpaths) and walked through. + +const repoRoot = process.cwd(); + +// The binary the sidecar spawns as the workflow-child. The graph walk derives +// its entrypoints from this file's actual imports, so anything the binary +// loads -- in any import form -- is part of the checked graph. +const CHILD_BINARY = "apps/sidecar/bin/workflow-child"; + +// The roots the binary is expected to import. The drift guard asserts the +// binary imports exactly these; a new root trips it so a human confirms the +// addition (and the walk covers the new root regardless). +const EXPECTED_CHILD_ROOTS = ["../src/workflow-substrate-factory", "@intx/workflow-host"]; + +// The module specifiers the binary VALUE-imports, in every runtime form +// (static `from`, side-effect `import`, dynamic `import()`, `require`). Bun's +// transpiler is the same machinery the bundler uses, so it ignores comments, +// strings, and member expressions like `Array.from(...)`, and -- crucially -- +// erases `import type` / type-only specifiers, keeping this list aligned with +// the runtime value graph the walk below captures. +function binaryImportSpecifiers(): string[] { + const raw = readFileSync(join(repoRoot, CHILD_BINARY), "utf8"); + // The transpiler does not accept the binary's + // `#!/usr/bin/env -S bun --conditions=intx-src` shebang. + const source = raw.replace(/^#![^\n]*\n/, ""); + const transpiler = new Bun.Transpiler({ loader: "ts" }); + const specs = new Set(transpiler.scanImports(source).map((i) => i.path)); + return [...specs].sort(); +} + +// Resolve the binary's imports to real entrypoint files. A root that cannot +// resolve throws here and fails the test loudly rather than yielding a graph +// built from an incomplete root set. +function binaryEntrypoints(): string[] { + const binDir = dirname(join(repoRoot, CHILD_BINARY)); + return binaryImportSpecifiers().map((spec) => Bun.resolveSync(spec, binDir)); +} + +/** + * Returns the package reason if a value-import specifier is forbidden in the + * child boot graph, or null otherwise. `@intx/db` / `drizzle-orm` are banned + * entirely; the `@intx/hub-sessions` and `@intx/hub-agent` *barrels* are banned + * (their `/substrate` and `/paths` subpaths are the supported db-free doors). + */ +function forbiddenReason(spec: string): string | null { + if (spec === "@intx/db" || spec.startsWith("@intx/db/")) { + return "control-plane database layer"; + } + if (spec === "drizzle-orm" || spec.startsWith("drizzle-orm/")) { + return "database ORM"; + } + if (spec === "@intx/hub-sessions") { + return "control-plane barrel; import @intx/hub-sessions/substrate instead"; + } + if (spec === "@intx/hub-agent") { + return "sidecar orchestrator barrel; import @intx/hub-agent/paths instead"; + } + return null; +} + +async function childValueImportGraph(entrypoints: string[]): Promise<{ + importers: Map>; + resolveFailures: string[]; + success: boolean; +}> { + const importers = new Map>(); + const resolveFailures: string[] = []; + const relativize = (p: string) => p.replace(`${repoRoot}/`, ""); + + const result = await Bun.build({ + entrypoints, + target: "bun", + throw: false, + plugins: [ + { + name: "track-value-imports", + setup(build) { + build.onResolve({ filter: /.*/ }, (args) => { + const spec = args.path; + const from = args.importer ? relativize(args.importer) : "entry"; + const seenFrom = importers.get(spec) ?? new Set(); + seenFrom.add(from); + importers.set(spec, seenFrom); + + // Relative imports: let Bun traverse natively. + if (spec.startsWith(".") || spec.startsWith("/")) return undefined; + // Leaves we record but do not walk into. + if (spec.startsWith("node:")) return { path: spec, external: true }; + if (!spec.startsWith("@intx/")) { + return { path: spec, external: true }; + } + // Workspace packages: resolve to real source so traversal continues + // through them (honoring `exports` subpaths like `/substrate`). A + // failure here would silently leave the subtree unwalked, so record + // it -- the test asserts there are none. + const fromDir = args.importer ? dirname(args.importer) : repoRoot; + try { + return { path: Bun.resolveSync(spec, fromDir) }; + } catch { + resolveFailures.push(`${spec} (from ${from})`); + return { path: spec, external: true }; + } + }); + }, + }, + ], + }); + + return { importers, resolveFailures, success: result.success }; +} + +describe("workflow-child boot graph", () => { + test("the child binary imports only the two known roots", () => { + expect(binaryImportSpecifiers()).toEqual([...EXPECTED_CHILD_ROOTS].sort()); + }); + + test("never value-imports the control-plane database or the orchestrator", async () => { + const { importers, resolveFailures, success } = + await childValueImportGraph(binaryEntrypoints()); + + expect(success).toBe(true); + // A workspace specifier that fails to resolve would drop its subtree from + // the walk while the build still succeeds; surface it rather than silently + // un-guarding that subtree. + expect(resolveFailures).toEqual([]); + + const reached = new Set(importers.keys()); + // Anti-vacuity: a workflow-child necessarily evaluates the workflow + // runtime, so a graph that did not reach it never walked the real child + // (e.g. empty entrypoints) and the forbidden check would pass over nothing. + expect(reached).toContain("@intx/workflow"); + + const violations = [...reached] + .map((spec) => ({ spec, reason: forbiddenReason(spec) })) + .filter((v): v is { spec: string; reason: string } => v.reason !== null); + + if (violations.length > 0) { + const detail = violations + .map((v) => { + const from = [...(importers.get(v.spec) ?? new Set())].join("\n "); + return ` ${v.spec} -- ${v.reason}\n imported by:\n ${from}`; + }) + .join("\n"); + throw new Error( + `The workflow-child boot graph reached forbidden modules:\n${detail}\n\n` + + "The spawned child must not evaluate the control-plane database or the " + + "sidecar orchestrator at boot. Use @intx/hub-sessions/substrate for the " + + "db-free repo/substrate symbols and @intx/hub-agent/paths for the deploy-tree " + + "and address helpers.", + ); + } + }); +}); diff --git a/apps/sidecar/src/workflow-closure-apply.test.ts b/apps/sidecar/src/workflow-closure-apply.test.ts new file mode 100644 index 0000000000..1cf3089f1c --- /dev/null +++ b/apps/sidecar/src/workflow-closure-apply.test.ts @@ -0,0 +1,268 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { promises as fs } from "node:fs"; +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; + +import type { RegistryConfig, TarballFetcher } from "@intx/tool-packaging"; +import type { ToolPackageManifest } from "@intx/types/tool-packages"; + +import { applyFrozenWorkflowClosure } from "./workflow-closure-apply"; + +const REGISTRY_NAME = "test-registry"; +const WORKFLOW_PACKAGE_NAME = "@fixture/frozen-workflow"; +const WORKFLOW_PACKAGE_VERSION = "1.4.2"; +const WORKFLOW_ID = "fixture-closure-workflow"; + +// The pinned code the frozen closure carries: an ESM entry whose default +// export is a plain, envelope-valid workflow definition. It imports nothing so +// the closure needs no dependency entries, keeping the fixture hermetic while +// still exercising the real fetch -> SRI-verify -> extract -> layout -> import +// path. +const WORKFLOW_ENTRY_SOURCE = `export default { + id: ${JSON.stringify(WORKFLOW_ID)}, + triggers: [], + steps: { done: { kind: "sleep", id: "done", durationMs: 1 } }, + stepOrder: ["done"], +}; +`; + +let scratchRoot: string; +let cacheRoot: string; +let instanceDir: string; + +beforeEach(async () => { + scratchRoot = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-wf-closure-apply-")); + cacheRoot = path.join(scratchRoot, "cache"); + instanceDir = path.join(scratchRoot, "instance"); + await fs.mkdir(cacheRoot, { recursive: true }); + await fs.mkdir(instanceDir, { recursive: true }); +}); + +afterEach(async () => { + await fs.rm(scratchRoot, { recursive: true, force: true }); +}); + +/** + * Pack the fixture workflow package into an npm-style tarball (with the + * `package/` prefix the loader strips) and return its bytes plus the SRI + * integrity string the frozen manifest pins. + */ +async function packWorkflowFixture(): Promise<{ + bytes: Uint8Array; + integrity: string; +}> { + const stagingDir = path.join(scratchRoot, "fixture-source"); + const packageDir = path.join(stagingDir, "package"); + await fs.mkdir(packageDir, { recursive: true }); + await fs.writeFile( + path.join(packageDir, "package.json"), + JSON.stringify({ + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: "index.js" }, + }), + ); + await fs.writeFile(path.join(packageDir, "index.js"), WORKFLOW_ENTRY_SOURCE); + + const tarballPath = path.join(stagingDir, "out.tgz"); + const proc = Bun.spawnSync(["tar", "-czf", tarballPath, "-C", stagingDir, "package"]); + if (!proc.success) { + throw new Error(`tar failed to pack the fixture: ${new TextDecoder().decode(proc.stderr)}`); + } + const bytes = await fs.readFile(tarballPath); + const integrity = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + return { bytes, integrity }; +} + +function registries(): ReadonlyMap { + return new Map([[REGISTRY_NAME, { url: "https://registry.invalid" }]]); +} + +describe("applyFrozenWorkflowClosure", () => { + test("materializes the pinned closure and loads the pinned code", async () => { + const fixture = await packWorkflowFixture(); + const manifest: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: fixture.integrity, + }, + }, + ], + }; + + // Record every fetch so the test can prove the sidecar fetched EXACTLY the + // frozen concrete version -- no packument round-trip, no re-resolution. + const fetched: string[] = []; + const fetchTarball: TarballFetcher = async (entry) => { + fetched.push(`${entry.name}@${entry.version}`); + return fixture.bytes; + }; + + const applied = await applyFrozenWorkflowClosure({ + source: { kind: "registry", registry: REGISTRY_NAME }, + closure: manifest, + instanceDir, + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + fetchTarball, + }); + + // The pinned code was evaluated to its validated definition. + expect(applied.definition.id).toBe(WORKFLOW_ID); + + // The frozen closure was applied byte-for-byte: the workflow package's + // package.json is present in the materialized store directory. + const pkgJson = JSON.parse( + await fs.readFile(path.join(applied.packageDir, "package.json"), "utf8"), + ); + expect(pkgJson.name).toBe(WORKFLOW_PACKAGE_NAME); + expect(pkgJson.version).toBe(WORKFLOW_PACKAGE_VERSION); + + // No re-resolution: the only fetch was for the exact pinned concrete + // version the frozen closure named. + expect(fetched).toEqual([`${WORKFLOW_PACKAGE_NAME}@${WORKFLOW_PACKAGE_VERSION}`]); + }); + + test("rejects fetched bytes whose SRI does not match the frozen closure", async () => { + const fixture = await packWorkflowFixture(); + const manifest: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + // A pinned integrity that does not describe the returned bytes. + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: `sha512-${createHash("sha512").update("tampered").digest("base64")}`, + }, + }, + ], + }; + const fetchTarball: TarballFetcher = async () => fixture.bytes; + + await expect( + applyFrozenWorkflowClosure({ + source: { kind: "registry", registry: REGISTRY_NAME }, + closure: manifest, + instanceDir, + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + fetchTarball, + }), + ).rejects.toThrow(/integrity\.mismatch/); + }); + + test("rejects a closure that does not pin exactly one top-level package", async () => { + const manifest: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [], + entries: [], + }; + await expect( + applyFrozenWorkflowClosure({ + source: { kind: "registry", registry: REGISTRY_NAME }, + closure: manifest, + instanceDir, + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + }), + ).rejects.toThrow(/exactly one top-level pin/); + }); + + test("rejects a source registry the sidecar is not configured for", async () => { + const manifest: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: `sha512-${createHash("sha512").update("x").digest("base64")}`, + }, + }, + ], + }; + await expect( + applyFrozenWorkflowClosure({ + source: { kind: "registry", registry: "unconfigured-registry" }, + closure: manifest, + instanceDir, + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + }), + ).rejects.toThrow(/is not in the sidecar registry config/); + }); + + test("materializes an asset-sourced closure from a mounted asset root", async () => { + // This is the durable-store read path: the deploy checked the source asset + // out at //, and both deploy and restore materialize + // from there with no HTTP fetch. Stage the tarball as a plain file where the + // loader resolves a kind:"asset" entry, then apply. + const fixture = await packWorkflowFixture(); + const assetId = "asset_deploy"; + const mountPath = "source-assets/asset_deploy/"; + const tarballRel = "tarballs/wf.tgz"; + const assetRoot = path.join(scratchRoot, "asset-store"); + const tarballAbs = path.join(assetRoot, mountPath, tarballRel); + await fs.mkdir(path.dirname(tarballAbs), { recursive: true }); + await fs.writeFile(tarballAbs, fixture.bytes); + + const manifest: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: tarballRel, + integrity: fixture.integrity, + }, + }, + }, + ], + }; + + const applied = await applyFrozenWorkflowClosure({ + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure: manifest, + instanceDir, + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + registries: registries(), + assetRoot, + assetMounts: new Map([[assetId, mountPath]]), + gitDirs: new Map(), + }); + + expect(applied.definition.id).toBe(WORKFLOW_ID); + }); +}); diff --git a/apps/sidecar/src/workflow-closure-materialization.test.ts b/apps/sidecar/src/workflow-closure-materialization.test.ts new file mode 100644 index 0000000000..7772ef1e4a --- /dev/null +++ b/apps/sidecar/src/workflow-closure-materialization.test.ts @@ -0,0 +1,609 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import fsNode, { promises as fs } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import git from "isomorphic-git"; + +import { base64Encode } from "@intx/types"; +import type { RegistryConfig, TarballFetcher } from "@intx/tool-packaging"; +import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; +import type { ToolPackageManifest } from "@intx/types/tool-packages"; + +import { createWorkflowClosureMaterializer } from "./workflow-closure-materialization"; + +const REGISTRY_NAME = "test-registry"; +const WORKFLOW_PACKAGE_NAME = "@fixture/wf-probe"; +const WORKFLOW_PACKAGE_VERSION = "1.0.0"; +const DEP_PACKAGE_NAME = "@fixture/dep"; +const DEP_PACKAGE_VERSION = "1.0.0"; +const WORKFLOW_ENTRY = "index.js"; + +// The pinned workflow entry writes a sentinel file as an import-time side +// effect. The materializer lays the closure out WITHOUT importing any author +// code, so the sentinel must never appear -- its absence proves no module was +// evaluated on the host. +const SENTINEL_ENV = "PROBE_MATERIALIZER_SENTINEL"; +const WORKFLOW_ENTRY_SOURCE = ` +import { writeFileSync } from "node:fs"; +const sentinel = process.env[${JSON.stringify(SENTINEL_ENV)}]; +if (sentinel !== undefined) writeFileSync(sentinel, "imported"); +export default { + id: "probe-materializer-fixture", + triggers: [], + steps: { done: { kind: "sleep", id: "done", durationMs: 1 } }, + stepOrder: ["done"], +}; +`; + +let scratchRoot: string; +let cacheRoot: string; +let materializerScratch: string; +let fixtureSourceRoot: string; + +beforeEach(async () => { + scratchRoot = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-wf-probe-materialize-")); + cacheRoot = path.join(scratchRoot, "cache"); + materializerScratch = path.join(scratchRoot, "probe-closures"); + fixtureSourceRoot = path.join(scratchRoot, "fixture-source"); + await fs.mkdir(cacheRoot, { recursive: true }); + await fs.mkdir(materializerScratch, { recursive: true }); + await fs.mkdir(fixtureSourceRoot, { recursive: true }); +}); + +afterEach(async () => { + await fs.rm(scratchRoot, { recursive: true, force: true }); +}); + +/** + * Pack a package directory into an npm-style tarball (with the `package/` + * prefix the loader strips) and return its bytes plus the SRI integrity the + * frozen manifest pins. + */ +async function packFixture( + pkgJson: Record, + files: Record, +): Promise<{ bytes: Uint8Array; integrity: string }> { + const stagingDir = path.join( + fixtureSourceRoot, + `${String(pkgJson.name).replace("/", "_")}-${String(pkgJson.version)}`, + ); + const packageDir = path.join(stagingDir, "package"); + await fs.mkdir(packageDir, { recursive: true }); + await fs.writeFile(path.join(packageDir, "package.json"), JSON.stringify(pkgJson)); + for (const [rel, contents] of Object.entries(files)) { + const dest = path.join(packageDir, rel); + await fs.mkdir(path.dirname(dest), { recursive: true }); + await fs.writeFile(dest, contents); + } + + const tarballPath = path.join(stagingDir, "out.tgz"); + const proc = Bun.spawnSync(["tar", "-czf", tarballPath, "-C", stagingDir, "package"]); + if (!proc.success) { + throw new Error(`tar failed to pack the fixture: ${new TextDecoder().decode(proc.stderr)}`); + } + const bytes = await fs.readFile(tarballPath); + const integrity = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + return { bytes, integrity }; +} + +function registries(): ReadonlyMap { + return new Map([[REGISTRY_NAME, { url: "https://registry.invalid" }]]); +} + +function materializerConfig(fetchTarball?: TarballFetcher) { + return { + cacheRoot, + cacheMaxBytes: 10_000_000, + registryMaxTarballBytes: 10_000_000, + maxAssetPayloadBytes: 50_000_000, + registries: registries(), + scratchRoot: materializerScratch, + ...(fetchTarball !== undefined ? { fetchTarball } : {}), + }; +} + +/** + * Build a git packfile whose single commit's tree holds `files`, mirroring the + * `createPack` output the hub delivers for an asset. Returns the pack bytes and + * the commit sha the probe frame pins. + */ +async function buildAssetPack( + files: Record, +): Promise<{ pack: Uint8Array; commitSha: string }> { + const sourceDir = path.join(fixtureSourceRoot, `asset-${randomUUID()}`); + await fs.mkdir(sourceDir, { recursive: true }); + await git.init({ fs: fsNode, dir: sourceDir, defaultBranch: "main" }); + for (const [rel, content] of Object.entries(files)) { + const abs = path.join(sourceDir, rel); + await fs.mkdir(path.dirname(abs), { recursive: true }); + await fs.writeFile(abs, content); + await git.add({ fs: fsNode, dir: sourceDir, filepath: rel }); + } + const commitSha = await git.commit({ + fs: fsNode, + dir: sourceDir, + message: "asset", + author: { name: "test", email: "test@test.dev" }, + }); + const oids = new Set([commitSha]); + const { commit } = await git.readCommit({ + fs: fsNode, + dir: sourceDir, + oid: commitSha, + }); + oids.add(commit.tree); + async function walkTree(treeOid: string): Promise { + const { tree } = await git.readTree({ + fs: fsNode, + dir: sourceDir, + oid: treeOid, + }); + for (const entry of tree) { + oids.add(entry.oid); + if (entry.type === "tree") await walkTree(entry.oid); + } + } + await walkTree(commit.tree); + const result = await git.packObjects({ + fs: fsNode, + dir: sourceDir, + oids: [...oids], + write: false, + }); + if (result.packfile === undefined) { + throw new Error("packObjects produced no packfile"); + } + return { pack: result.packfile, commitSha }; +} + +function probeFrame(closure: ToolPackageManifest, entry: string): WorkflowProbeRequestFrame { + return { + type: "workflow.probe.request", + requestId: "req-1", + source: { kind: "registry", registry: REGISTRY_NAME }, + closure, + entry, + }; +} + +describe("createWorkflowClosureMaterializer", () => { + test("lays out the frozen closure, resolves node_modules, and imports no author code", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + dependencies: { [DEP_PACKAGE_NAME]: DEP_PACKAGE_VERSION }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const dep = await packFixture( + { + name: DEP_PACKAGE_NAME, + version: DEP_PACKAGE_VERSION, + type: "module", + }, + { "index.js": "export const x = 1;\n" }, + ); + + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: workflow.integrity, + }, + }, + { + name: DEP_PACKAGE_NAME, + version: DEP_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: dep.integrity, + }, + }, + ], + }; + + const fetched: string[] = []; + const fetchTarball: TarballFetcher = async (entry) => { + fetched.push(`${entry.name}@${entry.version}`); + if (entry.name === WORKFLOW_PACKAGE_NAME) return workflow.bytes; + if (entry.name === DEP_PACKAGE_NAME) return dep.bytes; + throw new Error(`unexpected fetch for ${entry.name}@${entry.version}`); + }; + + const sentinelPath = path.join(scratchRoot, "import-sentinel"); + process.env[SENTINEL_ENV] = sentinelPath; + try { + const materialize = createWorkflowClosureMaterializer(materializerConfig(fetchTarball)); + const materialized = await materialize(probeFrame(closure, WORKFLOW_ENTRY)); + + // The workflow package's own package.json is present in the laid-out + // store directory. + const pkgJson = JSON.parse( + await fs.readFile(path.join(materialized.packageDir, "package.json"), "utf8"), + ); + expect(pkgJson.name).toBe(WORKFLOW_PACKAGE_NAME); + expect(pkgJson.version).toBe(WORKFLOW_PACKAGE_VERSION); + + // The dependency resolves through the laid-out node_modules graph. + const depPkgJson = JSON.parse( + await fs.readFile( + path.join(materialized.packageDir, "node_modules", "@fixture", "dep", "package.json"), + "utf8", + ), + ); + expect(depPkgJson.name).toBe(DEP_PACKAGE_NAME); + + // Both frozen entries were fetched exactly once, at their pinned + // concrete versions. + expect(fetched.sort()).toEqual([ + `${DEP_PACKAGE_NAME}@${DEP_PACKAGE_VERSION}`, + `${WORKFLOW_PACKAGE_NAME}@${WORKFLOW_PACKAGE_VERSION}`, + ]); + + // No author code ran: the entry module's import-time sentinel was + // never written. + await expect(fs.stat(sentinelPath)).rejects.toThrow(); + + // Cleanup removes the ephemeral scratch tree. + await materialized.cleanup(); + await expect(fs.stat(materialized.packageDir)).rejects.toThrow(); + } finally { + Reflect.deleteProperty(process.env, SENTINEL_ENV); + } + }); + + test("fails loud when the closure pins no top-level package", async () => { + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [], + entries: [], + }; + await expect(materialize(probeFrame(closure, WORKFLOW_ENTRY))).rejects.toThrow( + /exactly one top-level package/, + ); + }); + + test("fails loud when the closure pins more than one top-level package", async () => { + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [ + { name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }, + { name: DEP_PACKAGE_NAME, version: DEP_PACKAGE_VERSION }, + ], + entries: [], + }; + await expect(materialize(probeFrame(closure, WORKFLOW_ENTRY))).rejects.toThrow( + /exactly one top-level package/, + ); + }); + + test("materializes an asset-sourced closure from an inline-delivered pack", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const tarballPath = "tarballs/wf-probe-1.0.0.tgz"; + const assetId = "asset_probe"; + const mountPath = "package-registries/fixture/"; + const { pack, commitSha } = await buildAssetPack({ + [tarballPath]: workflow.bytes, + }); + + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: tarballPath, + integrity: workflow.integrity, + }, + }, + }, + ], + }; + + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + const materialized = await materialize({ + type: "workflow.probe.request", + requestId: "req-asset", + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure, + entry: WORKFLOW_ENTRY, + assets: [ + { + assetId, + mountPath, + pack: base64Encode(pack), + ref: "refs/heads/main", + commitSha, + }, + ], + }); + + try { + // The workflow package was laid out from the asset-delivered tarball, + // SRI-verified against the frozen closure entry -- no HTTP fetch. + const pkgJson = JSON.parse( + await fs.readFile(path.join(materialized.packageDir, "package.json"), "utf8"), + ); + expect(pkgJson.name).toBe(WORKFLOW_PACKAGE_NAME); + expect(pkgJson.version).toBe(WORKFLOW_PACKAGE_VERSION); + } finally { + await materialized.cleanup(); + } + }); + + test("fails loud when the inline asset payload exceeds the cap", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const assetId = "asset_probe"; + const { pack, commitSha } = await buildAssetPack({ + "tarballs/wf-probe-1.0.0.tgz": workflow.bytes, + }); + + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: "tarballs/wf-probe-1.0.0.tgz", + integrity: workflow.integrity, + }, + }, + }, + ], + }; + + const materialize = createWorkflowClosureMaterializer({ + ...materializerConfig(), + maxAssetPayloadBytes: 16, + }); + await expect( + materialize({ + type: "workflow.probe.request", + requestId: "req-cap", + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure, + entry: WORKFLOW_ENTRY, + assets: [ + { + assetId, + mountPath: "package-registries/fixture/", + pack: base64Encode(pack), + ref: "refs/heads/main", + commitSha, + }, + ], + }), + ).rejects.toThrow(/inline asset payload exceeds the 16-byte cap/); + }); + + test("rejects an asset-delivered tarball that fails its pinned integrity", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const assetId = "asset_probe"; + const tarballPath = "tarballs/wf-probe-1.0.0.tgz"; + const { pack, commitSha } = await buildAssetPack({ + [tarballPath]: workflow.bytes, + }); + + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + // A pinned integrity that does not describe the delivered tarball. + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: tarballPath, + integrity: `sha512-${createHash("sha512").update("tampered").digest("base64")}`, + }, + }, + }, + ], + }; + + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + await expect( + materialize({ + type: "workflow.probe.request", + requestId: "req-sri", + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure, + entry: WORKFLOW_ENTRY, + assets: [ + { + assetId, + mountPath: "package-registries/fixture/", + pack: base64Encode(pack), + ref: "refs/heads/main", + commitSha, + }, + ], + }), + ).rejects.toThrow(/did not match pinned integrity/); + }); + + test("rejects an asset source whose asset was not delivered", async () => { + const assetId = "asset_probe"; + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: "tarballs/wf.tgz", + integrity: `sha512-${createHash("sha512").update("x").digest("base64")}`, + }, + }, + }, + ], + }; + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + await expect( + materialize({ + type: "workflow.probe.request", + requestId: "req-undelivered", + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure, + entry: WORKFLOW_ENTRY, + assets: [], + }), + ).rejects.toThrow(/was not among the delivered assets/); + }); + + test("rejects a frame that delivers the same assetId twice", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const assetId = "asset_probe"; + const { pack, commitSha } = await buildAssetPack({ + "tarballs/wf-probe-1.0.0.tgz": workflow.bytes, + }); + const encoded = base64Encode(pack); + // The closure references the delivered asset so the first delivery clears + // the "delivered but referenced by no closure entry" guard and the second + // delivery is what trips the duplicate-delivery guard under test. + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: "tarballs/wf-probe-1.0.0.tgz", + integrity: "sha512-placeholder", + }, + }, + }, + ], + }; + const materialize = createWorkflowClosureMaterializer(materializerConfig()); + await expect( + materialize({ + type: "workflow.probe.request", + requestId: "req-dup", + source: { kind: "asset", assetId, package: { format: "tarball" } }, + closure, + entry: WORKFLOW_ENTRY, + assets: [ + { + assetId, + mountPath: "package-registries/a/", + pack: encoded, + ref: "refs/heads/main", + commitSha, + }, + { + assetId, + mountPath: "package-registries/b/", + pack: encoded, + ref: "refs/heads/main", + commitSha, + }, + ], + }), + ).rejects.toThrow(/is delivered more than once/); + }); + + test("fails loud when the frame entry disagrees with interchange.workflow", async () => { + const workflow = await packFixture( + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + type: "module", + interchange: { workflow: WORKFLOW_ENTRY }, + }, + { [WORKFLOW_ENTRY]: WORKFLOW_ENTRY_SOURCE }, + ); + const closure: ToolPackageManifest = { + schemaVersion: "1", + topLevel: [{ name: WORKFLOW_PACKAGE_NAME, version: WORKFLOW_PACKAGE_VERSION }], + entries: [ + { + name: WORKFLOW_PACKAGE_NAME, + version: WORKFLOW_PACKAGE_VERSION, + source: { + kind: "registry", + registry: REGISTRY_NAME, + integrity: workflow.integrity, + }, + }, + ], + }; + const fetchTarball: TarballFetcher = async () => workflow.bytes; + + const materialize = createWorkflowClosureMaterializer(materializerConfig(fetchTarball)); + await expect(materialize(probeFrame(closure, "./does-not-match.js"))).rejects.toThrow( + /does not match the materialized package's interchange.workflow/, + ); + }); +}); diff --git a/apps/sidecar/src/workflow-closure-materialization.ts b/apps/sidecar/src/workflow-closure-materialization.ts index 8b6622f58e..46d65b5072 100644 --- a/apps/sidecar/src/workflow-closure-materialization.ts +++ b/apps/sidecar/src/workflow-closure-materialization.ts @@ -50,7 +50,7 @@ import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; import type { ToolPackageManifest } from "@intx/types/tool-packages"; import { materializeWorkflowAssets } from "./source-asset-delivery"; -import { resolveHostPlatform } from "./tool-materialization"; +import { resolveHostPlatform } from "./sidecar-materialization-config"; import type { MaterializedWorkflowClosure, MaterializeWorkflowClosure, diff --git a/apps/sidecar/src/workflow-deployment-record.test.ts b/apps/sidecar/src/workflow-deployment-record.test.ts deleted file mode 100644 index 042d0891e1..0000000000 --- a/apps/sidecar/src/workflow-deployment-record.test.ts +++ /dev/null @@ -1,322 +0,0 @@ -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { - RESTORE_QUARANTINE_THRESHOLD, - clearWorkflowDeploymentRestoreFailure, - isWorkflowDeploymentRestoreQuarantined, - markWorkflowDeploymentRecordParked, - partitionScannedDeployments, - readWorkflowDeploymentRecord, - recordWorkflowDeploymentRestoreFailure, - scanWorkflowDeploymentRecords, - writeWorkflowDeploymentRecord, - type WorkflowDeploymentRecord, -} from "./workflow-deployment-record"; - -const RECORD_FILENAME = "deployment.json"; - -function recordPath(dataDir: string, deploymentId: string): string { - return path.join(dataDir, "workflow-runs", deploymentId, RECORD_FILENAME); -} - -async function makeDataDir(): Promise { - return await fs.mkdtemp(path.join(os.tmpdir(), "deployment-record-test-")); -} - -const baseRecord: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress: "run_parked-test@example.com", - definitionId: "def_1", - sources: {}, - approvedWireHash: "d".repeat(64), - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { - schemaVersion: "1", - topLevel: [{ name: "@x/wf", version: "1.0.0" }], - entries: [], - }, - }, -}; - -describe("markWorkflowDeploymentRecordParked", () => { - test("stamps parkedAt on an existing record without disturbing other fields", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - - await markWorkflowDeploymentRecordParked(dataDir, "dep_1"); - - const record = await readWorkflowDeploymentRecord(dataDir, "dep_1"); - expect(record?.parkedAt).toBeDefined(); - expect(record?.agentAddress).toBe(baseRecord.agentAddress); - expect(record?.definitionId).toBe(baseRecord.definitionId); - }); - - test("is a no-op when no record exists for the deployment", async () => { - const dataDir = await makeDataDir(); - - await markWorkflowDeploymentRecordParked(dataDir, "dep_missing"); - - await expect(readWorkflowDeploymentRecord(dataDir, "dep_missing")).resolves.toBeUndefined(); - }); -}); - -describe("record backward compatibility", () => { - test("a record written before the parkedAt field existed still loads as live", async () => { - const dataDir = await makeDataDir(); - // Simulate a pre-CL-6284 record on disk: no `parkedAt` key at all, not - // even `parkedAt: undefined`. - await writeWorkflowDeploymentRecord(dataDir, "dep_legacy", baseRecord); - - const record = await readWorkflowDeploymentRecord(dataDir, "dep_legacy"); - expect(record).toBeDefined(); - expect(record?.parkedAt).toBeUndefined(); - }); -}); - -describe("scanWorkflowDeploymentRecords parked/live distinction", () => { - test("surfaces parkedAt on scanned records so a caller can tell parked from live", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_live", baseRecord); - await writeWorkflowDeploymentRecord(dataDir, "dep_parked", baseRecord); - await markWorkflowDeploymentRecordParked(dataDir, "dep_parked"); - - const scanned = await scanWorkflowDeploymentRecords(dataDir); - const byId = new Map(scanned.map((s) => [s.deploymentId, s.record])); - - expect(byId.get("dep_live")?.parkedAt).toBeUndefined(); - expect(byId.get("dep_parked")?.parkedAt).toBeDefined(); - }); -}); - -describe("scanWorkflowDeploymentRecords reaps pre-cutover records", () => { - test("an unreadable old-format record (missing approvedWireHash/sourceRef) is reaped once, then boots quietly", async () => { - const dataDir = await makeDataDir(); - const preCutover = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress: "run_old-format@example.com", - definitionId: "def_1", - sources: {}, - }; - const filePath = recordPath(dataDir, "dep_old"); - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, JSON.stringify(preCutover)); - - const firstScan = await scanWorkflowDeploymentRecords(dataDir); - expect(firstScan).toHaveLength(0); - await expect(fs.access(filePath)).rejects.toThrow(); - - // Quiet second boot: the record is already gone, nothing left to reap. - const secondScan = await scanWorkflowDeploymentRecords(dataDir); - expect(secondScan).toHaveLength(0); - }); - - test("a parseable record is never reaped", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_live", baseRecord); - - const scanned = await scanWorkflowDeploymentRecords(dataDir); - - expect(scanned).toHaveLength(1); - expect(scanned[0]?.deploymentId).toBe("dep_live"); - await expect(readWorkflowDeploymentRecord(dataDir, "dep_live")).resolves.toBeDefined(); - }); - - test("a record invalid for a reason other than the old-format shape is warned about, not reaped", async () => { - const dataDir = await makeDataDir(); - const corrupt = { - version: 2, // wrong version -- not the recognized pre-cutover shape - agentAddress: "run_corrupt@example.com", - definitionId: "def_1", - sources: {}, - }; - const filePath = recordPath(dataDir, "dep_corrupt"); - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, JSON.stringify(corrupt)); - - const scanned = await scanWorkflowDeploymentRecords(dataDir); - - expect(scanned).toHaveLength(0); - // Still on disk -- never silently eaten. - await fs.access(filePath); - }); -}); - -describe("partitionScannedDeployments", () => { - test("splits scanned records into live (no parkedAt) and parked (parkedAt set)", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_live", baseRecord); - await writeWorkflowDeploymentRecord(dataDir, "dep_parked", baseRecord); - await markWorkflowDeploymentRecordParked(dataDir, "dep_parked"); - const scanned = await scanWorkflowDeploymentRecords(dataDir); - - const { live, parked } = partitionScannedDeployments(scanned); - - expect(live.map((s) => s.deploymentId)).toEqual(["dep_live"]); - expect(parked.map((s) => s.deploymentId)).toEqual(["dep_parked"]); - }); - - test("an all-live scan yields nothing to skip", () => { - const scanned = [ - { deploymentId: "dep_a", record: baseRecord }, - { deploymentId: "dep_b", record: baseRecord }, - ]; - - const { live, parked } = partitionScannedDeployments(scanned); - - expect(live).toHaveLength(2); - expect(parked).toHaveLength(0); - }); -}); - -describe("recordWorkflowDeploymentRestoreFailure", () => { - test("starts a kind's counter at 1 and persists reason/timestamp", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - - const updated = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "permanent", - reason: "address derives a different slug", - }); - - expect(updated?.restoreFailure).toEqual({ - kind: "permanent", - attempts: 1, - reason: "address derives a different slug", - lastAttemptAt: updated?.restoreFailure?.lastAttemptAt ?? "", - }); - const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_1"); - expect(onDisk?.restoreFailure?.attempts).toBe(1); - }); - - test("increments the counter across repeated failures of the same kind", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - - let record: WorkflowDeploymentRecord | undefined; - for (let i = 0; i < 3; i++) { - record = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "permanent", - reason: "still malformed", - }); - } - - expect(record?.restoreFailure?.attempts).toBe(3); - expect(record?.restoreFailure?.kind).toBe("permanent"); - }); - - test("a kind change resets the counter rather than adding to the other kind's count", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - - await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "permanent", - reason: "malformed", - }); - let record = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "permanent", - reason: "still malformed", - }); - expect(record?.restoreFailure?.attempts).toBe(2); - - record = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "transient", - reason: "provider not registered", - }); - - expect(record?.restoreFailure?.kind).toBe("transient"); - expect(record?.restoreFailure?.attempts).toBe(1); - }); - - test("is a no-op that returns undefined when the record is already gone (CL-7215: a reclaiming teardown raced ahead of this write)", async () => { - const dataDir = await makeDataDir(); - // Deliberately never written: simulates a teardown deleting the record - // before the boot loop's failure-recording catch acquires the lock. - - const updated = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_missing", { - kind: "transient", - reason: "provider not registered", - }); - - expect(updated).toBeUndefined(); - const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_missing"); - expect(onDisk).toBeUndefined(); - }); -}); - -describe("clearWorkflowDeploymentRestoreFailure", () => { - test("drops restoreFailure after a successful restore", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - const failed = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", { - kind: "transient", - reason: "provider not registered", - }); - - await clearWorkflowDeploymentRestoreFailure( - dataDir, - "dep_1", - failed as WorkflowDeploymentRecord, - ); - - const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_1"); - expect(onDisk?.restoreFailure).toBeUndefined(); - }); - - test("is a no-op when there is no failure to clear", async () => { - const dataDir = await makeDataDir(); - await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord); - - await clearWorkflowDeploymentRestoreFailure(dataDir, "dep_1", baseRecord); - - const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_1"); - expect(onDisk?.agentAddress).toBe(baseRecord.agentAddress); - }); -}); - -describe("isWorkflowDeploymentRestoreQuarantined", () => { - test("is false below the threshold and true at or above it, for permanent failures only", () => { - const belowThreshold: WorkflowDeploymentRecord = { - ...baseRecord, - restoreFailure: { - kind: "permanent", - attempts: RESTORE_QUARANTINE_THRESHOLD - 1, - reason: "malformed", - lastAttemptAt: new Date().toISOString(), - }, - }; - const atThreshold: WorkflowDeploymentRecord = { - ...baseRecord, - restoreFailure: { - kind: "permanent", - attempts: RESTORE_QUARANTINE_THRESHOLD, - reason: "malformed", - lastAttemptAt: new Date().toISOString(), - }, - }; - const transientAtSameCount: WorkflowDeploymentRecord = { - ...baseRecord, - restoreFailure: { - kind: "transient", - attempts: RESTORE_QUARANTINE_THRESHOLD + 5, - reason: "provider not registered", - lastAttemptAt: new Date().toISOString(), - }, - }; - - expect(isWorkflowDeploymentRestoreQuarantined(belowThreshold)).toBe(false); - expect(isWorkflowDeploymentRestoreQuarantined(atThreshold)).toBe(true); - expect(isWorkflowDeploymentRestoreQuarantined(baseRecord)).toBe(false); - // A transient failure never quarantines, no matter how high its own - // counter climbs -- it is tracked on a separate counter from the - // permanent one. - expect(isWorkflowDeploymentRestoreQuarantined(transientAtSameCount)).toBe(false); - }); -}); diff --git a/apps/sidecar/src/workflow-deployment-record.ts b/apps/sidecar/src/workflow-deployment-record.ts deleted file mode 100644 index 5239a50d4a..0000000000 --- a/apps/sidecar/src/workflow-deployment-record.ts +++ /dev/null @@ -1,417 +0,0 @@ -// Sidecar-local persistence of the per-deployment record needed to -// re-establish a workflow deployment across a sidecar PROCESS restart. The -// record is co-located with the deployment's workflow-run substrate at -// `${dataDir}/workflow-runs//deployment.json`, so a single -// teardown reclaims both and a boot scan can enumerate the active -// deployments beside the run state they resume. -// -// It carries only the inputs that are otherwise frame/in-memory only: -// `sources` (each step's ordered inference-source failover chain, threaded to -// the child via the spawn env and durable nowhere else), `sessionId` -// (inference-event -// correlation), `hubPublicKey` (the head's deploy-pack / inbound -// verification key, recorded only in memory today), and -// `referencedDefinitionHashes` (the hub-approved wire hash per referenced -// onTrigger body id, threaded to the child via the spawn env). The -// definition itself lives in `assets/workflow//workflow.json`, -// referenced by `definitionId`, and each step's grants live in its -// agent-state repo, so neither is duplicated here. - -import { mkdir, readdir, readFile, rm } from "node:fs/promises"; -import { dirname, join as pathJoin } from "node:path"; - -import { type } from "arktype"; - -import { getLogger } from "@intx/log"; -import { InferenceSource } from "@intx/types/runtime"; -import { SourceRefPin } from "@intx/types/sidecar"; -import { isErrnoNotFound } from "./conversation-state"; - -import { writeFileAtomicDurable } from "./atomic-write"; - -const logger = getLogger(["sidecar", "workflow-deployment-record"]); - -const RECORD_FILENAME = "deployment.json"; - -/** - * The on-disk deployment record. `version` guards the schema shape so a - * future reader can reject or migrate a stale record rather than parse it - * blindly. Validated at read time (the boot scan) at the trust boundary. - */ -export const WorkflowDeploymentRecord = type({ - version: "2", - agentAddress: "string > 0", - definitionId: "string > 0", - // The run's own tenant and principal, from the hub's signed deploy frame. - // Required since version 2: the child threads them into its substrate - // config so a step tool can address stock `/api/tenants/:tenantId/*` with - // the run bearer, and a restore that could not name them would stand up a - // child whose tools cannot reach the hub. A version-1 record fails this - // parse and is redeployed rather than restored. - tenantId: "string > 0", - principalId: "string > 0", - sources: { - "[string]": InferenceSource.array().atLeastLength(1), - }, - "sessionId?": "string > 0", - "hubPublicKey?": "string > 0", - // The hub-approved wire hash the restored child re-verifies its evaluated - // closure against, rather than a sidecar recompute of the inert projection - // -- the latter would collapse the out-of-band-pin property the re-verify - // barrier exists for. - approvedWireHash: "string > 0", - // The pin a restore re-runs the closure apply with: `source` names where the - // definition package comes from (no secret -- the registry token resolves - // from env at apply time), `closure` is the hub's frozen dependency set - // (concrete versions + integrity SRIs). Both rode the signed deploy frame. - sourceRef: SourceRefPin, - // Written only on the state-preserving hibernate teardown - // (`teardownDeployment({ reclaimDirs: false })`), never on deploy or - // rotation. Its presence is the durable answer to "did the hub park this - // deployment on purpose", as opposed to a record left behind by a crash - // or a bare process exit -- both of which leave no marker at all. Absent - // on every record written before this field existed and on every record - // for a deployment that has never been hibernated; `readWorkflowDeploymentRecord` - // and the boot scan both treat an absent marker as "live", so an old - // on-disk record keeps loading and restoring exactly as before. - "parkedAt?": "string > 0", - // Populated only after a failed boot-time restore attempt; absent for a - // record that has never failed restore, and dropped again the moment a - // restore succeeds (`clearWorkflowDeploymentRestoreFailure`). `kind` - // mirrors `WorkflowRestoreFailure.kind`: "permanent" counts toward - // `RESTORE_QUARANTINE_THRESHOLD`, "transient" is recorded for visibility - // only and never quarantines. `attempts` counts CONSECUTIVE failures of - // the SAME kind -- a kind change resets it to 1, so a transient failure - // streak can never inflate the permanent counter, or vice versa. - "restoreFailure?": { - kind: "'permanent' | 'transient'", - attempts: "number > 0", - reason: "string > 0", - lastAttemptAt: "string > 0", - }, -}); -export type WorkflowDeploymentRecord = typeof WorkflowDeploymentRecord.infer; - -export type RestoreFailureKind = NonNullable["kind"]; - -/** - * Thrown from inside the restore path to mark WHY a specific attempt - * failed. `kind` distinguishes a failure intrinsic to the record's own - * persisted bytes -- deterministic, will fail identically on every future - * boot too (a corrupt/misplaced record, a closure-derived definition that - * fails structural validation) -- from one that depends on this boot's - * environment and may clear on its own (an inference provider the sidecar - * cannot currently build). Only "permanent" failures count toward - * `RESTORE_QUARANTINE_THRESHOLD`; an ordinary (unwrapped) `Error` thrown - * anywhere else in the restore path is treated as "transient" by the boot - * loop -- the safe default, since misclassifying a recoverable failure as - * permanent stops future restore attempts for it. - */ -export class WorkflowRestoreFailure extends Error { - readonly kind: RestoreFailureKind; - constructor(kind: RestoreFailureKind, message: string) { - super(message); - this.name = "WorkflowRestoreFailure"; - this.kind = kind; - } -} - -/** - * Consecutive PERMANENT restore failures before a record stops being - * attempted and collapses into the boot scan's one-line summary instead - * of an individual per-record warning every boot. 3: permanent failures - * are deterministic (derived only from the record's own immutable - * closure pin and address, never from runtime environment), so they do - * not actually need repeated confirmation to be believed -- but a small - * buffer costs nothing and guards against a misclassification, while - * still bounding the "warn every boot forever" failure mode this - * quarantine exists to fix to a handful of boots. - */ -export const RESTORE_QUARANTINE_THRESHOLD = 3; - -function recordPath(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-runs", deploymentId, RECORD_FILENAME); -} - -// The format cutover added exactly two required fields (`approvedWireHash`, -// `sourceRef`); every pre-cutover record on disk fails validation with -// nothing wrong except those two missing. Recognizing that exact shape -- -// and only that shape -- is what lets the boot scan tell "deliberately -// obsolete, safe to reap" apart from "genuinely corrupt, must keep warning -// about": a record invalid for any other reason (a bad `version`, a -// malformed `sources` map, a `sourceRef` present but shaped wrong) still -// warns individually rather than being silently deleted. -const PRE_CUTOVER_MISSING_FIELDS = new Set(["approvedWireHash", "sourceRef"]); - -function isPreCutoverShape(errors: type.errors): boolean { - return [...errors].every( - (error) => - error.code === "required" && - error.path.length === 1 && - PRE_CUTOVER_MISSING_FIELDS.has(String(error.path[0])), - ); -} - -/** - * Read one deployment's record by id, for a caller that already knows the - * deployment id it needs and has no reason to re-scan the whole - * `workflow-runs/` tree. Returns `undefined` for a missing or unparseable - * record. - */ -export async function readWorkflowDeploymentRecord( - dataDir: string, - deploymentId: string, -): Promise { - let raw: string; - try { - raw = await readFile(recordPath(dataDir, deploymentId), "utf8"); - } catch (cause) { - if (isErrnoNotFound(cause)) return undefined; - throw cause; - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - return undefined; - } - const record = WorkflowDeploymentRecord(parsed); - if (record instanceof type.errors) return undefined; - return record; -} - -/** - * Persist a deployment record. Written after the deployment's slug is - * claimed and before the child is spawned, so a crash mid-spawn leaves a - * record the boot scan re-drives. Idempotent: it overwrites any existing - * record for the same deployment. - */ -export async function writeWorkflowDeploymentRecord( - dataDir: string, - deploymentId: string, - record: WorkflowDeploymentRecord, -): Promise { - const path = recordPath(dataDir, deploymentId); - await mkdir(dirname(path), { recursive: true }); - // Atomic + durable: this is the sole restore source for the - // deployment's `sources`/`hubPublicKey`, and a rotation overwrites the - // existing record in place, so an interrupted write must never expose a - // torn record the boot scan would then skip. Owner-only (0o600): the - // record names each source's `credentialId` and the run's routing - // identity, so it must not be world-readable on a shared host, matching - // the private-key writes elsewhere on the sidecar. - await writeFileAtomicDurable(path, JSON.stringify(record, null, 2), { - mode: 0o600, - }); -} - -/** - * Mark an existing deployment record as parked: stamp `parkedAt` with the - * current time and rewrite the record in place. Called from the - * state-preserving hibernate teardown, after the record has already been - * confirmed to exist (a hibernate never writes a fresh record, only - * annotates the one the original deploy or a later rotation left behind). - * A missing record is not an error -- there is nothing to mark, and the - * caller's own "record IS the durable state" invariant means this should - * not happen outside a race with a concurrent reclaim, which just leaves - * the reclaim as the last write. - */ -export async function markWorkflowDeploymentRecordParked( - dataDir: string, - deploymentId: string, -): Promise { - const existing = await readWorkflowDeploymentRecord(dataDir, deploymentId); - if (existing === undefined) return; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, { - ...existing, - parkedAt: new Date().toISOString(), - }); -} - -/** - * Record a failed boot-time restore attempt on an existing record: bump - * `attempts` (reset to 1 when `failure.kind` differs from the previously - * recorded kind, so a transient streak can never inflate the permanent - * counter or vice versa), stamp `reason`/`lastAttemptAt`, and persist. - * Returns the updated record so the caller can check - * `isWorkflowDeploymentRestoreQuarantined` without a re-read. - * - * Re-reads the record from disk rather than trusting the caller's - * in-memory copy (CL-7215): the boot-restore path's per-deployment lock - * serializes this against a reclaiming teardown that can delete the - * record out from under a dangling (timed-out but still-running) restore - * between when the boot loop scanned it and when this runs. Blindly - * writing the stale in-memory `record` would resurrect `deployment.json` - * with a false failure stamp for a deployment that was fully reclaimed -- - * the exact "durable record disagrees with reality" failure this ticket - * exists to close, just from the opposite direction. Returns `undefined` - * (no write) when the record is already gone, matching - * `markWorkflowDeploymentRecordParked`'s no-op-on-missing shape. - */ -export async function recordWorkflowDeploymentRestoreFailure( - dataDir: string, - deploymentId: string, - failure: { kind: RestoreFailureKind; reason: string }, -): Promise { - const existing = await readWorkflowDeploymentRecord(dataDir, deploymentId); - if (existing === undefined) return undefined; - const previous = existing.restoreFailure; - const attempts = - previous !== undefined && previous.kind === failure.kind ? previous.attempts + 1 : 1; - const updated: WorkflowDeploymentRecord = { - ...existing, - restoreFailure: { - kind: failure.kind, - attempts, - reason: failure.reason, - lastAttemptAt: new Date().toISOString(), - }, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, updated); - return updated; -} - -/** - * Clear a record's failure tracking after a restore succeeds. A no-op - * (no write) when there is nothing to clear, so a caller can call this - * unconditionally after every successful restore. - */ -export async function clearWorkflowDeploymentRestoreFailure( - dataDir: string, - deploymentId: string, - record: WorkflowDeploymentRecord, -): Promise { - if (record.restoreFailure === undefined) return; - const { restoreFailure: _restoreFailure, ...rest } = record; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, rest); -} - -/** - * Whether a record's next boot-time restore attempt should be skipped - * entirely: a "permanent"-kind failure that has reached - * `RESTORE_QUARANTINE_THRESHOLD` consecutive attempts. A "transient"-kind - * streak never quarantines, however high its own counter climbs -- see - * `WorkflowRestoreFailure`. - */ -export function isWorkflowDeploymentRestoreQuarantined(record: WorkflowDeploymentRecord): boolean { - return ( - record.restoreFailure?.kind === "permanent" && - record.restoreFailure.attempts >= RESTORE_QUARANTINE_THRESHOLD - ); -} - -/** - * Remove a deployment record. Called on undeploy and on a soft-failed - * deploy so a torn-down or never-completed deployment is not restored on - * the next boot. A missing record is not an error (`force`). - */ -export async function deleteWorkflowDeploymentRecord( - dataDir: string, - deploymentId: string, -): Promise { - await rm(recordPath(dataDir, deploymentId), { force: true }); -} - -/** A restorable deployment: its directory-derived id plus the validated record. */ -export interface ScannedWorkflowDeployment { - /** The `workflow-runs/` directory name the record was found under. */ - deploymentId: string; - record: WorkflowDeploymentRecord; -} - -/** - * Splits a boot scan into what still needs restoring and what a prior - * hibernate already put to sleep. `parkedAt` is the durable, locally - * decidable signal for "genuinely live" (CL-6282): a deployment the hub - * deliberately hibernated resumes on the next message or routine fire - * that addresses it, not by being pre-spawned at boot. - */ -export function partitionScannedDeployments(scanned: readonly ScannedWorkflowDeployment[]): { - live: ScannedWorkflowDeployment[]; - parked: ScannedWorkflowDeployment[]; -} { - const live: ScannedWorkflowDeployment[] = []; - const parked: ScannedWorkflowDeployment[] = []; - for (const entry of scanned) { - (entry.record.parkedAt === undefined ? live : parked).push(entry); - } - return { live, parked }; -} - -/** - * Enumerate the persisted deployment records under `workflow-runs/` so a - * boot-time restore can re-establish each deployment. Soft-fails per record: - * a missing `deployment.json`, unparseable JSON, or a record that fails schema - * validation is logged and skipped rather than wedging the whole boot -- one - * corrupt record must not strand every other deployment. An absent - * `workflow-runs/` directory is the legitimate first-boot case and yields an - * empty list, not an error. - * - * The returned `deploymentId` is the directory name; the caller cross-checks it - * against the record's own address before trusting it. - */ -export async function scanWorkflowDeploymentRecords( - dataDir: string, -): Promise { - const runsDir = pathJoin(dataDir, "workflow-runs"); - let entries; - try { - entries = await readdir(runsDir, { withFileTypes: true }); - } catch (cause) { - if (isErrnoNotFound(cause)) return []; - throw cause; - } - - const scanned: ScannedWorkflowDeployment[] = []; - let reapedCount = 0; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - const deploymentId = entry.name; - const path = recordPath(dataDir, deploymentId); - - let raw: string; - try { - raw = await readFile(path, "utf8"); - } catch (cause) { - // A run directory with no record: a crash between mkdir and the record - // write, or a run whose record was already reclaimed. Nothing to - // restore from -- skip. - if (isErrnoNotFound(cause)) { - logger.warn`skipping workflow-runs/${deploymentId}: no ${RECORD_FILENAME} to restore from`; - continue; - } - throw cause; - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} is not valid JSON: ${reason}`; - continue; - } - - const record = WorkflowDeploymentRecord(parsed); - if (record instanceof type.errors) { - if (isPreCutoverShape(record)) { - // Reap rather than warn-forever: this is the deliberate format - // cutover, and the hard-cutover/wipe policy applies to dev data. - // Removes this deployment's whole `workflow-runs//` directory - // (record plus its co-located run substrate) so the next boot has - // nothing left to warn about -- never touches a sibling - // deployment's directory or any other data store. - await rm(dirname(path), { recursive: true, force: true }); - reapedCount += 1; - continue; - } - logger.warn`skipping workflow-runs/${deploymentId}: ${RECORD_FILENAME} failed validation: ${record.summary}`; - continue; - } - scanned.push({ deploymentId, record }); - } - if (reapedCount > 0) { - logger.info`reaped ${reapedCount} pre-cutover deployment record(s)`; - } - return scanned; -} diff --git a/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts new file mode 100644 index 0000000000..689967adfc --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring-deploy-failure.test.ts @@ -0,0 +1,456 @@ +// Pins H-A1: a deploy-router rejection must not leave the +// `DeploymentAddressRegistry` populated. The multi-step branch defers +// `registerDeployment` until every step that can throw (asset +// materialization, `supervisor.spawn`) has succeeded. The link's +// `handleAgentDeploy` catches a rejection and sends `agent.error` +// without invoking `deployRouter.undeploy(frame)`, so a premature +// registration would retain a `(anchorRunId -> agentAddress)` mapping +// for a deployment that does not exist. The multi-step test drives that +// failure through a subprocess spawner that throws synchronously. +// +// The first test pins the router's frame-shape guard: a frame carrying +// neither `provisionStep` nor a workflow definition is rejected before +// any deploy work, so a malformed frame cannot leak registry state. + +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join as pathJoin } from "node:path"; + +import { describe, test, expect } from "bun:test"; +import { createInMemoryTransport } from "@intx/mail-memory"; +import { createEd25519Crypto, createNoopCredentialCipher, generateKeyPair } from "@intx/crypto"; +import type { RepoId, RepoStore } from "@intx/hub-sessions"; +import type { AgentDeployFrame } from "@intx/types/sidecar"; +import type { WorkflowDefinition } from "@intx/workflow"; +import type { SubprocessSpawner } from "@intx/workflow-host"; + +import { + createDeploymentAddressRegistry, + createMultistepDrainRouter, + createMultistepMailRouter, + createMultistepSignalRouter, +} from "./workflow-run-pack-client"; +import { createSidecarDeployRouter } from "./workflow-host-wiring"; + +function stubKeyStore(): Parameters[0]["keyStore"] { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + return { + async loadOrGenerateKey() { + // The single-step multi-step branch registers the agent's signing + // key on the host transport before `spawn()` (OUTBOUND half of + // mailbox ownership). Return a real keypair so that registration + // succeeds and the SPAWNER failure remains the failure this test + // exercises. + return { keyPair: await generateKeyPair(), isNew: false }; + }, + recordHubKey() { + /* no-op */ + }, + verifyDeployCommit() { + return true; + }, + forgetAgent() { + /* no-op */ + }, + } as unknown as Parameters[0]["keyStore"]; +} + +function stubFailingSessions(): Parameters[0]["sessions"] { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + return { + async provisionAgent() { + throw new Error("provisionAgent forced failure"); + }, + } as unknown as Parameters[0]["sessions"]; +} + +// Source-ref is the only deploy lineage: the router derives the runnable +// definition by materializing the pin's closure through this dependency. These +// spawn-failure tests need the closure to SUCCEED so the deploy reaches the +// spawn it exercises, so the stub returns a valid single-step live definition +// (its one step carries an agent so it survives `projectLiveToInert`), keyed to +// the frame's single source id `s1`. +function stubApplyFrozenWorkflowClosure( + definitionId: string, +): NonNullable[0]["applyFrozenWorkflowClosure"]> { + return (applyArgs) => + Promise.resolve({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a hand-built live definition cannot satisfy the full WorkflowDefinition nominal type; it stands in for a real closure evaluation + definition: { + id: definitionId, + triggers: [{ type: "manual" }], + stepOrder: ["s1"], + steps: { + s1: { + kind: "step", + id: "s1", + agent: { + id: "agent-s1", + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }, + }, + } as unknown as WorkflowDefinition, + packageDir: pathJoin(applyArgs.instanceDir, "package"), + deployDir: pathJoin(applyArgs.instanceDir, "deploy"), + }); +} + +function makeRouterDeps() { + const registry = createDeploymentAddressRegistry(); + const mailRouter = createMultistepMailRouter(); + const signalRouter = createMultistepSignalRouter(); + const drainRouter = createMultistepDrainRouter(); + const transport = createInMemoryTransport(); + return { registry, mailRouter, signalRouter, drainRouter, transport }; +} + +describe("deploy-failure registry leak", () => { + test("a frame carrying neither provisionStep nor a workflow is rejected before any deploy work", async () => { + const { registry, mailRouter, signalRouter, drainRouter, transport } = makeRouterDeps(); + + let spawnerInvoked = false; + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before sessions is touched + sessions: {} as Parameters[0]["sessions"], + keyStore: stubKeyStore(), + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- an unsupported frame throws before any repoStore usage + repoStore: {} as Parameters[0]["repoStore"], + signingKeySeed: new Uint8Array(32), + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: ({ runId, agentAddress }) => { + registry.record(runId, agentAddress); + }, + unregisterDeployment: ({ runId }) => { + registry.unregister(runId); + }, + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + multistepSubprocessSpawner: () => { + spawnerInvoked = true; + throw new Error("spawner must not run for an unsupported frame"); + }, + }); + + // Neither `provisionStep` nor a workflow definition: the router has + // no path to stage this frame through the substrate, so it rejects on + // shape before reaching any deploy work. + const frame: AgentDeployFrame = { + type: "agent.deploy", + agentAddress: "agent-unsupported@x.example", + agentId: "agent-unsupported", + hubPublicKey: "00".repeat(32), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- config is irrelevant; the frame is rejected on shape before config is read + config: {} as unknown as AgentDeployFrame["config"], + }; + + await expect(router.deploy(frame)).rejects.toThrow(/unsupported deploy frame/); + // The throw fires before any deploy work: no spawn, and the registry + // is never touched. + expect(spawnerInvoked).toBe(false); + const slug = "agent-unsupported-x-example"; + expect(registry.resolve(slug)).toBeNull(); + }); + + test("multi-step deploy: spawn-time failure leaves registry clean", async () => { + const { registry, mailRouter, signalRouter, drainRouter, transport } = makeRouterDeps(); + const sessions = stubFailingSessions(); + const keyStore = stubKeyStore(); + + const failingSpawner: SubprocessSpawner = () => { + throw new Error("spawner forced failure"); + }; + + const tmpDir = mkdtempSync(pathJoin(tmpdir(), "h-a1-deploy-failure-")); + + // The grants bridge writes `state/grants.json` to each step's + // agent-state repo before `spawn()`; supply a minimal RepoStore that + // honors `getRepoDir` + `writeTree` so the bridge succeeds and the + // SPAWNER failure is the one this test exercises. + const repoStoreStub: Partial = { + getRepoDir(repoId: RepoId): string { + return pathJoin(tmpDir, repoId.kind, repoId.id); + }, + writeTree(_p, repoId, _ref, content) { + const dir = pathJoin(tmpDir, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = pathJoin(dir, relPath); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, contents); + } + return Promise.resolve({ + commitSha: "stub-sha", + newlyTerminalRuns: [], + }); + }, + }; + + const router = createSidecarDeployRouter({ + sessions, + keyStore, + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure + repoStore: repoStoreStub as RepoStore, + signingKeySeed: new Uint8Array(32), + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: ({ runId, agentAddress }) => { + registry.record(runId, agentAddress); + }, + unregisterDeployment: ({ runId }) => { + registry.unregister(runId); + }, + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: tmpDir, + SIDECAR_SIGNING_PUBLIC_KEY: "00".repeat(32), + SIDECAR_SIGNING_PRIVATE_KEY: "00".repeat(32), + HUB_WS_URL: "ws://test", + SIDECAR_ID: "sc", + SIDECAR_TOKEN: "tok", + PATH: "/usr/bin", + // Source-ref materialization reads both byte caps from the substrate env. + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }, + multistepSubprocessSpawner: failingSpawner, + applyFrozenWorkflowClosure: stubApplyFrozenWorkflowClosure("wf-1"), + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + // Single-step projection: the deploy router parses the frame + // address into the legacy agent-state repo id, so it must carry the + // canonical `run_@` shape. + agentAddress: "run_mstep@x.example", + agentId: "mstep", + hubPublicKey: "00".repeat(32), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- multi-step branch does not consult config before failing + config: { + agentAddress: "run_mstep@x.example", + agentId: "mstep", + sessionId: "s", + sources: [], + defaultSource: "primary", + grants: [], + } as unknown as AgentDeployFrame["config"], + workflow: { + // Production always stamps the hub-approved wire hash; a placeholder + // satisfies the deploy path's fail-loud guard so this test reaches the + // spawn-failure behavior it exercises. + approvedWireHash: "a".repeat(64), + // Source-ref is the only deploy lineage: the frame carries no inline + // definition, only the pin the sidecar re-materializes (the injected + // closure stub above evaluates it to a single-step `s1` definition). + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + sources: { + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-x", + model: "claude-3-5", + }, + ], + }, + }, + }; + + let threw = false; + try { + await router.deploy(frame); + } catch { + threw = true; + } + expect(threw).toBe(true); + + const slug = "run_mstep-x-example"; + expect(registry.resolve(slug)).toBeNull(); + }); + + test("single-step spawn failure reverses the recorded hub key but preserves the head repo", async () => { + const { registry, mailRouter, signalRouter, drainRouter, transport } = makeRouterDeps(); + + const initedRepos: string[] = []; + const removedRepos: string[] = []; + const recordedHubKeys: string[] = []; + const forgottenAgents: string[] = []; + + // A sessions stub whose `initRepo` succeeds (so the single-step head + // proceeds to record the hub key and then fail at spawn) and whose + // `deleteAgentDir` is spied: the unwind must NOT call it, because the + // head repo directory also holds the durable identity keypair. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only initRepo runs on the happy path; deleteAgentDir is spied to prove it is NOT called + const sessions = { + initRepo: async (address: string) => { + initedRepos.push(address); + }, + deleteAgentDir: async (address: string) => { + removedRepos.push(address); + }, + } as unknown as Parameters[0]["sessions"]; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + const keyStore = { + async loadOrGenerateKey() { + return { keyPair: await generateKeyPair(), isNew: false }; + }, + recordHubKey(address: string) { + recordedHubKeys.push(address); + }, + verifyDeployCommit() { + return true; + }, + forgetAgent(address: string) { + forgottenAgents.push(address); + }, + } as unknown as Parameters[0]["keyStore"]; + + const failingSpawner: SubprocessSpawner = () => { + throw new Error("spawner forced failure"); + }; + + const tmpDir = mkdtempSync(pathJoin(tmpdir(), "single-step-unwind-")); + + const repoStoreStub: Partial = { + getRepoDir(repoId: RepoId): string { + return pathJoin(tmpDir, repoId.kind, repoId.id); + }, + writeTree(_p, repoId, _ref, content) { + const dir = pathJoin(tmpDir, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = pathJoin(dir, relPath); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, contents); + } + return Promise.resolve({ + commitSha: "stub-sha", + newlyTerminalRuns: [], + }); + }, + }; + + const router = createSidecarDeployRouter({ + sessions, + keyStore, + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub: only getRepoDir + writeTree are exercised before the spawn-time failure + repoStore: repoStoreStub as RepoStore, + signingKeySeed: new Uint8Array(32), + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: ({ runId, agentAddress }) => { + registry.record(runId, agentAddress); + }, + unregisterDeployment: ({ runId }) => { + registry.unregister(runId); + }, + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: tmpDir, + SIDECAR_SIGNING_PUBLIC_KEY: "00".repeat(32), + SIDECAR_SIGNING_PRIVATE_KEY: "00".repeat(32), + HUB_WS_URL: "ws://test", + SIDECAR_ID: "sc", + SIDECAR_TOKEN: "tok", + PATH: "/usr/bin", + // Source-ref materialization reads both byte caps from the substrate env. + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }, + multistepSubprocessSpawner: failingSpawner, + applyFrozenWorkflowClosure: stubApplyFrozenWorkflowClosure("wf-single"), + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + agentAddress: "run_single@x.example", + agentId: "single", + hubPublicKey: "00".repeat(32), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- single-step branch does not consult config before failing at spawn + config: { + agentAddress: "run_single@x.example", + agentId: "single", + sessionId: "s", + sources: [], + defaultSource: "primary", + grants: [], + } as unknown as AgentDeployFrame["config"], + workflow: { + // Placeholder hub-approved wire hash so the deploy path's fail-loud + // guard passes; production always stamps it. + approvedWireHash: "a".repeat(64), + // Source-ref is the only deploy lineage: the frame carries no inline + // definition, only the pin the sidecar re-materializes (the injected + // closure stub above evaluates it to a single-step `s1` definition). + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + sources: { + s1: [ + { + id: "primary", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-x", + model: "claude-3-5", + }, + ], + }, + }, + }; + + await expect(router.deploy(frame)).rejects.toThrow(); + + // The single-step head initialized its repo and recorded the hub key + // before the spawn failed. + expect(initedRepos).toEqual(["run_single@x.example"]); + expect(recordedHubKeys).toEqual(["run_single@x.example"]); + // The unwind reversed the in-memory hub key... + expect(forgottenAgents).toEqual(["run_single@x.example"]); + // ...but did NOT remove the head repo directory (it holds the durable + // identity keypair a rerouted head must keep across a failed redeploy). + expect(removedRepos).toEqual([]); + // Registry untouched. + expect(registry.resolve("run_single-x-example")).toBeNull(); + }); +}); diff --git a/apps/sidecar/src/workflow-host-wiring-source-assets.test.ts b/apps/sidecar/src/workflow-host-wiring-source-assets.test.ts new file mode 100644 index 0000000000..3d2bd7aae2 --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring-source-assets.test.ts @@ -0,0 +1,183 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { SourceRefPin } from "@intx/types/sidecar"; + +import { resolveDeploymentAssetMounts } from "./workflow-host-wiring"; + +// The durable source-asset store is what makes an asset-sourced deployment +// survive a restart: the deploy checks the assets out once, and both the deploy +// apply and the boot-time restore re-materialize from the store using mounts +// derived from the pin ALONE (restore has no re-delivery). These tests pin that +// derivation and its fail-loud presence check without spinning up a deployment. + +const DEPLOYMENT_ID = "dep-1"; + +function assetPin(assetIds: readonly string[]): SourceRefPin { + return { + source: { + kind: "asset", + assetId: assetIds[0] ?? "asset_top", + package: { format: "tarball" }, + }, + closure: { + schemaVersion: "1", + topLevel: [{ name: "@x/wf", version: "1.0.0" }], + entries: assetIds.map((assetId, i) => ({ + name: i === 0 ? "@x/wf" : `@x/dep-${String(i)}`, + version: "1.0.0", + source: { + kind: "asset", + assetId, + package: { + format: "tarball", + path: "tarballs/x.tgz", + integrity: "sha512-placeholder", + }, + }, + })), + }, + }; +} + +// A source-format pin: every member is a `format:"source"` entry backed by one +// asset checked out into the durable indexed-`.git` store. Restore derives the +// `assetId -> gitDir` map from this pin alone. +function sourcePin(assetId: string, packageDirs: readonly string[]): SourceRefPin { + return { + source: { + kind: "asset", + assetId, + package: { format: "source", commitSha: "commit-abc" }, + }, + closure: { + schemaVersion: "1", + topLevel: [{ name: "@x/wf", version: "1.0.0" }], + entries: packageDirs.map((packageDir, i) => ({ + name: i === 0 ? "@x/wf" : `@x/member-${String(i)}`, + version: "1.0.0", + source: { + kind: "asset", + assetId, + package: { + format: "source", + commitSha: "commit-abc", + packageDir, + treeOid: `tree-${String(i)}`, + }, + }, + })), + }, + }; +} + +let dataDir: string; + +beforeEach(async () => { + dataDir = await fs.mkdtemp(path.join(os.tmpdir(), "wf-deploy-source-assets-")); +}); + +afterEach(async () => { + await fs.rm(dataDir, { recursive: true, force: true }); +}); + +describe("resolveDeploymentAssetMounts", () => { + test("derives the pin's mount map when the store is populated", async () => { + const assetId = "asset_top"; + // Populate the durable store exactly as a deploy checkout would; restore + // reads only this, deriving the mount from the pin. + await fs.mkdir( + path.join(dataDir, "workflow-definition-sources", DEPLOYMENT_ID, "source-assets", assetId), + { recursive: true }, + ); + + const { assetRoot, assetMounts } = await resolveDeploymentAssetMounts( + dataDir, + DEPLOYMENT_ID, + assetPin([assetId]), + ); + + expect(assetRoot).toBe(path.join(dataDir, "workflow-definition-sources", DEPLOYMENT_ID)); + expect([...assetMounts]).toEqual([[assetId, `source-assets/${assetId}/`]]); + }); + + test("dedupes many entries backed by one asset into a single mount", async () => { + const assetId = "asset_top"; + await fs.mkdir( + path.join(dataDir, "workflow-definition-sources", DEPLOYMENT_ID, "source-assets", assetId), + { recursive: true }, + ); + // Two closure entries, same backing asset. + const { assetMounts } = await resolveDeploymentAssetMounts( + dataDir, + DEPLOYMENT_ID, + assetPin([assetId, assetId]), + ); + expect(assetMounts.size).toBe(1); + }); + + test("fails loud when a pinned asset is absent from the store", async () => { + await expect( + resolveDeploymentAssetMounts(dataDir, DEPLOYMENT_ID, assetPin(["asset_missing"])), + ).rejects.toThrow(/is not present in the durable store/); + }); + + test("derives the pin's gitDir map when the durable git store is populated", async () => { + const assetId = "asset_top"; + // Populate the durable indexed-`.git` store exactly as a deploy checkout + // would; restore reads only this, deriving the gitDir from the pin. Two + // members share one backing asset, so both resolve to one gitDir. + await fs.mkdir(path.join(dataDir, "workflow-definition-source-gits", DEPLOYMENT_ID, assetId), { + recursive: true, + }); + + const { gitDirs, assetMounts } = await resolveDeploymentAssetMounts( + dataDir, + DEPLOYMENT_ID, + sourcePin(assetId, [".", "packages/member-1"]), + ); + + // No tarball entries, so no plain-file mounts. + expect(assetMounts.size).toBe(0); + expect([...gitDirs]).toEqual([ + [assetId, path.join(dataDir, "workflow-definition-source-gits", DEPLOYMENT_ID, assetId)], + ]); + }); + + test("fails loud when a source pin's git store is absent", async () => { + await expect( + resolveDeploymentAssetMounts(dataDir, DEPLOYMENT_ID, sourcePin("asset_missing", ["."])), + ).rejects.toThrow(/has no indexed git store/); + }); + + test("returns an empty mount map for a registry-sourced pin", async () => { + const registryPin: SourceRefPin = { + source: { kind: "registry", registry: "npmjs" }, + closure: { + schemaVersion: "1", + topLevel: [{ name: "@x/wf", version: "1.0.0" }], + entries: [ + { + name: "@x/wf", + version: "1.0.0", + source: { + kind: "registry", + registry: "npmjs", + integrity: "sha512-placeholder", + }, + }, + ], + }, + }; + + const { assetMounts, gitDirs } = await resolveDeploymentAssetMounts( + dataDir, + DEPLOYMENT_ID, + registryPin, + ); + expect(assetMounts.size).toBe(0); + expect(gitDirs.size).toBe(0); + }); +}); diff --git a/apps/sidecar/src/workflow-host-wiring-spawner.test.ts b/apps/sidecar/src/workflow-host-wiring-spawner.test.ts new file mode 100644 index 0000000000..46ac1b2eb1 --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring-spawner.test.ts @@ -0,0 +1,213 @@ +// End-to-end exercise of the Bun.spawn-backed `defaultSubprocessSpawner` +// the wiring module exports. The supervisor's `wireChild` consumes +// four behaviours from the handle: +// +// 1. The control channel surfaces NDJSON lines the child writes +// to its stdout. A real child writes a signed `ready` envelope +// here; the test stub writes a sentinel NDJSON line. +// 2. The handle's `exited` future resolves with the child's +// terminal exit code so the supervisor can race spawn-time +// crashes against `readyPromise`. +// 3. A `Bun.spawn` that fails immediately (binary missing) settles +// `exited` with a non-zero code rapidly enough for that race +// to fire. +// 4. The child's runtime env is exactly the supervisor-supplied +// env -- unrelated `process.env` entries do not leak in. + +import { describe, test, expect, beforeAll, afterAll } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { defaultSubprocessSpawner } from "./workflow-host-wiring"; + +let tmpRoot: string; + +beforeAll(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "intx-spawner-test-")); +}); + +afterAll(async () => { + await fs.rm(tmpRoot, { recursive: true, force: true }); +}); + +async function writeChildScript(body: string): Promise { + const file = path.join(tmpRoot, `child-${String(Date.now())}.ts`); + await fs.writeFile(file, body, "utf-8"); + return file; +} + +describe("defaultSubprocessSpawner (Bun.spawn-backed)", () => { + test("surfaces the child's NDJSON output on controlReader and resolves exited with the exit code", async () => { + // Inline child: + // - reads CHILD_TOKEN out of the env and echoes it + // - reads SHOULD_NOT_LEAK to prove env isolation (not set by + // this test's spawner call, so the child must observe it + // as the sentinel string) + // - writes one byte sequence to fd 3 (event channel) + // - emits one NDJSON line on stdout + // - exits 0 + const childScript = ` +import fs from "node:fs"; +const token = process.env.CHILD_TOKEN ?? "MISSING"; +const leaked = process.env.SHOULD_NOT_LEAK ?? "UNSET"; +const event = new TextEncoder().encode("event-byte"); +const eventStream = fs.createWriteStream("", { fd: 3 }); +eventStream.write(event, () => { + process.stdout.write( + JSON.stringify({ probe: "ready", token, leaked }) + "\\n", + () => process.exit(0), + ); +}); +`; + const scriptPath = await writeChildScript(childScript); + + // The wiring module's spawner is a bare Bun.spawn binding; the + // production callsite invokes the binary directly. The script + // path here points to a TypeScript file which requires Bun's + // shebang to execute; the production binary at + // `apps/sidecar/bin/workflow-child` is + // `#!/usr/bin/env -S bun --conditions=intx-src`, so the wiring + // module spawns it as a bare argv entry. The + // test mirrors that by routing the spawn through a tiny + // wrapper script that points at Bun's runtime. + const wrapperPath = path.join(tmpRoot, `wrapper-${String(Date.now())}.sh`); + await fs.writeFile( + wrapperPath, + `#!/bin/sh\nexec "${process.execPath}" "${scriptPath}"\n`, + "utf-8", + ); + await fs.chmod(wrapperPath, 0o755); + + // Set SHOULD_NOT_LEAK on the test process; the spawner must NOT + // forward it to the child because the env arg below excludes + // it. The child observes the unset variable as "UNSET". + process.env.SHOULD_NOT_LEAK = "leaked"; + try { + const handle = defaultSubprocessSpawner({ + binaryPath: wrapperPath, + env: { CHILD_TOKEN: "spawner-test-value" }, + }); + + const ctrlIter = handle.controlReader.read(); + const first = await ctrlIter.next(); + expect(first.done).toBeFalsy(); + if (first.value === undefined) { + throw new Error("control reader yielded undefined first value"); + } + const parsed: unknown = JSON.parse(first.value); + if ( + typeof parsed !== "object" || + parsed === null || + !("probe" in parsed) || + !("token" in parsed) || + !("leaked" in parsed) + ) { + throw new Error(`child NDJSON payload missing expected fields: ${JSON.stringify(parsed)}`); + } + expect(parsed.probe).toBe("ready"); + expect(parsed.token).toBe("spawner-test-value"); + expect(parsed.leaked).toBe("UNSET"); + + // Drain one frame off the event channel. The child wrote one + // byte sequence to fd 3 before exiting; the supervisor's + // FrameReader contract yields one Uint8Array per kernel- + // delivered chunk. + const eventIter = handle.eventReader.read(); + const chunk = await eventIter.next(); + expect(chunk.done).toBeFalsy(); + if (chunk.value === undefined) { + throw new Error("event reader yielded undefined value"); + } + expect(new TextDecoder().decode(chunk.value)).toBe("event-byte"); + + const code = await handle.exited; + expect(code).toBe(0); + } finally { + delete process.env.SHOULD_NOT_LEAK; + } + }); + + test("spawn-time failure surfaces fast enough for the supervisor's readyPromise race", async () => { + // The supervisor's `wireChild` races `handle.exited` against + // the child's `readyPromise`. A child that fails to reach + // `ready` (the production binary's failure mode for malformed + // env: throws in `parseSpawnTimeEnv` before opening the + // control channel) must surface as `exited` settling with a + // non-zero code -- otherwise the supervisor wedges in + // `starting`. The inline child below exits 1 without writing + // a single byte; the spawner's `exited` future must observe + // the terminal code. + const childScript = `process.exit(7);`; + const scriptPath = await writeChildScript(childScript); + const wrapperPath = path.join(tmpRoot, `wrapper-fail-${String(Date.now())}.sh`); + await fs.writeFile( + wrapperPath, + `#!/bin/sh\nexec "${process.execPath}" "${scriptPath}"\n`, + "utf-8", + ); + await fs.chmod(wrapperPath, 0o755); + + const handle = defaultSubprocessSpawner({ + binaryPath: wrapperPath, + env: {}, + }); + const code = await handle.exited; + expect(code).toBe(7); + }); + + test("Bun.spawn synchronous throws on a missing binary surface to the caller", () => { + // The other half of the spawn-time race: a binary path that + // does not exist on disk surfaces as a synchronous throw from + // `Bun.spawn`, which the supervisor's `spawn(opts)` propagates + // to its caller. The supervisor never reaches the + // readyPromise race in this branch; the throw IS the failure + // signal. Pin the synchronous-throw behaviour here so a future + // Bun release that swaps to an async failure mode does not + // silently change the supervisor's spawn-error surface. + expect(() => { + defaultSubprocessSpawner({ + binaryPath: "/nonexistent-binary-for-spawner-test", + env: {}, + }); + }).toThrow(); + }); + + test("kill() forwards the recycle path's SIGTERM and SIGKILL signals", async () => { + // The supervisor's recycle path calls `handle.kill("SIGTERM")` + // and then `handle.kill("SIGKILL")`. Assert both crossings + // succeed; the bytes the supervisor sends are these exact + // strings. + const childScript = ` +import fs from "node:fs"; +process.stdout.write(JSON.stringify({ probe: "ready" }) + "\\n"); +const eventStream = fs.createWriteStream("", { fd: 3 }); +void eventStream; +await new Promise((r) => setTimeout(r, 5000)); +process.exit(0); +`; + const scriptPath = await writeChildScript(childScript); + const wrapperPath = path.join(tmpRoot, `wrapper-kill-${String(Date.now())}.sh`); + await fs.writeFile( + wrapperPath, + `#!/bin/sh\nexec "${process.execPath}" "${scriptPath}"\n`, + "utf-8", + ); + await fs.chmod(wrapperPath, 0o755); + + const handle = defaultSubprocessSpawner({ + binaryPath: wrapperPath, + env: {}, + }); + + const iter = handle.controlReader.read(); + const ready = await iter.next(); + expect(ready.done).toBeFalsy(); + + handle.kill("SIGTERM"); + handle.kill("SIGKILL"); + + const code = await handle.exited; + expect(typeof code).toBe("number"); + }); +}); diff --git a/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts new file mode 100644 index 0000000000..eedfc36eaf --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring-undeploy-supervisor.test.ts @@ -0,0 +1,456 @@ +// Pins C-A: the multi-step undeploy hook must shut down the per- +// deployment supervisor so the workflow-process child, its IPC pipes, +// and its event-channel fd are released. Without the shutdown the +// supervisor's reference outlives every other piece of routing state +// the undeploy hook tears down, leaking the child for the life of the +// sidecar. +// +// The harness drives the multi-step deploy through the same spawn +// handshake the existing wiring tests use, captures the +// `SubprocessHandle.kill` call, and asserts that `undeploy(frame)` +// invokes it and awaits the handle's `exited` settlement. + +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { createEd25519Crypto, createNoopCredentialCipher, generateKeyPair } from "@intx/crypto"; +import { hexEncode } from "@intx/types"; +import { createInMemoryTransport } from "@intx/mail-memory"; +import type { RepoId, RepoStore } from "@intx/hub-sessions"; +import { + createControlChannelSender, + type FrameReader, + type NdjsonReader, + type NdjsonWriter, + type SubprocessHandle, + type SubprocessSpawner, +} from "@intx/workflow-host"; +import type { AgentDeployFrame } from "@intx/types/sidecar"; +import type { WorkflowDefinition } from "@intx/workflow"; + +import { createSidecarDeployRouter, deriveDeploymentId } from "./workflow-host-wiring"; +import { + createMultistepDrainRouter, + createMultistepMailRouter, + createMultistepSignalRouter, +} from "./workflow-run-pack-client"; + +function createMemoryNdjsonStream() { + const buffer: string[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: NdjsonReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("buffer shift returned undefined"); + } + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + const writer: NdjsonWriter = { + write(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + return Promise.resolve(); + }, + }; + return { + writer, + reader, + inject(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + }, + close() { + done = true; + wake(); + }, + }; +} + +function createMemoryFrameStream() { + const buffer: Uint8Array[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: FrameReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("frame buffer shift returned undefined"); + } + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + return { + reader, + close() { + done = true; + wake(); + }, + }; +} + +function createSpawnTestRepoStore(tempBase: string): RepoStore { + const stub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(tempBase, repoId.kind, repoId.id); + }, + async writeTreePreservingPrefix(_p, _id, _ref, args) { + await args.merge(new Map()); + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + // The deploy router's grants bridge writes `state/grants.json` to + // each step's agent-state repo before `spawn()`. Mirror the + // `getRepoDir` layout so the write lands where the subsequent + // `assembleCredentialsSnapshot` working-tree read looks for it. + async writeTree(_p, repoId, _ref, content) { + const dir = path.join(tempBase, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = path.join(dir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, contents); + } + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented for this test`); + }; + }, + }); +} + +describe("createSidecarDeployRouter multi-step undeploy shuts the supervisor down", () => { + test("undeploy invokes the spawned child's kill and awaits exited", async () => { + // Per-spawn tracking. + type Spawn = { + handle: SubprocessHandle; + childToSupervisor: ReturnType; + supervisorToChild: ReturnType; + eventChildToSupervisor: ReturnType; + env: Record; + killed: boolean; + exitedResolved: boolean; + resolveExited: (code: number) => void; + }; + const spawns: Spawn[] = []; + + const spawner: SubprocessSpawner = ({ env }) => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const entry: Spawn = { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- assigned below + handle: undefined as unknown as SubprocessHandle, + supervisorToChild, + childToSupervisor, + eventChildToSupervisor, + env, + killed: false, + exitedResolved: false, + resolveExited: (code) => { + entry.exitedResolved = true; + resolveExit?.(code); + }, + }; + const handle: SubprocessHandle = { + pid: 5100 + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + entry.killed = true; + childToSupervisor.close(); + eventChildToSupervisor.close(); + entry.resolveExited(0); + }, + exited, + }; + entry.handle = handle; + spawns.push(entry); + return handle; + }; + + const transport = createInMemoryTransport(); + const keyPair = await generateKeyPair(); + const tempBase = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-undeploy-supervisor-")); + const dataDir = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-undeploy-supervisor-data-")); + const repoStore = createSpawnTestRepoStore(tempBase); + + const mailRouter = createMultistepMailRouter(); + const signalRouter = createMultistepSignalRouter(); + const drainRouter = createMultistepDrainRouter(); + + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) + sessions: { + provisionAgent: async () => { + throw new Error("single-step branch must not invoke provisionAgent"); + }, + persistHubPublicKey: async () => { + throw new Error("single-step branch must not invoke persistHubPublicKey"); + }, + initRepo: async () => undefined, + } as unknown as Parameters[0]["sessions"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn + keyStore: { + recordHubKey: () => undefined, + loadOrGenerateKey: async () => ({ + keyPair: await generateKeyPair(), + isNew: false, + }), + } as unknown as Parameters[0]["keyStore"], + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: () => { + /* no-op */ + }, + unregisterDeployment: () => { + /* no-op */ + }, + multistepSubprocessSpawner: spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: dataDir, + // Source-ref materialization reads both byte caps from the substrate env. + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }, + // Source-ref is the only deploy lineage: the router derives the runnable + // definition by materializing the pin's closure through this dependency. + // The stub returns a valid single-step `step-1` live definition (its step + // carries an agent so it survives `projectLiveToInert`) so the deploy + // reaches the spawn/undeploy behavior this test exercises. + applyFrozenWorkflowClosure: (applyArgs) => + Promise.resolve({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a hand-built live definition cannot satisfy the full WorkflowDefinition nominal type; it stands in for a real closure evaluation + definition: { + id: "wf-undeploy-supervisor", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { + "step-1": { + kind: "step", + id: "step-1", + agent: { + id: "agent-step-1", + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }, + }, + } as unknown as WorkflowDefinition, + packageDir: path.join(applyArgs.instanceDir, "package"), + deployDir: path.join(applyArgs.instanceDir, "deploy"), + }), + multistepMailRouter: mailRouter, + multistepSignalRouter: signalRouter, + multistepDrainRouter: drainRouter, + }); + + const frame: AgentDeployFrame = { + type: "agent.deploy", + // Single-step projection: the deploy router derives the sole + // step's agent-state repo from `parseAgentId(agentAddress)`, which + // requires the canonical `run_@` instance shape. + agentAddress: "run_undeploy-supervisor@example.com", + agentId: "undeploy-supervisor-agent", + hubPublicKey: "hub-pk", + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the multi-step branch does not read config + config: {} as AgentDeployFrame["config"], + workflow: { + // Placeholder hub-approved wire hash so the deploy path's fail-loud + // guard passes; production always stamps it. + approvedWireHash: "a".repeat(64), + // Source-ref is the only deploy lineage: the frame carries no inline + // definition, only the pin the sidecar re-materializes (the injected + // closure stub above evaluates it to a single-step `step-1` definition). + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + sources: { + "step-1": [ + { + id: "step-1", + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-step-1", + model: "claude-3-5", + }, + ], + }, + }, + }; + + const deployPromise = router.deploy(frame); + + while (spawns.length === 0) { + await new Promise((r) => setTimeout(r, 1)); + } + const spawn = spawns[0]; + if (spawn === undefined) throw new Error("unreachable"); + + const channelId = spawn.env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing from spawn env"); + } + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + spawn.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: spawn.handle.pid, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + + await deployPromise; + + expect(spawn.killed).toBe(false); + expect(spawn.exitedResolved).toBe(false); + + const undeploy = router.undeploy; + if (undeploy === undefined) { + throw new Error("router.undeploy is undefined"); + } + + // Pre-seed the on-disk per-step scratch the child roots under + // `/workflow-step-state//` and the durable + // conversation under `/agent-conversation-state//`. + // The warm subtree is the stable per-agent workspace (one dir, not + // one-per-message); a stale cold `runs//` subtree models a + // multi-step leftover the per-run cleanup did not drop. An unrelated + // deployment's step-state subtree must survive the undeploy sweep. + const anchorRunId = deriveDeploymentId(frame.agentAddress); + const stepStateRoot = path.join(dataDir, "workflow-step-state"); + const warmWorkspaceFile = path.join( + stepStateRoot, + anchorRunId, + "warm", + encodeURIComponent("step-1"), + "workspace", + "notes.txt", + ); + const coldLeftoverFile = path.join( + stepStateRoot, + anchorRunId, + "runs", + "run-stale", + "steps", + "step-1", + "attempt-1", + "workspace", + "scratch.txt", + ); + const otherDeploymentFile = path.join( + stepStateRoot, + "other-deployment", + "warm", + "step-1", + "workspace", + "keep.txt", + ); + const durableConversationFile = path.join( + dataDir, + "agent-conversation-state", + anchorRunId, + encodeURIComponent("step-1"), + "checkpoint.json", + ); + for (const file of [ + warmWorkspaceFile, + coldLeftoverFile, + otherDeploymentFile, + durableConversationFile, + ]) { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, "x"); + } + + await undeploy({ + type: "agent.undeploy", + agentAddress: frame.agentAddress, + reason: "test undeploy", + }); + + expect(spawn.killed).toBe(true); + expect(spawn.exitedResolved).toBe(true); + + // The deployment's whole step-state subtree is reclaimed -- warm + // stable workspace AND any cold leftover -- now that its supervisor + // and child are torn down. + await expect(fs.stat(path.join(stepStateRoot, anchorRunId))).rejects.toThrow(); + // A different deployment's scratch is untouched: the sweep is scoped + // to this deployment's `` subtree only. + expect(await fs.readFile(otherDeploymentFile, "utf8")).toBe("x"); + // The durable conversation lives under a DIFFERENT root and must + // survive so a re-deploy restores the prior conversation. + expect(await fs.readFile(durableConversationFile, "utf8")).toBe("x"); + }); +}); diff --git a/apps/sidecar/src/workflow-host-wiring.test.ts b/apps/sidecar/src/workflow-host-wiring.test.ts new file mode 100644 index 0000000000..375fefa49f --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring.test.ts @@ -0,0 +1,3917 @@ +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { type } from "arktype"; + +import { createEd25519Crypto, createNoopCredentialCipher, generateKeyPair } from "@intx/crypto"; +import { hexEncode } from "@intx/types"; +import { computeWireDefinitionHash } from "@intx/types/wire-definition-hash"; +import { createInMemoryTransport } from "@intx/mail-memory"; +import type { RepoId, RepoStore } from "@intx/hub-sessions"; +import { + createControlChannelSender, + type EventPayload, + type FrameReader, + type NdjsonReader, + type NdjsonWriter, + type SubprocessHandle, + type SubprocessSpawner, +} from "@intx/workflow-host"; +import type { WorkflowDefinition } from "@intx/workflow"; +import type { AgentDeployFrame } from "@intx/types/sidecar"; + +import { + assembleRunCredentialsSnapshot, + createSidecarDeployRouter, + createSidecarWorkflowSupervisor, + deriveDeploymentId, + STEP_INFERENCE_SOURCES_ENV_KEY, + validateWorkflowProjection, +} from "./workflow-host-wiring"; +import { + createDeploymentAddressRegistry, + createMultistepGrantsRouter, + createMultistepMailRouter, + createMultistepSourcesRouter, + type MultistepGrantsRouter, + type MultistepMailRouter, + type MultistepSourcesRouter, +} from "./workflow-run-pack-client"; +import { + scanWorkflowRunRecords, + writeWorkflowRunRecord, + type WorkflowRunRecord, +} from "./workflow-run-record"; + +function createMinimalStubRepoStore(): RepoStore { + const stub: Partial = { + getRepoDir(_repoId: RepoId): string { + return "/tmp/unused"; + }, + async writeTreePreservingPrefix(_p, _id, _ref, args) { + // The wiring test exercises signature attribution by driving a + // requestCancel; the merge callback runs once with an empty + // pre-image. + await args.merge(new Map()); + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; the wiring test exercises only getRepoDir + writeTreePreservingPrefix + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented for this test`); + }; + }, + }); +} + +describe("createSidecarWorkflowSupervisor", () => { + test("constructs the supervisor with the sidecar's bindings and signs CancelRequested via the host's signing key", async () => { + const transport = createInMemoryTransport(); + const keyPair = await generateKeyPair(); + const repoStore = createMinimalStubRepoStore(); + + const spawner: SubprocessSpawner = () => { + throw new Error("spawner not invoked in this test"); + }; + + const wired = createSidecarWorkflowSupervisor({ + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + workflowRunRepoId: { kind: "workflow-run", id: "wire-test" }, + workflowRunRef: "refs/heads/main", + runId: "wire-test", + stepCount: 1, + stepOrder: ["step1"], + deploymentMailAddress: "wire-test@example.com", + deriveStepAddress: ({ runId, stepId }) => `${runId}-${stepId}@example.com`, + substrateEnv: { DATA_DIR: "/tmp/wire" }, + dynamicSpawnEnv: () => ({}), + subprocessSpawner: spawner, + }); + + expect(typeof wired.supervisor.spawn).toBe("function"); + expect(wired.getCredentialsSnapshot()).toBeNull(); + + const result = await wired.supervisor.requestCancel({ + runId: "r-wire-1", + origin: "supervisor-operator", + reason: "wiring test", + at: "2026-01-01T00:00:00.000Z", + }); + expect(result.commitSha).toBe("stub-sha"); + expect(result.seq).toBe(0); + }); + + test("routeInbound rejects when no subscriber is registered so undelivered mail is withheld", async () => { + const transport = createInMemoryTransport(); + // generateKeyPair is async; this test only exercises the + // mail-routing path so we synthesize a 32-byte seed without + // calling crypto. + const fakeSeed = new Uint8Array(32); + const repoStore = createMinimalStubRepoStore(); + const wired = createSidecarWorkflowSupervisor({ + transport, + repoStore, + signingKeySeed: fakeSeed, + workflowRunRepoId: { kind: "workflow-run", id: "inbound" }, + workflowRunRef: "refs/heads/main", + runId: "inbound", + stepCount: 1, + stepOrder: ["step1"], + deploymentMailAddress: "inbound@example.com", + deriveStepAddress: ({ runId, stepId }) => `${runId}-${stepId}@example.com`, + substrateEnv: {}, + dynamicSpawnEnv: () => ({}), + subprocessSpawner: () => { + throw new Error("spawner not invoked in this test"); + }, + }); + // Without a subscriber the delivery is not durably accepted, so + // routeInbound rejects: the hub-link then withholds the ack and the hub + // redelivers, rather than silently dropping (which under the ack model + // would be an acked loss). + await expect(wired.routeInbound(new TextEncoder().encode("hello"))).rejects.toThrow( + /no active mail subscriber/, + ); + }); + + test("onRunStart fails a poisoned run and passes an unpoisoned one", async () => { + // A poisoned run (its `run.grants` write failed) must be rejected at the + // barrier rather than started under the deploy-time grant set. An + // unpoisoned run with a per-run grants file on disk resolves normally. + const tempBase = await createTempBaseDir("sidecar-poison-barrier-"); + const anchorRunId = "dep-poison"; + const cleanRunId = "run-clean"; + const grantsDir = path.join(tempBase, "workflow-run", anchorRunId, "runs", cleanRunId); + await fs.mkdir(grantsDir, { recursive: true }); + const runGrants = [{ id: "g1", resource: "tool:send-mail", effect: "allow" }]; + await fs.writeFile(path.join(grantsDir, "grants.json"), JSON.stringify({ grants: runGrants })); + + const readStub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(tempBase, repoId.kind, repoId.id); + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; onRunStart reads only via getRepoDir working-tree reads + const repoStore = new Proxy(readStub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented`); + }; + }, + }); + + const wired = createSidecarWorkflowSupervisor({ + transport: createInMemoryTransport(), + repoStore, + signingKeySeed: new Uint8Array(32), + workflowRunRepoId: { kind: "workflow-run", id: anchorRunId }, + workflowRunRef: "refs/heads/main", + runId: anchorRunId, + stepCount: 1, + stepOrder: ["step-1"], + deploymentMailAddress: `${anchorRunId}@example.com`, + deriveStepAddress: ({ runId: dep, stepId }) => `${dep}-${stepId}@example.com`, + substrateEnv: {}, + dynamicSpawnEnv: () => ({}), + subprocessSpawner: () => { + throw new Error("spawner not invoked in this test"); + }, + isRunPoisoned: (runId) => runId === "run-poisoned", + }); + + await expect(wired.onRunStart({ runId: "run-poisoned", anchorRunId })).rejects.toThrow( + /grants write failed/, + ); + + const snapshot = await wired.onRunStart({ + runId: cleanRunId, + anchorRunId, + }); + expect(snapshot.steps).toHaveLength(1); + expect(snapshot.steps[0]?.grants).toEqual(runGrants); + }); +}); + +describe("createSidecarDeployRouter provision-step (no-spawn) mode", () => { + test("a provisionStep frame inits the repo and records the hub key without spawning", async () => { + const transport = createInMemoryTransport(); + const keyPair = await generateKeyPair(); + + const initRepoCalls: string[] = []; + const recordHubKeyCalls: { address: string; hubKey: string }[] = []; + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep touches no RepoStore method; the Proxy throws if it ever does + const repoStore = new Proxy({} as RepoStore, { + get(_target, prop) { + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented`); + }; + }, + }); + + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only initRepo + sessions: { + initRepo: async (a: string) => { + initRepoCalls.push(a); + }, + } as unknown as Parameters[0]["sessions"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep exercises only recordHubKey + keyStore: { + recordHubKey: (a: string, h: string) => { + recordHubKeyCalls.push({ address: a, hubKey: h }); + }, + } as unknown as Parameters[0]["keyStore"], + senderKeyCache: { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: () => undefined, + registerDeployment: () => undefined, + unregisterDeployment: () => undefined, + }); + + const STEP_ADDR = "run_abc-step1@example.com"; + const HUB_KEY = "aa".repeat(32); + const result = await router.deploy({ + type: "agent.deploy", + agentAddress: STEP_ADDR, + agentId: "run_abc-step1", + hubPublicKey: HUB_KEY, + provisionStep: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- provisionStep never inspects config + config: {} as unknown as Parameters< + ReturnType["deploy"] + >[0]["config"], + }); + + // The step's agent-state repo is initialized and the hub key recorded, + // so the follow-up full-closure deploy pack applies into a repo and + // verifies against the recorded key. + expect(initRepoCalls).toEqual([STEP_ADDR]); + expect(recordHubKeyCalls).toEqual([{ address: STEP_ADDR, hubKey: HUB_KEY }]); + + // The ack carries the sidecar principal key (the hub discards it for a + // workflow-derived per-step address). + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + + // Nothing spawned: no supervisor, so no active address. + expect(router.activeAddresses()).toEqual([]); + }); +}); + +// -------------------------------------------------------------------- +// Multi-step branch tests +// -------------------------------------------------------------------- + +function createMemoryNdjsonStream() { + const buffer: string[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: NdjsonReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("buffer shift returned undefined"); + } + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + const writer: NdjsonWriter = { + write(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + return Promise.resolve(); + }, + }; + return { + writer, + reader, + inject(line: string) { + buffer.push(line.replace(/\n$/, "")); + wake(); + }, + flushed(): readonly string[] { + return buffer.slice(); + }, + close() { + done = true; + wake(); + }, + }; +} + +function createMemoryFrameStream() { + const buffer: Uint8Array[] = []; + let waiter: (() => void) | null = null; + let done = false; + function wake() { + const w = waiter; + waiter = null; + if (w) w(); + } + const reader: FrameReader = { + read(): AsyncIterableIterator { + return (async function* () { + while (true) { + if (buffer.length > 0) { + const next = buffer.shift(); + if (next === undefined) { + throw new Error("frame buffer shift returned undefined"); + } + yield next; + continue; + } + if (done) return; + await new Promise((resolve) => { + waiter = resolve; + }); + } + })(); + }, + }; + return { + reader, + inject(bytes: Uint8Array) { + buffer.push(bytes); + wake(); + }, + close() { + done = true; + wake(); + }, + }; +} + +function createTempBaseDir(prefix: string): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), prefix)); +} + +// The supervisor's `parked-correlations.request` control-frame discriminator. +// The re-emit-parked-correlations tests assert this reaches the downstream +// (supervisor -> child) stream when Trigger B fires. +const PARKED_REQUEST_TYPE = "parked-correlations.request"; + +// One downstream control line is the JSON serialization of a signed envelope: +// `{ envelope: { seq, channelId, payload }, sig }`. The re-emit tests inspect +// `envelope.payload.type` to detect a `parked-correlations.request` without +// verifying the signature (the supervisor's IPC public key is not exposed to +// the test). Structurally validate the envelope-carrying shape and return the +// payload discriminator, or `undefined` when the line is not a typed frame. +const DownstreamFrameShape = type({ + envelope: { + payload: { + "type?": "string", + "+": "ignore", + }, + "+": "ignore", + }, + "+": "ignore", +}); + +function downstreamPayloadType(line: string): string | undefined { + const parsed: unknown = JSON.parse(line); + const validated = DownstreamFrameShape(parsed); + if (validated instanceof type.errors) return undefined; + return validated.envelope.payload.type; +} + +/** + * Build a stub RepoStore whose `getRepoDir` resolves under the supplied + * tempBase. The multi-step branch's `assembleCredentialsSnapshot` + * reads `state/grants.json` from disk -- missing files are treated as + * empty grants, so a freshly-created tempBase produces an empty + * credentials snapshot which is what the wiring test wants. + */ +function createSpawnTestRepoStore(tempBase: string): RepoStore { + const stub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(tempBase, repoId.kind, repoId.id); + }, + async writeTreePreservingPrefix(_p, _id, _ref, args) { + await args.merge(new Map()); + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + // The deploy router's grants bridge writes `state/grants.json` to + // each step's agent-state repo before `spawn()`. Mirror the + // `getRepoDir` layout so the write lands where the subsequent + // `assembleCredentialsSnapshot` working-tree read looks for it. + async writeTree(_p, repoId, _ref, content) { + const dir = path.join(tempBase, repoId.kind, repoId.id); + for (const [relPath, contents] of Object.entries(content.files)) { + const full = path.join(dir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, contents); + } + return { commitSha: "stub-sha", newlyTerminalRuns: [] }; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; only getRepoDir + writeTreePreservingPrefix + writeTree exercised + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented for this test`); + }; + }, + }); +} + +type WorkflowProjection = NonNullable; +// A single source: each step's `sources` value is an ordered failover chain, +// so the fixture element type is the chain's member. +type InferenceSourceFixture = WorkflowProjection["sources"][string][number]; + +type MultistepDeployArgs = { + sources: WorkflowProjection["sources"]; + definition: { + id: string; + triggers: unknown[]; + stepOrder: string[]; + steps: Record; + }; + /** + * Override the deploy frame's `agentAddress`. Single-step projections + * are the agent-launch identity path: the deploy router derives the + * sole step's agent-state repo from `parseAgentId(agentAddress)`, which + * requires the canonical `run_@` shape. Tests that drive a + * single-step projection supply a valid instance address here; the + * default keeps the historical multi-step address for the multi-step + * tests (whose derived per-step repos do not parse the frame address). + */ + agentAddress?: string; + /** + * Hub-approved wire hash to stamp on the deploy frame's workflow -- the + * child's `DEFINITION_HASH`. Production always stamps it, so the helper + * defaults to a placeholder when unset; a test exercising re-verify passes a + * real computed hash here instead. + */ + approvedWireHash?: string; + /** + * Build a frame with NO `approvedWireHash` to exercise the deploy path's + * fail-loud guard (the sidecar refuses to recompute a hub-authority hash). + * Overrides the helper's default placeholder. + */ + omitApprovedWireHash?: boolean; +}; + +function makeInferenceSource(id: string): InferenceSourceFixture { + return { + id, + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: `sk-${id}`, + model: "claude-3-5", + }; +} + +/** + * Live definitions the source-ref deploy/restore path reconstructs, keyed by + * derived deployment id. `makeMultistepFrame` registers one per frame it builds; + * `buildMultistepFixture`'s default closure stub returns the matching entry so a + * deploy or restore evaluates the exact topology the test described. Module-level + * so a restore fixture built over the same on-disk data dir (a simulated + * restart) reads the same entry the deploy fixture registered. + */ +const deployDefinitionRegistry = new Map(); + +/** + * Upgrade a test's inert-ish definition to a valid LIVE definition: every step + * gets a real agent so the definition survives `projectLiveToInert` (a bare + * `{ kind: "step" }` with no agent throws). `stepOrder` is preserved verbatim, + * so a definition whose `stepOrder` names a step absent from `steps` still + * projects to the same "no such entry" throw the deploy path rejects on. + */ +function toLiveClosureDefinition( + definition: MultistepDeployArgs["definition"], +): WorkflowDefinition { + const steps: Record = {}; + for (const stepId of Object.keys(definition.steps)) { + steps[stepId] = { + kind: "step", + id: stepId, + agent: { + id: `agent-${stepId}`, + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a hand-built live definition cannot satisfy the full `WorkflowDefinition` nominal type; it stands in for a real closure evaluation, exactly as the restore test's `closureDefinition` does + return { + id: definition.id, + triggers: definition.triggers, + stepOrder: definition.stepOrder, + steps, + } as unknown as WorkflowDefinition; +} + +/** + * Build a source-ref deploy frame. The deploy lineage is source-ref only, so the + * frame carries NO inline `definition`: it pins each step's inference sources, + * the hub-approved wire hash, and a placeholder source-ref pin. The runnable + * definition is decoupled from the frame exactly as production decouples it -- + * the sidecar re-materializes it through the injected `applyFrozenWorkflowClosure`. + * The helper registers the intended live definition (the caller's `definition` + * arg, upgraded so each step carries a valid agent) under the frame's derived + * deployment id; `buildMultistepFixture`'s default closure stub looks it up by + * that id, so a deploy or restore reconstructs the exact topology the caller + * described without threading it through the frame. + */ +function makeMultistepFrame(args: MultistepDeployArgs): AgentDeployFrame { + const agentAddress = args.agentAddress ?? "multi@example.com"; + deployDefinitionRegistry.set( + deriveDeploymentId(agentAddress), + toLiveClosureDefinition(args.definition), + ); + return { + type: "agent.deploy", + agentAddress, + agentId: "multi-agent", + hubPublicKey: "hub-pk", + // The wire-side HarnessConfig has many required fields. On the + // workflow deploy path the router reads only `config.sessionId` + // and `config.grants`, both of which tolerate the empty + // placeholder (they resolve to `undefined`), so an opaque `{}` + // satisfies the surface contract for these tests. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path reads only config.sessionId/config.grants, which tolerate undefined + config: {} as AgentDeployFrame["config"], + workflow: { + sources: args.sources, + // Placeholder source-ref pin. The deploy/restore path re-materializes the + // definition through the injected closure stub, which keys off the + // deployment id -- not this pin's contents -- so the pin only has to be a + // well-formed `SourceRefPin`. + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + // Production always stamps the hub-approved hash; default a placeholder + // so a deploy test need not compute one, and only omit it when a test + // explicitly exercises the fail-loud guard. + ...(args.omitApprovedWireHash + ? {} + : { approvedWireHash: args.approvedWireHash ?? "a".repeat(64) }), + }, + }; +} + +function defaultMultistepSources(): WorkflowProjection["sources"] { + return { + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], + }; +} + +describe("validateWorkflowProjection", () => { + test("rejects an empty stepOrder", () => { + expect(() => + validateWorkflowProjection({ + definition: { id: "w-1", stepOrder: [], steps: {} }, + sources: {}, + }), + ).toThrow(/stepOrder must be a non-empty array/); + }); + + test("rejects a stepId that violates STEP_ID_PATTERN", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["bad.step"], + steps: { "bad.step": {} }, + }, + sources: { "bad.step": {} }, + }), + ).toThrow(/must match \^/); + }); + + test("rejects a missing sources entry for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: {}, + }), + ).toThrow(/sources is missing entry/); + }); + + test("rejects an empty sources chain for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": [] }, + }), + ).toThrow(/must be a non-empty array/); + }); + + test("rejects a non-array sources entry for a stepOrder id", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1"], + steps: { "step-1": {} }, + }, + sources: { "step-1": {} }, + }), + ).toThrow(/must be a non-empty array/); + }); + + test("accepts a well-formed projection", () => { + expect(() => + validateWorkflowProjection({ + definition: { + id: "w-1", + stepOrder: ["step-1", "step-2"], + steps: { "step-1": {}, "step-2": {} }, + }, + sources: { "step-1": [{}], "step-2": [{}] }, + }), + ).not.toThrow(); + }); +}); + +describe("createSidecarDeployRouter multi-step branch", () => { + async function buildMultistepFixture(opts: { + spawner: SubprocessSpawner; + publishWorkflowInferenceEvent?: ( + address: string, + event: EventPayload, + sessionId: string | undefined, + ) => void; + multistepBinaryPath?: string; + multistepSubstrateEnv?: Record; + multistepMailRouter?: MultistepMailRouter; + multistepGrantsRouter?: MultistepGrantsRouter; + /** + * Injectable sender-key cache. The co-delivery tests pass a spy (or a + * throwing stub) to observe the grants handler's cache write; omitted, a + * no-op cache satisfies the dependency without persisting anything. + */ + senderKeyCache?: Parameters[0]["senderKeyCache"]; + multistepSourcesRouter?: MultistepSourcesRouter; + registerDeployment?: (args: { runId: string; agentAddress: string }) => void; + /** + * Injectable deployment-address unregister hook. Defaults to a no-op. + * The self-termination retention test wires this to a real + * `deploymentAddressRegistry` so it can assert the reclaim does NOT + * remove the mapping (the buggy reclaim called this hook, which would + * strand the supervisor's terminal `RunFailed` commit). + */ + unregisterDeployment?: (args: { runId: string; agentAddress: string }) => void; + assertSourceBuildable?: Parameters< + typeof createSidecarDeployRouter + >[0]["assertSourceBuildable"]; + /** + * Reuse an existing transport instead of a fresh one. The restore + * tests deploy through one fixture, then build a SECOND fixture over + * the same on-disk data dir with a FRESH transport to model a sidecar + * process restart (the in-memory transport is process-local, so a + * restart starts with an empty registration table). + */ + transport?: ReturnType; + /** + * Spawn ready-handshake timeout (ms) threaded to every supervisor the + * router constructs. The ready-timeout test uses a small value with a + * spawner that never drives `ready`, asserting the deploy rejects with + * the threaded value echoed in the message. + */ + readyTimeoutMs?: number; + /** + * Fixed keypair the keyStore's `loadOrGenerateKey` returns for the head. + * The B-key test pins the single-step deploy ack to this agent key; when + * omitted a fresh keypair is minted per call as before. + */ + headKeyPair?: Awaited>; + /** + * Injectable deployment-record writer. The rotation-interleave tests + * pass a blockable/failing stub so a recycle can be driven into the + * rotation's persist window; omitted, the router uses the real writer. + */ + writeWorkflowRunRecord?: typeof writeWorkflowRunRecord; + /** + * Injectable closure materializer. A source-ref deploy/restore test passes + * a stub so the path runs without a live registry; omitted, the router + * uses the real `applyFrozenWorkflowClosure`. + */ + applyFrozenWorkflowClosure?: Parameters< + typeof createSidecarDeployRouter + >[0]["applyFrozenWorkflowClosure"]; + }) { + const transport = opts.transport ?? createInMemoryTransport(); + const keyPair = await generateKeyPair(); + const tempBase = await createTempBaseDir("sidecar-multistep-"); + const repoStore = createSpawnTestRepoStore(tempBase); + // The deploy router's source-ref branch materializes the pinned closure + // under `${SIDECAR_DATA_DIR}/workflow-definition-closures//` before + // invoking the spawner. The test fixture defaults the data dir to a + // per-test mkdtemp so the wiring tests do not have to touch a real /tmp + // path; callers can override `SIDECAR_DATA_DIR` (and any other key) by + // passing `multistepSubstrateEnv`. + const defaultSubstrateEnv: Record = { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-multistep-data-"), + // Source-ref is the only deploy lineage: every deploy materializes the + // pin's frozen closure, and the materializer requires both substrate byte + // caps in the env. Default them so a deploy test need not thread them; a + // test that overrides `multistepSubstrateEnv` keeps these unless it sets + // its own. + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }; + const mergedSubstrateEnv: Record = { + ...defaultSubstrateEnv, + ...(opts.multistepSubstrateEnv ?? {}), + }; + // The source-ref deploy/restore path derives the runnable definition by + // materializing the pin's closure through this injected dependency. The + // default stub returns the live definition `makeMultistepFrame` registered + // under the deployment id (the last segment of the per-deployment + // instance dir), so the deploy/restore evaluates the exact topology the + // frame described. A test that hand-writes a record (no frame) or wants a + // bespoke closure result passes its own `applyFrozenWorkflowClosure`. + const defaultApplyFrozenWorkflowClosure: NonNullable< + Parameters[0]["applyFrozenWorkflowClosure"] + > = (applyArgs) => { + const deploymentId = path.basename(applyArgs.instanceDir); + const definition = deployDefinitionRegistry.get(deploymentId); + if (definition === undefined) { + throw new Error( + `test default applyFrozenWorkflowClosure: no registered definition for deploymentId ${deploymentId} (instanceDir ${applyArgs.instanceDir}). Build the deploy frame via makeMultistepFrame/singleStepFrame, or pass an explicit applyFrozenWorkflowClosure when hand-writing a record.`, + ); + } + return Promise.resolve({ + definition, + packageDir: path.join(tempBase, "closure-package", deploymentId), + deployDir: path.join(tempBase, "closure-deploy", deploymentId), + }); + }; + const router = createSidecarDeployRouter({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the workflow path never invokes provisionAgent/persistHubPublicKey (single-step uses the narrow initRepo; the child mints its own key); the stubs throw if it does. initRepo is a no-op for the single-step head repo. + sessions: { + provisionAgent: async () => { + throw new Error("workflow branch must not invoke provisionAgent"); + }, + persistHubPublicKey: async () => { + throw new Error("workflow branch must not invoke persistHubPublicKey"); + }, + initRepo: async () => undefined, + } as unknown as Parameters[0]["sessions"], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; the single-step head deploy records the hub key for pack verification + keyStore: { + recordHubKey: () => undefined, + loadOrGenerateKey: async () => ({ + keyPair: opts.headKeyPair ?? (await generateKeyPair()), + isNew: false, + }), + // A single-step spawn failure unwinds the recorded hub key via + // forgetAgent; the fixture exercises that unwind, so the stub must + // honor the call. + forgetAgent: () => undefined, + } as unknown as Parameters[0]["keyStore"], + senderKeyCache: opts.senderKeyCache ?? { + get: () => undefined, + put: async () => undefined, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }, + transport, + repoStore, + signingKeySeed: keyPair.privateKey, + credentialCipher: createNoopCredentialCipher(), + createAgentCrypto: createEd25519Crypto, + assertSourceBuildable: opts.assertSourceBuildable ?? (() => undefined), + registerDeployment: opts.registerDeployment ?? (() => undefined), + unregisterDeployment: + opts.unregisterDeployment ?? + (() => { + /* no-op */ + }), + multistepSubprocessSpawner: opts.spawner, + ...(opts.multistepBinaryPath !== undefined + ? { multistepBinaryPath: opts.multistepBinaryPath } + : {}), + multistepSubstrateEnv: mergedSubstrateEnv, + ...(opts.publishWorkflowInferenceEvent !== undefined + ? { + publishWorkflowInferenceEvent: opts.publishWorkflowInferenceEvent, + } + : {}), + ...(opts.multistepMailRouter !== undefined + ? { multistepMailRouter: opts.multistepMailRouter } + : {}), + ...(opts.multistepGrantsRouter !== undefined + ? { multistepGrantsRouter: opts.multistepGrantsRouter } + : {}), + ...(opts.multistepSourcesRouter !== undefined + ? { multistepSourcesRouter: opts.multistepSourcesRouter } + : {}), + ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), + ...(opts.writeWorkflowRunRecord !== undefined + ? { + writeWorkflowRunRecord: opts.writeWorkflowRunRecord, + } + : {}), + applyFrozenWorkflowClosure: + opts.applyFrozenWorkflowClosure ?? defaultApplyFrozenWorkflowClosure, + }); + return { + router, + tempBase, + keyPair, + substrateEnv: mergedSubstrateEnv, + transport, + }; + } + + test("validates the projection, constructs SpawnOpts from the frame, drives spawn, and acks the deployment address's public key", async () => { + const supervisorIpcKeyPair = await generateKeyPair(); + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedBinary: string | undefined; + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ binaryPath, env }) => { + observedBinary = binaryPath; + observedEnv = env; + const handle: SubprocessHandle = { + pid: 7321, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const multiDataDir = await createTempBaseDir("sidecar-multi-data-"); + // The deployment address's own key -- what `loadOrGenerateKey` mints. + // Pin it so the ack assertion below is deterministic. + const deploymentKeyPair = await generateKeyPair(); + const { router } = await buildMultistepFixture({ + spawner, + headKeyPair: deploymentKeyPair, + multistepBinaryPath: "/fake/bin/multistep-workflow-child", + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: multiDataDir, + }, + }); + + // Hijack the supervisor's ipc keypair factory by routing through + // the test-construction surface: the router constructs the + // supervisor via createSidecarWorkflowSupervisor which does not + // expose ipcKeyPairFactory. The supervisor's default keypair is + // generated with generateKeyPair; the test signs the `ready` frame + // with whatever channelId the spawn-time env carries plus the + // child's keypair, and the supervisor accepts a bootstrap + // signature from any childPublicKey carried in the `ready` payload. + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-router-test", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + // The sidecar sources DEFINITION_HASH from the frame's hub-approved hash + // verbatim (never a recompute), so stamp the real wire hash and assert the + // child receives it. + const approvedWireHash = await computeWireDefinitionHash(definition); + const frame = makeMultistepFrame({ definition, sources, approvedWireHash }); + + const deployPromise = router.deploy(frame); + + // Wait until the spawner has been invoked. + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + + const env = observedEnv; + expect(observedBinary).toBe("/fake/bin/multistep-workflow-child"); + expect(env).toMatchObject({ + SIDECAR_DATA_DIR: multiDataDir, + DEPLOYMENT_ID: "multi-example-com", + MAILBOX_ADDRESS: "multi@example.com", + }); + expect(env.DEFINITION_HASH).toBe(approvedWireHash); + expect(env[STEP_INFERENCE_SOURCES_ENV_KEY]).toBe(JSON.stringify(sources)); + expect(env.IPC_CHANNEL_ID).toMatch(/^[0-9a-f]{32}$/); + + // Drive the `ready` handshake. + const channelId = env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 7321, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + + const result = await deployPromise; + // Every deployment -- single- or multi-step -- acks the deployment + // address's own public key, so the hub can verify ownership on + // reconnect. A multi-step deployment previously acked the supervisor + // principal key, which the hub discarded. The hex is a 64-character + // lowercase string. + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + expect(result.publicKey).toBe(hexEncode(deploymentKeyPair.publicKey)); + + // Teardown: kill the child so the spawn-time pumps unwind. + // Use unused supervisorToChild to silence the linter. + void supervisorToChild; + void supervisorIpcKeyPair; + }); + + test("sources the child's DEFINITION_HASH from the frame's hub-approved wire hash, not a sidecar recompute", async () => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + return { + pid: 9001, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + }; + + const multiDataDir = await createTempBaseDir("sidecar-approved-hash-"); + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-approved-hash", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + // A sentinel that is deliberately NOT the wire hash of `definition`, so an + // assertion that the child received it proves the value came from the + // frame (the hub authority) and was not recomputed at the sidecar. + const HUB_APPROVED_HASH = "hub-approved-sentinel-hash"; + const recomputed = await computeWireDefinitionHash(definition); + expect(HUB_APPROVED_HASH).not.toBe(recomputed); + + const frame = makeMultistepFrame({ + definition, + sources, + approvedWireHash: HUB_APPROVED_HASH, + }); + const deployPromise = router.deploy(frame); + + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const env = observedEnv; + // The child's DEFINITION_HASH is the frame's hub-approved hash, verbatim. + expect(env.DEFINITION_HASH).toBe(HUB_APPROVED_HASH); + + // Drive the child to exit so the deploy's spawn pumps unwind, then let the + // deploy settle (it rejects once the child dies mid-handshake, which is + // fine -- the env assertion above is the subject under test). + const channelId = env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + await deployPromise.catch(() => undefined); + void supervisorToChild; + }); + + test("a second same-address deploy is rejected mid-spawn and never deletes the live run record", async () => { + // Pins the synchronous single-flight reservation guard. The first + // deploy runs its durable writes and then suspends inside + // supervisor.spawn awaiting the child's `ready` handshake -- the window + // in which its reservation is held but `activeSupervisors` is not yet + // populated. A second same-address frame arriving in that window must be + // rejected at the reservation guard (its own message, distinct from the + // spawn-core backstop) before it touches durable state, so it cannot + // delete the first deploy's live record via the soft-fail catch. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let spawnCount = 0; + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + spawnCount += 1; + observedEnv = env; + const handle: SubprocessHandle = { + pid: 4242, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const multiDataDir = await createTempBaseDir("sidecar-concurrent-deploy-"); + const registered: string[] = []; + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir }, + registerDeployment: ({ agentAddress }) => { + registered.push(agentAddress); + }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-concurrent", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + const anchorRunId = deriveDeploymentId(frame.agentAddress); + const recordFile = path.join(multiDataDir, "workflow-runs", anchorRunId, "deployment.json"); + + const firstDeploy = router.deploy(frame); + // Wait until the first deploy has spawned: its record is on disk and it + // is now suspended in the ready handshake with the reservation held. + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const recordBefore = await fs.readFile(recordFile, "utf8"); + expect(recordBefore.length).toBeGreaterThan(0); + + // The loser is rejected at the reservation guard, not the spawn-core + // backstop -- the guard's message is the one asserted here. + await expect(router.deploy(frame)).rejects.toThrow( + /is already deployed; undeploy it before redeploying/, + ); + // It never reached the spawner and never deleted the live record. + expect(spawnCount).toBe(1); + const recordAfter = await fs.readFile(recordFile, "utf8"); + expect(recordAfter).toBe(recordBefore); + + // Drive the first deploy's ready handshake so it completes, then confirm + // the winner is the live, registered deployment. + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4242, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + + const result = await firstDeploy; + expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); + expect(registered).toEqual([frame.agentAddress]); + expect(router.activeAddresses()).toEqual([frame.agentAddress]); + + void supervisorToChild; + }); + + test("registers a multistepMailRouter handler against the deployment address once spawn succeeds", async () => { + // Drives the spawn handshake the same way the first multi-step + // test does, but injects a `multistepMailRouter` and asserts the + // deploy router registered a handler against the deployment's + // mail address by the time `deploy(frame)` resolves. The handler + // is what the sidecar hub-link's `mail.inbound` path dispatches + // through; without this registration, an inbound mail aimed at + // the deployment address falls into the legacy session path, + // which has no transport entry and no `sessions` row for the + // deployment address. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 9123, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const mailRouter = createMultistepMailRouter(); + const { router } = await buildMultistepFixture({ + spawner, + multistepMailRouter: mailRouter, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-mail-router-test", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + const deployPromise = router.deploy(frame); + + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 9123, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + + await deployPromise; + + // The handler must be installed against the deployment's mail + // address (`frame.agentAddress`), and tryRoute must claim it. + const claimed = mailRouter.tryRoute(frame.agentAddress, new Uint8Array([1, 2, 3])); + expect(claimed).not.toBeNull(); + // Settle the durable promise so its resolution/rejection is not left as an + // unhandled rejection; this assertion only checks the address is claimed, + // not the enqueue outcome for this synthetic payload. + await claimed?.catch(() => undefined); + + // Teardown. + void supervisorToChild; + }); + + test("a run.grants frame writes the run's grants to runs//grants.json in the workflow-run repo", async () => { + // Drives the same spawn handshake as the mail-router test, then routes + // a `run.grants` frame through the injected `multistepGrantsRouter` and + // asserts the handler the deploy router installed wrote the run's + // grants to `runs//grants.json` inside the deployment's + // `workflow-run` repo -- sibling to the run's `runs//events/` + // subtree. Nothing reads the grants back yet; the assertion is on the + // on-disk write (right repo, right path, right content). + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 9124, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const grantsRouter = createMultistepGrantsRouter(); + const { router, tempBase } = await buildMultistepFixture({ + spawner, + multistepGrantsRouter: grantsRouter, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-grants-router-test", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + const deployPromise = router.deploy(frame); + + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 9124, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + + await deployPromise; + + const runId = "run-abc"; + const stepGrants = [ + { + id: "grant-1", + resource: "tool:send-mail", + action: "invoke", + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: "prn_deployment", + }, + ]; + const routed = await grantsRouter.tryRoute({ + type: "run.grants", + agentAddress: frame.agentAddress, + runId, + stepGrants, + }); + expect(routed).toBe(true); + + // The write lands in the deployment's workflow-run repo at + // `runs//grants.json`, the run-owned sibling of `events/`. + const anchorRunId = deriveDeploymentId(frame.agentAddress); + const grantsFile = path.join( + tempBase, + "workflow-run", + anchorRunId, + "runs", + runId, + "grants.json", + ); + const onDisk: unknown = JSON.parse(await fs.readFile(grantsFile, "utf8")); + expect(onDisk).toEqual({ grants: stepGrants }); + + // Teardown. + void supervisorToChild; + }); + + // Deploy a multi-step deployment through the full spawn/ready handshake so + // the deploy router installs its grants handler, then hand back the pieces a + // co-delivery test needs to route a `run.grants` frame and inspect the write. + async function deployMultistepForGrants( + definitionId: string, + senderKeyCache: Parameters[0]["senderKeyCache"], + ): Promise<{ + grantsRouter: MultistepGrantsRouter; + agentAddress: string; + anchorRunId: string; + tempBase: string; + }> { + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + return { + pid: 9124, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + }; + + const grantsRouter = createMultistepGrantsRouter(); + const { router, tempBase } = await buildMultistepFixture({ + spawner, + multistepGrantsRouter: grantsRouter, + senderKeyCache, + }); + + const definition = { + id: definitionId, + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ + definition, + sources: defaultMultistepSources(), + }); + const deployPromise = router.deploy(frame); + + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 9124, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + await deployPromise; + void supervisorToChild; + + return { + grantsRouter, + agentAddress: frame.agentAddress, + anchorRunId: deriveDeploymentId(frame.agentAddress), + tempBase, + }; + } + + test("caches each co-delivered sender key before the run's grants land", async () => { + // The grants handler must cache the sender key BEFORE writing grants.json, + // so a durable grant is never missing the key its recipient verifies + // against. The spy records whether grants.json already exists when its + // `put` runs; it must not. + // A holder the spy reads at `put` time; its path is filled in only once the + // deploy resolves the run's on-disk location, so it starts empty. + const grantsFileRef: { path: string | undefined } = { path: undefined }; + const puts: { address: string; grantsExisted: boolean }[] = []; + const senderKeyCache = { + get: () => undefined, + put: async (address: string) => { + const grantsExisted = + grantsFileRef.path !== undefined && + (await fs + .stat(grantsFileRef.path) + .then(() => true) + .catch(() => false)); + puts.push({ address, grantsExisted }); + }, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }; + + const { grantsRouter, agentAddress, anchorRunId, tempBase } = await deployMultistepForGrants( + "wf-sender-key-order", + senderKeyCache, + ); + + const runId = "run-codeliver"; + const grantsFilePath = path.join( + tempBase, + "workflow-run", + anchorRunId, + "runs", + runId, + "grants.json", + ); + grantsFileRef.path = grantsFilePath; + const routed = await grantsRouter.tryRoute({ + type: "run.grants", + agentAddress, + runId, + stepGrants: [], + senderIdentities: [ + { + address: "sender@peer.example", + publicKey: hexEncode(new Uint8Array(32)), + }, + ], + }); + + expect(routed).toBe(true); + expect(puts).toEqual([{ address: "sender@peer.example", grantsExisted: false }]); + const grantsLanded = await fs + .stat(grantsFilePath) + .then(() => true) + .catch(() => false); + expect(grantsLanded).toBe(true); + }); + + test("a sender-key cache-write failure fails the run's grants", async () => { + // The cache write gates the grants write: if the key cannot be cached, the + // run must not start under a grant whose sender the recipient cannot + // verify. The handler's throw propagates and grants.json never lands. + const senderKeyCache = { + get: () => undefined, + put: async () => { + throw new Error("sender-key disk full"); + }, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }; + + const { grantsRouter, agentAddress, anchorRunId, tempBase } = await deployMultistepForGrants( + "wf-sender-key-fault", + senderKeyCache, + ); + + const runId = "run-cache-fault"; + await expect( + grantsRouter.tryRoute({ + type: "run.grants", + agentAddress, + runId, + stepGrants: [], + senderIdentities: [ + { + address: "sender@peer.example", + publicKey: hexEncode(new Uint8Array(32)), + }, + ], + }), + ).rejects.toThrow("sender-key disk full"); + + const grantsFilePath = path.join( + tempBase, + "workflow-run", + anchorRunId, + "runs", + runId, + "grants.json", + ); + const grantsLanded = await fs + .stat(grantsFilePath) + .then(() => true) + .catch(() => false); + expect(grantsLanded).toBe(false); + }); + + test("skips a malformed co-delivered key without failing the run's grants", async () => { + // A malformed key is a hub-side defect, keyless from the sidecar's view. + // Unlike a disk fault it must NOT poison the run -- otherwise a persistently + // bad key would wedge the run on every replay. The valid siblings still + // cache and the grants still land. + const puts: string[] = []; + const senderKeyCache = { + get: () => undefined, + put: async (address: string) => { + puts.push(address); + }, + evict: async () => undefined, + addresses: () => [], + rotatableAddresses: () => [], + }; + + const { grantsRouter, agentAddress, anchorRunId, tempBase } = await deployMultistepForGrants( + "wf-sender-key-malformed", + senderKeyCache, + ); + + const runId = "run-malformed"; + const routed = await grantsRouter.tryRoute({ + type: "run.grants", + agentAddress, + runId, + stepGrants: [], + senderIdentities: [ + // Wrong length: valid hex, but 16 bytes rather than 32. + { + address: "bad@peer.example", + publicKey: hexEncode(new Uint8Array(16)), + }, + { + address: "good@peer.example", + publicKey: hexEncode(new Uint8Array(32)), + }, + ], + }); + + expect(routed).toBe(true); + expect(puts).toEqual(["good@peer.example"]); + const grantsFilePath = path.join( + tempBase, + "workflow-run", + anchorRunId, + "runs", + runId, + "grants.json", + ); + const grantsLanded = await fs + .stat(grantsFilePath) + .then(() => true) + .catch(() => false); + expect(grantsLanded).toBe(true); + }); + + test("does not register a multistepMailRouter handler if spawn rejects", async () => { + const mailRouter = createMultistepMailRouter(); + const crashSpawner: SubprocessSpawner = () => { + throw new Error("ENOENT: binary missing"); + }; + const { router } = await buildMultistepFixture({ + spawner: crashSpawner, + multistepMailRouter: mailRouter, + }); + + const frame = makeMultistepFrame({ + agentAddress: "run_crash-noreg@example.com", + definition: { + id: "wf-crash-noreg", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ENOENT: binary missing/); + + expect(mailRouter.tryRoute(frame.agentAddress, new Uint8Array([1]))).toBeNull(); + }); + + test("a soft-failed deploy (spawn rejects) leaves no restore record", async () => { + const crashSpawner: SubprocessSpawner = () => { + throw new Error("ENOENT: binary missing"); + }; + const { router, substrateEnv } = await buildMultistepFixture({ + spawner: crashSpawner, + }); + const agentAddress = "run_softfail@example.com"; + const frame = makeMultistepFrame({ + agentAddress, + definition: { + id: "wf-softfail", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ENOENT/); + + // The record is written before the spawn, so the soft-failure catch must + // delete it -- a boot-time restore must not re-spawn a deploy that never + // completed. (A hard crash mid-spawn, by contrast, deliberately leaves + // the record for the restore to re-drive.) + const dataDir = substrateEnv.SIDECAR_DATA_DIR; + if (dataDir === undefined) throw new Error("fixture SIDECAR_DATA_DIR unset"); + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveDeploymentId(agentAddress), + "deployment.json", + ); + expect( + await fs.access(recordFile).then( + () => true, + () => false, + ), + ).toBe(false); + }); + + test("rejects a deploy whose step pins an unbuildable provider before spawning", async () => { + // The source-admission gate runs before any state is claimed or the + // child is spawned. A step whose pinned source names a provider the + // sidecar cannot build must reject the whole deploy synchronously -- + // the admission control property -- rather than spawning a child that + // fails when the step's inference first resolves. + let spawnCount = 0; + const trackingSpawner: SubprocessSpawner = () => { + spawnCount++; + throw new Error("spawn must not be reached for an inadmissible source"); + }; + const { router } = await buildMultistepFixture({ + spawner: trackingSpawner, + assertSourceBuildable: (source) => { + if (source.provider === "ghost-provider") { + throw new Error(`Source provider "${source.provider}" is not registered`); + } + }, + }); + + const frame = makeMultistepFrame({ + agentAddress: "run_unbuildable@example.com", + definition: { + id: "wf-unbuildable", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": [ + { + ...makeInferenceSource("step-1"), + provider: "ghost-provider", + }, + ], + }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ghost-provider.*not registered/); + expect(spawnCount).toBe(0); + }); + + test("a spawner that throws synchronously surfaces a structured rejection rather than hanging in starting", async () => { + // Simulates `Bun.spawn` failing to launch (binary missing, + // permissions error). The router must surface the rejection + // through `deploy(frame)` without leaving the supervisor wedged. + const crashSpawner: SubprocessSpawner = () => { + throw new Error("ENOENT: binary missing"); + }; + + const { router } = await buildMultistepFixture({ spawner: crashSpawner }); + + const frame = makeMultistepFrame({ + agentAddress: "run_crash@example.com", + definition: { + id: "wf-crash", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": [makeInferenceSource("step-1")], + }, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/ENOENT: binary missing/); + }); + + test("rejects a malformed workflow projection at the router boundary before spawn fires", async () => { + let spawnerInvoked = false; + const spawner: SubprocessSpawner = () => { + spawnerInvoked = true; + throw new Error("spawner must not run for an invalid projection"); + }; + + const { router } = await buildMultistepFixture({ spawner }); + + const frame = makeMultistepFrame({ + definition: { + id: "wf-bad", + triggers: [{ type: "manual" }], + // stepOrder mentions a step that has no steps[] entry + stepOrder: ["step-1", "step-missing"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": [makeInferenceSource("step-1")], + }, + }); + + // The closure-derived definition is structurally invalid: `stepOrder` names + // `step-missing`, which the `steps` record does not define. Source-ref is the + // only deploy lineage, so the definition is projected from the materialized + // closure BEFORE the projection guard; the live->inert projector rejects the + // dangling stepOrder entry at the router boundary, before any spawn fires. + await expect(router.deploy(frame)).rejects.toThrow( + /stepOrder names "step-missing" .* the steps record has no such entry/, + ); + expect(spawnerInvoked).toBe(false); + }); + + test("refuses to deploy a workflow frame carrying no approvedWireHash rather than recomputing it", async () => { + // The child re-verifies its own recompute against the HUB-approved wire + // hash. A frame that carries none is a wiring bug, not a legacy case: the + // sidecar must fail loud rather than substitute its own recompute, which + // would collapse the re-verify to a self-check. Production always stamps + // it, so only a malformed frame reaches this guard. Source-ref is the only + // deploy lineage, so the durable run record the deploy path builds before + // the spawn is the first gate to reject the missing hash. + let spawnerInvoked = false; + const spawner: SubprocessSpawner = () => { + spawnerInvoked = true; + throw new Error("spawner must not run for a hash-less frame"); + }; + + const { router } = await buildMultistepFixture({ spawner }); + + const frame = makeMultistepFrame({ + // A single-step deploy parses the frame address as a run address, so use + // the canonical `run_@` shape to reach the approved-wire-hash + // guard rather than tripping address parsing first. + agentAddress: "run_nohash@example.com", + definition: { + id: "wf-no-hash", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + omitApprovedWireHash: true, + }); + + await expect(router.deploy(frame)).rejects.toThrow(/must carry approvedWireHash/); + expect(spawnerInvoked).toBe(false); + }); + + test("does not drop the first upstream control frame the child sends after ready", async () => { + // The supervisor's `pumpUpstreamControl` consumes the same + // control-receive iterator `waitForReady` initialised. A buggy + // `waitForReady` that finalised the iterator on `ready` would + // silently drop the next upstream frame; a correct handoff + // surfaces a `recycle.request` as a real supervisor.recycle() + // call, which the supervisor implements by spawning a new child + // via the injected subprocessSpawner. Counting spawner + // invocations is the cleanest observable: 1 means the upstream + // frame was dropped; >=2 means the pump consumed it. + // + // The mock spawner serves a fresh control/event pair per call so + // the recycle path's own ready handshake completes; the test's + // child sender signs `ready` once per spawn. + type SpawnFixture = { + supervisorToChild: ReturnType; + childToSupervisor: ReturnType; + eventChildToSupervisor: ReturnType; + env: Record; + childIpcKeyPair: { privateKey: Uint8Array; publicKey: Uint8Array }; + }; + const spawns: SpawnFixture[] = []; + let resolveSpawnAdded: (() => void) | null = null; + const spawnAdded = (): Promise => + new Promise((resolve) => { + resolveSpawnAdded = resolve; + }); + const spawner: SubprocessSpawner = ({ env }) => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const handle: SubprocessHandle = { + pid: 4400 + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + // Capture the per-spawn streams synchronously so the test can + // drive the child side once the supervisor has wired the + // receiver. + const fixture: SpawnFixture = { + supervisorToChild, + childToSupervisor, + eventChildToSupervisor, + env, + // Mint a fresh child keypair per spawn; the supervisor's + // receiveControlChannel opens in bootstrap mode and pins on + // the per-spawn ready frame's `childPublicKey`. + childIpcKeyPair: { + publicKey: new Uint8Array(), + privateKey: new Uint8Array(), + }, + }; + spawns.push(fixture); + const r = resolveSpawnAdded; + resolveSpawnAdded = null; + if (r) r(); + return handle; + }; + + const { router } = await buildMultistepFixture({ spawner }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-handoff", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + // Helper to drive the child side of one spawn fixture's ready + // handshake, optionally chaining an upstream `recycle.request`. + async function driveReady( + fixture: SpawnFixture, + opts: { sendRecycleRequest: boolean }, + ): Promise { + const channelId = fixture.env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childIpcKeyPair = await generateKeyPair(); + fixture.childIpcKeyPair = childIpcKeyPair; + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + fixture.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 4400 + spawns.length, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + if (opts.sendRecycleRequest) { + await childSender.send({ + type: "recycle.request", + data: { reason: "iterator-handoff-test" }, + }); + } + } + + const deployPromise = router.deploy(frame); + + // Wait for the first spawn to land. + while (spawns.length === 0) { + await spawnAdded(); + } + const first = spawns[0]; + if (first === undefined) throw new Error("unreachable"); + // Drive ready + immediate recycle.request on the first spawn. + await driveReady(first, { sendRecycleRequest: true }); + + // The initial deploy's spawn() resolves once `ready` lands. The + // supervisor's pump consumes the recycle.request and kicks off a + // recycle, which calls the spawner a second time. + await deployPromise; + + // Wait for the recycle's respawn. + while (spawns.length < 2) { + await spawnAdded(); + } + const second = spawns[1]; + if (second === undefined) throw new Error("unreachable"); + // Drive ready on the second (recycle's) spawn so the recycle path + // unwinds cleanly. We do not assert on this spawn's effects; the + // assertion below covers the iterator-handoff invariant. + await driveReady(second, { sendRecycleRequest: false }); + // Allow the recycle path to settle its post-ready work. + await new Promise((r) => setTimeout(r, 25)); + + expect(spawns.length).toBeGreaterThanOrEqual(2); + }); + + test("multistepSubstrateEnv carries HUB_WS_URL, SIDECAR_ID, SIDECAR_TOKEN through to the spawn-time env", async () => { + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 7600, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + const bootEdgeDataDir = await createTempBaseDir("sidecar-boot-edge-data-"); + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: bootEdgeDataDir, + HUB_WS_URL: "ws://hub.example/sidecar-boot", + SIDECAR_ID: "sidecar-boot-1", + SIDECAR_TOKEN: "boot-token-abc", + }, + }); + const sources = defaultMultistepSources(); + const definition = { + id: "wf-boot-edge", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + const deployPromise = router.deploy(frame); + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + expect(observedEnv.HUB_WS_URL).toBe("ws://hub.example/sidecar-boot"); + expect(observedEnv.SIDECAR_ID).toBe("sidecar-boot-1"); + expect(observedEnv.SIDECAR_TOKEN).toBe("boot-token-abc"); + expect(observedEnv.SIDECAR_DATA_DIR).toBe(bootEdgeDataDir); + // Round out the spawn so the test exits cleanly. + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing from spawn env"); + } + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 7600, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + await deployPromise; + }); + + test("a registerDeployment failure before spawn unwinds the slug and leaves the address claimable", async () => { + // The multi-step partial-state unwind for the address-registry step. + // `registerDeployment` runs BEFORE `supervisor.spawn` -- the replay the + // spawn kicks off writes through the pack-pushing facade, which must + // resolve the deployment-address mapping, so the mapping has to exist + // before the spawn. A `registerDeployment` failure therefore throws + // before any child is spawned; the unwind must release the slug (and + // reverse nothing else, since nothing after it ran). The observable + // evidence is that (a) NO child was spawned for the failed deploy and + // (b) a subsequent deploy on the SAME address succeeds, which is only + // possible if the slug was released. + const childIpcKeyPair = await generateKeyPair(); + const spawnedHandles: { + pid: number; + killed: boolean; + supervisorToChild: ReturnType; + childToSupervisor: ReturnType; + eventChildToSupervisor: ReturnType; + }[] = []; + const observedEnvs: Record[] = []; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnvs.push(env); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const record = { + pid: 9000 + spawnedHandles.length, + killed: false, + supervisorToChild, + childToSupervisor, + eventChildToSupervisor, + }; + spawnedHandles.push(record); + const handle: SubprocessHandle = { + pid: record.pid, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + record.killed = true; + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + let registerCallCount = 0; + const multiDataDir = await createTempBaseDir("sidecar-multi-unwind-"); + const { router } = await buildMultistepFixture({ + spawner, + multistepBinaryPath: "/fake/bin/multistep-workflow-child", + multistepSubstrateEnv: { SIDECAR_DATA_DIR: multiDataDir }, + registerDeployment: () => { + registerCallCount += 1; + if (registerCallCount === 1) { + throw new Error("registerDeployment failure (synthetic)"); + } + }, + }); + + async function driveReadyFor(handleIndex: number, childPid: number): Promise { + while (spawnedHandles.length <= handleIndex) { + await new Promise((r) => setTimeout(r, 1)); + } + const env = observedEnvs[handleIndex]; + const channelId = env?.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing in observed env"); + } + const record = spawnedHandles[handleIndex]; + if (record === undefined) { + throw new Error(`spawnedHandles[${String(handleIndex)}] missing`); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + record.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + } + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-unwind-test", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + // The first deploy throws at `registerDeployment`, which now runs before + // `supervisor.spawn`, so it rejects WITHOUT spawning a child -- no ready + // handshake to drive. + let firstCaught: unknown; + try { + await router.deploy(frame); + } catch (err) { + firstCaught = err; + } + expect(firstCaught).toBeInstanceOf(Error); + expect(firstCaught instanceof Error && firstCaught.message).toMatch( + /registerDeployment failure \(synthetic\)/, + ); + + // No child was spawned for the failed deploy: the throw preceded spawn. + expect(spawnedHandles).toHaveLength(0); + + // Re-deploy on the SAME address must succeed. If the unwind missed the + // slug release, the second deploy would surface a phantom collision. The + // router's public contract is that a failed deploy leaves the address + // claimable again. This is the first deploy that actually spawns, so its + // ready handshake is at index 0. + const secondDeploy = router.deploy(frame); + await driveReadyFor(0, 9000); + const secondResult = await secondDeploy; + expect(secondResult.publicKey).toMatch(/^[0-9a-f]{64}$/); + expect(registerCallCount).toBe(2); + }); + + // ------------------------------------------------------------------ + // Boot-time restore of persisted workflow deployments + // ------------------------------------------------------------------ + + // A mock spawner that serves a fresh control/event channel per spawn and + // lets the test complete each child's `ready` handshake. Both `deploy` and + // `restoreWorkflowRuns` block on `supervisor.spawn` until `ready` + // lands, so every spawned child needs its handshake driven. + function makeReadyDrivingSpawner(pidBase: number) { + type Spawn = { + env: Record; + childToSupervisor: ReturnType; + eventChildToSupervisor: ReturnType; + childSender?: ReturnType; + }; + const spawns: Spawn[] = []; + // One-shot spawn failure. When armed, the NEXT spawner invocation throws + // instead of returning a handle, then disarms. Used to fail a recycle + // respawn so the supervisor tears down to `stopped` (a self-termination) + // without waiting on a ready-handshake timeout. + let failNext = false; + const spawner: SubprocessSpawner = ({ env }) => { + if (failNext) { + failNext = false; + throw new Error("makeReadyDrivingSpawner: armed spawn failure"); + } + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + spawns.push({ env, childToSupervisor, eventChildToSupervisor }); + const handle: SubprocessHandle = { + pid: pidBase + spawns.length, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + async function driveReadyFor( + index: number, + opts?: { sendRecycleRequest?: boolean }, + ): Promise { + while (spawns.length <= index) { + await new Promise((r) => setTimeout(r, 1)); + } + const spawn = spawns[index]; + if (spawn === undefined) { + throw new Error(`spawn ${String(index)} missing`); + } + const channelId = spawn.env.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID missing in spawn env"); + } + const childIpcKeyPair = await generateKeyPair(); + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + spawn.childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + // Retain the sender so a later recycle.request (recycleRequestFor) + // signs with the SAME keypair the supervisor pinned from this ready. + spawn.childSender = childSender; + await childSender.send({ + type: "ready", + data: { + childPid: pidBase + index, + childPublicKey: Buffer.from(childIpcKeyPair.publicKey).toString("hex"), + }, + }); + if (opts?.sendRecycleRequest === true) { + // Model the child asking to recycle; the supervisor's upstream pump + // consumes it and respawns. + await childSender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + } + } + return { + spawner, + driveReadyFor, + spawnCount: () => spawns.length, + envFor: (index: number): Record | undefined => spawns[index]?.env, + async recycleRequestFor(index: number): Promise { + const sender = spawns[index]?.childSender; + if (sender === undefined) { + throw new Error(`spawn ${String(index)} has no sender; drive its ready first`); + } + await sender.send({ + type: "recycle.request", + data: { reason: "sources-rotation-recycle" }, + }); + }, + // Arm a one-shot spawn failure for the next spawner invocation. + failNextSpawn(): void { + failNext = true; + }, + }; + } + + function isRegistered( + transport: ReturnType, + address: string, + ): boolean { + try { + transport.getTransportFor(address); + return true; + } catch { + return false; + } + } + + function recordExists(dataDir: string, anchorRunId: string): Promise { + return fs.access(path.join(dataDir, "workflow-runs", anchorRunId, "deployment.json")).then( + () => true, + () => false, + ); + } + + function singleStepFrame(agentAddress: string, definitionId: string): AgentDeployFrame { + return makeMultistepFrame({ + agentAddress, + definition: { + id: definitionId, + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { "step-1": [makeInferenceSource("step-1")] }, + }); + } + + test("restore re-spawns a persisted single-step deployment and re-registers its head on a fresh transport", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-restart-data-"); + const head = "run_restart@example.com"; + + // First process: deploy a single-step workflow. The deploy persists a + // restore record under `dataDir` and materializes its `workflow.json`. + const first = makeReadyDrivingSpawner(9100); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(singleStepFrame(head, "wf-restart")); + await first.driveReadyFor(0); + await deployPromise; + + // Second process (simulated restart): a FRESH transport (empty + // registration table) and fresh in-memory router state over the SAME + // on-disk data dir. + const second = makeReadyDrivingSpawner(9200); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // Nothing is registered before restore -- the restart started clean. + expect(isRegistered(freshTransport, head)).toBe(false); + + const restorePromise = routerB.restoreWorkflowRuns(); + await second.driveReadyFor(0); + await restorePromise; + + // The deployment was re-spawned exactly once and its head is live again. + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, head)).toBe(true); + }); + + test("restore soft-fails a record whose closure will not materialize and restores the rest", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-softfail-data-"); + const goodHead = "run_good@example.com"; + const badHead = "run_bad@example.com"; + + const first = makeReadyDrivingSpawner(9300); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployGood = routerA.deploy(singleStepFrame(goodHead, "wf-good")); + await first.driveReadyFor(0); + await deployGood; + const deployBad = routerA.deploy(singleStepFrame(badHead, "wf-bad")); + await first.driveReadyFor(1); + await deployBad; + + // Source-ref is the only deploy lineage: a restore re-materializes each + // record's pinned closure to derive its definition. Make the bad + // deployment's closure materialization fault (the pinned code no longer + // resolves) so its restore soft-fails, while the good one materializes and + // re-spawns. + const badDeploymentId = deriveDeploymentId(badHead); + const second = makeReadyDrivingSpawner(9400); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + applyFrozenWorkflowClosure: (applyArgs) => { + const deploymentId = path.basename(applyArgs.instanceDir); + if (deploymentId === badDeploymentId) { + throw new Error( + `closure materialization faulted for ${deploymentId}: pinned code did not resolve`, + ); + } + const definition = deployDefinitionRegistry.get(deploymentId); + if (definition === undefined) { + throw new Error( + `test applyFrozenWorkflowClosure: no registered definition for ${deploymentId}`, + ); + } + return Promise.resolve({ + definition, + packageDir: path.join(dataDir, "closure-package", deploymentId), + deployDir: path.join(dataDir, "closure-deploy", deploymentId), + }); + }, + }); + + // The good deployment re-spawns (exactly one handshake to drive); + // scan order is filesystem-dependent, but only the good record spawns. + const restorePromise = routerB.restoreWorkflowRuns(); + await second.driveReadyFor(0); + await restorePromise; + + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, goodHead)).toBe(true); + expect(isRegistered(freshTransport, badHead)).toBe(false); + // The failed record is KEPT on disk -- never deleted, unlike a + // soft-failed deploy -- so a later boot can retry it. + expect(await recordExists(dataDir, deriveDeploymentId(badHead))).toBe(true); + }); + + test("restore rejects a closure-derived definition whose stepOrder names a step with no matching entry", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-validator-data-"); + const head = "run_validator@example.com"; + const anchorRunId = deriveDeploymentId(head); + + // Write a well-formed source-ref record: it clears the record schema and the + // scan boundary, so restore reaches the projection gate. The closure it + // re-materializes, though, evaluates to a structurally invalid definition -- + // its `stepOrder` names a step the `steps` record does not define. The + // source-ref restore arm projects the closure-derived definition + // (`projectLiveToInert`) before spawning, so it rejects the dangling + // stepOrder entry and never spawns a child for a broken definition. + const record: WorkflowRunRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-missing-step", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + approvedWireHash: "a".repeat(64), + lineage: "source-ref", + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + }; + await writeWorkflowRunRecord(dataDir, anchorRunId, record, createNoopCredentialCipher()); + + const spawner = makeReadyDrivingSpawner(9500); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + applyFrozenWorkflowClosure: (applyArgs) => + Promise.resolve({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a hand-built live definition cannot satisfy the full WorkflowDefinition nominal type; this stands in for a closure that evaluates to a structurally invalid definition + definition: { + id: "wf-missing-step", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-missing"], + steps: { + "step-1": { + kind: "step", + id: "step-1", + agent: { + id: "agent-step-1", + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }, + }, + } as unknown as WorkflowDefinition, + packageDir: path.join(applyArgs.instanceDir, "package"), + deployDir: path.join(applyArgs.instanceDir, "deploy"), + }), + }); + + await router.restoreWorkflowRuns(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + }); + + test("restore soft-skips a malformed source-ref record (missing sourceRef) at the scan boundary", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-srcref-bad-"); + const head = "run_srcref@example.com"; + const deploymentId = deriveDeploymentId(head); + + // A source-ref record MUST carry a sourceRef pin + approvedWireHash -- the + // record schema's discriminated union on `lineage` requires them. Write a + // raw malformed one (lineage source-ref, none of the required fields) + // straight to the record path, bypassing the typed writer. + // `scanWorkflowRunRecords` validates against the schema and + // soft-skips it as corruption, so restore never reaches a spawn -- there is + // no bespoke source-ref guard in the restore loop to lean on. + const recordPath = path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); + await fs.mkdir(path.dirname(recordPath), { recursive: true }); + await fs.writeFile( + recordPath, + JSON.stringify({ + version: 1, + agentAddress: head, + definitionId: "wf-srcref", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + lineage: "source-ref", + }), + "utf8", + ); + + const spawner = makeReadyDrivingSpawner(9550); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowRuns(); + + // Rejected at the scan boundary: no child spawned, nothing registered. + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + }); + + test("restore re-materializes a source-ref deployment's closure and re-spawns it as source-ref", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-srcref-ok-"); + const head = "run_srcref_ok@example.com"; + const deploymentId = deriveDeploymentId(head); + + // A well-formed source-ref record: lineage source-ref with a sourceRef pin + // (source + closure) + approvedWireHash, exactly what the deploy path + // persists. + const record: WorkflowRunRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-srcref", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + approvedWireHash: "a".repeat(64), + lineage: "source-ref", + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + }; + await writeWorkflowRunRecord(dataDir, deploymentId, record, createNoopCredentialCipher()); + + // The source-ref restore arm reconstructs the definition from the + // re-materialized closure, NOT from the on-disk inert workflow.json. Write a + // DELIBERATELY-CORRUPT workflow.json to prove the arm never reads it: were + // the restore path to parse this file it would throw and soft-skip the + // record, so a successful restore below is proof the closure is the source + // of truth for the definition on this lineage. + const workflowJsonPath = path.join(dataDir, "assets", "workflow", "wf-srcref", "workflow.json"); + await fs.mkdir(path.dirname(workflowJsonPath), { recursive: true }); + await fs.writeFile(workflowJsonPath, "}{ not valid json", "utf8"); + + // Stub the closure materializer: record its inputs, and return a fake + // package dir plus the evaluated live definition the restore arm now + // projects to the inert wire shape -- the SAME + // `WorkflowProjectionDefinition(projectLiveToInert(...))` computation the + // deploy path applies. The definition (not the on-disk workflow.json) is the + // source of truth for the restored definition, so it must be a valid live + // definition whose projection covers the record's sources (`step-1`). + const fakePackageDir = path.join(dataDir, "fake-closure-package"); + const closureDefinition = { + id: "wf-srcref", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { + "step-1": { + kind: "step", + id: "step-1", + agent: { + id: "agent-1", + systemPrompt: "sys", + capabilities: [], + toolFactories: [], + inference: { sources: [] }, + }, + }, + }, + }; + const applyCalls: { registry: string; entryCount: number }[] = []; + const applyStub: NonNullable< + Parameters[0]["applyFrozenWorkflowClosure"] + > = (args) => { + if (args.source.kind !== "registry") { + throw new Error(`applyStub expected a registry source, got ${args.source.kind}`); + } + applyCalls.push({ + registry: args.source.registry, + entryCount: args.closure.entries.length, + }); + return Promise.resolve({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the stub stands in for a real closure evaluation; a minimal hand-built live definition cannot satisfy the full `WorkflowDefinition` nominal type + definition: closureDefinition as unknown as WorkflowDefinition, + packageDir: fakePackageDir, + deployDir: path.join(dataDir, "fake-deploy-dir"), + }); + }; + + const spawner = makeReadyDrivingSpawner(9560); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: dataDir, + SIDECAR_CACHE_MAX_BYTES: "1000000", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "1000000", + }, + applyFrozenWorkflowClosure: applyStub, + }); + + const restorePromise = router.restoreWorkflowRuns(); + await spawner.driveReadyFor(0); + await restorePromise; + + // Re-materialized (the frozen closure fed to the materializer) and spawned. + expect(applyCalls).toEqual([{ registry: "test-registry", entryCount: 0 }]); + expect(spawner.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, head)).toBe(true); + // The child came back on the source-ref (evaluate-the-closure) load path: + // the spawn env carries the freshly re-materialized closure package dir, so + // the child evaluates the pinned code rather than reading a definition off + // disk. Source-ref is the only lineage now, so the env always carries it. + const spawnEnv = spawner.envFor(0); + expect(spawnEnv?.CLOSURE_PACKAGE_DIR).toBe(fakePackageDir); + }); + + test("restore is a no-op for a deployment already live in this process", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-guard-data-"); + const head = "run_guard@example.com"; + + const spawner = makeReadyDrivingSpawner(9600); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-guard")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(spawner.spawnCount()).toBe(1); + + // The record is on disk and the address is live in this same process. A + // restore pass must NOT spawn a second child for an address the core's + // double-spawn guard already owns (the transition guard the B-reroute + // follow-up leans on). + await router.restoreWorkflowRuns(); + + expect(spawner.spawnCount()).toBe(1); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("a second deploy for a live address is rejected without orphaning its restore record", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-dup-data-"); + const head = "run_dup@example.com"; + const anchorRunId = deriveDeploymentId(head); + + const spawner = makeReadyDrivingSpawner(9700); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-dup")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(await recordExists(dataDir, anchorRunId)).toBe(true); + + // A second deploy for the already-live address must be rejected WITHOUT + // touching the running deployment's durable state. The reject fires + // before any overwrite; without it, deployMultiStep's catch would delete + // the live deployment's record and release its slug, silently breaking + // the next restart for a still-running agent. + await expect(router.deploy(singleStepFrame(head, "wf-dup"))).rejects.toThrow( + /already deployed/, + ); + expect(spawner.spawnCount()).toBe(1); + expect(await recordExists(dataDir, anchorRunId)).toBe(true); + expect(isRegistered(transport, head)).toBe(true); + }); + + test("a self-terminated deployment is reclaimed so its address redeploys without a manual undeploy", async () => { + const dataDir = await createTempBaseDir("sidecar-self-term-redeploy-data-"); + const head = "run_selfterm@example.com"; + const anchorRunId = deriveDeploymentId(head); + + const spawner = makeReadyDrivingSpawner(9750); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + // Deploy and bring the supervisor to `running`. + const deployPromise = router.deploy(singleStepFrame(head, "wf-selfterm")); + await spawner.driveReadyFor(0); + await deployPromise; + expect(router.activeAddresses()).toEqual([head]); + expect(isRegistered(transport, head)).toBe(true); + + // Drive a self-termination: a child-initiated recycle whose respawn spawn + // fails tears the supervisor down to `stopped` through the recycle-failure + // path, which fires `onSelfTerminate`. That drives the reclaim. + spawner.failNextSpawn(); + await spawner.recycleRequestFor(0); + + // The reclaim drops the address from the active map and releases its + // transport registration. The self-termination runs asynchronously off the + // supervisor's control pump, so poll for the reclaim to settle. + const deadline = Date.now() + 5_000; + while (router.activeAddresses().includes(head) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1)); + } + expect(router.activeAddresses()).toEqual([]); + expect(isRegistered(transport, head)).toBe(false); + + // The redeploy succeeds with no prior undeploy: the map slot is free and + // the transport is no longer registered (a stale registration would make + // the spawn core's `transport.register` throw "already registered"). The + // self-terminated deployment left its durable record and its step-state + // scratch behind; the redeploy overwrites the record destructively. + expect(await recordExists(dataDir, anchorRunId)).toBe(true); + const redeployPromise = router.deploy(singleStepFrame(head, "wf-selfterm")); + await spawner.driveReadyFor(1); + await redeployPromise; + expect(router.activeAddresses()).toEqual([head]); + expect(isRegistered(transport, head)).toBe(true); + expect(spawner.spawnCount()).toBe(2); + }); + + test("a reclaimed self-terminated address survives a following operator undeploy", async () => { + const dataDir = await createTempBaseDir("sidecar-self-term-undeploy-data-"); + const head = "run_selfterm_undeploy@example.com"; + + const spawner = makeReadyDrivingSpawner(9760); + const { router, transport } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-st-undeploy")); + await spawner.driveReadyFor(0); + await deployPromise; + + spawner.failNextSpawn(); + await spawner.recycleRequestFor(0); + const deadline = Date.now() + 5_000; + while (router.activeAddresses().includes(head) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1)); + } + expect(router.activeAddresses()).toEqual([]); + + // An operator undeploy following the reclaim is a clean no-op: the reclaim + // already dropped the supervisor and the transport registration, so + // undeploy's idempotent unregisters neither throw nor double-remove. This + // is the observable form of the reclaim/undeploy race resolution -- both + // are "if present, drop", so the loser no-ops. + const undeploy = router.undeploy; + if (undeploy === undefined) { + throw new Error("router.undeploy is undefined"); + } + await expect( + undeploy({ + type: "agent.undeploy", + agentAddress: head, + reason: "operator undeploy after self-termination", + }), + ).resolves.toBeUndefined(); + expect(router.activeAddresses()).toEqual([]); + expect(isRegistered(transport, head)).toBe(false); + }); + + test("the reclaim retains the deployment-address mapping so the terminal RunFailed commit can still resolve its run address", async () => { + // Regression guard for the reclaim/terminal-commit ordering hazard. The + // crash-loop latch commits its `RunFailed` tombstone AFTER teardown fires + // `onSelfTerminate`, and that commit resolves the deployment-address + // mapping to route the outbound pack push. A reclaim that dropped the + // mapping (via `unregisterDeployment`) stranded the commit with "no run + // address registered". This wires a REAL `deploymentAddressRegistry` + // through the register/unregister hooks -- the same registry the sidecar + // wires in `index.ts` -- and asserts the mapping survives the reclaim, so + // a subsequent `resolve` still returns the address the commit needs. + const dataDir = await createTempBaseDir("sidecar-self-term-mapping-data-"); + const head = "run_selfterm_mapping@example.com"; + const runId = deriveDeploymentId(head); + + const registry = createDeploymentAddressRegistry(); + + const spawner = makeReadyDrivingSpawner(9770); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + registerDeployment: ({ runId: id, agentAddress }) => { + registry.record(id, agentAddress); + }, + unregisterDeployment: ({ runId: id }) => { + registry.unregister(id); + }, + }); + + const deployPromise = router.deploy(singleStepFrame(head, "wf-st-mapping")); + await spawner.driveReadyFor(0); + await deployPromise; + // The deploy recorded the mapping before spawn (the replay's pack push + // resolves it), so it is resolvable while the supervisor is live. + expect(registry.resolve(runId)).toBe(head); + + // Drive the supervisor to a self-termination via the recycle-failure path. + spawner.failNextSpawn(); + await spawner.recycleRequestFor(0); + const deadline = Date.now() + 5_000; + while (router.activeAddresses().includes(head) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 1)); + } + expect(router.activeAddresses()).toEqual([]); + + // The reclaim dropped the redeploy gate but RETAINED the address mapping: + // the supervisor's own terminal `RunFailed` commit is the sole remaining + // consumer and must still resolve the address. A reclaim that unregistered + // the mapping would fail this assertion (and strand that commit). + expect(registry.resolve(runId)).toBe(head); + }); + + test("restore skips a record whose address does not derive its directory name", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-mismatch-data-"); + const head = "run_mismatch@example.com"; + // A record filed under a directory that is NOT its own derived slug -- + // a corrupt or misplaced record that must not be restored under the + // wrong slug. + const wrongDir = "not-the-right-slug"; + // Otherwise-valid source-ref record so the scan admits it and the restore + // loop reaches (and rejects on) the address-vs-directory mismatch -- not the + // schema. Source-ref is the only lineage, so it must carry the pin + hash. + const record: WorkflowRunRecord = { + version: 1, + agentAddress: head, + definitionId: "wf-mismatch", + sources: { "step-1": [makeInferenceSource("step-1")] }, + hubPublicKey: "hub-pk", + approvedWireHash: "a".repeat(64), + lineage: "source-ref", + sourceRef: { + source: { kind: "registry", registry: "test-registry" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + }, + }; + await writeWorkflowRunRecord(dataDir, wrongDir, record, createNoopCredentialCipher()); + + const spawner = makeReadyDrivingSpawner(9800); + const freshTransport = createInMemoryTransport(); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await router.restoreWorkflowRuns(); + + expect(spawner.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record is kept on a skip, not deleted. + expect(await recordExists(dataDir, wrongDir)).toBe(true); + }); + + test("restore soft-fails and keeps the record when the pinned source is no longer buildable", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-unbuildable-data-"); + const head = "run_unbuildable_restore@example.com"; + const anchorRunId = deriveDeploymentId(head); + + // First process: a permissive gate lets the deploy through, persisting + // the record and its workflow.json. + const first = makeReadyDrivingSpawner(9900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(singleStepFrame(head, "wf-unbuildable-restore")); + await first.driveReadyFor(0); + await deployPromise; + + // Restart with a gate that now rejects the pinned provider. + const second = makeReadyDrivingSpawner(10000); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + assertSourceBuildable: (source) => { + throw new Error(`Source provider "${source.provider}" is not registered`); + }, + }); + + await routerB.restoreWorkflowRuns(); + + expect(second.spawnCount()).toBe(0); + expect(isRegistered(freshTransport, head)).toBe(false); + // The record survives so a later boot, once the provider is buildable + // again, can retry it. + expect(await recordExists(dataDir, anchorRunId)).toBe(true); + }); + + test("restore isolates an unbuildable-provider record: it keeps the record and surfaces the failure while the healthy deployment still restores", async () => { + const dataDir = await createTempBaseDir("sidecar-restore-unbuildable-isolate-data-"); + const healthyHead = "run_healthy_isolate@example.com"; + const unbuildableHead = "run_unbuildable_isolate@example.com"; + const healthyId = deriveDeploymentId(healthyHead); + const unbuildableId = deriveDeploymentId(unbuildableHead); + + // The unbuildable deployment pins a source whose provider the restart's + // gate will reject; the healthy deployment keeps the default `anthropic` + // source the gate admits. Distinguishing on `provider` lets one + // `assertSourceBuildable` reject exactly one of the two restored records. + const unbuildableProvider = "phantom-provider"; + function unbuildableSingleStepFrame(): AgentDeployFrame { + return makeMultistepFrame({ + agentAddress: unbuildableHead, + definition: { + id: "wf-unbuildable-isolate", + triggers: [{ type: "manual" }], + stepOrder: ["step-1"], + steps: { "step-1": { kind: "step" } }, + }, + sources: { + "step-1": [{ ...makeInferenceSource("step-1"), provider: unbuildableProvider }], + }, + }); + } + + // First process: a permissive gate lets BOTH deploys through, persisting + // each record and its workflow.json. + const first = makeReadyDrivingSpawner(11400); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployHealthy = routerA.deploy(singleStepFrame(healthyHead, "wf-healthy-isolate")); + await first.driveReadyFor(0); + await deployHealthy; + const deployUnbuildable = routerA.deploy(unbuildableSingleStepFrame()); + await first.driveReadyFor(1); + await deployUnbuildable; + expect(await recordExists(dataDir, healthyId)).toBe(true); + expect(await recordExists(dataDir, unbuildableId)).toBe(true); + + // Restart: a fresh transport plus a gate that rejects ONLY the phantom + // provider. Capture the module's warn output through the default console + // sink (threshold "warning" routes `logger.warn` to `console.warn`) so we + // can assert the failure is surfaced loudly rather than silently dropped. + const warnCaptured: string[] = []; + // eslint-disable-next-line no-console -- intentionally spy on the default sink's warn target to prove the restore failure is surfaced + const originalWarn = console.warn; + // eslint-disable-next-line no-console + console.warn = (...parts: unknown[]) => { + warnCaptured.push(parts.map((part) => String(part)).join(" ")); + }; + try { + const second = makeReadyDrivingSpawner(11500); + const freshTransport = createInMemoryTransport(); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: freshTransport, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + assertSourceBuildable: (source) => { + if (source.provider === unbuildableProvider) { + throw new Error(`Source provider "${source.provider}" is not registered`); + } + }, + }); + + // Only the healthy deployment spawns, so its handshake is the sole one + // to drive; the unbuildable record faults before its spawner is ever + // reached. Restore is serial and per-record isolated, so scan order does + // not change the outcome. + const restorePromise = routerB.restoreWorkflowRuns(); + await second.driveReadyFor(0); + await restorePromise; + + // The unbuildable record did NOT strand the healthy one: it re-spawned + // exactly once and its head is routable again on the fresh transport. + expect(second.spawnCount()).toBe(1); + expect(isRegistered(freshTransport, healthyHead)).toBe(true); + + // The unbuildable deployment was not stood up: no spawn, no route. + expect(isRegistered(freshTransport, unbuildableHead)).toBe(false); + + // Its record survives -- restore keeps an unrestorable record so a later + // boot with the provider restored can retry it. + expect(await recordExists(dataDir, unbuildableId)).toBe(true); + + // The failure is surfaced loudly: a warning naming the failed deployment + // and the provider rejection reason, not a silent drop. + const failureWarn = warnCaptured.find( + (line) => + line.includes(unbuildableId) && + line.includes(unbuildableProvider) && + line.includes("is not registered"), + ); + expect(failureWarn).toBeDefined(); + } finally { + // eslint-disable-next-line no-console + console.warn = originalWarn; + } + }); + + test("a deploy whose child never signals ready times out and rejects", async () => { + const dataDir = await createTempBaseDir("sidecar-ready-timeout-data-"); + const head = "run_readytimeout@example.com"; + const anchorRunId = deriveDeploymentId(head); + + // A spawner whose child is created but never driven through the `ready` + // handshake. With a small threaded readyTimeoutMs the supervisor times + // out, kills the child, and rejects the spawn. The message echoes the + // threaded value, so this also proves readyTimeoutMs reaches the + // supervisor across the router's forwarding. + const spawner = makeReadyDrivingSpawner(10100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + readyTimeoutMs: 40, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + await expect(router.deploy(singleStepFrame(head, "wf-readytimeout"))).rejects.toThrow( + /did not emit ready within 40ms/, + ); + + // The deploy soft-failed, so its restore record was cleaned up -- a + // wedged deploy leaves nothing for a later boot to re-spawn. + expect(await recordExists(dataDir, anchorRunId)).toBe(false); + }); + + test("a single-step deploy acks the agent key, not the supervisor key", async () => { + // The single-step head is the deployed agent identity, so the ack must + // surface the agent key rather than the supervisor principal key. + const headKeyPair = await generateKeyPair(); + const spawner = makeReadyDrivingSpawner(10300); + const { router, keyPair: fixtureKeyPair } = await buildMultistepFixture({ + spawner: spawner.spawner, + headKeyPair, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-bkey-data-"), + }, + }); + + const deployPromise = router.deploy(singleStepFrame("run_bkey@example.com", "wf-bkey")); + await spawner.driveReadyFor(0); + const result = await deployPromise; + + // The supervisor principal key is derived from the fixture's signing seed + // (fixtureKeyPair); the head's agent key is the distinct headKeyPair. + expect(result.publicKey).toBe(Buffer.from(headKeyPair.publicKey).toString("hex")); + expect(result.publicKey).not.toBe(Buffer.from(fixtureKeyPair.publicKey).toString("hex")); + }); + + test("registers a sources-rotation handler for a single-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10600); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-single-"), + }, + }); + + const deployPromise = router.deploy( + singleStepFrame("run_srcsingle@example.com", "wf-srcsingle"), + ); + await spawner.driveReadyFor(0); + await deployPromise; + + // The single-step deploy registered a rotation handler, so an inbound + // sources.update for its address routes. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: "run_srcsingle@example.com", + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(true); + }); + + test("does not register a sources-rotation handler for a multi-step deployment", async () => { + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10700); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-sources-multi-"), + }, + }); + + const frame = makeMultistepFrame({ + definition: { + id: "wf-srcmulti", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }, + sources: { + "step-1": [makeInferenceSource("step-1")], + "step-2": [makeInferenceSource("step-2")], + }, + }); + const deployPromise = router.deploy(frame); + await spawner.driveReadyFor(0); + await deployPromise; + + // A multi-step deployment has no single warm agent to rotate, so no + // handler is registered and the inbound rotation is unrouted. + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: frame.agentAddress, + sources: [makeInferenceSource("primary")], + defaultSource: "primary", + }), + ).toBe(false); + }); + + test("a source rotation survives a recycle respawn", async () => { + // End-to-end guard for the rotation-survives-recycle fix: the single-step + // rotation handler mutates `currentSources`, `dynamicSpawnEnv` + // re-serializes it, and the recycle respawn's STEP_INFERENCE_SOURCES + // carries the ROTATED table -- not the frozen deploy-time one. + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(10800); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-rot-survive-"), + }, + }); + + const addr = "run_rotsurvive@example.com"; + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotsurvive")); + await spawner.driveReadyFor(0); + await deployPromise; + + // The initial spawn carries the deploy-time source table. + const initialEnv = spawner.envFor(0); + if (initialEnv === undefined) throw new Error("initial spawn env missing"); + expect(JSON.parse(initialEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [makeInferenceSource("step-1")], + }); + + // Rotate the single-step deployment's sources in place. + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // The child asks to recycle; the supervisor respawns. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + + // The recycle respawn's sources are the ROTATED table, proving the + // rotation survived the recycle (before the fix it reverted to the + // deploy-time list frozen in substrateEnv). + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect(JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [rotated], + }); + }); + + test("a source rotation survives a full sidecar restart", async () => { + // Restart-durability: a rotation is persisted into the run record, + // so a fresh sidecar process (a restore over the same data dir) respawns + // the deployment on the ROTATED sources, not the deploy-time ones. + const dataDir = await createTempBaseDir("sidecar-rot-restart-"); + const addr = "run_rotrestart@example.com"; + + // First process: deploy a single-step deployment and rotate its sources. + const sourcesRouter = createMultistepSourcesRouter(); + const first = makeReadyDrivingSpawner(10900); + const { router: routerA } = await buildMultistepFixture({ + spawner: first.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const deployPromise = routerA.deploy(singleStepFrame(addr, "wf-rotrestart")); + await first.driveReadyFor(0); + await deployPromise; + + const rotated = makeInferenceSource("rotated"); + expect( + await sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }), + ).toBe(true); + + // Second process (simulated restart): a fresh router over the SAME data + // dir restores the deployment from its durable record. + const second = makeReadyDrivingSpawner(11000); + const { router: routerB } = await buildMultistepFixture({ + spawner: second.spawner, + transport: createInMemoryTransport(), + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + const restorePromise = routerB.restoreWorkflowRuns(); + await second.driveReadyFor(0); + await restorePromise; + + // The restored spawn carries the ROTATED sources, read back from the + // durable record -- not the deploy-time list. + const restoredEnv = second.envFor(0); + if (restoredEnv === undefined) { + throw new Error("restored spawn env missing"); + } + expect(JSON.parse(restoredEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [rotated], + }); + }); + + test("a rotation whose persist fails leaves no partial effect", async () => { + // Atomicity: if the durable write rejects, the rotation takes NO net + // effect. The handler swaps currentSources synchronously, then rolls it + // back on a failed persist, so once the handler throws currentSources is + // on the deploy-time table (a recycle respawn AFTER the failure is + // unrotated) and deliverSources is never reached. Rolling the in-memory + // hint back keeps currentSources and the record in agreement in this + // no-interleaved-recycle failure case. + const dataDir = await createTempBaseDir("sidecar-rot-failatomic-"); + const addr = "run_rotfailatomic@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11100); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotfail")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Replace the run record file with a directory so the rotation's + // writeWorkflowRunRecord rejects (EISDIR) -- a deterministic, + // root-immune write fault (a chmod guard would be bypassed under root). + const recordFile = path.join( + dataDir, + "workflow-runs", + deriveDeploymentId(addr), + "deployment.json", + ); + await fs.rm(recordFile); + await fs.mkdir(recordFile); + + // The persist rejects, so the route rejects and nothing after the write + // runs. + await expect( + sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [makeInferenceSource("rotated")], + defaultSource: "rotated", + }), + ).rejects.toThrow(); + + // currentSources was rolled back: a recycle respawn AFTER the failed + // rotation carries the DEPLOY-TIME sources, proving the failed rotation + // left no net in-memory effect. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect(JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [makeInferenceSource("step-1")], + }); + }); + + test("a recycle interleaving the rotation persist respawns on the rotated sources", async () => { + // Interleave guard: the rotation swaps currentSources synchronously + // BEFORE the durable persist, so a recycle that lands inside the persist + // window respawns the child on the ROTATED sources -- consistent with + // what is being persisted -- rather than the stale deploy-time table. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-ok-"); + const addr = "run_rotinterok@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11200); + + // A persist that blocks on a test-controlled gate once armed, so a recycle + // can be driven into the rotation's persist window. The deploy's own + // persist (before the gate is armed) passes straight through to the real + // writer. + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowRunRecord = async (d, id, rec, cipher) => { + if (gate !== null) await gate; + await writeWorkflowRunRecord(d, id, rec, cipher); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowRunRecord: persist, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotinterok")); + await spawner.driveReadyFor(0); + await deployPromise; + + // Arm the gate so the rotation's persist blocks mid-window. + gate = new Promise((resolve) => { + release = resolve; + }); + + // Start the rotation but do not await it: the handler swaps currentSources + // synchronously, then parks on the blocked persist before deliverSources. + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // Drive a recycle while the persist is blocked. The respawn env must carry + // the ROTATED sources, because the synchronous swap already ran. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect(JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [rotated], + }); + + // Release the persist; the rotation completes without throwing and the + // durable record converges on the rotated sources. + release(); + await rotatePromise; + const scanned = await scanWorkflowRunRecords(dataDir, createNoopCredentialCipher()); + const record = scanned.find((s) => s.runId === deriveDeploymentId(addr)); + expect(record?.record.sources).toEqual({ "step-1": [rotated] }); + }); + + test("a recycle interleaving a failed rotation persist respawns on new sources then rolls back", async () => { + // The benign residual, pinned: a persist that fails WHILE a recycle + // interleaves leaves the just-respawned child transiently ahead on the + // rotated sources (the intended, self-healing direction), while + // currentSources rolls back to the deploy-time table so the NEXT recycle + // reverts the child to durable truth. + const dataDir = await createTempBaseDir("sidecar-rot-interleave-fail-"); + const addr = "run_rotinterfail@example.com"; + const sourcesRouter = createMultistepSourcesRouter(); + const spawner = makeReadyDrivingSpawner(11300); + + let gate: Promise | null = null; + let release: () => void = () => undefined; + const persist: typeof writeWorkflowRunRecord = async (d, id, rec, cipher) => { + if (gate !== null) { + await gate; + throw new Error("rotation persist boom"); + } + await writeWorkflowRunRecord(d, id, rec, cipher); + }; + + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSourcesRouter: sourcesRouter, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + writeWorkflowRunRecord: persist, + }); + + const deployPromise = router.deploy(singleStepFrame(addr, "wf-rotinterfail")); + await spawner.driveReadyFor(0); + await deployPromise; + + gate = new Promise((resolve) => { + release = resolve; + }); + + const rotated = makeInferenceSource("rotated"); + const rotatePromise = sourcesRouter.tryRoute({ + type: "sources.update", + agentAddress: addr, + sources: [rotated], + defaultSource: "rotated", + }); + + // A recycle interleaves the about-to-fail persist: the respawn still + // carries the ROTATED sources, because the swap already ran. + await spawner.recycleRequestFor(0); + while (spawner.spawnCount() < 2) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(1); + const respawnEnv = spawner.envFor(1); + if (respawnEnv === undefined) throw new Error("respawn env missing"); + expect(JSON.parse(respawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [rotated], + }); + + // Release the persist so it rejects; the rotation rolls currentSources + // back and rethrows. + release(); + await expect(rotatePromise).rejects.toThrow(/rotation persist boom/); + + // A SECOND recycle now respawns on the ROLLED-BACK deploy-time sources, + // healing the transient down to durable truth. + await spawner.recycleRequestFor(1); + while (spawner.spawnCount() < 3) { + await new Promise((r) => setTimeout(r, 1)); + } + await spawner.driveReadyFor(2); + const secondRespawnEnv = spawner.envFor(2); + if (secondRespawnEnv === undefined) { + throw new Error("second respawn env missing"); + } + expect(JSON.parse(secondRespawnEnv[STEP_INFERENCE_SOURCES_ENV_KEY] ?? "null")).toEqual({ + "step-1": [makeInferenceSource("step-1")], + }); + }); + + test("two addresses whose deriveDeploymentId slugs collide are rejected at the second deploy", async () => { + // deriveDeploymentId substitutes every disallowed character with + // `-`, so two distinct addresses can collapse to the same slug. The slug + // IS the workflow-run repoId, so a silent collision would let the second + // deploy overwrite the first deploy's repo state. claimSlug rejects the + // second deploy at the router edge, before any spawn or repo write. + const spawner = makeReadyDrivingSpawner(10400); + const { router } = await buildMultistepFixture({ + spawner: spawner.spawner, + multistepSubstrateEnv: { + SIDECAR_DATA_DIR: await createTempBaseDir("sidecar-collision-data-"), + }, + }); + + // `run_col.a@example.com` and `run_col-a@example.com` both project to + // `run_col-a-example-com` under the slug derivation. + const deployPromise = router.deploy(singleStepFrame("run_col.a@example.com", "wf-collide")); + await spawner.driveReadyFor(0); + await deployPromise; + + await expect( + router.deploy(singleStepFrame("run_col-a@example.com", "wf-collide")), + ).rejects.toThrow(/deriveDeploymentId collision/); + expect(spawner.spawnCount()).toBe(1); + }); + + test("reEmitParkedCorrelations reaches the live supervisor and sends a parked-correlations.request downstream", async () => { + // Trigger B: the hub-reconnect fan-out. After a deploy populates + // `activeSupervisors` for the deployment address, the router's + // address-dispatch wrapper must reach the live supervisor's own no-arg + // `reEmitParkedCorrelations`, which the supervisor implements by sending a + // `parked-correlations.request` control frame down to the child. The mock + // child never answers, so the supervisor's watchdog eventually fires; the + // router's fire-and-forget contract means the request frame lands on the + // downstream stream regardless, which is the observable evidence here. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 10500, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const dataDir = await createTempBaseDir("sidecar-reemit-data-"); + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-reemit", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + const deployPromise = router.deploy(frame); + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 10500, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + await deployPromise; + + // `activeSupervisors` is keyed by the frame's run address; the router + // routes the re-emit through that same key. + expect(router.activeAddresses()).toEqual([frame.agentAddress]); + + // Each downstream line is a signed envelope `{ envelope: { seq, channelId, + // payload }, sig }`; read `envelope.payload.type` without verifying the + // signature (the supervisor's IPC public key is not exposed to the test). + // The supervisor emits its own `parked-correlations.request` on spawn (a + // fresh child becoming addressable), so the trigger is observed as an + // increase in the downstream request count, not its first appearance. + function parkedRequestCount(): number { + return supervisorToChild + .flushed() + .filter((line) => downstreamPayloadType(line) === PARKED_REQUEST_TYPE).length; + } + + // Let the spawn-time Trigger A re-emit settle so the baseline captures it; + // the assertion below then proves the Trigger B call adds a request on top. + await new Promise((r) => setTimeout(r, 50)); + const baseline = parkedRequestCount(); + + router.reEmitParkedCorrelations(frame.agentAddress); + + // Fire-and-forget: poll the downstream stream until the router's request + // frame lands on top of the spawn-time baseline, bounded so a wiring break + // fails the test instead of hanging. + const deadline = Date.now() + 2000; + while (parkedRequestCount() <= baseline) { + if (Date.now() > deadline) { + throw new Error( + "reEmitParkedCorrelations did not add a parked-correlations.request to the downstream control stream", + ); + } + await new Promise((r) => setTimeout(r, 1)); + } + expect(parkedRequestCount()).toBeGreaterThan(baseline); + }); + + test("reEmitParkedCorrelations for an address with no active supervisor is a no-op", async () => { + // The edge boundary: the hub reports an address routable, but no live + // supervisor owns it (a torn-down or not-yet-respawned deployment). The + // router must skip it -- neither throw nor drive any downstream frame. + const childIpcKeyPair = await generateKeyPair(); + const supervisorToChild = createMemoryNdjsonStream(); + const childToSupervisor = createMemoryNdjsonStream(); + const eventChildToSupervisor = createMemoryFrameStream(); + let resolveExit: ((code: number) => void) | undefined; + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + let observedEnv: Record | undefined; + const spawner: SubprocessSpawner = ({ env }) => { + observedEnv = env; + const handle: SubprocessHandle = { + pid: 10600, + controlWriter: supervisorToChild.writer, + controlReader: childToSupervisor.reader, + eventReader: eventChildToSupervisor.reader, + kill: () => { + childToSupervisor.close(); + eventChildToSupervisor.close(); + resolveExit?.(0); + }, + exited, + }; + return handle; + }; + + const dataDir = await createTempBaseDir("sidecar-reemit-miss-data-"); + const { router } = await buildMultistepFixture({ + spawner, + multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, + }); + + const sources = defaultMultistepSources(); + const definition = { + id: "wf-reemit-miss", + triggers: [{ type: "manual" }], + stepOrder: ["step-1", "step-2"], + steps: { "step-1": { kind: "step" }, "step-2": { kind: "step" } }, + }; + const frame = makeMultistepFrame({ definition, sources }); + + const deployPromise = router.deploy(frame); + while (observedEnv === undefined) { + await new Promise((r) => setTimeout(r, 1)); + } + const channelId = observedEnv.IPC_CHANNEL_ID; + if (channelId === undefined) { + throw new Error("IPC_CHANNEL_ID not set in spawn-time env"); + } + const childSender = createControlChannelSender({ + privateKeySeed: childIpcKeyPair.privateKey, + channelId, + writer: { + write(line: string) { + childToSupervisor.inject(line); + return Promise.resolve(); + }, + }, + }); + await childSender.send({ + type: "ready", + data: { + childPid: 10600, + childPublicKey: hexEncode(childIpcKeyPair.publicKey), + }, + }); + await deployPromise; + + const missAddress = "run_nonexistent@wf.example"; + expect(router.activeAddresses()).not.toContain(missAddress); + + // The deployed supervisor emits its own `parked-correlations.request` on + // spawn; the miss-address re-emit must add nothing on top of that + // baseline. Count downstream requests before and after the no-op call. + function parkedRequestCount(): number { + return supervisorToChild + .flushed() + .filter((line) => downstreamPayloadType(line) === PARKED_REQUEST_TYPE).length; + } + // Settle the spawn-time Trigger A re-emit so it is folded into the baseline + // rather than racing the post-call window below. + await new Promise((r) => setTimeout(r, 50)); + const baseline = parkedRequestCount(); + + // The no-op skip must not throw for an address with no live supervisor. + expect(() => router.reEmitParkedCorrelations(missAddress)).not.toThrow(); + + // Give any errant fire-and-forget a chance to land, then confirm the miss + // added no downstream request for the missing address. + await new Promise((r) => setTimeout(r, 50)); + expect(parkedRequestCount()).toBe(baseline); + }); +}); + +describe("assembleRunCredentialsSnapshot", () => { + // getRepoDir mirrors the production `//` layout that + // `createSpawnTestRepoStore` uses, so a file written at + // `/workflow-run//runs//grants.json` lands where + // the sink's working-tree read looks, and a step's deploy-time grants land + // at `/agent-state//state/grants.json`. + function createReadStubRepoStore(tempBase: string): RepoStore { + const stub: Partial = { + getRepoDir(repoId: RepoId): string { + return path.join(tempBase, repoId.kind, repoId.id); + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; the sink reads only via getRepoDir working-tree reads + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub RepoStore: ${String(prop)} not implemented for this test`); + }; + }, + }); + } + + const anchorRunId = "dep-run-grants"; + const runId = "run-xyz"; + const stepOrder = ["step-1", "step-2"]; + const deriveStepAddress = ({ runId: dep, stepId }: { runId: string; stepId: string }) => + `${dep}-${stepId}@example.com`; + + async function writeRunGrantsFile(tempBase: string, contents: string): Promise { + const dir = path.join(tempBase, "workflow-run", anchorRunId, "runs", runId); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "grants.json"), contents); + } + + async function writeDeployTimeStepGrants( + tempBase: string, + stepId: string, + contents: string, + ): Promise { + const dir = path.join(tempBase, "agent-state", `${anchorRunId}-${stepId}`, "state"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "grants.json"), contents); + } + + test("a run with a per-run grants file reads those grants over deploy-time", async () => { + const tempBase = await createTempBaseDir("sidecar-run-grants-"); + const repoStore = createReadStubRepoStore(tempBase); + + const runGrants = [{ id: "run-grant", resource: "tool:send-mail", effect: "allow" }]; + await writeRunGrantsFile(tempBase, JSON.stringify({ grants: runGrants })); + // Deploy-time grants differ; the per-run read must win, so these are + // never surfaced. + await writeDeployTimeStepGrants( + tempBase, + "step-1", + JSON.stringify({ grants: [{ id: "deploy-grant" }] }), + ); + + const snapshot = await assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }); + + // Every step carries the single flat per-run grant set, keyed on the + // deploy-time per-step address, with a shared content hash. + expect(snapshot.steps).toHaveLength(2); + for (const step of snapshot.steps) { + expect(step.grants).toEqual(runGrants); + expect(step.address).toBe(`${anchorRunId}-${step.stepId}@example.com`); + } + expect(snapshot.steps[0]?.stepId).toBe("step-1"); + expect(snapshot.steps[1]?.stepId).toBe("step-2"); + expect(snapshot.steps[0]?.contentHash).toBe(snapshot.steps[1]?.contentHash ?? ""); + }); + + test("a run without a per-run grants file fails closed", async () => { + const tempBase = await createTempBaseDir("sidecar-run-grants-"); + const repoStore = createReadStubRepoStore(tempBase); + + // No per-run file. A deploy-time file is present so a silent fallback + // would surface deploy-time grants rather than failing; the run must fail + // closed instead of running against them. + await writeDeployTimeStepGrants( + tempBase, + "step-1", + JSON.stringify({ grants: [{ id: "deploy-grant-1" }] }), + ); + + await expect( + assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }), + ).rejects.toThrow(/has no grants file/); + }); + + test("a run with a malformed per-run grants file throws", async () => { + const tempBase = await createTempBaseDir("sidecar-run-grants-"); + const repoStore = createReadStubRepoStore(tempBase); + + // The file exists but is not valid JSON. A deploy-time file is present so + // a swallowed error would silently fall back rather than surface. + await writeRunGrantsFile(tempBase, "{ not valid json"); + await writeDeployTimeStepGrants( + tempBase, + "step-1", + JSON.stringify({ grants: [{ id: "deploy-grant" }] }), + ); + + await expect( + assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }), + ).rejects.toThrow(/is not valid JSON/); + + // A file that is valid JSON but violates the `{ grants: [] }` envelope + // also throws, rather than falling back -- the presence of the file + // implies a grants frame was delivered. + await writeRunGrantsFile(tempBase, JSON.stringify({ grants: "not-array" })); + await expect( + assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }), + ).rejects.toThrow(/failed validation/); + }); + + test("a re-dispatched run re-reads the durable per-run grants, not the deploy-time fallback", async () => { + // On a child respawn the supervisor's `replayProcessingToInbox` moves an + // already-consumed run's orphaned `processing/` entry back to `inbox/`, + // so the fresh dispatch loop re-dequeues it and the `onRunStart` grants + // barrier fires AGAIN for the same runId. The per-run grants file is a + // durable commit -- `readRunGrants` never deletes it -- so this second + // resolution must still read `runs//grants.json` and win over the + // deploy-time fallback, rather than inheriting the deployment's grants as + // if the run had never carried its own. + const tempBase = await createTempBaseDir("sidecar-run-grants-"); + const repoStore = createReadStubRepoStore(tempBase); + + const runGrants = [{ id: "run-grant", resource: "tool:send-mail", effect: "allow" }]; + await writeRunGrantsFile(tempBase, JSON.stringify({ grants: runGrants })); + // A deploy-time file is present so a spurious fallback (a deleted or + // missed per-run file on the second read) would surface as the wrong + // grants rather than an empty set. + await writeDeployTimeStepGrants( + tempBase, + "step-1", + JSON.stringify({ grants: [{ id: "deploy-grant" }] }), + ); + + const first = await assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }); + + // The re-dispatch: resolve the same run a second time with no rewrite of + // the grants file in between. + const second = await assembleRunCredentialsSnapshot({ + repoStore, + anchorRunId, + runId, + stepOrder, + deriveStepAddress, + }); + + for (const snapshot of [first, second]) { + expect(snapshot.steps).toHaveLength(2); + for (const step of snapshot.steps) { + expect(step.grants).toEqual(runGrants); + } + } + // The second resolution is byte-identical to the first: same flat grant + // set, same content hash across every step. + expect(second.steps.map((s) => s.contentHash)).toEqual(first.steps.map((s) => s.contentHash)); + }); +}); diff --git a/apps/sidecar/src/workflow-host-wiring.ts b/apps/sidecar/src/workflow-host-wiring.ts new file mode 100644 index 0000000000..8c1155c61b --- /dev/null +++ b/apps/sidecar/src/workflow-host-wiring.ts @@ -0,0 +1,2742 @@ +// Thin wiring module that constructs `createWorkflowSupervisor` with +// this sidecar's host-specific bindings: the existing mail-bus +// instance, the sidecar's Ed25519 signing keypair, the substrate +// RepoStore handle, and `Bun.spawn` as the subprocess spawner. Any +// logic that would benefit a future alternative-sidecar +// implementation lives inside `@intx/workflow-host`, not here. + +import { rm, stat } from "node:fs/promises"; +import { join as pathJoin } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { type } from "arktype"; + +import { derivePublicKeyBytes, signEd25519 } from "@intx/crypto"; +import { getLogger } from "@intx/log"; +import type { HubTransport } from "@intx/mail-memory"; +import { + parseAgentId, + workflowSourceAssetMountPath, + type Principal, + type RepoId, + type RepoStore, + type WorkflowRunSupervisorPrincipal, +} from "@intx/hub-sessions"; +import { + resolveInboundMailPolicy, + type AgentKeyStore, + type DeployRouter, + type DeployRouterResult, + type InboundMailPolicyRegistry, + type SenderKeyCache, + type SessionManager, +} from "@intx/hub-agent"; +import { + createWorkflowSupervisor, + hashGrants, + STEP_GRANTS_PATH, + STEP_GRANTS_REF, + wrapHubTransportAsMailBus, + type CredentialsSnapshot, + type CredentialsSnapshotStep, + type DeriveStepAddress, + type DeriveStepRepoId, + type DispatchTimingMark, + type FrameReader, + type HubTransportMailBusAdapter, + type NdjsonReader, + type NdjsonWriter, + type SpawnOpts, + type SubprocessHandle, + type SubprocessSpawner, + type SuspensionRegistration, + type WorkflowSupervisor, +} from "@intx/workflow-host"; +import { hexDecode, hexEncode, type CredentialCipher, type SignalKind } from "@intx/types"; +import { + parseInferenceEvent, + type ApprovalSnapshot, + type CryptoProvider, + type InferenceEvent, + type InferenceSource, + type KeyPair, +} from "@intx/types/runtime"; +import { + WorkflowProjectionDefinition, + type AgentDeployFrame, + type CredentialDelivery, + type SourceRefPin, +} from "@intx/types/sidecar"; +import { STEP_ID_PATTERN, projectLiveToInert } from "@intx/workflow"; +import { deriveWorkflowRunRepoId, inertFlatNamespaceStepIds } from "@intx/workflow-deploy"; + +import { applyFrozenWorkflowClosure, type AppliedWorkflowClosure } from "./workflow-closure-apply"; +import { + MAX_INLINE_ASSET_PAYLOAD_BYTES, + materializeWorkflowAssets, + sourceAssetGitDir, +} from "./source-asset-delivery"; +import { readRegistries } from "./sidecar-materialization-config"; + +import type { + MultistepDrainRouter, + MultistepGrantsRouter, + MultistepMailRouter, + MultistepSignalRouter, + MultistepSourcesRouter, + MultistepCredentialsRouter, +} from "./workflow-run-pack-client"; +import { + deleteWorkflowRunRecord, + scanWorkflowRunRecords, + writeWorkflowRunRecord, + type WorkflowRunRecord, +} from "./workflow-run-record"; +import { readRunGrants, runGrantsPath } from "./run-grants"; + +const logger = getLogger(["interchange", "sidecar", "workflow-host-wiring"]); + +// A raw Ed25519 public key is 32 bytes. +const ED25519_PUBLIC_KEY_BYTES = 32; + +/** + * Project an run address into the substrate-safe id of its + * workflow-run repo. Both deploy branches key `{ kind: "workflow-run", + * id }` by this slug, and the supervisor principal's `anchorRunId` + * must equal that id for the workflow-run kind handler's authz check to + * pass. The derivation is owned by `@intx/workflow-deploy` so the hub's + * read routes reconstruct the identical id; this thin delegator keeps + * the sidecar's call sites readable while the rationale and the + * substrate `SAFE_REPO_ID` contract live with the shared function. + * + * The name keeps "Deployment" deliberately: it derives the deploy-phase + * routing slug (the workflow-run repo id an address projects to), not a + * run identity, so it survives the run-first identity sweep. + */ +export function deriveDeploymentId(agentAddress: string): string { + return deriveWorkflowRunRepoId(agentAddress); +} + +/** + * The durable per-deployment store the sidecar checks a source-ref deployment's + * source assets out into. A SIBLING of the closure instance dir, not a child: + * `materializeDeploymentClosure` reclaims the closure dir on every apply and + * restore, but never this store, so the checked-out assets survive a restart + * and re-materialization needs no re-delivery. The store is reclaimed on + * redeploy (at the deploy call site) and on undeploy. + */ +function deploymentSourceAssetRoot(dataDir: string, deploymentId: string): string { + return pathJoin(dataDir, "workflow-definition-sources", deploymentId); +} + +/** + * The durable indexed-`.git` store root a pinned deployment's source-format + * asset entries are checked out from. Sibling of the plain-file source store; + * both survive restart so re-materialization needs no re-delivery. + */ +function deploymentSourceGitRoot(dataDir: string, deploymentId: string): string { + return pathJoin(dataDir, "workflow-definition-source-gits", deploymentId); +} + +/** + * The `assetId -> mountPath` map a pinned closure's TARBALL `kind:"asset"` + * entries resolve against, derived purely from the pin (via the shared + * mount-path helper) so deploy and restore agree without the frame's delivered + * assets. Source-format entries resolve through `deriveSourceGitDirs` instead. + */ +function deriveSourceAssetMounts(pin: SourceRefPin): Map { + const mounts = new Map(); + for (const entry of pin.closure.entries) { + if (entry.source.kind === "asset" && entry.source.package.format === "tarball") { + mounts.set(entry.source.assetId, workflowSourceAssetMountPath(entry.source.assetId)); + } + } + return mounts; +} + +/** + * The `assetId -> gitDir` map a pinned closure's SOURCE `kind:"asset"` entries + * check subtrees out of, derived purely from the pin so deploy and restore + * agree without re-delivery. + */ +function deriveSourceGitDirs(pin: SourceRefPin, gitRoot: string): Map { + const gitDirs = new Map(); + for (const entry of pin.closure.entries) { + if (entry.source.kind === "asset" && entry.source.package.format === "source") { + gitDirs.set(entry.source.assetId, sourceAssetGitDir(gitRoot, entry.source.assetId)); + } + } + return gitDirs; +} + +async function isExistingDir(dir: string): Promise { + try { + return (await stat(dir)).isDirectory(); + } catch (err) { + if (err instanceof Error && "code" in err && err.code === "ENOENT") { + return false; + } + throw err; + } +} + +/** + * Resolve the durable source-asset store root and `assetId -> mountPath` map a + * pinned deployment materializes its `kind:"asset"` closure entries from, + * asserting every referenced asset's mount directory is present on disk. This + * is a cheap early gate for a missing checkout; the loader still SRI-verifies + * each tarball's bytes at materialization. Derived purely from the pin, so the + * deploy path and the boot-time restore path (which has only the pin, no + * re-delivery) resolve the identical mounts. A missing mount is a broken + * deployment the hub must re-drive, so it fails loud rather than materializing + * against an absent store. + */ +export async function resolveDeploymentAssetMounts( + dataDir: string, + deploymentId: string, + pin: SourceRefPin, +): Promise<{ + assetRoot: string; + assetMounts: ReadonlyMap; + gitDirs: ReadonlyMap; +}> { + const assetRoot = deploymentSourceAssetRoot(dataDir, deploymentId); + const assetMounts = deriveSourceAssetMounts(pin); + for (const [assetId, mountPath] of assetMounts) { + const mountDir = pathJoin(assetRoot, mountPath); + if (!(await isExistingDir(mountDir))) { + throw new Error( + `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} is not present in the durable store at ${mountDir}; the deployment must be re-driven from the hub`, + ); + } + } + const gitRoot = deploymentSourceGitRoot(dataDir, deploymentId); + const gitDirs = deriveSourceGitDirs(pin, gitRoot); + for (const [assetId, gitDir] of gitDirs) { + if (!(await isExistingDir(gitDir))) { + throw new Error( + `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} has no indexed git store at ${gitDir}; the deployment must be re-driven from the hub`, + ); + } + } + return { assetRoot, assetMounts, gitDirs }; +} + +/** + * Read a substrate-config byte cap (`SIDECAR_CACHE_MAX_BYTES` / + * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) from the multi-step substrate env and + * parse it to a positive finite number. The boot edge resolves these once and + * threads them through the substrate env; a source-ref deploy needs them to + * size the tarball cache and per-fetch cap when it materializes the frozen + * workflow closure. A missing or non-numeric value is a boot-edge wiring bug, + * so it fails loud rather than defaulting. + */ +function requireSubstrateByteCap(env: Record, key: string): number { + const raw = env[key]; + if (raw === undefined) { + throw new Error( + `sidecar deploy router: ${key} must be present in the multi-step substrate env to materialize a frozen workflow closure`, + ); + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `sidecar deploy router: ${key} must be a positive finite number, got ${JSON.stringify(raw)}`, + ); + } + return parsed; +} + +/** + * Hub principal the deploy router presents when it writes a step's + * grants into the agent-state repo on the sidecar's substrate. The + * agent-state kind handler gates `writeTree` as hub-only; the deploy + * router is the local stand-in for the hub on the sidecar's disk, so it + * claims the hub principal for this single bookkeeping write. The child + * reads the same repo via the working-tree path (`getRepoDir`), which is + * not authorize-gated. + */ +const GRANTS_WRITE_PRINCIPAL: Principal = { kind: "hub" }; + +/** + * Per-deploy address/repo strategy. The single-step launched-agent + * deploy and the derived multi-step deploy disagree on how the per-step + * mail address and agent-state repo id are computed; both + * `deriveStepAddress` (consumed by the supervisor's credentialsSnapshot + * assembly for the step's mail address) and `deriveStepRepoId` (consumed + * by the same assembly to locate each step's grants) must agree on the + * choice, so they are minted together. + */ +type StepStrategy = { + deriveStepAddress: DeriveStepAddress; + deriveStepRepoId: DeriveStepRepoId; +}; + +/** + * Decide the per-step address/repo strategy from the projection's step + * count. + * + * `stepOrder.length === 1` is the single-agent deploy: the sole + * step keeps the deploy's own mail address, so its grants live in the + * agent-state repo keyed by `parseAgentId(address)`. The spawned child + * reads the agent's grants from where the run's identity already + * lives, and the deploy frame's run address is preserved (the + * workflow-run repo stays keyed by `deriveWorkflowRunRepoId(address)`). + * + * Any other step count is a derived multi-step deploy: each step gets a + * derived `-` mail address (via the router's + * `multistepDeriveStepAddress`) and a derived agent-state repo under the + * default `-` convention. + * + * NOTE: the supervisor's `deriveStepAddress` feeds the credentials + * snapshot's per-step mail `address` and the grants-repo derivation. It + * does NOT feed the child's on-disk tool read (`stepDeployTreeDir` in + * `step-agent-tools.ts`), which re-derives the step address from the + * deployment mailbox address independently. The deploy tree must + * therefore be staged at the address `stepDeployTreeDir` computes, + * regardless of this strategy's address choice. + */ +function createStepStrategy(args: { + legacyAddress: string; + stepOrder: readonly string[]; + multistepDeriveStepAddress: DeriveStepAddress; +}): StepStrategy { + if (args.stepOrder.length === 1) { + return { + deriveStepAddress: () => args.legacyAddress, + // `parseAgentId` is deferred into the closure rather than computed + // eagerly: the supervisor only invokes `deriveStepRepoId` while + // assembling the credentialsSnapshot inside `spawn()`, so a + // malformed address surfaces at the same point the rest of the + // spawn path would fault rather than ahead of the deploy router's + // other boundary checks. + deriveStepRepoId: () => ({ + kind: "agent-state", + id: parseAgentId(args.legacyAddress), + }), + }; + } + return { + deriveStepAddress: args.multistepDeriveStepAddress, + deriveStepRepoId: ({ runId, stepId }) => ({ + kind: "agent-state", + id: `${runId}-${stepId}`, + }), + }; +} + +/** + * Write a grant set into the sidecar's substrate as the canonical + * `{ grants: WireGrantRule[] }` envelope -- the shape + * `assembleCredentialsSnapshot` validates (`{ grants: unknown[] }`) and + * the child's `evaluateGrants` adapter narrows to `GrantRule[]`. + * + * Two destinations share this write machinery, selected by `runId`: + * + * - `runId` absent (deploy time): write every step's grants into its + * own `agent-state` repo at `STEP_GRANTS_PATH`, keyed by the same + * `deriveStepRepoId` the supervisor reads with, so read and write + * address the same repo. `stepOrder` here is the deployment's whole + * flat step-id namespace, not the bare top-level order, because the + * supervisor assembles a snapshot entry for every loop-body step too. + * The write is on the spawn critical path: a failure rejects the + * deploy (the caller's `finally` unwinds the partial state) rather + * than spawning a child that would fail every authorize closed + * against an empty grant set. + * + * - `runId` present (per-run delivery): write a single + * `runs//grants.json` into the deployment's `workflow-run` + * repo, sibling to that run's `runs//events/` subtree. The + * `stepOrder` / `deriveStepRepoId` fields are unused in this mode -- + * the destination is the one workflow-run repo keyed by + * `runId`, not a per-step fan-out. + * + * Both destinations use the same hub principal and `refs/heads/main` + * ref: the agent-state kind handler gates `writeTree` as hub-only, and + * the workflow-run repo pins its run subtree under the same moving ref. + */ +async function writeStepGrants(args: { + repoStore: RepoStore; + anchorRunId: string; + stepOrder: readonly string[]; + deriveStepRepoId: DeriveStepRepoId; + grants: readonly unknown[] | undefined; + runId?: string; +}): Promise { + // The deploy frame's validated HarnessConfig always carries a `grants` + // array (possibly empty); an absent array means "no grants", which + // serializes to the same fail-closed empty file the snapshot expects. + // Coerce here so the on-disk envelope is always a valid `{ grants: [] }` + // rather than `{}` (which the snapshot's validator rejects). + const grants = args.grants ?? []; + const serialized = JSON.stringify({ grants }, null, 2); + if (args.runId !== undefined) { + await args.repoStore.writeTree( + GRANTS_WRITE_PRINCIPAL, + { kind: "workflow-run", id: args.anchorRunId }, + STEP_GRANTS_REF, + { + files: { [runGrantsPath(args.runId)]: serialized }, + message: `Write run grants for ${args.runId}`, + }, + ); + return; + } + for (const stepId of args.stepOrder) { + const repoId = args.deriveStepRepoId({ + runId: args.anchorRunId, + stepId, + }); + await args.repoStore.writeTree(GRANTS_WRITE_PRINCIPAL, repoId, STEP_GRANTS_REF, { + files: { [STEP_GRANTS_PATH]: serialized }, + message: `Write step grants for ${stepId}`, + }); + } +} + +export type AssembleRunCredentialsSnapshotOpts = { + /** Substrate handle the sink reads the per-run grants file from. */ + repoStore: RepoStore; + /** Anchor run id keying the workflow-run repo the grants file lives in. */ + anchorRunId: string; + /** Run whose per-run grants file is read. */ + runId: string; + /** + * Every step id the snapshot must cover -- the deployment's flat step-id + * namespace, loop-body step ids included. The per-run grants apply + * uniformly across them. + */ + stepOrder: readonly string[]; + /** Per-step mail-address derivation. */ + deriveStepAddress: DeriveStepAddress; +}; + +/** + * Resolve a run's credentials snapshot for the `onRunStart` grants + * barrier from its per-run grants file. + * + * Every legitimate run birth path writes `runs//grants.json` in + * the deployment's workflow-run repo before the run dispatches -- the + * external trigger route and the mail-triggered path both ship a + * `run.grants` frame the sidecar writes, and a spawned child inherits its + * parent's grants directly at spawn without reaching this barrier. The + * per-run file IS the run's snapshot: the run's single flat grant set is + * applied uniformly across every step, keyed on each step's address. + * + * A missing file is therefore not an internal run inheriting deploy-time + * grants -- it is a run that reached its barrier with no grants written, + * so it FAILS CLOSED here rather than running under-authorized. A file + * that exists but is malformed also throws (via `readRunGrants`), for the + * same reason: the file's presence implies a grants frame was delivered, + * so a structural failure is a boundary bug, not a default. + */ +export async function assembleRunCredentialsSnapshot( + opts: AssembleRunCredentialsSnapshotOpts, +): Promise { + const runGrants = await readRunGrants({ + repoStore: opts.repoStore, + anchorRunId: opts.anchorRunId, + runId: opts.runId, + }); + if (runGrants === undefined) { + throw new Error( + `sidecar onRunStart: run ${opts.runId} has no grants file at ${runGrantsPath(opts.runId)}; refusing to start the run under-authorized`, + ); + } + const contentHash = await hashGrants(runGrants); + const steps: CredentialsSnapshotStep[] = opts.stepOrder.map((stepId) => ({ + stepId, + address: opts.deriveStepAddress({ + runId: opts.anchorRunId, + stepId, + }), + grants: runGrants, + contentHash, + })); + return { steps }; +} + +// The supervisor's `binaryPath` binding resolves to the sidecar's +// own `bin/workflow-child` script via `import.meta.resolve` against +// the `@intx/sidecar-app` package. The script lives next to this +// wiring module (`../bin/workflow-child`); resolving it statically +// at wiring-module load time keeps the production spawn surface +// independent of any runtime env override. Tests inject a sentinel +// path via the `binaryPath` opts override; production wiring +// closes over this constant. +const SIDECAR_WORKFLOW_CHILD_BINARY: string = (() => { + const url = import.meta.resolve("../bin/workflow-child"); + return fileURLToPath(url); +})(); + +/** + * Child fd the supervisor inherits the event-channel pipe on. The + * supervisor's spawn-time convention is: + * + * fd 0 stdin -- downstream control channel (supervisor -> child) + * fd 1 stdout -- upstream control channel (child -> supervisor) + * fd 2 stderr -- inherited so child diagnostics land on the + * sidecar's stderr + * fd 3 -- event-channel write side (child writes + * HMAC-authenticated InferenceEvent frames here; + * the supervisor reads the parent end as a + * `FrameReader`) + * + * The child opens fd 3 via `EVENT_CHANNEL_FD` in + * `@intx/workflow-host`'s `from-process-env`. The two ends of the + * pipe are provisioned by `Bun.spawn`'s `stdio` slot: setting + * `stdio[3] = "pipe"` makes Bun mint a pipe pair where the child + * inherits the write half at fd 3 and the parent receives the read + * half as a numeric fd at `proc.stdio[3]` in its own address space. + */ +const CHILD_EVENT_CHANNEL_FD = 3; + +/** + * Wrap a Bun `FileSink` as the supervisor's `NdjsonWriter`. The + * supervisor's control-channel sender writes one JSON line per + * frame (already including the trailing newline); the writer is + * responsible for passing the bytes through to the child's stdin + * without buffering across frames so each frame surfaces on the + * far side as soon as `write()` resolves. + */ +function ndjsonWriterFromFileSink(sink: Bun.FileSink): NdjsonWriter { + return { + async write(line: string): Promise { + const result = sink.write(line); + if (typeof result !== "number") await result; + const flushed = sink.flush(); + if (typeof flushed !== "number") await flushed; + }, + }; +} + +/** + * Wrap a Bun stdout `ReadableStream` as the supervisor's + * `NdjsonReader`. The pipe is a byte stream; this reader buffers + * partial chunks and yields one complete line per iteration. The + * receiver's iterator finalises only on EOF, which mirrors the + * `defaultControlReader` shape the child wires for `process.stdin`. + */ +function ndjsonReaderFromReadableStream(stream: ReadableStream): NdjsonReader { + return { + read(): AsyncIterableIterator { + return (async function* () { + const decoder = new TextDecoder("utf-8"); + let pending = ""; + const reader = stream.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (value !== undefined) { + pending += decoder.decode(value, { stream: true }); + let nl = pending.indexOf("\n"); + while (nl >= 0) { + const line = pending.slice(0, nl).replace(/\r$/, ""); + pending = pending.slice(nl + 1); + if (line.length > 0) yield line; + nl = pending.indexOf("\n"); + } + } + if (done) break; + } + if (pending.length > 0) yield pending; + } finally { + reader.releaseLock(); + } + })(); + }, + }; +} + +/** + * Wrap the parent-side read fd of the event-channel pipe as the + * supervisor's `FrameReader`. The child publishes one HMAC- + * authenticated envelope per `FileSink.write()` and the supervisor's + * `receiveEventChannel` parses each yielded `Uint8Array` as one + * complete envelope. The pipe is a byte stream; this reader yields + * each raw chunk the kernel delivers and trusts the sender's + * one-write-per-envelope discipline. The buffer-overflow / framing + * discipline lives in `receiveEventChannel`'s parser. + */ +function frameReaderFromFd(fd: number): FrameReader { + const stream = Bun.file(fd).stream(); + return { + read(): AsyncIterableIterator { + return (async function* () { + const reader = stream.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (value !== undefined && value.byteLength > 0) yield value; + if (done) break; + } + } finally { + reader.releaseLock(); + } + })(); + }, + }; +} + +/** + * Real `Bun.spawn`-backed subprocess spawner. Constructs a fresh env + * carrying exactly the trust anchors and substrate-config keys the + * supervisor passed in (no inheritance of the sidecar's process env); + * inherits stdio 0/1/2 as control + stderr; pipes fd 3 for the event + * channel and surfaces the parent-side read fd as the supervisor's + * `FrameReader`. + * + * Failure modes flow through the returned handle's `exited` promise. + * A `Bun.spawn` that fails to launch (binary missing, env malformed, + * `EXEC` error) settles `exited` with a non-zero code; the + * supervisor's `wireChild` races `exited` against `readyPromise` + * inside `spawn()` so a spawn-time crash surfaces as a rejected + * spawn rather than a wedged `starting` state. + */ +export const defaultSubprocessSpawner: SubprocessSpawner = ({ + binaryPath, + env, +}): SubprocessHandle => { + const proc = Bun.spawn([binaryPath], { + stdio: ["pipe", "pipe", "inherit", "pipe"], + env, + }); + const eventFd = proc.stdio[CHILD_EVENT_CHANNEL_FD]; + if (typeof eventFd !== "number") { + throw new Error( + `workflow-host-wiring: Bun.spawn did not return a numeric fd at stdio[${String(CHILD_EVENT_CHANNEL_FD)}] for the event channel; got ${typeof eventFd}`, + ); + } + return { + pid: proc.pid, + controlWriter: ndjsonWriterFromFileSink(proc.stdin), + controlReader: ndjsonReaderFromReadableStream(proc.stdout), + eventReader: frameReaderFromFd(eventFd), + kill(signal?: number | string): void { + // The supervisor's `SubprocessHandle.kill` widens the signal + // to `number | string`; Bun's `Subprocess.kill` accepts + // `number | NodeJS.Signals`. The supervisor's call sites pass + // `"SIGTERM"` / `"SIGKILL"` (recycle path) or no argument + // (shutdown path), which Bun handles directly. Cast at the + // boundary so the inner call matches Bun's narrower type + // without coercing valid input. + if (signal === undefined) { + proc.kill(); + return; + } + if (typeof signal === "number") { + proc.kill(signal); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- supervisor's kill widens to `string`; Bun's runtime accepts the same `"SIG*"` strings, narrowed back at the boundary. + proc.kill(signal as NodeJS.Signals); + }, + exited: proc.exited, + }; +}; + +export type CreateSidecarWorkflowSupervisorOpts = { + /** Sidecar's hub mail transport. */ + transport: HubTransport; + /** Substrate-shaped RepoStore the workflow-host's supervisor reads from. */ + repoStore: RepoStore; + /** Sidecar's 32-byte Ed25519 private key seed for principal signing. */ + signingKeySeed: Uint8Array; + /** Workflow-run repo identity for the deployment. */ + workflowRunRepoId: RepoId; + /** Workflow-run repo ref the supervisor commits events to. */ + workflowRunRef: string; + /** Deployment id baked into principal claims and address derivation. */ + runId: string; + /** + * Decrypted credential material for the deployment's tools (from the deploy + * frame). Delivered to the child on the pre-trigger barrier. Absent when the + * deployment binds no credentials. + */ + credentialDelivery?: CredentialDelivery; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). + * Threaded into the child's spawn-time env so its deploy-tree read + * collapses onto the head for a single-step deployment. + */ + stepCount: number; + /** + * Every step id in the deployment's flat step-id namespace: its + * `stepOrder` plus the step ids of every `loop` body it carries. The + * `onRunStart` grants sink walks these to assemble the per-run + * credentialsSnapshot, so the sink needs the ids rather than the bare + * count -- and it needs the loop-body ids too, because a loop iteration + * inherits the parent run's env and authorizes against this same snapshot. + */ + stepOrder: readonly string[]; + /** Deployment's mail address. */ + deploymentMailAddress: string; + /** Per-step mail-address derivation. */ + deriveStepAddress: DeriveStepAddress; + /** + * Optional override of the per-step `agent-state` repo identity the + * supervisor reads grants from while assembling the + * credentialsSnapshot. Defaults to the `-` + * convention; the single-step launched-agent deploy supplies a + * derivation that returns the legacy agent-state repo so the spawned + * child reads grants from the same repo the legacy agent identity + * keys. + */ + deriveStepRepoId?: DeriveStepRepoId; + /** Substrate-config keys propagated to the child via spawn-time env. */ + substrateEnv: Record; + /** + * Dynamic spawn-env fragment the supervisor recomputes on every spawn and + * recycle respawn (e.g. a live-rotated inference-source list). Its keys + * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. + */ + dynamicSpawnEnv: () => Record; + /** + * Override the subprocess spawner. Tests inject a deterministic + * mock; production defaults to the `Bun.spawn`-backed + * `defaultSubprocessSpawner`. + */ + subprocessSpawner?: SubprocessSpawner; + /** Override the `bin/workflow-child` path. */ + binaryPath?: string; + /** + * Optional per-message dispatch-timing observer, forwarded verbatim to + * the supervisor's `onDispatchTiming` binding. Absent in production; + * the deploy router wires it (off a benchmark env gate) only for the + * Phase 4.7 latency gate, which needs the supervisor to emit the + * per-message infra round-trip from inside the sidecar subprocess. + */ + onDispatchTiming?: (mark: DispatchTimingMark) => void; + /** + * D2 §10c forced-repack A/B toggle, forwarded verbatim to the + * supervisor's `repackEveryMessages` binding. Absent in production; + * the deploy router wires it (off the same benchmark env gate) only + * for the D2 attribution run. + */ + repackEveryMessages?: { everyMessages: number }; + /** + * Consumed-dedup retention horizon (ms), forwarded to the + * supervisor's `consumedRetentionMs` binding. The boot edge resolves + * the operator's `CONSUMED_RETENTION_MS` config; absent, the + * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). + */ + consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms), forwarded to the supervisor's + * `readyTimeoutMs` binding. The boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). + */ + readyTimeoutMs?: number; + /** + * Control-plane suspension sink forwarded to the supervisor's + * `onSuspensionRegister` binding. The supervisor stamps `runId` + + * `agentAddress` and invokes this when a workflow-process child reports a + * `park.notify`; production wiring routes it to the hub link so a + * `signal.correlation.register` frame reaches the hub. Absent means the + * deployment registers no suspensions. + */ + onSuspensionRegister?: (registration: SuspensionRegistration) => void; + /** + * Self-termination sink forwarded to the supervisor's `onSelfTerminate` + * binding. The supervisor invokes it when it drives itself to a terminal + * phase (crash-loop latch, channel crash, recycle failure); production wiring + * routes it to the deploy router's address reclaim. Absent means a + * self-termination is not surfaced to the host. + */ + onSelfTerminate?: (info: { phase: "stopped" | "crash-looping"; reason: string }) => void; + /** + * Predicate consulted at the `onRunStart` grants barrier: returns `true` + * for a runId whose `run.grants` write was attempted and FAILED, so the + * per-run grants file the run needs never landed. The barrier fails such a + * run loudly rather than starting it under-authorized. This is a + * fast-fail with a precise cause: an absent grants file already fails the + * barrier closed on its own (every run birth path writes the file before + * dispatch, so its absence is a defect), and this predicate lets a run + * whose write is KNOWN to have failed reject with that specific reason + * instead of a generic missing-file error. Absent means no run is + * poisoned. + */ + isRunPoisoned?: (runId: string) => boolean; +}; + +export type SidecarWorkflowSupervisor = { + supervisor: WorkflowSupervisor; + /** + * Hand a delivered inbound message off to the supervisor's mail + * subscription. The returned promise resolves once the message is durably + * accepted and rejects when it was not, so the hub-link can send a + * `mail.inbound.ack` only on resolution (resolve = ack, reject = withhold). + */ + routeInbound(message: Uint8Array): Promise; + /** Snapshot accessor that proxies the supervisor's credentials view. */ + getCredentialsSnapshot(): CredentialsSnapshot | null; + /** + * The per-run grants barrier the supervisor awaits before firing a run's + * trigger. Rejects for a poisoned run (its `run.grants` write failed) and + * otherwise resolves the run's credentials snapshot. Exposed so the barrier + * can be exercised without driving a full spawn. + */ + onRunStart(args: { runId: string; anchorRunId: string }): Promise; +}; + +/** + * Env key the multi-step branch uses to carry each step's ordered + * inference-source failover chain from `frame.workflow.sources` down to + * the workflow-process child. The substrate factory's `buildEnv` reads + * this and resolves a step's chain at step invocation, feeding it to the + * reactor for forward-only failover; the supervisor itself is opaque to + * the value (it is plumbed through `bindings.substrateEnv` verbatim). + * + * Listed here so the router and the future substrate-factory consumer + * spell the key the same way without a magic-string trip hazard. + */ +export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; + +/** + * Spawn-env key carrying every spawned body's per-step inference-source pins as + * a JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` map. The sidecar + * decrypts the sealed body sources from the run record and serializes the + * plaintext here so the run child resolves a body's sources without holding the + * sidecar's cipher key. Mirrors `STEP_INFERENCE_SOURCES` for the top level. + */ +export const WORKFLOW_BODY_SOURCES_ENV_KEY = "WORKFLOW_BODY_SOURCES"; + +/** + * A single env value cannot exceed the OS argument-string ceiling + * (`MAX_ARG_STRLEN`, 128 KiB on Linux); an over-long value makes the child + * `execve` fail opaquely at spawn. `WORKFLOW_BODY_SOURCES` sums over every body + * at every depth, so the deploy path validates the serialized size against this + * bound and rejects an over-large deployment loudly at deploy time rather than + * letting a much-later spawn fail. Held below the ceiling with headroom for the + * rest of the env block. + */ +const WORKFLOW_BODY_SOURCES_MAX_BYTES = 96 * 1024; + +/** + * Build the record's `bodySources` map from the deploy frame's referenced body + * definitions (the flat set of onTrigger sections and childWorkflow children, + * each already source-pinned by the hub). `undefined` when the deployment + * references no bodies, so the record omits the field. + */ +function buildBodySourcesMap( + referenced: NonNullable["referencedDefinitions"], +): WorkflowRunRecord["bodySources"] { + if (referenced === undefined || referenced.length === 0) { + return undefined; + } + const map: NonNullable = {}; + for (const ref of referenced) { + map[ref.definition.id] = ref.sources; + } + return map; +} + +/** + * Validate the wire-projected workflow definition at the deploy-router + * boundary. The arktype `AgentDeployFrame` validator enforces the + * wire shape (`id` is non-empty, `stepOrder` is `string[]`, `steps` + * is an object, `sources` covers every `stepOrder` entry); this + * function takes `unknown`-typed inputs so it can also gate callers + * that bypass the wire boundary, and it enforces the invariants the + * router and the downstream supervisor rely on: + * + * - `definition.id` is a non-empty string. The arktype shape + * already enforces this on the wire; the re-check here protects + * bypass callers and keeps the failure shape consistent with the + * other invariants this function owns. + * - `definition.stepOrder` is non-empty. The wire shape admits + * `[]`; a zero-step workflow has no semantics here. + * - Every `stepOrder` entry matches `STEP_ID_PATTERN` so per-step + * mail-address derivation never needs escaping at the substrate + * boundary. + * - Every `stepOrder` entry has a corresponding `steps[id]` entry. + * The wire shape lets `steps[id]` be `unknown` and lets the + * entry be absent; presence is required so the workflow-process + * child can resolve each step's primitive at run time. + * - Every `stepOrder` entry has a corresponding `sources[id]` + * entry, and that entry is a non-empty array (the step's ordered + * failover chain). The arktype narrow already enforces both; the + * re-check here surfaces a structured router-side error instead of + * an arktype validation failure at the wire boundary, which keeps + * the failure shape consistent with the rest of the validations + * this function owns. An empty chain would leave the reactor with + * no initial source, so it is rejected here rather than deferred to + * a deep-stack child failure. + * + * A rejection here surfaces as a thrown `Error` the link's deploy + * frame caller converts into a structured failure reply. + */ +export function validateWorkflowProjection(projection: { + definition: { id: unknown; stepOrder: unknown; steps: unknown }; + sources: unknown; +}): void { + const def = projection.definition; + if (typeof def.id !== "string" || def.id.length === 0) { + throw new Error("sidecar deploy router: workflow.definition.id must be a non-empty string"); + } + if (!Array.isArray(def.stepOrder) || def.stepOrder.length === 0) { + throw new Error( + "sidecar deploy router: workflow.definition.stepOrder must be a non-empty array", + ); + } + if (typeof def.steps !== "object" || def.steps === null) { + throw new Error("sidecar deploy router: workflow.definition.steps must be an object"); + } + if (typeof projection.sources !== "object" || projection.sources === null) { + throw new Error("sidecar deploy router: workflow.sources must be an object"); + } + const steps = def.steps; + const sources = projection.sources; + for (const stepId of def.stepOrder) { + if (typeof stepId !== "string" || stepId.length === 0) { + throw new Error( + "sidecar deploy router: workflow.definition.stepOrder entries must be non-empty strings", + ); + } + if (!STEP_ID_PATTERN.test(stepId)) { + throw new Error( + `sidecar deploy router: stepId ${JSON.stringify(stepId)} must match ${STEP_ID_PATTERN.source}`, + ); + } + if (!Object.prototype.hasOwnProperty.call(steps, stepId)) { + throw new Error( + `sidecar deploy router: workflow.definition.steps is missing entry for stepId ${JSON.stringify(stepId)}`, + ); + } + if (!Object.prototype.hasOwnProperty.call(sources, stepId)) { + throw new Error( + `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, + ); + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- sources is checked to be a non-null object above; this reads a value to re-check its array shape + const stepSources = (sources as Record)[stepId]; + if (!Array.isArray(stepSources) || stepSources.length === 0) { + throw new Error( + `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, + ); + } + } +} + +/** + * Derive the supervisor's principal public key from the sidecar's + * Ed25519 signing seed. The supervisor signs every workflow-run event + * with this key; the multi-step branch surfaces it to the link so the + * hub records the verifying key for the deployment's signed events. + */ +async function derivePrincipalPublicKeyHex(signingKeySeed: Uint8Array): Promise { + return hexEncode(await derivePublicKeyBytes(signingKeySeed)); +} + +/** + * The sidecar's `DeployRouter` plus the boot-time restore driver. The link + * routes `agent.deploy`/`agent.undeploy` through the `DeployRouter` surface; + * the sidecar boot edge additionally calls `restoreWorkflowRuns` once, + * before connecting to the hub, to re-establish the deployments a prior + * process persisted. The extra method is sidecar-app-only, so it rides on the + * concrete router type rather than the shared `DeployRouter` contract. + */ +export interface SidecarDeployRouter extends DeployRouter { + /** + * Re-establish every persisted workflow deployment on this sidecar's local + * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step + * head's mailbox/transport registration is live before the hub routes to it. + * Soft-fails per deployment: a record that cannot be restored (unbuildable + * provider, a source closure that cannot be re-materialized, spawn failure) + * is logged and left on disk for a later boot to retry -- it is never + * deleted here. + */ + restoreWorkflowRuns(): Promise; + /** + * The workflow-substrate deployment addresses (`run_@domain`) this router + * currently hosts a live supervisor for -- the set of addresses this + * sidecar can route mail to. The boot edge announces these to the hub on + * (re)connect so the hub re-registers them for routing: they are hub-minted + * and carry no per-address key; the allocation-authenticated announcement is + * what re-establishes their routes after a WS reconnect. Reflects + * `deploy`/`undeploy` + * and boot-time restore live, so a caller re-reads it per connect. + */ + activeAddresses(): string[]; + /** + * Trigger B: re-register every correlation the deployment at `address` is + * currently parked on, by asking its live supervisor to re-emit them to the + * hub. The boot edge calls this per address the hub link re-routes on an + * authenticated reconnect (`onWorkflowAddressesRoutable`), so a register frame + * the hub dropped from its bounded send queue during the outage is recovered. + * The address-dispatch wrapper around the supervisor's own no-arg + * `reEmitParkedCorrelations`; fire-and-forget (the driver is best-effort and + * watchdog-bounded inside the supervisor). + */ + reEmitParkedCorrelations(address: string): void; +} + +export function createSidecarDeployRouter(deps: { + sessions: SessionManager; + keyStore: AgentKeyStore; + /** + * Cache of hub-vouched sender public keys. The grants handler writes each + * co-delivered `senderIdentities` entry here before the run's grants land, + * so a durable grant is never missing the key its recipient needs to verify + * the sender's inbound mail. + */ + senderKeyCache: SenderKeyCache; + transport: HubTransport; + repoStore: RepoStore; + signingKeySeed: Uint8Array; + /** + * The sidecar cipher that seals credential material at rest. Used to seal + * each run record's inference-source apiKeys on persist and unseal them on + * the boot scan, so the durable record carries ciphertext rather than + * plaintext secrets. + */ + credentialCipher: CredentialCipher; + /** + * Per-agent crypto factory. Receives the agent's raw key pair and + * returns a `CryptoProvider` bound to it (production wires + * `@intx/crypto`'s `createEd25519Crypto`). The multi-step branch + * uses this to register the spawned single-step agent's signing key on + * the host transport before `spawn()`, so the supervisor's outbound + * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies + * with the AGENT's identity -- the OUTBOUND half of mailbox ownership + * (§3a). Without this registration the spawned agent's address has no + * `CryptoProvider` on the transport (nothing else registers one for + * it), and an outbound send would throw "address is not registered" + * rather than emit unsigned mail. + */ + createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; + /** + * Source-admission gate: throws if a step's pinned inference source + * names a provider this sidecar cannot build. The buildable-provider + * set is sidecar config (the boot edge's adapter registry), so this + * admission control lives at the sidecar -- the hub is a different + * process and cannot know a given sidecar's providers. Production wires + * the default harness builder's `canBuildSource` verbatim, so a rejected + * provider carries the same `"... is not registered"` message. + * + * Distinct from the orchestrator's operator-approval check + * (`pickStepInferenceSource`): that gates on whether the operator + * approved a `provider:model`; this gates on whether the provider is + * buildable at all. A source can be approved yet unbuildable. + */ + assertSourceBuildable: (source: InferenceSource) => void; + /** + * Record a `(runId -> agentAddress)` mapping the boot edge's + * workflow-run pack push facade consults when it must address an + * outbound pack frame. Fires once per inbound `agent.deploy` frame + * before the deployment's supervisor spawns, so the first pack push + * the child triggers sees the mapping. Tests that do not exercise + * the pack push path may pass a no-op. + */ + registerDeployment: (entry: { runId: string; agentAddress: string }) => void; + /** + * Symmetric removal hook for `registerDeployment`. Fires from the + * link's `agent.undeploy` path so the boot edge's + * `DeploymentAddressRegistry` drops the mapping when the deployment + * is torn down. A subsequent stale `writeTreePreservingPrefix` + * against the dead deployment's workflow-run ref surfaces + * structurally (`registry.resolve` returns `null`) rather than + * silently resolving to the prior address. Tests that do not + * exercise the pack push path may pass a no-op. + */ + unregisterDeployment: (entry: { runId: string; agentAddress: string }) => void; + /** + * Substrate-config env keys the multi-step branch propagates into + * the workflow-process child's spawn-time env (see + * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `workflow-substrate-factory.ts`). + * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step + * frame. Defaults to an empty record so a router built without + * substrate config (e.g. a test) needs no boot-edge threading. + */ + multistepSubstrateEnv?: Record; + /** + * Subprocess spawner the multi-step branch hands to the supervisor. + * Defaults to the production `Bun.spawn`-backed + * `defaultSubprocessSpawner`; tests inject a deterministic mock. + */ + multistepSubprocessSpawner?: SubprocessSpawner; + /** + * Optional override for the resolved `bin/workflow-child` path the + * multi-step branch hands to the supervisor. Production wiring uses + * the package-local default; tests inject a sentinel value so the + * mock spawner can assert on it. + */ + multistepBinaryPath?: string; + /** + * Callback the supervisor invokes for every verified InferenceEvent + * the workflow-process child publishes. The router threads the + * deployment's run address plus the deploy's session id through to + * the callback so a downstream fan-out can route each event to the + * hub timeline keyed to the right session. The `InferenceEvent` itself + * is sessionless; the session id rides alongside it, sourced from the + * deploy frame's `HarnessConfig.sessionId` per deployment. It is + * optional because a deploy frame need not carry a session id (a + * headless deployment with no hub-side session); the sink decides what + * an absent session id means. Defaults to a no-op; production wiring + * supplies the event publisher. + */ + publishWorkflowInferenceEvent?: ( + agentAddress: string, + event: InferenceEvent, + sessionId: string | undefined, + ) => void; + /** + * Callback the supervisor invokes for every control-plane suspension a + * workflow-process child reports (`park.notify`). The multi-step branch + * threads it into the supervisor's `onSuspensionRegister` binding so a + * parked run's correlation is registered at the hub (routing + approval + * rows). The supervisor stamps `runId` + `agentAddress` before + * invoking it. Defaults to a no-op; production wiring supplies the + * hub-link-backed publisher. + */ + publishWorkflowSuspension?: (registration: { + correlationId: string; + runId: string; + anchorRunId: string; + agentAddress: string; + kind: SignalKind; + approvalSnapshot?: ApprovalSnapshot; + }) => void; + /** + * Optional override for the multi-step branch's per-step mail-address + * derivation. Defaults to `${runId}-${stepId}@` + * derived from the frame's run address. Tests inject a deterministic + * factory. + */ + multistepDeriveStepAddress?: DeriveStepAddress; + /** + * Per-deployment-address mail handler registry the hub-link's + * `mail.inbound` path consults before falling back to the legacy + * session-routed delivery. The multi-step branch registers + * `wired.routeInbound` against the deployment's mail address once + * `supervisor.spawn` succeeds so inbound mail aimed at the + * deployment address flows into the supervisor's mail-bus + * subscription. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end mail loop can omit the binding; an absent registry + * simply means multi-step inbound mail cannot route through the + * hub-link until the wiring is plumbed. + */ + multistepMailRouter?: MultistepMailRouter; + /** + * Per-recipient-address registry of resolved inbound-mail admission + * policies the sidecar hub-link's `mail.inbound` seam enforces. The + * multi-step branch resolves this deployment's authored + * `inboundMailPolicy` into the registry once `supervisor.spawn` + * succeeds -- beside the mail-router registration -- and removes the + * entry in the same teardown, so a reused address never inherits a + * stale policy. + * + * Optional so tests that exercise the multi-step branch without the + * hub-link seam can omit it; an absent registry means an inbound frame + * for the address resolves to the fully-closed default and is rejected + * until the wiring is plumbed. + */ + inboundMailPolicyRegistry?: InboundMailPolicyRegistry; + /** + * Per-deployment-address signal handler registry the sidecar + * hub-link's `signal.deliver` path consults. The multi-step branch + * registers `wired.supervisor.deliverSignal` against the deployment's + * mail address once `supervisor.spawn` succeeds so a hub-side + * `signal.deliver` frame flows into the workflow-process child via + * the IPC's `signal.deliver` payload. The child commits the + * resulting `SignalReceived` event through its own substrate, + * preserving the workflow-run repo's single-writer invariant on the + * sidecar side. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end signal loop can omit the binding; an absent registry + * means hub-side signals cannot route through the hub-link until the + * wiring is plumbed. + */ + multistepSignalRouter?: MultistepSignalRouter; + /** + * Per-deployment-address drain handler registry the sidecar + * hub-link's `drain.deliver` path consults. The multi-step branch + * registers `wired.supervisor.drain` against the deployment's mail + * address once `supervisor.spawn` succeeds so a hub-side + * `drain.deliver` frame flows into the workflow-process child via + * the IPC's `drain` payload and arms the supervisor's per-run + * `drainTimeout` accumulators. Cancel-mode in-flight steps abort on + * the child side; wait-mode steps continue. Accumulators commit a + * signed `CancelRequested{origin: "supervisor-drain"}` against the + * workflow-run repo when the deadline expires. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end drain loop can omit the binding; an absent registry + * means hub-side drain frames cannot route through the hub-link until + * the wiring is plumbed. + */ + multistepDrainRouter?: MultistepDrainRouter; + /** + * Per-deployment-address grants handler registry the sidecar + * hub-link's `run.grants` path consults. The deploy router registers a + * handler against the deployment's mail address once `supervisor.spawn` + * succeeds, for single-step and multi-step deployments alike, so a + * hub-side `run.grants` frame writes the run's grants to + * `runs//grants.json` inside the deployment's `workflow-run` + * repo. The write is awaited in `tryRoute` so the frame's FIFO + * completion means the grants are durable on disk. + * + * Optional so tests that exercise the multi-step branch without an + * end-to-end grants loop can omit the binding; an absent registry + * means hub-side grants frames cannot route through the hub-link until + * the wiring is plumbed. + */ + multistepGrantsRouter?: MultistepGrantsRouter; + /** + * Per-deployment-address sources-rotation handler registry. Only a + * single-step warm deployment registers a handler (against the + * deployment's mail address once `supervisor.spawn` succeeds) so a + * rotation resolved for its address flows into + * `wired.supervisor.deliverSources` and on to the child's warm agent. A + * multi-step deployment registers none -- it has no single warm agent to + * rotate -- so `tryRoute` reports its address as unrouted. + * + * Optional so tests that exercise deploys without a rotation loop can + * omit the binding; an absent registry means no rotation handler is + * installed for any deployment. + */ + multistepSourcesRouter?: MultistepSourcesRouter; + /** + * Optional per-deployment credential-delivery handler registry. Every + * deployment with a supervisor registers a handler after `spawn` (not only + * warm single-step ones -- the material cell is per-child and read by every + * step's tool capabilities), and an inbound `credentials.update` for an + * unregistered (torn-down) address is unrouted. Optional so tests without a + * credential-delivery loop can omit the binding. + */ + multistepCredentialsRouter?: MultistepCredentialsRouter; + /** + * Optional per-message dispatch-timing observer the multi-step branch + * forwards to each supervisor it constructs. Resolved at the sidecar + * boot edge from the Phase 4.7 latency-gate env gate; absent in + * ordinary production. The supervisor runs in this sidecar subprocess, + * so the observer sees both ends of the per-message IPC round-trip in + * one process and can emit a parseable timing line the benchmark + * harness reads off the subprocess's output stream. + */ + onDispatchTiming?: (mark: DispatchTimingMark) => void; + /** + * D2 §10c forced-repack A/B toggle the multi-step branch forwards to + * each supervisor it constructs. Resolved at the sidecar boot edge from + * the same benchmark env gate; absent in ordinary production. + */ + repackEveryMessages?: { everyMessages: number }; + /** + * Consumed-dedup retention horizon (ms) forwarded to every supervisor + * the router constructs. The sidecar boot edge resolves the operator's + * `CONSUMED_RETENTION_MS` config; absent, the supervisor applies + * `DEFAULT_CONSUMED_RETENTION_MS` (24h). See the workflow-run kind + * handler for the operator-owned horizon invariant. + */ + consumedRetentionMs?: number; + /** + * Spawn ready-handshake timeout (ms) forwarded to every supervisor the + * router constructs. The sidecar boot edge resolves the operator's + * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies + * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never + * signals ready is killed and its spawn rejected rather than hanging + * the deploy or boot-time restore. + */ + readyTimeoutMs?: number; + /** + * Run-record writer, injectable so a test can block or fail the + * persist at a controlled point -- the natural seam for exercising a + * recycle that interleaves the source-rotation persist window. Defaults + * to the real `writeWorkflowRunRecord`; production never overrides + * it. + */ + writeWorkflowRunRecord?: typeof writeWorkflowRunRecord; + /** + * Materialize a source-ref deployment's frozen closure. Defaults to the real + * `applyFrozenWorkflowClosure` (registry fetch + SRI verify + layout); + * production never overrides it. A test seam so a unit test can drive the + * deploy/restore source-ref path without a live registry. + */ + applyFrozenWorkflowClosure?: typeof applyFrozenWorkflowClosure; +}): SidecarDeployRouter { + // Validate the signing seed at construction so a malformed key fails + // sidecar boot rather than the first multi-step deploy, where the + // public key is derived from it (`derivePrincipalPublicKeyHex`). The + // seed also signs every workflow-run event via the supervisor. + if (deps.signingKeySeed.length !== 32) { + throw new Error( + `sidecar deploy router: Ed25519 signing seed must be 32 bytes, got ${deps.signingKeySeed.length}`, + ); + } + const publishInferenceEvent = + deps.publishWorkflowInferenceEvent ?? + ((_address: string, _event: InferenceEvent, _sessionId: string | undefined): void => { + /* no-op default: tests and production-without-a-publisher + deployments do not consume events. */ + }); + const publishSuspension = + deps.publishWorkflowSuspension ?? + ((_registration: { + correlationId: string; + runId: string; + anchorRunId: string; + agentAddress: string; + kind: SignalKind; + approvalSnapshot?: ApprovalSnapshot; + }): void => { + /* no-op default: tests and production-without-a-publisher + deployments do not register suspensions. */ + }); + const multistepSubstrateEnv = deps.multistepSubstrateEnv ?? {}; + // Sidecar data dir the deployment's per-step scratch is rooted under + // (`/workflow-step-state//...`). Resolved once + // from the boot-edge substrate env so the undeploy hook can reclaim + // the whole subtree. Absent only when the router is wired without + // substrate config (a test that never spawns a child), in which case + // no child ever rooted scratch and the undeploy reclaim is correctly + // skipped. + const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; + const persistWorkflowRunRecord = deps.writeWorkflowRunRecord ?? writeWorkflowRunRecord; + const applyClosure = deps.applyFrozenWorkflowClosure ?? applyFrozenWorkflowClosure; + const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; + const multistepDeriveStepAddress: DeriveStepAddress = + deps.multistepDeriveStepAddress ?? (({ runId, stepId }) => `${runId}-${stepId}`); + + // Per-deployment supervisor tracking. The multi-step branch + // constructs one `SidecarWorkflowSupervisor` per `agent.deploy` + // frame; the supervisor owns the workflow-process child, its IPC + // pipes, and its event-channel fd. The undeploy hook consults this + // map to call `supervisor.shutdown()` so the child's lifetime ends + // with the deployment. + const activeSupervisors = new Map(); + + // Synchronous single-flight guard for the deploy path. The real supervisor + // does not exist until inside `spawnWorkflowRun`, so `deployMultiStep` + // cannot reserve its `activeSupervisors` slot up front; instead it records + // the address here synchronously, before its first await, and clears it in a + // finally once the deploy settles. `activeSupervisors` is populated only + // after `spawn` succeeds, so the has-check alone leaves a window in which two + // same-address frames both pass and the loser's unwind deletes the winner's + // live run record. This set closes that window: a second frame that + // arrives while the first is mid-deploy is rejected before it touches any + // durable state. Only the live deploy path reserves; the boot restore path + // is serial and relies on the `activeSupervisors` backstop instead. + const reservingDeployAddresses = new Set(); + + // Slug-collision tracking. `deriveDeploymentId` substitutes + // disallowed characters with `-`, which is deterministic but lossy: + // two distinct run addresses can collapse to the same slug, and + // a collision would let the second deploy silently overwrite the + // first deploy's workflow-run repo state (the slug IS the repoId). + // This map records the first-claimer; a subsequent deploy that + // produces the same slug from a different address is rejected at + // the router before any supervisor or repo state is touched. + const slugClaims = new Map(); + + function claimSlug(runId: string, agentAddress: string): void { + const existing = slugClaims.get(runId); + if (existing !== undefined && existing !== agentAddress) { + throw new Error( + `deriveDeploymentId collision: run addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to runId ${JSON.stringify(runId)}`, + ); + } + // A same-address re-claim is a defensive no-op: the `activeSupervisors` + // guard rejects a live re-deploy before claimSlug is re-invoked, and a + // failed or undeployed deploy releases the slug first, so in practice + // `existing` is only ever undefined or a different address here. + slugClaims.set(runId, agentAddress); + } + + function releaseSlug(runId: string, agentAddress: string): void { + const existing = slugClaims.get(runId); + if (existing === agentAddress) slugClaims.delete(runId); + } + + // Reclaim a deployment address whose supervisor drove ITSELF to a terminal + // phase (crash-loop latch, channel crash, recycle failure) without an + // operator undeploy. Drops the address's runtime/routing state so a redeploy + // of the same address succeeds without a manual undeploy: the `has`-guarded + // `activeSupervisors` entry is the redeploy gate, and `transport.register` + // throws on a double-register, so both must be released here. The routers and + // the slug are released too so a frame racing the reclaim is rejected at the + // boundary rather than dispatched into the dead supervisor, and the slug is + // free to re-claim. + // + // Three deliberate invariants: + // - This NEVER calls back into `wired.supervisor.*`. It runs as the + // supervisor's own `onSelfTerminate` sink, i.e. re-entrantly during the + // supervisor's terminal teardown; a call back in (e.g. `shutdown()`) + // would re-enter that teardown. The supervisor has already terminated, so + // there is nothing to shut down. + // - It does NOT delete the durable run record or on-disk state (unlike the + // `undeploy` hook). The run record mirrors the hub's deployment intent, + // and a self-termination is not a hub-initiated undeploy: the hub still + // believes the address is deployed. Leaving the record lets a boot restore + // re-spawn a fresh supervisor for the address, and a same-process redeploy + // overwrites the record destructively. This preserves the pre-existing + // boot-restore behavior exactly -- a self-terminated deployment already + // left its record on disk before this reclaim existed; the reclaim only + // drops in-memory routing state, never durable state. + // - It does NOT call `unregisterDeployment`, so the deployment address + // mapping (`deploymentAddressRegistry`) is RETAINED. That mapping is + // consulted only by the OUTBOUND workflow-run pack push + // (`registry.resolve`), which the supervisor itself drives; no inbound + // frame consults it (a frame racing the reclaim is rejected by the + // routers above, not routed through this mapping). The supervisor's own + // terminal `RunFailed` commit -- the crash-loop latch writes it AFTER its + // teardown fires this sink -- resolves this mapping, so dropping it here + // would strand that commit ("no run address registered for deployment") + // and leave the crash-loop with no durable failure tombstone. Retaining a + // stale entry never blocks a redeploy: `record` overwrites idempotently, + // and an operator `undeploy` or a process restart clears it. + // + // Fully synchronous with no `await` between the guard and the mutations, so it + // is idempotent and cannot interleave with a concurrent operator `undeploy` of + // the same address (both are "if present, drop"). + function reclaimSelfTerminatedSupervisor(args: { runId: string; agentAddress: string }): void { + if (!activeSupervisors.has(args.agentAddress)) return; + // Drop racing frames at the router boundary first, then unwind the + // underlying registrations -- the same ordering the undeploy hook uses. + deps.multistepMailRouter?.unregister(args.agentAddress); + deps.inboundMailPolicyRegistry?.unregister(args.agentAddress); + deps.multistepSignalRouter?.unregister(args.agentAddress); + deps.multistepDrainRouter?.unregister(args.agentAddress); + deps.multistepGrantsRouter?.unregister(args.agentAddress); + deps.multistepSourcesRouter?.unregister(args.agentAddress); + deps.multistepCredentialsRouter?.unregister(args.agentAddress); + activeSupervisors.delete(args.agentAddress); + deps.transport.unregister(args.agentAddress); + releaseSlug(args.runId, args.agentAddress); + } + + /** + * Remove the legacy `${dataDir}/assets/workflow//` staging a + * pre-sealed-record deploy wrote for a body (a world-readable plaintext + * `sources.json`, INTR-310). Body sources now ride sealed in the run record and + * reach the child through the spawn env, so the staging is retired; a redeploy + * of an address that predates the change reaches here and reclaims its orphaned + * file. A never-staged body is a no-op (`force`). + */ + async function sweepLegacyBodySources( + sidecarDataDir: string | undefined, + definitionId: string, + ): Promise { + if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the legacy body-source staging is reclaimed against this data dir", + ); + } + const bodyAssetDir = pathJoin(sidecarDataDir, "assets", "workflow", definitionId); + await rm(bodyAssetDir, { recursive: true, force: true }); + } + + /** + * The per-deployment inputs the shared spawn core needs to stand up a + * workflow deployment, independent of the live deploy frame. The live + * deploy path builds this from `frame`/`projection`; a boot-time restore + * path builds the same shape from the persisted run record. + */ + interface WorkflowDeploySpec { + agentAddress: string; + /** + * The runnable definition, projected to its inert wire shape. Source-ref is + * the only deploy lineage, so this is always the closure evaluation + * (`projectLiveToInert(applied.definition)`), NOT a frame-carried inline + * definition (the deploy frame carries none). Both the deploy path and the + * boot-time restore derive it the same way, from the materialized closure. + */ + definition: WorkflowProjectionDefinition; + sources: NonNullable["sources"]; + /** + * Per spawned-body inference-source pins, keyed by the body's definition id + * (the flat set of onTrigger sections and childWorkflow children). Persisted + * in the record and delivered to the run child as plaintext through the spawn + * env, so the child resolves a body's sources without holding the sidecar's + * cipher key. `undefined` when the deployment spawns no bodies. + */ + bodySources: WorkflowRunRecord["bodySources"]; + /** + * The run's unified credential-material cell (inference + tool secrets, plus + * tool bindings). Persisted sealed in the record and delivered to the child + * on the pre-trigger barrier; the child resolves both its inference sources + * and its tools from it by `credentialId`. `undefined` when the deployment + * binds no credentials and its sources need none. + */ + credentials: CredentialDelivery | undefined; + /** + * The hub-approved wire hash the deploy frame carried + * (`AgentDeployWorkflow.approvedWireHash`). The child's `DEFINITION_HASH` + * is sourced from this hub authority, NOT a sidecar recompute. Undefined + * only for a frame that carried no approved hash on the wire; the shared + * spawn core fails closed rather than substitute a recompute. + */ + approvedWireHash: string | undefined; + /** Correlates the child's inference events to the deploy's session. */ + sessionId: string | undefined; + /** + * Hub public key recorded at the head for deploy-pack verification and + * inbound hub-frame verification. Required for a single-step + * deployment (whose head IS the agent identity); undefined for a + * genuine multi-step deployment, which derives per-step addresses and + * records no head key. + */ + hubPublicKey: string | undefined; + /** + * Sidecar-local directory of the materialized workflow-definition closure, + * set after the frozen closure is applied. The spawn core threads it into + * the child's spawn env so the run child evaluates the pinned code to a live + * definition. Sidecar-local: it never travels on the hub deploy frame and is + * not persisted to the deployment record. Always present -- source-ref is + * the only deploy lineage. + */ + closurePackageDir: string; + /** + * The source-ref pin, carried so `buildWorkflowRunRecord` can persist it and + * a boot-time restore can re-run `applyFrozenWorkflowClosure` to + * re-materialize the pinned code. Its `source` carries no secret (the + * registry token is resolved from env at apply time); its `closure` is + * frozen versions + SRIs. + */ + sourceRef: NonNullable["sourceRef"]; + } + + /** + * Build the durable run record from a spec and a source table. The + * table is a parameter (not `spec.sources`) so the deploy path writes the + * deploy-time sources while the rotation handler writes the live-rotated + * ones -- both through one shape, so a rotation persists the same record a + * boot-time restore reseeds from. + */ + function buildWorkflowRunRecord( + spec: WorkflowDeploySpec, + sources: WorkflowRunRecord["sources"], + ): WorkflowRunRecord { + // The record schema requires both for a source-ref record -- the only + // lineage -- so a spec missing either is a wiring defect. Fail loudly here + // rather than persist a record the boot scan would then reject as corrupt. + if (spec.approvedWireHash === undefined) { + throw new Error( + `buildWorkflowRunRecord: a source-ref deployment (${spec.agentAddress}) must carry approvedWireHash`, + ); + } + return { + version: 2 as const, + agentAddress: spec.agentAddress, + definitionId: spec.definition.id, + sources, + ...(spec.bodySources !== undefined ? { bodySources: spec.bodySources } : {}), + ...(spec.credentials !== undefined ? { credentials: spec.credentials } : {}), + ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), + ...(spec.hubPublicKey !== undefined ? { hubPublicKey: spec.hubPublicKey } : {}), + lineage: "source-ref", + // Feeds the restored child's DEFINITION_HASH so it re-verifies the + // evaluated closure against the hub-approved pin, and the source-ref pin a + // restore re-runs applyFrozenWorkflowClosure with. + approvedWireHash: spec.approvedWireHash, + sourceRef: spec.sourceRef, + }; + } + + /** + * Materialize a source-ref deployment's frozen closure to its per-deployment + * instance dir and return the applied result. Owns the plumbing both the + * deploy path and the boot-time restore path share: the deterministic + * instance dir under `/workflow-definition-closures/`, + * the content-addressed cache root, the two substrate byte caps, and the + * registry table. The caller consumes the result: deploy validates + * `applied.definition` and takes `packageDir`; restore takes `packageDir` and + * re-carries the pin on its spec. + * + * The instance dir is force-reclaimed before the apply. `deploymentId` is + * deterministic per agent address, so a redeploy or a boot-restore reuses the + * same dir; a prior soft-failed deploy or a dead prior process can leave it + * half-materialized. The rm makes the apply write into a clean dir either + * way (a no-op on a never-deployed address). It is safe to reclaim only + * because no live reader holds the dir when this runs -- a precondition each + * caller establishes and notes at its call site. + */ + async function materializeDeploymentClosure( + dataDir: string, + deploymentId: string, + pin: SourceRefPin, + ): Promise { + const instanceDir = pathJoin(dataDir, "workflow-definition-closures", deploymentId); + await rm(instanceDir, { recursive: true, force: true }); + + // Tarball `kind:"asset"` entries read from the durable plain-file store; a + // source-format entry checks its subtree out of the durable indexed git + // store. Deriving and asserting both from the pin alone is what makes this + // symmetric on deploy and restore. + const { assetRoot, assetMounts, gitDirs } = await resolveDeploymentAssetMounts( + dataDir, + deploymentId, + pin, + ); + + return applyClosure({ + source: pin.source, + closure: pin.closure, + instanceDir, + cacheRoot: pathJoin(dataDir, "workflow-definition-closure-cache"), + cacheMaxBytes: requireSubstrateByteCap(multistepSubstrateEnv, "SIDECAR_CACHE_MAX_BYTES"), + registryMaxTarballBytes: requireSubstrateByteCap( + multistepSubstrateEnv, + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + ), + registries: readRegistries(), + assetRoot, + assetMounts, + gitDirs, + }); + } + + /** + * The single owner of the workflow-deployment spawn sequence: construct + * the supervisor, register the single-step agent's outbound key + head + * repo + hub key, spawn the workflow-process child, then register the + * live deployment (supervisor, mail/signal/drain routers, address + * mapping). Its `try/finally` unwinds every piece of partial state if any + * step throws, so a failed spawn leaks nothing. Both the live deploy path + * and the boot-time restore path route through here so the two can never + * diverge on how a deployment is stood up. Callers materialize the + * deploy-only durable state (the source closure, step grants) before calling. + */ + async function spawnWorkflowRun(spec: WorkflowDeploySpec): Promise { + // The run's credential material rides on `spec.credentials`: the sidecar now + // holds a cipher and seals it into the record, so both the live deploy and + // the boot-restore path carry it here (restore unseals it from the record), + // and it is delivered to the child on the pre-trigger barrier. + const credentialDelivery = spec.credentials; + // Fail loud if this address already has a live supervisor. Both single- + // and multi-step now register on the transport, so both carry the + // `transport.register` duplicate-throw backstop; this `has()` check is the + // primary early guard that gives a clean error before that lower-level + // throw and before the `activeSupervisors.set` below could clobber the + // running deployment's handle. Both the deploy path and the boot restore + // path route through here, so this is the single transition guard against + // a double-spawn -- notably a boot restore racing a legacy restore for the + // same address (the B-reroute follow-up relies on it). + if (activeSupervisors.has(spec.agentAddress)) { + throw new Error( + `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, + ); + } + const runId = deriveDeploymentId(spec.agentAddress); + + // Single-step launched-agent deploy vs. derived multi-step deploy. A + // one-step deployment keeps the deployment's own (legacy) mail address + // and its grants in the legacy agent-state repo keyed by the legacy + // instance id. A multi-step deployment derives `-` + // per step for both the mail address and the agent-state repo id. + const stepStrategy = createStepStrategy({ + legacyAddress: spec.agentAddress, + stepOrder: spec.definition.stepOrder, + multistepDeriveStepAddress, + }); + + // Every step id the deployment's credentials snapshot must cover. This is + // NOT `stepOrder`: a `loop` body runs in-process as a child run inheriting + // the parent's env, so a body step authorizes against the SAME snapshot the + // top-level steps do, keyed by its own plain step id. A snapshot built from + // `stepOrder` alone carries no entry for it and the child's authorize + // throws on the body's first tool call. The address/repo strategy above + // stays on `stepOrder`, because the head/step collapse is a property of the + // deployment's own step count, not of what a body can run. + const credentialStepIds = inertFlatNamespaceStepIds({ + definition: spec.definition, + context: "sidecar deploy router credentials snapshot: ", + }); + + // Unwind every piece of spawn state if any step in this block throws, + // so a failed spawn leaks no freshly-spawned workflow-process child, + // `activeSupervisors` entry, transport registration, or multistep + // router registration. (The deployment-address registration happens + // before spawn and is unwound by its own guard.) The ordering inside + // the finally is the reverse of the success-path registration order. + // The caller owns the deployment slug: it must + // claim the collision guard before any durable write and release it on + // failure, so the slug is not touched here. + let succeeded = false; + let wiredForUnwind: SidecarWorkflowSupervisor | undefined; + let supervisorRegistered = false; + let routersRegistered = false; + let agentTransportRegistered = false; + let hubKeyRecorded = false; + let deploymentRegistered = false; + try { + // The child's `DEFINITION_HASH` is the HUB-APPROVED wire hash the deploy + // frame carried (`spec.approvedWireHash`) -- the hub is the authority, so + // the child re-verifies its own recompute against it. Both feeds into this + // core carry it: the production hub deploy builder always stamps it, and + // the boot restore re-attaches it from the persisted record. A missing + // hash here is therefore a wiring bug, not a legacy case to paper over: + // substituting a sidecar recompute would make the child re-verify against + // the sidecar's own hash rather than the hub authority -- a circular check + // that gives false assurance. Fail loud instead. + if (spec.approvedWireHash === undefined) { + throw new Error( + `workflow deploy spawn (${spec.agentAddress}): the deploy spec carries no approvedWireHash. The hub deploy builder must stamp the hub-approved wire hash and a restore must re-attach it from the persisted record; the sidecar will not recompute it, which would collapse the child's re-verify to a self-check.`, + ); + } + const definitionHash = spec.approvedWireHash; + + // Per-deployment substrate-config keys the workflow-substrate-factory + // validator requires. The boot edge's `multistepSubstrateEnv` carries + // the boot-edge constants; the two workflow-run identity keys are + // derived per-deploy here. + // Spawned bodies' plaintext inference sources, serialized once for the + // deployment's lifetime (bodies do not live-rotate, unlike the top-level + // sources below). The child looks each body up by definition id and never + // holds the sidecar's cipher key. Guarded against the OS argument-string + // ceiling here, at deploy, so an over-large deployment fails loudly rather + // than at a later child `execve`. + const bodySourcesEnv = JSON.stringify(spec.bodySources ?? {}); + const bodySourcesBytes = Buffer.byteLength(bodySourcesEnv, "utf8"); + if (bodySourcesBytes > WORKFLOW_BODY_SOURCES_MAX_BYTES) { + throw new Error( + `sidecar deploy router: serialized ${WORKFLOW_BODY_SOURCES_ENV_KEY} is ${String(bodySourcesBytes)} bytes, over the ${String(WORKFLOW_BODY_SOURCES_MAX_BYTES)}-byte spawn-env limit; the deployment has too many spawned-body sources to deliver through the child env`, + ); + } + + const substrateEnv: Record = { + ...multistepSubstrateEnv, + WORKFLOW_RUN_REPO_ID: runId, + WORKFLOW_RUN_REF: "refs/heads/main", + // Thread the materialized closure's sidecar-local package dir so the + // run child EVALUATES the pinned code to a live definition and + // re-verifies by project-then-hash against `DEFINITION_HASH`. Source-ref + // is the only deploy lineage, so this is always present. Sidecar-local; + // carried on the frozen substrate env because the value is fixed for the + // deployment's lifetime. + CLOSURE_PACKAGE_DIR: spec.closurePackageDir, + [WORKFLOW_BODY_SOURCES_ENV_KEY]: bodySourcesEnv, + }; + // Live-rotatable per-step inference sources. Seeded from the deploy + // spec, then revised in place by the single-step sources-rotation + // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen + // `substrateEnv`: it is recomputed on every spawn and recycle respawn + // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of + // reverting to the deploy-time list. + let currentSources = spec.sources; + + // RunIds whose `run.grants` write was attempted and failed. The grants + // handler below records a runId here on a write failure; the grants + // barrier reads it through `isRunPoisoned` and fails the run rather than + // starting it under the empty deploy-time grant set. Scoped to this + // deployment's supervisor; a run that never had a `run.grants` frame is + // never added, so internal runs inherit the deployment's grants normally. + const poisonedRunIds = new Set(); + + const wired = createSidecarWorkflowSupervisor({ + transport: deps.transport, + repoStore: deps.repoStore, + signingKeySeed: deps.signingKeySeed, + workflowRunRepoId: { + kind: "workflow-run", + id: runId, + }, + workflowRunRef: "refs/heads/main", + runId, + stepCount: spec.definition.stepOrder.length, + stepOrder: credentialStepIds, + deploymentMailAddress: spec.agentAddress, + ...(credentialDelivery !== undefined ? { credentialDelivery } : {}), + deriveStepAddress: stepStrategy.deriveStepAddress, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + isRunPoisoned: (runId) => poisonedRunIds.has(runId), + // The supervisor stamps `runId` + `agentAddress` before + // invoking this; forward the fully-stamped registration to the + // hub-link-backed publisher so a `signal.correlation.register` frame + // reaches the hub for the parked run. + onSuspensionRegister: publishSuspension, + // Reclaim the deployment address when the supervisor drives itself to a + // terminal phase (crash-loop latch, channel crash, recycle failure) so + // the dead supervisor is dropped from `activeSupervisors` and the + // address is redeployable without a manual undeploy first. + onSelfTerminate: () => + reclaimSelfTerminatedSupervisor({ + runId, + agentAddress: spec.agentAddress, + }), + substrateEnv, + // Recomputed on every spawn AND recycle respawn. The rotation + // handler below revises `currentSources` in place, so a respawn + // re-serializes the current (possibly rotated) list rather than the + // frozen deploy-time value. + dynamicSpawnEnv: () => ({ + [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), + }), + subprocessSpawner: multistepSpawner, + ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } : {}), + ...(deps.onDispatchTiming !== undefined ? { onDispatchTiming: deps.onDispatchTiming } : {}), + ...(deps.repackEveryMessages !== undefined + ? { repackEveryMessages: deps.repackEveryMessages } + : {}), + ...(deps.consumedRetentionMs !== undefined + ? { consumedRetentionMs: deps.consumedRetentionMs } + : {}), + ...(deps.readyTimeoutMs !== undefined ? { readyTimeoutMs: deps.readyTimeoutMs } : {}), + }); + + // OUTBOUND half of mailbox ownership (§3a): register a signing key for + // the deployment mail address on the host transport so the supervisor + // signs the deployment's outbound mail. Every step -- single- or + // multi-step -- signs its outbound sends as `spec.agentAddress` (the + // one deployment mail address; no per-step sender reaches the host + // transport), so the transport MUST hold a `CryptoProvider` for it or + // `getTransportFor(senderAddress).send` throws "not registered". + // Registration happens before `spawn()` so the address is live the + // instant the first reply routes outbound. + const { keyPair } = await deps.keyStore.loadOrGenerateKey(spec.agentAddress); + deps.transport.register(spec.agentAddress, deps.createAgentCrypto(keyPair)); + agentTransportRegistered = true; + + // The deploy ack surfaces the deployment address's Ed25519 public key -- + // the key `loadOrGenerateKey` minted above and the workflow uses for + // signed content provenance. Every deployment acks it, single- and + // multi-step alike, so the Hub can publish the same identity. Reconnect + // ownership is established separately by the allocation credential. + const deploymentPublicKey = hexEncode(keyPair.publicKey); + if (spec.definition.stepOrder.length === 1) { + // A single-step workflow stages its deploy tree at the head (the + // lone step IS the head). Initialize the head's on-disk deploy-tree + // repo (idempotent) so the hub's deploy-pack push has a repo to + // apply into. The narrow `initRepo` (not `provisionAgent`) is + // deliberate: the supervised child mints its own keypair and + // persists no hub-agent config. + await deps.sessions.initRepo(spec.agentAddress); + + // Record the hub's public key at the head so the deploy-pack apply + // (and any inbound hub-signed frame) verifies against it. The + // verifier resolves the key from the in-memory key store's + // `recordHubKey` map, so a single-step deployment cannot stand up + // without it. + if (spec.hubPublicKey === undefined) { + throw new Error( + "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", + ); + } + deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); + hubKeyRecorded = true; + } + + // Warm-keep is the single-step launched-agent deploy: the sole step + // IS the long-lived agent, so the child warm-keeps it across + // messages. A multi-step deploy keeps instantiate-send-teardown per + // step. The signal is carried explicitly down through the spawn env. + const warmKeep = spec.definition.stepOrder.length === 1; + const spawnOpts: SpawnOpts = { + stepOrder: [...credentialStepIds], + definitionHash, + warmKeep, + onInferenceEvent: (event) => { + // The event arrives HMAC-verified over the child's event channel. + // Re-narrow it to the hub's `InferenceEvent` union; a parse + // failure means upstream corruption, so drop it loudly rather + // than forwarding an unvalidated payload onto the hub timeline. + const validated = parseInferenceEvent(event); + if (validated instanceof type.errors) { + logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; + return; + } + publishInferenceEvent(spec.agentAddress, validated, spec.sessionId); + }, + }; + + // Record the deployment-address mapping BEFORE `spawn`, because + // `spawn` kicks off `replayProcessingToInbox`, whose workflow-run + // substrate write routes through the boot-edge pack-pushing facade and + // resolves this mapping to address the outbound pack frame. Recording + // it after `spawn` (as the other registrations below are) loses the + // race: the replay's write throws "no run address registered" (a + // real defect masked as a swallowed best-effort warning in the + // supervisor's replay catch). Constraint ownership: the registry owns + // "address is resolvable"; the spawn path must satisfy that contract + // before the replay writes. The finally unwinds it on any failure + // between here and the end of the try. + deps.registerDeployment({ + runId, + agentAddress: spec.agentAddress, + }); + deploymentRegistered = true; + + // Surface spawn-time errors structurally: a subprocess spawner that + // crashes immediately rejects here, and the caller converts the + // rejection into a structured failure frame. The supervisor is + // registered against the deployment address only after spawn succeeds, + // so a spawn-time rejection leaves the registry untouched. + await wired.supervisor.spawn(spawnOpts); + wiredForUnwind = wired; + activeSupervisors.set(spec.agentAddress, wired); + supervisorRegistered = true; + + // Bind the deployment's mail address to this supervisor's + // `routeInbound` so the hub-link dispatches inbound mail into the + // supervisor's mail-bus subscription. Registration happens after + // `spawn` succeeds so a spawn-time rejection leaves the registry + // untouched. + deps.multistepMailRouter?.register(spec.agentAddress, (message) => + wired.routeInbound(message), + ); + // Resolve this deployment's authored inbound-mail policy into a total + // decision map ONCE and store it beside the mail-router registration, so + // the hub-link seam has the recipient's policy before any inbound frame + // for this address can route. + deps.inboundMailPolicyRegistry?.register( + spec.agentAddress, + resolveInboundMailPolicy(spec.definition.inboundMailPolicy), + ); + // Register the signal-delivery handler so a hub `signal.deliver` frame + // dispatches through the supervisor's `deliverSignal`. + deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { + await wired.supervisor.deliverSignal({ + runId: args.runId, + signalName: args.signalName, + signalId: args.signalId, + payload: args.payload, + }); + }); + // Register the drain handler so a hub `drain.deliver` frame dispatches + // through the supervisor's `drain`. + deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { + await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); + }); + // Register the grants handler so a hub `run.grants` frame writes the + // run's grants to `runs//grants.json` in the deployment's + // workflow-run repo. The `runId` selects the per-run destination; the + // step-fan-out fields are inert in that mode but the shared write + // machinery still takes them. + deps.multistepGrantsRouter?.register(spec.agentAddress, async (args) => { + try { + // Cache the co-delivered sender keys BEFORE the grants file lands, so + // "grant durable" implies "key durable": a cache-write fault falls + // into the catch below and poisons the run rather than starting it + // with a grant whose sender the recipient cannot verify. A malformed + // key is a hub-side defect that is keyless from here, so skip it and + // cache the valid ones (no weaker than the hub having omitted it) + // rather than wedge the run on every replay -- but log it at ERROR, + // naming the address and reason, so the hub bug surfaces loudly + // instead of being silently dropped. + for (const identity of args.senderIdentities ?? []) { + let publicKey: Uint8Array; + try { + publicKey = hexDecode(identity.publicKey); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + logger.error`Skipping unparseable sender key for ${identity.address} on run ${args.runId}: ${message}`; + continue; + } + if (publicKey.length !== ED25519_PUBLIC_KEY_BYTES) { + logger.error`Skipping wrong-length sender key for ${identity.address} on run ${args.runId}: got ${String(publicKey.length)} bytes`; + continue; + } + await deps.senderKeyCache.put(identity.address, publicKey); + } + await writeStepGrants({ + repoStore: deps.repoStore, + anchorRunId: runId, + stepOrder: spec.definition.stepOrder, + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: args.stepGrants, + runId: args.runId, + }); + } catch (cause) { + // A sender-key or per-run grants write did not land. Poison the runId + // so the grants barrier fails the run instead of starting it under + // the deploy-time grant set, then re-throw so the hub-link logs the + // durable-write failure loudly. + poisonedRunIds.add(args.runId); + throw cause; + } + // The per-run file now carries the (possibly overlaid) floor. Refresh a + // live child so a standing ("always") approval resolved while the run + // is running but not parked -- with no signal to piggyback the + // `deliverSignal` refresh on -- lowers its floor immediately. + // Best-effort and non-fatal: the same durable file governs the next + // barrier/respawn, so a skipped push never leaves the run under a stale + // floor for good. + await wired.supervisor.deliverGrants(args.runId); + }); + // Register the sources-rotation handler ONLY for a single-step warm + // deployment: it has one long-lived agent whose sources can be + // swapped in place. A multi-step deployment has no single warm agent, + // so it registers no handler and `tryRoute` reports its address as + // unrouted. + if (warmKeep) { + // A single-step deployment's source table has exactly one entry, + // keyed by the head step. Derive that key once here (the layer that + // owns the single-key invariant); `deliverSources` stays flat and + // stepId-agnostic. + const rotationStepId = spec.definition.stepOrder[0]; + if (rotationStepId === undefined) { + throw new Error("single-step deploy has no step id for sources rotation"); + } + deps.multistepSourcesRouter?.register(spec.agentAddress, async (args) => { + const rotated = { [rotationStepId]: args.sources }; + // Swap `currentSources` synchronously BEFORE the durable persist. + // `currentSources` is the process-local respawn hint the + // supervisor reads synchronously through `dynamicSpawnEnv`, so a + // recycle that interleaves the persist `await` must respawn the + // child on the SAME sources being persisted, not the stale prior + // table. The obvious inverse -- persist first, then swap -- is + // rejected: it leaves the child on the OLD sources during the + // persist window while the record has already moved to NEW, so a + // recycle there respawns stale and a restart would "correct" it, + // i.e. the running child contradicts durable intent. Swapping + // first makes the only residual disagreement child-ahead-of- + // durable on a failed persist, which the next recycle heals down + // to the rolled-back durable truth -- the benign direction. The + // wire boundary guarantees `args.sources[0]` is the default, + // which the recycle env form pins as the active source. + const prevSources = currentSources; + currentSources = rotated; + // The durable write still precedes the LIVE swap + // (`deliverSources`), preserving persist-before-externally-visible + // for state that outlives the process; only the process-local + // respawn hint moves ahead. On a failed persist, roll the hint + // back so `currentSources` and the record stay in agreement in the + // common (no interleaved recycle) failure case -- the invariant + // restart consistency depends on. Persistence lets the rotation + // survive a full sidecar restart, not just a recycle: the boot + // scan reseeds spec.sources from record.sources. Overwrites the + // deploy-time record in place. Skipped when no data dir was wired + // (a test router that never persists), matching the restore guard. + if (stepStateDataDir !== undefined) { + try { + await persistWorkflowRunRecord( + stepStateDataDir, + runId, + buildWorkflowRunRecord(spec, rotated), + deps.credentialCipher, + ); + } catch (cause) { + // Restoring unconditionally is safe because rotations for one + // deployment are serialized by the sidecar's per-connection + // inbound-frame queue: each hub frame, sources.update + // included, runs its handler to completion on that queue + // before the next frame's handler starts, so no second + // rotation is in flight whose committed table this rollback + // could clobber. This does NOT rely on the hub pacing its + // sends -- the hub dispatches sources.update fire-and-forget; + // the sidecar frame queue is the sole serializer. Parallelizing + // inbound-frame dispatch would break this rollback. + currentSources = prevSources; + throw cause; + } + } + await wired.supervisor.deliverSources({ + sources: args.sources, + defaultSource: args.defaultSource, + }); + }); + } + + // Register the credential-delivery handler for EVERY deployment (not only + // warm single-step ones): the material cell is per-child and read by + // every step's tool capabilities. The handler hands the delivery to the + // supervisor's `deliverCredentials`, which sends a `credentials-updated` + // control frame to the child where the material cell is swapped. No + // durable persist -- credential material never touches disk. + deps.multistepCredentialsRouter?.register(spec.agentAddress, async (args) => { + await wired.supervisor.deliverCredentials({ + delivery: args.delivery, + ...(args.revoke !== undefined ? { revoke: args.revoke } : {}), + }); + }); + routersRegistered = true; + + succeeded = true; + return { publicKey: deploymentPublicKey }; + } finally { + if (!succeeded) { + // Unwind in reverse registration order so each step undoes state + // the success path confirmed; ordering matches the `undeploy` hook. + if (routersRegistered) { + deps.multistepMailRouter?.unregister(spec.agentAddress); + deps.inboundMailPolicyRegistry?.unregister(spec.agentAddress); + deps.multistepSignalRouter?.unregister(spec.agentAddress); + deps.multistepDrainRouter?.unregister(spec.agentAddress); + deps.multistepGrantsRouter?.unregister(spec.agentAddress); + // Unregister unconditionally: the sources handler was registered + // only for a single-step deploy, but `unregister` is a no-op for + // an address that never registered one, so a multi-step unwind + // safely calls it too. + deps.multistepSourcesRouter?.unregister(spec.agentAddress); + deps.multistepCredentialsRouter?.unregister(spec.agentAddress); + } + if (supervisorRegistered) { + activeSupervisors.delete(spec.agentAddress); + } + if (wiredForUnwind !== undefined) { + await wiredForUnwind.supervisor.shutdown().catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`multi-step deploy unwind: supervisor.shutdown failed: ${message}`; + }); + } + if (agentTransportRegistered) { + // Drop the agent's transport registration so a failed deploy does + // not leave the address live with a dangling `CryptoProvider`. + deps.transport.unregister(spec.agentAddress); + } + if (hubKeyRecorded) { + // Reverse the single-step head's `recordHubKey` so a failed deploy + // leaves no in-memory hub key behind. `forgetAgent` also drops the + // agent keypair cache `loadOrGenerateKey` populated, which is safe: + // the transport registration is already unwound above, nothing reads + // that cache after unwind, and a redeploy reloads the keypair from + // disk. The on-disk deploy-tree repo `initRepo` created is + // deliberately NOT reversed. It is idempotent and the hub re-pushes + // the deploy pack on every redeploy, so it is benign residue; and + // decisively, the durable Ed25519 identity keypair lives inside that + // same directory (`keys/` nests under the agent repo dir), so + // removing the repo would destroy an identity a rerouted head must + // keep across a failed redeploy. + deps.keyStore.forgetAgent(spec.agentAddress); + } + if (deploymentRegistered) { + // Reverse the pre-spawn `registerDeployment`: drop the address + // mapping so a failed spawn leaves the boot-edge registry as it + // found it. Registered first (before spawn), unwound last. A + // subsequent stale workflow-run write for the dead deployment then + // surfaces structurally (`registry.resolve` returns null) rather + // than resolving to the address of a deployment that never came up. + deps.unregisterDeployment({ + runId, + agentAddress: spec.agentAddress, + }); + } + } + } + } + + /** + * Provision one step of a multi-step deploy WITHOUT spawning. The hub + * stages each step's deploy tree before firing the deployment-level + * workflow frame; a full-closure deploy pack still needs an initialized + * agent-state repo to apply into and the hub key recorded to verify the + * pack commit signature. This does exactly those two things -- the same + * harness-free `initRepo` + `recordHubKey` seam the single-step head uses + * -- and constructs no supervisor or child. The deployment-level workflow + * frame (fired once after every step is provisioned) spawns the child, + * which reads each step's staged deploy tree from disk. + * + * Returns the sidecar's principal public key so the link's + * `agent.deploy.ack` carries a key, matching the multi-step ack. A + * per-step address is workflow-derived and records no `agent_instance` + * key, so the hub discards this value. + */ + async function provisionStep(frame: AgentDeployFrame): Promise { + await deps.sessions.initRepo(frame.agentAddress); + deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); + return { + publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), + }; + } + + async function deployMultiStep( + frame: AgentDeployFrame, + projection: NonNullable, + ): Promise { + // Reject a re-deploy of an address already live OR mid-deploy in this + // process BEFORE touching any durable state. The durable writes below (the + // run record, the materialized closure, step grants) are destructive + // overwrites of state owned by whatever deployment currently holds the + // address; overwriting is only legal when this deploy owns the address. + // `activeSupervisors` catches an address whose deploy has completed; + // `reservingDeployAddresses` catches one whose deploy is still in flight. + // The map is populated only after `spawn` succeeds, so the has-check alone + // leaves a window in which two frames both pass and the loser's catch below + // deletes the winner's live record; the reservation set closes it. A + // re-deploy after `undeploy` passes: `undeploy` drops the + // `activeSupervisors` entry, and a failed or completed deploy has already + // cleared its reservation. + if ( + activeSupervisors.has(frame.agentAddress) || + reservingDeployAddresses.has(frame.agentAddress) + ) { + throw new Error( + `sidecar deploy router: ${frame.agentAddress} is already deployed; undeploy it before redeploying`, + ); + } + + const runId = deriveDeploymentId(frame.agentAddress); + + // Resolve the sidecar data dir once: the run record, the materialized + // closure, and the per-step scratch all root under it. Required for any + // deployment that spawns a child. + const dataDir = stepStateDataDir; + if (typeof dataDir !== "string" || dataDir.length === 0) { + throw new Error( + "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the run record and workflow-process child root under it", + ); + } + + // Claim the deployment slug BEFORE any durable write so a colliding runId + // (two distinct addresses projecting to the same slug) is rejected before + // the closure, the step grants, or the supervisor touch disk -- the + // router's "no repo state touched before rejection" guarantee. The claim is + // released on any failure below; a successful deploy keeps it (the undeploy + // hook releases it at teardown). The spawn core owns unwinding the + // supervisor and registrations it stands up; the slug is the caller's. + claimSlug(runId, frame.agentAddress); + // Hold the single-flight reservation across the async body below and clear + // it in the finally. Everything above is synchronous and throws before any + // durable write, so the reservation is only needed from the first await + // here onward; the top-of-method guard already consults this set for a + // concurrent frame, and claimSlug/runId derivation above cannot yield + // control before this point. + reservingDeployAddresses.add(frame.agentAddress); + try { + // Source-ref apply -- the only deploy lineage. Materialize EXACTLY the + // hub's frozen dependency `closure` and evaluate the PINNED CODE to the + // workflow definition; the deploy frame carries no inline definition to + // trust. The closure is applied byte-for-byte (concrete versions + + // integrity SRIs); the sidecar never re-resolves the pin at apply time. + // The re-evaluated projection IS the runnable definition, and the child's + // load-boundary re-verify recomputes the wire hash over the closure and + // fails closed if it diverges from the hub-approved hash, so a closure + // that no longer projects to the approved content cannot deploy. + // + // Check the frame's inline source assets out into the durable + // per-deployment store the closure materializes from. Reclaim the store + // first so a redeploy drops assets no longer referenced. This runs only on + // the DEPLOY path -- restore re-reads the store the deploy persisted, with + // no re-delivery -- so the checkout lives here, not in + // `materializeDeploymentClosure` (which also runs on restore). A + // registry-sourced pin delivers no assets and only clears the store. + const assetStore = deploymentSourceAssetRoot(dataDir, runId); + const gitStore = deploymentSourceGitRoot(dataDir, runId); + await rm(assetStore, { recursive: true, force: true }); + await rm(gitStore, { recursive: true, force: true }); + if (projection.assets !== undefined && projection.assets.length > 0) { + await materializeWorkflowAssets({ + assets: projection.assets, + closure: projection.sourceRef.closure, + assetRoot: assetStore, + gitDirRoot: gitStore, + maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, + }); + } + // Safe to reclaim the instance dir inside the helper: this deploy is + // single-flight-guarded (the reservation above) and the child is not yet + // spawned, so no live reader holds it. + const applied = await materializeDeploymentClosure(dataDir, runId, projection.sourceRef); + const validatedDefinition = WorkflowProjectionDefinition( + projectLiveToInert(applied.definition), + ); + if (validatedDefinition instanceof type.errors) { + throw new Error( + `sidecar deploy router: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`, + ); + } + const effectiveDefinition = validatedDefinition; + + // Structural invariants the wire arktype does not cover (non-empty + // stepOrder, every stepOrder entry backed by a `steps` entry AND a + // `sources` entry), checked against the closure-derived definition -- the + // frame carries none to cover. Mirrors the restore path. + validateWorkflowProjection({ + definition: effectiveDefinition, + sources: projection.sources, + }); + + // Source-admission gate: reject a deploy where any step pins an inference + // provider this sidecar cannot build. Every source in a step's failover + // chain must be buildable -- a chain with an unbuildable tail would fail + // only after the reactor failed over onto it -- so this iterates the whole + // list. The throw propagates back through the deploy frame so the hub's + // `deployWorkflow` rejects synchronously at deploy time. + for (const stepId of effectiveDefinition.stepOrder) { + const chain = projection.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + // Single-agent deploy vs. derived multi-step deploy. A one-step + // definition keeps the deploy's own mail address and its grants in the + // agent-state repo keyed by the run id; a multi-step definition derives + // `-` per step for both the mail address and the + // agent-state repo id, isolating each step's grants in its own repo. + const stepStrategy = createStepStrategy({ + legacyAddress: frame.agentAddress, + stepOrder: effectiveDefinition.stepOrder, + multistepDeriveStepAddress, + }); + + // The spec the shared spawn core consumes, and the durable record that + // lets a boot-time restore rebuild the SAME spec (definition re-evaluated + // from the pinned closure, grants from the step repos, and the record's + // frame/in-memory-only inputs: sources, session id, single-step hub key). + const spec: WorkflowDeploySpec = { + agentAddress: frame.agentAddress, + definition: effectiveDefinition, + sources: projection.sources, + bodySources: buildBodySourcesMap(projection.referencedDefinitions), + // The hub's unified credential-material cell (inference + tool secrets); + // persisted sealed in the record and delivered on the barrier. + credentials: projection.credentials, + approvedWireHash: projection.approvedWireHash, + sessionId: frame.config.sessionId, + hubPublicKey: effectiveDefinition.stepOrder.length === 1 ? frame.hubPublicKey : undefined, + // The sidecar-local dir of the just-materialized closure the spawn core + // threads into the child's env so it re-evaluates the pinned code. + closurePackageDir: applied.packageDir, + // The source-ref pin the record persists so a restore can + // re-materialize the closure. + sourceRef: projection.sourceRef, + }; + const record = buildWorkflowRunRecord(spec, spec.sources); + + // Persist the run record BEFORE the spawn so a crash mid-spawn leaves a + // record the boot scan re-drives (an idempotent re-spawn; the child's + // in-flight-run discovery resumes any run). A soft-failed deploy deletes + // it below, so only a crash-interrupted deploy leaves one. + await persistWorkflowRunRecord(dataDir, runId, record, deps.credentialCipher); + + // Sweep any legacy plaintext body-source file this deployment's bodies + // left behind. Body sources now ride sealed in the run record and reach + // the child through the spawn env, so the `assets/workflow//` + // staging is retired. A deployment first seen on this build never wrote + // one (the rm is a no-op); a redeploy across the upgrade -- or the reconnect + // re-push -- reaches here with its old world-readable plaintext file still + // on disk and removes it now that the record carries the sealed copy. + for (const referenced of projection.referencedDefinitions ?? []) { + await sweepLegacyBodySources(dataDir, referenced.definition.id); + } + + // Grants bridge: the spawned child does not see the frame; it reads + // each step's grants out of `state/grants.json` in the step's + // agent-state repo while the supervisor assembles the + // credentialsSnapshot. Write the operator-approved + // `frame.config.grants` to the same repo the supervisor reads via + // `deriveStepRepoId`, before the spawn core, so the read sees them. + // + // The write covers the deployment's whole flat step-id namespace, loop + // body steps included: the supervisor assembles a snapshot entry for each + // of them, and an entry read out of a repo that was never written carries + // an empty grant set, which denies the body every resource the deploy + // approved. + await writeStepGrants({ + repoStore: deps.repoStore, + anchorRunId: runId, + stepOrder: inertFlatNamespaceStepIds({ + definition: effectiveDefinition, + context: "sidecar deploy router grants bridge: ", + }), + deriveStepRepoId: stepStrategy.deriveStepRepoId, + grants: frame.config.grants, + }); + + // Hand off to the shared spawn core. + return await spawnWorkflowRun(spec); + } catch (cause) { + // Soft failure (this process survived, the deploy threw): drop the + // record and release the slug so the failed deploy is neither restored + // nor leaks its slug. The record delete must not mask the real deploy + // error or skip releasing the slug: a rejecting delete is logged (the + // orphaned record is a durable-state leak the next boot scan re-drives) + // but `cause` is still what propagates and the slug is still released. + try { + await deleteWorkflowRunRecord(dataDir, runId); + } catch (cleanupError) { + const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + logger.error`deploy cleanup: deleteWorkflowRunRecord failed for ${runId}: ${message}`; + } + releaseSlug(runId, frame.agentAddress); + throw cause; + } finally { + // Release the single-flight reservation whether the deploy succeeded or + // threw. On success the address is now in `activeSupervisors`, which the + // guard also consults, so a later re-deploy is still rejected. + reservingDeployAddresses.delete(frame.agentAddress); + } + } + + return { + async deploy(frame): Promise { + if (frame.provisionStep === true) { + return await provisionStep(frame); + } + if (frame.workflow !== undefined) { + return await deployMultiStep(frame, frame.workflow); + } + // Every deploy stages through the workflow-run substrate: a + // provision-step frame primes the per-step repo, and a workflow + // frame spawns the supervised child. A frame carrying neither is + // an unsupported shape -- there is no in-process fall-through. + throw new Error( + `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, + ); + }, + async undeploy(frame): Promise { + // Symmetric teardown for `deploy`: release the per-deployment + // routing state both branches install so a stale `signal.deliver` + // / `drain.deliver` / `mail.inbound` aimed at the dead deployment + // address is rejected by the router rather than dispatched into + // an orphan supervisor handler. The unregister calls are + // idempotent -- they are no-ops when no handler is registered. + // + // Routers come down BEFORE the supervisor's `shutdown()` so any + // hub-side frame racing the undeploy is dropped at the router + // boundary rather than dispatched into a supervisor that is in + // the middle of tearing its child down. The pattern is: drop + // racing frames first, then unwind the underlying resource. + const runId = deriveDeploymentId(frame.agentAddress); + deps.multistepMailRouter?.unregister(frame.agentAddress); + deps.inboundMailPolicyRegistry?.unregister(frame.agentAddress); + deps.multistepSignalRouter?.unregister(frame.agentAddress); + deps.multistepDrainRouter?.unregister(frame.agentAddress); + deps.multistepGrantsRouter?.unregister(frame.agentAddress); + // Unregister unconditionally (a no-op for a multi-step address that + // registered no sources handler), matching the sibling routers. + deps.multistepSourcesRouter?.unregister(frame.agentAddress); + deps.multistepCredentialsRouter?.unregister(frame.agentAddress); + // Shut the per-deployment supervisor down so the workflow-process + // child, its IPC pipes, and its event-channel fd are released. + // The supervisor's `shutdown()` is idempotent (returns early when + // the supervisor is already in `idle`/`stopped`) and handles the + // kill + `exited` await internally. The map entry is removed + // before the await so a subsequent re-deploy on the same address + // cannot observe a stale handle even if `shutdown()` rejects. + const wired = activeSupervisors.get(frame.agentAddress); + if (wired !== undefined) { + activeSupervisors.delete(frame.agentAddress); + await wired.supervisor.shutdown(); + // Drop the deployment address's transport registration installed at + // spawn (OUTBOUND half of mailbox ownership, §3a). Both single- and + // multi-step register the deployment address for outbound signing, so + // this tears down a real registration for either; `unregister` is a + // no-op only if the spawn failed before registering, so it is safe to + // call unconditionally for any spawned deployment. + deps.transport.unregister(frame.agentAddress); + // Reclaim the deployment's per-step local-disk scratch now that + // its supervisor + workflow-process child are torn down. The + // whole `workflow-step-state//` subtree goes: the + // warm single-step agent's stable workspace under `warm/` (the + // dir bounded keying parks per agent) AND any cold `runs//` + // subtrees a multi-step deploy's per-run cleanup did not already + // drop. Awaiting `shutdown()` above guarantees no child still + // holds the scratch, so this is a safe `rm -rf`. The durable + // conversation under `agent-conversation-state/` is a DIFFERENT + // root and is deliberately NOT touched here -- a re-deploy on the + // same address must restore the prior conversation from it. + if (stepStateDataDir !== undefined) { + await rm(pathJoin(stepStateDataDir, "workflow-step-state", runId), { + recursive: true, + force: true, + }); + } + } + // Drop the run record so a boot-time restore does not re-spawn a + // torn-down deployment, and reclaim a source-ref deployment's + // materialized closure tree AND its durable source-asset store. All run + // on every undeploy -- not only when a supervisor was active -- so state + // left behind by a crash-interrupted deploy, or by a source-ref restore + // that materialized the closure and then failed to spawn (registry down), + // is reclaimed too. A registry-sourced deployment never creates the source + // store, so its `force` remove is a no-op there. + if (stepStateDataDir !== undefined) { + await deleteWorkflowRunRecord(stepStateDataDir, runId); + await rm(pathJoin(stepStateDataDir, "workflow-definition-closures", runId), { + recursive: true, + force: true, + }); + await rm(deploymentSourceAssetRoot(stepStateDataDir, runId), { + recursive: true, + force: true, + }); + await rm(deploymentSourceGitRoot(stepStateDataDir, runId), { + recursive: true, + force: true, + }); + } + releaseSlug(runId, frame.agentAddress); + deps.unregisterDeployment({ + runId, + agentAddress: frame.agentAddress, + }); + }, + async restoreWorkflowRuns(): Promise { + const dataDir = stepStateDataDir; + if (dataDir === undefined) { + // No substrate config was wired (a test router that never spawns a + // child): nothing was ever persisted under this data dir, so there + // is nothing to restore. + return; + } + + const scanned = await scanWorkflowRunRecords(dataDir, deps.credentialCipher); + // Restore serially, not in parallel: deterministic boot-log ordering, + // one isolable warning per failed record, and no concurrent + // child-spawn / transport-register storm. Restore runs before + // `hubLink.connect()`, so there are no concurrent deploys to contend + // with. Each record's failure is caught so one bad deployment cannot + // strand the rest. + for (const { runId, record } of scanned) { + try { + // Integrity: the stored address must re-derive to its own directory + // name. A mismatch means a corrupt or misplaced record; skip it + // rather than restore a deployment under the wrong slug. (A source-ref + // record missing its source/closure/approvedWireHash is rejected + // earlier, at the scan boundary, by the record schema's discriminated + // union -- so no bespoke source-ref guard is needed here.) + const derived = deriveDeploymentId(record.agentAddress); + if (derived !== runId) { + logger.warn`skipping workflow deployment restore: ${record.agentAddress} derives slug ${derived}, not its directory ${runId}`; + continue; + } + + // Reconstruct this deployment's runnable definition. Source-ref is + // the only lineage: re-materialize the pinned closure and evaluate the + // pinned code to the live definition, then project it to the inert + // wire shape -- the SAME computation the deploy path applies + // (`WorkflowProjectionDefinition(projectLiveToInert(...))`). The + // closure IS the source of truth; no on-disk definition is read. The + // helper reclaims the instance dir first, which is safe here because + // the prior process (the only reader) is dead and restore is serial + // before `hubLink.connect()`, so no concurrent reader holds it. + // Registry-sourced entries fetch from the content-addressed closure + // cache (a hit populated on the original deploy, surviving restart); + // asset-sourced entries read from the durable source store the + // original deploy checked out (`materializeDeploymentClosure` derives + // the mounts from the pin, so no re-delivery is needed). Both are + // SRI-verified. A cache/store miss soft-fails the record (kept for the + // next boot), matching the `assertSourceBuildable` retry-on-later-boot + // behavior below. The schema guarantees a source-ref record carries a + // `sourceRef` pin, so no undefined-check is needed. + const applied = await materializeDeploymentClosure(dataDir, runId, record.sourceRef); + const validatedDefinition = WorkflowProjectionDefinition( + projectLiveToInert(applied.definition), + ); + if (validatedDefinition instanceof type.errors) { + logger.warn`skipping workflow deployment restore for ${record.agentAddress}: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`; + continue; + } + const definition: WorkflowProjectionDefinition = validatedDefinition; + const closurePackageDir = applied.packageDir; + + // Structural invariants the wire arktype does not cover (non-empty + // stepOrder, every stepOrder entry backed by a `steps` entry AND a + // `sources` entry). The closure eval skips the deploy frame's coverage + // narrow, so this is where its definition-vs-sources coverage is + // checked. + validateWorkflowProjection({ definition, sources: record.sources }); + + // Re-run the source-admission gate: refuse to restore a deployment + // whose pinned provider this sidecar can no longer build. Every + // source in a step's failover chain must be buildable, so this + // iterates the whole list. The record is KEPT (not deleted) so a + // later boot with the provider restored retries it. + for (const stepId of definition.stepOrder) { + const chain = record.sources[stepId]; + if (chain !== undefined) { + for (const source of chain) deps.assertSourceBuildable(source); + } + } + + const spec: WorkflowDeploySpec = { + agentAddress: record.agentAddress, + definition, + sources: record.sources, + // The record's unsealed body sources, re-delivered to the run child + // through the spawn env. `undefined` for a legacy record written + // before body sources were persisted here; the child then falls back + // to that deployment's on-disk plaintext file for the missing bodies. + bodySources: record.bodySources, + // The record's credential-material cell, unsealed by the boot scan + // and re-delivered to the child on the pre-trigger barrier so an + // offline restart restores both inference and tool credentials + // without the hub. `undefined` when the deployment bound none. + credentials: record.credentials, + // The hub-approved wire hash the original deploy persisted, so the + // restore re-spawn carries the same `DEFINITION_HASH` rather than a + // recompute. Always present -- the record schema requires it. + approvedWireHash: record.approvedWireHash, + sessionId: record.sessionId, + hubPublicKey: record.hubPublicKey, + // The sidecar-local dir of the just-materialized closure the spawn + // core threads into the child's env so it re-evaluates the pinned + // code. + closurePackageDir, + // Carry the source-ref pin so a post-restore source rotation -- + // which rebuilds the record from the spec -- re-persists it; without + // this a rotation would silently drop it and wedge the NEXT restart. + sourceRef: record.sourceRef, + }; + + // The slug is the caller's, matching `deployMultiStep`: claim before + // the spawn, release on failure. Unlike deploy's soft-fail, restore + // does NOT delete the record and does NOT re-materialize the step + // grants or the onTrigger body sources -- both are already on disk + // from the original deploy. A failed restore just warns and + // leaves the record for the next boot; there is deliberately no GC + // of a permanently-unrestorable record here (an operator reclaims it + // by undeploying the address). + // + // Release only a slug THIS pass newly claimed: if the address is + // already live (its slug still held by the running deployment), the + // core's double-spawn guard throws, and freeing the slug then would + // strand a live deployment's collision guard. `claimSlug` is a + // no-op for an already-held (runId, address) pair, so the + // pre-claim check distinguishes the two. + const slugNewlyClaimed = slugClaims.get(runId) !== record.agentAddress; + claimSlug(runId, record.agentAddress); + try { + await spawnWorkflowRun(spec); + logger.info`Restored workflow deployment for ${record.agentAddress}`; + } catch (cause) { + if (slugNewlyClaimed) { + releaseSlug(runId, record.agentAddress); + } + throw cause; + } + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`Failed to restore workflow deployment ${runId}: ${reason}`; + } + } + }, + activeAddresses(): string[] { + // `activeSupervisors` is keyed by deployment run address and holds + // exactly the deployments with a live supervisor (deploy and restore + // add; undeploy and spawn-unwind remove), so its keys are the addresses + // this sidecar can currently route mail to. + return [...activeSupervisors.keys()]; + }, + reEmitParkedCorrelations(address: string): void { + const wired = activeSupervisors.get(address); + if (wired === undefined) { + // Edge boundary: the hub reports this address routable but no live + // supervisor owns it -- a deployment torn down, or not yet respawned. + // Re-registering a torn-down deployment would be wrong, so skipping is + // correct. Logged at debug (not warn) so a supervisor unexpectedly + // missing for a live deployment is still traceable without crying wolf + // on the ordinary torn-down case. + logger.debug`re-emit on reconnect: no active supervisor for ${address}; skipping`; + return; + } + // Fire-and-forget: the driver is best-effort and watchdog-bounded inside + // the supervisor, so the reconnect fan-out never awaits it. The promise + // is dropped deliberately; the `.catch` is defense-in-depth since the + // driver already swallows its own query failures. + void wired.supervisor.reEmitParkedCorrelations().catch((cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + logger.warn`re-emit of parked correlations on hub reconnect failed for ${address}: ${message}`; + }); + }, + }; +} + +/** + * Logical mail-audit reference the supervisor stamps onto every + * inbox/processing/consumed envelope for sidecar-hosted deployments. + * The substrate does not dereference the value; it is a host-side + * pointer the audit consumer joins on. The mail audit is keyed by the + * deployment id plus the parsed messageId, which is unique per inbound + * message and stable across the FIFO pipeline's + * enqueue/dequeue/markConsumed transitions. + */ +export function deriveSidecarMailAuditRef(runId: string): ( + messageId: string, + rawMessage: Uint8Array, +) => { + store: string; + path: string; +} { + return (messageId, _rawMessage) => ({ + store: "sidecar-mail-audit", + path: `${runId}/${messageId}`, + }); +} + +/** + * Construct a per-deployment supervisor with the sidecar's bindings + * pre-wired. The router calls this once per multi-step `agent.deploy` + * frame to stand up the workflow-process child that hosts the + * deployment. + */ +export function createSidecarWorkflowSupervisor( + opts: CreateSidecarWorkflowSupervisorOpts, +): SidecarWorkflowSupervisor { + const mailBus: HubTransportMailBusAdapter = wrapHubTransportAsMailBus(opts.transport); + const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { + kind: "supervisor", + anchorRunId: opts.runId, + }; + // Per-run grants sink. The supervisor awaits this and pushes the + // resulting snapshot to the child before the run's `trigger.fire`; a + // throw propagates and the dispatch barrier fails the run rather than + // firing the trigger against absent grants. The snapshot is the run's + // own per-run grants file, which every legitimate birth path writes + // before dispatch (see `assembleRunCredentialsSnapshot`). + const onRunStart = (args: { + runId: string; + anchorRunId: string; + }): Promise => { + // Fail closed on a run whose `run.grants` write was recorded as failed: + // its per-run grants file never landed. This is the known-failed-write + // case, distinct from the general absent-file backstop in + // `assembleRunCredentialsSnapshot` -- both fail the run closed, but this + // one names the recorded write failure specifically so a poisoned run's + // diagnostics point at the failed push rather than a generic absence. + if (opts.isRunPoisoned?.(args.runId) === true) { + return Promise.reject( + new Error( + `sidecar onRunStart: run ${args.runId} grants write failed; refusing to start the run under-authorized`, + ), + ); + } + return assembleRunCredentialsSnapshot({ + repoStore: opts.repoStore, + anchorRunId: args.anchorRunId, + runId: args.runId, + stepOrder: opts.stepOrder, + deriveStepAddress: opts.deriveStepAddress, + }); + }; + const supervisor = createWorkflowSupervisor({ + repoStore: opts.repoStore, + signAsPrincipal: async (kind, payload) => { + const sig = await signEd25519(opts.signingKeySeed, payload); + return { sig, principalKind: kind }; + }, + mailBus, + subprocessSpawner: opts.subprocessSpawner ?? defaultSubprocessSpawner, + binaryPath: opts.binaryPath ?? SIDECAR_WORKFLOW_CHILD_BINARY, + substrateEnv: opts.substrateEnv, + dynamicSpawnEnv: opts.dynamicSpawnEnv, + workflowRunRepoId: opts.workflowRunRepoId, + workflowRunRef: opts.workflowRunRef, + anchorRunId: opts.runId, + stepCount: opts.stepCount, + deploymentMailAddress: opts.deploymentMailAddress, + readPrincipal: supervisorPrincipal, + deriveStepAddress: opts.deriveStepAddress, + onRunStart, + ...(opts.credentialDelivery !== undefined + ? { credentialDelivery: opts.credentialDelivery } + : {}), + ...(opts.onSuspensionRegister !== undefined + ? { onSuspensionRegister: opts.onSuspensionRegister } + : {}), + ...(opts.onSelfTerminate !== undefined ? { onSelfTerminate: opts.onSelfTerminate } : {}), + ...(opts.deriveStepRepoId !== undefined ? { deriveStepRepoId: opts.deriveStepRepoId } : {}), + deriveMailAuditRef: deriveSidecarMailAuditRef(opts.runId), + ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } : {}), + ...(opts.repackEveryMessages !== undefined + ? { repackEveryMessages: opts.repackEveryMessages } + : {}), + ...(opts.consumedRetentionMs !== undefined + ? { consumedRetentionMs: opts.consumedRetentionMs } + : {}), + ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), + }); + return { + supervisor, + routeInbound(message) { + return mailBus.routeInbound(opts.deploymentMailAddress, message); + }, + getCredentialsSnapshot: () => supervisor.getCredentialsSnapshot(), + onRunStart, + }; +} diff --git a/apps/sidecar/src/workflow-host-wiring/asset-materialization.ts b/apps/sidecar/src/workflow-host-wiring/asset-materialization.ts deleted file mode 100644 index 94b3f051d9..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/asset-materialization.ts +++ /dev/null @@ -1,65 +0,0 @@ -// Workflow-asset materialization on the sidecar's local substrate: the -// deploy-time `sources.json` disk-convention write an in-process onTrigger -// body child reads its inference-source pins back from. The body DEFINITION -// is never staged -- it is resolved in-memory from the parent's re-verified -// closure. - -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { dirname, join as pathJoin } from "node:path"; - -import type { AgentDeployFrame } from "@intx/types/sidecar"; - -/** - * Materialize an extracted onTrigger body's per-step inference-source pins to - * `${dataDir}/assets/workflow//sources.json`. A body child runs - * in-process with no process env and loses its env across a restart, so its - * sources must be durable on disk; the body invoker reads this file to build - * the body's inference-source resolver. Idempotent content-compare write. - */ -export async function materializeWorkflowSources( - sidecarDataDir: string | undefined, - definitionId: string, - sources: NonNullable["sources"], -): Promise { - if (typeof sidecarDataDir !== "string" || sidecarDataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the workflow-process child resolves the workflow-asset repo dir against this data dir", - ); - } - const sourcesAssetPath = pathJoin( - sidecarDataDir, - "assets", - "workflow", - definitionId, - "sources.json", - ); - const sourcesAssetBytes = JSON.stringify(sources, null, 2); - try { - await mkdir(dirname(sourcesAssetPath), { recursive: true }); - // Idempotent: only rewrite when the on-disk content differs. Treats a - // missing file as different. - let existing: string | null = null; - try { - existing = await readFile(sourcesAssetPath, "utf8"); - } catch (cause) { - if ( - !( - cause instanceof Error && - "code" in cause && - (cause as { code: unknown }).code === "ENOENT" - ) - ) { - throw cause; - } - } - if (existing !== sourcesAssetBytes) { - await writeFile(sourcesAssetPath, sourcesAssetBytes, "utf8"); - } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar deploy router: failed to materialize sources.json at ${sourcesAssetPath}: ${reason}`, - { cause }, - ); - } -} diff --git a/apps/sidecar/src/workflow-host-wiring/closure-staging.ts b/apps/sidecar/src/workflow-host-wiring/closure-staging.ts deleted file mode 100644 index b2f974a055..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/closure-staging.ts +++ /dev/null @@ -1,192 +0,0 @@ -// Staging for a deployment's frozen workflow-definition closure: the -// durable per-deployment stores its source assets are checked out into, -// and the apply that lays the closure out and evaluates the pinned code -// to a `WorkflowDefinition`. Both the deploy path and the boot-time -// restore path route through here, so the two resolve identical mounts -// from the pin alone -- restore has only the pin, never a re-delivery. - -import { rm, stat } from "node:fs/promises"; -import { join as pathJoin } from "node:path"; - -import { workflowSourceAssetMountPath } from "@intx/hub-sessions"; -import type { SourceRefPin } from "@intx/types/sidecar"; - -import { applyFrozenWorkflowClosure, type AppliedWorkflowClosure } from "../workflow-closure-apply"; -import { sourceAssetGitDir } from "../source-asset-delivery"; -import { parseToolRegistries } from "../tool-materialization"; - -/** - * The durable per-deployment store the sidecar checks a deployment's source - * assets out into. A SIBLING of the closure instance dir, not a child: - * `materializeDeploymentClosure` reclaims the closure dir on every apply and - * restore, but never this store, so the checked-out assets survive a restart - * and re-materialization needs no re-delivery. The store is reclaimed on - * redeploy (at the deploy call site) and on undeploy. - */ -export function deploymentSourceAssetRoot(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-definition-sources", deploymentId); -} - -/** - * The durable indexed-`.git` store root a pinned deployment's source-format - * asset entries are checked out from. Sibling of the plain-file source store; - * both survive restart so re-materialization needs no re-delivery. - */ -export function deploymentSourceGitRoot(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-definition-source-gits", deploymentId); -} - -/** - * The per-deployment directory a deployment's closure is laid out under. - * Deterministic per deployment id, so a redeploy or a boot restore reuses it. - */ -export function deploymentClosureInstanceDir(dataDir: string, deploymentId: string): string { - return pathJoin(dataDir, "workflow-definition-closures", deploymentId); -} - -function deriveSourceAssetMounts(pin: SourceRefPin): Map { - const mounts = new Map(); - for (const entry of pin.closure.entries) { - if (entry.source.kind === "asset" && entry.source.package.format === "tarball") { - mounts.set(entry.source.assetId, workflowSourceAssetMountPath(entry.source.assetId)); - } - } - return mounts; -} - -function deriveSourceGitDirs(pin: SourceRefPin, gitRoot: string): Map { - const gitDirs = new Map(); - for (const entry of pin.closure.entries) { - if (entry.source.kind === "asset" && entry.source.package.format === "source") { - gitDirs.set(entry.source.assetId, sourceAssetGitDir(gitRoot, entry.source.assetId)); - } - } - return gitDirs; -} - -async function isExistingDir(dir: string): Promise { - try { - return (await stat(dir)).isDirectory(); - } catch (err) { - if (err instanceof Error && "code" in err && err.code === "ENOENT") { - return false; - } - throw err; - } -} - -/** - * Resolve the durable source-asset store root and the `assetId -> mountPath` / - * `assetId -> gitDir` maps a pinned deployment materializes its - * `kind: "asset"` closure entries from, asserting every referenced asset is - * present on disk. A cheap early gate for a missing checkout; the loader still - * SRI-verifies each tarball's bytes at materialization. A missing mount is a - * broken deployment the hub must re-drive, so it fails loud rather than - * materializing against an absent store. - */ -export async function resolveDeploymentAssetMounts( - dataDir: string, - deploymentId: string, - pin: SourceRefPin, -): Promise<{ - assetRoot: string; - assetMounts: ReadonlyMap; - gitDirs: ReadonlyMap; -}> { - const assetRoot = deploymentSourceAssetRoot(dataDir, deploymentId); - const assetMounts = deriveSourceAssetMounts(pin); - for (const [assetId, mountPath] of assetMounts) { - const mountDir = pathJoin(assetRoot, mountPath); - if (!(await isExistingDir(mountDir))) { - throw new Error( - `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} is not present in the durable store at ${mountDir}; the deployment must be re-driven from the hub`, - ); - } - } - const gitRoot = deploymentSourceGitRoot(dataDir, deploymentId); - const gitDirs = deriveSourceGitDirs(pin, gitRoot); - for (const [assetId, gitDir] of gitDirs) { - if (!(await isExistingDir(gitDir))) { - throw new Error( - `resolveDeploymentAssetMounts: source asset ${JSON.stringify(assetId)} for deployment ${deploymentId} has no indexed git store at ${gitDir}; the deployment must be re-driven from the hub`, - ); - } - } - return { assetRoot, assetMounts, gitDirs }; -} - -/** - * Read a substrate-config byte cap (`SIDECAR_CACHE_MAX_BYTES` / - * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) from the multi-step substrate env and - * parse it to a positive finite number. The boot edge resolves these once and - * threads them through the substrate env; the closure apply needs them to size - * the tarball cache and the per-fetch cap. A missing or non-numeric value is a - * boot-edge wiring bug, so it fails loud rather than defaulting. - */ -export function requireSubstrateByteCap(env: Record, key: string): number { - const raw = env[key]; - if (raw === undefined) { - throw new Error( - `sidecar deploy router: ${key} must be present in the multi-step substrate env to materialize a frozen workflow closure`, - ); - } - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error( - `sidecar deploy router: ${key} must be a positive finite number, got ${JSON.stringify(raw)}`, - ); - } - return parsed; -} - -/** - * Materialize a deployment's frozen closure to its per-deployment instance dir - * and evaluate the pinned code. The instance dir is force-reclaimed first: the - * id is deterministic per address, so a redeploy or a boot restore reuses the - * same dir and a prior soft-failed deploy can leave it half-materialized. Safe - * only because no live reader holds the dir when this runs -- a precondition - * each caller establishes. - */ -export async function materializeDeploymentClosure(args: { - dataDir: string; - deploymentId: string; - pin: SourceRefPin; - substrateEnv: Record; -}): Promise { - const instanceDir = deploymentClosureInstanceDir(args.dataDir, args.deploymentId); - await rm(instanceDir, { recursive: true, force: true }); - - const { assetRoot, assetMounts, gitDirs } = await resolveDeploymentAssetMounts( - args.dataDir, - args.deploymentId, - args.pin, - ); - - return applyFrozenWorkflowClosure({ - source: args.pin.source, - closure: args.pin.closure, - instanceDir, - cacheRoot: pathJoin(args.dataDir, "workflow-definition-closure-cache"), - cacheMaxBytes: requireSubstrateByteCap(args.substrateEnv, "SIDECAR_CACHE_MAX_BYTES"), - registryMaxTarballBytes: requireSubstrateByteCap( - args.substrateEnv, - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - ), - registries: parseToolRegistries( - requireSubstrateEntry(args.substrateEnv, "SIDECAR_TOOL_REGISTRIES"), - ), - assetRoot, - assetMounts, - gitDirs, - }); -} - -function requireSubstrateEntry(env: Record, key: string): string { - const raw = env[key]; - if (raw === undefined) { - throw new Error( - `sidecar deploy router: ${key} must be present in the multi-step substrate env to materialize a frozen workflow closure`, - ); - } - return raw; -} diff --git a/apps/sidecar/src/workflow-host-wiring/event-channel-hmac.test.ts b/apps/sidecar/src/workflow-host-wiring/event-channel-hmac.test.ts deleted file mode 100644 index b5759268f8..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/event-channel-hmac.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -// CL-6396: a body child's stamped `childRunId` is inside the HMAC-signed -// envelope, not a sidecar-side annotation after verify. This roundtrip -// is the production event-channel path (`encodeEnvelope` → HMAC → -// `receiveEventChannel` → `ReceivedEvent`), so a missing field cannot -// hide behind an in-process object that never went over the wire. -import { expect, test } from "bun:test"; -import { - createEventChannelSender, - decodeEnvelope, - encodeEnvelope, - generateChannelId, - generateHmacKey, - receiveEventChannel, - type EventPayload, - type FrameReader, - type FrameWriter, -} from "@intx/workflow-host"; - -const EVENT = { - type: "inference.start" as const, - seq: 1, - data: { model: "stub" }, -}; - -function createMemoryPipe(): { - writer: FrameWriter; - reader: FrameReader; - close: () => void; -} { - const chunks: Uint8Array[] = []; - let closed = false; - let wake: (() => void) | null = null; - function notify() { - const waiting = wake; - wake = null; - waiting?.(); - } - return { - writer: { - write(bytes) { - chunks.push(bytes); - notify(); - }, - }, - reader: { - async *read() { - for (;;) { - while (chunks.length > 0) { - const next = chunks.shift(); - if (next !== undefined) yield next; - } - if (closed) return; - await new Promise((resolve) => { - wake = resolve; - }); - } - }, - }, - close() { - closed = true; - notify(); - }, - }; -} - -test("stamped childRunId survives encodeEnvelope, HMAC verify, and ReceivedEvent", async () => { - const hmacKey = generateHmacKey(); - const channelId = generateChannelId(); - const envelopeBytes = encodeEnvelope({ - seq: 1, - channelId, - payload: EVENT, - childRunId: "turn__0", - }); - expect(decodeEnvelope(envelopeBytes).childRunId).toBe("turn__0"); - - const pipe = createMemoryPipe(); - const sender = createEventChannelSender({ - hmacKey, - channelId, - writer: pipe.writer, - }); - const received: { event: EventPayload; childRunId?: string }[] = []; - const crashes: string[] = []; - const consume = (async () => { - for await (const frame of receiveEventChannel({ - hmacKey, - channelId, - reader: pipe.reader, - onCrash: (reason) => crashes.push(reason), - })) { - received.push(frame); - } - })(); - - await sender.send(EVENT, "turn__0"); - const deadline = Date.now() + 2_000; - while (received.length === 0) { - if (crashes.length > 0) { - throw new Error(`event channel crashed: ${crashes.join("; ")}`); - } - if (Date.now() > deadline) { - throw new Error("timed out waiting for HMAC-verified event"); - } - await new Promise((resolve) => setTimeout(resolve, 5)); - } - pipe.close(); - await consume; - - expect(crashes).toEqual([]); - expect(received).toEqual([{ event: EVENT, childRunId: "turn__0" }]); -}); diff --git a/apps/sidecar/src/workflow-host-wiring/index.ts b/apps/sidecar/src/workflow-host-wiring/index.ts deleted file mode 100644 index 52a8d55d80..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/index.ts +++ /dev/null @@ -1,2225 +0,0 @@ -// Thin wiring module that constructs `createWorkflowSupervisor` with -// this sidecar's host-specific bindings: the existing mail-bus -// instance, the sidecar's Ed25519 signing keypair, the substrate -// RepoStore handle, and `Bun.spawn` as the subprocess spawner. Any -// logic that would benefit a future alternative-sidecar -// implementation lives inside `@intx/workflow-host`, not here. - -import { rm } from "node:fs/promises"; -import { join as pathJoin } from "node:path"; - -import { type } from "arktype"; - -import { derivePublicKeyBytes } from "@intx/crypto"; -import { getLogger } from "@intx/log"; -import type { HubTransport } from "@intx/mail-memory"; -import { parseAgentId, type RepoStore } from "@intx/hub-sessions"; -import type { - AgentKeyStore, - DeployRouter, - DeployRouterResult, - SessionManager, -} from "@intx/hub-agent"; -import { reportError } from "@corbits/error-sink"; -import { - type DeriveStepAddress, - type DispatchTimingMark, - type SpawnOpts, - type SubprocessSpawner, - type SuspensionRegistration, -} from "@intx/workflow-host"; -import { hexEncode, isRunAddress } from "@intx/types"; -import { - parseInferenceEvent, - type CryptoProvider, - type InferenceEvent, - type InferenceSource, - type KeyPair, -} from "@intx/types/runtime"; -import { WorkflowProjectionDefinition, type AgentDeployFrame } from "@intx/types/sidecar"; -import { projectLiveToInert } from "@intx/workflow"; - -import type { - MultistepCredentialsRouter, - MultistepDrainRouter, - MultistepGrantsRouter, - MultistepMailRouter, - MultistepSignalRouter, - MultistepSourcesRouter, -} from "../workflow-run-pack-client"; -import { - WorkflowRestoreFailure, - clearWorkflowDeploymentRestoreFailure, - deleteWorkflowDeploymentRecord, - isWorkflowDeploymentRestoreQuarantined, - markWorkflowDeploymentRecordParked, - partitionScannedDeployments, - readWorkflowDeploymentRecord, - recordWorkflowDeploymentRestoreFailure, - scanWorkflowDeploymentRecords, - writeWorkflowDeploymentRecord, - type WorkflowDeploymentRecord, -} from "../workflow-deployment-record"; -import { runWithConcurrency } from "../concurrency"; -import { - reapExpiredHibernationSnapshots as reapVaultSnapshots, - restoreAgentIdentity, - snapshotAgentIdentity, -} from "../hibernated-agent-identity-vault"; -import { isErrnoNotFound } from "../conversation-state"; -import { computeWireDefinitionHash, validateWorkflowProjection } from "./wire-validation"; -import { defaultSubprocessSpawner } from "./transport"; - -export { defaultSubprocessSpawner }; -import { createSidecarWorkflowSupervisor, type SidecarWorkflowSupervisor } from "./supervisor"; - -export { deriveSidecarMailAuditRef, type CreateSidecarWorkflowSupervisorOpts } from "./supervisor"; -export { createSidecarWorkflowSupervisor, type SidecarWorkflowSupervisor }; -import { createStepStrategy, deriveDeploymentId, writeStepGrants } from "./step-strategy"; - -export { deriveDeploymentId }; -import { materializeWorkflowSources } from "./asset-materialization"; -import { - deploymentSourceAssetRoot, - deploymentSourceGitRoot, - materializeDeploymentClosure, -} from "./closure-staging"; -import { - MAX_INLINE_ASSET_PAYLOAD_BYTES, - materializeWorkflowAssets, -} from "../source-asset-delivery"; - -export { computeWireDefinitionHash, validateWorkflowProjection }; - -const logger = getLogger(["sidecar", "workflow-host-wiring"]); - -/** - * Env key the multi-step branch uses to carry each step's ordered - * inference-source failover chain from `frame.workflow.sources` down to - * the workflow-process child. The substrate factory's `buildEnv` reads - * this and resolves a step's chain at step invocation, feeding it to the - * reactor for forward-only failover; the supervisor itself is opaque to - * the value (it is plumbed through `bindings.substrateEnv` verbatim). - * - * Listed here so the router and the future substrate-factory consumer - * spell the key the same way without a magic-string trip hazard. - */ -export const STEP_INFERENCE_SOURCES_ENV_KEY = "STEP_INFERENCE_SOURCES"; - -/** - * How long a teardown (`teardownDeployment`, and the process-exit drain) - * waits for a supervisor's own graceful `shutdown()` (child signal + await - * exit) before also sending SIGKILL to the child directly via - * `hardKillChild`. Bounds a teardown's worst case to this window rather - * than however long a wedged child's own shutdown sequencing takes. - */ -export const CHILD_KILL_ESCALATION_MS = 3000; - -/** - * How long `teardownDeployment` waits for the supervisor's own `drain` to - * let an in-flight step settle -- so its resulting workflow-run event - * commit (and the pack push that commit triggers) lands -- before the - * child comes down. Bounded so a wedged or long-running step cannot hang - * teardown indefinitely; mirrors `CHILD_KILL_ESCALATION_MS`'s bounded-wait - * shape. - */ -export const TEARDOWN_DRAIN_DEADLINE_MS = 5000; - -/** - * How many workflow deployment records `restoreWorkflowDeployments` restores - * at once. Bounded, not unbounded, so a host with many live records at boot - * cannot storm the OS with concurrent `Bun.spawn` calls all at once; 8 is - * comfortably below typical per-process fd/thread pressure from a handful of - * child processes while still cutting a boot with dozens of records from - * minutes of serial restore to a few bounded rounds. - */ -export const RESTORE_CONCURRENCY = 8; - -/** - * Ceiling on a single `restoreDeploymentFromRecord` attempt. A restore - * candidate can wedge indefinitely -- a stalled closure fetch, a spawn that - * never reports back -- and with a bounded worker pool one wedged record - * pins its worker (and, at the extreme, every worker) for the rest of - * boot. A restore that exceeds this deadline is treated as an ordinary - * transient failure: counted on the record's `restoreFailure` counter, - * logged, and skipped, so the boot moves on and the record gets another - * attempt next boot (or quarantines after `RESTORE_QUARANTINE_THRESHOLD` - * consecutive permanent failures). Overridable per router - * (`deps.restoreAttemptTimeoutMs`) so a test can exercise the timeout path - * in milliseconds instead of waiting out the real ceiling. - */ -export const RESTORE_ATTEMPT_TIMEOUT_MS = 30_000; - -/** - * The `reason` this router's own idle-hibernate sweep passes to - * `teardownDeployment` for every idle eviction, and the reason - * `undeploy(frame)` matches on to choose the state-preserving teardown - * flavor (`reclaimDirs: false` — deployment record, step-state, and slug - * all survive) over the destructive default a caller-initiated undeploy - * gets. CL-8187: idle hibernation is this router's own decision now — it - * tracks per-deployment last activity itself (see `lastActivityMs`) and - * tears an idle run down directly, rather than waiting for a hub-driven - * `agent.undeploy` tagged with this reason. The tag stays a named export - * so a caller-initiated undeploy can still choose to hibernate rather - * than reclaim. - */ -export const IDLE_HIBERNATE_UNDEPLOY_REASON = "idle-hibernate"; - -/** - * Bounds a boot-restore attempt to `timeoutMs` -- so `restoreWorkflowDeployments`'s - * bounded worker pool frees this slot and moves on to the next record rather - * than waiting on a wedged restore forever -- WITHOUT quietly abandoning the - * restore itself (CL-7215). `work` receives an `AbortSignal` that fires the - * instant the deadline wins, so a restore attempt that has not yet started - * its closure fetch or its spawn can cancel itself for real rather than - * running either to completion unobserved. Neither the closure fetch - * (`@intx/tool-packaging`'s `applyAtomic`) nor the workflow-process spawn - * (`@intx/workflow-host`'s `Supervisor.spawn`) accept a signal of their own, - * so a restore already past both of `restoreDeploymentFromRecord`'s - * checkpoints when the deadline fires keeps running -- but it corrects its - * own durable record once it learns the true outcome instead of leaving a - * live deployment recorded as a boot failure; see the `signal.aborted` - * check around `spawnWorkflowDeployment` in `restoreDeploymentFromRecord`. - */ -function withRestoreTimeout( - work: (signal: AbortSignal) => Promise, - deploymentId: string, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - return new Promise((resolve, reject) => { - const message = `restore of ${deploymentId} exceeded ${timeoutMs}ms`; - const timer = setTimeout(() => { - controller.abort(new Error(message)); - reject(new Error(message)); - }, timeoutMs); - work(controller.signal).then( - (value) => { - clearTimeout(timer); - resolve(value); - }, - (error: unknown) => { - clearTimeout(timer); - reject(error); - }, - ); - }); -} - -/** - * Cooperative cancellation checkpoint for `restoreDeploymentFromRecord` - * (CL-7215): called immediately before starting a phase of work that is - * still avoidable -- the closure fetch, or the spawn itself -- so a restore - * whose caller has already timed out stops advancing instead of spending - * that work on an attempt nobody is waiting on anymore. - */ -function throwIfRestoreAborted(signal: AbortSignal, deploymentId: string): void { - if (signal.aborted) { - throw new Error( - `restore of ${deploymentId} was cancelled after its boot-restore attempt timed out`, - ); - } -} - -/** - * Await a supervisor's graceful `shutdown()`, escalating to a direct - * SIGKILL of its child if `shutdown()` hasn't settled within - * `CHILD_KILL_ESCALATION_MS` -- still awaiting `shutdown()` to completion - * either way, never abandoning it. - */ -async function shutdownSupervisorWithEscalation(wired: SidecarWorkflowSupervisor): Promise { - const timer = setTimeout(() => { - wired.hardKillChild(); - }, CHILD_KILL_ESCALATION_MS); - timer.unref?.(); - try { - await wired.supervisor.shutdown(); - } finally { - clearTimeout(timer); - } -} - -/** - * Derive the supervisor's principal public key from the sidecar's - * Ed25519 signing seed. The supervisor signs every workflow-run event - * with this key; the multi-step branch surfaces it to the link so the - * hub records the verifying key for the deployment's signed events. - */ -async function derivePrincipalPublicKeyHex(signingKeySeed: Uint8Array): Promise { - return hexEncode(await derivePublicKeyBytes(signingKeySeed)); -} - -/** - * The sidecar's `DeployRouter` plus the boot-time restore driver. The link - * routes `agent.deploy`/`agent.undeploy` through the `DeployRouter` surface; - * the sidecar boot edge additionally calls `restoreWorkflowDeployments` once, - * before connecting to the hub, to re-establish the deployments a prior - * process persisted. The extra method is sidecar-app-only, so it rides on the - * concrete router type rather than the shared `DeployRouter` contract. - */ -export interface SidecarDeployRouter extends DeployRouter { - /** - * Re-establish every persisted workflow deployment on this sidecar's local - * substrate. Runs once at boot, before `hubLink.connect()`, so a single-step - * head's mailbox/transport registration is live before the hub routes to it. - * Soft-fails per deployment: a record that cannot be restored (unbuildable - * provider, corrupt `deployment.json`, spawn failure) is logged and left on - * disk for a later boot to retry -- it is never deleted here. A failure - * that is deterministic given only the record's own bytes (a - * `WorkflowRestoreFailure("permanent", ...)`, e.g. a corrupt/misplaced - * record or a closure-derived definition that fails validation) is - * tracked on the record and, after `RESTORE_QUARANTINE_THRESHOLD` - * consecutive such failures, quarantined: later boots skip it entirely - * rather than re-attempting and re-warning. A failure that depends on - * this boot's environment (an unbuildable inference provider, and - * anything else not explicitly classified permanent) never quarantines - * and is retried every boot for as long as the record exists. - */ - restoreWorkflowDeployments(): Promise; - /** - * Sweep this sidecar's hibernated-agent-identity vault - * (`hibernated-agent-identity-vault.ts`) for snapshots older than its - * stated retention window and delete them, returning how many were - * reaped for observability. Independent of `restoreWorkflowDeployments` - * -- an orphaned snapshot (its address hibernated, then permanently - * torn down without ever redeploying) has no relationship to that scan - * and this does not touch its concurrency. Intended to run once at - * boot, in either order relative to the restore scan. - */ - reapExpiredHibernationSnapshots(): Promise; - /** - * The workflow-substrate deployment addresses (`ins_dep_...`) this router - * currently hosts a live supervisor for -- the set of addresses this - * sidecar can route mail to. The boot edge announces these to the hub on - * (re)connect so the hub re-registers them for routing: they are hub-minted - * and carry no per-address key, so unlike single-agent sessions they are - * not re-established by the challenge flow, and without this announcement - * the hub drops their route on a WS reconnect. Reflects `deploy`/`undeploy` - * and boot-time restore live, so a caller re-reads it per connect. - */ - activeAddresses(): string[]; - /** - * Process-exit drain: shut down every live supervisor so each - * workflow-process child is released before the host exits, while - * leaving all durable state -- deployment records and the durable - * conversation root included -- on disk for the next boot's - * `restoreWorkflowDeployments`. Distinct from `undeploy`, which - * additionally reclaims the deployment's records and scratch. - */ - shutdownAll(): Promise; - /** - * Shared teardown body. `undeploy` calls this itself, choosing - * `reclaimDirs` from `frame.reason`: `IDLE_HIBERNATE_UNDEPLOY_REASON` - * gets `false` (a state-preserving "hibernate"); every other reason - * gets the destructive `true`. This router's own idle-hibernate sweep - * calls it directly with `reclaimDirs: false`, the same flavor. Also - * exposed directly on the router surface for a caller that wants to - * choose the flavor itself. See `reclaimDirs`'s doc comment on the - * internal `teardownDeployment` for the exact split between the two - * flavors. - */ - teardownDeployment(agentAddress: string, opts: { reclaimDirs: boolean }): Promise; -} - -export function createSidecarDeployRouter(deps: { - sessions: SessionManager; - keyStore: AgentKeyStore; - transport: HubTransport; - repoStore: RepoStore; - signingKeySeed: Uint8Array; - /** - * Per-agent crypto factory. Receives the agent's raw key pair and - * returns a `CryptoProvider` bound to it (production wires - * `@intx/crypto`'s `createEd25519Crypto`). The multi-step branch - * uses this to register the spawned single-step agent's signing key on - * the host transport before `spawn()`, so the supervisor's outbound - * mail path (`MailBusBindings.sendOutbound`) signs the agent's replies - * with the AGENT's identity -- the OUTBOUND half of mailbox ownership - * (the mailbox-ownership contract). Without this registration the spawned agent's address has no - * `CryptoProvider` on the transport (nothing else registers one for - * it), and an outbound send would throw "address is not registered" - * rather than emit unsigned mail. - */ - createAgentCrypto: (keyPair: KeyPair) => CryptoProvider; - /** - * Source-admission gate: throws if a step's pinned inference source - * names a provider this sidecar cannot build. The buildable-provider - * set is sidecar config (the boot edge's adapter registry), so this - * admission control lives at the sidecar -- the hub is a different - * process and cannot know a given sidecar's providers. Production wires - * the default harness builder's `canBuildSource` verbatim, so a rejected - * provider carries the same `"... is not registered"` message. - * - * Distinct from the orchestrator's operator-approval check - * (`pickStepInferenceSource`): that gates on whether the operator - * approved a `provider:model`; this gates on whether the provider is - * buildable at all. A source can be approved yet unbuildable. - */ - assertSourceBuildable: (source: InferenceSource) => void; - /** - * Record a `(deploymentId -> agentAddress)` mapping the boot edge's - * workflow-run pack push facade consults when it must address an - * outbound pack frame. Fires once per inbound `agent.deploy` frame - * before the deployment's supervisor spawns, so the first pack push - * the child triggers sees the mapping. Tests that do not exercise - * the pack push path may pass a no-op. - */ - registerDeployment: (entry: { deploymentId: string; agentAddress: string }) => void; - /** - * Symmetric removal hook for `registerDeployment`. Fires from the - * link's `agent.undeploy` path so the boot edge's - * `DeploymentAddressRegistry` drops the mapping when the deployment - * is torn down. A subsequent stale `writeTreePreservingPrefix` - * against the dead deployment's workflow-run ref surfaces - * structurally (`registry.resolve` returns `null`) rather than - * silently resolving to the prior address. Tests that do not - * exercise the pack push path may pass a no-op. - */ - unregisterDeployment: (entry: { deploymentId: string; agentAddress: string }) => void; - /** - * Control-plane suspension sink threaded verbatim to every deployment's - * supervisor as `onSuspensionRegister`. Production wires this to the - * sidecar's hub link (`HubLink.sendSignalCorrelationRegister`) so an - * ask-rail suspension's approval snapshot reaches the hub as a - * `signal.correlation.register` frame and the hub co-writes the run's - * routing + approval rows. Optional so a test that does not exercise the - * approval-park path needs no hub-link stub; omitting it in production - * means a workflow-child ask-suspend parks with no approval ever - * registered. - */ - registerSuspension?: (registration: SuspensionRegistration) => void; - /** - * Substrate-config env keys the multi-step branch propagates into - * the workflow-process child's spawn-time env (see - * `SIDECAR_SUBSTRATE_CONFIG_KEYS` in `workflow-substrate-factory.ts`). - * The router merges `STEP_INFERENCE_SOURCES` on top per multi-step - * frame. Defaults to an empty record so a router built without - * substrate config (e.g. a test) needs no boot-edge threading. - */ - multistepSubstrateEnv?: Record; - /** - * Subprocess spawner the multi-step branch hands to the supervisor. - * Defaults to the production `Bun.spawn`-backed - * `defaultSubprocessSpawner`; tests inject a deterministic mock. - */ - multistepSubprocessSpawner?: SubprocessSpawner; - /** - * Optional override for the resolved `bin/workflow-child` path the - * multi-step branch hands to the supervisor. Production wiring uses - * the package-local default; tests inject a sentinel value so the - * mock spawner can assert on it. - */ - multistepBinaryPath?: string; - /** - * Callback the supervisor invokes for every verified InferenceEvent - * the workflow-process child publishes. The router threads the - * deployment's agent address plus the deploy's session id through to - * the callback so a downstream fan-out can route each event to the - * hub timeline keyed to the right session. The `InferenceEvent` itself - * is sessionless; the session id rides alongside it, sourced from the - * deploy frame's `HarnessConfig.sessionId` per deployment. It is - * optional because a deploy frame need not carry a session id (a - * headless deployment with no hub-side session); the sink decides what - * an absent session id means. Defaults to a no-op; production wiring - * supplies the event publisher. - */ - publishWorkflowInferenceEvent?: ( - agentAddress: string, - event: InferenceEvent, - sessionId: string | undefined, - childRunId?: string, - ) => void; - /** - * Optional override for the multi-step branch's per-step mail-address - * derivation. Defaults to `${deploymentId}-${stepId}@` - * derived from the frame's agent address. Tests inject a deterministic - * factory. - */ - multistepDeriveStepAddress?: DeriveStepAddress; - /** - * Per-deployment-address mail handler registry the hub-link's - * `mail.inbound` path consults before falling back to the legacy - * session-routed delivery. The multi-step branch registers - * `wired.routeInbound` against the deployment's mail address once - * `supervisor.spawn` succeeds so inbound mail aimed at the - * deployment address flows into the supervisor's mail-bus - * subscription. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end mail loop can omit the binding; an absent registry - * simply means multi-step inbound mail cannot route through the - * hub-link until the wiring is plumbed. - */ - multistepMailRouter?: MultistepMailRouter; - /** - * Per-deployment-address signal handler registry the sidecar - * hub-link's `signal.deliver` path consults. The multi-step branch - * registers `wired.supervisor.deliverSignal` against the deployment's - * mail address once `supervisor.spawn` succeeds so a hub-side - * `signal.deliver` frame flows into the workflow-process child via - * the IPC's `signal.deliver` payload. The child commits the - * resulting `SignalReceived` event through its own substrate, - * preserving the workflow-run repo's single-writer invariant on the - * sidecar side. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end signal loop can omit the binding; an absent registry - * means hub-side signals cannot route through the hub-link until the - * wiring is plumbed. - */ - multistepSignalRouter?: MultistepSignalRouter; - /** - * Per-deployment-address drain handler registry the sidecar - * hub-link's `drain.deliver` path consults. The multi-step branch - * registers `wired.supervisor.drain` against the deployment's mail - * address once `supervisor.spawn` succeeds so a hub-side - * `drain.deliver` frame flows into the workflow-process child via - * the IPC's `drain` payload and arms the supervisor's per-run - * `drainTimeout` accumulators. Cancel-mode in-flight steps abort on - * the child side; wait-mode steps continue. Accumulators commit a - * signed `CancelRequested{origin: "supervisor-drain"}` against the - * workflow-run repo when the deadline expires. - * - * Optional so tests that exercise the multi-step branch without an - * end-to-end drain loop can omit the binding; an absent registry - * means hub-side drain frames cannot route through the hub-link until - * the wiring is plumbed. - */ - multistepDrainRouter?: MultistepDrainRouter; - /** - * Per-deployment-address grants handler registry the sidecar - * hub-link's `run.grants` path consults. Both deploy branches - * register a handler against the deployment's mail address once - * `supervisor.spawn` succeeds so a hub-side `run.grants` frame writes - * the run's grants to `runs//grants.json` in the deployment's - * workflow-run repo -- durable next to the run's events, and shipped - * to the hub with the repo's pack flow. - * - * Optional so tests that exercise deploys without a grants loop can - * omit the binding; an absent registry means inbound `run.grants` - * frames cannot route through the hub-link until the wiring is - * plumbed. - */ - multistepGrantsRouter?: MultistepGrantsRouter; - /** - * Per-deployment-address sources-rotation handler registry. Only a - * single-step warm deployment registers a handler (against the - * deployment's mail address once `supervisor.spawn` succeeds) so a - * rotation resolved for its address flows into - * `wired.supervisor.deliverSources` and on to the child's warm agent. A - * multi-step deployment registers none -- it has no single warm agent to - * rotate -- so `tryRoute` reports its address as unrouted. - * - * Optional so tests that exercise deploys without a rotation loop can - * omit the binding; an absent registry means no rotation handler is - * installed for any deployment. - */ - multistepSourcesRouter?: MultistepSourcesRouter; - /** - * Per-deployment-address credential-delivery handler registry the sidecar - * hub-link's `credentials.update` path consults. Registered for EVERY - * deployment (not only warm single-step ones) once `supervisor.spawn` - * succeeds -- the material cell is per-child and read by every step's - * tool capabilities -- so a hub-side `credentials.update` frame dispatches - * through the supervisor's `deliverCredentials`, refreshing the child's - * material cell without ever persisting the secret to disk. - * - * Optional so tests that exercise deploys without a credential-rotation - * loop can omit the binding; an absent registry means an inbound - * `credentials.update` frame is unrouted for every deployment. - */ - multistepCredentialsRouter?: MultistepCredentialsRouter; - /** - * Optional per-message dispatch-timing observer the multi-step branch - * forwards to each supervisor it constructs. Resolved at the sidecar - * boot edge from a benchmark env gate; absent in ordinary - * production. The supervisor runs in this sidecar subprocess, - * so the observer sees both ends of the per-message IPC round-trip in - * one process and can emit a parseable timing line the benchmark - * harness reads off the subprocess's output stream. - */ - onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** - * forced-repack A/B toggle the multi-step branch forwards to - * each supervisor it constructs. Resolved at the sidecar boot edge from - * the same benchmark env gate; absent in ordinary production. - */ - repackEveryMessages?: { everyMessages: number }; - /** - * Consumed-dedup retention horizon (ms) forwarded to every supervisor - * the router constructs. The sidecar boot edge resolves the operator's - * `CONSUMED_RETENTION_MS` config; absent, the supervisor applies - * `DEFAULT_CONSUMED_RETENTION_MS` (24h). See the workflow-run kind - * handler for the operator-owned horizon invariant. - */ - consumedRetentionMs?: number; - /** - * Spawn ready-handshake timeout (ms) forwarded to every supervisor the - * router constructs. The sidecar boot edge resolves the operator's - * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies - * `DEFAULT_READY_TIMEOUT_MS` (30s). A child that spawns but never - * signals ready is killed and its spawn rejected rather than hanging - * the deploy or boot-time restore. - */ - readyTimeoutMs?: number; - /** - * Ceiling (ms) on a single boot-time `restoreDeploymentFromRecord` - * attempt; see `RESTORE_ATTEMPT_TIMEOUT_MS`'s doc comment for what the - * bound protects. Defaults to `RESTORE_ATTEMPT_TIMEOUT_MS`; overridable - * so a test can exercise the timeout-and-late-settle path in - * milliseconds instead of the real 30s ceiling. - */ - restoreAttemptTimeoutMs?: number; - /** - * Deployment-record writer, injectable so a test can block or fail the - * persist at a controlled point -- the natural seam for exercising a - * recycle that interleaves the source-rotation persist window. Defaults - * to the real `writeWorkflowDeploymentRecord`; production never overrides - * it. - */ - writeWorkflowDeploymentRecord?: typeof writeWorkflowDeploymentRecord; - /** - * Closure materializer, injectable so a test can stand in for the real - * fetch + SRI-verify + layout + evaluate pass without publishing a package. - * Defaults to the real `materializeDeploymentClosure`; production never - * overrides it. - */ - materializeDeploymentClosure?: typeof materializeDeploymentClosure; - /** - * How long a deployment may sit with no observed inbound mail before - * this router's own sweep hibernates it (`teardownDeployment` with - * `reclaimDirs: false`, tagged `IDLE_HIBERNATE_UNDEPLOY_REASON`). - * CL-8187: idle hibernation moved from a hub-driven decision to this - * router's own — the boot edge threads the operator's - * `CHAT_IDLE_HIBERNATE_MS` here. Undefined (the default) disables the - * sweep entirely: nothing is tracked, no interval runs, matching the - * behavior before this option existed. - */ - idleHibernateMs?: number; - /** - * How often the idle-hibernate sweep runs. Defaults to half - * `idleHibernateMs`, floored at 5s. Ignored when `idleHibernateMs` is - * unset. - */ - idleHibernateSweepIntervalMs?: number; -}): SidecarDeployRouter { - // Validate the signing seed at construction so a malformed key fails - // sidecar boot rather than the first multi-step deploy, where the - // public key is derived from it (`derivePrincipalPublicKeyHex`). The - // seed also signs every workflow-run event via the supervisor. - if (deps.signingKeySeed.length !== 32) { - throw new Error( - `sidecar deploy router: Ed25519 signing seed must be 32 bytes, got ${deps.signingKeySeed.length}`, - ); - } - const publishInferenceEvent = - deps.publishWorkflowInferenceEvent ?? - (( - _address: string, - _event: InferenceEvent, - _sessionId: string | undefined, - _childRunId?: string, - ): void => { - /* no-op default: tests and production-without-a-publisher - deployments do not consume events. */ - }); - const multistepSubstrateEnv = deps.multistepSubstrateEnv ?? {}; - // Sidecar data dir the deployment's per-step scratch is rooted under - // (`/workflow-step-state//...`). Resolved once - // from the boot-edge substrate env so the undeploy hook can reclaim - // the whole subtree. Absent only when the router is wired without - // substrate config (a test that never spawns a child), in which case - // no child ever rooted scratch and the undeploy reclaim is correctly - // skipped. - const stepStateDataDir = multistepSubstrateEnv.SIDECAR_DATA_DIR; - const persistDeploymentRecord = - deps.writeWorkflowDeploymentRecord ?? writeWorkflowDeploymentRecord; - const applyClosure = deps.materializeDeploymentClosure ?? materializeDeploymentClosure; - const restoreAttemptTimeoutMs = deps.restoreAttemptTimeoutMs ?? RESTORE_ATTEMPT_TIMEOUT_MS; - const multistepSpawner = deps.multistepSubprocessSpawner ?? defaultSubprocessSpawner; - const multistepDeriveStepAddress: DeriveStepAddress = - deps.multistepDeriveStepAddress ?? (({ runId, stepId }) => `${runId}-${stepId}`); - - // Per-deployment supervisor tracking. The multi-step branch - // constructs one `SidecarWorkflowSupervisor` per `agent.deploy` - // frame; the supervisor owns the workflow-process child, its IPC - // pipes, and its event-channel fd. The undeploy hook consults this - // map to call `supervisor.shutdown()` so the child's lifetime ends - // with the deployment. - const activeSupervisors = new Map(); - - // CL-8187: per-deployment last-activity clock for the idle-hibernate - // sweep below. Stamped at spawn (a freshly deployed run starts - // "active") and on every inbound mail frame the multistep mail router - // dispatches -- the same choke point every triggered turn passes - // through. An address with no entry (never spawned by this process, - // or already torn down) is never swept. - const lastActivityMs = new Map(); - - function recordActivity(agentAddress: string): void { - lastActivityMs.set(agentAddress, Date.now()); - } - - let idleHibernateInterval: ReturnType | undefined; - if (deps.idleHibernateMs !== undefined) { - const idleHibernateMs = deps.idleHibernateMs; - const sweepIntervalMs = - deps.idleHibernateSweepIntervalMs ?? Math.max(idleHibernateMs / 2, 5_000); - idleHibernateInterval = setInterval(() => { - void (async () => { - const now = Date.now(); - for (const agentAddress of [...activeSupervisors.keys()]) { - const last = lastActivityMs.get(agentAddress); - if (last === undefined || now - last < idleHibernateMs) continue; - try { - await teardownDeployment(agentAddress, { reclaimDirs: false }); - logger.info`idle-hibernate sweep: torn down ${agentAddress} after ${String(idleHibernateMs)}ms of inactivity`; - } catch (cause) { - reportError(cause, { - operation: "workflow-host-wiring.idleHibernateSweep", - agentId: agentAddress, - }); - } - } - })(); - }, sweepIntervalMs); - idleHibernateInterval.unref?.(); - } - - // CL-7215: per-deployment FIFO serialization for the boot-restore - // failure record. Three independent writers can touch the same on-disk - // `deployment.json` around a timed-out restore -- the boot loop's own - // catch (`recordWorkflowDeploymentRestoreFailure`), if the abandoned - // restore later finishes anyway `restoreDeploymentFromRecord`'s - // late-settle correction (`clearWorkflowDeploymentRestoreFailure`), and - // `teardownDeployment`'s reclaim/park writes if an operator undeploys - // or hibernates the address while a restore is still dangling -- and - // without serialization a correction's disk READ can land before the - // catch's disk WRITE, observe nothing to clear, and no-op, leaving the - // catch's later write to permanently mismark a live deployment as - // failed (or, symmetrically, the catch's write can land after a - // reclaiming teardown deleted the record, resurrecting it with a false - // failure for a deployment that no longer exists -- - // `recordWorkflowDeploymentRestoreFailure` re-reads from disk and - // no-ops on a missing record specifically to make that resurrection - // impossible). Queuing every writer through this lock, keyed by - // deployment id, makes whichever runs last observe every prior one's - // completed write; combined with the catch consulting - // `activeSupervisors` (see below) while holding the lock, the final - // on-disk state is correct regardless of ordering. - const deploymentRecordLocks = new Map>(); - function withDeploymentRecordLock(deploymentId: string, fn: () => Promise): Promise { - const prior = deploymentRecordLocks.get(deploymentId) ?? Promise.resolve(); - const settled = prior.then(fn, fn); - deploymentRecordLocks.set( - deploymentId, - settled.then( - () => undefined, - () => undefined, - ), - ); - return settled; - } - - // Synchronous single-flight guard for the deploy path. The real supervisor - // does not exist until inside `spawnWorkflowDeployment`, so `deployMultiStep` - // cannot reserve its `activeSupervisors` slot up front; instead it records - // the address here synchronously, before its first await, and clears it in a - // finally once the deploy settles. `activeSupervisors` is populated only - // after `spawn` succeeds, so the has-check alone leaves a window in which two - // same-address frames both pass and the loser's unwind deletes the winner's - // live deployment record. This set closes that window: a second frame that - // arrives while the first is mid-deploy is rejected before it touches any - // durable state. Only the live deploy path reserves; the boot restore path - // is serial and relies on the `activeSupervisors` backstop instead. - const reservingDeployAddresses = new Set(); - - // Slug-collision tracking. `deriveDeploymentId` substitutes - // disallowed characters with `-`, which is deterministic but lossy: - // two distinct agent addresses can collapse to the same slug, and - // a collision would let the second deploy silently overwrite the - // first deploy's workflow-run repo state (the slug IS the repoId). - // This map records the first-claimer; a subsequent deploy that - // produces the same slug from a different address is rejected at - // the router before any supervisor or repo state is touched. - const slugClaims = new Map(); - - function claimSlug(deploymentId: string, agentAddress: string): void { - const existing = slugClaims.get(deploymentId); - if (existing !== undefined && existing !== agentAddress) { - throw new Error( - `deriveDeploymentId collision: agent addresses ${JSON.stringify(existing)} and ${JSON.stringify(agentAddress)} both project to deploymentId ${JSON.stringify(deploymentId)}`, - ); - } - // A same-address re-claim is a defensive no-op: the `activeSupervisors` - // guard rejects a live re-deploy before claimSlug is re-invoked, and a - // failed or undeployed deploy releases the slug first, so in practice - // `existing` is only ever undefined or a different address here. - slugClaims.set(deploymentId, agentAddress); - } - - function releaseSlug(deploymentId: string, agentAddress: string): void { - const existing = slugClaims.get(deploymentId); - if (existing === agentAddress) slugClaims.delete(deploymentId); - } - - // Reclaim a deployment address whose supervisor drove ITSELF to a terminal - // phase (crash-loop latch, channel crash, recycle failure) without an - // operator undeploy, so a redeploy of the same address succeeds without a - // manual undeploy first. Runs re-entrantly as the supervisor's own - // `onSelfTerminate` sink, so it never calls back into `wired.supervisor.*`. - // It drops only in-memory routing state: the durable deployment record and - // on-disk step state stay (the hub still believes the address is deployed, - // and a boot restore re-spawns from the record), and the deployment - // address registry is RETAINED because the supervisor's own terminal - // `RunFailed` commit -- written after this sink fires -- resolves its run - // address through it. Fully synchronous, so it cannot interleave with a - // concurrent operator undeploy of the same address. - function reclaimSelfTerminatedSupervisor(args: { - deploymentId: string; - agentAddress: string; - }): void { - if (!activeSupervisors.has(args.agentAddress)) return; - deps.multistepMailRouter?.unregister(args.agentAddress); - deps.multistepSignalRouter?.unregister(args.agentAddress); - deps.multistepDrainRouter?.unregister(args.agentAddress); - deps.multistepGrantsRouter?.unregister(args.agentAddress); - deps.multistepSourcesRouter?.unregister(args.agentAddress); - deps.multistepCredentialsRouter?.unregister(args.agentAddress); - activeSupervisors.delete(args.agentAddress); - lastActivityMs.delete(args.agentAddress); - deps.transport.unregister(args.agentAddress); - releaseSlug(args.deploymentId, args.agentAddress); - } - - /** - * The per-deployment inputs the shared spawn core needs to stand up a - * workflow deployment, independent of the live deploy frame. The live - * deploy path builds this from `frame`/`projection`; a boot-time restore - * path builds the same shape from the persisted deployment record. - */ - interface WorkflowDeploySpec { - agentAddress: string; - /** - * The runnable definition, projected to its inert wire shape. Always the - * closure evaluation (`projectLiveToInert(applied.definition)`), never a - * frame-carried inline definition -- the deploy frame carries none. Both - * the deploy path and the boot-time restore derive it the same way, from - * the materialized closure. - */ - definition: WorkflowProjectionDefinition; - sources: NonNullable["sources"]; - /** Correlates the child's inference events to the deploy's session. */ - sessionId: string | undefined; - /** - * The run's own tenant and principal, from the hub's signed deploy frame - * (`HarnessConfig`). Threaded into the child's substrate config so a step - * tool can address stock `/api/tenants/:tenantId/*` with the run bearer. - * Persisted on the deployment record so a boot-time restore reseeds the - * same pair rather than losing it. - */ - tenantId: string; - principalId: string; - /** - * Hub public key recorded at the head for deploy-pack verification and - * inbound hub-frame verification. Required for a single-step - * deployment (whose head IS the agent identity); undefined for a - * genuine multi-step deployment, which derives per-step addresses and - * records no head key. - */ - hubPublicKey: string | undefined; - /** - * The hub-approved wire hash the deploy frame carried. The child's - * `DEFINITION_HASH` is sourced from this hub authority, never a sidecar - * recompute, so a closure that no longer projects to the approved content - * cannot run. - */ - approvedWireHash: string; - /** - * Sidecar-local directory of the materialized closure. The spawn core - * threads it into the child's env so the run child evaluates the pinned - * code to a live definition. Never travels on the deploy frame and is - * never persisted -- a restore re-materializes it from `sourceRef`. - */ - closurePackageDir: string; - /** - * The source-ref pin the deployment record persists so a boot-time restore - * can re-materialize the pinned code. Its `source` carries no secret (the - * registry token resolves from env at apply time); its `closure` is frozen - * versions + SRIs. - */ - sourceRef: NonNullable["sourceRef"]; - /** - * Decrypted credential material from the deploy frame's - * `workflow.credentials`, threaded to the supervisor's - * `credentialDelivery` binding so the child's materialRef is seeded - * before the first trigger. Frame-only and never persisted (secrets - * stay off disk), so the boot-restore path rebuilds the spec without - * it and the deployment waits for the hub's `credentials.update` push. - */ - credentials: NonNullable["credentials"]; - } - - /** - * Build the durable deployment record from a spec and a source table. The - * table is a parameter (not `spec.sources`) so the deploy path writes the - * deploy-time sources while the rotation handler writes the live-rotated - * ones -- both through one shape, so a rotation persists the same record a - * boot-time restore reseeds from. - */ - function buildDeploymentRecord( - spec: WorkflowDeploySpec, - sources: WorkflowDeploymentRecord["sources"], - ): WorkflowDeploymentRecord { - // `exactOptionalPropertyTypes` rejects `{ field: undefined }` for these - // optional record fields, so an absent value must omit the key entirely - // rather than assign `undefined` to it -- hence one conditional-spread - // per optional field, folded into this single literal. - return { - version: 2 as const, - agentAddress: spec.agentAddress, - definitionId: spec.definition.id, - tenantId: spec.tenantId, - principalId: spec.principalId, - sources, - ...(spec.sessionId !== undefined ? { sessionId: spec.sessionId } : {}), - ...(spec.hubPublicKey !== undefined ? { hubPublicKey: spec.hubPublicKey } : {}), - approvedWireHash: spec.approvedWireHash, - sourceRef: spec.sourceRef, - }; - } - - /** - * The single owner of the workflow-deployment spawn sequence: construct - * the supervisor, register the single-step agent's outbound key + head - * repo + hub key, spawn the workflow-process child, then register the - * live deployment (supervisor, mail/signal/drain routers, address - * mapping). Its `try/finally` unwinds every piece of partial state if any - * step throws, so a failed spawn leaks nothing. Both the live deploy path - * and the boot-time restore path route through here so the two can never - * diverge on how a deployment is stood up. Callers materialize the - * deploy-only durable state (the source closure, step grants) before - * calling. - */ - async function spawnWorkflowDeployment(spec: WorkflowDeploySpec): Promise { - // Fail loud if this address already has a live supervisor. Both single- - // and multi-step now register on the transport, so both carry the - // `transport.register` duplicate-throw backstop; this `has()` check is the - // primary early guard that gives a clean error before that lower-level - // throw and before the `activeSupervisors.set` below could clobber the - // running deployment's handle. Both the deploy path and the boot restore - // path route through here, so this is the single transition guard against - // a double-spawn -- notably a boot restore racing a legacy restore for the - // same address. - if (activeSupervisors.has(spec.agentAddress)) { - throw new Error( - `sidecar deploy router: a supervisor is already active for ${spec.agentAddress}; refusing to spawn a second`, - ); - } - const deploymentId = deriveDeploymentId(spec.agentAddress); - - // Single-step launched-agent deploy vs. derived multi-step deploy. A - // one-step deployment keeps the deployment's own (legacy) mail address - // and its grants in the legacy agent-state repo keyed by the legacy - // instance id. A multi-step deployment derives `-` - // per step for both the mail address and the agent-state repo id. - const stepStrategy = createStepStrategy({ - legacyAddress: spec.agentAddress, - stepOrder: spec.definition.stepOrder, - multistepDeriveStepAddress, - }); - - // Unwind every piece of spawn state if any step in this block throws, - // so a failed spawn leaks no freshly-spawned workflow-process child, - // `activeSupervisors` entry, transport registration, or multistep - // router registration. (The deployment-address registration happens - // before spawn and is unwound by its own guard.) The ordering inside - // the finally is the reverse of the success-path registration order. - // The caller owns the deployment slug: it must - // claim the collision guard before any durable write and release it on - // failure, so the slug is not touched here. - let succeeded = false; - let wiredForUnwind: SidecarWorkflowSupervisor | undefined; - let supervisorRegistered = false; - let routersRegistered = false; - let agentTransportRegistered = false; - let hubKeyRecorded = false; - let deploymentRegistered = false; - try { - // The hub-approved hash the frame carried, not a sidecar recompute: the - // child re-verifies its evaluated closure against the hub's authority. - const definitionHash = spec.approvedWireHash; - - // Warm-keep is the single-step launched-agent deploy: the sole step - // IS the long-lived agent, so the child warm-keeps it across - // messages. A multi-step deploy keeps instantiate-send-teardown per - // step. Computed early because both the recycle-policy wiring below - // and the spawn opts further down key off it. - const warmKeep = spec.definition.stepOrder.length === 1; - - // Per-deployment substrate-config keys the workflow-substrate-factory - // validator requires. The boot edge's `multistepSubstrateEnv` carries - // the boot-edge constants; the definition identity, the workflow-run - // identity keys and the materialized closure dir are derived per-deploy - // here. `CLOSURE_PACKAGE_DIR` is what makes the run child EVALUATE the - // pinned code and re-verify by project-then-hash against - // `DEFINITION_HASH`. - const substrateEnv: Record = { - ...multistepSubstrateEnv, - WORKFLOW_DEFINITION_ID: spec.definition.id, - WORKFLOW_TENANT_ID: spec.tenantId, - WORKFLOW_PRINCIPAL_ID: spec.principalId, - WORKFLOW_RUN_REPO_ID: deploymentId, - WORKFLOW_RUN_REF: "refs/heads/main", - CLOSURE_PACKAGE_DIR: spec.closurePackageDir, - }; - // Live-rotatable per-step inference sources. Seeded from the deploy - // spec, then revised in place by the single-step sources-rotation - // handler below. `STEP_INFERENCE_SOURCES` is NOT in the frozen - // `substrateEnv`: it is recomputed on every spawn and recycle respawn - // via `dynamicSpawnEnv`, so a rotation survives a recycle instead of - // reverting to the deploy-time list. - let currentSources = spec.sources; - - const wiredBaseConfig = { - transport: deps.transport, - repoStore: deps.repoStore, - signingKeySeed: deps.signingKeySeed, - workflowRunRepoId: { - kind: "workflow-run" as const, - id: deploymentId, - }, - workflowRunRef: "refs/heads/main", - deploymentId, - stepCount: spec.definition.stepOrder.length, - stepOrder: spec.definition.stepOrder, - warmKeep, - deploymentMailAddress: spec.agentAddress, - deriveStepAddress: stepStrategy.deriveStepAddress, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - substrateEnv, - // Recomputed on every spawn AND recycle respawn. The rotation - // handler below revises `currentSources` in place, so a respawn - // re-serializes the current (possibly rotated) list rather than the - // frozen deploy-time value. - dynamicSpawnEnv: () => ({ - [STEP_INFERENCE_SOURCES_ENV_KEY]: JSON.stringify(currentSources), - }), - subprocessSpawner: multistepSpawner, - // `exactOptionalPropertyTypes` rejects `{ field: undefined }` for - // these optional supervisor-config fields, so an absent value must - // omit the key entirely -- hence one conditional-spread per optional - // field, folded into this single literal rather than a chain of - // named intermediate configs. - ...(deps.registerSuspension !== undefined - ? { onSuspensionRegister: deps.registerSuspension } - : {}), - onSelfTerminate: () => - reclaimSelfTerminatedSupervisor({ - deploymentId, - agentAddress: spec.agentAddress, - }), - ...(spec.credentials !== undefined ? { credentialDelivery: spec.credentials } : {}), - ...(deps.multistepBinaryPath !== undefined ? { binaryPath: deps.multistepBinaryPath } : {}), - ...(deps.onDispatchTiming !== undefined ? { onDispatchTiming: deps.onDispatchTiming } : {}), - ...(deps.repackEveryMessages !== undefined - ? { repackEveryMessages: deps.repackEveryMessages } - : {}), - ...(deps.consumedRetentionMs !== undefined - ? { consumedRetentionMs: deps.consumedRetentionMs } - : {}), - ...(deps.readyTimeoutMs !== undefined ? { readyTimeoutMs: deps.readyTimeoutMs } : {}), - }; - const wired = createSidecarWorkflowSupervisor(wiredBaseConfig); - - // Restore a hibernated identity directory, if this address has one - // vaulted, BEFORE the `loadOrGenerateKey` call below: it is a no-op - // for an address that was never hibernated (an ordinary fresh - // deploy), and otherwise puts the preserved keypair back on disk so - // that call loads it (`isNew: false`) instead of minting a new one. - const identityRestore = await (stepStateDataDir !== undefined - ? restoreAgentIdentity(stepStateDataDir, spec.agentAddress) - : Promise.resolve({ restored: false })); - - // OUTBOUND half of mailbox ownership: register a signing key for - // the deployment mail address on the host transport so the supervisor - // signs the deployment's outbound mail. Every step -- single- or - // multi-step -- signs its outbound sends as `spec.agentAddress` (the - // one deployment mail address; no per-step sender reaches the host - // transport), so the transport MUST hold a `CryptoProvider` for it or - // `getTransportFor(senderAddress).send` throws "not registered". - // Registration happens before `spawn()` so the address is live the - // instant the first reply routes outbound. - const { keyPair, isNew: keyIsNew } = await deps.keyStore.loadOrGenerateKey(spec.agentAddress); - // A restored vault entry that did not yield an existing on-disk key - // means the restore itself is broken (a corrupt or partial - // snapshot) -- the exact "wake silently rotates identity" failure - // this workaround exists to make impossible to ship unnoticed. - if (identityRestore.restored && keyIsNew) { - reportError( - new Error( - "restored a hibernated agent identity snapshot but " + - "loadOrGenerateKey still minted a fresh keypair; the " + - "restored snapshot was missing or corrupt key material", - ), - { - operation: "workflow-host-wiring.restoreAgentIdentity", - agentId: spec.agentAddress, - }, - ); - } - deps.transport.register(spec.agentAddress, deps.createAgentCrypto(keyPair)); - agentTransportRegistered = true; - - // The public key the deploy ack surfaces to the hub is the deployment - // address's own Ed25519 key -- the one `loadOrGenerateKey` minted above, - // which `AgentKeyStore.signChallenge(spec.agentAddress)` also signs - // reconnect challenges with. EVERY deployment acks it, single- and - // multi-step alike, so the hub can verify the reconnect ownership - // challenge for both: a single-step head records it into - // `agent_instance.publicKey`; a workflow-derived deployment records it on - // its `workflow_deployment` row. A multi-step deployment previously acked - // the supervisor principal key -- which the hub discarded and which does - // NOT match what `signChallenge` signs with -- so its address could be - // re-claimed on reconnect without proof; carrying the deployment key - // closes that. - const deploymentPublicKey = hexEncode(keyPair.publicKey); - if (spec.definition.stepOrder.length === 1) { - // A single-step workflow stages its deploy tree at the head (the - // lone step IS the head). Initialize the head's on-disk deploy-tree - // repo (idempotent) so the hub's deploy-pack push has a repo to - // apply into. The narrow `initRepo` (not `provisionAgent`) is - // deliberate: the supervised child mints its own keypair and - // persists no hub-agent config. - await deps.sessions.initRepo(spec.agentAddress); - - // Record the hub's public key at the head so the deploy-pack apply - // (and any inbound hub-signed frame) verifies against it. The - // verifier resolves the key from the in-memory key store's - // `recordHubKey` map, so a single-step deployment cannot stand up - // without it. - if (spec.hubPublicKey === undefined) { - throw new Error( - "sidecar deploy router: a single-step workflow deployment requires a hubPublicKey to record at the head; none was supplied", - ); - } - deps.keyStore.recordHubKey(spec.agentAddress, spec.hubPublicKey); - hubKeyRecorded = true; - } - - const stepOrder = [...spec.definition.stepOrder]; - const spawnOpts: SpawnOpts = { - stepOrder, - definitionHash, - warmKeep, - onInferenceEvent: (event) => { - // The event arrives HMAC-verified over the child's event channel. - // Re-narrow it to the hub's `InferenceEvent` union; a parse - // failure means upstream corruption, so drop it loudly rather - // than forwarding an unvalidated payload onto the hub timeline. - const validated = parseInferenceEvent(event); - if (validated instanceof type.errors) { - logger.warn`dropping workflow inference event for ${spec.agentAddress}: ${validated.summary}`; - return; - } - // This hook fires for the workflow's own top-level step events, - // never a spawned body's -- a body's events carry their own - // childRunId through the separate RunSuspendableChild onEvent - // seam, not this per-deployment onInferenceEvent. - publishInferenceEvent(spec.agentAddress, validated, spec.sessionId); - }, - }; - - // Record the deployment-address mapping BEFORE `spawn`, because - // `spawn` kicks off `replayProcessingToInbox`, whose workflow-run - // substrate write routes through the boot-edge pack-pushing facade and - // resolves this mapping to address the outbound pack frame. Recording - // it after `spawn` (as the other registrations below are) loses the - // race: the replay's write throws "no agent address registered" (a - // real defect masked as a swallowed best-effort warning in the - // supervisor's replay catch). Constraint ownership: the registry owns - // "address is resolvable"; the spawn path must satisfy that contract - // before the replay writes. The finally unwinds it on any failure - // between here and the end of the try. - deps.registerDeployment({ - deploymentId, - agentAddress: spec.agentAddress, - }); - deploymentRegistered = true; - - // Surface spawn-time errors structurally: a subprocess spawner that - // crashes immediately rejects here, and the caller converts the - // rejection into a structured failure frame. The supervisor is - // registered against the deployment address only after spawn succeeds, - // so a spawn-time rejection leaves the registry untouched. - await wired.supervisor.spawn(spawnOpts); - wiredForUnwind = wired; - activeSupervisors.set(spec.agentAddress, wired); - supervisorRegistered = true; - recordActivity(spec.agentAddress); - - // Bind the deployment's mail address to this supervisor's - // `routeInbound` so the hub-link dispatches inbound mail into the - // supervisor's mail-bus subscription. Registration happens after - // `spawn` succeeds so a spawn-time rejection leaves the registry - // untouched. - deps.multistepMailRouter?.register(spec.agentAddress, async (message) => { - recordActivity(spec.agentAddress); - return wired.routeInbound(message); - }); - // Register the signal-delivery handler so a hub `signal.deliver` frame - // dispatches through the supervisor's `deliverSignal`. - deps.multistepSignalRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.deliverSignal({ - runId: args.runId, - signalName: args.signalName, - signalId: args.signalId, - payload: args.payload, - }); - }); - // Register the drain handler so a hub `drain.deliver` frame dispatches - // through the supervisor's `drain`. - deps.multistepDrainRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.drain({ deadlineMs: args.deadlineMs }); - }); - // Register the grants handler so a hub `run.grants` frame writes the - // run's grants to `runs//grants.json` in the deployment's - // workflow-run repo. The `runId` selects the per-run destination; the - // step-fan-out fields are inert in that mode but the shared write - // machinery still takes them. A write failure re-throws so the - // hub-link logs the durable-write failure loudly. - deps.multistepGrantsRouter?.register(spec.agentAddress, async (args) => { - await writeStepGrants({ - repoStore: deps.repoStore, - deploymentId, - stepOrder: spec.definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: args.stepGrants, - runId: args.runId, - }); - // The per-run file now carries the (possibly overlaid) floor. Refresh - // a live child so a standing ("always") approval resolved while the - // run is running but not parked lowers its floor immediately; the - // same durable file governs the next barrier/respawn, so a skipped - // push never leaves the run under a stale floor for good. - await wired.supervisor.deliverGrants(args.runId); - }); - // Register the sources-rotation handler ONLY for a single-step warm - // deployment: it has one long-lived agent whose sources can be - // swapped in place. A multi-step deployment has no single warm agent, - // so it registers no handler and `tryRoute` reports its address as - // unrouted. - if (warmKeep) { - // A single-step deployment's source table has exactly one entry, - // keyed by the head step. Derive that key once here (the layer that - // owns the single-key invariant); `deliverSources` stays flat and - // stepId-agnostic. - const rotationStepId = spec.definition.stepOrder[0]; - if (rotationStepId === undefined) { - throw new Error("single-step deploy has no step id for sources rotation"); - } - deps.multistepSourcesRouter?.register(spec.agentAddress, async (args) => { - const rotated = { [rotationStepId]: args.sources }; - // Swap `currentSources` synchronously BEFORE the durable persist. - // `currentSources` is the process-local respawn hint the - // supervisor reads synchronously through `dynamicSpawnEnv`, so a - // recycle that interleaves the persist `await` must respawn the - // child on the SAME sources being persisted, not the stale prior - // table. The obvious inverse -- persist first, then swap -- is - // rejected: it leaves the child on the OLD sources during the - // persist window while the record has already moved to NEW, so a - // recycle there respawns stale and a restart would "correct" it, - // i.e. the running child contradicts durable intent. Swapping - // first makes the only residual disagreement child-ahead-of- - // durable on a failed persist, which the next recycle heals down - // to the rolled-back durable truth -- the benign direction. The - // wire boundary guarantees `args.sources[0]` is the default, - // which the recycle env form pins as the active source. - const prevSources = currentSources; - currentSources = rotated; - // The durable write still precedes the LIVE swap - // (`deliverSources`), preserving persist-before-externally-visible - // for state that outlives the process; only the process-local - // respawn hint moves ahead. On a failed persist, roll the hint - // back so `currentSources` and the record stay in agreement in the - // common (no interleaved recycle) failure case -- the invariant - // restart consistency depends on. Persistence lets the rotation - // survive a full sidecar restart, not just a recycle: the boot - // scan reseeds spec.sources from record.sources. Overwrites the - // deploy-time record in place. Skipped when no data dir was wired - // (a test router that never persists), matching the restore guard. - if (stepStateDataDir !== undefined) { - try { - await persistDeploymentRecord( - stepStateDataDir, - deploymentId, - buildDeploymentRecord(spec, rotated), - ); - } catch (cause) { - // Restoring unconditionally is safe because rotations for one - // deployment are serialized by the sidecar's per-connection - // inbound-frame queue: each hub frame, sources.update - // included, runs its handler to completion on that queue - // before the next frame's handler starts, so no second - // rotation is in flight whose committed table this rollback - // could clobber. This does NOT rely on the hub pacing its - // sends -- the hub dispatches sources.update fire-and-forget; - // the sidecar frame queue is the sole serializer. Parallelizing - // inbound-frame dispatch would break this rollback. - currentSources = prevSources; - throw cause; - } - } - await wired.supervisor.deliverSources({ - sources: args.sources, - defaultSource: args.defaultSource, - }); - }); - } - - // Register the credential-delivery handler for EVERY deployment (not - // only warm single-step ones): the material cell is per-child and read - // by every step's tool capabilities. The handler hands the delivery to - // the supervisor's `deliverCredentials`, which sends a - // `credentials-updated` control frame to the child where the material - // cell is swapped. No durable persist -- credential material never - // touches disk. - deps.multistepCredentialsRouter?.register(spec.agentAddress, async (args) => { - await wired.supervisor.deliverCredentials({ - delivery: args.delivery, - }); - }); - routersRegistered = true; - - succeeded = true; - return { publicKey: deploymentPublicKey }; - } finally { - if (!succeeded) { - // Unwind in reverse registration order so each step undoes state - // the success path confirmed; ordering matches the `undeploy` hook. - if (routersRegistered) { - deps.multistepMailRouter?.unregister(spec.agentAddress); - deps.multistepSignalRouter?.unregister(spec.agentAddress); - deps.multistepDrainRouter?.unregister(spec.agentAddress); - deps.multistepGrantsRouter?.unregister(spec.agentAddress); - // Unregister unconditionally: the sources handler was registered - // only for a single-step deploy, but `unregister` is a no-op for - // an address that never registered one, so a multi-step unwind - // safely calls it too. - deps.multistepSourcesRouter?.unregister(spec.agentAddress); - deps.multistepCredentialsRouter?.unregister(spec.agentAddress); - } - if (supervisorRegistered) { - activeSupervisors.delete(spec.agentAddress); - } - if (wiredForUnwind !== undefined) { - await wiredForUnwind.supervisor.shutdown().catch((cause) => { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`multi-step deploy unwind: supervisor.shutdown failed: ${message}`; - }); - } - if (agentTransportRegistered) { - // Drop the agent's transport registration so a failed deploy does - // not leave the address live with a dangling `CryptoProvider`. - deps.transport.unregister(spec.agentAddress); - } - if (hubKeyRecorded) { - // Reverse the single-step head's `recordHubKey` so a failed deploy - // leaves no in-memory hub key behind. `forgetAgent` also drops the - // agent keypair cache `loadOrGenerateKey` populated, which is safe: - // the transport registration is already unwound above, nothing reads - // that cache after unwind, and a redeploy reloads the keypair from - // disk. The on-disk deploy-tree repo `initRepo` created is - // deliberately NOT reversed. It is idempotent and the hub re-pushes - // the deploy pack on every redeploy, so it is benign residue; and - // decisively, the durable Ed25519 identity keypair lives inside that - // same directory (`keys/` nests under the agent repo dir), so - // removing the repo would destroy an identity a rerouted head must - // keep across a failed redeploy. - deps.keyStore.forgetAgent(spec.agentAddress); - } - if (deploymentRegistered) { - // Reverse the pre-spawn `registerDeployment`: drop the address - // mapping so a failed spawn leaves the boot-edge registry as it - // found it. Registered first (before spawn), unwound last. A - // subsequent stale workflow-run write for the dead deployment then - // surfaces structurally (`registry.resolve` returns null) rather - // than resolving to the address of a deployment that never came up. - deps.unregisterDeployment({ - deploymentId, - agentAddress: spec.agentAddress, - }); - } - } - } - } - - /** - * Provision one step of a multi-step deploy WITHOUT spawning. The hub - * stages each step's deploy tree before firing the deployment-level - * workflow frame; a full-closure deploy pack still needs an initialized - * agent-state repo to apply into and the hub key recorded to verify the - * pack commit signature. This does exactly those two things -- the same - * harness-free `initRepo` + `recordHubKey` seam the single-step head uses - * -- and constructs no supervisor or child. The deployment-level workflow - * frame (fired once after every step is provisioned) spawns the child, - * which reads each step's staged deploy tree from disk. - * - * Returns the sidecar's principal public key so the link's - * `agent.deploy.ack` carries a key, matching the multi-step ack. A - * per-step address is workflow-derived and records no `agent_instance` - * key, so the hub discards this value. - */ - async function provisionStep(frame: AgentDeployFrame): Promise { - await deps.sessions.initRepo(frame.agentAddress); - deps.keyStore.recordHubKey(frame.agentAddress, frame.hubPublicKey); - return { - publicKey: await derivePrincipalPublicKeyHex(deps.signingKeySeed), - }; - } - - async function deployMultiStep( - frame: AgentDeployFrame, - projection: NonNullable, - ): Promise { - // A re-deploy of an address with a live supervisor acks idempotently, - // BEFORE touching any durable state: the resident deployment already - // owns the address, its persisted key is what reconnect challenges - // sign with, and nothing below may overwrite its repo state. Acking - // (rather than rejecting) matters because the hub's deploy-reject - // handler unroutes the address — a wake racing a hub-restart - // reconnect would otherwise tear down the just-verified route and - // every later wake would re-trip this same guard, a permanent - // wake/reject loop against a run the sidecar never stopped. - if (activeSupervisors.has(frame.agentAddress)) { - const { keyPair } = await deps.keyStore.loadOrGenerateKey(frame.agentAddress); - return { publicKey: hexEncode(keyPair.publicKey) }; - } - // A deploy still in flight for the address stays a hard reject: two - // concurrent frames racing the same address must have one loser, or - // the loser's cleanup deletes the winner's live record. The - // reservation set closes the window `activeSupervisors` alone leaves - // (it is populated only after `spawn` succeeds). A re-deploy after - // `undeploy` passes: `undeploy` drops the `activeSupervisors` entry, - // and a failed or completed deploy has already cleared its - // reservation. - if (reservingDeployAddresses.has(frame.agentAddress)) { - throw new Error( - `sidecar deploy router: a deploy is already in flight for ${frame.agentAddress}`, - ); - } - - const deploymentId = deriveDeploymentId(frame.agentAddress); - - // Claim the deployment slug BEFORE any durable write so a colliding - // deploymentId (two distinct addresses projecting to the same slug) is - // rejected before the closure, the step grants, or the supervisor touch - // disk -- the router's "no repo state touched before rejection" - // guarantee. The claim is released on any failure below; a successful - // deploy keeps it (the undeploy hook releases it at teardown). The spawn - // core owns unwinding the supervisor and registrations it stands up; the - // slug is the caller's. - // - // Resolve the sidecar data dir once: the deployment record, the - // materialized closure, and the per-step scratch all root under it. - const dataDir = stepStateDataDir; - if (typeof dataDir !== "string" || dataDir.length === 0) { - throw new Error( - "sidecar deploy router: SIDECAR_DATA_DIR must be present in the multi-step substrate env; the deployment record and workflow-process child root under it", - ); - } - - claimSlug(deploymentId, frame.agentAddress); - // Hold the single-flight reservation across the async body below and clear - // it in the finally. Everything above is synchronous and throws before any - // durable write, so the reservation is only needed from the first await - // here onward; the top-of-method guard already consults this set for a - // concurrent frame, and claimSlug/deploymentId derivation above cannot - // yield control before this point. - reservingDeployAddresses.add(frame.agentAddress); - try { - // Source-ref apply -- the only deploy lineage. Materialize EXACTLY the - // hub's frozen dependency closure and evaluate the PINNED CODE to the - // workflow definition; the frame carries no inline definition to trust. - // The closure is applied byte-for-byte (concrete versions + integrity - // SRIs) and never re-resolved here. - // - // Check the frame's inline source assets out into the durable - // per-deployment store the closure materializes from, reclaiming it - // first so a redeploy drops assets no longer referenced. This runs only - // on the DEPLOY path -- restore re-reads the store this deploy persisted - // -- so the checkout lives here rather than inside the shared - // materializer. A registry-sourced pin delivers no assets and only - // clears the store. - const assetStore = deploymentSourceAssetRoot(dataDir, deploymentId); - const gitStore = deploymentSourceGitRoot(dataDir, deploymentId); - await rm(assetStore, { recursive: true, force: true }); - await rm(gitStore, { recursive: true, force: true }); - if (projection.assets !== undefined && projection.assets.length > 0) { - await materializeWorkflowAssets({ - assets: projection.assets, - closure: projection.sourceRef.closure, - assetRoot: assetStore, - gitDirRoot: gitStore, - maxAssetPayloadBytes: MAX_INLINE_ASSET_PAYLOAD_BYTES, - }); - } - // Safe to reclaim the instance dir inside the helper: this deploy is - // single-flight-guarded by the reservation above and the child is not - // yet spawned, so no live reader holds it. - const applied = await applyClosure({ - dataDir, - deploymentId, - pin: projection.sourceRef, - substrateEnv: multistepSubstrateEnv, - }); - const validatedDefinition = WorkflowProjectionDefinition( - projectLiveToInert(applied.definition), - ); - if (validatedDefinition instanceof type.errors) { - throw new Error( - `sidecar deploy router: workflow definition loaded from the frozen closure failed projection validation: ${validatedDefinition.summary}`, - ); - } - const definition: WorkflowProjectionDefinition = validatedDefinition; - - // Structural invariants the wire arktype does not cover (non-empty - // stepOrder, every stepOrder entry backed by a `steps` entry AND a - // `sources` entry), checked against the closure-derived definition -- - // the frame carries none to cover. Mirrors the restore path. - validateWorkflowProjection({ definition, sources: projection.sources }); - - // Source-admission gate: reject a deploy where any step pins an - // inference provider this sidecar cannot build. Every source in a step's - // failover chain must be buildable -- a chain with an unbuildable tail - // would fail only after the reactor failed over onto it -- so this - // iterates the whole list. The throw propagates back through the deploy - // frame so the hub's `deployWorkflow` rejects synchronously. - for (const stepId of definition.stepOrder) { - const chain = projection.sources[stepId]; - if (chain !== undefined) { - for (const source of chain) deps.assertSourceBuildable(source); - } - } - - // Single-step launched-agent deploy vs. derived multi-step deploy. A - // one-step definition keeps the deployment's own mail address and its - // grants in the agent-state repo keyed by the instance id; a multi-step - // definition derives `-` per step for both the - // mail address and the agent-state repo id, isolating each step's - // grants in its own repo. - const stepStrategy = createStepStrategy({ - legacyAddress: frame.agentAddress, - stepOrder: definition.stepOrder, - multistepDeriveStepAddress, - }); - - // The child re-verifies its evaluated closure against the hub's approved - // hash. A frame that carried none has no anchor to verify against, so - // fail closed rather than substitute a sidecar recompute. - if (projection.approvedWireHash === undefined) { - throw new Error( - `sidecar deploy router: the deploy frame for ${frame.agentAddress} carries no approvedWireHash; the child has no hub-approved anchor to re-verify the evaluated closure against`, - ); - } - - const spec: WorkflowDeploySpec = { - agentAddress: frame.agentAddress, - definition, - sources: projection.sources, - sessionId: frame.config.sessionId, - tenantId: frame.config.tenantId, - principalId: frame.config.principalId, - hubPublicKey: definition.stepOrder.length === 1 ? frame.hubPublicKey : undefined, - approvedWireHash: projection.approvedWireHash, - closurePackageDir: applied.packageDir, - sourceRef: projection.sourceRef, - credentials: projection.credentials, - }; - const record = buildDeploymentRecord(spec, spec.sources); - - // Persist the deployment record BEFORE the spawn so a crash mid-spawn - // leaves a record the boot scan re-drives (an idempotent re-spawn; the - // child's in-flight-run discovery resumes any run). A soft-failed deploy - // deletes it below, so only a crash-interrupted deploy leaves one. - await persistDeploymentRecord(dataDir, deploymentId, record); - - // Materialize each extracted onTrigger section body's per-step inference - // sources. The body DEFINITION is not staged: the run child resolves each - // body in-memory from the parent's re-verified closure and hard-fails - // rather than reading a body definition off disk. The sources ride on - // disk (not through env) because the body child is in-process and loses - // its env across a restart. - for (const referenced of projection.referencedDefinitions ?? []) { - await materializeWorkflowSources(dataDir, referenced.definition.id, referenced.sources); - } - - // Grants bridge: the spawned child does not see the frame; it reads - // each step's grants out of `state/grants.json` in the step's - // agent-state repo while the supervisor assembles the - // credentialsSnapshot. Write the operator-approved - // `frame.config.grants` to the same repo the supervisor reads via - // `deriveStepRepoId`, before the spawn core, so the read sees them. - await writeStepGrants({ - repoStore: deps.repoStore, - deploymentId, - stepOrder: definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: frame.config.grants, - }); - - // Per-run grants bridge, for the same reason the per-step write - // above exists: the barrier must find the file before the run can - // read it. A single-step deploy is self-anchored -- its run id IS - // the address's instance id -- and `spawn` replays any mail already - // sitting in the inbox, which births that run immediately. The - // hub's own `run.grants` frame cannot win that race: it is sent - // only after the deploy ack returns, so a wake with mail pending - // reached `onRunStart` with no grants file and failed the run - // closed. Writing the operator-approved `frame.config.grants` here - // -- the same set the frame carries for a self-anchored run -- - // closes it; the hub's later frame rewrites identical bytes, and - // `writeTree` without a `clearPrefix` is purely additive, so it - // never disturbs the run's events. Guarded on `isRunAddress`: only - // a run address names a self-anchored run id. - if (definition.stepOrder.length === 1 && isRunAddress(frame.agentAddress)) { - await writeStepGrants({ - repoStore: deps.repoStore, - deploymentId, - stepOrder: definition.stepOrder, - deriveStepRepoId: stepStrategy.deriveStepRepoId, - grants: frame.config.grants, - runId: parseAgentId(frame.agentAddress), - }); - } - - // Hand off to the shared spawn core. - return await spawnWorkflowDeployment(spec); - } catch (cause) { - // Soft failure (this process survived, the deploy threw): drop the - // record and release the slug so the failed deploy is neither restored - // nor leaks its slug. The record delete must not mask the real deploy - // error or skip releasing the slug: a rejecting delete is logged (the - // orphaned record is a durable-state leak the next boot scan re-drives) - // but `cause` is still what propagates and the slug is still released. - try { - await deleteWorkflowDeploymentRecord(dataDir, deploymentId); - } catch (cleanupError) { - const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); - logger.error`deploy cleanup: deleteWorkflowDeploymentRecord failed for ${deploymentId}: ${message}`; - } - releaseSlug(deploymentId, frame.agentAddress); - throw cause; - } finally { - // Release the single-flight reservation whether the deploy succeeded or - // threw. On success the address is now in `activeSupervisors`, which the - // guard also consults, so a later re-deploy is still rejected. - reservingDeployAddresses.delete(frame.agentAddress); - } - } - - /** - * Whether a closure-materialization failure is an ENOENT reading the - * workflow package's `package.json` (or a `package.json`-shaped read - * inside the staged closure) -- the CL-6640 crash-loop shape: - * `applyFrozenWorkflowClosure` stages fresh into a new deploy-id - * directory on every call and immediately reads back from that same - * directory in the same `await` chain, so a missing file there is never - * this boot's timing -- it is corrupt/incomplete PERSISTED input (the - * tarball cache, or a durable source-asset checkout) that reproduces - * identically on every future retry. `readPackageJSON` - * (`@intx/workflow-host`) wraps the raw ENOENT in a describing `Error` - * with `{ cause }`, so both the direct code and the wrapped cause's code - * are checked. - */ - function isMissingClosureStagingFailure(cause: unknown): boolean { - if (!(cause instanceof Error)) return false; - if (isErrnoNotFound(cause)) return true; - return "cause" in cause && isErrnoNotFound((cause as { cause?: unknown }).cause); - } - - /** - * Re-establish one persisted deployment from its on-disk record -- the - * shared core of the boot-time restore loop and the CL-5477 idle-reap - * wake path. Applies exactly the gates the live deploy path applies - * (address integrity, wire arktype, tool-metadata-equivalent structural - * projection, source admission). Every failure throws so the caller's - * failure-accounting (`recordWorkflowDeploymentRestoreFailure`) sees it; - * the record is never deleted here. A failure that is intrinsic to the - * record's own persisted bytes -- deterministic, will recur identically - * on every future boot -- throws a `WorkflowRestoreFailure("permanent", - * ...)`: address-derivation mismatch, closure-derived-definition - * validation, and a missing/incomplete closure staging directory - * (CL-6640 -- see `isMissingClosureStagingFailure`) are the only such - * gates below. Every other failure (an unbuildable inference provider, a - * spawn-core throw) is a plain `Error`, which the caller treats as - * "transient" -- it depends on this boot's environment, not the - * record's content, and may clear on a later boot with no change to the - * record at all. - * - * Returns `"deferred-to-wake"` instead of restoring for a single-step - * ("warm-keep") deployment (CL-6648): that shape is exactly what a - * chat launch deploys, and it already has a working lazy-wake port - * (`@corbits/chat`'s `wakeByAddressBounded` -> `wakeByAddress` -> - * `prepareProvisionedDeployment`) that re-resolves - * inference sources fresh against the tenant's LIVE catalog - * on every wake -- `record.sources` is only ever a snapshot of what - * resolved at the deployment's last deploy or rotation. Restoring one - * eagerly here would instead replay that frozen snapshot forever, - * including a chain whose credential died after the freeze, with no - * later trigger to ever refresh it (a restored deployment reads as - * "already live" to every future wake check, so the self-healing wake - * path never fires for it again). Deferring leaves the address - * unroutable until its next message or routine fire, at which point the - * ordinary wake path redeploys it against a current resolution -- the - * same re-derive-at-wake property this ticket asks for, without - * inventing a new sidecar-to-hub source-resolution channel (the sidecar - * has no hub DB access and no pre-connect RPC to one; see CL-6648). A - * true multi-step workflow deployment has no such wake port today (open - * follow-up), so it keeps restoring eagerly from its frozen `sources` - * below, unchanged. - */ - async function restoreDeploymentFromRecord( - dataDir: string, - deploymentId: string, - record: WorkflowDeploymentRecord, - signal: AbortSignal, - ): Promise<"restored" | "deferred-to-wake" | "pruned"> { - // Integrity: the stored address must re-derive to its own directory - // name. A mismatch means a corrupt or misplaced record -- permanent, - // since re-deriving the same address on a later boot yields the same - // mismatch every time. - const derived = deriveDeploymentId(record.agentAddress); - if (derived !== deploymentId) { - throw new WorkflowRestoreFailure( - "permanent", - `${record.agentAddress} derives slug ${derived}, not its directory ${deploymentId}`, - ); - } - - // A record whose address the platform's own parser rejects is - // permanently unrestorable -- it predates the current run-address - // scheme (e.g. legacy "ins_" prefixes) and no later boot can ever - // revive it. A wake never reaches this branch (a wake only fires for - // an address the boot scan already accepted), but the boot restore - // loop and this shared core must agree, so the check lives here once. - if (!isRunAddress(record.agentAddress)) { - await deleteWorkflowDeploymentRecord(dataDir, deploymentId); - logger.info`Pruned unrestorable workflow deployment record ${deploymentId} (legacy address ${record.agentAddress})`; - return "pruned"; - } - - // Single-step ("warm-keep") deployment: defer to the wake path rather - // than eagerly restoring from a frozen `sources` snapshot. `sources` - // always carries exactly one entry per `stepOrder` id (the wire - // boundary's own invariant -- see `validateWorkflowProjection`), so a - // single entry here means a single step without needing to - // re-materialize the closure just to find out. See this function's - // doc comment for why deferring is what makes CL-6648's re-derive - // property hold without a new sidecar-to-hub RPC. - if (Object.keys(record.sources).length === 1) { - return "deferred-to-wake"; - } - - // CL-7215: the closure fetch below is real, avoidable work -- bail - // before starting it rather than spending it on an attempt the caller - // has already timed out and recorded as failed. - throwIfRestoreAborted(signal, deploymentId); - - // Reconstruct this deployment's runnable definition: re-materialize the - // pinned closure and evaluate the pinned code, then project it to the - // inert wire shape -- the SAME computation the deploy path applies. The - // closure IS the source of truth; no on-disk definition is read. The - // helper reclaims the instance dir first, safe here because the prior - // process (the only reader) is dead and restore is serial before - // `hubLink.connect()`. Asset-sourced entries read from the durable source - // store the original deploy checked out, so no re-delivery is needed; a - // store miss soft-fails the record (kept for the next boot). - let applied: Awaited>; - try { - applied = await applyClosure({ - dataDir, - deploymentId, - pin: record.sourceRef, - substrateEnv: multistepSubstrateEnv, - }); - } catch (cause) { - if (isMissingClosureStagingFailure(cause)) { - // The closure re-materializes fresh into a brand-new staging - // directory on every single restore attempt (a new deploy-id is - // minted each call; see `applyFrozenWorkflowClosure`), so an ENOENT - // reading the package it JUST staged is not this boot's - // environment -- it is a corrupt/incomplete PERSISTED input (the - // pinned closure's tarball cache entry, or the durable source-asset - // checkout `resolveDeploymentAssetMounts` already validated) that - // re-derives byte-for-byte the same broken result on every future - // boot too. Permanent, so it quarantines instead of warning forever - // while silently never making progress. - throw new WorkflowRestoreFailure( - "permanent", - cause instanceof Error ? cause.message : String(cause), - ); - } - throw cause; - } - const validatedDefinition = WorkflowProjectionDefinition( - projectLiveToInert(applied.definition), - ); - if (validatedDefinition instanceof type.errors) { - // Permanent: the closure is an immutable pin, so re-evaluating it on - // a later boot yields the identical (broken) projection every time. - throw new WorkflowRestoreFailure( - "permanent", - `workflow definition loaded from the frozen closure failed projection validation for ${record.agentAddress}: ${validatedDefinition.summary}`, - ); - } - const definition: WorkflowProjectionDefinition = validatedDefinition; - - // Structural invariants the wire arktype does not cover. The closure eval - // skips the deploy frame's coverage narrow, so this is where the - // definition-vs-sources coverage is checked. Also permanent, for the - // same reason as the projection check above: both the definition and - // `record.sources` are immutable once persisted. - try { - validateWorkflowProjection({ definition, sources: record.sources }); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new WorkflowRestoreFailure("permanent", reason); - } - - // Re-run the source-admission gate: refuse to restore a deployment - // whose pinned provider this sidecar can no longer build. Every - // source in a step's failover chain must be buildable, so this - // iterates the whole list. Transient, not permanent: the buildable- - // provider set is this boot's adapter registry, not anything the - // record itself carries, so a later boot with the provider restored - // retries and can succeed with the SAME record unchanged. The record - // is KEPT (not deleted) either way. - for (const stepId of definition.stepOrder) { - const chain = record.sources[stepId]; - if (chain !== undefined) { - for (const source of chain) { - try { - deps.assertSourceBuildable(source); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new WorkflowRestoreFailure("transient", reason); - } - } - } - } - - const spec: WorkflowDeploySpec = { - agentAddress: record.agentAddress, - definition, - sources: record.sources, - sessionId: record.sessionId, - tenantId: record.tenantId, - principalId: record.principalId, - hubPublicKey: record.hubPublicKey, - approvedWireHash: record.approvedWireHash, - closurePackageDir: applied.packageDir, - // Re-carried so a post-restore source rotation -- which rebuilds the - // record from the spec -- re-persists the pin; without this a rotation - // would silently drop it and wedge the NEXT restart. - sourceRef: record.sourceRef, - // Frame-only, never persisted: a restore (boot-time OR a wake) waits for - // the hub's next `credentials.update` push, exactly like a redeploy of a - // deployment that predates a credentials push. - credentials: undefined, - }; - - // The slug is the caller's, matching `deployMultiStep`: claim before - // the spawn, release on failure. Unlike deploy's soft-fail, restore - // does NOT delete the record and does NOT re-materialize the step grants - // or the onTrigger body sources -- both are already on disk from the - // original deploy. A failed restore's caller (the boot loop, below) - // tracks the failure on the record and always leaves the record itself - // in place for the next boot; there is deliberately no GC of the record - // FILE here (an operator reclaims it by undeploying the address) -- - // only of the ATTENTION a permanently-unrestorable one demands, via - // the quarantine threshold below. - // - // Release only a slug THIS pass newly claimed: if the address is - // already live (its slug still held by the running deployment), the - // core's double-spawn guard throws, and freeing the slug then would - // strand a live deployment's collision guard. `claimSlug` is a - // no-op for an already-held (deploymentId, address) pair, so the - // pre-claim check distinguishes the two. A PARKED deployment keeps - // its slug claimed, so a wake respawn lands in this already-held arm. - const slugNewlyClaimed = slugClaims.get(deploymentId) !== record.agentAddress; - claimSlug(deploymentId, record.agentAddress); - try { - // CL-7215: checked INSIDE this try, after the slug is claimed, so a - // cancellation here unwinds through the same catch below that - // releases it -- an abort must never leak a claimed slug. - // `spawnWorkflowDeployment` itself has no cancellation point (no - // signal reaches `@intx/workflow-host`'s `Supervisor.spawn`), so this - // is the last point this restore can still avoid starting it. - throwIfRestoreAborted(signal, deploymentId); - await spawnWorkflowDeployment(spec); - if (signal.aborted) { - // The caller's own timeout already fired -- and its catch may - // already have recorded this attempt as a boot failure -- but - // `spawnWorkflowDeployment` just registered a live supervisor for - // this address regardless (CL-7215). Reconcile the durable record - // against whatever the current on-disk truth is, under the same - // lock the boot loop's catch and `teardownDeployment` use: without - // it, this read could land BEFORE the catch's write does, observe - // nothing to correct, and no-op -- leaving the catch's write to - // permanently mismark a live deployment once it lands afterward. - // The lock guarantees whichever writer runs last observes every - // prior one's completed write. - // - // Three outcomes are possible for what this read finds: - // - a `restoreFailure` stamp: the boot loop's catch ran first and - // falsely marked this now-live deployment failed -- clear it. - // - the record itself is GONE: a reclaiming `teardownDeployment` - // ran while this spawn was in flight. `activeSupervisors` had no - // entry for this address yet at that point (this spawn had not - // resolved), so the teardown's own `wired` check missed it and - // left the routers this spawn just (re-)registered live, with no - // durable record backing them -- an orphaned live deployment, - // arguably worse than the original false-failure bug this ticket - // started from. Unwind it (below, outside the lock). - // - neither: nothing raced this restore; leave it as a normal - // successful (if late) restore. - let reclaimedDuringSpawn = false; - try { - const corrected = await withDeploymentRecordLock(deploymentId, async () => { - const onDisk = await readWorkflowDeploymentRecord(dataDir, deploymentId); - if (onDisk === undefined) { - reclaimedDuringSpawn = true; - return false; - } - if (onDisk.restoreFailure === undefined) return false; - await clearWorkflowDeploymentRestoreFailure(dataDir, deploymentId, onDisk); - return true; - }); - if (corrected) { - logger.warn`Workflow deployment ${record.agentAddress} finished restoring after its boot-restore attempt had already timed out; corrected its record so it no longer claims the deployment failed to restore`; - } - } catch (correctionError) { - reportError(correctionError, { - operation: "workflow-host-wiring.restoreDeploymentFromRecord.lateRestoreCorrection", - agentId: record.agentAddress, - }); - } - if (reclaimedDuringSpawn) { - // Called OUTSIDE the lock above: `teardownDeployment` acquires - // the same per-deployment lock itself, and this restore's own - // correction closure has already returned by this point, so - // there is no reentrant deadlock. - logger.warn`Workflow deployment ${record.agentAddress} finished spawning after being torn down while its boot-restore attempt was still in flight; unwinding the orphaned live supervisor`; - try { - await teardownDeployment(record.agentAddress, { - reclaimDirs: true, - }); - } catch (unwindError) { - reportError(unwindError, { - operation: - "workflow-host-wiring.restoreDeploymentFromRecord.unwindReclaimedDuringSpawn", - agentId: record.agentAddress, - }); - } - } - } - logger.info`Restored workflow deployment for ${record.agentAddress}`; - return "restored"; - } catch (cause) { - if (slugNewlyClaimed) { - releaseSlug(deploymentId, record.agentAddress); - } - throw cause; - } - } - - /** - * Shared teardown body for `undeploy` (`reclaimDirs: true` -- forget the - * deployment entirely) and the state-preserving "hibernate" flavor - * (`reclaimDirs: false` -- keep the deployment record and on-disk step - * state so a later `deploy` call for the same address resumes rather than - * starts fresh). The hub-side reap that calls the hibernate flavor is a - * separate lane; this function only needs to exist and be correct here. - * - * Routers come down BEFORE the supervisor's `shutdown()` so any hub-side - * frame racing the teardown is dropped at the router boundary rather than - * dispatched into a supervisor mid child-teardown. Before the child is - * killed, `wired.supervisor.drain` is given a bounded window to let an - * in-flight step settle so its workflow-run event commit (and the pack - * push that commit triggers) has a chance to land -- an unconditional - * immediate kill would drop that commit on the floor. - */ - async function teardownDeployment( - agentAddress: string, - opts: { reclaimDirs: boolean }, - ): Promise { - // Snapshot the agent's identity directory (its reconnect-challenge - // keypair) BEFORE anything else, and before this function returns: - // the published `@intx/hub-agent` package's own undeploy handling - // destroys that directory unconditionally, but only AFTER this hook - // returns -- this is a compensating workaround for CL-6239 (still - // open: the real fix is a non-destructive upstream undeploy), not a - // substitute for it. Skipped for a reclaiming (non-hibernate) - // teardown, which is expected to destroy the identity along with - // everything else. - if (!opts.reclaimDirs && stepStateDataDir !== undefined) { - await snapshotAgentIdentity(stepStateDataDir, agentAddress); - } - const deploymentId = deriveDeploymentId(agentAddress); - deps.multistepMailRouter?.unregister(agentAddress); - deps.multistepSignalRouter?.unregister(agentAddress); - deps.multistepDrainRouter?.unregister(agentAddress); - deps.multistepGrantsRouter?.unregister(agentAddress); - // Unregister unconditionally: the sources handler was registered only - // for a single-step deploy, but `unregister` is a no-op for an address - // that never registered one, so an unconditional call here is safe for - // a multi-step deployment too. - deps.multistepSourcesRouter?.unregister(agentAddress); - deps.multistepCredentialsRouter?.unregister(agentAddress); - - lastActivityMs.delete(agentAddress); - const wired = activeSupervisors.get(agentAddress); - if (wired !== undefined) { - activeSupervisors.delete(agentAddress); - await wired.supervisor - .drain({ deadlineMs: TEARDOWN_DRAIN_DEADLINE_MS }) - .catch((cause: unknown) => { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`teardownDeployment: pre-shutdown drain failed for ${agentAddress}: ${reason}`; - }); - await shutdownSupervisorWithEscalation(wired); - // Drop the deployment address's transport registration installed at - // spawn (OUTBOUND half of mailbox ownership). Both single- and - // multi-step register the deployment address for outbound signing, so - // this tears down a real registration for either; `unregister` is a - // no-op only if the spawn failed before registering, so it is safe to - // call unconditionally for any spawned deployment. A relaunch - // re-registers it fresh, whether this teardown reclaims dirs or not. - deps.transport.unregister(agentAddress); - // Reclaim the deployment's per-step local-disk scratch now that its - // supervisor + workflow-process child are torn down. Skipped for a - // hibernate: the parked run's per-step scratch (and warm workspace) - // must survive so a later relaunch resumes against it. The durable - // conversation under `agent-conversation-state/` is a DIFFERENT root - // and is deliberately NOT touched here either way -- a re-deploy on - // the same address must restore the prior conversation from it. - if (opts.reclaimDirs && stepStateDataDir !== undefined) { - await rm(pathJoin(stepStateDataDir, "workflow-step-state", deploymentId), { - recursive: true, - force: true, - }); - } - } - // Drop the deployment record so a boot-time restore does not re-spawn a - // torn-down deployment. Runs on every reclaiming teardown -- not only - // when a supervisor was active -- so a record left behind by a - // crash-interrupted deploy is reclaimed too. Skipped for a hibernate: - // the record IS the durable state a later relaunch reads to resume. - if (opts.reclaimDirs && stepStateDataDir !== undefined) { - // CL-7215: serialized through the same per-deployment lock the - // boot-restore path uses. Without it, a teardown racing a dangling - // (timed-out but still-running) restore's correction could delete - // the record out from under it, or clobber a `restoreFailure` write - // it lands mid-teardown -- neither of the boot-restore path's own - // two writers accounts for a third, unlocked writer touching the - // same file. - await withDeploymentRecordLock(deploymentId, () => - deleteWorkflowDeploymentRecord(stepStateDataDir, deploymentId), - ); - } - // Stamp the kept record as parked. This is the ONLY durable signal that - // distinguishes "the hub parked this deployment on purpose" from "this - // record is here because the process crashed or exited while the - // deployment was still live" -- both leave the record on disk with no - // marker otherwise. The boot scan reads this to REPORT parked-vs-live - // counts today; it does not yet change what it spawns (that cutover is - // CL-6282). - if (!opts.reclaimDirs && stepStateDataDir !== undefined) { - // CL-7215: same lock as above, for the same reason -- a hibernate's - // parked-stamp read-modify-write must not interleave with a dangling - // restore's own record writes for this deploymentId. - await withDeploymentRecordLock(deploymentId, () => - markWorkflowDeploymentRecordParked(stepStateDataDir, deploymentId), - ); - } - if (opts.reclaimDirs) { - releaseSlug(deploymentId, agentAddress); - } - deps.unregisterDeployment({ deploymentId, agentAddress }); - } - - return { - async deploy(frame): Promise { - if (frame.provisionStep === true) { - return await provisionStep(frame); - } - if (frame.workflow !== undefined) { - return await deployMultiStep(frame, frame.workflow); - } - // Every deploy stages through the workflow-run substrate: a - // provision-step frame primes the per-step repo, and a workflow - // frame spawns the supervised child. A frame carrying neither is - // an unsupported shape -- there is no in-process fall-through. - throw new Error( - `sidecar deploy router: unsupported deploy frame for ${frame.agentAddress}; a deploy must carry provisionStep or a workflow definition`, - ); - }, - async undeploy(frame): Promise { - // `frame.reason` rides the wire from `sendAgentUndeploy(address, - // reason)` verbatim (`AgentUndeployFrame.reason`, `@intx/types`). - // `IDLE_HIBERNATE_UNDEPLOY_REASON` (this router's own tag, also - // used by its idle-hibernate sweep above) tells "sleep it, a - // relaunch will resume it" apart from every other undeploy reason - // (channel deletion, member removal, ...), which still gets the - // destructive default. - const reclaimDirs = frame.reason !== IDLE_HIBERNATE_UNDEPLOY_REASON; - await teardownDeployment(frame.agentAddress, { reclaimDirs }); - }, - teardownDeployment, - async restoreWorkflowDeployments(): Promise { - const dataDir = stepStateDataDir; - if (dataDir === undefined) { - // No substrate config was wired (a test router that never spawns a - // child): nothing was ever persisted under this data dir, so there - // is nothing to restore. - return; - } - - const scanned = await scanWorkflowDeploymentRecords(dataDir); - // CL-6282: `parkedAt` (written by `markWorkflowDeploymentRecordParked` - // on a state-preserving hibernate teardown) is the durable, locally - // decidable answer to "is this deployment genuinely live" -- a - // deployment the hub deliberately put to sleep does not need to be - // running to be correct; it resumes the moment a message or routine - // fire addresses it again (the same wake path `ensureAwake`/`deployAtHead` - // already use for an idle-slept deployment that never crashed at all). - // Restoring it anyway at every boot is exactly the "restart dead - // stuff" cost this cutover removes. A record with no `parkedAt` - // (never hibernated, or predates the field) restores as before -- - // absent is "assume live", matching `readWorkflowDeploymentRecord`'s - // own backward-compatible read. - const { live, parked } = partitionScannedDeployments(scanned); - const alreadyQuarantinedCount = live.filter(({ record }) => - isWorkflowDeploymentRestoreQuarantined(record), - ).length; - logger.info`Boot scan found ${scanned.length} deployment record(s): ${parked.length} parked (left asleep; will wake on the next message or routine fire), ${alreadyQuarantinedCount} quarantined, ${live.length - alreadyQuarantinedCount} to attempt (single-step deployments among them defer to their wake path instead of restoring frozen sources -- CL-6648)`; - // Bounded-parallel, not fully parallel: `RESTORE_CONCURRENCY` caps how - // many workflow-process children spawn at once so a boot with many - // live deployments cannot storm the host, while still restoring far - // faster than one-at-a-time. Restore runs before `hubLink.connect()`, - // so there are no concurrent hub-driven deploys to contend with. Each - // worker handles its own quarantine skip/record/clear bookkeeping and - // logging inline (rather than via `runWithConcurrency`'s returned - // failure list) so one record's outcome never depends on another's - // ordering, matching the isolation the old serial loop's per-iteration - // `try`/`catch` gave. - let skippedQuarantinedCount = 0; - let deferredToWakeCount = 0; - await runWithConcurrency(live, RESTORE_CONCURRENCY, async ({ deploymentId, record }) => { - // Skip WITHOUT attempting: a permanently unrestorable record that - // has already crossed RESTORE_QUARANTINE_THRESHOLD gets neither a - // spawn attempt nor a per-record warning this boot -- both are - // pointless for a deterministic failure that has already been - // reported that many times. It still counts toward the one - // summary line below, and the record itself is untouched (an - // operator reclaims it by undeploying the address). - if (isWorkflowDeploymentRestoreQuarantined(record)) { - skippedQuarantinedCount += 1; - return; - } - try { - const outcome = await withRestoreTimeout( - (signal) => restoreDeploymentFromRecord(dataDir, deploymentId, record, signal), - deploymentId, - restoreAttemptTimeoutMs, - ); - if (outcome === "deferred-to-wake") { - deferredToWakeCount += 1; - return; - } - if (record.restoreFailure !== undefined) { - await clearWorkflowDeploymentRestoreFailure(dataDir, deploymentId, record); - } - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - const kind = cause instanceof WorkflowRestoreFailure ? cause.kind : "transient"; - // CL-7215: serialized against `restoreDeploymentFromRecord`'s - // own late-settle correction AND `teardownDeployment`'s record - // writes via the same lock, and gated on `activeSupervisors` -- - // in-memory, always set synchronously the instant - // `spawnWorkflowDeployment` actually succeeds, unlike a disk - // snapshot -- so a restore that timed out here but has ALREADY - // gone live by the time this runs never gets a boot failure - // recorded against it in the first place. See the lock's own - // doc comment for why every writer must go through it. - await withDeploymentRecordLock(deploymentId, async () => { - if (activeSupervisors.has(record.agentAddress)) { - logger.warn`Workflow deployment ${deploymentId} timed out during boot restore but finished spawning before its failure could be recorded; leaving its record as a live, successful restore`; - return; - } - const updated = await recordWorkflowDeploymentRestoreFailure(dataDir, deploymentId, { - kind, - reason, - }); - if (updated === undefined) { - // The record is gone -- a concurrent reclaiming teardown - // won the lock first and deleted it. Nothing to mark: the - // deployment was torn down on purpose, not left claiming a - // false restore failure. - return; - } - if (isWorkflowDeploymentRestoreQuarantined(updated)) { - const attempts = updated.restoreFailure?.attempts ?? 0; - logger.warn`Workflow deployment ${deploymentId} failed to restore ${attempts} consecutive times and is now quarantined -- it will not be retried again until the address is undeployed. Last failure: ${reason}`; - } else { - logger.warn`Failed to restore workflow deployment ${deploymentId}: ${reason}`; - } - }); - } - }); - if (skippedQuarantinedCount > 0) { - logger.warn`Skipped ${skippedQuarantinedCount} quarantined workflow deployment record(s) (permanent restore failures, already reported); undeploy an address to clear its record`; - } - if (deferredToWakeCount > 0) { - logger.info`Deferred ${deferredToWakeCount} single-step workflow deployment(s) to their lazy-wake path instead of restoring from a frozen sources snapshot (CL-6648); they redeploy against a current catalog resolution on their next message or routine fire`; - } - }, - async reapExpiredHibernationSnapshots(): Promise { - if (stepStateDataDir === undefined) return 0; - const { reapedEntries } = await reapVaultSnapshots(stepStateDataDir); - if (reapedEntries.length > 0) { - logger.info`Reaped ${reapedEntries.length} expired hibernated-agent-identity snapshot(s)`; - } - return reapedEntries.length; - }, - activeAddresses(): string[] { - // `activeSupervisors` holds exactly the deployments with a live - // supervisor -- the set this sidecar can currently route mail to. - return [...activeSupervisors.keys()]; - }, - async shutdownAll(): Promise { - if (idleHibernateInterval !== undefined) { - clearInterval(idleHibernateInterval); - } - // Process-exit drain, NOT an undeploy: every live supervisor is shut - // down so each workflow-process child, its IPC pipes, and its - // event-channel fd are released before the host exits, but every - // deployment record, routing registration source of truth, and the - // durable conversation root stay on disk untouched -- the next boot's - // `restoreWorkflowDeployments` re-establishes each deployment from - // them. - await Promise.all( - [...activeSupervisors.entries()].map(async ([address, wired]) => { - activeSupervisors.delete(address); - try { - await shutdownSupervisorWithEscalation(wired); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - logger.warn`Drain: supervisor shutdown for ${address} failed: ${reason}`; - } - }), - ); - }, - }; -} diff --git a/apps/sidecar/src/workflow-host-wiring/step-strategy.test.ts b/apps/sidecar/src/workflow-host-wiring/step-strategy.test.ts deleted file mode 100644 index 753764276b..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/step-strategy.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { expect, test } from "bun:test"; -import type { RepoId, RepoStore } from "@intx/hub-sessions"; - -import { writeStepGrants } from "./step-strategy"; - -type CapturedWrite = { - repoId: RepoId; - ref: string; - files: Record; -}; - -function makeCapturingRepoStore(): { - store: RepoStore; - writes: CapturedWrite[]; -} { - const writes: CapturedWrite[] = []; - const store = { - writeTree: ( - _principal: unknown, - repoId: RepoId, - ref: string, - args: { files: Record }, - ) => { - writes.push({ repoId, ref, files: args.files }); - return Promise.resolve("sha"); - }, - } as unknown as RepoStore; - return { store, writes }; -} - -test("per-run mode writes runs//grants.json into the workflow-run repo", async () => { - const { store, writes } = makeCapturingRepoStore(); - await writeStepGrants({ - repoStore: store, - deploymentId: "dep-1", - stepOrder: ["step-1", "step-2"], - deriveStepRepoId: ({ runId, stepId }) => ({ - kind: "agent-state", - id: `${runId}-${stepId}`, - }), - grants: [{ resource: "tool:echo", action: "invoke" }], - runId: "run-1", - }); - expect(writes).toHaveLength(1); - const write = writes[0]; - expect(write?.repoId).toEqual({ kind: "workflow-run", id: "dep-1" }); - expect(Object.keys(write?.files ?? {})).toEqual(["runs/run-1/grants.json"]); - expect(JSON.parse(String(write?.files["runs/run-1/grants.json"]))).toEqual({ - grants: [{ resource: "tool:echo", action: "invoke" }], - }); -}); - -test("step fan-out mode writes one agent-state grants file per step", async () => { - const { store, writes } = makeCapturingRepoStore(); - await writeStepGrants({ - repoStore: store, - deploymentId: "dep-1", - stepOrder: ["step-1", "step-2"], - deriveStepRepoId: ({ runId, stepId }) => ({ - kind: "agent-state", - id: `${runId}-${stepId}`, - }), - grants: undefined, - }); - expect(writes.map((w) => w.repoId)).toEqual([ - { kind: "agent-state", id: "dep-1-step-1" }, - { kind: "agent-state", id: "dep-1-step-2" }, - ]); - for (const write of writes) { - expect(JSON.parse(String(write.files["state/grants.json"]))).toEqual({ - grants: [], - }); - } -}); diff --git a/apps/sidecar/src/workflow-host-wiring/step-strategy.ts b/apps/sidecar/src/workflow-host-wiring/step-strategy.ts deleted file mode 100644 index bad44fabec..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/step-strategy.ts +++ /dev/null @@ -1,173 +0,0 @@ -// Per-deploy step address/repo strategy and grants writing: the -// deployment-id derivation, the single-step vs. multi-step choice of -// per-step mail address and agent-state repo id, and the pre-spawn -// write of every step's grants into its agent-state repo. - -import { parseAgentId, type Principal, type RepoStore } from "@intx/hub-sessions"; -import { - STEP_GRANTS_PATH, - STEP_GRANTS_REF, - type DeriveStepAddress, - type DeriveStepRepoId, -} from "@intx/workflow-host"; -import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy"; - -import { getLogger } from "@intx/log"; -import { runGrantsPath } from "../run-grants"; - -/** - * Project an agent address into the substrate-safe id of its - * workflow-run repo. Both deploy branches key `{ kind: "workflow-run", - * id }` by this slug, and the supervisor principal's `deploymentId` - * must equal that id for the workflow-run kind handler's authz check to - * pass. The derivation is owned by `@intx/workflow-deploy` so the hub's - * read routes reconstruct the identical id; this thin delegator keeps - * the sidecar's call sites readable while the rationale and the - * substrate `SAFE_REPO_ID` contract live with the shared function. - */ -export function deriveDeploymentId(agentAddress: string): string { - return deriveWorkflowRunRepoId(agentAddress); -} - -/** - * Hub principal the deploy router presents when it writes a step's - * grants into the agent-state repo on the sidecar's substrate. The - * agent-state kind handler gates `writeTree` as hub-only; the deploy - * router is the local stand-in for the hub on the sidecar's disk, so it - * claims the hub principal for this single bookkeeping write. The child - * reads the same repo via the working-tree path (`getRepoDir`), which is - * not authorize-gated. - */ -const GRANTS_WRITE_PRINCIPAL: Principal = { kind: "hub" }; - -/** - * Per-deploy address/repo strategy. The single-step launched-agent - * deploy and the derived multi-step deploy disagree on how the per-step - * mail address and agent-state repo id are computed; both - * `deriveStepAddress` (consumed by the supervisor's credentialsSnapshot - * assembly for the step's mail address) and `deriveStepRepoId` (consumed - * by the same assembly to locate each step's grants) must agree on the - * choice, so they are minted together. - */ -export type StepStrategy = { - deriveStepAddress: DeriveStepAddress; - deriveStepRepoId: DeriveStepRepoId; -}; - -/** - * Decide the per-step address/repo strategy from the projection's step - * count. - * - * `stepOrder.length === 1` is the agent-launch identity deploy: the sole - * step IS the legacy launched agent, so its grants live in the legacy - * agent-state repo keyed by `parseAgentId(legacyAddress)`. This is - * exactly the repo the legacy agent identity keys, so the spawned child - * reads the agent's grants from where the agent's identity already - * lives, and the deployment frame's `ins_` address is preserved - * (the deploy-ack listener finds the `agent_instance` row, the - * workflow-run repo stays keyed by `deriveWorkflowRunRepoId(legacy)`). - * - * Any other step count is a derived multi-step deploy: each step gets a - * derived `-` mail address (via the router's - * `multistepDeriveStepAddress`) and a derived agent-state repo under the - * default `-` convention. - * - * NOTE: the supervisor's `deriveStepAddress` feeds the credentials - * snapshot's per-step mail `address` and the grants-repo derivation. It - * does NOT feed the child's on-disk tool read (`stepDeployTreeDir` in - * `step-agent-tools.ts`), which re-derives the step address from the - * deployment mailbox address independently. The deploy tree must - * therefore be staged at the address `stepDeployTreeDir` computes, - * regardless of this strategy's address choice. - */ -export function createStepStrategy(args: { - legacyAddress: string; - stepOrder: readonly string[]; - multistepDeriveStepAddress: DeriveStepAddress; -}): StepStrategy { - if (args.stepOrder.length === 1) { - return { - deriveStepAddress: () => args.legacyAddress, - // `parseAgentId` is deferred into the closure rather than computed - // eagerly: the supervisor only invokes `deriveStepRepoId` while - // assembling the credentialsSnapshot inside `spawn()`, so a - // malformed address surfaces at the same point the rest of the - // spawn path would fault rather than ahead of the deploy router's - // other boundary checks. - deriveStepRepoId: () => ({ - kind: "agent-state", - id: parseAgentId(args.legacyAddress), - }), - }; - } - return { - deriveStepAddress: args.multistepDeriveStepAddress, - deriveStepRepoId: ({ runId, stepId }) => ({ - kind: "agent-state", - id: `${runId}-${stepId}`, - }), - }; -} - -const logger = getLogger(["sidecar", "step-grants"]); - -/** - * Write every step's grants into its agent-state repo so the - * supervisor's `assembleCredentialsSnapshot` (invoked inside `spawn()`) - * reads them off the working tree at `STEP_GRANTS_PATH`. The on-disk - * shape is `{ grants: WireGrantRule[] }` -- the envelope - * `assembleCredentialsSnapshot` validates (`{ grants: unknown[] }`) and - * the child's `evaluateGrants` adapter narrows to `GrantRule[]`. - * - * The same `deriveStepRepoId` the supervisor reads with keys the write, - * so read and write address the same repo. The write is on the spawn - * critical path: a failure rejects the deploy (the caller's `finally` - * unwinds the partial state) rather than spawning a child that would - * fail every authorize closed against an empty grant set. - */ -export async function writeStepGrants(args: { - repoStore: RepoStore; - deploymentId: string; - stepOrder: readonly string[]; - deriveStepRepoId: DeriveStepRepoId; - grants: readonly unknown[] | undefined; - /** - * When present, selects the per-run mode: one write of - * `runs//grants.json` into the deployment's `workflow-run` repo - * (the destination the hub's `run.grants` frame targets) instead of the - * per-step agent-state fan-out. The step fan-out fields are inert in - * that mode but the shared write machinery still takes them. - */ - runId?: string; -}): Promise { - // The deploy frame's validated HarnessConfig always carries a `grants` - // array (possibly empty); an absent array means "no grants", which - // serializes to the same fail-closed empty file the snapshot expects. - // Coerce here so the on-disk envelope is always a valid `{ grants: [] }` - // rather than `{}` (which the snapshot's validator rejects). - const grants = args.grants ?? []; - const serialized = JSON.stringify({ grants }, null, 2); - if (args.runId !== undefined) { - await args.repoStore.writeTree( - GRANTS_WRITE_PRINCIPAL, - { kind: "workflow-run", id: args.deploymentId }, - STEP_GRANTS_REF, - { - files: { [runGrantsPath(args.runId)]: serialized }, - message: `Write run grants for ${args.runId}`, - }, - ); - logger.info`Wrote run grants for run ${args.runId} at ${runGrantsPath(args.runId)} in workflow-run repo ${args.deploymentId}`; - return; - } - for (const stepId of args.stepOrder) { - const repoId = args.deriveStepRepoId({ - runId: args.deploymentId, - stepId, - }); - await args.repoStore.writeTree(GRANTS_WRITE_PRINCIPAL, repoId, STEP_GRANTS_REF, { - files: { [STEP_GRANTS_PATH]: serialized }, - message: `Write step grants for ${stepId}`, - }); - } -} diff --git a/apps/sidecar/src/workflow-host-wiring/supervisor.test.ts b/apps/sidecar/src/workflow-host-wiring/supervisor.test.ts deleted file mode 100644 index 77ef5e5865..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/supervisor.test.ts +++ /dev/null @@ -1,360 +0,0 @@ -// Regression test for the ask-approval-registration wiring gap: a -// workflow-child's `ask`-effect suspend only reaches the hub as a -// `signal.correlation.register` frame (and gets an approval row) when -// `createSidecarWorkflowSupervisor` forwards its caller's -// `onSuspensionRegister` sink into `createWorkflowSupervisor`'s bindings. -// Before this fix the sink was silently dropped: the supervisor logged -// "no onSuspensionRegister sink is wired" and the suspended run parked -// forever with zero rows in the `approval` table. - -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, mock, test } from "bun:test"; -import type { RepoStore } from "@intx/hub-sessions"; - -import { runGrantsPath } from "../run-grants"; - -mock.module("@intx/workflow-host", () => ({ - createWorkflowSupervisor: mock((config: unknown) => { - capturedConfig = config; - return { - getCredentialsSnapshot: () => null, - deliverCredentials: async () => {}, - }; - }), - wrapHubTransportAsMailBus: () => ({ - routeInbound: async () => {}, - }), - hashGrants: async (grants: readonly unknown[]) => `stub-hash:${JSON.stringify(grants)}`, -})); - -let capturedConfig: unknown; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeRepoStore(): Promise<{ store: RepoStore; dir: string }> { - const dir = await mkdtemp(path.join(tmpdir(), "supervisor-onrunstart-")); - tempDirs.push(dir); - const store = { - getRepoDir: () => dir, - } as unknown as RepoStore; - return { store, dir }; -} - -const { createSidecarWorkflowSupervisor, DEFAULT_MAX_GRANTS_AGE_MS } = await import("./supervisor"); - -test("createSidecarWorkflowSupervisor forwards onSuspensionRegister to the workflow-host supervisor", () => { - const registerSuspension = mock(() => {}); - - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-1" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-1", - stepCount: 1, - deploymentMailAddress: "dep-1@local", - deriveStepAddress: () => "dep-1-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - onSuspensionRegister: registerSuspension, - }); - - expect(capturedConfig).toMatchObject({ - onSuspensionRegister: registerSuspension, - }); -}); - -test("createSidecarWorkflowSupervisor omits onSuspensionRegister when the caller supplies none", () => { - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-2" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-2", - stepCount: 1, - deploymentMailAddress: "dep-2@local", - deriveStepAddress: () => "dep-2-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - }); - - expect( - (capturedConfig as { onSuspensionRegister?: unknown }).onSuspensionRegister, - ).toBeUndefined(); -}); - -// Same wiring-gap class as onSuspensionRegister above (CL: MCP live proof): -// the hub delivers decrypted credential material on the deploy frame's -// `workflow.credentials`, and the workflow-host supervisor forwards its -// `credentialDelivery` binding to the child on the pre-trigger barrier — -// but before this fix `createSidecarWorkflowSupervisor` never accepted the -// field, so the child's materialRef stayed null and every -// `credentials.resolve("mcp.")` failed "no credential is bound". -test("createSidecarWorkflowSupervisor forwards credentialDelivery to the workflow-host supervisor", () => { - const delivery = { - bindings: [ - { - handle: "mcp.exa", - credentialId: "cred_1", - consumer: "tool:@corbits/mcp-tools", - }, - ], - materials: [ - { - credentialId: "cred_1", - providerKey: "http", - origin: "https://mcp.exa.ai/mcp", - secret: "unauthenticated-mcp-server", - }, - ], - }; - - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-3" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-3", - stepCount: 1, - deploymentMailAddress: "dep-3@local", - deriveStepAddress: () => "dep-3-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - credentialDelivery: delivery, - }); - - expect(capturedConfig).toMatchObject({ credentialDelivery: delivery }); -}); - -test("createSidecarWorkflowSupervisor omits credentialDelivery when the deploy carries none", () => { - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-4" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-4", - stepCount: 1, - deploymentMailAddress: "dep-4@local", - deriveStepAddress: () => "dep-4-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - }); - - expect((capturedConfig as { credentialDelivery?: unknown }).credentialDelivery).toBeUndefined(); -}); - -// CL-6242: the vendor's recycle policy (`RecyclePolicyBounds` / -// `readRssBytes` / `readGrantsAgeMs`, `types.ts:473-499`) was never armed -// for a warm-keep deployment, so a long-lived single-step agent ran -// forever on the grants it was spawned with -- no bound ever forced a -// respawn onto fresh grants. -test("createSidecarWorkflowSupervisor arms the grants-age recycle policy for a warm-keep deployment", () => { - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-5" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-5", - stepCount: 1, - deploymentMailAddress: "dep-5@local", - deriveStepAddress: () => "dep-5-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - warmKeep: true, - }); - - expect(capturedConfig).toMatchObject({ - recyclePolicy: { maxGrantsAgeMs: DEFAULT_MAX_GRANTS_AGE_MS }, - }); - expect(typeof (capturedConfig as { readGrantsAgeMs?: unknown }).readGrantsAgeMs).toBe("function"); - // No RSS reader: see the module-level comment in `supervisor.ts` on why - // `maxRssBytes`/`readRssBytes` are never wired. - expect((capturedConfig as { readRssBytes?: unknown }).readRssBytes).toBeUndefined(); -}); - -test("createSidecarWorkflowSupervisor omits the recycle policy for a non-warm-keep deployment", () => { - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-6" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-6", - stepCount: 2, - deploymentMailAddress: "dep-6@local", - deriveStepAddress: () => "dep-6-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - warmKeep: false, - }); - - expect((capturedConfig as { recyclePolicy?: unknown }).recyclePolicy).toBeUndefined(); - expect((capturedConfig as { readGrantsAgeMs?: unknown }).readGrantsAgeMs).toBeUndefined(); -}); - -test("readGrantsAgeMs reports undefined until deliverCredentials fires, then a non-negative age", async () => { - const wired = createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: {} as never, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-7" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-7", - stepCount: 1, - deploymentMailAddress: "dep-7@local", - deriveStepAddress: () => "dep-7-step@local", - stepOrder: ["step"], - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - warmKeep: true, - }); - - const readGrantsAgeMs = (capturedConfig as { readGrantsAgeMs: () => number | undefined }) - .readGrantsAgeMs; - expect(readGrantsAgeMs()).toBeUndefined(); - - await wired.supervisor.deliverCredentials({ - delivery: { bindings: [], materials: [] }, - } as never); - - const age = readGrantsAgeMs(); - expect(age).toBeGreaterThanOrEqual(0); -}); - -// Port of upstream Interchange's `onRunStart` grants sink: the per-run -// credential/grants barrier the vendor supervisor awaits before every -// `trigger.fire`. -test("createSidecarWorkflowSupervisor wires onRunStart to assemble the run's credentialsSnapshot from its per-run grants file", async () => { - const { store, dir } = await makeRepoStore(); - const filePath = path.join(dir, runGrantsPath("run-1")); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile( - filePath, - JSON.stringify({ grants: [{ resource: "tool:echo", action: "invoke" }] }), - ); - - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: store, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-8" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-8", - stepCount: 1, - stepOrder: ["step-a"], - deploymentMailAddress: "dep-8@local", - deriveStepAddress: ({ stepId }) => `dep-8-${stepId}@local`, - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - }); - - const onRunStart = ( - capturedConfig as { - onRunStart: (args: { runId: string; anchorRunId: string }) => Promise; - } - ).onRunStart; - const snapshot = await onRunStart({ runId: "run-1", anchorRunId: "dep-8" }); - - expect(snapshot).toEqual({ - steps: [ - { - stepId: "step-a", - address: "dep-8-step-a@local", - grants: [{ resource: "tool:echo", action: "invoke" }], - contentHash: `stub-hash:${JSON.stringify([{ resource: "tool:echo", action: "invoke" }])}`, - }, - ], - }); -}); - -// Every run birth path writes `runs//grants.json` before the run -// dispatches — `deployAtHead` produces the `run.grants` frame the sidecar -// writes — so an absent file is a run that would start under-authorized, -// and the barrier fails it closed rather than inventing an empty grant set. -test("onRunStart fails a run with no per-run grants file closed", async () => { - const { store } = await makeRepoStore(); - - createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: store, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-9" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-9", - stepCount: 1, - stepOrder: ["step-a"], - deploymentMailAddress: "dep-9@local", - deriveStepAddress: ({ stepId }) => `dep-9-${stepId}@local`, - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - }); - - const onRunStart = ( - capturedConfig as { - onRunStart: (args: { runId: string; anchorRunId: string }) => Promise; - } - ).onRunStart; - - await expect(onRunStart({ runId: "run-missing", anchorRunId: "dep-9" })).rejects.toThrow( - /has no grants file; failing closed/, - ); -}); - -test("onRunStart refreshes readGrantsAgeMs, mirroring deliverCredentials's observation point", async () => { - const { store, dir } = await makeRepoStore(); - const filePath = path.join(dir, runGrantsPath("run-1")); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, JSON.stringify({ grants: [] })); - - const wired = createSidecarWorkflowSupervisor({ - transport: {} as never, - repoStore: store, - signingKeySeed: new Uint8Array(32), - workflowRunRepoId: { kind: "workflow-run", id: "dep-10" }, - workflowRunRef: "refs/heads/main", - deploymentId: "dep-10", - stepCount: 1, - stepOrder: ["step-a"], - deploymentMailAddress: "dep-10@local", - deriveStepAddress: ({ stepId }) => `dep-10-${stepId}@local`, - substrateEnv: {}, - dynamicSpawnEnv: () => ({}), - warmKeep: true, - }); - - const readGrantsAgeMs = (capturedConfig as { readGrantsAgeMs: () => number | undefined }) - .readGrantsAgeMs; - const onRunStart = ( - capturedConfig as { - onRunStart: (args: { runId: string; anchorRunId: string }) => Promise; - } - ).onRunStart; - expect(readGrantsAgeMs()).toBeUndefined(); - - await onRunStart({ runId: "run-1", anchorRunId: "dep-10" }); - - expect(readGrantsAgeMs()).toBeGreaterThanOrEqual(0); - // deliverCredentials (a rotation) still refreshes the same observation - // point independently. - await wired.supervisor.deliverCredentials({ - delivery: { bindings: [], materials: [] }, - } as never); - expect(readGrantsAgeMs()).toBeGreaterThanOrEqual(0); -}); diff --git a/apps/sidecar/src/workflow-host-wiring/supervisor.ts b/apps/sidecar/src/workflow-host-wiring/supervisor.ts deleted file mode 100644 index 8f2cf5ab22..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/supervisor.ts +++ /dev/null @@ -1,440 +0,0 @@ -// Per-deployment supervisor construction with the sidecar's bindings -// pre-wired: the hub-transport mail-bus adapter, the Ed25519 principal -// signer, the resolved `bin/workflow-child` binary, and the mail-audit -// reference derivation the supervisor stamps onto every envelope. - -import { signEd25519 } from "@intx/crypto"; -import type { HubTransport } from "@intx/mail-memory"; -import type { RepoId, RepoStore, WorkflowRunSupervisorPrincipal } from "@intx/hub-sessions"; -import { - createWorkflowSupervisor, - hashGrants, - wrapHubTransportAsMailBus, - type CredentialsSnapshot, - type CredentialsSnapshotStep, - type DeriveStepAddress, - type DeriveStepRepoId, - type DispatchTimingMark, - type HubTransportMailBusAdapter, - type PrincipalSigner, - type RecyclePolicyBounds, - type SubprocessHandle, - type SubprocessSpawner, - type SuspensionRegistration, - type WorkflowSupervisor, -} from "@intx/workflow-host"; - -import { getLogger } from "@intx/log"; - -import { defaultSubprocessSpawner, SIDECAR_WORKFLOW_CHILD_BINARY } from "./transport"; -import { readRunGrants } from "../run-grants"; - -const logger = getLogger(["sidecar", "workflow-host-wiring", "supervisor"]); - -/** - * Conservative default for the warm-keep recycle policy's grants-staleness - * bound. A warm-keep child's agent runs for the deployment's whole - * lifetime on the grants pushed by `onRunStart` at each run's dispatch - * barrier; this forces a clean respawn -- which re-reads every step's - * grants from the repo store during `recycle`'s respawn step -- rather - * than let a long-lived agent run indefinitely on hours-old material - * between runs. 24h mirrors - * `DEFAULT_CONSUMED_RETENTION_MS`'s order of magnitude. - */ -export const DEFAULT_MAX_GRANTS_AGE_MS = 24 * 60 * 60 * 1000; - -/** - * The vendor's recycle policy also supports a `maxRssBytes` bound - * (`RecyclePolicyBounds.maxRssBytes`, consulted via a `readRssBytes` - * reader the host supplies -- see `types.ts:473-499`). This wiring - * deliberately does NOT arm it: the reader's contract is "the workflow- - * process child's current resident-set size", but the only pid this - * wiring ever observes is the one returned by the FIRST `spawn()` - * (`SpawnResult.pid`) -- `recycle()`'s own return value (`RecycleAttempt`) - * never surfaces the respawned child's pid, and recycle can be triggered - * by the operator, the policy itself, or the child's own - * `recycle.request` control frame, any of which silently retires the pid - * this wiring would otherwise be reading. A reader that keeps polling a - * stale (possibly OS-recycled) pid across a respawn would make the bound - * actively wrong rather than merely absent, so it is left unwired until - * the vendor surfaces the live child's pid across a recycle. - */ - -// `assembleRunCredentialsSnapshot` and the `onRunStart` wiring below are -// ported from upstream Interchange's sidecar (apps/sidecar/src/ -// workflow-host-wiring.ts), adapted only to this directory's `deploymentId` -// naming (upstream: `anchorRunId`) for the deployment-level id. - -export type AssembleRunCredentialsSnapshotOpts = { - /** Substrate handle the sink reads the per-run grants file from. */ - repoStore: RepoStore; - /** Deployment id keying the workflow-run repo the grants file lives in. */ - deploymentId: string; - /** Run whose per-run grants file is read. */ - runId: string; - /** Step ids in `stepOrder`; the per-run grants apply uniformly across them. */ - stepOrder: readonly string[]; - /** Per-step mail-address derivation. */ - deriveStepAddress: DeriveStepAddress; -}; - -/** - * Resolve a run's credentials snapshot for the `onRunStart` grants - * barrier from its per-run grants file. - * - * Every legitimate run birth path writes `runs//grants.json` in - * the deployment's workflow-run repo before the run dispatches -- the - * external trigger route and the mail-triggered path both ship a - * `run.grants` frame the sidecar writes, and a spawned child inherits its - * parent's grants directly at spawn without reaching this barrier. The - * per-run file IS the run's snapshot: the run's single flat grant set is - * applied uniformly across every step, keyed on each step's address. - * - * A missing file is therefore not an internal run inheriting deploy-time - * grants -- it is a run that reached its barrier with no grants written, - * so it FAILS CLOSED here rather than running under-authorized. A file - * that exists but is malformed also throws (via `readRunGrants`), for the - * same reason: the file's presence implies a grants frame was delivered, - * so a structural failure is a boundary bug, not a default. - */ -export async function assembleRunCredentialsSnapshot( - opts: AssembleRunCredentialsSnapshotOpts, -): Promise { - const runGrants = await readRunGrants({ - repoStore: opts.repoStore, - deploymentId: opts.deploymentId, - runId: opts.runId, - }); - if (runGrants === undefined) { - throw new Error( - `assembleRunCredentialsSnapshot: run ${opts.runId} of deployment ${opts.deploymentId} has no grants file; failing closed`, - ); - } - const contentHash = await hashGrants(runGrants); - const steps: CredentialsSnapshotStep[] = opts.stepOrder.map((stepId) => ({ - stepId, - address: opts.deriveStepAddress({ - runId: opts.deploymentId, - stepId, - }), - grants: runGrants, - contentHash, - })); - return { steps }; -} - -export type CreateSidecarWorkflowSupervisorOpts = { - /** Sidecar's hub mail transport. */ - transport: HubTransport; - /** Substrate-shaped RepoStore the workflow-host's supervisor reads from. */ - repoStore: RepoStore; - /** Sidecar's 32-byte Ed25519 private key seed for principal signing. */ - signingKeySeed: Uint8Array; - /** Workflow-run repo identity for the deployment. */ - workflowRunRepoId: RepoId; - /** Workflow-run repo ref the supervisor commits events to. */ - workflowRunRef: string; - /** Deployment id baked into principal claims and address derivation. */ - deploymentId: string; - /** - * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`). - * Threaded into the child's spawn-time env so its deploy-tree read - * collapses onto the head for a single-step deployment. - */ - stepCount: number; - /** - * Step ids in the deployed `WorkflowDefinition`'s `stepOrder`. The - * `onRunStart` grants sink walks these to assemble the per-run - * credentialsSnapshot from each step's per-run grants file, so the sink - * needs the ordered ids rather than the bare count. - */ - stepOrder: readonly string[]; - /** Deployment's mail address. */ - deploymentMailAddress: string; - /** Per-step mail-address derivation. */ - deriveStepAddress: DeriveStepAddress; - /** - * Optional override of the per-step `agent-state` repo identity the - * supervisor reads grants from while assembling the - * credentialsSnapshot. Defaults to the `-` - * convention; the single-step launched-agent deploy supplies a - * derivation that returns the legacy agent-state repo so the spawned - * child reads grants from the same repo the legacy agent identity - * keys. - */ - deriveStepRepoId?: DeriveStepRepoId; - /** Substrate-config keys propagated to the child via spawn-time env. */ - substrateEnv: Record; - /** - * Dynamic spawn-env fragment the supervisor recomputes on every spawn and - * recycle respawn (e.g. a live-rotated inference-source list). Its keys - * layer over `substrateEnv`. See the `dynamicSpawnEnv` supervisor binding. - */ - dynamicSpawnEnv: () => Record; - /** - * Override the subprocess spawner. Tests inject a deterministic - * mock; production defaults to the `Bun.spawn`-backed - * `defaultSubprocessSpawner`. - */ - subprocessSpawner?: SubprocessSpawner; - /** Override the `bin/workflow-child` path. */ - binaryPath?: string; - /** - * Optional per-message dispatch-timing observer, forwarded verbatim to - * the supervisor's `onDispatchTiming` binding. Absent in production; - * the deploy router wires it (off a benchmark env gate) only when - * measuring dispatch latency, which needs the supervisor to emit the - * per-message infra round-trip from inside the sidecar subprocess. - */ - onDispatchTiming?: (mark: DispatchTimingMark) => void; - /** - * forced-repack A/B toggle, forwarded verbatim to the - * supervisor's `repackEveryMessages` binding. Absent in production; - * the deploy router wires it (off the same benchmark env gate) only - * when measuring how repack frequency contributes to latency. - */ - repackEveryMessages?: { everyMessages: number }; - /** - * Consumed-dedup retention horizon (ms), forwarded to the - * supervisor's `consumedRetentionMs` binding. The boot edge resolves - * the operator's `CONSUMED_RETENTION_MS` config; absent, the - * supervisor applies `DEFAULT_CONSUMED_RETENTION_MS` (24h). - */ - consumedRetentionMs?: number; - /** - * Spawn ready-handshake timeout (ms), forwarded to the supervisor's - * `readyTimeoutMs` binding. The boot edge resolves the operator's - * `CHILD_READY_TIMEOUT_MS` config; absent, the supervisor applies - * `DEFAULT_READY_TIMEOUT_MS` (30s). - */ - readyTimeoutMs?: number; - /** - * Whether this deployment's child warm-keeps its agent across messages - * (the single-step launched-agent deploy; see `SpawnOpts.warmKeep`). - * Gates the recycle policy this wiring arms: a warm-keep child is the - * only shape whose grants can go stale over a long-lived process, so - * `true` arms `recyclePolicy.maxGrantsAgeMs` (`DEFAULT_MAX_GRANTS_AGE_MS`) - * and its `readGrantsAgeMs` reader. A per-message multi-step child tears - * down and respawns fresh every message and has no long-running grants - * to police, so `false` (the default) leaves the recycle policy unarmed. - */ - warmKeep?: boolean; - /** - * Control-plane suspension sink, forwarded verbatim to the supervisor's - * `onSuspensionRegister` binding. Production wires this to the sidecar's - * hub link (`HubLink.sendSignalCorrelationRegister`) so an ask-rail - * suspension's approval snapshot reaches the hub as a - * `signal.correlation.register` frame; the hub co-writes the run's - * routing + approval rows from it. Omitted, a workflow-child suspend - * never registers an approval and the run parks invisibly forever. - */ - onSuspensionRegister?: (registration: SuspensionRegistration) => void; - /** - * Self-termination sink forwarded to the supervisor's `onSelfTerminate` - * binding: invoked when the supervisor drives itself to a terminal phase - * (crash-loop latch, channel crash, recycle failure). Production wiring - * routes it to the deploy router's address reclaim. - */ - onSelfTerminate?: (info: { phase: "stopped" | "crash-looping"; reason: string }) => void; - /** - * Decrypted credential material from the deploy frame's - * `workflow.credentials`, forwarded verbatim to the supervisor's - * `credentialDelivery` binding so the child's materialRef is seeded on - * the pre-trigger barrier. Omitted, every tool-package - * `credentials.resolve(handle)` fails "no credential is bound" even - * though the hub resolved and delivered the material on the frame. - * Absent on the boot-restore path by construction: secrets are never - * persisted sidecar-side, so a restored deployment waits for the hub's - * `credentials.update` push. - */ - credentialDelivery?: import("@intx/types/sidecar").CredentialDelivery; -}; - -export type SidecarWorkflowSupervisor = { - supervisor: WorkflowSupervisor; - /** - * Hand a delivered inbound message off to the supervisor's mail - * subscription. The returned promise resolves once the message is durably - * accepted and rejects when it was not, so the hub-link can send a - * `mail.inbound.ack` only on resolution (resolve = ack, reject = withhold). - */ - routeInbound(message: Uint8Array): Promise; - /** Snapshot accessor that proxies the supervisor's credentials view. */ - getCredentialsSnapshot(): CredentialsSnapshot | null; - /** - * Direct SIGKILL of the most recently spawned child, bypassing the - * supervisor's own graceful `shutdown()` sequencing entirely. The CL-5477 - * idle-reap park path's shutdown escalation (`shutdownSupervisorWithEscalation`) - * calls this from a bare `setTimeout` when `shutdown()` has not settled - * within the escalation window, so a wedged child cannot block a park or - * a process-exit drain forever. Safe to call at any point, including - * mid-await inside `shutdown()`'s own teardown: `Bun.spawn`'s `kill()` is - * idempotent against an already-exited or already-killed process. - */ - hardKillChild(): void; -}; - -/** - * Logical mail-audit reference the supervisor stamps onto every - * inbox/processing/consumed envelope for sidecar-hosted deployments. - * The substrate does not dereference the value; it is a host-side - * pointer the audit consumer joins on. The mail audit is keyed by the - * deployment id plus the parsed messageId, which is unique per inbound - * message and stable across the FIFO pipeline's - * enqueue/dequeue/markConsumed transitions. - */ -export function deriveSidecarMailAuditRef(deploymentId: string): ( - messageId: string, - rawMessage: Uint8Array, -) => { - store: string; - path: string; -} { - return (messageId, _rawMessage) => ({ - store: "sidecar-mail-audit", - path: `${deploymentId}/${messageId}`, - }); -} - -/** - * Construct a per-deployment supervisor with the sidecar's bindings - * pre-wired. The router calls this once per multi-step `agent.deploy` - * frame to stand up the workflow-process child that hosts the - * deployment. - */ -export function createSidecarWorkflowSupervisor( - opts: CreateSidecarWorkflowSupervisorOpts, -): SidecarWorkflowSupervisor { - const mailBus: HubTransportMailBusAdapter = wrapHubTransportAsMailBus(opts.transport); - // Tracks the most recently spawned child's handle so `hardKillChild` can - // reach it directly. The `WorkflowSupervisor` interface exposes no raw - // handle access, so wrapping the spawner -- the same injection point - // tests already use -- is the only seam that sees it. Re-assigned on - // every spawn call, including a recycle's respawn, so it always points - // at the live child. - let currentHandle: SubprocessHandle | undefined; - const baseSpawner = opts.subprocessSpawner ?? defaultSubprocessSpawner; - const trackingSpawner: SubprocessSpawner = (spawnArgs) => { - const handle = baseSpawner(spawnArgs); - currentHandle = handle; - return handle; - }; - // Wall-clock of the most recent credentials push -- either an `onRunStart` - // grants-barrier snapshot or a `deliverCredentials` rotation -- read by - // `readGrantsAgeMs` below. `undefined` until the first delivery, which - // matches the vendor reader's contract: "or `undefined` if no refresh - // has been observed yet" (`types.ts:483-488`). - let lastGrantsRefreshAt: number | undefined; - const supervisorPrincipal: WorkflowRunSupervisorPrincipal = { - kind: "supervisor", - anchorRunId: opts.deploymentId, - }; - // Per-run grants sink. The supervisor awaits this and pushes the - // resulting snapshot to the child before the run's `trigger.fire`; a - // throw propagates and the dispatch barrier fails the run rather than - // firing the trigger against absent grants. The snapshot is the run's - // own per-run grants file, which every legitimate birth path writes - // before dispatch (see `assembleRunCredentialsSnapshot`). Once wired, - // this is the SOLE grants push for the deployment's lifetime (the - // vendor supervisor suppresses its own spawn-time push), so it is also - // the observation point `readGrantsAgeMs` below keys off for a - // warm-keep deployment's staleness bound. - const onRunStart = async (args: { - runId: string; - anchorRunId: string; - }): Promise => { - const snapshot = await assembleRunCredentialsSnapshot({ - repoStore: opts.repoStore, - deploymentId: args.anchorRunId, - runId: args.runId, - stepOrder: opts.stepOrder, - deriveStepAddress: opts.deriveStepAddress, - }); - lastGrantsRefreshAt = Date.now(); - return snapshot; - }; - // Recycle policy is armed only for a warm-keep child (see `warmKeep`'s - // doc comment); a per-message multi-step child has nothing for it to - // police. `maxRssBytes`/`readRssBytes` are intentionally absent -- see - // the module-level comment above `DEFAULT_MAX_GRANTS_AGE_MS`. - const recyclePolicy: RecyclePolicyBounds | undefined = opts.warmKeep - ? { maxGrantsAgeMs: DEFAULT_MAX_GRANTS_AGE_MS } - : undefined; - // `exactOptionalPropertyTypes` rejects `{ field: undefined }` for these - // optional supervisor-config fields, so an absent value must omit the key - // entirely -- hence one conditional-spread per optional field, folded - // into this single literal rather than a chain of named intermediate - // configs. - const supervisorConfig = { - repoStore: opts.repoStore, - signAsPrincipal: (async (kind, payload) => { - const sig = await signEd25519(opts.signingKeySeed, payload); - return { sig, principalKind: kind }; - }) satisfies PrincipalSigner, - mailBus, - subprocessSpawner: trackingSpawner, - binaryPath: opts.binaryPath ?? SIDECAR_WORKFLOW_CHILD_BINARY, - substrateEnv: opts.substrateEnv, - dynamicSpawnEnv: opts.dynamicSpawnEnv, - onRunStart, - workflowRunRepoId: opts.workflowRunRepoId, - workflowRunRef: opts.workflowRunRef, - anchorRunId: opts.deploymentId, - stepCount: opts.stepCount, - deploymentMailAddress: opts.deploymentMailAddress, - readPrincipal: supervisorPrincipal, - deriveStepAddress: opts.deriveStepAddress, - deriveMailAuditRef: deriveSidecarMailAuditRef(opts.deploymentId), - ...(opts.onSuspensionRegister !== undefined - ? { onSuspensionRegister: opts.onSuspensionRegister } - : {}), - ...(opts.onSelfTerminate !== undefined ? { onSelfTerminate: opts.onSelfTerminate } : {}), - ...(opts.credentialDelivery !== undefined - ? { credentialDelivery: opts.credentialDelivery } - : {}), - ...(opts.deriveStepRepoId !== undefined ? { deriveStepRepoId: opts.deriveStepRepoId } : {}), - ...(opts.onDispatchTiming !== undefined ? { onDispatchTiming: opts.onDispatchTiming } : {}), - ...(opts.repackEveryMessages !== undefined - ? { repackEveryMessages: opts.repackEveryMessages } - : {}), - ...(opts.consumedRetentionMs !== undefined - ? { consumedRetentionMs: opts.consumedRetentionMs } - : {}), - ...(opts.readyTimeoutMs !== undefined ? { readyTimeoutMs: opts.readyTimeoutMs } : {}), - ...(recyclePolicy !== undefined - ? { - recyclePolicy, - readGrantsAgeMs: () => - lastGrantsRefreshAt === undefined ? undefined : Date.now() - lastGrantsRefreshAt, - } - : {}), - }; - const supervisor = createWorkflowSupervisor(supervisorConfig); - return { - supervisor: { - ...supervisor, - async deliverCredentials( - args: Parameters[0], - ): Promise { - await supervisor.deliverCredentials(args); - lastGrantsRefreshAt = Date.now(); - }, - }, - routeInbound(message) { - return mailBus.routeInbound(opts.deploymentMailAddress, message); - }, - getCredentialsSnapshot: () => supervisor.getCredentialsSnapshot(), - hardKillChild() { - // This runs from a bare `setTimeout` callback on the shutdown-escalation - // path: an uncaught throw here becomes an unhandled rejection that - // skips a clean park/exit, which is exactly the failure mode this - // exists to eliminate. - try { - currentHandle?.kill("SIGKILL"); - } catch (cause) { - const message = cause instanceof Error ? cause.message : String(cause); - logger.warn`hardKillChild: kill("SIGKILL") threw: ${message}`; - } - }, - }; -} diff --git a/apps/sidecar/src/workflow-host-wiring/transport.ts b/apps/sidecar/src/workflow-host-wiring/transport.ts deleted file mode 100644 index 653a75a785..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/transport.ts +++ /dev/null @@ -1,194 +0,0 @@ -// Subprocess/IPC transport adapter for the supervisor: the resolved -// `bin/workflow-child` binary path, the event-channel fd convention, -// the NDJSON control-channel reader/writer wrappers over Bun's pipe -// primitives, and the `Bun.spawn`-backed `SubprocessSpawner`. Zero -// deploy-semantics knowledge lives here. - -import { fileURLToPath } from "node:url"; - -import type { - FrameReader, - NdjsonReader, - NdjsonWriter, - SubprocessHandle, - SubprocessSpawner, -} from "@intx/workflow-host"; - -// The supervisor's `binaryPath` binding resolves to the sidecar's -// own `bin/workflow-child` script via `import.meta.resolve` against -// the `@intx/sidecar-app` package. The script lives next to this -// wiring module (`../bin/workflow-child`); resolving it statically -// at wiring-module load time keeps the production spawn surface -// independent of any runtime env override. Tests inject a sentinel -// path via the `binaryPath` opts override; production wiring -// closes over this constant. -export const SIDECAR_WORKFLOW_CHILD_BINARY: string = (() => { - const url = import.meta.resolve("../../bin/workflow-child"); - return fileURLToPath(url); -})(); - -/** - * Child fd the supervisor inherits the event-channel pipe on. The - * supervisor's spawn-time convention is: - * - * fd 0 stdin -- downstream control channel (supervisor -> child) - * fd 1 stdout -- upstream control channel (child -> supervisor) - * fd 2 stderr -- inherited so child diagnostics land on the - * sidecar's stderr - * fd 3 -- event-channel write side (child writes - * HMAC-authenticated InferenceEvent frames here; - * the supervisor reads the parent end as a - * `FrameReader`) - * - * The child opens fd 3 via `EVENT_CHANNEL_FD` in - * `@intx/workflow-host`'s `from-process-env`. The two ends of the - * pipe are provisioned by `Bun.spawn`'s `stdio` slot: setting - * `stdio[3] = "pipe"` makes Bun mint a pipe pair where the child - * inherits the write half at fd 3 and the parent receives the read - * half as a numeric fd at `proc.stdio[3]` in its own address space. - */ -const CHILD_EVENT_CHANNEL_FD = 3; - -/** - * Wrap a Bun `FileSink` as the supervisor's `NdjsonWriter`. The - * supervisor's control-channel sender writes one JSON line per - * frame (already including the trailing newline); the writer is - * responsible for passing the bytes through to the child's stdin - * without buffering across frames so each frame surfaces on the - * far side as soon as `write()` resolves. - */ -function ndjsonWriterFromFileSink(sink: Bun.FileSink): NdjsonWriter { - return { - async write(line: string): Promise { - const result = sink.write(line); - if (typeof result !== "number") await result; - const flushed = sink.flush(); - if (typeof flushed !== "number") await flushed; - }, - }; -} - -/** - * Wrap a Bun stdout `ReadableStream` as the supervisor's - * `NdjsonReader`. The pipe is a byte stream; this reader buffers - * partial chunks and yields one complete line per iteration. The - * receiver's iterator finalises only on EOF, which mirrors the - * `defaultControlReader` shape the child wires for `process.stdin`. - */ -function ndjsonReaderFromReadableStream(stream: ReadableStream): NdjsonReader { - return { - read(): AsyncIterableIterator { - return (async function* () { - const decoder = new TextDecoder("utf-8"); - let pending = ""; - const reader = stream.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (value !== undefined) { - pending += decoder.decode(value, { stream: true }); - let nl = pending.indexOf("\n"); - while (nl >= 0) { - const line = pending.slice(0, nl).replace(/\r$/, ""); - pending = pending.slice(nl + 1); - if (line.length > 0) yield line; - nl = pending.indexOf("\n"); - } - } - if (done) break; - } - if (pending.length > 0) yield pending; - } finally { - reader.releaseLock(); - } - })(); - }, - }; -} - -/** - * Wrap the parent-side read fd of the event-channel pipe as the - * supervisor's `FrameReader`. The child publishes one HMAC- - * authenticated envelope per `FileSink.write()` and the supervisor's - * `receiveEventChannel` parses each yielded `Uint8Array` as one - * complete envelope. The pipe is a byte stream; this reader yields - * each raw chunk the kernel delivers and trusts the sender's - * one-write-per-envelope discipline. The buffer-overflow / framing - * discipline lives in `receiveEventChannel`'s parser. - */ -function frameReaderFromFd(fd: number): FrameReader { - const stream = Bun.file(fd).stream(); - return { - read(): AsyncIterableIterator { - return (async function* () { - const reader = stream.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (value !== undefined && value.byteLength > 0) yield value; - if (done) break; - } - } finally { - reader.releaseLock(); - } - })(); - }, - }; -} - -/** - * Real `Bun.spawn`-backed subprocess spawner. Constructs a fresh env - * carrying exactly the trust anchors and substrate-config keys the - * supervisor passed in (no inheritance of the sidecar's process env); - * inherits stdio 0/1/2 as control + stderr; pipes fd 3 for the event - * channel and surfaces the parent-side read fd as the supervisor's - * `FrameReader`. - * - * Failure modes flow through the returned handle's `exited` promise. - * A `Bun.spawn` that fails to launch (binary missing, env malformed, - * `EXEC` error) settles `exited` with a non-zero code; the - * supervisor's `wireChild` races `exited` against `readyPromise` - * inside `spawn()` so a spawn-time crash surfaces as a rejected - * spawn rather than a wedged `starting` state. - */ -export const defaultSubprocessSpawner: SubprocessSpawner = ({ - binaryPath, - env, -}): SubprocessHandle => { - const proc = Bun.spawn([binaryPath], { - stdio: ["pipe", "pipe", "inherit", "pipe"], - env, - }); - const eventFd = proc.stdio[CHILD_EVENT_CHANNEL_FD]; - if (typeof eventFd !== "number") { - throw new Error( - `workflow-host-wiring: Bun.spawn did not return a numeric fd at stdio[${String(CHILD_EVENT_CHANNEL_FD)}] for the event channel; got ${typeof eventFd}`, - ); - } - return { - pid: proc.pid, - controlWriter: ndjsonWriterFromFileSink(proc.stdin), - controlReader: ndjsonReaderFromReadableStream(proc.stdout), - eventReader: frameReaderFromFd(eventFd), - kill(signal?: number | string): void { - // The supervisor's `SubprocessHandle.kill` widens the signal - // to `number | string`; Bun's `Subprocess.kill` accepts - // `number | NodeJS.Signals`. The supervisor's call sites pass - // `"SIGTERM"` / `"SIGKILL"` (recycle path) or no argument - // (shutdown path), which Bun handles directly. Cast at the - // boundary so the inner call matches Bun's narrower type - // without coercing valid input. - if (signal === undefined) { - proc.kill(); - return; - } - if (typeof signal === "number") { - proc.kill(signal); - return; - } - // Boundary type assertion: supervisor's kill widens to `string`; Bun's runtime accepts the same `"SIG*"` strings, narrowed back at the boundary. - proc.kill(signal as NodeJS.Signals); - }, - exited: proc.exited, - }; -}; diff --git a/apps/sidecar/src/workflow-host-wiring/wire-validation.ts b/apps/sidecar/src/workflow-host-wiring/wire-validation.ts deleted file mode 100644 index 1ea25d7ddf..0000000000 --- a/apps/sidecar/src/workflow-host-wiring/wire-validation.ts +++ /dev/null @@ -1,136 +0,0 @@ -// Wire-boundary validation and content hashing for the deploy -// router: projection validation of the wire-projected workflow -// definition and the canonical-JSON content hash the supervisor and -// the workflow-process child consume as the deployment's -// content-addressed handle. - -import { hexEncode } from "@intx/types"; -import { STEP_ID_PATTERN } from "@intx/workflow"; - -/** - * Validate the wire-projected workflow definition at the deploy-router - * boundary. The arktype `AgentDeployFrame` validator enforces the - * wire shape (`id` is non-empty, `stepOrder` is `string[]`, `steps` - * is an object, `sources` covers every `stepOrder` entry); this - * function takes `unknown`-typed inputs so it can also gate callers - * that bypass the wire boundary, and it enforces the invariants the - * router and the downstream supervisor rely on: - * - * - `definition.id` is a non-empty string. The arktype shape - * already enforces this on the wire; the re-check here protects - * bypass callers and keeps the failure shape consistent with the - * other invariants this function owns. - * - `definition.stepOrder` is non-empty. The wire shape admits - * `[]`; a zero-step workflow has no semantics here. - * - Every `stepOrder` entry matches `STEP_ID_PATTERN` so per-step - * mail-address derivation never needs escaping at the substrate - * boundary. - * - Every `stepOrder` entry has a corresponding `steps[id]` entry. - * The wire shape lets `steps[id]` be `unknown` and lets the - * entry be absent; presence is required so the workflow-process - * child can resolve each step's primitive at run time. - * - Every `stepOrder` entry has a corresponding `sources[id]` - * entry, and that entry is a non-empty array (the step's ordered - * failover chain). The arktype narrow already enforces both; the - * re-check here surfaces a structured router-side error instead of - * an arktype validation failure at the wire boundary, which keeps - * the failure shape consistent with the rest of the validations - * this function owns. An empty chain would leave the reactor with - * no initial source, so it is rejected here rather than deferred to - * a deep-stack child failure. - * - * A rejection here surfaces as a thrown `Error` the link's deploy - * frame caller converts into a structured failure reply. - */ -export function validateWorkflowProjection(projection: { - definition: { id: unknown; stepOrder: unknown; steps: unknown }; - sources: unknown; -}): void { - const def = projection.definition; - if (typeof def.id !== "string" || def.id.length === 0) { - throw new Error("sidecar deploy router: workflow.definition.id must be a non-empty string"); - } - if (!Array.isArray(def.stepOrder) || def.stepOrder.length === 0) { - throw new Error( - "sidecar deploy router: workflow.definition.stepOrder must be a non-empty array", - ); - } - if (typeof def.steps !== "object" || def.steps === null) { - throw new Error("sidecar deploy router: workflow.definition.steps must be an object"); - } - if (typeof projection.sources !== "object" || projection.sources === null) { - throw new Error("sidecar deploy router: workflow.sources must be an object"); - } - const steps = def.steps; - const sources = projection.sources; - for (const stepId of def.stepOrder) { - if (typeof stepId !== "string" || stepId.length === 0) { - throw new Error( - "sidecar deploy router: workflow.definition.stepOrder entries must be non-empty strings", - ); - } - if (!STEP_ID_PATTERN.test(stepId)) { - throw new Error( - `sidecar deploy router: stepId ${JSON.stringify(stepId)} must match ${STEP_ID_PATTERN.source}`, - ); - } - if (!Object.prototype.hasOwnProperty.call(steps, stepId)) { - throw new Error( - `sidecar deploy router: workflow.definition.steps is missing entry for stepId ${JSON.stringify(stepId)}`, - ); - } - if (!Object.prototype.hasOwnProperty.call(sources, stepId)) { - throw new Error( - `sidecar deploy router: workflow.sources is missing entry for stepId ${JSON.stringify(stepId)}`, - ); - } - // Boundary type assertion: sources is checked to be a non-null object above; this reads a value to re-check its array shape - const stepSources = (sources as Record)[stepId]; - if (!Array.isArray(stepSources) || stepSources.length === 0) { - throw new Error( - `sidecar deploy router: workflow.sources[${JSON.stringify(stepId)}] must be a non-empty array (the step's ordered inference-source failover chain)`, - ); - } - } -} - -/** - * Project a value into a canonical JSON string with deterministically - * sorted object keys. Used at the router boundary to mint a stable - * content hash of the wire-projected workflow definition; the - * orchestrator's hand-off task computes the same hash from the same - * canonical form, so a downstream verifier comparing the two values - * sees byte equality. - * - * The shape mirrors the canonicalizer the runlocal repo-store uses - * for equality checks (`packages/workflow/src/runlocal/repo-store.ts`). - */ -function canonicalJsonStringify(value: unknown): string { - if (value === null || typeof value !== "object") { - return JSON.stringify(value); - } - if (Array.isArray(value)) { - return `[${value.map(canonicalJsonStringify).join(",")}]`; - } - const entries = Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); - return `{${entries - .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJsonStringify(v)}`) - .join(",")}}`; -} - -/** - * Compute the deploy router's content hash for the wire-projected - * workflow definition. SHA-256 of the canonical JSON of the - * `WorkflowDefinition` projection, hex-encoded. The supervisor and the - * workflow-process child read the value out of the spawn-time env - * verbatim; it is the deployment's content-addressed handle. - * - * The router computes this locally so the multi-step branch does not - * round-trip the hub for a hash the orchestrator's hand-off task will - * also derive deterministically from the same canonical form. - */ -export async function computeWireDefinitionHash(definition: unknown): Promise { - const canonical = canonicalJsonStringify(definition); - const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(canonical)); - return hexEncode(new Uint8Array(digest)); -} diff --git a/apps/sidecar/src/workflow-probe-handler.test.ts b/apps/sidecar/src/workflow-probe-handler.test.ts new file mode 100644 index 0000000000..d73b2b685b --- /dev/null +++ b/apps/sidecar/src/workflow-probe-handler.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test } from "bun:test"; + +import { hexDecode, hexEncode } from "@intx/types"; +import type { WorkflowProbeRequestFrame } from "@intx/types/sidecar"; +import { encodeEnvelope, signHmac, type FrameEnvelope } from "@intx/workflow-host"; + +import { + createWorkflowProbeExecutor, + enrichProbeError, + type MaterializeWorkflowClosure, + type ProbeChildSpawner, +} from "./workflow-probe-handler"; + +describe("enrichProbeError", () => { + test("adds a dependencies/devDependencies hint to a module-not-found", () => { + const enriched = enrichProbeError( + new Error("Cannot find module '@wf/lib' from '/x/workflow.mjs'"), + ); + expect(enriched).toMatch(/could not resolve "@wf\/lib"/); + expect(enriched).toMatch(/"dependencies" rather than "devDependencies"/); + }); + + test("matches the 'Cannot find package' phrasing too", () => { + const enriched = enrichProbeError( + new Error('Cannot find package "left-pad" imported from /x/workflow.mjs'), + ); + expect(enriched).toMatch(/could not resolve "left-pad"/); + }); + + test("passes a non-resolution error through unchanged", () => { + const enriched = enrichProbeError(new Error("boom, author code threw")); + expect(enriched).toBe("boom, author code threw"); + }); +}); + +function probeFrame(): WorkflowProbeRequestFrame { + return { + type: "workflow.probe.request", + requestId: "probe-req-1", + source: { kind: "registry", registry: "npmjs" }, + closure: { schemaVersion: "1", topLevel: [], entries: [] }, + entry: "./workflow.js", + }; +} + +describe("createWorkflowProbeExecutor", () => { + test("returns a written result even when the child exit resolves before the read", async () => { + // A one-shot child writes its result line and then exits promptly, so both + // the buffered line and `handle.exited` become ready together. A handle + // whose `exited` is ALREADY resolved and whose stdout carries a valid signed + // result reproduces that race deterministically: the former `exit` race arm + // would win and discard the written result; racing only the line must + // return it. + const projection = { + id: "race-fixture", + triggers: [], + stepOrder: [], + steps: {}, + }; + const raceSpawner: ProbeChildSpawner = ({ env }) => { + const channelId = env["PROBE_IPC_CHANNEL_ID"]; + const hmacHex = env["PROBE_IPC_HMAC_KEY"]; + if (channelId === undefined || hmacHex === undefined) { + throw new Error("probe spawn env missing channel id / hmac key"); + } + const hmacKey = hexDecode(hmacHex); + const stdout = new ReadableStream({ + async start(controller) { + const payload = { + ok: true as const, + projection, + grants: ["cap:probe-race"], + grantWalkSnapshot: { + perStep: [{ stepId: "s1", grants: ["cap:probe-race"], grantEffects: {} }], + grantRequirements: [], + }, + wireHash: "a".repeat(64), + }; + const envelope: FrameEnvelope = { seq: 0, channelId, payload }; + const mac = hexEncode(await signHmac(encodeEnvelope(envelope), hmacKey)); + controller.enqueue(new TextEncoder().encode(`${JSON.stringify({ envelope, mac })}\n`)); + controller.close(); + }, + }); + return { + pid: 4242, + stdout, + exited: Promise.resolve(0), + kill: () => { + /* mock handle: already exited, nothing to reap */ + }, + }; + }; + + const materialize: MaterializeWorkflowClosure = () => + Promise.resolve({ + packageDir: "/unused-by-the-race-spawner", + cleanup: () => Promise.resolve(), + }); + + const executor = createWorkflowProbeExecutor({ + materialize, + spawnProbeChild: raceSpawner, + }); + + const result = await executor.probe(probeFrame()); + expect(result.projection).toEqual(projection); + expect(result.grants).toEqual(["cap:probe-race"]); + expect(result.grantWalkSnapshot).toEqual({ + perStep: [{ stepId: "s1", grants: ["cap:probe-race"], grantEffects: {} }], + grantRequirements: [], + }); + expect(result.wireHash).toBe("a".repeat(64)); + }); +}); diff --git a/apps/sidecar/src/workflow-probe-handler.ts b/apps/sidecar/src/workflow-probe-handler.ts index 2118a9abef..4bd0fc6456 100644 --- a/apps/sidecar/src/workflow-probe-handler.ts +++ b/apps/sidecar/src/workflow-probe-handler.ts @@ -211,6 +211,7 @@ export const defaultProbeChildSpawner: ProbeChildSpawner = ({ proc.kill(signal); return; } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the probe reaper passes "SIGTERM"/"SIGKILL"; Bun's runtime accepts the same "SIG*" strings, narrowed back at the boundary. proc.kill(signal as NodeJS.Signals); }, exited: proc.exited, diff --git a/apps/sidecar/src/workflow-run-pack-client-undeploy.test.ts b/apps/sidecar/src/workflow-run-pack-client-undeploy.test.ts new file mode 100644 index 0000000000..e33b125339 --- /dev/null +++ b/apps/sidecar/src/workflow-run-pack-client-undeploy.test.ts @@ -0,0 +1,100 @@ +// Pins the symmetric unregister contract on the multi-step routers and +// the `DeploymentAddressRegistry`. After a deployment is torn down, the +// boot-edge undeploy path must invoke `unregister` on every router and +// the registry so subsequent `signal.deliver` / `drain.deliver` / +// `mail.inbound` frames aimed at the dead deployment address are +// rejected by the router (`tryRoute` returns false) rather than +// dispatched into the orphaned supervisor handler. + +import { describe, test, expect } from "bun:test"; + +import { + createDeploymentAddressRegistry, + createMultistepDrainRouter, + createMultistepMailRouter, + createMultistepSignalRouter, +} from "./workflow-run-pack-client"; + +describe("multistep router lifecycle: unregister", () => { + test("MailRouter.unregister drops the handler so stale frames are not claimed", () => { + const router = createMultistepMailRouter(); + const delivered: Uint8Array[] = []; + router.register("dep-A@x.example", async (msg) => { + delivered.push(msg); + }); + + router.unregister("dep-A@x.example"); + + const claimed = router.tryRoute("dep-A@x.example", new Uint8Array([1])); + expect(claimed).toBeNull(); + expect(delivered).toHaveLength(0); + }); + + test("MailRouter.unregister of an unknown address is a no-op", () => { + const router = createMultistepMailRouter(); + expect(() => { + router.unregister("never-registered@x.example"); + }).not.toThrow(); + }); + + test("SignalRouter.unregister drops the handler so stale frames are not claimed", async () => { + const router = createMultistepSignalRouter(); + const delivered: string[] = []; + router.register("dep-A@x.example", async (args) => { + delivered.push(args.signalId); + }); + + router.unregister("dep-A@x.example"); + + const claimed = await router.tryRoute({ + type: "signal.deliver", + agentAddress: "dep-A@x.example", + runId: "run-stale", + signalName: "approve", + signalId: "sig-stale", + payload: null, + }); + expect(claimed).toBe(false); + expect(delivered).toEqual([]); + }); + + test("DrainRouter.unregister drops the handler so stale frames are not claimed", async () => { + const router = createMultistepDrainRouter(); + const delivered: number[] = []; + router.register("dep-A@x.example", async (args) => { + delivered.push(args.deadlineMs); + }); + + router.unregister("dep-A@x.example"); + + const claimed = await router.tryRoute({ + type: "drain.deliver", + agentAddress: "dep-A@x.example", + deadlineMs: 999, + }); + expect(claimed).toBe(false); + expect(delivered).toEqual([]); + }); + + test("DeploymentAddressRegistry exposes a removal API that breaks the anchorRunId mapping", () => { + const registry = createDeploymentAddressRegistry(); + registry.record("dep-A", "agent-a@x.example"); + + expect(registry.resolve("dep-A")).toBe("agent-a@x.example"); + registry.unregister("dep-A"); + expect(registry.resolve("dep-A")).toBeNull(); + }); + + test("DeploymentAddressRegistry.unregister of an unknown anchorRunId is a no-op", () => { + const registry = createDeploymentAddressRegistry(); + expect(() => { + registry.unregister("never-recorded"); + }).not.toThrow(); + }); + + test("DeploymentAddressRegistry's public surface includes the removal API", () => { + const registry = createDeploymentAddressRegistry(); + const keys = Object.keys(registry).sort(); + expect(keys).toEqual(["record", "resolve", "unregister"]); + }); +}); diff --git a/apps/sidecar/src/workflow-run-pack-client.test.ts b/apps/sidecar/src/workflow-run-pack-client.test.ts index 297815668f..b2976d8d87 100644 --- a/apps/sidecar/src/workflow-run-pack-client.test.ts +++ b/apps/sidecar/src/workflow-run-pack-client.test.ts @@ -1,9 +1,8 @@ import { describe, test, expect } from "bun:test"; import type { InferenceSource } from "@intx/types/runtime"; -import { CredentialsUpdateFrame } from "@intx/types/sidecar"; -import { type } from "arktype"; -import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions"; +import type { CredentialDelivery } from "@intx/types/sidecar"; +import type { RepoId, RepoStore } from "@intx/hub-sessions"; import { createDeploymentAddressRegistry, @@ -14,7 +13,6 @@ import { createMultistepCredentialsRouter, createWorkflowRunPackClient, createWorkflowRunPackPushingRepoStore, - type WorkflowRunPackClient, } from "./workflow-run-pack-client"; function createRecordingUnderlyingRepoStore(): { @@ -54,13 +52,6 @@ function createRecordingUnderlyingRepoStore(): { newlyTerminalRuns: [], }; }, - async writeTreeDelta(principal, repoId, ref) { - preserveCalls.push({ principal, repoId, ref }); - return { - commitSha: `sha-${String(preserveCalls.length)}`, - newlyTerminalRuns: [], - }; - }, async resolveRef(_principal, _repoId, _ref) { // The client's empty-delta guard compares the current ref tip against // the last commit it acked. Return a fixed tip distinct from the @@ -81,6 +72,7 @@ function createRecordingUnderlyingRepoStore(): { packedTipCommits.push({ repoId, ref, commitSha }); }, }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- in-test stub; the unused RepoStore methods are guarded by the Proxy below const store = new Proxy(stub as RepoStore, { get(target, prop, receiver) { const value = Reflect.get(target, prop, receiver); @@ -185,7 +177,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { async push(opts) { pushed.push(opts); }, - markRestored() {}, }, registry, }); @@ -237,7 +228,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { await gate; pushOrder.push("exit"); }, - markRestored() {}, }, registry, }); @@ -286,7 +276,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { pushCount += 1; if (idx === 0) await firstGate; }, - markRestored() {}, }, registry, }); @@ -333,7 +322,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { throw new Error("hub_rejected: non_fast_forward"); } }, - markRestored() {}, }, registry, }); @@ -363,7 +351,7 @@ describe("createWorkflowRunPackPushingRepoStore", () => { const registry = createDeploymentAddressRegistry(); const facade = createWorkflowRunPackPushingRepoStore({ underlying: store, - packClient: { push: () => Promise.resolve(), markRestored() {} }, + packClient: { push: () => Promise.resolve() }, registry, }); await facade.flushWorkflowRunPushes( @@ -382,7 +370,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { async push(opts) { pushed.push(opts); }, - markRestored() {}, }, registry, }); @@ -408,7 +395,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { underlying: store, packClient: { push: () => Promise.resolve(), - markRestored() {}, }, registry, }); @@ -424,15 +410,15 @@ describe("createWorkflowRunPackPushingRepoStore", () => { message: "append", }, ), - ).rejects.toThrow(/no agent address registered/); + ).rejects.toThrow(/no run address registered/); }); test("markAddressUnroutable holds a push until notifyAddressRoutable resumes it", async () => { // The reconnect ordering contract. A WS disconnect blocks the address: // a write that lands while blocked schedules no wire push, because a - // push shipped on the fresh, not-yet-challenged connection is dropped by - // the hub as "unrouted". The reconnect challenge lifts the block and the - // held push ships then -- after the hub has re-routed the address. + // push shipped on the fresh, not-yet-registered connection is dropped by + // the hub as "unrouted". The reconnect route announcement lifts the block, + // and the held push ships after the hub has re-routed the address. const { store } = createRecordingUnderlyingRepoStore(); const registry = createDeploymentAddressRegistry(); registry.record("dep-blocked", "agent-blocked@example.com"); @@ -443,7 +429,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { async push() { pushCount += 1; }, - markRestored() {}, }, registry, }); @@ -470,7 +455,7 @@ describe("createWorkflowRunPackPushingRepoStore", () => { // in-flight transfer) and latches its error. There is no later local // write to re-arm the coalescing loop, so without the routable-again // re-drive the run would strand forever. notifyAddressRoutable re-ships - // the un-acked commits once the challenge re-routes the address. + // the un-acked commits once reconnect registration re-routes the address. const { store } = createRecordingUnderlyingRepoStore(); const registry = createDeploymentAddressRegistry(); registry.record("dep-cancelled", "agent-cancelled@example.com"); @@ -484,7 +469,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { throw new Error("transfer cancelled: Connection lost"); } }, - markRestored() {}, }, registry, }); @@ -523,7 +507,6 @@ describe("createWorkflowRunPackPushingRepoStore", () => { async push() { pushCount += 1; }, - markRestored() {}, }, registry, }); @@ -650,6 +633,17 @@ describe("createMultistepCredentialsRouter", () => { expect(received[0]?.delivery).toEqual(delivery); }); + test("a frame's revoke list threads through to the handler", async () => { + const router = createMultistepCredentialsRouter(); + const received: { delivery: typeof delivery; revoke?: string[] }[] = []; + router.register("dep@integration.interchange", async (args) => { + received.push(args); + }); + expect(await router.tryRoute({ ...frame, revoke: ["cred_1"] })).toBe(true); + expect(received).toHaveLength(1); + expect(received[0]?.revoke).toEqual(["cred_1"]); + }); + test("registration is per-address; an unrelated address falls through", async () => { const router = createMultistepCredentialsRouter(); router.register("dep-a@integration.interchange", async () => undefined); @@ -668,19 +662,28 @@ describe("createMultistepCredentialsRouter", () => { expect(await router.tryRoute(frame)).toBe(false); }); - test("a malformed delivery is rejected at the wire boundary, so it never reaches the router", () => { - // Validation belongs to the frame schema, not to this adapter: hub-link's - // `handleMessage` narrows every inbound frame through `HubFrame` (which - // unions `CredentialsUpdateFrame`) and answers a malformed one with an - // error frame before any router is consulted. So the contract to assert is - // the boundary's, and the router carries no defensive re-validation of a - // shape it can no longer be handed. + test("rejects a malformed delivery for a registered address without dispatching", async () => { + const router = createMultistepCredentialsRouter(); + let called = false; + router.register("dep@integration.interchange", async () => { + called = true; + }); + // A malformed delivery would crash the child's control-channel receiver on + // its `CredentialsUpdateFrame` narrow, so the router rejects before + // dispatch and the hub-link turns the throw into a truthful session.error. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- deliberately malformed delivery to exercise the pre-dispatch validation const malformed = { - type: "credentials.update", - agentAddress: "dep@integration.interchange", - delivery: { bindings: [{ handle: "x" }], materials: [] }, - }; - expect(CredentialsUpdateFrame(malformed)).toBeInstanceOf(type.errors); + bindings: [{ handle: "x" }], + materials: [], + } as unknown as CredentialDelivery; + await expect( + router.tryRoute({ + type: "credentials.update", + agentAddress: "dep@integration.interchange", + delivery: malformed, + }), + ).rejects.toThrow(); + expect(called).toBe(false); }); test("propagates the handler's rejection (deliverCredentials throwing) verbatim", async () => { @@ -1047,58 +1050,3 @@ describe("createMultistepSignalRouter", () => { expect(claimed).toBe(false); }); }); - -test("unregister reclaim drops the push slot so flush does not wait on an in-flight push", async () => { - const REF = "refs/heads/main"; - const DEPLOYMENT_ID = "dep-reclaim"; - const AGENT_ADDRESS = "run_reclaim@bench.localhost"; - const REPO_ID: RepoId = { kind: "workflow-run", id: DEPLOYMENT_ID }; - const PRINCIPAL: Principal = { kind: "hub" }; - const { store: underlying } = createRecordingUnderlyingRepoStore(); - let pushCalls = 0; - let releaseFirst!: () => void; - const firstPush = new Promise((resolve) => { - releaseFirst = resolve; - }); - const packClient: WorkflowRunPackClient = { - async push() { - pushCalls += 1; - if (pushCalls === 1) await firstPush; - }, - markRestored() {}, - }; - const registry = createDeploymentAddressRegistry(); - registry.record(DEPLOYMENT_ID, AGENT_ADDRESS); - const store = createWorkflowRunPackPushingRepoStore({ - underlying, - packClient, - registry, - }); - const deltaArgs = { - computeDelta: async () => ({ puts: {}, deletes: [] as const }), - changedPathPrefixes: undefined, - message: "test", - }; - - await store.writeTreeDelta(PRINCIPAL, REPO_ID, REF, deltaArgs); - expect(pushCalls).toBe(1); - - store.markAddressUnroutable(AGENT_ADDRESS); - store.reclaimPushState({ - deploymentId: DEPLOYMENT_ID, - agentAddress: AGENT_ADDRESS, - }); - - await Promise.race([ - store.flushWorkflowRunPushes(REPO_ID, REF), - new Promise((_, reject) => { - setTimeout(() => reject(new Error("flush hung on a residual slot")), 50); - }), - ]); - - registry.record(DEPLOYMENT_ID, AGENT_ADDRESS); - await store.writeTreeDelta(PRINCIPAL, REPO_ID, REF, deltaArgs); - expect(pushCalls).toBe(2); - - releaseFirst(); -}); diff --git a/apps/sidecar/src/workflow-run-pack-client.ts b/apps/sidecar/src/workflow-run-pack-client.ts index e8e9193249..b70504a505 100644 --- a/apps/sidecar/src/workflow-run-pack-client.ts +++ b/apps/sidecar/src/workflow-run-pack-client.ts @@ -18,11 +18,11 @@ import { type } from "arktype"; import { getLogger } from "@intx/log"; import { SourcesUpdatedData } from "@intx/workflow-host"; import type { InferenceSource } from "@intx/types/runtime"; -import type { CredentialDelivery } from "@intx/types/sidecar"; +import { CredentialDelivery, type SenderIdentity } from "@intx/types/sidecar"; import type { RepoId, RepoStore, WorkflowRunSupervisorPrincipal } from "@intx/hub-sessions"; import type { HubLink } from "@intx/hub-agent"; -const logger = getLogger(["sidecar", "workflow-run-pack-client"]); +const logger = getLogger(["interchange", "sidecar", "workflow-run-pack-client"]); export type WorkflowRunPackClient = { /** @@ -70,6 +70,15 @@ export function createWorkflowRunPackClient( } return { + markRestored(repoId, ref, commitSha) { + if (repoId.kind !== "workflow-run") { + throw new Error( + `workflow-run pack client: restored repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, + ); + } + substrate.commitPackedTip(repoId, ref, commitSha); + lastAckedSha.set(ackKey(repoId, ref), commitSha); + }, async push({ agentAddress, repoId, ref }) { if (repoId.kind !== "workflow-run") { throw new Error( @@ -106,42 +115,33 @@ export function createWorkflowRunPackClient( substrate.commitPackedTip(repoId, ref, commitSha); lastAckedSha.set(ackKey(repoId, ref), commitSha); }, - markRestored(repoId, ref, commitSha) { - if (repoId.kind !== "workflow-run") { - throw new Error( - `workflow-run pack client: restored repoId.kind must be "workflow-run", got ${JSON.stringify(repoId.kind)}`, - ); - } - substrate.commitPackedTip(repoId, ref, commitSha); - lastAckedSha.set(ackKey(repoId, ref), commitSha); - }, }; } /** * Mapping registry the boot-edge substrate facade consults to resolve - * `repoId.id` (the workflow-run deploymentId, which the deploy router + * `repoId.id` (the workflow-run runId, which the deploy router * derives by slugging the agent's mail address) back into the * agentAddress carried on every outbound pack frame. Populated by the * deploy router as each `agent.deploy` frame lands. */ export type DeploymentAddressRegistry = { - record(deploymentId: string, agentAddress: string): void; - resolve(deploymentId: string): string | null; - unregister(deploymentId: string): void; + record(runId: string, agentAddress: string): void; + resolve(runId: string): string | null; + unregister(runId: string): void; }; export function createDeploymentAddressRegistry(): DeploymentAddressRegistry { const table = new Map(); return { - record(deploymentId, agentAddress) { - table.set(deploymentId, agentAddress); + record(runId, agentAddress) { + table.set(runId, agentAddress); }, - resolve(deploymentId) { - return table.get(deploymentId) ?? null; + resolve(runId) { + return table.get(runId) ?? null; }, - unregister(deploymentId) { - table.delete(deploymentId); + unregister(runId) { + table.delete(runId); }, }; } @@ -276,10 +276,16 @@ export function createMultistepSignalRouter(): MultistepSignalRouter { * `runs//events/` subtree. The write is awaited so the frame's FIFO * completion means the grants are durable on disk before the next frame is * processed. + * + * `senderIdentities` carries the run's authorized senders' hub-vouched keys, + * co-delivered on the same `run.grants` frame. The handler caches each one + * before the grants write, so a grant that lands durably is never missing the + * key its recipient needs to verify the sender's mail. */ export type MultistepGrantsHandler = (args: { runId: string; stepGrants: readonly unknown[]; + senderIdentities?: readonly SenderIdentity[]; }) => Promise; /** @@ -290,11 +296,12 @@ export type MultistepGrantsHandler = (args: { * deployments alike; the handler writes the run's grants into the * deployment's workflow-run repo. * - * The registry lives at the sidecar's host layer for the same boundary - * reason as `MultistepMailRouter` / `MultistepSignalRouter`: the routing - * decision is a concrete sidecar host concern, and the workflow-host - * package stays agnostic to which transport surface its supervisor handle - * rides on. + * The registry lives at the sidecar's host layer (not inside the + * workflow-host library) for the same boundary reason as + * `MultistepMailRouter` / `MultistepSignalRouter`: the routing decision + * is a concrete sidecar host concern, and the workflow-host package + * stays agnostic to which transport surface its supervisor handle rides + * on. */ export type MultistepGrantsRouter = { register(address: string, handler: MultistepGrantsHandler): void; @@ -304,77 +311,12 @@ export type MultistepGrantsRouter = { agentAddress: string; runId: string; stepGrants: readonly unknown[]; + senderIdentities?: readonly SenderIdentity[]; }): Promise; }; export function createMultistepGrantsRouter(): MultistepGrantsRouter { const handlers = new Map(); - return { - register(address, handler) { - handlers.set(address, handler); - }, - unregister(address) { - handlers.delete(address); - }, - async tryRoute(frame) { - const handler = handlers.get(frame.agentAddress); - if (handler === undefined) { - // The run's grants never reach disk, so its `onRunStart` barrier - // fails closed and the run dies before its first step. Silence - // here made that look like an unexplained run failure. - logger.error`run.grants for run ${frame.runId} dropped: no grants handler registered for deployment ${frame.agentAddress}; the run will fail its onRunStart barrier closed`; - return false; - } - await handler({ runId: frame.runId, stepGrants: frame.stepGrants }); - return true; - }, - }; -} - -// The `MultistepCredentialsRouter` family below is ported from upstream -// Interchange's sidecar (apps/sidecar/src/workflow-host-wiring.ts, -// `credentials.update` handler), adapted only to -// this directory's split (the router type lives beside its `Grants` / -// `Sources` siblings here rather than inline in the wiring module). Before -// this port, no handler registry existed for `credentials.update`, so an -// inbound rotation or revocation frame was unrouted for every deployment. - -/** - * Per-deployment credential-delivery handler the deploy router installs - * against the `MultistepCredentialsRouter` after a supervisor's `spawn` - * succeeds. The handler hands the delivery to the supervisor's - * `deliverCredentials`, which sends a `credentials-updated` control frame to - * the child where the material cell is swapped. No durable persist -- - * credential material never touches disk. - */ -export type MultistepCredentialsHandler = (args: { delivery: CredentialDelivery }) => Promise; - -/** - * Per-deployment-address credential-delivery handler registry the sidecar - * hub-link consults on every inbound `credentials.update` frame. Registered - * for EVERY deployment (not only warm single-step ones) once - * `wired.supervisor.spawn` succeeds: the material cell is per-child and read - * by every step's tool capabilities. An inbound `credentials.update` for an - * unregistered (torn-down) address is unrouted. - * - * The registry lives at the sidecar's host layer for the same boundary - * reason as `MultistepMailRouter` / `MultistepGrantsRouter`: the routing - * decision is a concrete sidecar host concern, and the workflow-host - * package stays agnostic to which transport surface its supervisor handle - * rides on. - */ -export type MultistepCredentialsRouter = { - register(address: string, handler: MultistepCredentialsHandler): void; - unregister(address: string): void; - tryRoute(frame: { - type: "credentials.update"; - agentAddress: string; - delivery: CredentialDelivery; - }): Promise; -}; - -export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { - const handlers = new Map(); return { register(address, handler) { handlers.set(address, handler); @@ -385,7 +327,13 @@ export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { async tryRoute(frame) { const handler = handlers.get(frame.agentAddress); if (handler === undefined) return false; - await handler({ delivery: frame.delivery }); + await handler({ + runId: frame.runId, + stepGrants: frame.stepGrants, + ...(frame.senderIdentities !== undefined + ? { senderIdentities: frame.senderIdentities } + : {}), + }); return true; }, }; @@ -526,6 +474,75 @@ export function createMultistepSourcesRouter(): MultistepSourcesRouter { }; } +/** + * Per-deployment credential-delivery handler the deploy router installs + * against the `MultistepCredentialsRouter` after a supervisor's `spawn` + * succeeds. The handler hands the delivery to the supervisor's + * `deliverCredentials`, which sends a `credentials-updated` control IPC frame + * to the workflow-process child, where the material cell is swapped in place. + * + * Unlike sources rotation, this registers for ANY deployment (single- or + * multi-step): the material cell is per-child and read by every step's tool + * capabilities, so there is no single-warm-agent restriction. There is no + * durable persist -- credential material never touches disk (it is re-resolved + * by the hub on reconnect). + */ +export type MultistepCredentialsHandler = (args: { + delivery: CredentialDelivery; + revoke?: string[]; +}) => Promise; + +/** + * Per-deployment-address credential-delivery handler registry. Mirrors + * `MultistepSourcesRouter`: `credentials.update` is a REQUEST/ACK frame, so a + * registered address that throws surfaces as a `session.error` and an + * unregistered address returns `false` (unrouted). Lives at the sidecar host + * layer for the same boundary reason -- the workflow-host package stays + * agnostic to the transport surface its supervisor rides on. + */ +export type MultistepCredentialsRouter = { + register(address: string, handler: MultistepCredentialsHandler): void; + unregister(address: string): void; + tryRoute(frame: { + type: "credentials.update"; + agentAddress: string; + delivery: CredentialDelivery; + revoke?: string[]; + }): Promise; +}; + +export function createMultistepCredentialsRouter(): MultistepCredentialsRouter { + const handlers = new Map(); + return { + register(address, handler) { + handlers.set(address, handler); + }, + unregister(address) { + handlers.delete(address); + }, + async tryRoute(frame) { + const handler = handlers.get(frame.agentAddress); + // Registration check first: an unregistered (torn-down) address is + // unrouted -- reported as `false`, its payload never inspected. + if (handler === undefined) return false; + // Validate the delivery BEFORE dispatch: a malformed delivery would + // reach the child's control-channel receiver and crash it on + // `CredentialsUpdateFrame`'s narrow. Rejecting here throws, and the + // hub-link turns the throw into a truthful `session.error` instead of + // acking and detonating the child. + const validated = CredentialDelivery(frame.delivery); + if (validated instanceof type.errors) { + throw new Error(validated.summary); + } + await handler({ + delivery: frame.delivery, + ...(frame.revoke !== undefined ? { revoke: frame.revoke } : {}), + }); + return true; + }, + }; +} + /** * Boot-edge facade around the substrate-shaped `RepoStore`. Forwards * every method to the underlying store; intercepts the @@ -585,7 +602,7 @@ export function createMultistepSourcesRouter(): MultistepSourcesRouter { */ export type WorkflowRunPackPushingRepoStoreOpts = { underlying: RepoStore; - packClient: WorkflowRunPackClient; + packClient: Pick; registry: DeploymentAddressRegistry; }; @@ -594,8 +611,9 @@ export type WorkflowRunPackPushingRepoStoreOpts = { * per-(repoId.id, ref) pack-push pipeline to drain. The `RepoStore` * shape is unchanged so call sites that consume `RepoStore` keep * working; `flushWorkflowRunPushes` is opt-in for code that - * genuinely needs hub-side visibility (shutdown, integration tests). - * Call sites that don't need it pay zero cost. + * genuinely needs hub-side visibility (shutdown, integration tests, + * end-to-end benchmarks). Call sites that don't need it pay zero + * cost. */ export type WorkflowRunPackPushingRepoStore = RepoStore & { /** @@ -609,7 +627,7 @@ export type WorkflowRunPackPushingRepoStore = RepoStore & { /** * Re-drive any workflow-run push for `agentAddress` that a disconnect * cancelled. Called when the hub-link observes the deployment address - * become routable again after a reconnect challenge. For each slot bound + * become routable again after an authenticated reconnect. For each slot bound * to `agentAddress` whose last push attempt failed (its `lastError` is * latched), it re-arms the coalescing loop so a fresh `createPack` re-ships * the un-acked commits -- the liveness path a synchronous single-step run @@ -618,26 +636,19 @@ export type WorkflowRunPackPushingRepoStore = RepoStore & { * A re-ship that fails again re-latches without self-retrying, so a * genuinely unrecoverable failure still surfaces loudly on the next local * write rather than spinning. It is gated on the address being routable - * again (the caller only fires post-challenge) so the re-ship cannot race + * again (the caller only fires post-reconnect) so the re-ship cannot race * ahead of the hub re-routing the address. */ notifyAddressRoutable: (agentAddress: string) => void; /** * Block workflow-run pushes for `agentAddress` until the next * `notifyAddressRoutable`. Called when the hub-link observes its WS drop: - * the address's hub route is gone until the reconnect challenge re-proves - * ownership, so a push shipped in the interim is dropped by the hub as + * the address's hub route is gone until the authenticated reconnect restores + * it, so a push shipped in the interim is dropped by the hub as * "unrouted". Holding the push at the block -- rather than shipping and - * failing -- is what lets the reconnect re-ship wait for the challenge. + * failing -- is what lets the reconnect re-ship wait for route restoration. */ markAddressUnroutable: (agentAddress: string) => void; - /** - * Drop coalescing slots and routability blocks for a torn-down - * deployment. Called from the boot edge's `unregisterDeployment` - * hook so an undeploy does not leave a slot (and its blocked address) - * in this process-wide map forever. - */ - reclaimPushState: (entry: { deploymentId: string; agentAddress: string }) => void; }; export function createWorkflowRunPackPushingRepoStore( @@ -660,14 +671,14 @@ export function createWorkflowRunPackPushingRepoStore( } // Addresses whose hub route was dropped and has not been re-established by a - // reconnect challenge. Absent means routable -- the steady state, and the + // authenticated reconnect. Absent means routable -- the steady state, and the // first-connect state (a deployment routes via its `agent.deploy`, not a - // challenge, so it is never blocked before its first push). An address is + // reconnect, so it is never blocked before its first push). An address is // added on `markAddressUnroutable` (WS disconnect) and removed on - // `notifyAddressRoutable` (challenge passed). A push for a blocked address + // `notifyAddressRoutable` (reconnect sent). A push for a blocked address // is held: the coalescing loop pauses with `dirty` still set rather than // shipping to a hub that has not yet re-routed the address -- which is what - // makes the reconnect re-ship wait for the challenge instead of racing + // makes the reconnect re-ship wait for route restoration instead of racing // ahead of it and being dropped as "unrouted". const blockedAddresses = new Set(); @@ -680,7 +691,7 @@ export function createWorkflowRunPackPushingRepoStore( function startLoop(slot: Slot, repoId: RepoId, ref: string): void { if (slot.inFlight !== null) return; // Hold the push while the address is not routable (dropped, awaiting the - // reconnect challenge). Leave `dirty` set and start no loop: the loop + // authenticated reconnect). Leave `dirty` set and start no loop: the loop // resumes when `notifyAddressRoutable` clears the block and re-arms it. // Shipping now would race ahead of the hub re-routing the address, and // the frames would be dropped as "unrouted". @@ -689,7 +700,7 @@ export function createWorkflowRunPackPushingRepoStore( while (slot.dirty) { // Re-check routability each iteration: a disconnect mid-drain must // pause the loop rather than push into a severed link. Leave `dirty` - // set so the post-challenge resume re-ships. + // set so the post-reconnect resume re-ships. if (blockedAddresses.has(slot.agentAddress)) break; slot.dirty = false; try { @@ -727,7 +738,7 @@ export function createWorkflowRunPackPushingRepoStore( } else { // The agentAddress is derived from a stable per-deployment // mapping; refreshing it on every call keeps the slot in sync - // if the registry ever re-resolves the same deploymentId to + // if the registry ever re-resolves the same runId to // a different address (today it does not, but the contract is // "look up at push time", not "cache forever"). slot.agentAddress = agentAddress; @@ -746,24 +757,15 @@ export function createWorkflowRunPackPushingRepoStore( function markAddressUnroutable(agentAddress: string): void { // The hub route for this address just dropped (WS disconnect). Block its - // pushes until the reconnect challenge re-routes it. A push already + // pushes until the authenticated reconnect re-routes it. A push already // in-flight when the link dropped rejects through `packSender.cancelAll` // and latches its error; the block stops the coalescing loop from - // immediately re-shipping on the fresh (not-yet-challenged) connection. + // immediately re-shipping on the fresh (not-yet-registered) connection. blockedAddresses.add(agentAddress); } - function reclaimPushState(entry: { deploymentId: string; agentAddress: string }): void { - blockedAddresses.delete(entry.agentAddress); - for (const [key, slot] of slots) { - if (slot.repoId.id === entry.deploymentId) { - slots.delete(key); - } - } - } - function notifyAddressRoutable(agentAddress: string): void { - // The reconnect challenge re-routed this address on the hub. Clear the + // The authenticated reconnect re-routed this address on the hub. Clear the // block and re-drive so a push the disconnect cancelled -- or one held // while the block was up -- ships now. This is the liveness path a // synchronous single-step run lacks: with all its events in one batch it @@ -818,13 +820,12 @@ export function createWorkflowRunPackPushingRepoStore( listRefs: underlying.listRefs.bind(underlying), resolveHead: underlying.resolveHead.bind(underlying), getRepoDir: underlying.getRepoDir.bind(underlying), - subscribe: underlying.subscribe.bind(underlying), openCommittedReads: underlying.openCommittedReads.bind(underlying), openCommittedReadsAtCommit: underlying.openCommittedReadsAtCommit.bind(underlying), + subscribe: underlying.subscribe.bind(underlying), flushWorkflowRunPushes, notifyAddressRoutable, markAddressUnroutable, - reclaimPushState, async writeTreePreservingPrefix(principal, repoId, ref, args) { if (repoId.kind === "workflow-run") { const latched = takeLatchedError(repoId, ref); @@ -839,7 +840,7 @@ export function createWorkflowRunPackPushingRepoStore( const agentAddress = registry.resolve(repoId.id); if (agentAddress === null) { throw new Error( - `workflow-run pack push: no agent address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, ); } schedulePush(agentAddress, repoId, ref); @@ -859,7 +860,7 @@ export function createWorkflowRunPackPushingRepoStore( const agentAddress = registry.resolve(repoId.id); if (agentAddress === null) { throw new Error( - `workflow-run pack push: no agent address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, + `workflow-run pack push: no run address registered for deployment ${repoId.id}; the deploy router must record the mapping before the supervisor commits run events`, ); } schedulePush(agentAddress, repoId, ref); diff --git a/apps/sidecar/src/workflow-run-pack-restore.test.ts b/apps/sidecar/src/workflow-run-pack-restore.test.ts index 907a1daa81..95c9410e9d 100644 --- a/apps/sidecar/src/workflow-run-pack-restore.test.ts +++ b/apps/sidecar/src/workflow-run-pack-restore.test.ts @@ -30,7 +30,7 @@ test("restored refs survive replacement and the next sidecar commit fast-forward dataDir: path.join(root, "replacement"), signingKey: targetKey, }); - const agentAddress = "ins_dep_restore@workflow.test"; + const agentAddress = "run_restore@workflow.test"; const repoId: RepoId = { kind: "workflow-run", id: deriveWorkflowRunRepoId(agentAddress), diff --git a/apps/sidecar/src/workflow-run-record.test.ts b/apps/sidecar/src/workflow-run-record.test.ts index bd7eefc3b5..dca3d946d2 100644 --- a/apps/sidecar/src/workflow-run-record.test.ts +++ b/apps/sidecar/src/workflow-run-record.test.ts @@ -4,6 +4,8 @@ import os from "node:os"; import path from "node:path"; import { type } from "arktype"; +import { createEnvKeyCredentialCipher } from "@intx/crypto"; +import type { CredentialDelivery } from "@intx/types/sidecar"; import { WorkflowRunRecord, @@ -12,6 +14,12 @@ import { scanWorkflowRunRecords, } from "./workflow-run-record"; +// Real ciphers (not a noop) so the tests exercise actual seal/unseal. A second +// cipher under a different key drives the decrypt-failure path (a rotated +// sidecar key). +const CIPHER = createEnvKeyCredentialCipher(new Uint8Array(32).fill(1)); +const OTHER_KEY_CIPHER = createEnvKeyCredentialCipher(new Uint8Array(32).fill(2)); + async function makeDataDir(): Promise { return fs.mkdtemp(path.join(os.tmpdir(), "wdr-")); } @@ -29,8 +37,35 @@ async function fileExists(p: string): Promise { } } +/** Read the raw on-disk record (still sealed) and validate its schema shape. */ +async function readRawRecord(dataDir: string, anchorRunId: string): Promise { + const raw = await fs.readFile(recordPath(dataDir, anchorRunId), "utf8"); + const parsed = WorkflowRunRecord(JSON.parse(raw)); + if (parsed instanceof type.errors) { + throw new Error(`record failed validation: ${parsed.summary}`); + } + return parsed; +} + +/** Build a credential-material cell (no tool bindings) from id -> secret. */ +function deliveryOf(materials: Record): CredentialDelivery { + return { + bindings: [], + materials: Object.entries(materials).map(([credentialId, secret]) => ({ + credentialId, + providerKey: "anthropic", + origin: "https://api.example/anthropic", + secret, + })), + }; +} + +// Fixtures are the unified format: `sources`/`bodySources` are non-secret config +// carrying a `credentialId` per source, and every secret lives in `credentials` +// (the material cell), sealed to ciphertext on disk by `writeWorkflowRunRecord` +// and unsealed on read by `scanWorkflowRunRecords`. const SINGLE_STEP: WorkflowRunRecord = { - version: 1, + version: 2, agentAddress: "run_abc123@tenant.example", definitionId: "wf_abc123", sources: { @@ -39,11 +74,12 @@ const SINGLE_STEP: WorkflowRunRecord = { id: "anthropic:mock", provider: "anthropic", baseURL: "https://api.example/anthropic", - credentialId: "sk-x", + credentialId: "cred-x", model: "claude-mock", }, ], }, + credentials: deliveryOf({ "cred-x": "sk-x" }), sessionId: "ses_1", hubPublicKey: "deadbeef", // Source-ref is the only lineage: every record carries the pin + approved @@ -63,7 +99,7 @@ const SINGLE_STEP: WorkflowRunRecord = { // A multi-step deployment records no head hub key and may carry no session // id -- both optional fields absent. const MULTI_STEP: WorkflowRunRecord = { - version: 1, + version: 2, agentAddress: "run_xyz@tenant.example", definitionId: "wf_xyz", sources: { @@ -72,7 +108,7 @@ const MULTI_STEP: WorkflowRunRecord = { id: "anthropic:mock", provider: "anthropic", baseURL: "https://api.example/anthropic", - credentialId: "sk-y", + credentialId: "cred-y", model: "claude-mock", }, ], @@ -81,11 +117,12 @@ const MULTI_STEP: WorkflowRunRecord = { id: "openai:mock", provider: "openai", baseURL: "https://api.example/openai", - credentialId: "sk-z", + credentialId: "cred-z", model: "gpt-mock", }, ], }, + credentials: deliveryOf({ "cred-y": "sk-y", "cred-z": "sk-z" }), approvedWireHash: "e".repeat(64), lineage: "source-ref", sourceRef: { @@ -102,7 +139,7 @@ const MULTI_STEP: WorkflowRunRecord = { // a sourceRef pin (source + closure) + approvedWireHash for lineage // "source-ref". const SOURCE_REF: WorkflowRunRecord = { - version: 1, + version: 2, agentAddress: "ins_dep_src@tenant.example", definitionId: "wf_src", sources: { @@ -111,11 +148,12 @@ const SOURCE_REF: WorkflowRunRecord = { id: "anthropic:mock", provider: "anthropic", baseURL: "https://api.example/anthropic", - credentialId: "sk-s", + credentialId: "cred-s", model: "claude-mock", }, ], }, + credentials: deliveryOf({ "cred-s": "sk-s" }), approvedWireHash: "c".repeat(64), lineage: "source-ref", sourceRef: { @@ -130,18 +168,180 @@ const SOURCE_REF: WorkflowRunRecord = { }, }; +// A deployment that spawns bodies (an onTrigger section and a childWorkflow +// child): the top-level `sources` plus a `bodySources` table keyed by each +// body's definition id, and one `credentials` cell backing every source's +// `credentialId` -- top-level and per-body alike. +const WITH_BODIES: WorkflowRunRecord = { + version: 2, + agentAddress: "ins_dep_bodies@tenant.example", + definitionId: "wf_bodies", + sources: { + "step-1": [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + credentialId: "cred-top", + model: "claude-mock", + }, + ], + }, + bodySources: { + "wf_bodies:onTrigger:0": { + "body-step": [ + { + id: "anthropic:mock", + provider: "anthropic", + baseURL: "https://api.example/anthropic", + credentialId: "cred-body-a", + model: "claude-mock", + }, + ], + }, + "wf_bodies:child:handler": { + plan: [ + { + id: "openai:mock", + provider: "openai", + baseURL: "https://api.example/openai", + credentialId: "cred-body-b", + model: "gpt-mock", + }, + ], + }, + }, + credentials: deliveryOf({ + "cred-top": "sk-top", + "cred-body-a": "sk-body-a", + "cred-body-b": "sk-body-b", + }), + approvedWireHash: "f".repeat(64), + lineage: "source-ref", + sourceRef: { + source: { kind: "registry", registry: "npm" }, + closure: { + schemaVersion: "1", + topLevel: [{ name: "@x/wf", version: "1.0.0" }], + entries: [], + }, + }, +}; + describe("workflow run record store", () => { - test("round-trips a source-ref record (source + closure + approvedWireHash)", async () => { + test("seal/unseal round-trips a source-ref record through disk", async () => { const dataDir = await makeDataDir(); const anchorRunId = "src-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, SOURCE_REF); + await writeWorkflowRunRecord(dataDir, anchorRunId, SOURCE_REF, CIPHER); + + // The scan unseals the credential cell, so the restored record matches the + // original. + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + expect(scanned.map((s) => s.record)).toEqual([SOURCE_REF]); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("seals each credential secret at rest and unseals it on scan", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "abc123-tenant-example"; + await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); + + // The record embeds credential secrets (sealed) plus the deployment's + // identity, so it must not be group/world readable on a shared host. + const stat = await fs.stat(recordPath(dataDir, anchorRunId)); + expect(stat.mode & 0o077).toBe(0); + + // On disk the record is version 2 and each material's secret is ciphertext + // -- NOT the plaintext the caller handed in. The source config carries only + // the non-secret `credentialId`. + const onDisk = await readRawRecord(dataDir, anchorRunId); + expect(onDisk.version).toBe(2); + expect(onDisk.sources["step-1"]?.[0]?.credentialId).toBe("cred-x"); + const sealedSecret = onDisk.credentials?.materials[0]?.secret; + expect(typeof sealedSecret).toBe("string"); + expect(sealedSecret).not.toBe("sk-x"); + + // The scan unseals it back to the plaintext for the restored run. + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + expect(scanned[0]?.record.credentials?.materials[0]?.secret).toBe("sk-x"); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("seals every credential in the cell, top-level and per-body alike", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "bodies-tenant-example"; + await writeWorkflowRunRecord(dataDir, anchorRunId, WITH_BODIES, CIPHER); + + // On disk every material secret is ciphertext, distinct from its plaintext, + // and the AAD is namespaced by credential id so two materials never seal to + // the same ciphertext even were their plaintext to match. + const onDisk = await readRawRecord(dataDir, anchorRunId); + const sealed = new Map( + (onDisk.credentials?.materials ?? []).map((m) => [m.credentialId, m.secret]), + ); + expect(sealed.get("cred-top")).not.toBe("sk-top"); + expect(sealed.get("cred-body-a")).not.toBe("sk-body-a"); + expect(sealed.get("cred-body-b")).not.toBe("sk-body-b"); + expect(new Set(sealed.values()).size).toBe(3); + + // The scan unseals the whole cell back to the plaintext the caller handed in. + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + expect(scanned).toEqual([{ runId: anchorRunId, record: WITH_BODIES }]); - const raw = await fs.readFile(recordPath(dataDir, anchorRunId), "utf8"); - const parsed = WorkflowRunRecord(JSON.parse(raw)); - if (parsed instanceof type.errors) { - throw new Error(`record failed validation: ${parsed.summary}`); + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("soft-skips a credential decrypt failure, dropping the whole run", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "bodies-corrupt-example"; + await writeWorkflowRunRecord(dataDir, anchorRunId, WITH_BODIES, CIPHER); + + // Corrupt ONE material's sealed secret in place under the correct key. The + // other materials are intact, so this isolates a single-credential decrypt + // failure: the whole run is dropped rather than half-restored. + const onDisk = await readRawRecord(dataDir, anchorRunId); + const material = onDisk.credentials?.materials[0]; + if (material === undefined) { + throw new Error("fixture is missing a material to corrupt"); } - expect(parsed).toEqual(SOURCE_REF); + material.secret = "not-a-valid-ciphertext"; + await fs.writeFile(recordPath(dataDir, anchorRunId), JSON.stringify(onDisk, null, 2), "utf8"); + + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + expect(scanned).toEqual([]); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("round-trips a record with the optional fields absent (multi-step)", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "run_xyz-tenant-example"; + await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP, CIPHER); + + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + const restored = scanned[0]?.record; + expect(restored).toEqual(MULTI_STEP); + expect(restored !== undefined && "hubPublicKey" in restored).toBe(false); + expect(restored !== undefined && "sessionId" in restored).toBe(false); + + await fs.rm(dataDir, { recursive: true, force: true }); + }); + + test("round-trips a record that binds no credentials", async () => { + const dataDir = await makeDataDir(); + const anchorRunId = "no-creds-1"; + // A deployment whose sources need no secret carries no `credentials` cell; + // the write seals nothing and the scan unseals nothing. + const { credentials: _omit, ...noCreds } = SOURCE_REF; + void _omit; + await writeWorkflowRunRecord(dataDir, anchorRunId, noCreds, CIPHER); + + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); + const restored = scanned[0]?.record; + expect(restored).toEqual(noCreds); + expect(restored !== undefined && "credentials" in restored).toBe(false); await fs.rm(dataDir, { recursive: true, force: true }); }); @@ -157,7 +357,7 @@ describe("workflow run record store", () => { const rejects = (r: unknown): boolean => WorkflowRunRecord(r) instanceof type.errors; const base = { - version: 1, + version: 2, agentAddress: "ins_dep_bad@tenant.example", definitionId: "wf_bad", sources: SOURCE_REF.sources, @@ -182,43 +382,6 @@ describe("workflow run record store", () => { expect(rejects({ ...base, sourceRef, approvedWireHash })).toBe(false); }); - test("round-trips a schema-valid record through disk (single-step)", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "abc123-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP); - - // The record names source credentialIds, so it must not be group/world - // readable on a shared host. - const stat = await fs.stat(recordPath(dataDir, anchorRunId)); - expect(stat.mode & 0o077).toBe(0); - - const raw = await fs.readFile(recordPath(dataDir, anchorRunId), "utf8"); - const parsed = WorkflowRunRecord(JSON.parse(raw)); - if (parsed instanceof type.errors) { - throw new Error(`record failed validation: ${parsed.summary}`); - } - expect(parsed).toEqual(SINGLE_STEP); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - - test("round-trips a record with the optional fields absent (multi-step)", async () => { - const dataDir = await makeDataDir(); - const anchorRunId = "run_xyz-tenant-example"; - await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP); - - const raw = await fs.readFile(recordPath(dataDir, anchorRunId), "utf8"); - const parsed = WorkflowRunRecord(JSON.parse(raw)); - if (parsed instanceof type.errors) { - throw new Error(`record failed validation: ${parsed.summary}`); - } - expect(parsed).toEqual(MULTI_STEP); - expect("hubPublicKey" in parsed).toBe(false); - expect("sessionId" in parsed).toBe(false); - - await fs.rm(dataDir, { recursive: true, force: true }); - }); - test("overwriting a record leaves the new one and no temp orphan", async () => { const dataDir = await makeDataDir(); const anchorRunId = "rotated-1"; @@ -226,13 +389,13 @@ describe("workflow run record store", () => { // A source rotation overwrites the existing record in place. The // atomic write must replace it cleanly, leaving only the record and // no `.tmp` staging file behind. - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP); - await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP); + await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); + await writeWorkflowRunRecord(dataDir, anchorRunId, MULTI_STEP, CIPHER); const dir = path.join(dataDir, "workflow-runs", anchorRunId); expect(await fs.readdir(dir)).toEqual(["deployment.json"]); - const scanned = await scanWorkflowRunRecords(dataDir); + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); expect(scanned.map((s) => s.record)).toEqual([MULTI_STEP]); await fs.rm(dataDir, { recursive: true, force: true }); @@ -245,7 +408,7 @@ describe("workflow run record store", () => { // No-op when the record was never written. await deleteWorkflowRunRecord(dataDir, anchorRunId); - await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP); + await writeWorkflowRunRecord(dataDir, anchorRunId, SINGLE_STEP, CIPHER); expect(await fileExists(recordPath(dataDir, anchorRunId))).toBe(true); await deleteWorkflowRunRecord(dataDir, anchorRunId); @@ -260,16 +423,16 @@ describe("scanWorkflowRunRecords", () => { const dataDir = await makeDataDir(); // First boot: nothing has been deployed, so `workflow-runs/` does not // exist. That is the legitimate empty case, not an error. - expect(await scanWorkflowRunRecords(dataDir)).toEqual([]); + expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); await fs.rm(dataDir, { recursive: true, force: true }); }); test("returns every schema-valid record keyed by its directory name", async () => { const dataDir = await makeDataDir(); - await writeWorkflowRunRecord(dataDir, "dep-a", SINGLE_STEP); - await writeWorkflowRunRecord(dataDir, "dep-b", MULTI_STEP); + await writeWorkflowRunRecord(dataDir, "dep-a", SINGLE_STEP, CIPHER); + await writeWorkflowRunRecord(dataDir, "dep-b", MULTI_STEP, CIPHER); - const scanned = await scanWorkflowRunRecords(dataDir); + const scanned = await scanWorkflowRunRecords(dataDir, CIPHER); const byId = new Map(scanned.map((s) => [s.runId, s.record])); expect(byId.size).toBe(2); expect(byId.get("dep-a")).toEqual(SINGLE_STEP); @@ -278,29 +441,31 @@ describe("scanWorkflowRunRecords", () => { await fs.rm(dataDir, { recursive: true, force: true }); }); - test("soft-fails a corrupt or schema-invalid record while returning the valid ones", async () => { + test("soft-skips a record sealed under a different key", async () => { const dataDir = await makeDataDir(); - await writeWorkflowRunRecord(dataDir, "dep-valid", SINGLE_STEP); - - // A directory whose record is not valid JSON. - const corruptDir = path.join(dataDir, "workflow-runs", "dep-corrupt"); - await fs.mkdir(corruptDir, { recursive: true }); - await fs.writeFile(path.join(corruptDir, "deployment.json"), "{ not json"); - - // A directory whose record parses but fails the schema (missing fields). - const invalidDir = path.join(dataDir, "workflow-runs", "dep-invalid"); - await fs.mkdir(invalidDir, { recursive: true }); - await fs.writeFile(path.join(invalidDir, "deployment.json"), JSON.stringify({ version: 1 })); - - // A bare run directory with no record at all. - await fs.mkdir(path.join(dataDir, "workflow-runs", "dep-empty"), { - recursive: true, - }); + // A record whose credential cell was sealed under one key cannot be + // unsealed under a rotated/wrong key: the whole run is dropped as + // corruption, not half-restored. + await writeWorkflowRunRecord(dataDir, "wrong-key", SINGLE_STEP, CIPHER); + expect(await scanWorkflowRunRecords(dataDir, OTHER_KEY_CIPHER)).toEqual([]); + await fs.rm(dataDir, { recursive: true, force: true }); + }); - const scanned = await scanWorkflowRunRecords(dataDir); - expect(scanned.map((s) => s.runId)).toEqual(["dep-valid"]); - expect(scanned[0]?.record).toEqual(SINGLE_STEP); + test("soft-skips a record whose JSON is corrupt", async () => { + const dataDir = await makeDataDir(); + const dir = path.join(dataDir, "workflow-runs", "dep-corrupt"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "deployment.json"), "{ not json", "utf8"); + expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); + await fs.rm(dataDir, { recursive: true, force: true }); + }); + test("soft-skips a record that fails schema validation", async () => { + const dataDir = await makeDataDir(); + const dir = path.join(dataDir, "workflow-runs", "dep-invalid"); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "deployment.json"), JSON.stringify({ version: 2 }), "utf8"); + expect(await scanWorkflowRunRecords(dataDir, CIPHER)).toEqual([]); await fs.rm(dataDir, { recursive: true, force: true }); }); }); diff --git a/apps/sidecar/src/workflow-run-record.ts b/apps/sidecar/src/workflow-run-record.ts index 6eaf352f64..934be40f44 100644 --- a/apps/sidecar/src/workflow-run-record.ts +++ b/apps/sidecar/src/workflow-run-record.ts @@ -22,11 +22,10 @@ import { dirname, join as pathJoin } from "node:path"; import { type } from "arktype"; -import { reportError } from "@corbits/error-sink"; - import { getLogger } from "@intx/log"; +import { credentialAad, type CredentialCipher } from "@intx/types"; import { InferenceSource } from "@intx/types/runtime"; -import { SourceRefPin } from "@intx/types/sidecar"; +import { CredentialDelivery, SourceRefPin } from "@intx/types/sidecar"; import { writeFileAtomicDurable } from "./atomic-write"; @@ -42,13 +41,39 @@ function isENOENT(cause: unknown): boolean { } // The base fields every run record carries, spread into the full schema below. +// `version` is 2 for the unified credential format: `sources`/`bodySources` are +// non-secret config carrying a `credentialId` per source, and the run's secret +// material lives once in `credentials` (the delivery cell) with each material's +// secret sealed under the sidecar cipher. A pre-unification record (inline +// per-source secrets) does not satisfy this schema and is soft-skipped at the +// scan boundary; the hub re-pushes its deployment on reconnect. const workflowRunRecordBase = { - version: "1", + version: "1 | 2", agentAddress: "string > 0", definitionId: "string > 0", + // Top-level per-step inference sources. Non-secret config only: each source + // carries a `credentialId` referencing an entry in `credentials`, no inline + // secret, so this rides in the clear. sources: { "[string]": InferenceSource.array().atLeastLength(1), }, + // Per spawned-body inference-source config, keyed by the body's definition id + // (an onTrigger section or a childWorkflow child; a nested loop body folds its + // step sources into its container's table, so it is not a separate entry). + // Non-secret, like `sources`: each carries a `credentialId` into `credentials`. + // Optional: a deployment with no bodies carries none. + "bodySources?": { + "[string]": { + "[string]": InferenceSource.array().atLeastLength(1), + }, + }, + // The run's credential-material cell: the ONE at-rest home for every secret, + // inference and tool alike. `bindings` and each material's `credentialId`/ + // `providerKey`/`origin` ride in the clear; only each material's `secret` is + // sealed under the sidecar cipher (see `transformDeliveryMaterials`). Optional: + // a deployment that binds no credentials and whose sources need none carries + // none. Restored + re-delivered to the child on the pre-trigger barrier. + "credentials?": CredentialDelivery, "sessionId?": "string > 0", "hubPublicKey?": "string > 0", } as const; @@ -88,26 +113,75 @@ function recordPath(dataDir: string, runId: string): string { return pathJoin(dataDir, "workflow-runs", runId, RECORD_FILENAME); } +// The AAD column binding a credential's sealed secret to its id, so a ciphertext +// cannot be swapped between credentials (or between runs) and still decrypt. +function credentialSecretColumn(credentialId: string): string { + return `credential:${credentialId}:secret`; +} + +// Transform each credential material's `secret` in the delivery under +// `transform` (encrypt on write, decrypt on scan), binding each to its run and +// credential. The non-secret fields -- each material's `credentialId`/ +// `providerKey`/`origin` and the whole `bindings` array -- ride in the clear. +// Returns a fresh delivery so the caller's in-memory record is not mutated. A +// decrypt failure (a rotated/wrong key or a tampered blob) throws, which the +// scan caller treats as record corruption and soft-skips the whole run. +async function transformDeliveryMaterials( + delivery: CredentialDelivery, + runId: string, + transform: (secret: string, aad: string) => Promise, +): Promise { + return { + bindings: delivery.bindings, + materials: await Promise.all( + delivery.materials.map(async (material) => ({ + ...material, + secret: await transform( + material.secret, + credentialAad(runId, credentialSecretColumn(material.credentialId)), + ), + })), + ), + }; +} + /** * Persist a run record. Written after the run's slug is claimed and before * the child is spawned, so a crash mid-spawn leaves a record the boot scan * re-drives. Idempotent: it overwrites any existing record for the same run. + * + * Every credential secret in the run's `credentials` cell -- inference and tool + * alike -- is sealed under the sidecar `cipher` before it touches disk, so the + * record carries ciphertext (version 2). The `sources`/`bodySources` config is + * non-secret (each references a credential by id) and rides in the clear. */ export async function writeWorkflowRunRecord( dataDir: string, runId: string, record: WorkflowRunRecord, + cipher: CredentialCipher, ): Promise { const path = recordPath(dataDir, runId); await mkdir(dirname(path), { recursive: true }); - // Atomic + durable: this is the sole restore source for the - // run's `sources`/`hubPublicKey`, and a rotation overwrites the - // existing record in place, so an interrupted write must never expose a - // torn record the boot scan would then skip. Owner-only (0o600): the - // record names each source's `credentialId` and the run's routing - // identity, so it must not be world-readable on a shared host, matching - // the private-key writes elsewhere on the sidecar. - await writeFileAtomicDurable(path, JSON.stringify(record, null, 2), { + const sealed: WorkflowRunRecord = { + ...record, + version: 2, + ...(record.credentials !== undefined + ? { + credentials: await transformDeliveryMaterials(record.credentials, runId, (secret, aad) => + cipher.encrypt(secret, aad), + ), + } + : {}), + }; + // Atomic + durable: this is the sole restore source for the run's credential + // material, and a rotation overwrites the existing record in place, so an + // interrupted write must never expose a torn record the boot scan would then + // skip. Owner-only (0o600): the sealed secrets are ciphertext, but the record + // still names each source's provider/baseURL and the deployment's identity, so + // it stays off a shared host's world-readable set, matching the private-key + // writes elsewhere on the sidecar. + await writeFileAtomicDurable(path, JSON.stringify(sealed, null, 2), { mode: 0o600, }); } @@ -140,7 +214,10 @@ export interface ScannedWorkflowRun { * The returned `runId` is the directory name; the caller cross-checks it * against the record's own address before trusting it. */ -export async function scanWorkflowRunRecords(dataDir: string): Promise { +export async function scanWorkflowRunRecords( + dataDir: string, + cipher: CredentialCipher, +): Promise { const runsDir = pathJoin(dataDir, "workflow-runs"); let entries; try { @@ -174,14 +251,8 @@ export async function scanWorkflowRunRecords(dataDir: string): Promise + cipher.decrypt(secret, aad), + ), + }; + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + logger.warn`skipping workflow-runs/${runId}: sealed credential decrypt failed: ${reason}`; + continue; + } + } + scanned.push({ runId, record: restored }); } return scanned; } diff --git a/apps/sidecar/src/workflow-substrate-factory-abort.test.ts b/apps/sidecar/src/workflow-substrate-factory-abort.test.ts new file mode 100644 index 0000000000..07da44a100 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-abort.test.ts @@ -0,0 +1,316 @@ +// A parent abort of an in-process child tears it down LOCALLY, settling it +// FAILED, not cancelled. An in-process child writes through the workflow-run +// proxy substrate, which cannot sign the supervisor `CancelRequested` a +// control-plane cancel needs (the kind handler refuses a workflow-process-signed +// cancel), so the child never self-cancels: the parent abort aborts the child's +// own controller directly, its parked step fails, and the run settles `failed` +// under its own principal with no `CancelRequested` in the log. +// +// Both in-process spawners are covered: the terminal-only `createSidecarRunChild` +// and the suspendable `createSidecarSpawnSuspendableChild`. Each abort is driven +// while the child is live and parked on an approval -- the reachable drain-window +// path -- so the test exercises the real teardown, not a child that already +// settled. + +import { describe, test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { generateKeyPair } from "@intx/crypto"; +import type { ApprovalSnapshot, KeyPair } from "@intx/types/runtime"; +import { defineAgent } from "@intx/agent"; +import { builtinCredentialProviders, createCredentialProviderRegistry } from "@intx/harness"; +import { evaluateGrants } from "@intx/authz"; +import type { GrantRule } from "@intx/authz"; +import { + createRepoStore, + workflowRunKindHandler, + WORKFLOW_RUN_GITIGNORE_PATH, +} from "@intx/hub-sessions"; +import type { AuthorizeFn, RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions"; +import { createWorkflowRunRepoStore } from "@intx/workflow-host"; +import { + createInMemoryScheduler, + createInMemoryRepoStore, + defineWorkflow, + step, + type WorkflowDefinition, + type WorkflowEvent, +} from "@intx/workflow"; + +import { + createSidecarRunChild, + createSidecarSpawnSuspendableChild, + type SidecarChildStepInvoker, +} from "./workflow-substrate-factory"; +import { runGrantsPath } from "./run-grants"; + +const REF = "refs/heads/main"; +const DEPLOYMENT_ID = "deployment-abort"; +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: DEPLOYMENT_ID, +}; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: DEPLOYMENT_ID, +}; +const BODY_STEP_AGENT_ID = "wallet-spend"; +const CORRELATION_ID = "corr-abort-1"; +const SNAPSHOT: ApprovalSnapshot = { + name: BODY_STEP_AGENT_ID, + description: "spend from the shared wallet", + inputSchema: { amount: "number" }, + arguments: { amount: 100 }, +}; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +// `buildChildRunEnv` reads each spawned run's `assets/workflow//sources.json` +// eagerly; the park-forever invoker never resolves inference, but the read still +// happens, so stage a minimal sources file for the body definition id. +let bodySourcesDataDir: string; + +const noopOnEvent = (): void => { + /* the event sink is not asserted in these tests */ +}; + +beforeAll(async () => { + signingKey = await generateKeyPair(); + bodySourcesDataDir = await makeTempDir("abort-assets-"); + const dir = path.join(bodySourcesDataDir, "assets", "workflow", "body"); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile( + path.join(dir, "sources.json"), + JSON.stringify({ + s: [ + { + id: "anthropic:m", + provider: "anthropic", + baseURL: "http://localhost:1", + credentialId: "sk-x", + model: "m", + }, + ], + }), + ); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { + /* best effort */ + }); + } +}); + +async function makeTempDir(prefix: string): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(d); + return d; +} + +function grant(resource: string, action: string): GrantRule { + return { + id: `grant-${resource}-${action}`, + resource, + action, + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; +} + +function bodyDefinition(id: string): WorkflowDefinition { + const agent = defineAgent({ + id: BODY_STEP_AGENT_ID, + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }); + return defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { s: step({ agent }) }, + }); +} + +// An invoker that parks the body step on an approval and never returns an +// output: the child stays parked until the parent abort cancels it. +const parkForever: SidecarChildStepInvoker = async () => ({ + suspend: { + correlationId: CORRELATION_ID, + kind: "approval", + approvalSnapshot: SNAPSHOT, + }, +}); + +const evaluateGrantsAdapter: Parameters< + typeof createSidecarRunChild +>[0]["evaluateGrants"] = async ({ resource, action, grants }) => { + const result = await evaluateGrants( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the snapshot's grants are typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar, so the seeded rows narrow here + [...(grants as readonly GrantRule[])], + resource, + action, + ); + return { effect: result.effect, matchingGrants: [], resolvedBy: null }; +}; + +async function makeSubstrate(prefix: string): Promise> { + const dataDir = await makeTempDir(prefix); + const substrate = createRepoStore({ + dataDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + return substrate; +} + +async function seedRunGrants( + substrate: ReturnType, + runId: string, + grants: readonly GrantRule[], +): Promise { + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { + [runGrantsPath(runId)]: JSON.stringify({ grants }, null, 2), + }, + message: `seed grants for ${runId}`, + }); +} + +function sharedDeps(substrate: ReturnType) { + return { + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: REF, + principal: PRINCIPAL, + scheduler: createInMemoryScheduler({ + repoStore: createInMemoryRepoStore(), + clock: () => new Date(), + }), + invokeStep: parkForever, + evaluateGrants: evaluateGrantsAdapter, + dataDir: bodySourcesDataDir, + bodySources: {}, + credentialProviders: createCredentialProviderRegistry(builtinCredentialProviders()), + }; +} + +function reader(substrate: ReturnType) { + return createWorkflowRunRepoStore({ + substrate, + repoId: WORKFLOW_RUN_REPO_ID, + principal: PRINCIPAL, + ref: REF, + }); +} + +// Poll the child's durable log until its step parks (a SignalAwaited is +// committed), so the abort lands while the child is genuinely in flight. +async function waitForChildPark( + substrate: ReturnType, + childRunId: string, +): Promise { + const r = reader(substrate); + for (let i = 0; i < 300; i += 1) { + const events = await r.read(childRunId); + if (events.some((e) => e.kind === "SignalAwaited")) return; + await new Promise((res) => setTimeout(res, 10)); + } + throw new Error(`timed out waiting for child ${childRunId} to park`); +} + +// Assert the child's durable log shows a LOCAL fail teardown: a StepFailed and +// the terminal RunFailed, and NO CancelRequested (the in-process child cannot +// sign one, so the teardown never writes one). +async function expectLocalFailTeardown( + substrate: ReturnType, + childRunId: string, +): Promise { + const events: readonly WorkflowEvent[] = await reader(substrate).read(childRunId); + expect(events.some((e) => e.kind === "StepFailed")).toBe(true); + expect(events.some((e) => e.kind === "RunFailed")).toBe(true); + expect(events.some((e) => e.kind === "CancelRequested")).toBe(false); +} + +describe("in-process child parent-abort", () => { + test("createSidecarRunChild settles failed via local teardown", async () => { + const substrate = await makeSubstrate("abort-terminal-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [grant(`tool:${BODY_STEP_AGENT_ID}`, "invoke")]); + + const runChild = createSidecarRunChild(sharedDeps(substrate)); + const abort = new AbortController(); + const childRunId = "run-body-terminal"; + const settled = runChild( + { + definition: bodyDefinition("body"), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "s", + signal: abort.signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ); + + await waitForChildPark(substrate, childRunId); + abort.abort(); + + const result = await settled; + expect(result.terminalStatus).toBe("failed"); + await expectLocalFailTeardown(substrate, childRunId); + }); + + test("createSidecarSpawnSuspendableChild settles failed via local teardown", async () => { + const substrate = await makeSubstrate("abort-suspendable-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [grant(`tool:${BODY_STEP_AGENT_ID}`, "invoke")]); + + const spawn = createSidecarSpawnSuspendableChild(sharedDeps(substrate)); + const abort = new AbortController(); + const childRunId = "run-body-suspendable"; + const handle = await spawn( + { + definition: bodyDefinition("body"), + definitionRef: REF, + childRunId, + input: { text: "event-0" }, + parentRunId, + parentStepId: "section", + signal: abort.signal, + depth: 0, + maxChildSpawnDepth: 32, + }, + () => undefined, + ); + + // The body parked on an approval and the handle surfaced it: the child is + // live. Abort the parent now. + const parked = await handle.next(); + expect(parked.kind).toBe("park"); + abort.abort(); + + const terminal = await handle.next(); + expect(terminal.kind).toBe("terminal"); + if (terminal.kind !== "terminal") throw new Error("expected a terminal"); + expect(terminal.terminalStatus).toBe("failed"); + await expectLocalFailTeardown(substrate, childRunId); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-adapter-load.test.ts b/apps/sidecar/src/workflow-substrate-factory-adapter-load.test.ts new file mode 100644 index 0000000000..03d1af2998 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-adapter-load.test.ts @@ -0,0 +1,62 @@ +import { describe, test, expect } from "bun:test"; +import type { LastCycleSource } from "@intx/types/runtime"; +import type { AdapterFactory, ProviderAdapter } from "@intx/inference"; +import { loadAdapterRegistry } from "@intx/inference/providers"; + +import { parseAdapterManifest } from "./workflow-substrate-factory"; + +// Failure-isolation hedge for the cross-process custom-adapter path: +// proves the child-side load path (parse the serialized manifest, then +// `loadAdapterRegistry` with an injected importer, then resolve) in +// isolation, so a red forked-child integration test points at the +// spawn/IPC plumbing rather than at adapter loading. No fixture module +// is loaded from disk -- the importer is injected. + +function createStubAdapter(url: string): ProviderAdapter { + return { + buildRequest: () => ({ url, headers: {}, body: "" }), + parseResponse: () => [], + parseJSONResponse: () => [], + }; +} + +function createSource(provider: string): LastCycleSource { + return { sourceId: "src-1", provider, model: "test-model" }; +} + +describe("child-side adapter manifest load path", () => { + test("parses a serialized manifest and resolves the custom adapter", async () => { + const customAdapter = createStubAdapter("https://custom.invalid"); + const make: AdapterFactory = () => customAdapter; + + const manifest = parseAdapterManifest( + JSON.stringify([{ provider: "custom", specifier: "custom-pkg", export: "make" }]), + ); + const registry = await loadAdapterRegistry(manifest, { + import: (specifier) => { + expect(specifier).toBe("custom-pkg"); + return Promise.resolve({ make }); + }, + }); + + expect(registry.resolve(createSource("custom"))).toBe(customAdapter); + // Built-ins remain resolvable alongside the custom provider. + expect(typeof registry.resolve(createSource("anthropic")).buildRequest).toBe("function"); + }); + + test("an empty serialized manifest yields just the built-ins", async () => { + const registry = await loadAdapterRegistry(parseAdapterManifest("[]")); + + expect(registry.has("anthropic")).toBe(true); + expect(registry.has("custom")).toBe(false); + }); + + test("a malformed manifest fails loud naming the env key", () => { + expect(() => parseAdapterManifest("{not json")).toThrow( + /SIDECAR_ADAPTER_MANIFEST is not valid JSON/, + ); + expect(() => parseAdapterManifest(JSON.stringify([{ provider: 1 }]))).toThrow( + /SIDECAR_ADAPTER_MANIFEST failed validation/, + ); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-child-depth.test.ts b/apps/sidecar/src/workflow-substrate-factory-child-depth.test.ts new file mode 100644 index 0000000000..8492744297 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-child-depth.test.ts @@ -0,0 +1,301 @@ +// The childWorkflow spawn-depth ceiling on the REAL in-process spawn path. +// +// `createSidecarRunChild` recurses on itself: a childWorkflow runs its child in +// the same process against the same shared workflow-run repo, threading `depth` +// by value through each rung (no serialization, no reset). So this harness -- +// a real on-disk substrate plus the real `createSidecarRunChild` / +// `createInMemorySpawnChild` / `runtimeRun` seam -- exercises the identical spawn +// path a deployed chain runs; the hub deploy-frame and the workflow-host +// subprocess wrapper are the only things a full roundtrip would add, and neither +// is where the depth guard lives. +// +// A nested chain lets depth ACCUMULATE rung over rung until the guard fires, +// rather than pre-loading a single `depth + 1 > max` arithmetic check (that is +// covered by the runLocal and child-depth unit tests). With the ceiling lowered +// to 2, the parent (depth 0) spawns outer (1) and outer spawns mid (2) for real +// -- proving depth climbs past 0 -- and only mid spawning leaf (depth 3) trips +// the guard, landing a clean `StepFailed` that names the offending depth. + +import { describe, test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { defineAgent } from "@intx/agent"; +import { builtinCredentialProviders, createCredentialProviderRegistry } from "@intx/harness"; +import { evaluateGrants } from "@intx/authz"; +import type { GrantRule } from "@intx/authz"; +import { + createRepoStore, + workflowRunKindHandler, + WORKFLOW_RUN_GITIGNORE_PATH, +} from "@intx/hub-sessions"; +import type { AuthorizeFn, RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions"; +import { createWorkflowRunRepoStore } from "@intx/workflow-host"; +import { + createInMemoryScheduler, + createInMemoryRepoStore, + childWorkflow, + defineWorkflow, + step, + type WorkflowDefinition, + type WorkflowEvent, +} from "@intx/workflow"; + +import { createSidecarRunChild, type SidecarChildStepInvoker } from "./workflow-substrate-factory"; +import { runGrantsPath } from "./run-grants"; + +const REF = "refs/heads/main"; +const DEPLOYMENT_ID = "deployment-child-depth"; +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: DEPLOYMENT_ID, +}; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: DEPLOYMENT_ID, +}; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +// `buildChildRunEnv` reads each spawned rung's `sources.json` eagerly, keyed by +// the rung's rewritten ref -- the `__` handle the deploy +// mints, which accumulates as the chain descends. The intermediate rungs carry +// no agent step, but the read still happens, so stage a minimal file for every +// rung that runs its env: the top (`depth-parent`), outer +// (`depth-parent__spawn`), and mid (`depth-parent__spawn__spawn`). The leaf rung +// is never reached because the guard fires before mid spawns it. +const RUNG_SOURCE_REFS = [ + "depth-parent", + "depth-parent__spawn", + "depth-parent__spawn__spawn", +] as const; +let sourcesDataDir: string; + +// The guard fires before any step runs, so no invoker is ever called. +const noopInvoker: SidecarChildStepInvoker = () => { + throw new Error("child-depth: no step should run before the guard fires"); +}; +const noopOnEvent = (): void => { + /* the event sink is not asserted in this test */ +}; + +beforeAll(async () => { + signingKey = await generateKeyPair(); + sourcesDataDir = await makeTempDir("child-depth-assets-"); + for (const ref of RUNG_SOURCE_REFS) { + await stageSources(ref); + } +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { + /* best effort */ + }); + } +}); + +async function makeTempDir(prefix: string): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(d); + return d; +} + +async function stageSources(ref: string): Promise { + const dir = path.join(sourcesDataDir, "assets", "workflow", ref); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile( + path.join(dir, "sources.json"), + JSON.stringify({ + work: [ + { + id: "anthropic:m", + provider: "anthropic", + baseURL: "http://localhost:1", + credentialId: "sk-x", + model: "m", + }, + ], + }), + ); +} + +function grant(resource: string, action: string): GrantRule { + return { + id: `grant-${resource}-${action}`, + resource, + action, + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; +} + +const evaluateGrantsAdapter: Parameters< + typeof createSidecarRunChild +>[0]["evaluateGrants"] = async ({ resource, action, grants }) => { + const result = await evaluateGrants( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the snapshot's grants are typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar, so the seeded rows narrow here + [...(grants as readonly GrantRule[])], + resource, + action, + ); + return { effect: result.effect, matchingGrants: [], resolvedBy: null }; +}; + +// parent -> outer -> mid -> leaf, each spawning the next inline child. Depths: +// parent 0, outer 1, mid 2, leaf 3. Only the leaf carries an agent step; the +// guard fires before it ever runs. +function nestedChain(): WorkflowDefinition { + const leaf = defineWorkflow({ + id: "depth-leaf", + trigger: { type: "manual" }, + steps: { + work: step({ + agent: defineAgent({ + id: "depth-leaf-agent", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }), + }), + }, + }); + const mid = defineWorkflow({ + id: "depth-mid", + trigger: { type: "manual" }, + steps: { spawn: childWorkflow({ definition: leaf }) }, + }); + const outer = defineWorkflow({ + id: "depth-outer", + trigger: { type: "manual" }, + steps: { spawn: childWorkflow({ definition: mid }) }, + }); + return defineWorkflow({ + id: "depth-parent", + trigger: { type: "manual" }, + steps: { spawn: childWorkflow({ definition: outer }) }, + }); +} + +async function makeSubstrate(): Promise> { + const dataDir = await makeTempDir("child-depth-substrate-"); + const substrate = createRepoStore({ + dataDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + return substrate; +} + +async function seedRunGrants( + substrate: ReturnType, + runId: string, + grants: readonly GrantRule[], +): Promise { + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { + [runGrantsPath(runId)]: JSON.stringify({ grants }, null, 2), + }, + message: `seed grants for ${runId}`, + }); +} + +function reader(substrate: ReturnType) { + return createWorkflowRunRepoStore({ + substrate, + repoId: WORKFLOW_RUN_REPO_ID, + principal: PRINCIPAL, + ref: REF, + }); +} + +// The runId of the single child a rung spawned (its `ChildSpawned`). +async function spawnedChildRunId( + substrate: ReturnType, + runId: string, +): Promise { + const events: readonly WorkflowEvent[] = await reader(substrate).read(runId); + const spawned = events.find((e) => e.kind === "ChildSpawned"); + if (spawned === undefined || spawned.kind !== "ChildSpawned") { + throw new Error(`run ${runId} has no ChildSpawned event`); + } + return spawned.childRunId; +} + +describe("createSidecarRunChild spawn-depth ceiling", () => { + test("a chain past the ceiling fails loud naming the offending depth", async () => { + const substrate = await makeSubstrate(); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [grant("inference.source:anthropic:m", "invoke")]); + + const runChild = createSidecarRunChild({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: REF, + principal: PRINCIPAL, + scheduler: createInMemoryScheduler({ + repoStore: createInMemoryRepoStore(), + clock: () => new Date(), + }), + invokeStep: noopInvoker, + evaluateGrants: evaluateGrantsAdapter, + dataDir: sourcesDataDir, + bodySources: {}, + credentialProviders: createCredentialProviderRegistry(builtinCredentialProviders()), + }); + + // Run the top of the chain at depth 0 with the ceiling lowered to 2. + const topRunId = "run-top"; + const result = await runChild( + { + definition: nestedChain(), + definitionRef: REF, + childRunId: topRunId, + input: null, + parentRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 0, + maxChildSpawnDepth: 2, + }, + noopOnEvent, + ); + + // The top run settled failed: the deep spawn was rejected and the failure + // propagated up every rung's spawn step. + expect(result.terminalStatus).toBe("failed"); + + // Depth climbed for real: the parent spawned outer (depth 1) and outer + // spawned mid (depth 2) before the guard fired -- two successful in-process + // spawns, not a single pre-loaded arithmetic trip. + const outerRunId = await spawnedChildRunId(substrate, topRunId); + const midRunId = await spawnedChildRunId(substrate, outerRunId); + + // mid (depth 2) spawning leaf (depth 3) tripped the guard: its spawn step + // failed loud, and the error names the offending depth and ceiling verbatim. + // mid never committed a ChildSpawned (the guard fires before it). + const midEvents: readonly WorkflowEvent[] = await reader(substrate).read(midRunId); + expect(midEvents.some((e) => e.kind === "ChildSpawned")).toBe(false); + const midStepFailed = midEvents.find((e) => e.kind === "StepFailed"); + if (midStepFailed === undefined || midStepFailed.kind !== "StepFailed") { + throw new Error(`mid run ${midRunId} has no StepFailed event`); + } + expect(midStepFailed.error.message).toContain( + 'childWorkflow spawn depth 3 exceeds the maximum 2 at step "spawn"', + ); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-child-grants.test.ts b/apps/sidecar/src/workflow-substrate-factory-child-grants.test.ts new file mode 100644 index 0000000000..5d59d608d9 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-child-grants.test.ts @@ -0,0 +1,441 @@ +// A spawned child inherits the grants of the run that spawned it, CAPPED at +// what the child body itself declares. +// +// `createSidecarRunChild` reads the parent run's +// `runs//grants.json` as the ceiling, re-walks the child body to +// learn what it declares, and binds the child's `env.authorize` to the +// intersection: a parent grant the child body declares survives, a parent-only +// grant the child never declares is dropped. It persists that same capped set +// under the child's own `runs//grants.json`, so a grandchild's +// ceiling is the capped set -- not the raw parent set. A child whose parent has +// no grants file fails closed at spawn. +// +// The substrate here is a real on-disk workflow-run repo. The child's injected +// `invokeStep` calls the credentials-backed `authorize` the runtime env +// carries, so a declared resource resolves `allow` and an undeclared one +// resolves fail-closed -- exercising the cap end to end. + +import { describe, test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { generateKeyPair } from "@intx/crypto"; +import type { KeyPair } from "@intx/types/runtime"; +import { defineAgent } from "@intx/agent"; +import { builtinCredentialProviders, createCredentialProviderRegistry } from "@intx/harness"; +import { evaluateGrants } from "@intx/authz"; +import type { GrantRule } from "@intx/authz"; +import { + createRepoStore, + workflowRunKindHandler, + WORKFLOW_RUN_GITIGNORE_PATH, +} from "@intx/hub-sessions"; +import type { AuthorizeFn, RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions"; +import { + createInMemoryScheduler, + createInMemoryRepoStore, + defineWorkflow, + step, + type StepInvokeResult, + type WorkflowDefinition, +} from "@intx/workflow"; + +import { createSidecarRunChild, type SidecarChildStepInvoker } from "./workflow-substrate-factory"; +import { readRunGrants, runGrantsPath } from "./run-grants"; + +const REF = "refs/heads/main"; +const DEPLOYMENT_ID = "deployment-child-grants"; +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: DEPLOYMENT_ID, +}; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: DEPLOYMENT_ID, +}; + +// The child body's single agent declares this inference source, so the +// capability walk emits it as a grant the child body declares -- a parent +// grant for it survives the cap. +const DECLARED_RESOURCE = "inference.source:anthropic:m"; +// The child body declares nothing that covers this, so a parent grant for it +// is dropped from the child's inherited set. +const UNDECLARED_RESOURCE = "tool:parent-only"; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +// The real child runtime resolves each step's inference source from a staged +// `assets/workflow//sources.json`. These tests use a recording invoker +// that ignores the sources, but `buildChildRunEnv` reads the file eagerly, so +// stage a minimal one per child definition id these tests spawn. +let childSourcesDataDir: string; + +// These tests assert grant capping, not event emission, so the sink is noop. +const noopOnEvent = (): void => { + /* the event sink is not asserted in these tests */ +}; + +async function stageChildSources(defId: string): Promise { + const dir = path.join(childSourcesDataDir, "assets", "workflow", defId); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile( + path.join(dir, "sources.json"), + JSON.stringify({ + s: [ + { + id: "anthropic:m", + provider: "anthropic", + baseURL: "http://localhost:1", + credentialId: "sk-x", + model: "m", + }, + ], + }), + ); +} + +beforeAll(async () => { + signingKey = await generateKeyPair(); + childSourcesDataDir = await makeTempDir("child-grants-assets-"); + await stageChildSources("child-wf"); + await stageChildSources("grandchild-wf"); +}); + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { + /* best effort */ + }); + } +}); + +async function makeTempDir(prefix: string): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(d); + return d; +} + +function grant(resource: string, action: string): GrantRule { + return { + id: `grant-${resource}-${action}`, + resource, + action, + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; +} + +// The one-step child definition every spawn in these tests runs. Its single +// agent is toolless and declares the anthropic:m inference source, so +// `DECLARED_RESOURCE` is the one grant the child body declares. +const CHILD_STEP_AGENT_ID = "wallet-spend"; +function childDefinition(id: string): WorkflowDefinition { + const agent = defineAgent({ + id: CHILD_STEP_AGENT_ID, + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }); + return defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { s: step({ agent }) }, + }); +} + +// Grant evaluator that delegates the decision to `@intx/authz` against the +// credentials snapshot's grants alone. Unlike the production adapter it does +// NOT merge any per-step tool-mark floor grants, so a decision here reflects +// only the capped grant set the child inherited. +const evaluateGrantsAdapter: SidecarRunChildDepsEvaluator = async ({ + resource, + action, + grants, +}) => { + const result = await evaluateGrants( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the snapshot's grants are typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar, so the seeded rows narrow here + [...(grants as readonly GrantRule[])], + resource, + action, + ); + return { effect: result.effect, matchingGrants: [], resolvedBy: null }; +}; +type SidecarRunChildDepsEvaluator = Parameters[0]["evaluateGrants"]; + +// Build a real on-disk workflow-run substrate and seed its genesis tree. +async function makeSubstrate(prefix: string): Promise> { + const dataDir = await makeTempDir(prefix); + const substrate = createRepoStore({ + dataDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + return substrate; +} + +// Seed a run's grants file the way the hub's `run.grants` delivery does: a +// single `runs//grants.json` under the workflow-run repo. +async function seedRunGrants( + substrate: ReturnType, + runId: string, + grants: readonly GrantRule[], +): Promise { + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { + [runGrantsPath(runId)]: JSON.stringify({ grants }, null, 2), + }, + message: `seed grants for ${runId}`, + }); +} + +// An invoker that authorizes a fixed list of resources against the child's env +// authorize and records each decision, so a test can assert exactly which +// inherited (capped) grants the child holds. +function recordingInvoker( + record: { decisions: { resource: string; effect: string | null }[] }, + probeResources: readonly string[], +): SidecarChildStepInvoker { + return async (req, authorize, _sourcesRef, _onEvent): Promise => { + for (const resource of probeResources) { + const decision = await authorize(resource, "invoke", req.authzContext); + record.decisions.push({ resource, effect: decision.effect }); + } + return { output: null }; + }; +} + +function makeRunChild( + substrate: ReturnType, + invokeStep: SidecarChildStepInvoker, +): ReturnType { + return createSidecarRunChild({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: REF, + principal: PRINCIPAL, + scheduler: createInMemoryScheduler({ + repoStore: createInMemoryRepoStore(), + clock: () => new Date(), + }), + invokeStep, + evaluateGrants: evaluateGrantsAdapter, + dataDir: childSourcesDataDir, + bodySources: {}, + credentialProviders: createCredentialProviderRegistry(builtinCredentialProviders()), + }); +} + +describe("createSidecarRunChild grant capping", () => { + test("a child's inherited grants are capped at what its body declares", async () => { + const substrate = await makeSubstrate("child-grants-cap-"); + const parentRunId = "run-parent"; + // Parent holds a grant the child body declares and one it does not. + await seedRunGrants(substrate, parentRunId, [ + grant(DECLARED_RESOURCE, "invoke"), + grant(UNDECLARED_RESOURCE, "invoke"), + ]); + + const record = { + decisions: [] as { resource: string; effect: string | null }[], + }; + const runChild = makeRunChild( + substrate, + recordingInvoker(record, [DECLARED_RESOURCE, UNDECLARED_RESOURCE]), + ); + + const childRunId = "run-child"; + const result = await runChild( + { + definition: childDefinition("child-wf"), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ); + + expect(result.terminalStatus).toBe("completed"); + // The declared resource resolves `allow`; the undeclared one was dropped + // from the child's inherited set and resolves fail-closed `null`. The + // positive control (declared -> allow) proves the null is a genuine cap, + // not an empty grant view that would deny everything. + expect(record.decisions).toEqual([ + { resource: DECLARED_RESOURCE, effect: "allow" }, + { resource: UNDECLARED_RESOURCE, effect: null }, + ]); + // The child persisted ONLY the declared grant as its own file, so a + // grandchild inherits the capped set rather than the raw parent set. + const childGrants = await readRunGrants({ + repoStore: substrate, + anchorRunId: DEPLOYMENT_ID, + runId: childRunId, + }); + expect(childGrants).toEqual([grant(DECLARED_RESOURCE, "invoke")]); + }); + + test("the grandchild ceiling is the capped set, not the raw parent set", async () => { + const substrate = await makeSubstrate("child-grants-multihop-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [ + grant(DECLARED_RESOURCE, "invoke"), + grant(UNDECLARED_RESOURCE, "invoke"), + ]); + + const record = { + decisions: [] as { resource: string; effect: string | null }[], + }; + const runChild = makeRunChild( + substrate, + recordingInvoker(record, [DECLARED_RESOURCE, UNDECLARED_RESOURCE]), + ); + + // Hop 1: parent -> child. Writes runs/run-child/grants.json (capped). + const childRunId = "run-child"; + await runChild( + { + definition: childDefinition("child-wf"), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ); + + // Hop 2: child -> grandchild. The grandchild's parent is the child, so it + // reads the child's capped grants file as its ceiling. + const grandchildRunId = "run-grandchild"; + const grandResult = await runChild( + { + definition: childDefinition("grandchild-wf"), + definitionRef: REF, + childRunId: grandchildRunId, + input: null, + parentRunId: childRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ); + + expect(grandResult.terminalStatus).toBe("completed"); + // Both hops authorize the declared resource `allow`; the undeclared one is + // absent at BOTH hops -- dropped at hop 1 and never reachable at hop 2, + // because the grandchild's ceiling is the child's capped file. + expect(record.decisions).toEqual([ + { resource: DECLARED_RESOURCE, effect: "allow" }, + { resource: UNDECLARED_RESOURCE, effect: null }, + { resource: DECLARED_RESOURCE, effect: "allow" }, + { resource: UNDECLARED_RESOURCE, effect: null }, + ]); + const grandchildGrants = await readRunGrants({ + repoStore: substrate, + anchorRunId: DEPLOYMENT_ID, + runId: grandchildRunId, + }); + expect(grandchildGrants).toEqual([grant(DECLARED_RESOURCE, "invoke")]); + }); + + test("an existing child grants file is read back, not recomputed (write-once)", async () => { + const substrate = await makeSubstrate("child-grants-write-once-"); + const parentRunId = "run-parent"; + // The parent holds the grant the child body declares, so a RECOMPUTE would + // cap to and persist `[DECLARED_RESOURCE]`. + await seedRunGrants(substrate, parentRunId, [grant(DECLARED_RESOURCE, "invoke")]); + + // Pre-seed the CHILD's own grants file with a sentinel the cap would never + // produce (the child body declares `anthropic:m`, not this). A run's + // authorization ceiling is fixed at birth, so a re-spawn against the same + // childRunId must READ THIS BACK rather than recompute -- the property that + // keeps a resume re-drive from racing/clobbering the run's event subtree. + const childRunId = "run-child"; + const sentinel = grant("inference.source:sentinel:preseeded", "invoke"); + await seedRunGrants(substrate, childRunId, [sentinel]); + + const record = { + decisions: [] as { resource: string; effect: string | null }[], + }; + const runChild = makeRunChild(substrate, recordingInvoker(record, [DECLARED_RESOURCE])); + + const result = await runChild( + { + definition: childDefinition("child-wf"), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ); + + expect(result.terminalStatus).toBe("completed"); + // The persisted file is untouched -- the sentinel, not a recomputed + // `[DECLARED_RESOURCE]`. + const childGrants = await readRunGrants({ + repoStore: substrate, + anchorRunId: DEPLOYMENT_ID, + runId: childRunId, + }); + expect(childGrants).toEqual([sentinel]); + // The child authorized against the read-back sentinel, so the body's + // declared resource -- which the sentinel does NOT cover -- fails closed. + expect(record.decisions).toEqual([{ resource: DECLARED_RESOURCE, effect: null }]); + }); + + test("a child whose parent has no grants file fails closed at spawn", async () => { + const substrate = await makeSubstrate("child-grants-absent-"); + // No grants file seeded for the parent run. + const parentRunId = "run-parent-ungranted"; + + const record = { + decisions: [] as { resource: string; effect: string | null }[], + }; + const runChild = makeRunChild(substrate, recordingInvoker(record, [DECLARED_RESOURCE])); + + await expect( + runChild( + { + definition: childDefinition("child-wf"), + definitionRef: REF, + childRunId: "run-child", + input: null, + parentRunId, + parentStepId: "s", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 32, + }, + noopOnEvent, + ), + ).rejects.toThrow(/has no grants file/); + // The child never ran a step, so no authorize decision was recorded. + expect(record.decisions).toEqual([]); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-parked-approval.test.ts b/apps/sidecar/src/workflow-substrate-factory-parked-approval.test.ts new file mode 100644 index 0000000000..81571f6fd1 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-parked-approval.test.ts @@ -0,0 +1,395 @@ +// `loadParkedApproval` durable reads: the sidecar binding recovers a parked +// correlation's approval snapshot for the child's re-registration enumeration. +// +// Two layouts, verified against the production read helpers: +// - COLD (multi-step): the snapshot lives in the per-attempt isogit step +// store on disk; the read loads it, and returns undefined (without +// manufacturing a repo) when the store dir is absent. +// - WARM (single-step): the snapshot lives in the durable conversation +// store, mirrored to the workflow-run substrate under `agent-state//`. +// The read reconstructs it from the substrate WITHOUT going through the live +// registry -- proving a respawned child (whose live store is unbuilt) still +// recovers the snapshot. + +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import type { ApprovalSnapshot, PendingOperation } from "@intx/types/runtime"; +import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions/substrate"; +import { createIsogitStore } from "@intx/storage-isogit/node"; + +import { + readColdParkedApprovalSnapshot, + readColdParkedPendingOperations, + readWarmParkedApprovalSnapshot, + readWarmParkedPendingOperations, + stepStorageRoot, + toParkedApprovalOps, +} from "./workflow-substrate-factory"; +import { createDurableConversationStore } from "./conversation-state"; + +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: "parked-approval", +}; +const WORKFLOW_RUN_REF = "refs/heads/main"; +const PRINCIPAL: Principal = { kind: "workflow-process" }; +const EMPTY_USAGE = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, +}; + +const snapshot: ApprovalSnapshot = { + name: "charge_card", + description: "Charge the customer's card", + inputSchema: { type: "object" }, + arguments: { amount: 100 }, +}; + +function pendingApproval( + correlationId: string, + approvalSnapshot?: ApprovalSnapshot, +): PendingOperation { + return { + correlationId, + kind: "approval", + registeredAt: 0, + gateId: `gate-${correlationId}`, + ...(approvalSnapshot !== undefined ? { approvalSnapshot } : {}), + }; +} + +async function makeTempDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "parked-approval-")); +} + +const testSigner = (payload: string): Promise => Promise.resolve(`sig:${payload.length}`); + +describe("readColdParkedApprovalSnapshot", () => { + test("loads the snapshot from a parked step's on-disk store", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "run-1", + stepId: "s", + attempt: 1, + }; + const store = await createIsogitStore(stepStorageRoot(coordinate), testSigner); + await store.writeMetadata({ + pendingOperations: [pendingApproval("corr-1", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + + const got = await readColdParkedApprovalSnapshot({ + ...coordinate, + correlationId: "corr-1", + }); + expect(got).toEqual(snapshot); + }); + + test("returns undefined without creating a repo when the store is absent", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "missing", + stepId: "s", + attempt: 1, + }; + + const got = await readColdParkedApprovalSnapshot({ + ...coordinate, + correlationId: "corr-x", + }); + expect(got).toBeUndefined(); + // The read is a read: a missing store is never manufactured on disk. + await expect(fs.stat(stepStorageRoot(coordinate))).rejects.toThrow(); + }); + + test("returns undefined for a correlation with no matching pending op", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "run-2", + stepId: "s", + attempt: 1, + }; + const store = await createIsogitStore(stepStorageRoot(coordinate), testSigner); + await store.writeMetadata({ + pendingOperations: [pendingApproval("corr-a", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + + const got = await readColdParkedApprovalSnapshot({ + ...coordinate, + correlationId: "corr-other", + }); + expect(got).toBeUndefined(); + }); + + test("returns undefined for a matching op that carries no snapshot", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "run-3", + stepId: "s", + attempt: 1, + }; + const store = await createIsogitStore(stepStorageRoot(coordinate), testSigner); + await store.writeMetadata({ + pendingOperations: [pendingApproval("corr-1")], + tokenUsage: EMPTY_USAGE, + }); + + const got = await readColdParkedApprovalSnapshot({ + ...coordinate, + correlationId: "corr-1", + }); + expect(got).toBeUndefined(); + }); +}); + +/** + * Read every file under `/` into a path->bytes map, keyed by + * the repo-relative path, so a `writeTreePreservingPrefix` merge callback sees + * the prior subtree the way the real substrate presents it. + */ +async function readPrefixEntries( + repoDir: string, + prefix: string, +): Promise> { + const entries = new Map(); + const prefixDir = path.join(repoDir, prefix); + let names: string[]; + try { + names = await fs.readdir(prefixDir, { recursive: true }); + } catch { + return entries; + } + for (const name of names) { + const full = path.join(prefixDir, name); + if (!(await fs.stat(full)).isFile()) continue; + entries.set(`${prefix}${name}`, await fs.readFile(full)); + } + return entries; +} + +/** + * A substrate stub that persists `writeTreePreservingPrefix` to disk under + * `getRepoDir`, so a durable-conversation mirror round-trips through the real + * checkpoint/WAL layout the read reconstructs from. Any other method surfaces + * as a precise failure. + */ +function createStubSubstrate(baseDir: string): RepoStore { + const repoDirFor = (repoId: RepoId): string => path.join(baseDir, repoId.kind, repoId.id); + const stub: Partial = { + getRepoDir: repoDirFor, + async writeTreePreservingPrefix(_principal, repoId, _ref, args) { + const repoDir = repoDirFor(repoId); + const existing = await readPrefixEntries(repoDir, args.preservePrefix); + const merged = await args.merge(existing); + await fs.rm(path.join(repoDir, args.preservePrefix), { + recursive: true, + force: true, + }); + for (const [relPath, content] of Object.entries(merged)) { + const full = path.join(repoDir, relPath); + await fs.mkdir(path.dirname(full), { recursive: true }); + await fs.writeFile(full, content); + } + return { commitSha: "deadbeefcafef00d", newlyTerminalRuns: [] }; + }, + }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- test stub; missing methods surface as a precise failure via the proxy + return new Proxy(stub as RepoStore, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (value !== undefined) return value; + return () => { + throw new Error(`stub substrate: ${String(prop)} not implemented for this test`); + }; + }, + }); +} + +describe("readWarmParkedApprovalSnapshot", () => { + test("reconstructs the snapshot from the durable substrate mirror", async () => { + const substrate = createStubSubstrate(await makeTempDir()); + const store = await createDurableConversationStore({ + localStoreDir: await makeTempDir(), + signer: testSigner, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: "s", + }); + // Suspend-time state: the parked approval's pending op, then mirrored to + // the substrate at the run boundary. No live registry is involved on the + // read side, mirroring a respawned child whose warm store is unbuilt. + await store.storage.writeMetadata({ + pendingOperations: [pendingApproval("corr-1", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + await store.mirrorToSubstrate(); + + const got = await readWarmParkedApprovalSnapshot({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + stepId: "s", + correlationId: "corr-1", + }); + expect(got).toEqual(snapshot); + }); + + test("returns undefined when no durable state exists for the agent", async () => { + const substrate = createStubSubstrate(await makeTempDir()); + + const got = await readWarmParkedApprovalSnapshot({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + stepId: "never-ran", + correlationId: "corr-x", + }); + expect(got).toBeUndefined(); + }); + + test("returns undefined for a matching op that carries no snapshot", async () => { + const substrate = createStubSubstrate(await makeTempDir()); + const store = await createDurableConversationStore({ + localStoreDir: await makeTempDir(), + signer: testSigner, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: "s", + }); + await store.storage.writeMetadata({ + pendingOperations: [pendingApproval("corr-1")], + tokenUsage: EMPTY_USAGE, + }); + await store.mirrorToSubstrate(); + + const got = await readWarmParkedApprovalSnapshot({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + stepId: "s", + correlationId: "corr-1", + }); + expect(got).toBeUndefined(); + }); +}); + +// The pending-operations readers back the resume classifier's `readParkedApprovalOps` +// binding (the crash-mid-park recovery hook), where the snapshot readers back the +// re-registration enumeration. Both project the same durable store; these pin the +// enumeration variant returns every parked op, not just one matched by correlationId. +describe("readColdParkedPendingOperations", () => { + test("returns every pending operation from a parked step's on-disk store", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "run-1", + stepId: "s", + attempt: 1, + }; + const store = await createIsogitStore(stepStorageRoot(coordinate), testSigner); + await store.writeMetadata({ + pendingOperations: [pendingApproval("corr-1", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + + const got = await readColdParkedPendingOperations(coordinate); + expect(got).toEqual([pendingApproval("corr-1", snapshot)]); + }); + + test("returns an empty list without creating a repo when the store is absent", async () => { + const dataDir = await makeTempDir(); + const coordinate = { + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId: "missing", + stepId: "s", + attempt: 1, + }; + + const got = await readColdParkedPendingOperations(coordinate); + expect(got).toEqual([]); + await expect(fs.stat(stepStorageRoot(coordinate))).rejects.toThrow(); + }); +}); + +describe("readWarmParkedPendingOperations", () => { + test("reconstructs the pending operations from the durable substrate mirror", async () => { + const substrate = createStubSubstrate(await makeTempDir()); + const store = await createDurableConversationStore({ + localStoreDir: await makeTempDir(), + signer: testSigner, + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: WORKFLOW_RUN_REF, + principal: PRINCIPAL, + agentKey: "s", + }); + await store.storage.writeMetadata({ + pendingOperations: [pendingApproval("corr-1", snapshot)], + tokenUsage: EMPTY_USAGE, + }); + await store.mirrorToSubstrate(); + + const got = await readWarmParkedPendingOperations({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + stepId: "s", + }); + expect(got).toEqual([pendingApproval("corr-1", snapshot)]); + }); + + test("returns an empty list when no durable state exists for the agent", async () => { + const substrate = createStubSubstrate(await makeTempDir()); + + const got = await readWarmParkedPendingOperations({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + stepId: "never-ran", + }); + expect(got).toEqual([]); + }); +}); + +describe("toParkedApprovalOps", () => { + test("projects approval ops to correlationId plus the epoch-ms deadline", async () => { + const withDeadline: PendingOperation = { + correlationId: "corr-timeout", + kind: "approval", + registeredAt: 0, + gateId: "gate-corr-timeout", + timeoutAt: 1_700_000_000_000, + }; + + const got = toParkedApprovalOps([pendingApproval("corr-1", snapshot), withDeadline]); + + expect(got).toEqual([ + { correlationId: "corr-1" }, + { correlationId: "corr-timeout", timeoutAtMs: 1_700_000_000_000 }, + ]); + }); + + test("omits timeoutAtMs for an indefinite-hold park", async () => { + const got = toParkedApprovalOps([pendingApproval("corr-1")]); + expect(got).toEqual([{ correlationId: "corr-1" }]); + const first = got[0]; + if (first === undefined) throw new Error("unreachable"); + expect("timeoutAtMs" in first).toBe(false); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts new file mode 100644 index 0000000000..50cb92eb1b --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-step-storage.test.ts @@ -0,0 +1,364 @@ +// Per-step scratch keying for `createSidecarStepBuildEnv` (#3 leak fix). +// +// `stepStorageRoot` rooted every step invocation's workspace + tool +// scratch under the per-message `runId`, and nothing ever reclaimed it, +// so a long-lived deployment's `workflow-step-state/` grew without +// bound. The fix keys the warm single-step agent's scratch STABLY per +// agent (so the cached agent reuses one workspace across runs and the +// warm case is bounded to one dir per agent) while the cold/multi-step +// path keeps its per-run keying (reclaimed at run completion / undeploy +// elsewhere). +// +// These tests pin the keying directly off the production `buildEnv` the +// substrate factory wires: +// - WARM (`durableConversation` present): two different runIds produce +// the SAME `env.workdir`, and a file written for run-1 is visible in +// the env built for run-2 -- the workspace-continuity the stable key +// buys. +// - COLD (no `durableConversation`): two different runIds produce +// DIFFERENT `env.workdir`s, each under that run's subtree -- the +// per-run keying the run-completion cleanup reclaims. + +import { describe, test, expect } from "bun:test"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import type { + AuditStore, + ContextStore, + InferenceSource, + PendingOperation, + ToolCall, +} from "@intx/types/runtime"; +import type { RepoId } from "@intx/hub-sessions"; +import { createBuiltinRegistry } from "@intx/inference/providers"; +import { createIsogitStore } from "@intx/storage-isogit/node"; +import type { ChildOutboundMailBridge, SourcesSnapshotRef, StepEnvBase } from "@intx/workflow-host"; +import { scopedStepId, type StepInvokeRequest } from "@intx/workflow"; + +import { + createSidecarStepBuildEnv, + stepStorageRoot, + type SidecarStepBuildEnvDeps, +} from "./workflow-substrate-factory"; +import type { DurableConversationRegistry } from "./conversation-state"; + +const STEP_ID = "step-1"; +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: "deployment-keying", +}; + +const SOURCE: InferenceSource = { + id: STEP_ID, + provider: "anthropic", + baseURL: "https://api.anthropic.com", + credentialId: "sk-keying", + model: "claude-keying", +}; + +function stubOutboundMailBridge(): ChildOutboundMailBridge { + return { + submit: () => Promise.reject(new Error("unused in buildEnv keying test")), + handleResult: () => undefined, + cancelAll: () => undefined, + pendingCount: 0, + }; +} + +// A warm `durableConversation` whose `acquire(stepId)` returns a storage +// the env builder files into `env.storage`. `buildEnv` never invokes the +// store's methods, so a structural double-cast stub is sufficient and is +// the documented test-stub escape hatch for a wide library interface. +function stubDurableConversationRegistry(): DurableConversationRegistry { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv only files `.storage` into the env; it never calls the store's methods, so a structural stub cannot be satisfied field-by-field + const storage = {} as ContextStore & AuditStore; + const store = { + storage, + mirrorToSubstrate: () => Promise.resolve(), + restoreFromSubstrate: () => Promise.resolve(), + }; + return { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the registry's full DurableConversationStore surface is not exercised by buildEnv + acquire: () => Promise.resolve(store as never), + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- same: only `.storage` is read + get: () => store as never, + }; +} + +function buildDeps(opts: { + dataDir: string; + durableConversation?: DurableConversationRegistry; +}): SidecarStepBuildEnvDeps { + return { + dataDir: opts.dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + signer: (payload: string) => Promise.resolve(`sig:${payload.length}`), + mailboxAddress: "run_deployment-keying@example.com", + stepCount: 1, + outboundMailBridge: stubOutboundMailBridge(), + cache: { cacheMaxBytes: 1_000_000, registryMaxTarballBytes: 1_000_000 }, + adapters: createBuiltinRegistry(), + recordToolMarkFloor: () => undefined, + sourceTools: false, + ...(opts.durableConversation !== undefined + ? { durableConversation: opts.durableConversation } + : {}), + }; +} + +function sourcesRefFor(chain: InferenceSource[] = [SOURCE]): SourcesSnapshotRef { + return { current: { [STEP_ID]: chain } }; +} + +function requestForRun(runId: string): StepInvokeRequest { + return { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv reads only authzContext; the agent definition is never consulted here + agent: {} as StepInvokeRequest["agent"], + input: null, + authzContext: { stepId: STEP_ID, runId, attempt: 1 }, + signal: new AbortController().signal, + }; +} + +async function makeTempDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), "sidecar-step-keying-")); +} + +describe("createSidecarStepBuildEnv per-step scratch keying", () => { + test("warm path keys one stable workdir across runs and preserves files", async () => { + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv( + buildDeps({ + dataDir, + durableConversation: stubDurableConversationRegistry(), + }), + ); + + const sourcesRef = sourcesRefFor(); + const env1: StepEnvBase = await buildEnv(requestForRun("run-1"), sourcesRef); + // A file the agent would write during run-1's turn. + const marker = path.join(env1.workdir, "turn-1.txt"); + await fs.writeFile(marker, "carried"); + + const env2: StepEnvBase = await buildEnv(requestForRun("run-2"), sourcesRef); + + // Stable keying: a different runId resolves to the SAME workdir, so + // the warm case is bounded to one dir per agent (not one-per-message) + // and the workspace survives across runs/respawn. + expect(env2.workdir).toBe(env1.workdir); + // The warm workdir lives under the stable `warm//` sub-root, + // never under any run's `runs//` subtree. + expect(env1.workdir).toContain( + path.join("workflow-step-state", WORKFLOW_RUN_REPO_ID.id, "warm"), + ); + expect(env1.workdir).not.toContain(path.join("runs", "run-1")); + // Continuity: the file written in run-1 is visible in run-2's env. + expect(await fs.readFile(path.join(env2.workdir, "turn-1.txt"), "utf8")).toBe("carried"); + }); + + test("cold path keys a distinct per-run workdir under that run's subtree", async () => { + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); + + const env1: StepEnvBase = await buildEnv(requestForRun("run-1"), sourcesRef); + const env2: StepEnvBase = await buildEnv(requestForRun("run-2"), sourcesRef); + + // Per-run keying: each run gets its own workdir, rooted under that + // run's `runs//` subtree -- exactly what the run-completion + // cleanup reclaims at run granularity. + expect(env2.workdir).not.toBe(env1.workdir); + expect(env1.workdir).toContain( + path.join("workflow-step-state", WORKFLOW_RUN_REPO_ID.id, "runs", "run-1"), + ); + expect(env2.workdir).toContain(path.join("runs", "run-2")); + // The cold path never parks scratch under the warm sub-root. + expect(env1.workdir).not.toContain(path.join("warm", STEP_ID)); + }); + + test("feeds the reactor the whole failover chain with the head pinned as default", async () => { + // The reactor resolves its initial source by `defaultSource` and fails + // over forward across `sources`; the child must hand it the full ordered + // chain, not just the active source, or cross-provider failover is lost. + const failoverSource: InferenceSource = { + id: "failover", + provider: "openai", + baseURL: "https://api.openai.com", + credentialId: "sk-failover", + model: "gpt-failover", + }; + const chain = [SOURCE, failoverSource]; + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + + const env: StepEnvBase = await buildEnv(requestForRun("run-1"), sourcesRefFor(chain)); + + // The whole chain reaches the reactor, ordered, with element 0 as the + // initial source. + expect(env.sources).toEqual(chain); + expect(env.defaultSource).toBe(SOURCE.id); + }); + + test("resolves sources from the mutable ref at build time", async () => { + // The build reads the source table through the ref, so a rotation that + // writes the ref before a build is reflected in the agent that build + // constructs (the cold-window path; a warm already-built agent swaps + // sources through the warm cache, not here). + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const sourcesRef = sourcesRefFor(); + + const before: StepEnvBase = await buildEnv(requestForRun("run-1"), sourcesRef); + expect(before.sources).toEqual([SOURCE]); + + const rotated: InferenceSource = { + id: "rotated", + provider: "openai", + baseURL: "https://api.openai.com", + credentialId: "sk-rotated", + model: "gpt-rotated", + }; + sourcesRef.current = { [STEP_ID]: [rotated] }; + + const after: StepEnvBase = await buildEnv(requestForRun("run-2"), sourcesRef); + expect(after.sources).toEqual([rotated]); + expect(after.defaultSource).toBe(rotated.id); + }); + + test("a map iteration resolves the base step's source while keeping scratch scoped", async () => { + // Deploy pins one source per base step, keyed by the base step id; a map + // iteration runs under a scoped step id `[]`. The source + // table holds only the base id, so without base resolution the build + // would throw "no InferenceSource pinned". The scratch, by contrast, + // stays keyed by the scoped id so concurrent iterations never collide. + const dataDir = await makeTempDir(); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + const scopedId = scopedStepId(STEP_ID, 0); + + const scopedRequest: StepInvokeRequest = { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv reads only authzContext; the agent definition is never consulted here + agent: {} as StepInvokeRequest["agent"], + input: null, + authzContext: { stepId: scopedId, runId: "run-1", attempt: 1 }, + signal: new AbortController().signal, + }; + const env: StepEnvBase = await buildEnv(scopedRequest, sourcesRefFor()); + + // The scoped iteration resolves the base step's pinned source. + expect(env.sources).toEqual([SOURCE]); + expect(env.defaultSource).toBe(SOURCE.id); + // Per-iteration scratch stays keyed by the scoped id, not the base. + expect(env.workdir).toContain(path.join("steps", scopedId)); + }); +}); + +// A re-dispatchable ask-rail approval gate always carries a `suspendedCall` +// (the call to re-run on approval). An async-tool pending marker uses the +// same `kind: "approval"` but never sets `suspendedCall`, and on resume the +// reactor clears its gate WITHOUT re-dispatching. The cold-path resume +// keying assertion must therefore find a `suspendedCall`-bearing op for the +// resumed correlationId, mirroring the reactor's own discriminator. +const SUSPENDED_CALL: ToolCall = { + id: "call-1", + name: "charge_card", + arguments: { amount: 100 }, +}; + +function approvalOp( + correlationId: string, + opts: { suspendedCall?: ToolCall } = {}, +): PendingOperation { + return { + correlationId, + kind: "approval", + registeredAt: 0, + gateId: `gate-${correlationId}`, + ...(opts.suspendedCall !== undefined ? { suspendedCall: opts.suspendedCall } : {}), + }; +} + +async function seedColdStore( + dataDir: string, + runId: string, + pendingOperations: PendingOperation[], +): Promise { + const store = await createIsogitStore( + stepStorageRoot({ + dataDir, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + runId, + stepId: STEP_ID, + attempt: 1, + }), + (payload: string) => Promise.resolve(`sig:${payload.length}`), + ); + await store.writeMetadata({ + pendingOperations, + tokenUsage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }, + }); +} + +function approvalResumeRequest(runId: string, correlationId: string): StepInvokeRequest { + return { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- buildEnv reads only authzContext and resume; the agent definition is never consulted here + agent: {} as StepInvokeRequest["agent"], + input: null, + authzContext: { stepId: STEP_ID, runId, attempt: 1 }, + resume: { + correlationId, + decision: { outcome: "approved" }, + kind: "approval", + }, + signal: new AbortController().signal, + }; +} + +describe("createSidecarStepBuildEnv cold-path approval-resume keying", () => { + test("rejects a resume whose matching pending op carries no suspendedCall", async () => { + const dataDir = await makeTempDir(); + // A pending op that matches the correlationId but is not a + // re-dispatchable gate (no suspendedCall) -- an async-tool marker, not + // the ask-rail approval the resume must find. + await seedColdStore(dataDir, "run-1", [approvalOp("corr-1")]); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + + await expect( + buildEnv(approvalResumeRequest("run-1", "corr-1"), sourcesRefFor()), + ).rejects.toThrow(/keying violation/); + }); + + test("accepts a resume that finds a suspendedCall-bearing approval gate", async () => { + const dataDir = await makeTempDir(); + await seedColdStore(dataDir, "run-1", [ + approvalOp("corr-1", { suspendedCall: SUSPENDED_CALL }), + ]); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + + const env: StepEnvBase = await buildEnv( + approvalResumeRequest("run-1", "corr-1"), + sourcesRefFor(), + ); + + expect(env.workdir).toContain(path.join("runs", "run-1")); + }); + + test("rejects a resume whose correlationId has no pending op at all", async () => { + const dataDir = await makeTempDir(); + // A real gate exists, but for a different correlationId: the resumed + // correlationId finds nothing -- the wrong-attempt forever-hang guard. + await seedColdStore(dataDir, "run-1", [approvalOp("other", { suspendedCall: SUSPENDED_CALL })]); + const buildEnv = createSidecarStepBuildEnv(buildDeps({ dataDir })); + + await expect( + buildEnv(approvalResumeRequest("run-1", "corr-1"), sourcesRefFor()), + ).rejects.toThrow(/keying violation/); + }); +}); diff --git a/apps/sidecar/src/workflow-substrate-factory-suspendable-child.test.ts b/apps/sidecar/src/workflow-substrate-factory-suspendable-child.test.ts new file mode 100644 index 0000000000..7ca236be44 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory-suspendable-child.test.ts @@ -0,0 +1,364 @@ +// An onTrigger body child, spawned through the real sidecar seam, parks on +// an approval and resumes on the correlated grant. +// +// `createSidecarSpawnSuspendableChild` runs the body definition against a +// real on-disk workflow-run substrate and hands back the live handle +// `runOnTrigger` drives. This test exercises that handle end to end: the +// body step's injected `invokeStep` suspends as an approval, so the child +// runtime parks on `signalName(correlationId)`; `handle.next()` surfaces the +// park with the step's snapshot; `handle.resume` delivers the grant on the +// child's own signal channel, and the re-invoked step completes the run. +// +// A second case aborts the parent signal while the body is parked and proves +// the child cancels and the handle surfaces a terminal rather than hanging -- +// the abort threads through `handle.cancel` and the run settles. + +import { describe, test, expect, afterAll, beforeAll } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { generateKeyPair } from "@intx/crypto"; +import type { ApprovalSnapshot, KeyPair } from "@intx/types/runtime"; +import { defineAgent } from "@intx/agent"; +import { builtinCredentialProviders, createCredentialProviderRegistry } from "@intx/harness"; +import { evaluateGrants } from "@intx/authz"; +import type { GrantRule } from "@intx/authz"; +import { + createRepoStore, + workflowRunKindHandler, + WORKFLOW_RUN_GITIGNORE_PATH, +} from "@intx/hub-sessions"; +import type { AuthorizeFn, RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions"; +import { + childWorkflow, + createInMemoryScheduler, + createInMemoryRepoStore, + defineWorkflow, + step, + type WorkflowDefinition, + type WorkflowEvent, +} from "@intx/workflow"; + +import { createWorkflowRunRepoStore } from "@intx/workflow-host"; + +import { + createSidecarSpawnSuspendableChild, + type SidecarChildStepInvoker, +} from "./workflow-substrate-factory"; +import { runGrantsPath } from "./run-grants"; + +const REF = "refs/heads/main"; +const DEPLOYMENT_ID = "deployment-suspendable-child"; +const WORKFLOW_RUN_REPO_ID: RepoId = { + kind: "workflow-run", + id: DEPLOYMENT_ID, +}; +const allowAll: AuthorizeFn = () => ({ allowed: true }); +const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: DEPLOYMENT_ID, +}; + +const BODY_STEP_AGENT_ID = "wallet-spend"; +const CORRELATION_ID = "corr-approval-1"; +const SNAPSHOT: ApprovalSnapshot = { + name: BODY_STEP_AGENT_ID, + description: "spend from the shared wallet", + inputSchema: { amount: "number" }, + arguments: { amount: 100 }, +}; + +const tempDirs: string[] = []; +let signingKey: KeyPair; +// `buildChildRunEnv` reads the body run's `assets/workflow//sources.json` +// eagerly; the suspend/resume mock invoker ignores inference, but the read still +// happens, so stage a minimal sources file for the body definition id. +let bodySourcesDataDir: string; + +beforeAll(async () => { + signingKey = await generateKeyPair(); + bodySourcesDataDir = await makeTempDir("suspendable-assets-"); + // The approval-park body ("body-wf") carries an agent step "s"; the + // depth body ("depth-body") carries only a childWorkflow step (no + // inference), but `buildChildRunEnv` reads its sources eagerly all the same. + await stageBodySources("body-wf"); + await stageBodySources("depth-body"); +}); + +async function stageBodySources(id: string): Promise { + const dir = path.join(bodySourcesDataDir, "assets", "workflow", id); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile( + path.join(dir, "sources.json"), + JSON.stringify({ + s: [ + { + id: "anthropic:m", + provider: "anthropic", + baseURL: "http://localhost:1", + credentialId: "sk-x", + model: "m", + }, + ], + }), + ); +} + +afterAll(async () => { + for (const d of tempDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { + /* best effort */ + }); + } +}); + +async function makeTempDir(prefix: string): Promise { + const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(d); + return d; +} + +function grant(resource: string, action: string): GrantRule { + return { + id: `grant-${resource}-${action}`, + resource, + action, + effect: "allow", + origin: "creator", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + }; +} + +// The one-step body definition every spawn runs. Its single step's injected +// invoker suspends as an approval on the first invocation and completes on +// the resume re-invocation. +function bodyDefinition(id: string): WorkflowDefinition { + const agent = defineAgent({ + id: BODY_STEP_AGENT_ID, + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }); + return defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { s: step({ agent }) }, + }); +} + +const evaluateGrantsAdapter: Parameters< + typeof createSidecarSpawnSuspendableChild +>[0]["evaluateGrants"] = async ({ resource, action, grants }) => { + const result = await evaluateGrants( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the snapshot's grants are typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar, so the seeded rows narrow here + [...(grants as readonly GrantRule[])], + resource, + action, + ); + return { effect: result.effect, matchingGrants: [], resolvedBy: null }; +}; + +async function makeSubstrate(prefix: string): Promise> { + const dataDir = await makeTempDir(prefix); + const substrate = createRepoStore({ + dataDir, + signingKey, + handlers: { "workflow-run": workflowRunKindHandler }, + authorize: allowAll, + }); + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, + message: "genesis", + }); + return substrate; +} + +async function seedRunGrants( + substrate: ReturnType, + runId: string, + grants: readonly GrantRule[], +): Promise { + await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { + files: { + [runGrantsPath(runId)]: JSON.stringify({ grants }, null, 2), + }, + message: `seed grants for ${runId}`, + }); +} + +// An invoker that suspends as an approval on its first invocation and, on the +// resume re-invocation, records the delivered decision and completes. +function suspendThenComplete(record: { resumeDecision?: unknown }): SidecarChildStepInvoker { + return async (req) => { + if (req.resume === undefined) { + return { + suspend: { + correlationId: CORRELATION_ID, + kind: "approval", + approvalSnapshot: SNAPSHOT, + }, + }; + } + record.resumeDecision = req.resume.decision; + return { output: { echoed: req.resume.decision } }; + }; +} + +function makeSpawner( + substrate: ReturnType, + invokeStep: SidecarChildStepInvoker, +): ReturnType { + return createSidecarSpawnSuspendableChild({ + substrate, + workflowRunRepoId: WORKFLOW_RUN_REPO_ID, + workflowRunRef: REF, + principal: PRINCIPAL, + scheduler: createInMemoryScheduler({ + repoStore: createInMemoryRepoStore(), + clock: () => new Date(), + }), + invokeStep, + evaluateGrants: evaluateGrantsAdapter, + dataDir: bodySourcesDataDir, + bodySources: {}, + credentialProviders: createCredentialProviderRegistry(builtinCredentialProviders()), + }); +} + +describe("createSidecarSpawnSuspendableChild", () => { + test("surfaces a body approval park, resumes on the grant, and completes", async () => { + const substrate = await makeSubstrate("suspendable-approval-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [grant(`tool:${BODY_STEP_AGENT_ID}`, "invoke")]); + + const record: { resumeDecision?: unknown } = {}; + const spawn = makeSpawner(substrate, suspendThenComplete(record)); + + const handle = await spawn( + { + definition: bodyDefinition("body-wf"), + definitionRef: REF, + childRunId: "run-body-0", + input: { text: "event-0" }, + parentRunId, + parentStepId: "section", + signal: new AbortController().signal, + depth: 0, + maxChildSpawnDepth: 32, + }, + () => undefined, + ); + + // The body step suspended -> the handle surfaces the approval park on the + // reserved correlation, carrying the step's snapshot. + const parked = await handle.next(); + expect(parked.kind).toBe("park"); + if (parked.kind !== "park") throw new Error("expected a park"); + expect(parked.park.correlationId).toBe(CORRELATION_ID); + expect(parked.park.approvalSnapshot).toEqual(SNAPSHOT); + + // Grant it: resume delivers the decision on the child's own signal + // channel, unblocking the parked step. + const decision = { outcome: "approved" as const, note: "ok" }; + await handle.resume(CORRELATION_ID, decision); + + // The re-invoked step ran with the delivered decision and the run + // completed. + const terminal = await handle.next(); + expect(terminal.kind).toBe("terminal"); + if (terminal.kind !== "terminal") throw new Error("expected a terminal"); + expect(terminal.terminalStatus).toBe("completed"); + expect(record.resumeDecision).toEqual(decision); + }); + + test("threads depth so a body's childWorkflow grandchild trips the ceiling", async () => { + const substrate = await makeSubstrate("suspendable-depth-"); + const parentRunId = "run-parent"; + await seedRunGrants(substrate, parentRunId, [grant("inference.source:anthropic:m", "invoke")]); + + // The invoker must never run: the depth guard fires before the grandchild's + // agent step is reached. + const spawn = makeSpawner(substrate, () => { + throw new Error("depth: no step should run before the guard fires"); + }); + + // Spawn the body AT depth 1 with the ceiling lowered to 1. The body runs at + // depth 1 (passed through unchanged), so its `childWorkflow` spawns the + // grandchild at depth 2 > 1 and trips the guard. WITHOUT depth threading the + // body would run at depth 0 and the grandchild at depth 1 <= 1 -- no trip -- + // so this failure proves the container's depth reached the body run. + const childRunId = "run-depth-body"; + const handle = await spawn( + { + definition: childSpawningBody(), + definitionRef: REF, + childRunId, + input: null, + parentRunId, + parentStepId: "section", + signal: new AbortController().signal, + depth: 1, + maxChildSpawnDepth: 1, + }, + () => undefined, + ); + + // The body's childWorkflow step failed loud, so the body run settles failed. + const terminal = await handle.next(); + expect(terminal.kind).toBe("terminal"); + if (terminal.kind !== "terminal") throw new Error("expected a terminal"); + expect(terminal.terminalStatus).toBe("failed"); + + // The guard fired before the body committed a ChildSpawned, and the + // StepFailed names the tripped depth 2 (body depth 1 + 1), not the depth 1 a + // reset-at-the-body-boundary would give. + const bodyEvents: readonly WorkflowEvent[] = await reader(substrate).read(childRunId); + expect(bodyEvents.some((e) => e.kind === "ChildSpawned")).toBe(false); + const stepFailed = bodyEvents.find((e) => e.kind === "StepFailed"); + if (stepFailed === undefined || stepFailed.kind !== "StepFailed") { + throw new Error(`body run ${childRunId} has no StepFailed event`); + } + expect(stepFailed.error.message).toContain( + 'childWorkflow spawn depth 2 exceeds the maximum 1 at step "spawn"', + ); + }); +}); + +// A body whose single step spawns a `childWorkflow` grandchild. The grandchild +// carries an agent step that is never reached: the depth guard fires first. +function childSpawningBody(): WorkflowDefinition { + const leaf = defineWorkflow({ + id: "depth-leaf", + trigger: { type: "manual" }, + steps: { + work: step({ + agent: defineAgent({ + id: "depth-leaf-agent", + systemPrompt: "s", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "anthropic", model: "m" }] }, + }), + }), + }, + }); + return defineWorkflow({ + id: "depth-body", + trigger: { type: "manual" }, + steps: { spawn: childWorkflow({ definition: leaf }) }, + }); +} + +function reader(substrate: ReturnType) { + return createWorkflowRunRepoStore({ + substrate, + repoId: WORKFLOW_RUN_REPO_ID, + principal: PRINCIPAL, + ref: REF, + }); +} diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts new file mode 100644 index 0000000000..374ff4fd92 --- /dev/null +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -0,0 +1,2627 @@ +// Substrate factory the sidecar's `bin/workflow-child` hands to +// `runWorkflowChildFromProcessEnv`. The factory closes over the +// production substrate (`createAgentRepoStore`-backed `RepoStore`), +// the host-process scheduler singleton (adapted to the runtime's +// `Scheduler` shape), and the sidecar's grant-rule evaluator. +// +// The factory consumes the workflow-host's typed `SubstrateFactoryEnv` +// -- the parsed `SpawnTimeEnv` plus a narrow `substrateConfig` +// record carrying only the keys the binary listed in +// `RunWorkflowChildFromProcessEnvOpts.substrateConfigKeys`. The +// factory does not read `process.env` itself; the binary owns the +// only crossing of that boundary. +// +// Single-writer architecture: the workflow-run repo's ref has exactly +// one writer at a time -- the supervisor. The child opens a bare +// `createAgentRepoStore` against the shared on-disk data dir for +// read-only operations (`getRepoDir`, `subscribe`, `resolveRef`, +// etc.) and exposes a proxy `RepoStore` whose +// `writeTreePreservingPrefix` forwards every write over the control +// IPC into the supervisor's substrate. The supervisor's substrate is +// wrapped with the boot-edge pack-push facade, so the hub push fires +// as part of the supervisor's normal write path -- the child does +// not open its own pack-push pipeline. + +import fs from "node:fs"; +import path from "node:path"; + +import { type } from "arktype"; + +import { InferenceSource } from "@intx/types/runtime"; +import type { + ApprovalSnapshot, + AuditStore, + ContextStore, + InboundMessage, + InferenceEvent, + MessageTransport, + PendingOperation, +} from "@intx/types/runtime"; +import type { RuntimeCapabilities } from "@intx/types/runtime-capabilities"; +import { evaluateGrants } from "@intx/authz"; +import type { GrantRule } from "@intx/authz"; +import { AdapterManifest, createDependencies, type AdapterRegistry } from "@intx/inference"; +import { loadAdapterRegistry } from "@intx/inference/providers"; +import type { AnnotatedPluginFactory, DirectorRegistry } from "@intx/agent"; +import { createDefaultDirectorRegistry } from "@intx/agent"; +import { + builtinCredentialProviders, + createCredentialProviderRegistry, + createHarnessRuntimeCapabilities, + driveConnectorReplies, + type AgentEventStream, + type ConnectorReplyDrain, + type CredentialProviderRegistry, +} from "@intx/harness"; +import { createSSHSignature } from "@intx/crypto"; +import { + createAgentRepoStore, + WORKFLOW_RUN_AGENT_STATE_PREFIX, + type Principal, + type RepoId, + type RepoStore, + type WorkflowRunWorkflowProcessPrincipal, +} from "@intx/hub-sessions/substrate"; +import { createIsogitStore } from "@intx/storage-isogit/node"; +import { + adaptHostScheduler, + createChildMailboxReader, + createCredentialsBackedAuthorize, + createMailboxWatchRegistry, + createProxyWorkflowRunRepoStore, + createSupervisorBackedTransport, + createWorkflowHostScheduler, + createWorkflowRunBlobSubstrate, + createWorkflowRunRepoStore, + createWorkflowHostSignalChannel, + createInMemorySpawnChild, + createInMemorySpawnSuspendableChild, + createWorkflowStepInvoker, + hashGrants, + loadWorkflowLoopFnsFromClosure, + loadWorkflowPluginFactoriesFromClosure, + loadWorkflowPluginToolDefinitionsFromClosure, + type ChildMailboxReader, + type ChildOutboundMailBridge, + type CredentialsSnapshot, + type CredentialsSnapshotRef, + type GrantEvaluator, + type LoadParkedApproval, + type RunChildWorkflow, + type RunSuspendableChild, + type RunWorkflowChildBindings, + type SourcesSnapshotRef, + type StepEnvBase, + type SubstrateFactory, + type SubstrateFactoryEnv, + type SupervisorBackedTransportInbound, +} from "@intx/workflow-host"; +import { + baseStepId, + collectDeclaredPluginNames, + createLoopIterationHandle, + createNoopDrainController, + createSuspendableChildHandle, + eagerlyResolveLoopFns, + emptyState, + enumerateInlineLoopBodies, + rewriteInlineChildWorkflowBodies, + runtimeRun, + walkWorkflowSteps, + LOOP_BODY_DESCENT, + type LoopFnRegistry, + type ParkedApprovalOp, + type ReadParkedApprovalOps, + type Scheduler, + type StepInvokeRequest, + type StepInvokeResult, + type WorkflowAuthorizeFn, + type WorkflowDefinition, + type WorkflowRuntimeEnv, +} from "@intx/workflow"; +import { type PluginToolDefinitions } from "@intx/workflow-deploy"; + +import { + attachStepCredentialWiring, + attachStepTools, + createToolBearingAgentFactory, + deriveToolMarkFloorGrants, + materializeStepTools, + type StepToolCacheConfig, + type StepToolMaterialization, +} from "./step-agent-tools"; +import { + createInferenceCredentialResolver, + type CredentialMaterialCell, +} from "./step-credential-capabilities"; +import { readRunGrants, runGrantsPath } from "./run-grants"; +import { collectDeclaredResources, filterGrantsToDeclaredResources } from "./child-grant-filter"; +import { + createDurableConversationRegistry, + isErrnoNotFound, + reconstructDurableConversation, + type DurableConversationRegistry, +} from "./conversation-state"; + +// The child does not construct a workflow-run pack-push pipeline of +// its own. The supervisor owns the workflow-run repo's write +// contract; the supervisor's substrate is wrapped at the sidecar's +// boot edge with the pack-pushing facade so any successful workflow- +// run write fires the hub push automatically. The child's proxy +// `RepoStore` forwards `writeTreePreservingPrefix` over IPC into the +// supervisor's wrapped substrate. + +/** + * Required substrate-config keys the sidecar's binary forwards into + * the factory's `substrateConfig` slot. Listed here so the binary + * passes the same names to the helper; the helper enforces + * presence-and-non-empty against this allowlist before the factory + * runs. + * + * `HUB_WS_URL`, `SIDECAR_ID`, and `SIDECAR_TOKEN` carry the + * hub-connection trust anchors the child needs to ship workflow-run + * pack pushes back to the hub. The sidecar's deploy router populates + * these via the supervisor's `substrateEnv` plumbing + * (`multistepSubstrateEnv` on `createSidecarDeployRouter`), threaded + * from the boot edge's own env reads. + */ +export const SIDECAR_SUBSTRATE_CONFIG_KEYS = [ + "SIDECAR_DATA_DIR", + "WORKFLOW_RUN_REPO_ID", + "WORKFLOW_RUN_REF", + "SIDECAR_SIGNING_PUBLIC_KEY", + "SIDECAR_SIGNING_PRIVATE_KEY", + "HUB_WS_URL", + "SIDECAR_ID", + "SIDECAR_TOKEN", + "STEP_INFERENCE_SOURCES", + "WORKFLOW_BODY_SOURCES", + "SIDECAR_CACHE_MAX_BYTES", + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + "SIDECAR_ADAPTER_MANIFEST", +] as const; + +const SubstrateConfig = type({ + SIDECAR_DATA_DIR: "string > 0", + WORKFLOW_RUN_REPO_ID: "string > 0", + WORKFLOW_RUN_REF: "string > 0", + SIDECAR_SIGNING_PUBLIC_KEY: "string > 0", + SIDECAR_SIGNING_PRIVATE_KEY: "string > 0", + HUB_WS_URL: "string > 0", + SIDECAR_ID: "string > 0", + SIDECAR_TOKEN: "string > 0", + STEP_INFERENCE_SOURCES: "string > 0", + // JSON `{ [definitionId]: { [stepId]: InferenceSource[] } }` of every spawned + // body's plaintext inference sources, decrypted sidecar-side from the run + // record. Always serialized by the deploy router (at least "{}" when the + // deployment spawns no bodies), so a missing key child-side is a serialization + // bug and must fail loud here. + WORKFLOW_BODY_SOURCES: "string > 0", + // Per-step tool-loader caps. The supervisor threads the boot edge's + // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` + // through `substrateEnv` so the child's per-step tool materialization is + // bounded by the sidecar's boot-edge-resolved caps. Validated as + // positive-finite-number strings at this boundary. + SIDECAR_CACHE_MAX_BYTES: "string > 0", + SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", + // JSON-encoded custom inference adapter manifest. Required: the boot + // edge always serializes it into `substrateEnv` (defaulting to "[]" + // when no custom adapters are configured), so a missing key child-side + // is a serialization bug and must fail loud here, exactly like the + // byte-cap fields. Validated as a non-empty string at this boundary; + // its JSON shape is re-validated against `AdapterManifest` in + // `parseAdapterManifest` before any module is imported. + SIDECAR_ADAPTER_MANIFEST: "string > 0", +}).onUndeclaredKey("ignore"); + +/** + * Parse a substrate-config cap entry (`SIDECAR_CACHE_MAX_BYTES` / + * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) into a positive finite number. + * The boot edge already validated these via the `config.ts` readers + * before serializing them into `substrateEnv`; this re-parse at the + * child boundary keeps the typed-config contract honest rather than + * trusting the wire blindly. + */ +function parseByteCap(raw: string, name: string): number { + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + throw new Error( + `sidecar workflow-child substrate config: ${name} must be a positive finite number; got ${JSON.stringify(raw)}`, + ); + } + return n; +} + +/** + * Per-step inference-source table parsed from the spawn-time + * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes + * `frame.workflow.sources` (a `Record`) as + * JSON and threads it through the supervisor's `substrateEnv`; the + * factory parses and validates the table once at construction time and + * seeds it into the run loop's mutable sources reference, which each + * `buildEnv` reads. Each value is the step's ordered + * failover chain -- element 0 is the active source, the tail are the + * reactor's forward-only failover targets -- so the list is non-empty. + */ +const StepInferenceSourceTable = type({ + "[string]": InferenceSource.array().atLeastLength(1), +}); +type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; + +/** + * Parse and validate the JSON-encoded `STEP_INFERENCE_SOURCES` entry + * the supervisor threaded through `substrateEnv`. A malformed JSON + * payload, a non-object root, or a value that does not match + * `Record` is rejected at the boundary with + * a structured error rather than being deferred to a deep-stack + * `buildEnv` failure. + */ +function parseStepInferenceSources(raw: string): StepInferenceSourceTable { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES is not valid JSON: ${reason}`, + ); + } + const validated = StepInferenceSourceTable(parsed); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES failed validation: ${validated.summary}`, + ); + } + return validated; +} + +/** + * Every spawned body's per-step inference-source table, keyed by the body's + * definition id. Parsed from the `WORKFLOW_BODY_SOURCES` env entry the deploy + * router serializes from the run record's decrypted body sources. Empty when the + * deployment spawns no bodies. + */ +const BodyInferenceSources = type({ + "[string]": StepInferenceSourceTable, +}); +type BodyInferenceSources = typeof BodyInferenceSources.infer; + +/** + * Parse and validate the JSON-encoded `WORKFLOW_BODY_SOURCES` entry. Mirrors + * `parseStepInferenceSources`: a malformed payload is rejected at the boundary + * rather than deep in a body spawn. + */ +function parseBodyInferenceSources(raw: string): BodyInferenceSources { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES is not valid JSON: ${reason}`, + ); + } + const validated = BodyInferenceSources(parsed); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config: WORKFLOW_BODY_SOURCES failed validation: ${validated.summary}`, + ); + } + return validated; +} + +/** + * Parse and validate the JSON-encoded `SIDECAR_ADAPTER_MANIFEST` entry + * the supervisor threaded through `substrateEnv` from the boot edge's + * `readAdapterManifest`. + * + * Trust boundary: the child's substrate config is operator-supplied + * (the supervisor's `Bun.spawn` env), so this re-validation is + * defense-in-depth at the deserialization boundary, NOT a trust + * upgrade. The manifest was already trusted operator config on the + * parent side; the same channel already carries the sidecar's signing + * private key, so it is not a lower-trust surface. Re-asserting the + * shape here keeps the typed-config contract honest rather than + * importing modules off an unvalidated wire value. + * + * Host contract for custom adapters: a manifest `specifier` must + * resolve from BOTH the sidecar's and this child's module-resolution + * roots (the child is a separate `bun` process spawned by the + * supervisor), and an adapter module MUST be import-side-effect-free — + * it is imported once per process by `loadAdapterRegistry`, and any + * top-level side effect would run independently in the parent and in + * every child. + */ +export function parseAdapterManifest(raw: string): AdapterManifest { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + throw new Error( + "sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST is not valid JSON", + { cause }, + ); + } + const validated = AdapterManifest(parsed); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, + ); + } + return validated; +} + +/** + * Resolve the per-step inference-source failover chain from the table a + * build reads. The supervisor's multi-step branch only invokes + * a step whose `stepId` appears in `frame.workflow.sources`; a lookup + * miss here is a programmer error in the supervisor, not a wire-side + * failure, and the resolver surfaces it with the missing base step id + * named (plus the scoped invocation id, for a map iteration). The + * returned list is the step's ordered chain (element 0 the active + * source, the tail the reactor's failover targets); the table's arktype + * guarantees it is non-empty. + * + * A `map` iteration runs under a scoped step id `[]`, but + * deploy pins one source per base step, so the scoped id is resolved to + * its base before the lookup -- every iteration shares the base step's + * pinned source. `baseStepId` is the identity on an unscoped id, so a + * plain step is unaffected. + */ +function createStepInferenceSourceResolver( + table: StepInferenceSourceTable, +): (stepId: string) => InferenceSource[] { + return (stepId: string): InferenceSource[] => { + const base = baseStepId(stepId); + const sources = table[base]; + if (sources === undefined) { + const scopedNote = + base === stepId ? "" : ` (normalized from scoped invocation id ${JSON.stringify(stepId)})`; + throw new Error( + `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(base)}${scopedNote}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, + ); + } + return sources; + }; +} + +function hexDecode(hex: string, name: string): Uint8Array { + if (hex.length % 2 !== 0) { + throw new Error(`${name} must be even-length hex; got ${String(hex.length)} chars`); + } + const out = new Uint8Array(hex.length / 2); + for (let i = 0; i < out.length; i += 1) { + const byte = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + if (Number.isNaN(byte)) { + throw new Error(`${name} contains non-hex characters`); + } + out[i] = byte; + } + return out; +} + +/** + * Dependency overrides accepted by `createSidecarSubstrateFactory`. + * Production callers omit these to get the default-disk-backed bare + * store and the IPC-bridge-backed substrate proxy; tests inject an + * in-memory bare store and/or an explicit substrate-write bridge. + */ +interface SidecarSubstrateFactoryDeps { + /** + * Override the bare-store constructor. Production callers omit this + * to get the `createAgentRepoStore`-backed `RepoStore` against + * `SIDECAR_DATA_DIR`; tests inject an in-memory recording stub. + * + * The bare store backs the child's read-only operations + * (`getRepoDir`, `subscribe`, `resolveRef`, `listRefs`, + * `resolveHead`, `createPack`). The child's workflow-run writes do + * NOT flow through this store; the proxy `RepoStore` forwards them + * over IPC into the supervisor's substrate. + */ + createBareRepoStore?: (config: { + dataDir: string; + signingKey: { publicKey: Uint8Array; privateKey: Uint8Array }; + }) => RepoStore; +} + +/** + * `CommitSigner` the per-step isogit stores use to sign every commit. + * The factory's Ed25519 signing keypair (the same key the child's bare + * `RepoStore` carries) is bound into an `sshsig`-shaped signer so the + * per-step agent-state commits are attributable to the sidecar's + * substrate identity, matching the signing surface the production + * `RepoStore` uses for workflow-run writes. + */ +function createStepStorageSigner(signingKey: { + publicKey: Uint8Array; + privateKey: Uint8Array; +}): (payload: string) => Promise { + return (payload: string) => + Promise.resolve(createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey)); +} + +/** + * Root directory for a single step invocation's agent-state storage and + * workspace, derived from the sidecar data dir and the run/step/attempt + * coordinates the workflow runtime owns. + * + * The per-step agent storage is a distinct isogit repo, deliberately + * rooted OUTSIDE the workflow-run repo's working tree. The workflow-run + * repo's single writer is the supervisor, and its working tree carries + * the run-event log under `runs//events/...`; nesting a second + * git repo inside that tree would collide with the event subtree and + * with the supervisor's write contract. Rooting the per-step store under + * a dedicated `workflow-step-state/` sibling subtree keyed by the + * workflow-run repo id keeps every step's storage isolated per run and + * per step while never touching the run-event tree. + * + * Resume-attempt invariant: a suspended agent step commits its pending-op + * + turns under the `attempt-N` directory where N is the step's attempt + * at suspend time. On crash-resume the runtime (`runStep` in + * `packages/workflow`) recovers that attempt from the reduced state's + * `currentAttempt` and re-invokes with it, so this function reopens the + * SAME `attempt-N` store and the reactor's `rehydrateGates` finds the + * pending-op the delivered decision correlates against. This is why the + * resume path recovers `currentAttempt` rather than assuming attempt 1: + * attempt 1 is correct only for a step that never retried before + * suspending; a retried-then-suspended step lives under `attempt-2`+, and + * reopening `attempt-1` would rehydrate an empty store and hang on a + * decision that correlates nothing. `createSidecarStepBuildEnv` asserts + * this invariant loudly on the cold path (a resume opening a store that + * lacks the resumed correlationId's pending-op throws). + */ +export function stepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "runs", + args.runId, + "steps", + args.stepId, + `attempt-${String(args.attempt)}`, + ); +} + +/** + * Root directory for a single workflow-run subtree's per-step scratch: + * `/workflow-step-state//runs//`. The cold + * (multi-step) path's per-step `stepStorageRoot` nests under this, so + * reclaiming this subtree on run completion drops every step/attempt the + * run produced in one `rm -rf`. Kept distinct from `stepStorageRoot` so + * the deletion granularity (a whole run, not a single step/attempt) is + * expressed at the call site that owns run-completion cleanup. + */ +function runStepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "runs", + args.runId, + ); +} + +/** + * Stable per-agent scratch root for the WARM single-step agent's + * workspace + tool materialization (tarball-cache + apply-state). Keyed + * by the step identity exactly like the durable conversation store's + * `agent-conversation-state///` (conversation-state.ts), + * NOT by the arbitrary first-message runId. Keying it stably is what + * bounds the warm case: the cached agent reuses ONE workspace across + * every message in the child's lifetime, and that same workspace is + * re-derived (and so survives) across a child respawn, instead of + * stranding a fresh per-runId subtree each time. The whole subtree is + * reclaimed on undeploy, when the deployment's supervisor + child are + * already torn down. Rooted under a `warm/` sibling of the cold `runs/` + * subtree so the undeploy sweep of `workflow-step-state//` + * reclaims both with one removal and the two keyings never collide. + */ +function warmStepStorageRoot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + stepId: string; +}): string { + return path.join( + args.dataDir, + "workflow-step-state", + args.workflowRunRepoId.id, + "warm", + encodeURIComponent(args.stepId), + ); +} + +async function directoryExists(dir: string): Promise { + try { + return (await fs.promises.stat(dir)).isDirectory(); + } catch (cause) { + if (isErrnoNotFound(cause)) return false; + throw cause; + } +} + +function findApprovalSnapshot( + pendingOperations: readonly PendingOperation[], + correlationId: string, +): ApprovalSnapshot | undefined { + return pendingOperations.find((op) => op.correlationId === correlationId)?.approvalSnapshot; +} + +/** + * Resolve the full RFC 5322 References chain for a reply whose parent is + * `inReplyTo`, from the deployment's committed substrate mailbox. Opens a + * fresh committed snapshot, locates the message whose Message-Id equals + * `inReplyTo`, and returns that message's own References followed by its + * Message-Id -- the complete ancestry a reply threaded onto it must carry. + * + * Returns `undefined` when the parent is not in the mailbox (the first reply + * on a fresh thread, or an id that never arrived). The message identifiers are + * carried through verbatim; the transport's send path (`buildReferences` in + * `@intx/mail-memory`) owns filtering non-RFC identifiers out of the emitted + * `References` header, so this reader does not pre-validate them. + */ +async function resolveMailboxReferences( + reader: ChildMailboxReader, + inReplyTo: string, +): Promise { + const store = await reader.open(); + const parent = store.messages.find((m) => m.envelope.messageId === inReplyTo); + if (parent === undefined) return undefined; + return [...parent.envelope.references, parent.envelope.messageId]; +} + +/** + * Read a cold (multi-step) parked step's durable pending operations from its + * on-disk per-attempt isogit store. The store is written at suspend and + * survives while the run is non-terminal -- a parked step keeps the run + * in-flight, so the run-completion reclamation (`cleanupRunStorage`) never + * fires against it. + * + * Returns an empty list when the store directory is absent rather than + * manufacturing an empty repo on the read path: `createIsogitStore` calls + * `initAgentRepo`, which would `mkdir` and init a fresh repo for a + * non-existent dir. The `directoryExists` guard keeps the read a read -- on an + * existing store `initAgentRepo` finds a repo and commits nothing, so no + * signer is needed (`load()` never signs). + */ +export async function readColdParkedPendingOperations(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; +}): Promise { + const storeDir = stepStorageRoot({ + dataDir: args.dataDir, + workflowRunRepoId: args.workflowRunRepoId, + runId: args.runId, + stepId: args.stepId, + attempt: args.attempt, + }); + if (!(await directoryExists(storeDir))) return []; + const store = await createIsogitStore(storeDir); + const { pendingOperations } = await store.load(); + return pendingOperations; +} + +export async function readColdParkedApprovalSnapshot(args: { + dataDir: string; + workflowRunRepoId: RepoId; + runId: string; + stepId: string; + attempt: number; + correlationId: string; +}): Promise { + return findApprovalSnapshot(await readColdParkedPendingOperations(args), args.correlationId); +} + +/** + * Read a warm (single-step) parked agent's durable pending operations from + * substrate state. A warm agent's pending operations live in its durable + * conversation store, mirrored to the workflow-run substrate under + * `agent-state//`. + * + * Reconstructs that state read-only -- deliberately NOT through + * `DurableConversationRegistry.acquire`, whose first acquire writes and + * commits a substrate restore into the live store and would front-run the warm + * agent's own restore ordering. A respawned child has not rebuilt the live + * store when re-registration runs (resume re-parks without re-invoking the + * step), so the substrate is the only place the pending operations live at that + * moment. Returns an empty list when no durable state exists for the agent. + */ +export async function readWarmParkedPendingOperations(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + stepId: string; +}): Promise { + const agentStateDir = path.join( + args.substrate.getRepoDir(args.workflowRunRepoId), + WORKFLOW_RUN_AGENT_STATE_PREFIX, + encodeURIComponent(args.stepId), + ); + const reconstructed = await reconstructDurableConversation(agentStateDir, args.stepId); + if (reconstructed === null) return []; + return reconstructed.pendingOperations; +} + +export async function readWarmParkedApprovalSnapshot(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + stepId: string; + correlationId: string; +}): Promise { + return findApprovalSnapshot(await readWarmParkedPendingOperations(args), args.correlationId); +} + +/** + * Project a parked step's durable pending operations down to the minimal + * approval records the resume classifier needs. Filters to `approval` (the only + * control-plane kind today) and keeps only the correlationId and the optional + * epoch-ms deadline; the runtime reconstructs the lost `SignalAwaited` from + * those alone, and must not see the reactor's pending-operation internals. + */ +export function toParkedApprovalOps(pendingOperations: PendingOperation[]): ParkedApprovalOp[] { + return pendingOperations + .filter((op) => op.kind === "approval") + .map((op) => ({ + correlationId: op.correlationId, + ...(op.timeoutAt !== undefined ? { timeoutAtMs: op.timeoutAt } : {}), + })); +} + +export interface SidecarStepBuildEnvDeps { + dataDir: string; + workflowRunRepoId: RepoId; + signer: (payload: string) => Promise; + /** + * Deployment mailbox address the supervisor threaded into the child + * (`MAILBOX_ADDRESS`). Used to locate each step's on-disk deploy tree + * for tool materialization (see `stepDeployTreeDir`) AND as the step + * agent's outbound mail `address`: the supervisor signs the agent's + * outbound mail as this address through the host transport (OUTBOUND + * half of mailbox ownership, §3a). For the single-agent deploy this is + * the run address the host registered the agent's `CryptoProvider` + * against. + */ + mailboxAddress: string; + /** + * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), + * threaded from the host through the spawn-time env. Selects the + * head/step collapse when locating a step's deploy tree + * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step + * deployment reads at the head, a multi-step deployment at the per-step + * address, matching the host's producer push. + */ + stepCount: number; + /** + * Child-side outbound-mail bridge over the upstream control channel + * (OUTBOUND half of mailbox ownership, §3a). The per-step env builder + * wraps it in a supervisor-backed `MessageTransport` it supplies as + * the step agent's `env.transport`; the agent's mail tools call + * `transport.send`, which routes through the bridge to the supervisor + * for the actual signed send. The step agent never holds the signing + * key. + */ + outboundMailBridge: ChildOutboundMailBridge; + /** + * Inbound local IMAP surface for the step agent's supervisor-backed + * transport (INBOUND half of mailbox ownership, §3b). Bundles the child + * mailbox reader (a fresh committed snapshot of the deployment's substrate + * `INBOX` per read), the shared mailbox watch registry (the same instance the + * child's control loop fires `mailbox.notify` into), and the sender-key + * resolver `fetchFull` verifies signatures against. When present, the warm + * agent's `mail_read` / `mail_search` / `mail_wait` resolve against the + * committed mailbox rather than throwing "not wired". Absent for a build that + * owns no inbound mailbox (a spawned child or an onTrigger body), whose + * transport inbound stays inert. + */ + inbound?: SupervisorBackedTransportInbound; + /** Per-step tool-loader caps (cache + registry tarball size). */ + cache: StepToolCacheConfig; + /** + * Adapter registry the step agent resolves inference adapters through. + * The child builds this eagerly at boot from the validated + * `SIDECAR_ADAPTER_MANIFEST` (built-ins merged with operator custom + * adapters) and the env builder sets it on `env.deps`, so a step whose + * source names a custom provider resolves in the child exactly as it + * does on the sidecar main path. Without it the step agent would fall + * back to `createAgent`'s built-ins-only default and a custom-provider + * source would fail to resolve at run time. + */ + adapters: AdapterRegistry; + /** + * Durable-conversation registry for the warm single-step agent + * (design §3c). When present, the env builder swaps the per-run isogit + * `ContextStore` for a per-agent durable store whose conversation is + * mirrored to the workflow-run substrate, and restores the prior + * conversation from the substrate before returning the env (so the + * agent's reactor `load()` and the warm cache's lazy build see the + * restored turns -- including the respawn-rebuild path). Absent for a + * multi-step deploy, whose per-step agents are not warm/long-lived and + * need no cross-run conversation durability. + */ + durableConversation?: DurableConversationRegistry; + /** + * Record the tool-mark floor grants derived from the step's + * materialized tool factories, keyed by the step's base id. The env + * builder is the only place the child holds the loaded factories' + * static `definitions` (name + approval mark), so it derives the floor + * here; the grant evaluator (a sibling closure in the substrate + * factory) reads the recorded floor by base step id and merges it under + * the credentials snapshot's grants at authorization time. Keyed by + * base id (matching the credentials snapshot and inference-source + * table) so a `map` iteration's scoped id shares its base step's floor. + * The recording persists across a warm agent's messages: the env + * builder runs once (first build) but the floor it records must be + * available for every later tool call the warm agent makes. + */ + recordToolMarkFloor: (baseStepId: string, grants: GrantRule[]) => void; + /** + * Feed the step agent's OWN evaluated tool factories into the + * materialization slot, instead of reading a pinned tool-package manifest + * off the deploy tree. Set for the source-ref lineage, whose child holds + * the live `WorkflowDefinition` (re-verified against the approved wire hash + * at run start) and threads each step's live `agent.toolFactories` in as + * `req.agent.toolFactories`. Those are the same `AnnotatedToolFactory` + * callables `materializeStepTools` would produce, bare-named (no + * namespacing loader runs), so the existing tool-bearing agent factory + * consumes them unchanged. + * + * The source-ref lineage stages no `tool-packages-manifest.json`, so + * `materializeStepTools` would find nothing and return empty tools; this + * arm is what actually runs a source workflow's tools. No tool-mark floor + * is recorded on this lineage: a source tool's runtime name is the bare + * `definition.name` the probe's capability walk already emitted a + * `tool:` grant for into the frozen credentials snapshot, so the + * snapshot authorizes it directly (unlike a pinned tool, whose namespaced + * name the walk never saw and which the floor exists to compensate for). + */ + sourceTools: boolean; + /** + * Sidecar-local directory of the materialized workflow-definition closure + * (`env.spawn.closurePackageDir`), present only on the source-ref lineage. + * The source arm materializes each step agent's declared plugin packages + * (`req.agent.plugins`) from this already-laid-out closure -- resolving them + * through the workflow package's `node_modules/` graph, no re-download -- and + * feeds the resulting plugin factories into the per-step plugin chain. A + * plugin package contributes no agent-visible tool factory, so this closure + * load is the only channel that reaches an `env.plugins` a source workflow's + * posix bundle consumes. Undefined on a build that materializes pinned tool + * packages instead, which reads no closure. + */ + closurePackageDir?: string; +} + +/** + * Materialize the plugin factories a source-ref step agent declares + * (`req.agent.plugins`) from the workflow's frozen closure. Returns an + * empty list when the agent declares no plugins. Fails closed when plugins + * are declared but the source-ref closure dir is absent -- that pairing is a + * wiring bug (the source arm always carries `closurePackageDir`), and + * silently dropping the plugins would run the workflow without the tools it + * declared and leave the reactor to fail closed on the un-loaded tool. + * + * The closure is already laid out on disk from the deploy-side apply, so + * this is a load-only pass -- no re-download -- resolving each plugin + * package through the workflow package's `node_modules/` graph and + * collecting its `definePlugin` factories. + */ +async function materializeSourcePluginFactories( + deps: SidecarStepBuildEnvDeps, + req: StepInvokeRequest, +): Promise { + const plugins = req.agent.plugins ?? []; + if (plugins.length === 0) { + return []; + } + if (deps.closurePackageDir === undefined) { + throw new Error( + `sidecar workflow-child step invoker: step agent ${JSON.stringify(req.agent.id)} declares plugins ${JSON.stringify(plugins)} but the source-ref closure package dir is absent; a source workflow's plugin factories can only be materialized from its frozen closure`, + ); + } + return loadWorkflowPluginFactoriesFromClosure({ + packageDir: deps.closurePackageDir, + plugins, + }); +} + +/** + * Build the step-invoker `buildEnv` callback the workflow-host's + * adapter consumes. Pulled out of `createSidecarSubstrateFactory` so + * the per-step env construction is observable without standing up the + * full substrate. + * + * The closure reads the per-step source table from the mutable + * reference passed per build, derives the `stepId` / `runId` / `attempt` + * from the runtime's `AuthorizeContext`, resolves the per-step + * `InferenceSource` from that table, and stands up + * a per-step isogit `ContextStore` (also serving as the audit store) + * plus a per-step workspace directory rooted under the run. A + * construction failure (mkdir, isogit init) surfaces here rather than + * being papered over with a stub: the single-step path now always runs + * a real agent against real storage. + */ +/** + * The per-run credential inputs a build resolves the step's `credentials` + * wiring from. The live material cell and the grants resolver ride in from the + * run child (per-run state); the provider registry is sidecar-static and + * combined in at the invoke-step boundary. Absent when no credential material + * was threaded, which leaves the step's inference reader and tool credentials + * unwired. + */ +interface SidecarStepCredentialContext { + readonly materialCell: CredentialMaterialCell; + /** + * The step's grants, resolved live by base step id. Typed `unknown[]` at + * this boundary (the run child owns no grant grammar); the sidecar casts to + * `GrantRule[]` here where the grammar is known, exactly as + * `evaluateGrantsAdapter` does. + */ + readonly resolveStepGrants: (stepId: string) => readonly unknown[]; + readonly providers: CredentialProviderRegistry; +} + +export function createSidecarStepBuildEnv( + deps: SidecarStepBuildEnvDeps, +): ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + credentialContext?: SidecarStepCredentialContext, +) => Promise { + return async ( + req: StepInvokeRequest, + sourcesRef: SourcesSnapshotRef, + credentialContext?: SidecarStepCredentialContext, + ): Promise => { + // Resolve against the live table each build so a source rotation that + // wrote `sourcesRef.current` before this build is reflected in the + // agent this build constructs. A warm agent that is already built does + // not pass through here again, so a rotation does not reach it through + // this path -- this ref covers only a build that has not happened yet. + const resolveStepInferenceSource = createStepInferenceSourceResolver(sourcesRef.current); + const { stepId, runId, attempt } = req.authzContext; + if (stepId === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.stepId is required for per-step InferenceSource resolution; the workflow runtime must populate stepId on every step-originated invocation", + ); + } + if (runId === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.runId is required to root per-step storage under the run; the workflow runtime must populate runId on every step-originated invocation", + ); + } + if (attempt === undefined) { + throw new Error( + "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", + ); + } + const sources = resolveStepInferenceSource(stepId); + // The resolver's arktype guarantees a non-empty chain; assert it here so + // the reactor's initial-source pin (element 0) is a checked fact rather + // than an unchecked index. + const activeSource = sources[0]; + if (activeSource === undefined) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, + ); + } + + // Root the per-step scratch (workspace + tool tarball-cache + + // apply-state). The cold (multi-step) path keys it per + // run/step/attempt: each run rebuilds the agent and its scratch, and + // the run's whole `runs//` subtree is reclaimed on run + // completion. The warm single-step path (`durableConversation` + // present) keys it STABLY per agent so the cached agent reuses one + // workspace across every message -- bounding the warm case to one + // dir per agent and letting that workspace survive child respawn -- + // and the subtree is reclaimed on undeploy. The two keyings live + // under disjoint `runs/` and `warm/` sub-roots so neither sweep + // touches the other's tree, and the durable conversation under + // `agent-conversation-state/` is a different root that neither + // sweep touches. + const storeDir = + deps.durableConversation !== undefined + ? warmStepStorageRoot({ + dataDir: deps.dataDir, + workflowRunRepoId: deps.workflowRunRepoId, + stepId, + }) + : stepStorageRoot({ + dataDir: deps.dataDir, + workflowRunRepoId: deps.workflowRunRepoId, + runId, + stepId, + attempt, + }); + // Conversation storage. For the warm single-step agent the + // conversation must survive child respawn, so it is backed by a + // per-agent durable store whose content is mirrored to the + // workflow-run substrate (design §3c); building it here restores the + // prior conversation before the agent's reactor loads. A multi-step + // deploy (no durable registry) keeps the per-run isogit store: its + // per-step agents are not warm/long-lived and have no cross-run + // conversation to carry. The workdir + tools stay per-run in both + // cases -- only the conversation context is durable across runs. + const storage: ContextStore & AuditStore = + deps.durableConversation !== undefined + ? (await deps.durableConversation.acquire(stepId)).storage + : await createIsogitStore(storeDir, deps.signer); + + // Cold-path resume keying assertion (correct-by-construction guard for + // the resume-attempt invariant documented on `stepStorageRoot`). An + // APPROVAL resume re-invocation delivers the correlated decision to the + // reactor, which rehydrates its gate from THIS store's pending + // operations. A re-dispatchable ask-rail approval gate carries a + // `suspendedCall` (the call the approval re-runs); an async-tool pending + // marker shares `kind: "approval"` but has no `suspendedCall`, and on + // resume the reactor clears its gate WITHOUT re-running the approved + // call. So the resume must find a `suspendedCall`-bearing op for its + // correlationId, mirroring the reactor's own discriminator. Two failure + // modes make that false: the runtime reopened the wrong `attempt`'s + // store (`stepStorageRoot` above), so no pending op matches at all and + // the reactor comes up gateless -- a silent forever-hang; or the only + // match is an async marker, so the gate clears and the approved call is + // silently skipped. Make either loud here, at the single seam that both + // opened the store AND knows an approval resume must find its gate, + // rather than letting it surface as a hang or a skipped call. The + // warm path keys its durable store per agent (not per attempt) and + // rehydrates from a different lifecycle, so + // this assertion is cold-path only. An `"input"` resume is exempt by + // construction: its correlationId names the runtime-minted re-arm channel + // awaiting the step's next trigger, not a reactor gate, so no pending + // operation ever exists for it and the delivered decision is sent as a + // plain next turn (no correlation to rehydrate). + if ( + deps.durableConversation === undefined && + req.resume !== undefined && + req.resume.kind === "approval" + ) { + const resumeCorrelationId = req.resume.correlationId; + const loaded = await storage.load(); + const hasPendingGate = loaded.pendingOperations.some( + (op) => op.correlationId === resumeCorrelationId && op.suspendedCall !== undefined, + ); + if (!hasPendingGate) { + throw new Error( + `sidecar workflow-child step invoker buildEnv: resume of step ${JSON.stringify(stepId)} (run ${JSON.stringify(runId)}, attempt ${String(attempt)}) reopened a ContextStore with no re-dispatchable approval gate for correlationId ${JSON.stringify(resumeCorrelationId)}. The cold-path store is keyed by attempt (${storeDir}); a resume that finds no suspendedCall-bearing pending operation here means it reopened the wrong attempt's store (the reactor would come up gateless and the decision would correlate against nothing) or matched only an async pending marker (the reactor would clear the gate without re-running the approved call). This is a keying violation, not a recoverable state.`, + ); + } + } + + const workdir = path.join(storeDir, "workspace"); + await fs.promises.mkdir(workdir, { recursive: true }); + + // Assemble the step's tool runtime. Two arms: + // + // - Source-ref lineage (`sourceTools`): run the step agent's OWN + // evaluated tool factories. The child holds the live re-verified + // `WorkflowDefinition`, and the runtime threads each step's live agent + // in as `req.agent`, so `req.agent.toolFactories` are the real callable + // `AnnotatedToolFactory`s -- the same shape `materializeStepTools` + // produces (`LoadedToolFactory === AnnotatedToolFactory`), + // bare-named because no namespacing loader runs. The source deploy + // stages no `tool-packages-manifest.json`, so `materializeStepTools` + // would find nothing here; feeding the evaluated factories straight + // into the slot is what runs a source workflow's tools. `packageName` + // is the factory's bundle `id` (the credential Gate-2 key; a mail/plain + // tool declares no credentials, so it stays inert). Plugin factories -- + // which have no agent slot, so Option A cannot carry them -- are + // materialized separately from the frozen closure the child already + // holds (`materializeSourcePluginFactories`) and fed into the SAME + // `pluginFactories` slot `materializeStepTools` fills for pinned + // packages, so the existing per-step plugin chain wires them onto + // `env.plugins`. + // + // - Pinned tool packages (`materializeStepTools`): materialize the pinned + // tool-package closure (posix, LSP, mail, ...) from its on-disk deploy + // tree, rooted per step under `storeDir` so concurrent steps in one + // child never collide on the tarball cache or the apply-state tree. A + // step with no manifest yields empty tools (the legitimate + // `rawManifestBytes === undefined` case); a present-but-broken manifest + // surfaces loudly through `materializeStepTools` rather than degrading to + // empty tools. + const materialization: StepToolMaterialization = + deps.sourceTools === true + ? { + factories: req.agent.toolFactories.map((factory) => ({ + packageName: factory.id, + declaredCredentials: [], + factory, + })), + pluginFactories: await materializeSourcePluginFactories(deps, req), + } + : await materializeStepTools({ + dataDir: deps.dataDir, + mailboxAddress: deps.mailboxAddress, + stepId, + stepCount: deps.stepCount, + storeDir, + cache: deps.cache, + }); + + // Derive and record the step's tool-mark floor from the just-loaded + // factories' static definitions. A pinned tool loads here in the + // child and never reached the hub's capability walk, so its + // `tool:` grant is absent from the credentials snapshot; the + // recorded floor is what lets the grant evaluator authorize a pinned + // tool against its own static mark. Keyed by base step id so the + // evaluator's `baseStepId(stepId)` lookup resolves for both a plain + // step and a `map` iteration's scoped id. + // + // Skipped on the source-ref lineage: a source tool's runtime name is the + // bare `definition.name` the capability walk already emitted a `tool:` + // grant for, so the credentials snapshot authorizes it directly with the + // correct effect -- no floor is needed, and recording one would blur that + // invariant. + if (deps.sourceTools !== true) { + deps.recordToolMarkFloor( + baseStepId(stepId), + deriveToolMarkFloorGrants(materialization.factories.map((f) => f.factory)), + ); + } + + // Supervisor-backed transport for the step agent's mail tools (both + // halves of mailbox ownership, §3a OUTBOUND and §3b INBOUND). Outbound + // (`send`) routes over the control IPC to the supervisor, which performs + // the actual signed send through the host transport as `address`. + // `address` is the deployment mailbox address: the same identity the host + // registered the agent's `CryptoProvider` against, so the outbound mail + // carries the agent's signature with parity to the in-process path. + // Inbound (`deps.inbound`, present for the warm single-step agent) makes + // `mail_read` / `mail_search` / `mail_wait` resolve locally against a fresh + // committed snapshot of the deployment's substrate `INBOX`; a build that + // owns no inbound mailbox (a spawned child) leaves it undefined and the + // inbound methods stay inert. Both `transport` and `address` are the env + // keys `@intx/tools-mail`'s sidecar bundle declares in its `requires`. + const transport = createSupervisorBackedTransport( + deps.outboundMailBridge, + deps.mailboxAddress, + deps.inbound, + ); + + // The host owns capability assembly: it builds the RuntimeCapabilities + // bag here (currently `mail.transport`) and puts it on `env.capabilities`, + // so a bundle consumes the assembled bag rather than re-wrapping a raw + // env key of its own. `@intx/tools-mail`'s sidecar bundle (`requires: + // ["capabilities", "address"]`) resolves `mail.transport` from it. + const capabilities = createHarnessRuntimeCapabilities({ transport }); + + // The step env carries `toolCwd`, `transport`, `address`, and the + // assembled `capabilities` beyond `BaseEnv`. `toolCwd` is the working + // tree the posix tools operate on; `capabilities` is what the mail bundle + // reads; `transport` remains a raw env surface for tool packages that + // read it directly. `address` is observability-only. These widen the + // returned `StepEnvBase` structurally, which the buildEnv return type + // (`StepEnvBase`) accepts (a wider object is assignable to the narrower + // type). + const env: StepEnvBase & { + toolCwd: string; + transport: MessageTransport; + address: string; + capabilities: RuntimeCapabilities; + } = { + // Feed the reactor the step's full ordered failover chain and pin + // its initial source to element 0. The reactor resolves the initial + // source by id and fails over forward through `sources`, so this + // restores cross-source failover inside the workflow-child. + sources, + defaultSource: activeSource.id, + storage, + workdir, + // The per-step workspace is both the lock boundary and the tree the + // filesystem tools operate on; a deployed step gets a throwaway + // scratch dir, so the two coincide. + toolCwd: workdir, + audit: storage, + directors: createDefaultDirectorRegistry(), + // Resolve inference adapters through the child's boot-built + // registry (built-ins + operator custom adapters), so a + // custom-provider step source resolves in the child the same way + // it does on the sidecar main path rather than hitting + // `createAgent`'s built-ins-only default. + deps: createDependencies(deps.adapters), + transport, + address: deps.mailboxAddress, + capabilities, + }; + // Carry the materialized tool runtime to the tool-bearing + // `agentFactory` via the env's symbol-keyed slot. The step-invoker + // adapter spreads this env (`{ ...envBase, authorize }`) before + // handing it to `agentFactory`; object spread preserves own + // symbol-keyed properties, so the slot survives the spread. + attachStepTools(env, materialization); + // Attach the credential wiring so the `agentFactory` can assemble each + // bundle's consumer-scoped `credentials` capability. Grants are wired as a + // THUNK, resolved (and cast to the sidecar's `GrantRule` grammar, the same + // cast `evaluateGrantsAdapter` makes) only when a package actually needs a + // capability -- a step with no credential-consuming package never reads + // them, so a self-discovery resume that precedes the grants barrier does not + // fault on a missing snapshot. Omitted when no credential context was + // threaded, in which case a credential-consuming tool fails closed and loud + // at its own `resolve("credentials")`. + if (credentialContext !== undefined) { + // Inference resolves its source's secret from the SAME live cell tool + // credentials resolve from, by `credentialId`, so the step's sources carry + // no inline key and the child never holds the cipher key. + env.readCurrentMaterial = createInferenceCredentialResolver(credentialContext.materialCell); + attachStepCredentialWiring(env, { + materialCell: credentialContext.materialCell, + resolveGrants: () => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- resolveStepGrants returns unknown[] at the run-child boundary; the sidecar owns the GrantRule grammar + credentialContext.resolveStepGrants(stepId) as readonly GrantRule[], + providers: credentialContext.providers, + }); + } + return env; + }; +} + +/** + * Real per-step invoker for a spawned child's steps -- a childWorkflow child + * and an onTrigger section body alike. It widens the workflow-runtime + * `StepInvoker` with the child's credentials-backed `authorize` (the seam that + * resolves each tool call against the run's grants), the child's own per-step + * inference `sourcesRef` (built fresh per spawn from the env-delivered body + * sources, disjoint from the top-level's mutable table), and the parent run's + * `onEvent` funnel (so the child's live inference events reach the hub + * timeline). + */ +export type SidecarChildStepInvoker = ( + req: StepInvokeRequest, + authorize: WorkflowAuthorizeFn, + sourcesRef: SourcesSnapshotRef, + onEvent: (event: InferenceEvent) => void, + credentialContext?: SidecarStepCredentialContext, +) => Promise; + +/** + * Inputs required to construct the sidecar's in-process child runtime. + * Lifted out of `createSidecarSubstrateFactory` so the implementation + * is exercisable in isolation (the co-located test wires a hand-built + * substrate/principal/scheduler/invokeStep against this surface). + * + * Sub-namespace scoping: the child runtime is invoked with + * `runId: childRunId`. The runtime body threads that id through every + * `repoStore.read/append/subscribe` call, every `blobs.recordOutput` + * call, and every `signalChannel.deliver/awaitNext` call. The host- + * adapter implementations (`createWorkflowRunRepoStore`, + * `createWorkflowRunBlobSubstrate`, `createWorkflowHostSignalChannel`) + * each compute their on-disk path as `runs//...` against the + * supplied workflow-run repo. The net effect is that the child's + * events land under `runs//events/.json` of the + * parent's workflow-run repo, sibling to the parent's own + * `runs//...` subtree. + * + * Substrate identity: the child reuses the parent's wrapped `RepoStore` + * (the workflow-run pack-pushing wrap installed by the factory) so a + * successful child write fires the same hub pack push the parent's + * writes do. The substrate's signing principal (a workflow-process + * principal scoped to the parent's anchorRunId) is reused verbatim + * because the child runs under the same supervisor authority. + */ +interface SidecarRunChildDeps { + /** Wrapped workflow-run substrate (the factory's `substrate`). */ + substrate: RepoStore; + /** Workflow-run repo identifying the parent's deployment. */ + workflowRunRepoId: RepoId; + /** Workflow-run ref the child reads/writes against. */ + workflowRunRef: string; + /** Principal the child presents on every substrate operation. */ + principal: Principal; + /** Host-process scheduler singleton; shared with the parent. */ + scheduler: Scheduler; + /** + * Step invoker the child runtime delegates per-step invocations to. + * + * The in-process child runs a WorkflowDefinition whose stepIds are + * disjoint from the parent's. The parent's + * `STEP_INFERENCE_SOURCES`-pinned `buildStepEnv` knows only the + * parent's stepIds and throws on any other id; routing the child's + * step invocations through that closure surfaces a misleading + * "no InferenceSource pinned" error for every child step. Callers + * therefore supply a SEPARATE invokeStep for the child. The substrate + * factory's default (`childInvokeStep` in + * `createSidecarSubstrateFactory`) runs a real, tool-bearing agent + * against the child's own staged sources; a test may inject its own. + * One invoker serves every spawned child's steps -- a childWorkflow + * child, an onTrigger section body, and a body's own grandchildren -- + * because each of them runs real inference and so must be able to call + * tools. + * + * The invoker receives the child's credentials-backed `authorize` + * alongside the request, mirroring the workflow-process child's + * `ChildStepInvoker`: the runtime body calls `env.invokeStep` with the + * request only, so the invoker -- not the runtime -- is the seam that + * gates each tool call against the run's grants. + */ + invokeStep: SidecarChildStepInvoker; + /** + * Every spawned body's plaintext inference-source table, keyed by definition + * id, decrypted sidecar-side from the run record and delivered through the + * spawn env. Each body path resolves its own table from here by definition id, + * so the child never holds the sidecar cipher key. A body whose id is absent -- + * a deployment restored from a legacy record written before body sources moved + * into the record -- falls back to the on-disk `dataDir` file. + */ + bodySources: BodyInferenceSources; + /** + * Sidecar data dir. The legacy fallback for a body absent from `bodySources` + * reads its `assets/workflow//sources.json`; the terminal + * childWorkflow / body paths resolve per-step storage under it. Required. + */ + dataDir?: string; + /** + * Grant evaluator the child's credentials-backed `authorize` delegates + * each `(resource, action)` decision to. The parent factory owns the + * sidecar's grant-rule grammar and supplies its adapter here so the + * child resolves authorization against the same evaluator the parent's + * steps use. + */ + evaluateGrants: GrantEvaluator; + /** + * Sidecar-static credential provider registry, shared with the top level. + * The child's build combines it with the run's live material cell and the + * child's capped grants to assemble each bundle's consumer `credentials` + * capability, and to resolve its inference source secret against the same + * material. + */ + credentialProviders: CredentialProviderRegistry; + /** Director registry the child runtime uses; defaults to the canonical built-ins. */ + directors?: DirectorRegistry; + /** + * Sidecar-local directory of the materialized workflow-definition closure + * (`env.spawn.closurePackageDir`), the source-ref lineage's only closure. + * `buildChildRunEnv` loads the child body's declared plugin tool definitions + * from it so a plugin-contributed `tool:` the child loads is declared + * when capping the child's inherited grants. A child body that declares no + * plugin package needs no closure read; absent on a lineage that stages no + * closure. + */ + closurePackageDir?: string; + /** Clock for timestamp generation; defaults to `() => new Date()`. */ + clock?: () => Date; + /** + * Random id generator for run ids, signal ids, timer ids; defaults to + * a monotonic counter combined with a random suffix. + */ + newId?: (prefix: string) => string; +} + +/** + * Write a spawned child's inherited grants to its own + * `runs//grants.json` in the deployment's workflow-run repo. + * + * The write goes through the child proxy substrate's + * `writeTreePreservingPrefix` -- the only write path the proxy forwards + * to the supervisor. It names the shallow `runs//` prefix, whose + * merge input is only that level's flat blobs (the prefix read does not + * recurse), so the write rebuilds the level with just `grants.json`. This is + * safe ONLY because the sole caller (`capAndPersistChildGrants`) is write-once: + * it invokes this only when no grants file exists yet, i.e. at the run's birth + * before the runtime appends any event, so the `runs//` subtree is + * empty and nothing is dropped. Invoking it over a populated run would delete + * that run's committed `events/`/`blobs/` subtrees. A later event append names + * the nested `runs//events/` prefix, so its own subtree-delete does + * not reach `grants.json` one level up -- the sibling survives. + */ +async function writeChildRunGrants(args: { + substrate: RepoStore; + workflowRunRepoId: RepoId; + principal: Principal; + ref: string; + childRunId: string; + grants: readonly unknown[]; +}): Promise { + const prefix = `runs/${args.childRunId}/`; + const grantsFile = runGrantsPath(args.childRunId); + const serialized = JSON.stringify({ grants: args.grants }, null, 2); + await args.substrate.writeTreePreservingPrefix(args.principal, args.workflowRunRepoId, args.ref, { + preservePrefix: prefix, + merge: async (existing) => { + const files: Record = {}; + for (const [k, v] of existing) files[k] = v; + files[grantsFile] = serialized; + return files; + }, + message: `Write inherited run grants for ${args.childRunId}`, + }); +} + +/** + * Construct the `RunChildWorkflow` callback the spawn-child adapter + * delegates to. The returned callback, when invoked with the parent + * runtime's attribution + the parent-allocated `childRunId` + the + * resolved `WorkflowDefinition`, builds a fresh `WorkflowRuntimeEnv` + * scoped to `childRunId`, invokes `runtimeRun`, and returns the + * child's terminal status. + * + * Abort propagation: the parent-supplied `signal` is wired to the child + * run's local-abort seam (`RuntimeRunOptions.localAbort`). If it aborts -- + * mid-flight or pre-aborted -- the child's own cancel controller aborts, an + * in-flight step fails, and the returned promise resolves with + * `terminalStatus: "failed"`. There is no durable `CancelRequested`: an + * in-process child writes through the proxy substrate and cannot sign the + * supervisor cancel one would require. + * + * Resource lifecycle: the child's per-run signal channel handle is + * `stop()`ped in a finally block so any background `subscribeKind` + * loop tied to the child's runId tears down before the callback + * returns. The blob substrate, repo store, and scheduler entries are + * either per-call (no handle to dispose) or shared with the parent + * (the scheduler). + */ +export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkflow { + const directors = deps.directors ?? createDefaultDirectorRegistry(); + const clock = deps.clock ?? defaultClock; + const newId = deps.newId ?? defaultNewId; + // No `controlPlanePrincipal`: an in-process child tears down through the + // runtime's local-abort seam (see the `runtimeRun` call below), not a durable + // `CancelRequested`, so it never writes a control-plane cancel this repo store + // would have to sign. Run-body events use the workflow-process `principal`. + // Created once and shared across every child this factory spawns (the + // runtime scopes reads/subscribes by runId), so sibling and grandchild + // spawns route through one repo-store handle rather than a fresh one each. + const repoStore = createWorkflowRunRepoStore({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + }); + // Self-referential `RunChildWorkflow` so a child env's recursive + // `spawnChild` (built via `createInMemorySpawnChild` in `buildChildRunEnv`) + // can route grandchild spawns back through the same adapter. Each invocation + // builds a per-runId env that itself wires a `spawnChild` slot whose + // `runChild` is this same `runChild` constant -- the recursion bottoms + // out when a rung's `WorkflowDefinition` has no `childWorkflow` + // primitive. Sub-namespace scoping continues to hold at every depth + // because `childRunId` flows verbatim into the per-rung + // `blobs`/`signalChannel`/`runtimeRun` calls, keeping every rung's + // events under `runs//...` of the parent's workflow-run repo. + const runChild: RunChildWorkflow = async ( + { definition, childRunId, input, parentRunId, signal, depth, maxChildSpawnDepth }, + onEvent, + credentialMaterial, + ) => { + const { + env, + signalChannel, + definition: rewrittenDefinition, + } = await buildChildRunEnv({ + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + onEvent, + ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), + }); + try { + // Thread this rung's depth/ceiling into the child run so its own + // childWorkflow spawns keep counting against the tree-wide bound (the + // guard lives in the runtime's runChildWorkflow). Without this the + // in-process sidecar recursion would reset to depth 0 each rung. + // A parent abort tears the child down through `runtimeRun`'s local-abort + // seam, not a control-plane cancel: the in-process child writes through + // the workflow-run PROXY substrate, which cannot sign the supervisor + // `CancelRequested` a cancel needs (the kind handler refuses a + // workflow-process-signed cancel). Local teardown aborts the child's own + // controller directly, so an in-flight step fails and the child settles + // `failed` under its own principal -- no durable cancel, no wedge while the + // parent awaits the terminal below. + const handle = runtimeRun(rewrittenDefinition, env, { + runId: childRunId, + triggerPayload: input, + depth, + maxChildSpawnDepth, + localAbort: signal, + }); + const result = await handle.complete; + return { terminalStatus: result.terminalStatus }; + } finally { + await signalChannel.stop(); + } + }; + return runChild; +} + +/** + * Construct the `RunSuspendableChild` callback the suspendable-spawn + * adapter delegates to. The park-aware analog of + * {@link createSidecarRunChild}: instead of awaiting the child's terminal, + * it returns a live {@link SuspendableChildHandle} the caller (`runOnTrigger`) + * drives across the body's approval parks. + * + * Park surfacing: the built env's `onPark` sink translates the child body's + * control-plane parks into the handle's `next()` stream. An `"approval"` + * park is queued for the caller to proxy up on the same correlation; the + * caller's granted decision returns through `resume`, which delivers it on + * the child's own signal channel so the parked step unblocks. A body that + * parks on a control-plane `"input"` channel is a nested onTrigger re-arm + * the suspendable seam does not service -- the caller proxies approvals + * only, so nothing would ever deliver that input and the child would park + * forever. Rather than drop the park and hang, `onPark` surfaces it as a + * hard error on `next()` and tears the child down locally so the section run + * ends loudly (failed). + * + * Signal-channel lifecycle: unlike `createSidecarRunChild`, which stops the + * channel in a `finally` around a single awaited terminal, this keeps the + * channel alive across every park (so `resume` can deliver) and ties its + * teardown to the run's terminal -- the one lifecycle moment every path + * funnels through (normal completion, a parent-abort local teardown, an + * illegal-input-park local teardown, or a runtime failure). Tearing down + * per-`next()` would leak the channel when a parent abort makes `runOnTrigger` + * stop calling `next()` mid-park. + * + * Abort propagation: the parent-supplied `signal` tears the body down through + * `createSuspendableChildHandle`'s local-teardown seam -- the in-process body + * runs under the `workflow-process` principal and cannot sign the control-plane + * `CancelRequested` a durable cancel needs, so on abort its own step fails and + * the run settles `failed` (not `cancelled`), and the terminal-tied teardown + * runs. + */ +export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): RunSuspendableChild { + const directors = deps.directors ?? createDefaultDirectorRegistry(); + const clock = deps.clock ?? defaultClock; + const newId = deps.newId ?? defaultNewId; + // No `controlPlanePrincipal`: an in-process body tears down through the shared + // handle's local-abort seam (`createSuspendableChildHandle`), not a durable + // `CancelRequested`, so it never writes a control-plane cancel this repo store + // would have to sign. Run-body events use the workflow-process `principal`. + const repoStore = createWorkflowRunRepoStore({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + }); + // A body's own `childWorkflow` grandchildren spawn terminal-only: the + // suspendable seam is exercised only by onTrigger sections, and + // `buildChildRunEnv` wires the body env's `spawnChild` (not + // `spawnSuspendableChild`), so a nested onTrigger inside a body fails loud + // rather than silently spawning. + const runChild = createSidecarRunChild(deps); + + return async ( + { + definition, + childRunId, + input, + parentRunId, + signal, + depth, + maxChildSpawnDepth, + resumeFromEvents, + }, + onEvent, + credentialMaterial, + ) => { + const { + env: baseEnv, + signalChannel, + definition: rewrittenDefinition, + } = await buildChildRunEnv({ + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + // The live event sink is always threaded (both a body step and a + // grandchild childWorkflow step run a real agent). The body's own steps + // and its childWorkflow grandchildren, built via the internal + // `createSidecarRunChild(deps)` above, both run through the tool-bearing + // `deps.invokeStep`. + onEvent, + // The run's live credential-material cell so the body's inference resolves + // its source secret against the parent's current delivery; a grandchild + // spawned from the body inherits it through the recursive spawnChild. + ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), + }); + + return createSuspendableChildHandle(baseEnv, { + definition: rewrittenDefinition, + childRunId, + input, + depth, + maxChildSpawnDepth, + ...(resumeFromEvents !== undefined ? { resumeFromEvents } : {}), + signal, + cleanup: () => signalChannel.stop(), + }); + }; +} + +/** + * Read the parent run's grants, cap them to what `definition` declares, and + * persist the capped set as the child run's own `runs//grants.json` + * -- the file the next spawn hop (a childWorkflow grandchild) reads back as its + * ceiling. Returns the capped grants so a caller building a fresh child env can + * also key its credentials snapshot on them. + * + * `definition` MUST be the PRE-rewrite body, its childWorkflow grandchildren + * still INLINE: `collectDeclaredResources` skips a `{ ref }` body, so a + * rewritten definition would drop the grandchild's declared resources and + * under-authorize it. Every birth path materializes a run's grants file, so a + * missing parent file is a defect, not a run that legitimately holds none -- + * fail closed. The cap only removes rules (the parent stays the ceiling), + * matching the top level's "authority bounded by declared capabilities" model. + * + * The grants file is WRITE-ONCE per run: a run's authorization ceiling is fixed + * at its birth, so this reads back an existing `runs//grants.json` + * rather than rewriting it. Re-writing on a resume re-drive is not just + * redundant, it is CORRUPTING: `writeChildRunGrants` commits at the shallow + * `runs//` prefix through a substrate writer separate from the + * runtime's event-log writer, so a re-write racing the runtime's replay + * re-appends on the shared repo regresses another run's event seq (a + * single-writer-invariant violation), and its wholesale subtree-delete drops the + * run's committed `events/`/`blobs/` (the prefix read is non-recursive, so the + * merge cannot carry them forward). The read-back returns the same capped set + * the birth-time write persisted, so a fresh child-env caller keys its + * credentials snapshot on identical grants either way. + */ +async function capAndPersistChildGrants(args: { + deps: SidecarRunChildDeps; + directors: ReturnType; + definition: WorkflowDefinition; + childRunId: string; + parentRunId: string; +}): Promise { + const { deps, directors, definition, childRunId, parentRunId } = args; + const existingChildGrants = await readRunGrants({ + repoStore: deps.substrate, + anchorRunId: deps.workflowRunRepoId.id, + runId: childRunId, + }); + if (existingChildGrants !== undefined) return existingChildGrants; + const parentGrants = await readRunGrants({ + repoStore: deps.substrate, + anchorRunId: deps.workflowRunRepoId.id, + runId: parentRunId, + }); + if (parentGrants === undefined) { + throw new Error( + `sidecar runChild: parent run ${parentRunId} has no grants file at ${runGrantsPath(parentRunId)}; refusing to spawn child ${childRunId} under-authorized`, + ); + } + const pluginDefs: PluginToolDefinitions = + deps.closurePackageDir === undefined + ? new Map() + : await loadWorkflowPluginToolDefinitionsFromClosure({ + packageDir: deps.closurePackageDir, + plugins: collectDeclaredPluginNames(definition), + }); + const declaredResources = collectDeclaredResources(definition, directors, pluginDefs); + const childGrants = filterGrantsToDeclaredResources(parentGrants, declaredResources); + await writeChildRunGrants({ + substrate: deps.substrate, + workflowRunRepoId: deps.workflowRunRepoId, + principal: deps.principal, + ref: deps.workflowRunRef, + childRunId, + grants: childGrants, + }); + return childGrants; +} + +/** + * Build the per-childRunId `WorkflowRuntimeEnv` a spawned child runs + * against: inherit the parent's grants, assemble the child's credentials + * snapshot, and wire the per-run repo store / blob substrate / signal + * channel plus a recursive `spawnChild`. Returned alongside the child's + * signal channel so the caller can `stop()` it once the child settles. + * Shared by the child-drive callers so the env construction lives in one + * place. + */ +async function buildChildRunEnv(args: { + deps: SidecarRunChildDeps; + directors: ReturnType; + clock: () => Date; + newId: (prefix: string) => string; + repoStore: ReturnType; + runChild: RunChildWorkflow; + definition: WorkflowDefinition; + childRunId: string; + parentRunId: string; + /** + * Per-run live inference-event sink, threaded from the parent run's event + * channel. Required for both paths -- a childWorkflow step and an onTrigger + * body step both run a real agent whose inference the hub stream must see, so + * a missing sink is a wiring defect, not a silent drop. + */ + onEvent: (event: InferenceEvent) => void; + /** + * The parent run's live credential-material cell. Threaded so the child's + * inference resolves its source secret by `credentialId` against the run's + * current delivery -- reached live through the shared reference on a rotation. + * Absent when a non-sidecar executor carries no credential material, in which + * case no credential context is assembled and the child's inference reader + * stays unset. + */ + materialCell?: CredentialMaterialCell; +}): Promise<{ + env: WorkflowRuntimeEnv; + signalChannel: ReturnType; + definition: WorkflowDefinition; +}> { + const { + deps, + directors, + clock, + newId, + repoStore, + runChild, + definition, + childRunId, + parentRunId, + onEvent, + materialCell, + } = args; + // A rung may itself embed a child (grandchild recursion) as an inline + // `childWorkflow`. Lift each to an internal `{ ref }` and run the rewritten + // definition whose children are refs -- the shape the runtime dispatches -- + // and keep the lifted definitions in an in-memory map the rung's own + // terminal resolver serves from, so a grandchild spawns with no on-disk read + // at any depth. Returned to the caller so `runtimeRun` drives the rewritten + // form. + const { workflow: rewrittenDefinition, bodies: grandchildBodies } = + rewriteInlineChildWorkflowBodies(definition); + const grandchildMap = new Map(grandchildBodies.map((b) => [b.ref, b.definition])); + // Enumerate any `loop` bodies nested in this spawned body so a loop can run + // here (the runtime needs `env.loopFns` + `env.spawnLoopIteration`, wired + // below). A loop body stays inline on its primitive, so mint a ref-keyed copy + // for the loop-iteration host, keep the pre-rewrite form for the per-iteration + // grant cap, and merge each loop body's own childWorkflow grandchildren into + // `grandchildMap` so a grandchild spawned from a loop body resolves in-memory. + // Mirrors the top-level loop-body registration in run-child.ts. + const loopBodies = enumerateInlineLoopBodies(definition); + const loopBodiesMap = new Map(); + const loopBodyPreRewrite = new Map(); + for (const loopBody of loopBodies) { + loopBodyPreRewrite.set(loopBody.ref, loopBody.definition); + const bodyRewrite = rewriteInlineChildWorkflowBodies(loopBody.definition); + loopBodiesMap.set(loopBody.ref, bodyRewrite.workflow); + for (const grandchild of bodyRewrite.bodies) { + grandchildMap.set(grandchild.ref, grandchild.definition); + } + } + // Resolve the loop while/carry fns from the closure so a loop in this body can + // run. Only load when the body contains a loop. If it declares a loop but no + // closure is wired, that is a wiring defect -- fail loud, mirroring the + // establish-time posture rather than deferring to a mid-run resolve failure. + let loopFns: LoopFnRegistry | undefined; + if (loopBodies.length > 0) { + if (deps.closurePackageDir === undefined) { + throw new Error( + "sidecar child: a loop is nested in this spawned body but deps.closurePackageDir is missing; the loop while/carry fns cannot be resolved", + ); + } + loopFns = await loadWorkflowLoopFnsFromClosure({ + packageDir: deps.closurePackageDir, + }); + eagerlyResolveLoopFns( + [rewrittenDefinition, ...loopBodiesMap.values(), ...grandchildMap.values()], + loopFns, + ); + } + // Read the parent's grants, cap them to what this body declares, and persist + // the capped set as the child's own `runs//grants.json` (the file + // a grandchild reads as its ceiling). `definition` here is the childWorkflow- + // inline form, so the cap keeps a grandchild's declared resources. + const childGrants = await capAndPersistChildGrants({ + deps, + directors, + definition, + childRunId, + parentRunId, + }); + // The child's credentials snapshot applies the capped grant set + // uniformly across every step the child definition declares, keyed on + // each step's id (the same shape the deploy-time and per-run snapshot + // assemblies produce). The in-process child has no per-step mail + // address, so the snapshot's `address` mirrors the step id -- + // `createCredentialsBackedAuthorize` reads only `grants`. + // + // "Every step the child definition declares" reaches past `stepOrder` into + // this body's `loop` bodies: a loop iteration runs under the inherited env, + // so its steps authorize against THIS snapshot under their own plain ids. A + // snapshot built from `stepOrder` alone leaves a loop nested one rung down + // with no entry, and the authorize throws on the body's first tool call. The + // capped set applies to them unchanged -- the cap is taken once, against the + // spawned body, and a loop inherits its enclosing run's authority rather than + // narrowing it again. + const credentialStepIds = new Set(); + walkWorkflowSteps({ + definition: rewrittenDefinition, + descent: LOOP_BODY_DESCENT, + context: "sidecar child credentials snapshot: ", + visit: ({ stepId }) => { + credentialStepIds.add(stepId); + }, + }); + const contentHash = await hashGrants(childGrants); + const credentialsSnapshot: CredentialsSnapshot = { + steps: [...credentialStepIds].map((stepId) => ({ + stepId, + address: stepId, + grants: childGrants, + contentHash, + })), + }; + const blobs = createWorkflowRunBlobSubstrate({ + substrate: deps.substrate, + repoId: deps.workflowRunRepoId, + principal: deps.principal, + runId: childRunId, + ref: deps.workflowRunRef, + }); + const signalChannel = createWorkflowHostSignalChannel({ + repoStore: deps.substrate, + principal: deps.principal, + repoId: deps.workflowRunRepoId, + ref: deps.workflowRunRef, + runId: childRunId, + readState: () => emptyState(childRunId), + newId: () => newId("sig"), + clock, + }); + // The child's `env.authorize` binds to the inherited credentials + // snapshot: each `(resource, action)` decision looks up the step's + // grants and delegates to the parent factory's grant evaluator. The + // runtime body stores this on the env; the child's `invokeStep` + // wrapper below is the seam that consults it per tool call, and an + // action step's `EffectContext` calls it directly for each effect. + const credentialsRef: CredentialsSnapshotRef = { + current: credentialsSnapshot, + }; + const authorize = createCredentialsBackedAuthorize(credentialsRef, deps.evaluateGrants); + const drain = createNoopDrainController(rewrittenDefinition); + // Both the terminal childWorkflow path and the onTrigger BODY path run a real + // agent that resolves inference against its OWN per-step source table, keyed by + // the rewritten ref, and funnels live events to the parent run's channel. So + // `onEvent` is required for both; a missing one is a wiring defect, not a + // silent drop. + if (onEvent === undefined) { + throw new Error( + "sidecar child: onEvent is missing; child inference events would be silently dropped from the hub stream", + ); + } + const childOnEvent = onEvent; + // Resolve fresh per spawn into a `sourcesRef` disjoint from the top-level's + // mutable table, so a top-level source rotation never leaks into a child. The + // sources are delivered plaintext through the spawn env (`deps.bodySources`), + // decrypted sidecar-side, so the child never holds the cipher key. + const sourcesRef: SourcesSnapshotRef = { + current: await resolveBodyStepSources(deps, rewrittenDefinition.id), + }; + // Recursive `spawnChild`: a grandchild embedded inline in this rung is + // resolved from the in-memory map lifted above and flows back into this same + // `runChild` callback. Inject THIS run's event funnel (mirroring the rung-0 + // wrap in run-child.ts) so the grandchild's agent steps ride this run's + // channel; the runtime env keeps the narrow `SpawnChildWorkflow` (no event + // slot). Sub-namespace scoping holds at every depth via `childRunId`. + const spawnHost = createInMemorySpawnChild({ + bodies: grandchildMap, + runChild, + }); + const spawnChild: WorkflowRuntimeEnv["spawnChild"] = (spawnInput) => + spawnHost(spawnInput, childOnEvent, materialCell); + // Assemble the per-step credential context from the run's live material cell + // (threaded in from the parent) when one is present, so the child's inference + // resolves its source secret by `credentialId` against the parent's current + // delivery. Every spawned child's steps are tool-bearing, so the context + // resolves the child's capped grants (mirroring the top level's snapshot + // lookup) and carries the sidecar-static providers for the tool `credentials` + // capability. Absent when no material was threaded (a non-sidecar executor), + // which leaves the child's inference reader unset. + const childCredentialContext: SidecarStepCredentialContext | undefined = + materialCell === undefined + ? undefined + : { + materialCell, + resolveStepGrants: (stepId) => { + const entry = credentialsSnapshot.steps.find( + (step) => step.stepId === baseStepId(stepId), + ); + if (entry === undefined) { + throw new Error( + `sidecar child credential wiring: credentials snapshot has no entry for step ${baseStepId(stepId)}`, + ); + } + return entry.grants; + }, + providers: deps.credentialProviders, + }; + // Per-step invocation seam. The runtime body invokes `env.invokeStep` with + // the request alone; the wrapper forwards the child's credentials-backed + // authorize (so each tool call gates against the inherited grants), the run's + // `sourcesRef`, the event funnel, and the credential context (so the step's + // inference reads the run's live material). One invoker serves both the + // childWorkflow path and the onTrigger body path: each runs a real + // tool-bearing agent through the source-tools arm against the same + // `sourcesRef`. + const invokeStep: WorkflowRuntimeEnv["invokeStep"] = (req) => + deps.invokeStep(req, authorize, sourcesRef, childOnEvent, childCredentialContext); + const env: WorkflowRuntimeEnv = { + repoStore, + scheduler: deps.scheduler, + signalChannel, + blobs, + directors, + authorize, + invokeStep, + spawnChild, + clock, + newId, + drain, + ...(loopFns !== undefined ? { loopFns } : {}), + }; + // Wire loop-iteration spawning for a `loop` nested in this body. Assigned + // AFTER the env literal because the iteration host closes over `env`: a loop + // iteration re-enters THIS body env, so a nested loop composes and inherits + // the body's step invoker, capped grants, and in-memory spawnChild. + // + // This deliberately REPLICATES the top-level loop host in run-child.ts rather + // than sharing a helper. The two live in different packages and differ in the + // grants seam (here `capAndPersistChildGrants` is in scope and called + // directly; the top level injects it as a binding), and the loop-in-body + // crash-resume path is not yet proven identical to the top level's. Extract a + // shared helper only once a deployed loop-in-body resume test shows the two + // paths match. + // + // Boundary: the suspendable-child seam services APPROVAL parks only. A loop + // iteration in a body that awaits an externally-delivered signal, or re-arms + // an onTrigger, inherits that approvals-only limit and cannot park on an + // external input channel. + if (loopFns !== undefined) { + const loopIterationHost = createInMemorySpawnSuspendableChild({ + bodies: loopBodiesMap, + runSuspendableChild: async (loopInput, _onEvent) => { + const preRewriteBody = loopBodyPreRewrite.get(loopInput.definitionRef); + if (preRewriteBody === undefined) { + throw new Error( + `sidecar child: no pre-rewrite loop body for ref ${loopInput.definitionRef}`, + ); + } + // Cap the iteration's grants against the body's own grants (the + // iteration's parent IS the body run) and persist them before the + // iteration appends its first event. + await capAndPersistChildGrants({ + deps, + directors, + definition: preRewriteBody, + childRunId: loopInput.childRunId, + parentRunId: loopInput.parentRunId, + }); + const iterationSignalChannel = createWorkflowHostSignalChannel({ + repoStore: deps.substrate, + principal: deps.principal, + repoId: deps.workflowRunRepoId, + ref: deps.workflowRunRef, + runId: loopInput.childRunId, + readState: () => emptyState(loopInput.childRunId), + newId: () => newId("sig"), + clock, + }); + return createLoopIterationHandle(env, { + definition: loopInput.definition, + childRunId: loopInput.childRunId, + input: loopInput.input, + depth: loopInput.depth, + maxChildSpawnDepth: loopInput.maxChildSpawnDepth, + ...(loopInput.resumeFromEvents !== undefined + ? { resumeFromEvents: loopInput.resumeFromEvents } + : {}), + signal: loopInput.signal, + signalChannel: iterationSignalChannel, + cleanup: () => iterationSignalChannel.stop(), + }); + }, + }); + env.spawnLoopIteration = (spawnInput) => loopIterationHost(spawnInput, childOnEvent); + } + return { env, signalChannel, definition: rewrittenDefinition }; +} + +/** + * Resolve a spawned body's per-step inference sources by definition id. The + * primary path is sidecar-mediated: the sources arrive plaintext in + * `deps.bodySources`, decrypted sidecar-side from the sealed run record, so the + * child holds no cipher key. + * + * LEGACY FALLBACK: a body whose id is absent from the delivered set -- a + * deployment restored from a record written before body sources moved into the + * record -- reads its on-disk plaintext `sources.json`. That file predates the + * change and is plaintext, so the fallback constructs no cipher and the child + * stays key-free. Removable once no restorable record predates the + * record-carried body sources (after the reconnect re-push has re-persisted + * every live deployment). + */ +async function resolveBodyStepSources( + deps: SidecarRunChildDeps, + definitionId: string, +): Promise { + const delivered = deps.bodySources[definitionId]; + if (delivered !== undefined) { + return delivered; + } + if (deps.dataDir === undefined) { + throw new Error( + `sidecar child: body ${definitionId} is absent from the delivered sources and deps.dataDir is missing, so its legacy on-disk sources cannot be read`, + ); + } + return readChildStepInferenceSources(deps.dataDir, definitionId); +} + +/** + * Legacy fallback reader (see `resolveBodyStepSources`): a spawned body's + * plaintext per-step pins from `${dataDir}/assets/workflow//sources.json`, + * staged by a pre-record deploy. Parsed through the same + * `parseStepInferenceSources` boundary the top-level `STEP_INFERENCE_SOURCES` env + * entry uses. Only reached for a body the delivered set does not carry, so a + * missing or malformed file is a defect and surfaces loudly. + */ +async function readChildStepInferenceSources( + dataDir: string, + childRef: string, +): Promise { + const sourcesPath = path.join(dataDir, "assets", "workflow", childRef, "sources.json"); + let raw: string; + try { + raw = await fs.promises.readFile(sourcesPath, "utf8"); + } catch (cause) { + const reason = cause instanceof Error ? cause.message : String(cause); + throw new Error( + `sidecar child: failed to read child inference sources at ${sourcesPath}: ${reason}`, + { cause }, + ); + } + return parseStepInferenceSources(raw); +} + +function defaultClock(): Date { + return new Date(); +} + +let runChildIdCounter = 0; +function defaultNewId(prefix: string): string { + runChildIdCounter += 1; + return `${prefix}-${String(runChildIdCounter)}-${Math.random().toString(36).slice(2, 8)}`; +} + +/** + * Build a `SubstrateFactory` closed over the supplied dependency + * overrides. The production export `createSubstrate` is the + * default-deps call. + * + * Construction order: + * 1. Narrow the `substrateConfig` record against the typed schema. + * A missing or empty key already threw inside the helper; this + * pass enforces the exact shape the factory consumes. + * 2. Open a bare `RepoStore` via `createAgentRepoStore` against the + * sidecar's data dir and Ed25519 keypair. This store backs the + * child's read-only operations against the workflow-run repo; + * the on-disk repo is shared with the supervisor's substrate so + * reads see whatever the supervisor has committed. + * 3. Construct a proxy `RepoStore` whose + * `writeTreePreservingPrefix` forwards over the upstream control + * channel via the substrate-write bridge. The supervisor's + * handler runs its own substrate's `writeTreePreservingPrefix` + * (wrapped at the boot edge with the pack-push facade) under the + * per-repo lock and replies with the resulting `commitSha`. + * 4. Start the host-process scheduler singleton against the proxy + * substrate, then adapt it to the runtime's `Scheduler` shape. + * 5. Construct the production `invokeStep` and `spawnChild` + * adapters. + * 6. Return the `RunWorkflowChildBindings` the runtime body + * consumes, with the proxy store in the `substrate` slot. + */ +export function createSidecarSubstrateFactory( + deps: SidecarSubstrateFactoryDeps = {}, +): SubstrateFactory { + const createBareRepoStore = + deps.createBareRepoStore ?? + (({ dataDir, signingKey }) => createAgentRepoStore({ dataDir, signingKey }).repoStore); + + return async (env: SubstrateFactoryEnv) => { + const validated = SubstrateConfig(env.substrateConfig); + if (validated instanceof type.errors) { + throw new Error( + `sidecar workflow-child substrate config failed validation: ${validated.summary}`, + ); + } + + const stepInferenceSources = parseStepInferenceSources(validated.STEP_INFERENCE_SOURCES); + + // Build the child's adapter registry eagerly at boot from the + // operator-supplied manifest. `loadAdapterRegistry` imports every + // custom module now, so a bad specifier crashes the child loudly at + // construction rather than silently degrading to built-ins-only at + // first resolve. The closure registry the sidecar built at its own + // boot edge cannot cross the fork; the child rebuilds an equivalent + // one from the serialized-and-revalidated manifest. + const childAdapterRegistry = await loadAdapterRegistry( + parseAdapterManifest(validated.SIDECAR_ADAPTER_MANIFEST), + ); + + const signingKey = { + publicKey: hexDecode(validated.SIDECAR_SIGNING_PUBLIC_KEY, "SIDECAR_SIGNING_PUBLIC_KEY"), + privateKey: hexDecode(validated.SIDECAR_SIGNING_PRIVATE_KEY, "SIDECAR_SIGNING_PRIVATE_KEY"), + }; + + const bareStore: RepoStore = createBareRepoStore({ + dataDir: validated.SIDECAR_DATA_DIR, + signingKey, + }); + + const workflowRunRepoId = { + kind: "workflow-run" as const, + id: validated.WORKFLOW_RUN_REPO_ID, + }; + const principal: WorkflowRunWorkflowProcessPrincipal = { + kind: "workflow-process", + anchorRunId: env.spawn.anchorRunId, + }; + + // Proxy substrate: writes are forwarded over IPC into the + // supervisor's substrate; reads consult the bare on-disk store. + // The supervisor is the sole writer of the workflow-run ref so + // the child's writes never race against the supervisor's + // claim-check writes (inbox / processing / consumed). + const substrate: RepoStore = createProxyWorkflowRunRepoStore({ + bareStore, + bridge: env.substrateWriteBridge, + workflowRunRepoId, + }); + + // INBOUND half of mailbox ownership (§3b). One watch registry per child, + // created at boot and shared by BOTH the step agent's supervisor-backed + // transport (its `watch` registers callbacks here, backing `mail_wait`) and + // the child's control loop (which fires each `mailbox.notify` into it). The + // registry rides out on the returned bindings so `runWorkflowChild` routes + // notifications to this same instance. The child mailbox reader opens a + // fresh committed snapshot of the deployment's substrate `INBOX` per read, + // over the same substrate handle / repo id / principal / ref the mail-part + // reader uses, so a read taken after a `mailbox.notify` observes the message + // the supervisor just committed. + const mailboxWatchRegistry = createMailboxWatchRegistry(); + const transportInbound: SupervisorBackedTransportInbound = { + reader: createChildMailboxReader({ + substrate, + repoId: workflowRunRepoId, + principal, + ref: validated.WORKFLOW_RUN_REF, + }), + watchRegistry: mailboxWatchRegistry, + // The child holds no sender-key registry, so signature verification is not + // yet possible here; `fetchFull` reports every message's signature status + // as "unknown". A future sender-key surface would replace this resolver. + getCrypto: () => undefined, + // The write methods (setFlags / clearFlags / expunge) route through this + // bridge to the supervisor, the sole mailbox writer, instead of flushing + // the run ref from the child. + mutationBridge: env.mailboxMutationBridge, + }; + + const hostScheduler = createWorkflowHostScheduler({ + repoStore: substrate, + principal, + listActiveDeployments: () => [workflowRunRepoId], + ref: validated.WORKFLOW_RUN_REF, + clock: () => new Date(), + }); + await hostScheduler.start(); + const scheduler = adaptHostScheduler(hostScheduler); + + // The single-step / top-level path runs a real agent. The per-step + // env builder stands up real per-step storage/workdir/audit/directors + // rooted under the run (see `createSidecarStepBuildEnv`), resolving + // the per-step `InferenceSource` from the pinned table; the real + // step-invoker instantiates the step's agent via `createAgent`, + // delivers the resolved input as a synthesized inbound message, and + // captures the agent's reply as the step output. + const stepToolCache: StepToolCacheConfig = { + cacheMaxBytes: parseByteCap(validated.SIDECAR_CACHE_MAX_BYTES, "SIDECAR_CACHE_MAX_BYTES"), + registryMaxTarballBytes: parseByteCap( + validated.SIDECAR_REGISTRY_MAX_TARBALL_BYTES, + "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", + ), + }; + + // The single-step / top-level path runs a real agent with REAL + // tools materialized in-child. The per-step env builder stands up + // real per-step storage/workdir/audit/directors rooted under the + // run (see `createSidecarStepBuildEnv`), resolves the per-step + // `InferenceSource`, and materializes the step's pinned + // tool-package closure (posix, LSP, mail, ...) from its on-disk + // deploy tree -- rooted per step so concurrent steps in one child + // never collide on the tarball cache or apply-state. The + // tool-bearing `agentFactory` below attaches those factories to the + // step's `AgentDefinition` and builds the plugin chain. + // Durable-conversation registry for the warm single-step agent + // (design §3c). Built only when the deployment is warm-kept: the + // sole long-lived agent's conversation must survive child respawn, + // so it is mirrored to the workflow-run substrate at a per-agent + // path. A multi-step deploy leaves this `undefined` -- its per-step + // agents are not warm/long-lived (§3b), so they carry no cross-run + // conversation and keep the per-run isogit store. The registry lives + // for the child's lifetime; on respawn the child rebuilds it empty + // and each store restores its prior snapshot from the substrate on + // first acquire. + const conversationSigner = createStepStorageSigner(signingKey); + const durableConversation: DurableConversationRegistry | undefined = env.spawn.warmKeep + ? createDurableConversationRegistry({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + substrate, + principal, + signer: conversationSigner, + }) + : undefined; + + // Per-step tool-mark floor grants, keyed by base step id. The step + // env builder derives and records each step's floor from its + // materialized (pinned) tool factories; the grant evaluator reads it + // by base step id and merges it under the credentials snapshot's + // grants so a pinned tool authorizes against its own static mark. The + // map lives for the factory's (child's) lifetime, so a warm agent's + // floor -- recorded on its single first build -- remains available + // for every later tool call it makes. + const toolMarkFloorByStep = new Map(); + + const buildStepEnv = createSidecarStepBuildEnv({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + signer: conversationSigner, + mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, + outboundMailBridge: env.outboundMailBridge, + cache: stepToolCache, + adapters: childAdapterRegistry, + recordToolMarkFloor: (stepId, grants) => { + toolMarkFloorByStep.set(stepId, grants); + }, + // Source-ref is the only deploy lineage: the child runs each step agent's + // own evaluated tool factories (fed from `req.agent.toolFactories`) from + // the materialized closure, never a pinned tool-package manifest off a + // deploy tree. + sourceTools: true, + // The materialized closure dir the source arm materializes each step + // agent's declared plugin packages from. Always present (source-ref only). + closurePackageDir: env.spawn.closurePackageDir, + // Activate the warm agent's inbound mail surface: `mail_read` / + // `mail_search` / `mail_wait` resolve against the deployment's committed + // substrate `INBOX` through this bundle. The spawned-child build below + // omits it (a spawned child owns no warm inbound mailbox). + inbound: transportInbound, + ...(durableConversation !== undefined ? { durableConversation } : {}), + }); + + // The tool-bearing agent factory reads the materialized tool + // runtime off the per-step env (set by `buildStepEnv` via + // `attachStepTools`), attaches the loaded tool factories to the + // step's `AgentDefinition`, builds the plugin chain on + // `env.plugins`, and wraps `agent.close()` so every plugin (the LSP + // subprocess included) and tool bundle is torn down with the agent + // on every exit path. The factory is stateless across steps, so it + // is pinned once here and shared by every per-step invoker built + // below. + const stepAgentFactory = createToolBearingAgentFactory(); + + // The credential provider registry that shapes a delivered credential into + // a mediated handle. Built once here from the sidecar-static built-ins (the + // origin-pinned http provider) and shared by every per-step build; the + // per-run material and grants ride in separately at each invoke. + const credentialProviders = createCredentialProviderRegistry(builtinCredentialProviders()); + + // Spawned-child step build env (INTR-310). Every spawned child's steps -- + // a childWorkflow child's and an onTrigger section body's alike -- run real, + // TOOL-BEARING agents through the same source-tools arm the top level uses: + // the child holds the live re-verified definition, so each step agent + // carries live `toolFactories` fed from `req.agent.toolFactories`, and the + // source arm materializes declared plugins from the shared closure. Tools + // are available wherever inference runs, so a body agent gets the same + // build a top-level step agent gets (see the tool-availability invariant in + // `packages/workflow/README.md`). + // + // Built COLD per invocation -- no `durableConversation`/warm hooks (a + // fan-out branch and a section body are each a fresh run per spawn) and no + // `inbound` (a spawned child owns no warm inbound mailbox; inbound-reading + // tools stay inert). The source arm records no tool-mark floor (a source + // tool's bare `tool:` grant is already in the credentials snapshot), + // so the floor recorder throw-asserts that invariant. + // + // The source arm also keeps the body's tools scoped correctly by + // construction: a body child runs under the PARENT deployment's + // `mailboxAddress`/`stepCount`, so a body step whose id collides with a + // parent step id would read the PARENT step's tools if tools were resolved + // off the deploy tree. Feeding each agent its own evaluated + // `req.agent.toolFactories` never consults that tree. + const coldChildBuildStepEnv = createSidecarStepBuildEnv({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + signer: conversationSigner, + mailboxAddress: env.spawn.mailboxAddress, + stepCount: env.spawn.stepCount, + outboundMailBridge: env.outboundMailBridge, + cache: stepToolCache, + adapters: childAdapterRegistry, + recordToolMarkFloor: () => { + throw new Error("source-tools child build-env must not record a tool-mark floor"); + }, + sourceTools: true, + closurePackageDir: env.spawn.closurePackageDir, + }); + // Spawned-child step invoker (INTR-310). It runs a real agent through + // `createWorkflowStepInvoker`, resolving inference against the child's own + // per-step `sourcesRef` (staged at deploy, read fresh per spawn) and + // funnelling live events to the parent run's channel. `buildChildRunEnv` + // threads in the run's `credentialContext` (the live material cell, the + // child's capped grants, the sidecar-static providers), so the build + // attaches each bundle's `credentials` capability and the step's inference + // resolves its source secret against the run's live material. Absent when no + // material was threaded, in which case a tool that declares a credential + // consumer fails closed and loud at its own `resolve("credentials")`, never + // silently. Wired as `childRunDeps.invokeStep`, so it covers every spawned + // child's steps: a childWorkflow child, an onTrigger section body, and a + // body's own childWorkflow grandchildren. + const childInvokeStep: SidecarChildStepInvoker = ( + req, + authorize, + sourcesRef, + onEvent, + credentialContext, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + buildEnv: (buildReq) => coldChildBuildStepEnv(buildReq, sourcesRef, credentialContext), + agentFactory: stepAgentFactory, + sourcesRef, + onEvent, + })(req); + + // Adapt the workflow-runtime `StepInvoker` shape onto the host's + // `ChildStepInvoker` shape. The host's `onEvent` is the child's + // per-run event-channel sink: the runtime body passes it per step, + // and the chain from here is `onEvent -> child event-channel sender + // -> supervisor -> publishWorkflowInferenceEvent -> hub timeline`. + // + // The `authorize` argument is the child's credentials-backed + // authorize closure (`createCredentialsBackedAuthorize`), threaded + // in from `run-child.ts`'s runtime env. The step agent's runtime + // gates EVERY tool call through `env.authorize` with + // `resource = tool:`, `action = "invoke"` (the inference + // layer's authz before-tool extension); using the credentials-backed + // authorize here means each tool call resolves against the per-step + // grant snapshot the supervisor assembled from the agent's + // `state/grants.json` and pushed over the control IPC. A tool the + // agent's grants do not allow is blocked; a granted tool runs. The + // operator gate at deploy time (the capability walk's `tool:` + // approval) and this runtime grant check are complementary: the walk + // bounds the toolset the deploy may carry, the grant snapshot decides + // which of those the agent may invoke at run time. + // + // A fresh `createWorkflowStepInvoker` is built per invocation so the + // adapter subscribes the step agent's event stream to THIS step's + // `onEvent`. The per-step env builder and the tool-bearing agent + // factory are pinned (closed over above); the event sink and the + // authorize closure vary per step. + // + // The `warmCache` (design §3b) is the run-loop's per-deployment + // warm-agent cache, present only for the single-step long-lived + // deployment the deploy projection marked a warm candidate. When + // supplied, the adapter builds the agent once and reuses it across + // messages; when absent, it keeps instantiate-send-teardown per + // step. Forwarding it here is the only warm-keep wiring this binding + // needs -- the adapter and the run-loop own the rest of the + // lifecycle. + // Run-boundary durability flush (design §3c). When the deployment is + // warm-kept, mirror the warm agent's conversation snapshot to the + // workflow-run substrate after each message's send settles. The key + // is the step identity, the same key the env builder filed the + // durable store under, so the hook resolves the right per-agent + // store. Absent for a multi-step deploy (no durable registry). + const onRunBoundary: ((key: string) => Promise) | undefined = + durableConversation !== undefined + ? async (key: string) => { + await durableConversation.get(key).mirrorToSubstrate(); + } + : undefined; + + // Connector-thread seed (design §3c). When the deployment is warm-kept, + // route each mail-derived inbound message onto the warm agent's + // connector thread before its send, so the reply path has thread state. + // The key is the step identity, the same key the durable store is filed + // under. Absent for a multi-step deploy (no durable registry). + const seedInbound: ((key: string, message: InboundMessage) => Promise) | undefined = + durableConversation !== undefined + ? async (key: string, message: InboundMessage) => { + await durableConversation.get(key).seedInbound(message); + } + : undefined; + + // Connector reply drain (design §3c). When the deployment is + // warm-kept, drive the warm agent's outbound replies through the shared + // connector reply drain: on each `connector.reply` the agent emits, + // compose a threaded reply from the durable store's connector thread, + // send it through the outbound bridge (`bridge.submit`, the same signed- + // send path the agent's own supervisor-backed transport uses), then + // advance the thread from the receipt. Established once per warm agent + // over its lifetime stream; the returned drain handle carries the lifetime + // `done` promise the step-invoker folds into the warm entry so eviction + // drains it, plus the per-turn settle barrier the warm step gates each + // reply turn on. The key is the step identity, the same key the durable + // store is filed under. Absent for a multi-step deploy (no durable + // registry). + const driveReplies: + | ((key: string, stream: AgentEventStream) => ConnectorReplyDrain) + | undefined = + durableConversation !== undefined + ? (key: string, stream: AgentEventStream) => + driveConnectorReplies({ + stream, + composeReply: () => durableConversation.get(key).composeReply(), + send: (message) => env.outboundMailBridge.submit(env.spawn.mailboxAddress, message), + // Build the full RFC 5322 References chain from the deployment's + // committed mailbox: locate the parent by its Message-Id and + // return its own References plus its Message-Id, so a reply + // carries the whole conversational ancestry rather than a + // single element. The reader opens a fresh committed snapshot, + // so a parent committed just before this reply is visible. A + // parent miss (the first reply on a fresh thread, a malformed + // id) returns undefined and the transport derives [inReplyTo]. + resolveReferences: (inReplyTo) => + resolveMailboxReferences(transportInbound.reader, inReplyTo), + onReplySent: (receipt) => durableConversation.get(key).onReplySent(receipt), + }) + : undefined; + + const invokeStep: RunWorkflowChildBindings["invokeStep"] = async ( + req, + onEvent, + authorize, + warmCache, + sourcesRef, + credentialWiring, + mailPartReader, + ) => + createWorkflowStepInvoker({ + workflowAuthorize: authorize, + // Combine the per-run credential wiring (the live material cell and + // the step-grants resolver, ridden in from the run child) with the + // sidecar-static provider registry, so `buildStepEnv` attaches a + // complete credential context and the agentFactory can assemble each + // bundle's consumer-scoped `credentials` capability. + buildEnv: (buildReq) => + buildStepEnv(buildReq, sourcesRef, { + materialCell: credentialWiring.materialRef, + resolveStepGrants: credentialWiring.resolveStepGrants, + providers: credentialProviders, + }), + agentFactory: stepAgentFactory, + onEvent, + sourcesRef, + mailPartReader, + ...(warmCache !== undefined ? { warmCache } : {}), + ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), + ...(seedInbound !== undefined ? { seedInbound } : {}), + ...(driveReplies !== undefined ? { driveReplies } : {}), + })(req); + + const evaluateGrantsAdapter: GrantEvaluator = async ({ resource, action, stepId, grants }) => { + // Merge the step's pinned-tool floor grants (derived and recorded + // by the env builder from the step's materialized factories) under + // the credentials snapshot's grants. The floor supplies the + // `tool:` authority a pinned tool never got from the hub's + // capability walk. It is ADDITIVE: `evaluateGrants` ranks by + // specificity then effect, so a declared `deny` (priority 2) still + // beats the derived `ask`/`allow` and an explicit denial is + // honored -- the floor only raises the minimum authority to the + // tool's static mark. + // + // A missing floor entry (`?? []`) contributes no rows: this can + // only ever fail MORE closed (a pinned tool the hub also did not + // grant stays denied, the pre-#68 behavior), never open a hole, so + // it is safe as an additive default. The floor is keyed by base + // step id, so a `map` iteration's scoped id resolves to its base + // step's floor. + const floor = toolMarkFloorByStep.get(baseStepId(stepId)) ?? []; + const result = await evaluateGrants( + // The credentialsSnapshot's grants are typed as + // `readonly unknown[]` so the workflow-host package does not + // depend on the sidecar's grant-rule grammar. The sidecar owns + // that grammar; the cast surfaces here at the boundary where + // the typed grant shape is known. + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- credentialsSnapshot.steps[*].grants is typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar + [...(grants as readonly GrantRule[]), ...floor], + resource, + action, + ); + return { + effect: result.effect, + matchingGrants: [], + resolvedBy: null, + }; + }; + + const childRunDeps: SidecarRunChildDeps = { + substrate, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + principal, + scheduler, + invokeStep: childInvokeStep, + // Plaintext body sources decrypted sidecar-side from the run record; each + // body path resolves its own table from here by definition id. + bodySources: parseBodyInferenceSources(validated.WORKFLOW_BODY_SOURCES), + dataDir: validated.SIDECAR_DATA_DIR, + evaluateGrants: evaluateGrantsAdapter, + // Shared with the top level's `buildStepEnv`: a spawned child's build + // combines it with the run's live material and capped grants. + credentialProviders, + // The shared closure the child re-walks to cap its inherited grants at + // its declared capabilities. Source-ref only, so always present here. + closurePackageDir: env.spawn.closurePackageDir, + }; + // Terminal childWorkflow executor. `run-child` builds the in-memory + // resolver from this plus the lifted-body map it extracts after loading + // the parent's re-verified definition, so an owned inline child spawns + // with no on-disk asset read. + const runChild = createSidecarRunChild(childRunDeps); + + // An onTrigger section runs each event's body as a suspendable child. + // Source-ref is the only deploy lineage: `run-child` builds the in-memory + // body resolver from this raw executor plus the lifted-body map it extracts + // after re-evaluating the parent's closure, so a body resolves in-process + // with no on-disk read and no separate per-body re-verify (the parent's + // re-verify already covers every inline body). + const runSuspendableChild = createSidecarSpawnSuspendableChild(childRunDeps); + + // Per-run scratch reclamation for the cold (multi-step) path. The + // run-loop fires this once each run reaches its terminal status; it + // drops the run's whole `workflow-step-state//runs//` + // subtree (every step/attempt the run produced), which nothing + // reopens after terminal (resume reads the substrate run log, not + // local step state). + // + // Parked-step safety: reclamation keys on the RUN's terminal status, + // and a step parked on a signal (`awaiting-signal`) keeps the run + // non-terminal, so this never fires while a suspended step's + // `attempt-N` store still holds a live pending-op the resume path must + // reopen. Any future per-STEP reclamation must preserve that invariant + // -- it MUST exclude an `awaiting-signal` step, whose `attempt-N` store + // is the exact store a later crash-resume reopens to rehydrate the + // gate; dropping it would reproduce the empty-store hang the + // resume-attempt recovery closes. + // + // Built only for the cold path: a warm deploy + // roots its single agent's scratch per agent under the disjoint + // `warm/` sub-root (reclaimed on undeploy), and the run-loop's own + // `warmKeep` gate already suppresses the per-run call there, so + // leaving this undefined for warm deploys keeps the path-owning + // module's intent explicit. `rm -rf` semantics via `recursive + + // force` so a run that never wrote scratch (no buildEnv reached) is + // a no-op rather than an ENOENT throw. + const cleanupRunStorage: ((runId: string) => Promise) | undefined = env.spawn.warmKeep + ? undefined + : (runId: string) => + fs.promises.rm( + runStepStorageRoot({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + }), + { recursive: true, force: true }, + ); + + // Recover a parked correlation's approval snapshot for the child's + // re-registration enumeration. Wired unconditionally (unlike + // `cleanupRunStorage`, which is cold-only): a warm agent parks on approval + // just as a cold one does, and the branch on `warmKeep` selects the durable + // read -- cold reads the per-attempt isogit store, warm reconstructs the + // agent's durable conversation state from the substrate. + const loadParkedApproval: LoadParkedApproval = ({ runId, stepId, attempt, correlationId }) => + env.spawn.warmKeep + ? readWarmParkedApprovalSnapshot({ + substrate, + workflowRunRepoId, + stepId, + correlationId, + }) + : readColdParkedApprovalSnapshot({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + stepId, + attempt, + correlationId, + }); + + // Enumerate a crashed step's durable pending approval operations for the + // resume classifier, off the same cold/warm durable read as + // `loadParkedApproval`. Where that binding is a lookup by a known + // correlationId (answering the supervisor's re-registration), this is the + // enumeration the classifier needs when the correlationId never reached the + // log -- the crash-across-park case: read the pending operations, project + // to the minimal approval records the runtime reconstructs `SignalAwaited` + // from. + const readParkedApprovalOps: ReadParkedApprovalOps = async ({ runId, stepId, attempt }) => + toParkedApprovalOps( + env.spawn.warmKeep + ? await readWarmParkedPendingOperations({ + substrate, + workflowRunRepoId, + stepId, + }) + : await readColdParkedPendingOperations({ + dataDir: validated.SIDECAR_DATA_DIR, + workflowRunRepoId, + runId, + stepId, + attempt, + }), + ); + + const bindings: RunWorkflowChildBindings = { + substrate, + workflowRunRepoId, + workflowRunRef: validated.WORKFLOW_RUN_REF, + principal, + invokeStep, + initialSources: stepInferenceSources, + runChild, + runSuspendableChild, + // A loop iteration runs under the inherited env (not `buildChildRunEnv`), + // so it is the one body birth path that writes no grants file of its own. + // Materialize it here -- capping the container run's grants to the loop + // body's declared resources -- so the body's childWorkflow grandchild + // spawn is authorized. `definition` is the PRE-rewrite loop body. + materializeLoopIterationGrants: async ({ parentRunId, childRunId, definition }) => { + await capAndPersistChildGrants({ + deps: childRunDeps, + directors: childRunDeps.directors ?? createDefaultDirectorRegistry(), + definition, + childRunId, + parentRunId, + }); + }, + scheduler, + evaluateGrants: evaluateGrantsAdapter, + loadParkedApproval, + readParkedApprovalOps, + // The same registry the warm agent's transport registers `watch` + // callbacks into; `runWorkflowChild` routes each `mailbox.notify` to it. + mailboxWatchRegistry, + ...(cleanupRunStorage !== undefined ? { cleanupRunStorage } : {}), + }; + return bindings; + }; +} + +/** + * Production substrate factory. The sidecar's + * `bin/workflow-child` binary calls + * `runWorkflowChildFromProcessEnv(createSubstrate, { substrateConfigKeys: SIDECAR_SUBSTRATE_CONFIG_KEYS })` + * and the helper invokes this factory with the parsed env. The + * factory is the default-deps variant of + * `createSidecarSubstrateFactory`; deployments that need a recording + * hub sink (tests, alternate hosts) construct their own via + * `createSidecarSubstrateFactory`. + */ +export const createSubstrate: SubstrateFactory = createSidecarSubstrateFactory(); diff --git a/apps/sidecar/src/workflow-substrate-factory/child-runtime.ts b/apps/sidecar/src/workflow-substrate-factory/child-runtime.ts deleted file mode 100644 index c2c273e04a..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/child-runtime.ts +++ /dev/null @@ -1,719 +0,0 @@ -// In-process child runtime for the workflow-process child: the -// terminal-only `runChild` adapter, the park-aware suspendable-child -// adapter an onTrigger section drives across approval parks, the shared -// per-childRunId env construction, and the body's on-disk -// inference-source read. Sub-namespace scoping keeps every child's -// events under `runs//...` of the parent's workflow-run -// repo; the child reuses the parent's wrapped substrate and principal. - -import fs from "node:fs"; -import path from "node:path"; - -import { createDefaultDirectorRegistry } from "@intx/agent"; -import type { DirectorRegistry } from "@intx/agent"; -import type { Principal, RepoId, RepoStore } from "@intx/hub-sessions/substrate"; -import { signalName } from "@intx/types"; -import type { InferenceEvent } from "@intx/types/runtime"; -import { - createNoopDrainController, - emptyState, - rewriteInlineChildWorkflowBodies, - runtimeRun, - type Scheduler, - type SpawnChildWorkflow, - type StepInvokeRequest, - type StepInvokeResult, - type StepInvoker, - type SuspendableChildPark, - type WorkflowAuthorizeFn, - type WorkflowDefinition, - type WorkflowRuntimeEnv, -} from "@intx/workflow"; -import { - createWorkflowHostSignalChannel, - createWorkflowRunBlobSubstrate, - createWorkflowRunRepoStore, - createInMemorySpawnChild, - type CredentialWiring, - type RunChildWorkflow, - type RunSuspendableChild, - type SourcesSnapshotRef, -} from "@intx/workflow-host"; -import type { MailPartReader } from "@intx/types/runtime"; -import { reportError } from "@corbits/error-sink"; - -import { parseStepInferenceSources, type StepInferenceSourceTable } from "./config"; - -function fireAndForget( - work: Promise, - operation: string, - childRunId: string, - report: typeof reportError, -): void { - void work.catch((error: unknown) => { - report(error, { - operation, - extra: { childRunId }, - }); - }); -} - -/** - * Real per-step invoker for an onTrigger BODY child, distinct from the - * `StepInvoker` stub the shared `SidecarRunChildDeps.invokeStep` carries for - * childWorkflow spawns (which stay `ChildStepNotImplementedError` until - * childWorkflow agent execution is built). The body invoker runs a real - * agent through `createWorkflowStepInvoker`, so it widens the stub with the - * body's own per-step inference `sourcesRef` -- built fresh per body spawn - * from the body's on-disk `sources.json`, disjoint from the top-level's - * mutable source table so a top-level source rotation never leaks into a - * body. It also carries an `onEvent` funnel that attributes the body child's - * live inference events to the body run id on the hub timeline, the - * workflow-typed `authorize` the spawn seam threaded from the parent child - * (CL-6448: the credentials-backed authorize, so a body agent's tool calls - * gate through the same per-step grant snapshot a top-level step's do), - * the parent's live `CredentialWiring` for tool bundles that declare a - * `credentials` capability, and the parent's `MailPartReader` so a body's - * attachments-only inbound mail resolves its parts instead of throwing. - */ -export type SidecarBodyStepInvoker = ( - req: StepInvokeRequest, - authorize: WorkflowAuthorizeFn, - sourcesRef: SourcesSnapshotRef, - onEvent: (event: InferenceEvent) => void, - credentialWiring?: CredentialWiring, - mailPartReader?: MailPartReader, -) => Promise; - -/** - * Inputs required to construct the sidecar's in-process child runtime. - * Lifted out of `createSidecarSubstrateFactory` so the implementation - * is exercisable in isolation (the co-located test wires a hand-built - * substrate/principal/scheduler/invokeStep against this surface). - * - * Sub-namespace scoping: the child runtime is invoked with - * `runId: childRunId`. The runtime body threads that id through every - * `repoStore.read/append/subscribe` call, every `blobs.recordOutput` - * call, and every `signalChannel.deliver/awaitNext` call. The host- - * adapter implementations (`createWorkflowRunRepoStore`, - * `createWorkflowRunBlobSubstrate`, `createWorkflowHostSignalChannel`) - * each compute their on-disk path as `runs//...` against the - * supplied workflow-run repo. The net effect is that the child's - * events land under `runs//events/.json` of the - * parent's workflow-run repo, sibling to the parent's own - * `runs//...` subtree. - * - * Substrate identity: the child reuses the parent's wrapped `RepoStore` - * (the workflow-run pack-pushing wrap installed by the factory) so a - * successful child write fires the same hub pack push the parent's - * writes do. The substrate's signing principal (a workflow-process - * principal scoped to the parent's deploymentId) is reused verbatim - * because the child runs under the same supervisor authority. - */ -export interface SidecarRunChildDeps { - /** Wrapped workflow-run substrate (the factory's `substrate`). */ - substrate: RepoStore; - /** Workflow-run repo identifying the parent's deployment. */ - workflowRunRepoId: RepoId; - /** Workflow-run ref the child reads/writes against. */ - workflowRunRef: string; - /** Principal the child presents on every substrate operation. */ - principal: Principal; - /** Host-process scheduler singleton; shared with the parent. */ - scheduler: Scheduler; - /** - * Step invoker the child runtime delegates per-step invocations to. - * - * The in-process child runs a WorkflowDefinition whose stepIds are - * disjoint from the parent's. The parent's - * `STEP_INFERENCE_SOURCES`-pinned `buildStepEnv` knows only the - * parent's stepIds and throws on any other id; routing the child's - * step invocations through that closure surfaces a misleading - * "no InferenceSource pinned" error for every child step. Callers - * therefore supply a SEPARATE invokeStep for the child that does - * not consult the parent's pinned source table. The substrate - * factory's default wires a stub that mirrors the parent's stub - * step output (`{ output: { reply: req.agent.id, turn: null } }`) - * without resolving any per-step InferenceSource -- threading the - * child's WorkflowDefinition-derived sources into a real inference - * call is not yet built, for the same reason the parent's stub - * exists. - */ - invokeStep: StepInvoker; - /** - * Real per-step invoker used ONLY for an onTrigger body child's own steps. - * `createSidecarSpawnSuspendableChild` wires it onto the body env; the - * shared `invokeStep` stub above stays the seam for childWorkflow spawns - * and for the body's own childWorkflow grandchildren. Absent leaves the - * body on the stub (the behavior an isolation test may exercise). - */ - bodyInvokeStep?: SidecarBodyStepInvoker; - /** - * Sidecar data dir, used ONLY on the body path to read a body's on-disk - * `assets/workflow//sources.json` and build the body's per-step - * inference `sourcesRef`. Required whenever `bodyInvokeStep` is wired. - */ - dataDir?: string; - /** Director registry the child runtime uses; defaults to the canonical built-ins. */ - directors?: DirectorRegistry; - /** Clock for timestamp generation; defaults to `() => new Date()`. */ - clock?: () => Date; - /** - * Random id generator for run ids, signal ids, timer ids; defaults to - * a monotonic counter combined with a random suffix. - */ - newId?: (prefix: string) => string; - /** - * Test seam only — no production caller ever sets this. Defaults to - * `@intx/workflow`'s `runtimeRun`. Exists because bun's `mock.module` - * replaces a module process-wide and cannot be undone for later files - * in the same `bun test` process. - */ - runtimeRun?: typeof runtimeRun; - /** - * Test seam only — no production caller ever sets this. Defaults to - * `@corbits/error-sink`'s `reportError`. Paired with `runtimeRun` so a - * cancel-rejection test can observe reports without a process-wide - * module swap. - */ - reportError?: typeof reportError; - /** - * Test seam only — no production caller ever sets this. Defaults to - * `@intx/workflow-host`'s `createWorkflowHostSignalChannel`. Paired with - * `reportError` so a stop-rejection test can observe reports without a - * process-wide module swap. - */ - createSignalChannel?: typeof createWorkflowHostSignalChannel; -} - -/** - * Construct the `RunChildWorkflow` callback the spawn-child adapter - * delegates to. The returned callback, when invoked with the parent - * runtime's attribution + the parent-allocated `childRunId` + the - * resolved `WorkflowDefinition`, builds a fresh `WorkflowRuntimeEnv` - * scoped to `childRunId`, invokes `runtimeRun`, and returns the - * child's terminal status. - * - * Abort propagation: the parent-supplied `signal` is observed at every - * runtime observation point. If the signal aborts mid-flight the - * runtime body's cancel cascade fires and the returned promise - * resolves with `terminalStatus: "cancelled"`. A pre-aborted signal is - * handled by the spawn-child adapter's entry-time short-circuit; the - * runChild callback itself does not see the pre-abort case. - * - * Resource lifecycle: the child's per-run signal channel handle is - * `stop()`ped in a finally block so any background `subscribeKind` - * loop tied to the child's runId tears down before the callback - * returns. The blob substrate, repo store, and scheduler entries are - * either per-call (no handle to dispose) or shared with the parent - * (the scheduler). - */ -export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkflow { - const directors = deps.directors ?? createDefaultDirectorRegistry(); - const clock = deps.clock ?? defaultClock; - const newId = deps.newId ?? defaultNewId; - const startRun = deps.runtimeRun ?? runtimeRun; - const report = deps.reportError ?? reportError; - // The in-process child runs under real supervisor authority; its - // control-plane cancel (`CancelRequested`) must be signed by a supervisor - // principal, which the kind handler requires and the substrate authorizes - // for this deployment. Run-body events keep their workflow-process - // attribution. - const supervisorPrincipal = { - kind: "supervisor" as const, - deploymentId: deps.workflowRunRepoId.id, - }; - // Created once and shared across every child this factory spawns (the - // runtime scopes reads/subscribes by runId), so sibling and grandchild - // spawns route through one repo-store handle rather than a fresh one each. - const repoStore = createWorkflowRunRepoStore({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - controlPlanePrincipal: supervisorPrincipal, - ref: deps.workflowRunRef, - }); - // Self-referential `RunChildWorkflow` so a child env's recursive - // `spawnChild` (wired inside `buildChildRunEnv`) can route grandchild - // spawns back through the same adapter. Each invocation builds a - // per-runId env that itself wires an in-memory `spawnChild` resolver whose - // `runChild` is this same `runChild` constant -- the recursion bottoms out - // when a rung's `WorkflowDefinition` has no `childWorkflow` primitive. - // Sub-namespace scoping continues to hold at every depth because - // `childRunId` flows verbatim into the per-rung - // `blobs`/`signalChannel`/`runtimeRun` calls, keeping every rung's - // events under `runs//...` of the parent's workflow-run repo. - const runChild: RunChildWorkflow = async ( - { definition, childRunId, input, signal, depth, maxChildSpawnDepth }, - onEvent, - ) => { - const { - env, - signalChannel, - definition: rewrittenDefinition, - } = buildChildRunEnv({ - deps, - directors, - clock, - newId, - repoStore, - runChild, - definition, - childRunId, - onEvent, - }); - try { - const handle = startRun(rewrittenDefinition, env, { - runId: childRunId, - triggerPayload: input, - depth, - maxChildSpawnDepth, - }); - // The resulting `CancelRequested` is written under the supervisor - // principal wired into this run's repo store (see - // `controlPlanePrincipal` above): the kind handler requires a - // supervisor signer for any cancel origin, so a - // workflow-process-signed cancel would be refused and a parent abort - // would surface as a failed rather than cancelled child. - const cancelOnAbort = (): void => { - fireAndForget( - handle.cancel("supervisor-operator", "parent cancelled"), - "sidecar.child-runtime.cancel", - childRunId, - report, - ); - }; - if (signal.aborted) { - cancelOnAbort(); - } else { - signal.addEventListener("abort", cancelOnAbort, { once: true }); - } - try { - const result = await handle.complete; - return { terminalStatus: result.terminalStatus }; - } finally { - signal.removeEventListener("abort", cancelOnAbort); - } - } finally { - await signalChannel.stop(); - } - }; - return runChild; -} - -/** - * Construct the `RunSuspendableChild` callback the suspendable-spawn - * adapter delegates to. The park-aware analog of - * {@link createSidecarRunChild}: instead of awaiting the child's terminal, - * it returns a live `SuspendableChildHandle` the caller (`runOnTrigger`) - * drives across the body's approval parks. - * - * Park surfacing: the built env's `onPark` sink translates the child body's - * control-plane parks into the handle's `next()` stream. An `"approval"` - * park is queued for the caller to proxy up on the same correlation; the - * caller's granted decision returns through `resume`, which delivers it on - * the child's own signal channel so the parked step unblocks. A body that - * parks on a control-plane `"input"` channel is a nested onTrigger re-arm - * the suspendable seam does not service -- the caller proxies approvals - * only, so nothing would ever deliver that input and the child would park - * forever. Rather than drop the park and hang, `onPark` surfaces it as a - * hard error on `next()` and cancels the child so the section run ends - * loudly. - * - * Signal-channel lifecycle: unlike `createSidecarRunChild`, which stops the - * channel in a `finally` around a single awaited terminal, this keeps the - * channel alive across every park (so `resume` can deliver) and ties its - * teardown to the run's terminal -- the one lifecycle moment every path - * funnels through (normal completion, cancel-on-abort, an - * illegal-input-park cancel, or a runtime failure). Tearing down - * per-`next()` would leak the channel when a parent abort makes - * `runOnTrigger` stop calling `next()` mid-park. - * - * Abort propagation: the parent-supplied `signal` is threaded via - * `handle.cancel` exactly as `createSidecarRunChild` does; the child - * runtime takes no abort signal of its own. - */ -export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): RunSuspendableChild { - const directors = deps.directors ?? createDefaultDirectorRegistry(); - const clock = deps.clock ?? defaultClock; - const newId = deps.newId ?? defaultNewId; - const startRun = deps.runtimeRun ?? runtimeRun; - const report = deps.reportError ?? reportError; - // The in-process body child runs under real supervisor authority; its - // control-plane cancel (`CancelRequested`) must be signed by a supervisor - // principal, which the kind handler requires and the substrate authorizes - // for this deployment. Run-body events keep their workflow-process - // attribution. - const supervisorPrincipal = { - kind: "supervisor" as const, - deploymentId: deps.workflowRunRepoId.id, - }; - const repoStore = createWorkflowRunRepoStore({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - controlPlanePrincipal: supervisorPrincipal, - ref: deps.workflowRunRef, - }); - // A body's own `childWorkflow` grandchildren spawn terminal-only: the - // suspendable seam is exercised only by onTrigger sections, and - // `buildChildRunEnv` wires the body env's `spawnChild` (not - // `spawnSuspendableChild`), so a nested onTrigger inside a body fails - // loud rather than silently spawning. - const runChild = createSidecarRunChild(deps); - - return async ( - { - definition, - childRunId, - input, - resumeFromEvents, - signal, - depth, - maxChildSpawnDepth, - authorize: inputAuthorize, - credentialWiring, - mailPartReader, - }, - onEvent, - ) => { - const { - env: baseEnv, - signalChannel, - definition: rewrittenDefinition, - } = buildChildRunEnv({ - deps, - directors, - clock, - newId, - repoStore, - runChild, - definition, - childRunId, - onEvent, - }); - - // The BODY env runs real agent steps when the factory wired a body - // invoker; the body's own childWorkflow grandchildren, built via the - // internal `createSidecarRunChild(deps)` above, do NOT get it and stay - // on the stub. The live event sink is paired with the body invoker: it - // feeds the body's inference to the parent run's event channel. - // The body's per-step sources are read fresh per spawn from the body's - // on-disk `sources.json` when the body invoker is wired; the read is - // awaited here (not inside `buildChildRunEnv`) so the env construction - // stays synchronous for both paths. - let invokeStep = baseEnv.invokeStep; - if (deps.bodyInvokeStep !== undefined) { - if (deps.dataDir === undefined) { - throw new Error( - "sidecar body child: bodyInvokeStep is wired but deps.dataDir is missing; the body's sources.json cannot be resolved", - ); - } - const bodySourcesRef: SourcesSnapshotRef = { - current: await readBodyStepInferenceSources(deps.dataDir, rewrittenDefinition.id), - }; - const bodyInvokeStep = deps.bodyInvokeStep; - const authorize = inputAuthorize ?? baseEnv.authorize; - invokeStep = (req) => - bodyInvokeStep(req, authorize, bodySourcesRef, onEvent, credentialWiring, mailPartReader); - } - - // FIFO the caller drains via `next()`: each entry is either an approval - // park to proxy up, a signal park to relay, or a fatal illegal-park - // error. A single waiter slot suffices because `next()` has exactly one - // consumer (`runOnTrigger`) driving it sequentially, mirroring the - // signal channel's single-consumer shape. - type BodyEvent = - | { kind: "park"; park: SuspendableChildPark } - | { kind: "signal-park"; name: string } - | { kind: "error"; error: Error }; - const events: BodyEvent[] = []; - let wake: (() => void) | null = null; - const notify = (): void => { - if (wake !== null) { - const resolve = wake; - wake = null; - resolve(); - } - }; - - const env: WorkflowRuntimeEnv = { - ...baseEnv, - invokeStep, - onPark: (park) => { - if (park.parkKind === "approval") { - events.push({ - kind: "park", - park: - park.approvalSnapshot !== undefined - ? { - correlationId: park.correlationId, - approvalSnapshot: park.approvalSnapshot, - } - : { correlationId: park.correlationId }, - }); - } else { - events.push({ - kind: "error", - error: new Error( - `onTrigger body ${childRunId} parked on a control-plane input ` + - `channel (${park.correlationId}); a suspendable body may not ` + - `re-arm an input park -- the section proxies approvals only, ` + - `so this park has no resolver`, - ), - }); - } - notify(); - }, - // A body `awaitSignal` gate on an author name: surface it so the - // section proxies it up as a signal-relay await and relays the - // resolved signal back via `deliverSignal`. Without this the body - // would park on the signal channel with nothing upstream to route a - // delivery to it. - onSignalPark: (park) => { - events.push({ kind: "signal-park", name: park.name }); - notify(); - }, - }; - - // On resume, drive the run from its durable log; the body step re-parks - // silently (a re-park does not re-fire onPark), and the caller relays - // the grant via resume on the correlation it recovered from its own - // log. On a fresh spawn, seed the run with the event's trigger payload. - const handle = startRun( - rewrittenDefinition, - env, - resumeFromEvents !== undefined - ? { runId: childRunId, resumeFromEvents, depth, maxChildSpawnDepth } - : { - runId: childRunId, - triggerPayload: input, - depth, - maxChildSpawnDepth, - }, - ); - - // The resulting `CancelRequested` is written under the supervisor - // principal wired into this run's repo store (see - // `controlPlanePrincipal` above): the kind handler requires a - // supervisor signer for any cancel origin, so a workflow-process-signed - // cancel would be refused and a parent abort would surface as a failed - // rather than cancelled child. - const cancelOnAbort = (): void => { - fireAndForget( - handle.cancel("supervisor-operator", "parent cancelled"), - "sidecar.child-runtime.cancel", - childRunId, - report, - ); - }; - if (signal.aborted) { - cancelOnAbort(); - } else { - signal.addEventListener("abort", cancelOnAbort, { once: true }); - } - - let settled: { - terminalStatus: "completed" | "failed" | "cancelled"; - } | null = null; - let failure: Error | null = null; - void handle.complete - .then((result) => { - settled = { terminalStatus: result.terminalStatus }; - }) - .catch((cause: unknown) => { - failure = cause instanceof Error ? cause : new Error(String(cause)); - }) - .finally(() => { - signal.removeEventListener("abort", cancelOnAbort); - fireAndForget( - signalChannel.stop(), - "sidecar.child-runtime.signal-channel-stop", - childRunId, - report, - ); - notify(); - }); - - return { - next: async () => { - for (;;) { - const event = events.shift(); - if (event !== undefined) { - if (event.kind === "error") { - // The body re-armed an input park nothing will resolve. - // Cancel the child so its terminal (and the channel teardown - // tied to it) fires, then surface the error: the throw lands - // the section run's terminal via `runOnTrigger`. - fireAndForget( - handle.cancel( - "supervisor-operator", - "onTrigger body re-armed an unsupported input park", - ), - "sidecar.child-runtime.cancel", - childRunId, - report, - ); - throw event.error; - } - if (event.kind === "signal-park") { - return { kind: "signal-park", name: event.name }; - } - return { kind: "park", park: event.park }; - } - if (failure !== null) throw failure; - if (settled !== null) { - return { - kind: "terminal", - terminalStatus: settled.terminalStatus, - }; - } - await new Promise((resolve) => { - wake = resolve; - }); - } - }, - resume: async (correlationId, decision) => { - await signalChannel.deliver(signalName(correlationId), decision); - }, - deliverSignal: async (name, payload, signalId) => { - await signalChannel.deliver(name, payload, signalId); - }, - }; - }; -} - -/** - * Build the per-childRunId `WorkflowRuntimeEnv` a spawned child runs - * against: wire the per-run blob substrate / signal channel plus a - * recursive `spawnChild`, with the child's `env.authorize` kept as the - * throwing stub (the sub-namespace child does not inherit a per-step - * credentials snapshot from the parent). Returned alongside the child's - * signal channel so the caller owns its `stop()` lifecycle. Shared by the - * terminal-only and suspendable child paths so the env construction lives - * in one place. - */ -function buildChildRunEnv(args: { - deps: SidecarRunChildDeps; - directors: DirectorRegistry; - clock: () => Date; - newId: (prefix: string) => string; - repoStore: ReturnType; - runChild: RunChildWorkflow; - definition: WorkflowDefinition; - childRunId: string; - onEvent: (event: InferenceEvent) => void; -}): { - env: WorkflowRuntimeEnv; - signalChannel: ReturnType; - definition: WorkflowDefinition; -} { - const { deps, directors, clock, newId, repoStore, runChild, definition } = args; - const childRunId = args.childRunId; - // A rung may itself embed a grandchild as an inline `childWorkflow`. Lift - // each to an internal `{ ref }` and run the rewritten definition whose - // children are refs -- the shape the runtime dispatches -- keeping the - // lifted definitions in an in-memory map this rung's own resolver serves - // from, so a grandchild spawns with no on-disk read at any depth. - const { workflow: rewrittenDefinition, bodies: grandchildBodies } = - rewriteInlineChildWorkflowBodies(definition); - const grandchildMap = new Map(grandchildBodies.map((body) => [body.ref, body.definition])); - const blobs = createWorkflowRunBlobSubstrate({ - substrate: deps.substrate, - repoId: deps.workflowRunRepoId, - principal: deps.principal, - runId: childRunId, - ref: deps.workflowRunRef, - }); - const startChannel = deps.createSignalChannel ?? createWorkflowHostSignalChannel; - const signalChannel = startChannel({ - repoStore: deps.substrate, - principal: deps.principal, - repoId: deps.workflowRunRepoId, - ref: deps.workflowRunRef, - runId: childRunId, - readState: () => emptyState(childRunId), - newId: () => newId("sig"), - clock, - }); - // The child's `env.authorize` slot is the workflow-typed authorize - // the runtime body stores; the runtime body never reads it - // directly. Step invocations route through `invokeStep`, which - // wires its own `BaseEnv.authorize` against the workflow-typed - // callback. Throwing here surfaces a precise "no credentialsRef - // installed" error if a future wiring forgets to inject one. - const authorize: WorkflowAuthorizeFn = () => { - // The slot is intentionally throwing: a step that actually calls - // `env.authorize` is asking for a per-step credentials snapshot - // that the sub-namespace child does not yet inherit from the - // parent's `runWorkflowChild` credentialsRef. The slot is - // observable to tests that wire an `invokeStep` bypassing - // authorize. - throw new Error( - "sidecar runChild authorize: per-step credentials snapshot is not threaded through the spawn-child seam; the child runtime cannot resolve a workflow-typed authorize call", - ); - }; - const drain = createNoopDrainController(rewrittenDefinition); - // Recursive `spawnChild`: a grandchild embedded inline in this rung is - // resolved from the in-memory map lifted above and flows back into this - // same `runChild` callback. The runtime body's `runChildWorkflow` contract - // is depth-agnostic; the in-memory resolver makes the sidecar's adapter - // depth-agnostic too, with no on-disk read at any rung. - const hostSpawnChild = createInMemorySpawnChild({ - bodies: grandchildMap, - runChild, - }); - const spawnChild: SpawnChildWorkflow = (spawnInput) => hostSpawnChild(spawnInput, args.onEvent); - const env: WorkflowRuntimeEnv = { - repoStore, - scheduler: deps.scheduler, - signalChannel, - blobs, - directors, - authorize, - invokeStep: deps.invokeStep, - spawnChild, - clock, - newId, - drain, - }; - return { env, signalChannel, definition: rewrittenDefinition }; -} - -/** - * Read an onTrigger body's per-step inference-source pins from - * `${dataDir}/assets/workflow//sources.json`, staged beside the - * body definition at deploy time. Parsed and validated through the same - * `parseStepInferenceSources` boundary the top-level - * `STEP_INFERENCE_SOURCES` env entry uses. A body's sources file is - * guaranteed present (the deploy router materializes it for every - * referenced body), so a missing or malformed file is a defect and - * surfaces loudly rather than degrading to empty pins. - */ -async function readBodyStepInferenceSources( - dataDir: string, - bodyRef: string, -): Promise { - const sourcesPath = path.join(dataDir, "assets", "workflow", bodyRef, "sources.json"); - let raw: string; - try { - raw = await fs.promises.readFile(sourcesPath, "utf8"); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar body child: failed to read body inference sources at ${sourcesPath}: ${reason}`, - { cause }, - ); - } - return parseStepInferenceSources(raw); -} - -function defaultClock(): Date { - return new Date(); -} - -let runChildIdCounter = 0; -function defaultNewId(prefix: string): string { - runChildIdCounter += 1; - return `${prefix}-${String(runChildIdCounter)}-${Math.random().toString(36).slice(2, 8)}`; -} diff --git a/apps/sidecar/src/workflow-substrate-factory/compactors.test.ts b/apps/sidecar/src/workflow-substrate-factory/compactors.test.ts deleted file mode 100644 index c459e715e4..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/compactors.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -// Tests for the summarize-older-turns compactor (CL-6204): a director that -// registers this name and fires `caps.compact("summarize-older-turns", ...)` -// gets a bounded, deterministic fold of everything older than the newest -// `keep` turns instead of silent context-window truncation. -import { expect, test } from "bun:test"; - -import type { ConversationTurn, StrategyContext } from "@intx/types/runtime"; - -import { - SUMMARIZE_BUDGETED_TURNS_NAME, - SUMMARIZE_OLDER_TURNS_NAME, - createBudgetedContextCompactor, - createSummarizeOlderTurnsCompactor, - estimateTurnsChars, -} from "./compactors"; - -function textTurn( - role: ConversationTurn["role"], - text: string, - timestamp: number, -): ConversationTurn { - return { role, content: [{ type: "text", text }], timestamp }; -} - -function makeCtx(): StrategyContext { - return { - state: { - turns: [], - lastCheckpointHash: undefined, - } as unknown as StrategyContext["state"], - trigger: "director:test", - }; -} - -test("passes turns through unfolded when at or under the keep count", async () => { - const compactor = createSummarizeOlderTurnsCompactor({ keep: 12 }); - const turns = Array.from({ length: 12 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `turn ${i}`, i), - ); - - const result = await compactor.apply(turns, makeCtx()); - - expect(result.output).toEqual(turns); - expect(result.record.reason).toBe("within-keep-window"); -}); - -test("keeps the newest K turns verbatim and folds the rest into one synthetic summary turn", async () => { - const compactor = createSummarizeOlderTurnsCompactor({ keep: 12 }); - const turns = Array.from({ length: 20 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `turn ${i}`, i), - ); - - const result = await compactor.apply(turns, makeCtx()); - - // 1 synthetic summary turn + the newest 12 verbatim turns. - expect(result.output).toHaveLength(13); - const [summary, ...kept] = result.output; - expect(summary).toBeDefined(); - expect(summary?.role).toBe("system"); - expect(kept).toEqual(turns.slice(turns.length - 12)); - expect(result.record.reason).toBe("folded-older-turns"); - expect(result.record.decisions["foldedCount"]).toBe(8); - expect(result.record.strategy).toBe(SUMMARIZE_OLDER_TURNS_NAME); -}); - -test("the fold is deterministic for the same input", async () => { - const compactor = createSummarizeOlderTurnsCompactor({ keep: 4 }); - const turns = Array.from({ length: 10 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `message number ${i}`, i), - ); - - const first = await compactor.apply(turns, makeCtx()); - const second = await compactor.apply(turns, makeCtx()); - - expect(first.output).toEqual(second.output); -}); - -test("caps each folded turn's contribution so the summary stays bounded regardless of turn count or length", async () => { - const compactor = createSummarizeOlderTurnsCompactor({ - keep: 2, - maxSummaryChars: 500, - }); - const turns = Array.from({ length: 200 }, (_, i) => - textTurn("user", `a very long message repeated many times `.repeat(20) + String(i), i), - ); - - const result = await compactor.apply(turns, makeCtx()); - const summary = result.output[0]; - - expect(summary?.content[0]?.type).toBe("text"); - const text = summary?.content[0]?.type === "text" ? summary.content[0].text : ""; - expect(text.length).toBeLessThanOrEqual(500); -}); - -test("non-text content blocks fold to a short placeholder rather than being dropped silently", async () => { - const compactor = createSummarizeOlderTurnsCompactor({ keep: 1 }); - const turns: ConversationTurn[] = [ - { - role: "assistant", - timestamp: 1, - content: [{ type: "tool_call", id: "c1", name: "search", arguments: {} }], - }, - textTurn("user", "second", 2), - textTurn("assistant", "third", 3), - ]; - - const result = await compactor.apply(turns, makeCtx()); - const summary = result.output[0]; - const text = summary?.content[0]?.type === "text" ? summary.content[0].text : ""; - expect(text).toContain("tool_call"); -}); - -test("carries the compactor's name and a version for the manifest record", () => { - const compactor = createSummarizeOlderTurnsCompactor({ keep: 12 }); - expect(compactor.name).toBe(SUMMARIZE_OLDER_TURNS_NAME); - expect(compactor.version).toBe("1"); -}); - -test("createBudgetedContextCompactor: a short conversation under budget is untouched", async () => { - const turns = Array.from({ length: 6 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `turn ${i}`, i), - ); - const compactor = createBudgetedContextCompactor(estimateTurnsChars(turns) + 1_000); - - const result = await compactor.apply(turns, makeCtx()); - - expect(result.output).toEqual(turns); - expect(result.record.reason).toBe("within-budget"); -}); - -test("createBudgetedContextCompactor: a conversation past the budget is folded rather than resent whole", async () => { - const turns = Array.from({ length: 40 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `message number ${i}`, i), - ); - // A budget generous enough for only the newest handful of turns. - const budgetChars = estimateTurnsChars(turns.slice(-6)); - const compactor = createBudgetedContextCompactor(budgetChars); - - const result = await compactor.apply(turns, makeCtx()); - - expect(result.record.reason).toBe("folded-older-turns"); - expect(result.record.strategy).toBe(SUMMARIZE_BUDGETED_TURNS_NAME); - expect(result.output.length).toBeLessThan(turns.length); - const [summary] = result.output; - expect(summary?.role).toBe("system"); -}); - -test("estimateTurnsChars measures a tool_result's real content size, not a placeholder", () => { - const largeResult = "x".repeat(20_000); - const turn: ConversationTurn = { - role: "user", - timestamp: 0, - content: [ - { - type: "tool_result", - callId: "call_1", - content: [{ type: "text", text: largeResult }], - }, - ], - }; - - // The old estimator measured `excerptBlock`'s placeholder - // (`[tool_result call_1]`, ~20 chars) instead of the real payload. - expect(estimateTurnsChars([turn])).toBeGreaterThanOrEqual(20_000); -}); - -test("estimateTurnsChars measures a tool_call's real argument size, not a placeholder", () => { - const largeArgs = { query: "y".repeat(15_000) }; - const turn: ConversationTurn = { - role: "assistant", - timestamp: 0, - content: [{ type: "tool_call", id: "c1", name: "search", arguments: largeArgs }], - }; - - expect(estimateTurnsChars([turn])).toBeGreaterThanOrEqual(15_000); -}); - -test("estimateTurnsChars: ten turns each carrying a 20,000-char tool_result sum to their real size, not 10 placeholders", () => { - const turns: ConversationTurn[] = Array.from({ length: 10 }, (_, i) => ({ - role: "user" as const, - timestamp: i, - content: [ - { - type: "tool_result" as const, - callId: `call_${String(i)}`, - content: [{ type: "text" as const, text: "z".repeat(20_000) }], - }, - ], - })); - - const chars = estimateTurnsChars(turns); - - // A hard limit sized for real conversations (e.g. 32,000 chars) must - // see this as over budget -- the old placeholder-based estimator - // returned ~160 chars for the same turns and let it through silently. - expect(chars).toBeGreaterThan(32_000); - expect(chars).toBeGreaterThanOrEqual(200_000); -}); - -test("createBudgetedContextCompactor: folded output never exceeds the budget it folded to", async () => { - // Realistic scale (comparable to `resolveContextBudgetChars` at a - // small `numCtx`, e.g. ~4900 chars for numCtx=2048): a budget bigger - // than `maxSummaryChars` (4000) but not by much, the exact regime - // where an uncounted summary previously pushed the total over. - const turns = Array.from({ length: 40 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `message number ${i} `.repeat(25), i), - ); - const budgetChars = 6_000; - const compactor = createBudgetedContextCompactor(budgetChars); - - const result = await compactor.apply(turns, makeCtx()); - - expect(result.record.reason).toBe("folded-older-turns"); - expect(estimateTurnsChars(result.output)).toBeLessThanOrEqual(budgetChars); -}); - -test("createBudgetedContextCompactor: always keeps a minimum verbatim tail even under a near-zero budget", async () => { - const turns = Array.from({ length: 10 }, (_, i) => - textTurn(i % 2 === 0 ? "user" : "assistant", `message ${i}`, i), - ); - const compactor = createBudgetedContextCompactor(1); - - const result = await compactor.apply(turns, makeCtx()); - - // 1 synthetic summary turn + at least the floor of kept turns. - expect(result.output.length).toBeGreaterThanOrEqual(2); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/compactors.ts b/apps/sidecar/src/workflow-substrate-factory/compactors.ts deleted file mode 100644 index 6b64e36dc7..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/compactors.ts +++ /dev/null @@ -1,388 +0,0 @@ -// Two `Compactor`s (per `@intx/types/runtime`'s `ContextStrategy< -// ConversationTurn[], ConversationTurn[]>`), both deterministic textual -// folds, not LLM calls: `ContextStrategy.apply` receives only `{ state, -// trigger }` (`StrategyContext`) -- no inference handle -- so an -// LLM-based summary is not cleanly available inside this seam today. -// -// - `summarize-older-turns`: fixed `keep` (default 12), registered on -// `env.compactors` for a director that names it via -// `caps.compact("summarize-older-turns", reason)`. -// - `summarize-budgeted-turns` (CL-6204): the one actually fired, by -// `WorkbenchDirector` (see `./workbench-director`) via the same -// `caps.compact` reactor action. `@intx/agent`'s `createAgent` does -// not thread a `ContextTransform` seam through `BaseEnv` at all (only -// `env.compactors` reaches `createReactorAssembly` -- confirmed -// against `@intx/agent`'s `agent.js`), so the reactor's `compact` -// action is the only seam actually available; `WorkbenchDirector` -// fires it only at the one point in the reactor's event flow where -// `compact`'s exclusion from `infer` in the same cycle cannot stall a -// reply -- see that file's header comment for exactly which point and -// why. -// -// Both folds are deliberately lossy -- a bounded recap of what was -// active in the folded turns, not a durable record. Anything a channel -// agent needs to recall past the working window is firm memory's job -// (`@corbits/memory/tools`'s `memory_search`/`memory_add`, already -// wired for the assistant workflow), not this recap's. - -import type { - Compactor, - ContentBlock, - ConversationTurn, - StrategyContext, - StrategyResult, -} from "@intx/types/runtime"; - -export const SUMMARIZE_OLDER_TURNS_NAME = "summarize-older-turns"; -const SUMMARIZE_OLDER_TURNS_VERSION = "1"; - -const DEFAULT_KEEP = 12; -const DEFAULT_MAX_SUMMARY_CHARS = 4_000; -const DEFAULT_MAX_EXCERPT_CHARS_PER_TURN = 200; - -export type SummarizeOlderTurnsOptions = { - /** Newest-turn count kept verbatim. Defaults to 12. */ - keep?: number; - /** Hard cap on the synthetic summary turn's total text length. */ - maxSummaryChars?: number; - /** Per-folded-turn excerpt cap, applied before the total cap. */ - maxExcerptCharsPerTurn?: number; -}; - -export const SUMMARIZE_BUDGETED_TURNS_NAME = "summarize-budgeted-turns"; -const SUMMARIZE_BUDGETED_TURNS_VERSION = "1"; -const MIN_KEPT_TURNS = 4; - -function turnChars(turn: ConversationTurn): number { - let total = 0; - for (const block of turn.content) { - total += blockPayloadChars(block); - } - return total; -} - -/** - * A media block's real payload size: base64 data / a URL / a file - * reference, whichever the source carries. Shared by top-level media - * blocks and the media items nested in a `tool_result`'s `content`. - */ -function mediaSourceChars(source: { - kind: "base64" | "file-reference" | "url"; - data?: string; - url?: string; - reference?: string; -}): number { - switch (source.kind) { - case "base64": - return source.data?.length ?? 0; - case "url": - return source.url?.length ?? 0; - case "file-reference": - return source.reference?.length ?? 0; - } -} - -/** - * A `tool_result` content item's real size: text length, or the - * underlying media source's size for every other item kind. - */ -function toolResultItemChars( - item: Extract["content"][number], -): number { - return item.type === "text" ? item.text.length : mediaSourceChars(item.source); -} - -/** - * A block's true payload size -- what actually ships to the model -- - * as distinct from {@link excerptBlock}'s human-readable placeholder. - * `tool_call.arguments` and `tool_result.content` carry real request/ - * response payloads that can dwarf the rest of a turn; a budget - * estimator blind to them silently undercounts by orders of magnitude. - */ -function blockPayloadChars(block: ContentBlock): number { - switch (block.type) { - case "text": - return block.text.length; - case "refusal": - return block.reason.length; - case "thinking": - return block.thinking.length; - case "redacted_thinking": - return block.data.length; - case "citation": - return block.citedText.length; - case "safety_rating": - return block.blockReason.length; - case "code_execution_request": - return block.code.length; - case "code_execution_result": - return (block.stdout?.length ?? 0) + (block.stderr?.length ?? 0); - case "image": - case "audio": - case "video": - case "document": - return mediaSourceChars(block.source); - case "tool_call": - return block.name.length + JSON.stringify(block.arguments).length; - case "tool_result": { - let total = 0; - for (const item of block.content) { - total += toolResultItemChars(item); - } - if (block.detail !== undefined) { - total += JSON.stringify(block.detail).length; - } - return total; - } - default: - return 0; - } -} - -/** Total character length of a turn list -- the budget check's estimate. */ -export function estimateTurnsChars(turns: ConversationTurn[]): number { - let total = 0; - for (const turn of turns) { - total += turnChars(turn); - } - return total; -} - -/** - * Newest-first walk that keeps turns while their combined length stays - * under `budgetChars`, with a floor of `MIN_KEPT_TURNS` so the most - * recent exchange is never folded away even when it alone exceeds - * budget (the honest-overflow case a director checks separately). - */ -function countTurnsWithinBudget(turns: ConversationTurn[], budgetChars: number): number { - let runningChars = 0; - let kept = 0; - for (let i = turns.length - 1; i >= 0; i--) { - const turn = turns[i]; - if (turn === undefined) continue; - const chars = turnChars(turn); - if (kept >= MIN_KEPT_TURNS && runningChars + chars > budgetChars) { - break; - } - runningChars += chars; - kept += 1; - } - return kept; -} - -function roleLabel(role: ConversationTurn["role"]): string { - return role === "user" ? "User" : role === "assistant" ? "Assistant" : "System"; -} - -/** - * First line of readable content from a turn's blocks, capped at - * `maxChars`. Text/refusal blocks contribute their words; every other - * block kind contributes a short `[]` placeholder so folding - * never silently drops a turn's shape -- only its detail. - */ -function excerptTurn(turn: ConversationTurn, maxChars: number): string { - const parts: string[] = []; - for (const block of turn.content) { - parts.push(excerptBlock(block)); - } - const joined = parts.join(" ").replace(/\s+/g, " ").trim(); - return joined.length > maxChars ? `${joined.slice(0, maxChars)}…` : joined; -} - -function excerptBlock(block: ContentBlock): string { - switch (block.type) { - case "text": - return block.text; - case "refusal": - return block.reason; - case "tool_call": - return `[tool_call ${block.name}]`; - case "tool_result": - return `[tool_result ${block.callId}]`; - default: - return `[${block.type}]`; - } -} - -/** - * Builds a `summarize-older-turns` `Compactor`. Keeps the newest `keep` - * turns verbatim; everything older is replaced by one synthetic `system` - * turn whose text is a numbered, per-turn excerpt list, truncated to - * `maxSummaryChars` total. Pure and deterministic: the same input always - * folds to the same output, with no randomness and no external call. - */ -export function createSummarizeOlderTurnsCompactor( - options: SummarizeOlderTurnsOptions = {}, -): Compactor { - const keep = options.keep ?? DEFAULT_KEEP; - const maxSummaryChars = options.maxSummaryChars ?? DEFAULT_MAX_SUMMARY_CHARS; - const maxExcerptCharsPerTurn = - options.maxExcerptCharsPerTurn ?? DEFAULT_MAX_EXCERPT_CHARS_PER_TURN; - if (!Number.isInteger(keep) || keep < 0) { - throw new Error( - `createSummarizeOlderTurnsCompactor: keep must be a non-negative integer, got ${String(keep)}`, - ); - } - - return { - name: SUMMARIZE_OLDER_TURNS_NAME, - version: SUMMARIZE_OLDER_TURNS_VERSION, - async apply( - turns: ConversationTurn[], - _ctx: StrategyContext, - ): Promise> { - if (turns.length <= keep) { - return { - output: turns, - record: { - strategy: SUMMARIZE_OLDER_TURNS_NAME, - version: SUMMARIZE_OLDER_TURNS_VERSION, - parameters: { keep, maxSummaryChars, maxExcerptCharsPerTurn }, - reason: "within-keep-window", - decisions: { turnCount: turns.length }, - }, - }; - } - - return foldOlderTurns(turns, keep, { - strategy: SUMMARIZE_OLDER_TURNS_NAME, - version: SUMMARIZE_OLDER_TURNS_VERSION, - maxSummaryChars, - maxExcerptCharsPerTurn, - }); - }, - }; -} - -/** - * Shared fold: keeps the newest `keep` turns verbatim and replaces - * everything older with one synthetic `system` turn of numbered, - * per-turn excerpts, truncated to `maxSummaryChars`. The recap is - * deliberately lossy -- it is a bounded reminder of what was active in - * the folded turns, not a durable record. Anything a channel agent - * needs to recall past the working window belongs in firm memory - * (`@corbits/memory/tools`'s `memory_search`/`memory_add`), not in this - * recap. - */ -function foldOlderTurns( - turns: ConversationTurn[], - keep: number, - meta: { - strategy: string; - version: string; - maxSummaryChars: number; - maxExcerptCharsPerTurn: number; - }, -): StrategyResult { - const splitAt = turns.length - keep; - const older = turns.slice(0, splitAt); - const kept = turns.slice(splitAt); - - const lines = older.map( - (turn, i) => - `${String(i + 1)}. ${roleLabel(turn.role)}: ${excerptTurn(turn, meta.maxExcerptCharsPerTurn)}`, - ); - let summaryText = - `[Summary of ${String(older.length)} earlier turns, folded to stay ` + - `within context]\n${lines.join("\n")}`; - if (summaryText.length > meta.maxSummaryChars) { - summaryText = `${summaryText.slice(0, meta.maxSummaryChars - 1)}…`; - } - - const summaryTurn: ConversationTurn = { - role: "system", - content: [{ type: "text", text: summaryText }], - timestamp: older[0]?.timestamp ?? Date.now(), - }; - - return { - output: [summaryTurn, ...kept], - record: { - strategy: meta.strategy, - version: meta.version, - parameters: { - keep, - maxSummaryChars: meta.maxSummaryChars, - maxExcerptCharsPerTurn: meta.maxExcerptCharsPerTurn, - }, - reason: "folded-older-turns", - decisions: { - foldedCount: older.length, - keptCount: kept.length, - summaryChars: summaryText.length, - }, - }, - }; -} - -export type BudgetedContextCompactorOptions = { - /** Hard cap on the synthetic summary turn's total text length. */ - maxSummaryChars?: number; - /** Per-folded-turn excerpt cap, applied before the total cap. */ - maxExcerptCharsPerTurn?: number; -}; - -/** - * Builds a `summarize-budgeted-turns` `Compactor`: folds turns older - * than the newest ones that fit `budgetChars` (with a floor of - * `MIN_KEPT_TURNS` verbatim), instead of a fixed turn count. - * `budgetChars` should come from `resolveContextBudgetChars` so the - * fold point tracks the model's actual context window rather than one - * constant shared by every model. - * - * Registered on `env.compactors` (see `./step-env`) and fired by - * `WorkbenchDirector` via `caps.compact("summarize-budgeted-turns", …)` - * only at the one point in the reactor's event flow where doing so - * cannot stall a reply -- see `./workbench-director`'s header comment. - */ -export function createBudgetedContextCompactor( - budgetChars: number, - options: BudgetedContextCompactorOptions = {}, -): Compactor { - const maxSummaryChars = options.maxSummaryChars ?? DEFAULT_MAX_SUMMARY_CHARS; - const maxExcerptCharsPerTurn = - options.maxExcerptCharsPerTurn ?? DEFAULT_MAX_EXCERPT_CHARS_PER_TURN; - - return { - name: SUMMARIZE_BUDGETED_TURNS_NAME, - version: SUMMARIZE_BUDGETED_TURNS_VERSION, - async apply( - turns: ConversationTurn[], - _ctx: StrategyContext, - ): Promise> { - // First check against the full budget, exactly as when no fold is - // needed at all -- a conversation already under budget must stay - // untouched rather than being folded pre-emptively to make room - // for a summary turn nothing will produce. - if (countTurnsWithinBudget(turns, budgetChars) >= turns.length) { - return { - output: turns, - record: { - strategy: SUMMARIZE_BUDGETED_TURNS_NAME, - version: SUMMARIZE_BUDGETED_TURNS_VERSION, - parameters: { - budgetChars, - maxSummaryChars, - maxExcerptCharsPerTurn, - }, - reason: "within-budget", - decisions: { turnCount: turns.length }, - }, - }; - } - - // Folding does happen: reserve the summary turn's own worst-case - // size out of the budget so kept-turns chars + summary chars - // together stay within `budgetChars`, instead of the summary - // landing on top of an already-full budget. - const keepBudgetChars = Math.max(0, budgetChars - maxSummaryChars); - const keep = countTurnsWithinBudget(turns, keepBudgetChars); - return foldOlderTurns(turns, keep, { - strategy: SUMMARIZE_BUDGETED_TURNS_NAME, - version: SUMMARIZE_BUDGETED_TURNS_VERSION, - maxSummaryChars, - maxExcerptCharsPerTurn, - }); - }, - }; -} diff --git a/apps/sidecar/src/workflow-substrate-factory/config.test.ts b/apps/sidecar/src/workflow-substrate-factory/config.test.ts deleted file mode 100644 index 7801c44eb8..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/config.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { expect, test } from "bun:test"; - -import { deriveHubHttpUrl } from "./config"; - -test("derives an https origin from a wss:// hub URL", () => { - expect(deriveHubHttpUrl("wss://hub.example.com/api/sidecars/ws")).toBe("https://hub.example.com"); -}); - -test("derives an http origin from a ws:// hub URL", () => { - expect(deriveHubHttpUrl("ws://localhost:3000/api/sidecars/ws")).toBe("http://localhost:3000"); -}); - -test("preserves a non-default port", () => { - expect(deriveHubHttpUrl("wss://hub.example.com:8443/ws")).toBe("https://hub.example.com:8443"); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/config.ts b/apps/sidecar/src/workflow-substrate-factory/config.ts deleted file mode 100644 index 5aab3b0640..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/config.ts +++ /dev/null @@ -1,232 +0,0 @@ -// Substrate-config parsing for the sidecar's workflow-process child: -// the required substrate-config key allowlist, the typed schema the -// factory narrows the config record against, and the per-key parsers -// (byte caps, the per-step inference-source table, and the custom -// inference adapter manifest). - -import { type } from "arktype"; - -import { AdapterManifest } from "@intx/inference"; -import { InferenceSource } from "@intx/types/runtime"; - -/** - * Required substrate-config keys the sidecar's binary forwards into - * the factory's `substrateConfig` slot. Listed here so the binary - * passes the same names to the helper; the helper enforces - * presence-and-non-empty against this allowlist before the factory - * runs. - * - * `HUB_WS_URL`, `SIDECAR_ID`, and `SIDECAR_TOKEN` carry the - * hub-connection trust anchors the child needs to ship workflow-run - * pack pushes back to the hub. The sidecar's deploy router populates - * these via the supervisor's `substrateEnv` plumbing - * (`multistepSubstrateEnv` on `createSidecarDeployRouter`), threaded - * from the boot edge's own env reads. - */ -export const SIDECAR_SUBSTRATE_CONFIG_KEYS = [ - "SIDECAR_DATA_DIR", - "WORKFLOW_DEFINITION_ID", - "WORKFLOW_TENANT_ID", - "WORKFLOW_PRINCIPAL_ID", - "WORKFLOW_RUN_REPO_ID", - "WORKFLOW_RUN_REF", - "SIDECAR_SIGNING_PUBLIC_KEY", - "SIDECAR_SIGNING_PRIVATE_KEY", - "HUB_WS_URL", - "SIDECAR_ID", - "SIDECAR_TOKEN", - "STEP_INFERENCE_SOURCES", - "SIDECAR_CACHE_MAX_BYTES", - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - "SIDECAR_ADAPTER_MANIFEST", - "SIDECAR_TOOL_REGISTRIES", -] as const; - -export const SubstrateConfig = type({ - SIDECAR_DATA_DIR: "string > 0", - // The deployed definition's own id, for the workflow-run-authenticated - // capabilities route a step tool calls. Identity only: the definition - // itself is evaluated from the closure, never read from a repo. - WORKFLOW_DEFINITION_ID: "string > 0", - // The run's own tenant and principal, taken from the hub's signed deploy - // frame (`HarnessConfig.tenantId` / `.principalId`). A step tool addresses - // stock `/api/tenants/:tenantId/*` with them and the run bearer; without - // them a tool package has no way to name the tenant it is running for. - WORKFLOW_TENANT_ID: "string > 0", - WORKFLOW_PRINCIPAL_ID: "string > 0", - WORKFLOW_RUN_REPO_ID: "string > 0", - WORKFLOW_RUN_REF: "string > 0", - SIDECAR_SIGNING_PUBLIC_KEY: "string > 0", - SIDECAR_SIGNING_PRIVATE_KEY: "string > 0", - HUB_WS_URL: "string > 0", - SIDECAR_ID: "string > 0", - SIDECAR_TOKEN: "string > 0", - STEP_INFERENCE_SOURCES: "string > 0", - // Per-step tool-loader caps. The supervisor threads the boot edge's - // resolved `SIDECAR_CACHE_MAX_BYTES` / `SIDECAR_REGISTRY_MAX_TARBALL_BYTES` - // through `substrateEnv` so the child's per-step tool materialization is - // bounded by the sidecar's boot-edge-resolved caps. Validated as - // positive-finite-number strings at this boundary. - SIDECAR_CACHE_MAX_BYTES: "string > 0", - SIDECAR_REGISTRY_MAX_TARBALL_BYTES: "string > 0", - // JSON-encoded custom inference adapter manifest. Required: the boot - // edge always serializes it into `substrateEnv` (defaulting to "[]" - // when no custom adapters are configured), so a missing key child-side - // is a serialization bug and must fail loud here, exactly like the - // byte-cap fields. Validated as a non-empty string at this boundary; - // its JSON shape is re-validated against `AdapterManifest` in - // `parseAdapterManifest` before any module is imported. - SIDECAR_ADAPTER_MANIFEST: "string > 0", - // JSON-encoded tool-registry list. Required: the boot edge always - // serializes it (defaulting to the public npmjs registry when the - // operator pinned none), so a missing key child-side is a - // serialization bug and must fail loud here. Its JSON shape is - // re-validated by `parseToolRegistries` before any tool loads — - // per-step materialization never falls back to a default of its own. - SIDECAR_TOOL_REGISTRIES: "string > 0", -}).onUndeclaredKey("ignore"); - -/** - * Parse a substrate-config cap entry (`SIDECAR_CACHE_MAX_BYTES` / - * `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`) into a positive finite number. - * The boot edge already validated these via the `config.ts` readers - * before serializing them into `substrateEnv`; this re-parse at the - * child boundary keeps the typed-config contract honest rather than - * trusting the wire blindly. - */ -export function parseByteCap(raw: string, name: string): number { - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) { - throw new Error( - `sidecar workflow-child substrate config: ${name} must be a positive finite number; got ${JSON.stringify(raw)}`, - ); - } - return n; -} - -/** - * Per-step inference-source table parsed from the spawn-time - * `STEP_INFERENCE_SOURCES` env entry. The deploy router serializes - * `frame.workflow.sources` (a `Record`) as - * JSON and threads it through the supervisor's `substrateEnv`; the - * factory parses and validates the table once at construction time and - * seeds it into the run loop's mutable sources reference, which each - * `buildEnv` reads. Each value is the step's ordered - * failover chain -- element 0 is the active source, the tail are the - * reactor's forward-only failover targets -- so the list is non-empty. - */ -export const StepInferenceSourceTable = type({ - "[string]": InferenceSource.array().atLeastLength(1), -}); -export type StepInferenceSourceTable = typeof StepInferenceSourceTable.infer; - -/** - * Parse and validate the JSON-encoded `STEP_INFERENCE_SOURCES` entry - * the supervisor threaded through `substrateEnv`. A malformed JSON - * payload, a non-object root, or a value that does not match - * `Record` is rejected at the boundary with - * a structured error rather than being deferred to a deep-stack - * `buildEnv` failure. - */ -export function parseStepInferenceSources(raw: string): StepInferenceSourceTable { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - const reason = cause instanceof Error ? cause.message : String(cause); - throw new Error( - `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES is not valid JSON: ${reason}`, - { cause }, - ); - } - const validated = StepInferenceSourceTable(parsed); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config: STEP_INFERENCE_SOURCES failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** - * Parse and validate the JSON-encoded `SIDECAR_ADAPTER_MANIFEST` entry - * the supervisor threaded through `substrateEnv` from the boot edge's - * `readAdapterManifest`. - * - * Trust boundary: the child's substrate config is operator-supplied - * (the supervisor's `Bun.spawn` env), so this re-validation is - * defense-in-depth at the deserialization boundary, NOT a trust - * upgrade. The manifest was already trusted operator config on the - * parent side; the same channel already carries the sidecar's signing - * private key, so it is not a lower-trust surface. Re-asserting the - * shape here keeps the typed-config contract honest rather than - * importing modules off an unvalidated wire value. - * - * Host contract for custom adapters: a manifest `specifier` must - * resolve from BOTH the sidecar's and this child's module-resolution - * roots (the child is a separate `bun` process spawned by the - * supervisor), and an adapter module MUST be import-side-effect-free — - * it is imported once per process by `loadAdapterRegistry`, and any - * top-level side effect would run independently in the parent and in - * every child. - */ -export function parseAdapterManifest(raw: string): AdapterManifest { - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new Error( - "sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST is not valid JSON", - { cause }, - ); - } - const validated = AdapterManifest(parsed); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config: SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, - ); - } - return validated; -} - -/** - * Resolve the per-step inference-source failover chain from the table a - * build reads. The supervisor's multi-step branch only invokes - * a step whose `stepId` appears in `frame.workflow.sources`; a lookup - * miss here is a programmer error in the supervisor, not a wire-side - * failure, and the resolver surfaces it with the missing `stepId` - * named. The returned list is the step's ordered chain (element 0 the - * active source, the tail the reactor's failover targets); the table's - * arktype guarantees it is non-empty. - */ -/** - * Derives the hub's plain HTTP origin from its `HUB_WS_URL` trust anchor - * (`ws://` -> `http://`, `wss://` -> `https://`, same host/port, no - * path). Used to reach the workflow-artifacts HTTP surface - * (`@corbits/artifacts-hub`'s `createWorkflowArtifactRoutes`, CL-6000) - * with the same `SIDECAR_TOKEN` the child already carries for pack-push - * — a second `HUB_HTTP_URL` substrate-config key would just be another - * name for the same origin the operator already configured once. - */ -export function deriveHubHttpUrl(hubWsUrl: string): string { - const url = new URL(hubWsUrl); - url.protocol = url.protocol === "wss:" ? "https:" : "http:"; - url.pathname = "/"; - url.search = ""; - url.hash = ""; - return url.origin; -} - -export function createStepInferenceSourceResolver( - table: StepInferenceSourceTable, -): (stepId: string) => InferenceSource[] { - return (stepId: string): InferenceSource[] => { - const sources = table[stepId]; - if (sources === undefined) { - throw new Error( - `sidecar workflow-child step invoker buildEnv: no InferenceSource pinned for stepId ${JSON.stringify(stepId)}; the supervisor must populate frame.workflow.sources for every stepOrder entry`, - ); - } - return sources; - }; -} diff --git a/apps/sidecar/src/workflow-substrate-factory/context-budget.test.ts b/apps/sidecar/src/workflow-substrate-factory/context-budget.test.ts deleted file mode 100644 index e8d6db2282..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/context-budget.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -// Tests for the context-budget resolver: quirks `numCtx` pin, then the -// advertised catalog window for the model name, never a baked 8k-token -// (32k-char) cap for frontier models. -import { expect, test } from "bun:test"; - -import { - advertisedContextWindowTokens, - readNumCtxHint, - resolveContextBudgetChars, - resolveContextWindowTokens, - resolveHardContextLimitChars, -} from "./context-budget"; - -test("readNumCtxHint: reads a per-model override over the default", () => { - const quirks = { - default: { numCtx: 8_000 }, - perModel: { "gpt-oss:20b": { numCtx: 128_000 } }, - }; - - expect(readNumCtxHint(quirks, "gpt-oss:20b")).toBe(128_000); - expect(readNumCtxHint(quirks, "other-model")).toBe(8_000); -}); - -test("readNumCtxHint: unrecognized or absent quirks resolve to undefined", () => { - expect(readNumCtxHint(undefined, "claude")).toBeUndefined(); - expect(readNumCtxHint(null, "claude")).toBeUndefined(); - expect(readNumCtxHint({ unrelated: true }, "claude")).toBeUndefined(); - expect(readNumCtxHint("not-an-object", "claude")).toBeUndefined(); -}); - -test("readNumCtxHint: a top-level numCtx pin is visible without the Ollama bag shape", () => { - expect(readNumCtxHint({ numCtx: 200_000 }, "claude-sonnet-5")).toBe(200_000); -}); - -test("resolveContextBudgetChars: a bigger numCtx yields a bigger budget", () => { - const small = resolveContextBudgetChars({ default: { numCtx: 32_000 } }, "qwen3"); - const large = resolveContextBudgetChars({ default: { numCtx: 128_000 } }, "gpt-oss:20b"); - - expect(large).toBeGreaterThan(small); -}); - -test("resolveContextWindowTokens: quirks pin wins over the advertised catalog window", () => { - expect(resolveContextWindowTokens({ default: { numCtx: 8_192 } }, "claude-sonnet-5")).toBe(8_192); -}); - -test("resolveContextWindowTokens: a frontier catalog model gets its advertised window, not a 32k-char cap", () => { - const claudeTokens = resolveContextWindowTokens(undefined, "claude-sonnet-5"); - const claudeHard = resolveHardContextLimitChars(undefined, "claude-sonnet-5"); - const oldDefaultHardChars = 8_000 * 4; - - expect(claudeTokens).toBe(200_000); - expect(claudeHard).toBeGreaterThan(oldDefaultHardChars); - expect(claudeHard).toBe(200_000 * 4); -}); - -test("resolveContextWindowTokens: an Ollama catalog model gets its native window without quirks", () => { - expect(resolveContextWindowTokens(undefined, "gpt-oss:20b")).toBe(131_072); - expect(resolveContextWindowTokens(undefined, "qwen3.8:27b")).toBe(32_768); - expect(resolveHardContextLimitChars(undefined, "gpt-oss:20b")).toBeGreaterThan( - resolveHardContextLimitChars(undefined, "qwen3.8:27b"), - ); -}); - -test("advertisedContextWindowTokens: a relay-prefixed name still matches the catalog model", () => { - expect(advertisedContextWindowTokens("anthropic/claude-sonnet-5")).toBe(200_000); - expect(advertisedContextWindowTokens("openai/gpt-4.1")).toBe(1_047_576); -}); - -test("resolveContextBudgetChars: unknown model falls back to a hosted-sized window, not 8k tokens", () => { - const withoutQuirks = resolveContextBudgetChars(undefined, "unknown-model"); - const hard = resolveHardContextLimitChars(undefined, "unknown-model"); - - expect(withoutQuirks).toBeGreaterThan(0); - expect(Number.isFinite(withoutQuirks)).toBe(true); - expect(hard).toBeGreaterThan(8_000 * 4); - expect(hard).toBe(128_000 * 4); -}); - -test("resolveHardContextLimitChars: sits above the headroomed budget for the same source", () => { - const quirks = { default: { numCtx: 32_000 } }; - - const budget = resolveContextBudgetChars(quirks, "m"); - const hardLimit = resolveHardContextLimitChars(quirks, "m"); - - expect(hardLimit).toBeGreaterThan(budget); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/context-budget.ts b/apps/sidecar/src/workflow-substrate-factory/context-budget.ts deleted file mode 100644 index 72b8dd121a..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/context-budget.ts +++ /dev/null @@ -1,171 +0,0 @@ -// Per-model context-window budget for the workbench director's -// compaction gate. -// -// Window size is the resolved model's advertised native context, not a -// baked 8k-token (32k-char) cap: -// -// 1. `InferenceSource.quirks` -- the same `numCtx` override shape -// `@corbits/ollama-adapter` resolves (`OllamaAdapterConfig`'s -// `default`/`perModel` bag), plus a few other common token-window -// field names operators actually pin. -// 2. The pinned catalog's advertised native window for that model -// name (Ollama table entries and hosted-family prefixes). -// 3. A modern hosted-model fallback (128k tokens), used only when -// neither quirks nor the catalog name the model. Compact (see -// `COMPACTION_HEADROOM`) remains the safety net inside that -// window; `CONTEXT_OVERFLOW_MESSAGE` is reserved for the hard -// edge. -// -// `quirks` is `unknown` at this boundary (an operator-authored, -// per-source passthrough bag), so every read here is defensive rather -// than a validating parse. -// -// Token counts are estimated from character counts -// (`CHARS_PER_TOKEN_ESTIMATE`) -- no tokenizer is available at this -// layer -- so the budget is deliberately conservative (see -// `COMPACTION_HEADROOM`), leaving room for the system prompt, tool -// definitions, and the model's own reply inside the same window. - -const FALLBACK_CONTEXT_WINDOW_TOKENS = 128_000; -const CHARS_PER_TOKEN_ESTIMATE = 4; -const COMPACTION_HEADROOM = 0.6; - -// Exact names from `@corbits/inference-catalog`'s Ollama native-window -// table, then longest-prefix families for hosted catalog models. -// Longest prefix wins so `gpt-4.1` is not swallowed by `gpt-4`. -const ADVERTISED_CONTEXT_WINDOWS: readonly (readonly [string, number])[] = [ - ["qwen3.5:9b-mlx", 32_768], - ["qwen3.8:27b", 32_768], - ["llama3.1:8b", 131_072], - ["gpt-oss:20b", 131_072], - ["gpt-4-turbo", 128_000], - ["gpt-4.1", 1_047_576], - ["llama3.1", 131_072], - ["gpt-oss", 131_072], - ["gpt-4o", 128_000], - ["claude", 200_000], - ["gemini", 1_048_576], - ["gpt-5", 400_000], - ["gpt-4", 8_192], - ["grok", 256_000], - ["qwen3", 32_768], - ["llama", 131_072], - ["kimi", 128_000], - ["deepseek", 128_000], - ["minimax", 128_000], - ["glm", 128_000], - ["o1", 200_000], - ["o3", 200_000], - ["o4", 200_000], -]; - -function positiveTokenCount(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; -} - -function readWindowFromBag(bag: Record): number | undefined { - return ( - positiveTokenCount(bag.numCtx) ?? - positiveTokenCount(bag.contextWindow) ?? - positiveTokenCount(bag.contextLength) ?? - positiveTokenCount(bag.maxContextTokens) - ); -} - -function catalogModelKey(model: string): string { - const trimmed = model.trim().toLowerCase(); - const slash = trimmed.lastIndexOf("/"); - return slash === -1 ? trimmed : trimmed.slice(slash + 1); -} - -/** - * Advertised native context window in tokens for a catalog model name. - * Exact Ollama table entries win; otherwise the longest matching hosted - * family prefix. `undefined` if this module has no published window for - * the name. - */ -export function advertisedContextWindowTokens(model: string): number | undefined { - const key = catalogModelKey(model); - if (key.length === 0) { - return undefined; - } - let best: { prefixLength: number; tokens: number } | undefined; - for (const [prefix, tokens] of ADVERTISED_CONTEXT_WINDOWS) { - if (key === prefix) { - return tokens; - } - if (key.startsWith(prefix) && (best === undefined || prefix.length > best.prefixLength)) { - best = { prefixLength: prefix.length, tokens }; - } - } - return best?.tokens; -} - -/** - * Best-effort read of a per-model window hint off an - * `InferenceSource.quirks` bag. Understands `@corbits/ollama-adapter`'s - * `OllamaAdapterConfig` (`{ default?: { numCtx? }, perModel?: { [model]: - * { numCtx? } } }`) and a few other common token-window field names. - * Returns `undefined` for any unrecognized shape rather than throwing. - */ -export function readNumCtxHint(quirks: unknown, model: string): number | undefined { - if (typeof quirks !== "object" || quirks === null) { - return undefined; - } - const bag = quirks as Record; - const perModel = bag.perModel; - if (typeof perModel === "object" && perModel !== null) { - const entry = (perModel as Record)[model]; - if (typeof entry === "object" && entry !== null) { - const fromPerModel = readWindowFromBag(entry as Record); - if (fromPerModel !== undefined) { - return fromPerModel; - } - } - } - const base = bag.default; - if (typeof base === "object" && base !== null) { - const fromDefault = readWindowFromBag(base as Record); - if (fromDefault !== undefined) { - return fromDefault; - } - } - return readWindowFromBag(bag); -} - -/** - * Resolved context window in tokens for one `InferenceSource`: quirks - * pin, then the advertised catalog window, then the hosted-model - * fallback. Compact headroom is applied by the char-budget helpers, - * not here. - */ -export function resolveContextWindowTokens(quirks: unknown, model: string): number { - return ( - readNumCtxHint(quirks, model) ?? - advertisedContextWindowTokens(model) ?? - FALLBACK_CONTEXT_WINDOW_TOKENS - ); -} - -function toChars(tokens: number, headroom: number): number { - return Math.floor(tokens * CHARS_PER_TOKEN_ESTIMATE * headroom); -} - -/** - * Resolve the compaction budget, in characters, for one `InferenceSource`. - * Applies `COMPACTION_HEADROOM` on top of the resolved window so - * compaction fires with room left, not at the hard edge. - */ -export function resolveContextBudgetChars(quirks: unknown, model: string): number { - return toChars(resolveContextWindowTokens(quirks, model), COMPACTION_HEADROOM); -} - -/** - * The raw (no-headroom) character budget for one source -- the point - * past which content can no longer be assumed to fit the model's window - * at all, used to detect the unrecoverable case where even the turns a - * compactor must keep verbatim are already too large. - */ -export function resolveHardContextLimitChars(quirks: unknown, model: string): number { - return toChars(resolveContextWindowTokens(quirks, model), 1); -} diff --git a/apps/sidecar/src/workflow-substrate-factory/index.ts b/apps/sidecar/src/workflow-substrate-factory/index.ts deleted file mode 100644 index 271a6ed3cd..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/index.ts +++ /dev/null @@ -1,759 +0,0 @@ -// Substrate factory the sidecar's `bin/workflow-child` hands to -// `runWorkflowChildFromProcessEnv`. The factory closes over the -// production substrate (`createAgentRepoStore`-backed `RepoStore`), -// the host-process scheduler singleton (adapted to the runtime's -// `Scheduler` shape), and the sidecar's grant-rule evaluator. -// -// The factory consumes the workflow-host's typed `SubstrateFactoryEnv` -// -- the parsed `SpawnTimeEnv` plus a narrow `substrateConfig` -// record carrying only the keys the binary listed in -// `RunWorkflowChildFromProcessEnvOpts.substrateConfigKeys`. The -// factory does not read `process.env` itself; the binary owns the -// only crossing of that boundary. -// -// Single-writer architecture: the workflow-run repo's ref has exactly -// one writer at a time -- the supervisor. The child opens a bare -// `createAgentRepoStore` against the shared on-disk data dir for -// read-only operations (`getRepoDir`, `subscribe`, `resolveRef`, -// etc.) and exposes a proxy `RepoStore` whose -// `writeTreePreservingPrefix` forwards every write over the control -// IPC into the supervisor's substrate. The supervisor's substrate is -// wrapped with the boot-edge pack-push facade, so the hub push fires -// as part of the supervisor's normal write path -- the child does -// not open its own pack-push pipeline. - -import fs from "node:fs"; - -import { type } from "arktype"; - -import { evaluateGrants } from "@intx/authz"; -import type { GrantRule } from "@intx/authz"; -import { - builtinCredentialProviders, - createCredentialProviderRegistry, - driveConnectorReplies, - type AgentEventStream, - type ConnectorReplyDrain, -} from "@intx/harness"; -import { - createHttpRawAuthorizationCredentialProvider, - createHttpXApiKeyCredentialProvider, - createHttpXManusApiKeyCredentialProvider, - createMcpStreamableHttpCredentialProvider, -} from "@corbits/credential-providers"; -import { loadAdapterRegistry } from "@intx/inference/providers"; -import { createSSHSignature } from "@intx/crypto"; -import { - createAgentRepoStore, - type RepoStore, - type WorkflowRunWorkflowProcessPrincipal, -} from "@intx/hub-sessions/substrate"; -import { - adaptHostScheduler, - createProxyWorkflowRunRepoStore, - createWorkflowHostScheduler, - createWorkflowStepInvoker, - type GrantEvaluator, - type LoadParkedApproval, - type RunWorkflowChildBindings, - type SubstrateFactory, - type SubstrateFactoryEnv, - createChildMailboxReader, - createMailboxWatchRegistry, - type SupervisorBackedTransportInbound, -} from "@intx/workflow-host"; -import { type ReadParkedApprovalOps, type StepInvoker } from "@intx/workflow"; -import type { InboundMessage } from "@intx/types/runtime"; - -import { createToolBearingAgentFactory, type StepToolCacheConfig } from "../step-agent-tools"; -import { - createDurableConversationRegistry, - prepareConversationForOriginatingWorkbench, - type DurableConversationRegistry, -} from "../conversation-state"; -import { originatingWorkbenchIdFromRequest } from "../originating-workbench"; -import { parseToolRegistries } from "../tool-materialization"; -import { - deriveHubHttpUrl, - parseAdapterManifest, - parseByteCap, - parseStepInferenceSources, - SIDECAR_SUBSTRATE_CONFIG_KEYS, - SubstrateConfig, -} from "./config"; -import { runStepStorageRoot } from "./storage-paths"; -import { - readColdParkedApprovalSnapshot, - readColdParkedPendingOperations, - readWarmParkedApprovalSnapshot, - readWarmParkedPendingOperations, - toParkedApprovalOps, -} from "./parked-approvals"; -import { createSidecarStepBuildEnv } from "./step-env"; - -export { createSidecarStepBuildEnv }; -export { type SidecarStepBuildEnvDeps } from "./step-env"; -import { - createSidecarRunChild, - createSidecarSpawnSuspendableChild, - type SidecarBodyStepInvoker, - type SidecarRunChildDeps, -} from "./child-runtime"; - -export { createSidecarRunChild, createSidecarSpawnSuspendableChild, type SidecarBodyStepInvoker }; - -export { parseAdapterManifest, SIDECAR_SUBSTRATE_CONFIG_KEYS }; - -// The child does not construct a workflow-run pack-push pipeline of -// its own. The supervisor owns the workflow-run repo's write -// contract; the supervisor's substrate is wrapped at the sidecar's -// boot edge with the pack-pushing facade so any successful workflow- -// run write fires the hub push automatically. The child's proxy -// `RepoStore` forwards `writeTreePreservingPrefix` over IPC into the -// supervisor's wrapped substrate. - -/** - * Thrown by the child-runtime step invoker when a `childWorkflow` (or a - * `map` nested inside one) reaches a per-step agent invocation. Real - * per-step child execution -- threading the child - * `WorkflowDefinition`-derived inference sources, tools, and grants into - * a real agent, backed by deploy-side child asset staging and capability - * approval -- is not yet built. - * - * Failing here is deliberate. Returning a fabricated success output would - * report a child run `completed` whose agent never ran, a silent - * correctness trap; a loud, structured failure is the honest behavior for - * an unbuilt seam. - */ -export class ChildStepNotImplementedError extends Error { - constructor(agentId: string, stepId: string | undefined) { - super( - `childWorkflow per-step execution is not implemented; ` + - `the child runtime cannot run a real per-step agent for step ${JSON.stringify(stepId)} (agent ${JSON.stringify(agentId)})`, - ); - this.name = "ChildStepNotImplementedError"; - } -} - -function hexDecode(hex: string, name: string): Uint8Array { - if (hex.length % 2 !== 0) { - throw new Error(`${name} must be even-length hex; got ${String(hex.length)} chars`); - } - const out = new Uint8Array(hex.length / 2); - for (let i = 0; i < out.length; i += 1) { - const byte = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); - if (Number.isNaN(byte)) { - throw new Error(`${name} contains non-hex characters`); - } - out[i] = byte; - } - return out; -} - -/** - * Dependency overrides accepted by `createSidecarSubstrateFactory`. - * Production callers omit these to get the default-disk-backed bare - * store and the IPC-bridge-backed substrate proxy; tests inject an - * in-memory bare store and/or an explicit substrate-write bridge. - */ -interface SidecarSubstrateFactoryDeps { - /** - * Override the bare-store constructor. Production callers omit this - * to get the `createAgentRepoStore`-backed `RepoStore` against - * `SIDECAR_DATA_DIR`; tests inject an in-memory recording stub. - * - * The bare store backs the child's read-only operations - * (`getRepoDir`, `subscribe`, `resolveRef`, `listRefs`, - * `resolveHead`, `createPack`). The child's workflow-run writes do - * NOT flow through this store; the proxy `RepoStore` forwards them - * over IPC into the supervisor's substrate. - */ - createBareRepoStore?: (config: { - dataDir: string; - signingKey: { publicKey: Uint8Array; privateKey: Uint8Array }; - }) => RepoStore; -} - -/** - * `CommitSigner` the per-step isogit stores use to sign every commit. - * The factory's Ed25519 signing keypair (the same key the child's bare - * `RepoStore` carries) is bound into an `sshsig`-shaped signer so the - * per-step agent-state commits are attributable to the sidecar's - * substrate identity, matching the signing surface the production - * `RepoStore` uses for workflow-run writes. - */ -function createStepStorageSigner(signingKey: { - publicKey: Uint8Array; - privateKey: Uint8Array; -}): (payload: string) => Promise { - return (payload: string) => - Promise.resolve(createSSHSignature(payload, signingKey.privateKey, signingKey.publicKey)); -} - -/** - * Build a `SubstrateFactory` closed over the supplied dependency - * overrides. The production export `createSubstrate` is the - * default-deps call. - * - * Construction order: - * 1. Narrow the `substrateConfig` record against the typed schema. - * A missing or empty key already threw inside the helper; this - * pass enforces the exact shape the factory consumes. - * 2. Open a bare `RepoStore` via `createAgentRepoStore` against the - * sidecar's data dir and Ed25519 keypair. This store backs the - * child's read-only operations against the workflow-run repo; - * the on-disk repo is shared with the supervisor's substrate so - * reads see whatever the supervisor has committed. - * 3. Construct a proxy `RepoStore` whose - * `writeTreePreservingPrefix` forwards over the upstream control - * channel via the substrate-write bridge. The supervisor's - * handler runs its own substrate's `writeTreePreservingPrefix` - * (wrapped at the boot edge with the pack-push facade) under the - * per-repo lock and replies with the resulting `commitSha`. - * 4. Start the host-process scheduler singleton against the proxy - * substrate, then adapt it to the runtime's `Scheduler` shape. - * 5. Construct the production `invokeStep` and `spawnChild` - * adapters. - * 6. Return the `RunWorkflowChildBindings` the runtime body - * consumes, with the proxy store in the `substrate` slot. - */ -export function createSidecarSubstrateFactory( - deps: SidecarSubstrateFactoryDeps = {}, -): SubstrateFactory { - const createBareRepoStore = - deps.createBareRepoStore ?? - (({ dataDir, signingKey }) => createAgentRepoStore({ dataDir, signingKey }).repoStore); - - return async (env: SubstrateFactoryEnv) => { - const validated = SubstrateConfig(env.substrateConfig); - if (validated instanceof type.errors) { - throw new Error( - `sidecar workflow-child substrate config failed validation: ${validated.summary}`, - ); - } - - const stepInferenceSources = parseStepInferenceSources(validated.STEP_INFERENCE_SOURCES); - - // Build the child's adapter registry eagerly at boot from the - // operator-supplied manifest. `loadAdapterRegistry` imports every - // custom module now, so a bad specifier crashes the child loudly at - // construction rather than silently degrading to built-ins-only at - // first resolve. The closure registry the sidecar built at its own - // boot edge cannot cross the fork; the child rebuilds an equivalent - // one from the serialized-and-revalidated manifest. - const childAdapterRegistry = await loadAdapterRegistry( - parseAdapterManifest(validated.SIDECAR_ADAPTER_MANIFEST), - ); - - const signingKey = { - publicKey: hexDecode(validated.SIDECAR_SIGNING_PUBLIC_KEY, "SIDECAR_SIGNING_PUBLIC_KEY"), - privateKey: hexDecode(validated.SIDECAR_SIGNING_PRIVATE_KEY, "SIDECAR_SIGNING_PRIVATE_KEY"), - }; - - const bareStore: RepoStore = createBareRepoStore({ - dataDir: validated.SIDECAR_DATA_DIR, - signingKey, - }); - - const workflowRunRepoId = { - kind: "workflow-run" as const, - id: validated.WORKFLOW_RUN_REPO_ID, - }; - const principal: WorkflowRunWorkflowProcessPrincipal = { - kind: "workflow-process", - anchorRunId: env.spawn.anchorRunId, - }; - - // Proxy substrate: writes are forwarded over IPC into the - // supervisor's substrate; reads consult the bare on-disk store. - // The supervisor is the sole writer of the workflow-run ref so - // the child's writes never race against the supervisor's - // claim-check writes (inbox / processing / consumed). - const substrate: RepoStore = createProxyWorkflowRunRepoStore({ - bareStore, - bridge: env.substrateWriteBridge, - workflowRunRepoId, - }); - - const hostScheduler = createWorkflowHostScheduler({ - repoStore: substrate, - principal, - listActiveDeployments: () => [workflowRunRepoId], - ref: validated.WORKFLOW_RUN_REF, - clock: () => new Date(), - }); - await hostScheduler.start(); - const scheduler = adaptHostScheduler(hostScheduler); - - // The single-step / top-level path runs a real agent. The per-step - // env builder stands up real per-step storage/workdir/audit/directors - // rooted under the run (see `createSidecarStepBuildEnv`), resolving - // the per-step `InferenceSource` from the pinned table; the real - // step-invoker instantiates the step's agent via `createAgent`, - // delivers the resolved input as a synthesized inbound message, and - // captures the agent's reply as the step output. - const stepToolCache: StepToolCacheConfig = { - cacheMaxBytes: parseByteCap(validated.SIDECAR_CACHE_MAX_BYTES, "SIDECAR_CACHE_MAX_BYTES"), - registryMaxTarballBytes: parseByteCap( - validated.SIDECAR_REGISTRY_MAX_TARBALL_BYTES, - "SIDECAR_REGISTRY_MAX_TARBALL_BYTES", - ), - }; - - // The single-step / top-level path runs a real agent with REAL - // tools materialized in-child. The per-step env builder stands up - // real per-step storage/workdir/audit/directors rooted under the - // run (see `createSidecarStepBuildEnv`), resolves the per-step - // `InferenceSource`, and materializes the step's pinned - // tool-package closure (posix, LSP, mail, ...) from its on-disk - // deploy tree -- rooted per step so concurrent steps in one child - // never collide on the tarball cache or apply-state. The - // tool-bearing `agentFactory` below attaches those factories to the - // step's `AgentDefinition` and builds the plugin chain. - // Durable-conversation registry for the warm single-step agent. - // Built only when the deployment is warm-kept: the - // sole long-lived agent's conversation must survive child respawn, - // so it is mirrored to the workflow-run substrate at a per-agent - // path. A multi-step deploy leaves this `undefined` -- its per-step - // agents are not warm/long-lived, so they carry no cross-run - // conversation and keep the per-run isogit store. The registry lives - // for the child's lifetime; on respawn the child rebuilds it empty - // and each store restores its prior snapshot from the substrate on - // first acquire. - const conversationSigner = createStepStorageSigner(signingKey); - const durableConversation: DurableConversationRegistry | undefined = env.spawn.warmKeep - ? createDurableConversationRegistry({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - substrate, - principal, - signer: conversationSigner, - }) - : undefined; - - // INBOUND half of mailbox ownership. One watch registry per child, - // shared by the step agent's supervisor-backed transport (its `watch` - // registers callbacks here, backing `mail_wait`) and the child's control - // loop (which fires each `mailbox.notify` into it); it rides out on the - // bindings so `runWorkflowChild` routes notifications to this instance. - // The reader opens a fresh committed snapshot of the deployment's - // substrate `INBOX` per read, so a read after a `mailbox.notify` sees the - // message the supervisor just committed. Every step env -- top-level or - // onTrigger body -- gets the surface, since the fork's bodies are - // tool-bearing agents. - const mailboxWatchRegistry = createMailboxWatchRegistry(); - const transportInbound: SupervisorBackedTransportInbound = { - reader: createChildMailboxReader({ - substrate, - repoId: workflowRunRepoId, - principal, - ref: validated.WORKFLOW_RUN_REF, - }), - watchRegistry: mailboxWatchRegistry, - // The child holds no sender-key registry, so `fetchFull` reports every - // message's signature status as "unknown". - getCrypto: () => undefined, - // Flag/expunge writes ride to the supervisor, the sole mailbox writer. - mutationBridge: env.mailboxMutationBridge, - }; - - const buildStepEnvBaseOpts = { - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - signer: conversationSigner, - registries: parseToolRegistries(validated.SIDECAR_TOOL_REGISTRIES), - mailboxAddress: env.spawn.mailboxAddress, - stepCount: env.spawn.stepCount, - outboundMailBridge: env.outboundMailBridge, - inbound: transportInbound, - cache: stepToolCache, - adapters: childAdapterRegistry, - hubArtifactsUrl: deriveHubHttpUrl(validated.HUB_WS_URL), - sidecarToken: validated.SIDECAR_TOKEN, - definitionId: validated.WORKFLOW_DEFINITION_ID, - tenantId: validated.WORKFLOW_TENANT_ID, - principalId: validated.WORKFLOW_PRINCIPAL_ID, - }; - const buildStepEnv = createSidecarStepBuildEnv( - durableConversation !== undefined - ? { ...buildStepEnvBaseOpts, durableConversation } - : buildStepEnvBaseOpts, - ); - - // Credential provider registry: the platform's built-in `http` - // (Bearer) provider (`@intx/harness`'s `builtinCredentialProviders`) - // plus this workbench's own `http-raw-authorization` plugin (for a - // provider row whose API expects the raw secret in `authorization` - // with no `Bearer ` prefix -- Linear's convention) and - // `http-x-api-key` plugin (for a provider row whose API expects the - // secret in an `x-api-key` header -- Exa's and ScrapeCreators' - // convention) and `http-x-manus-api-key` plugin (for Manus, which - // expects `x-manus-api-key`), all from `@corbits/credential-providers` - // rather than forking or reaching around the vendored plugin. Fixed - // for the child's lifetime -- unlike the per-step wiring below, the - // set of provider plugins is not something a rotation or - // `credentials-updated` frame changes. - const credentialProviders = createCredentialProviderRegistry([ - ...builtinCredentialProviders(), - createHttpRawAuthorizationCredentialProvider(), - createHttpXApiKeyCredentialProvider(), - createHttpXManusApiKeyCredentialProvider(), - createMcpStreamableHttpCredentialProvider(), - ]); - - // The tool-bearing agent factory reads the materialized tool - // runtime off the per-step env (set by `buildStepEnv` via - // `attachStepTools`), attaches the loaded tool factories to the - // step's `AgentDefinition`, builds the plugin chain on - // `env.plugins`, and wraps `agent.close()` so every plugin (the LSP - // subprocess included) and tool bundle is torn down with the agent - // on every exit path. It also shapes a consumer-scoped `credentials` - // capability for any tool package that declares one, reading the - // per-step `CredentialWiring` `buildStepEnv` attached to the env - // (see `attachStepCredentials`). The factory is stateless across - // steps, so it is pinned once here and shared by every per-step - // invoker built below. - const stepAgentFactory = createToolBearingAgentFactory({ - providers: credentialProviders, - }); - - // Child-runtime step invoker. The in-process `runChild` (see - // `createSidecarRunChild` below) runs a separate WorkflowDefinition - // whose stepIds are disjoint from the parent's, and deploy does not - // stage the child definition's per-step assets (inference sources, - // tool trees) or walk its capabilities. Running a real per-step agent - // for a `childWorkflow` / `map` fan-out step is therefore not - // implemented yet. The `authorize` - // argument -- the child's credentials-backed authorize -- is unused - // here for the same reason: no agent runs to gate. - // - // This is a deliberate hard stop, not a fabricated result. A fake - // success output (the shape this once returned) reported a child run - // `completed` whose agent never ran -- a silent correctness trap. - // Failing loudly surfaces the child step as `StepFailed` with a - // structured error instead. The `spawnChild` / - // `runChild` recursion and the sub-namespace scoping around it are - // real and exercised right up to this seam. - const childInvokeStep: StepInvoker = (req) => - Promise.reject(new ChildStepNotImplementedError(req.agent.id, req.authzContext.stepId)); - - // onTrigger BODY step invoker (CL-6448). Unlike a childWorkflow child, - // an onTrigger section body IS staged: its definition and per-step - // inference sources land on disk beside each other at deploy. A body - // agent step runs for real through the same `createWorkflowStepInvoker` - // and the same `buildStepEnv` the top level uses -- so a warm-kept - // section deployment's body turns share the per-agent durable - // conversation store (each turn's agent loads every prior turn, keyed - // by the body's stable stepId across `turn__` occurrences) and - // materialize the deployment's staged tool manifest (the head/step - // collapse reads the folded launch's own staged pins for a single-step - // deployment). The agent itself stays cold per occurrence; the mirror - // in the `finally` below is the body path's run-boundary durability - // flush, matching the warm top-level path's `onRunBoundary`. The - // per-body `sourcesRef` is threaded in per spawn, disjoint from the - // top level's. `onEvent` is the per-run event funnel from the parent - // run's event channel, so a body agent's live inference events reach - // the hub stream (per-run attribution stays durable via - // runs//events/). - const bodyInvokeStep: SidecarBodyStepInvoker = async ( - req, - authorize, - sourcesRef, - onEvent, - credentialWiring, - mailPartReader, - ) => { - const bodyStepId = req.authzContext.stepId; - if (durableConversation !== undefined && bodyStepId !== undefined) { - await prepareConversationForOriginatingWorkbench({ - registry: durableConversation, - agentKey: bodyStepId, - originatingWorkbenchId: originatingWorkbenchIdFromRequest(req), - }); - } - try { - return await createWorkflowStepInvoker({ - workflowAuthorize: authorize, - buildEnv: (buildReq) => buildStepEnv(buildReq, sourcesRef, credentialWiring), - agentFactory: stepAgentFactory, - sourcesRef, - onEvent, - ...(mailPartReader !== undefined ? { mailPartReader } : {}), - })(req); - } finally { - if (durableConversation !== undefined && bodyStepId !== undefined) { - // `peek`, not `get`: a build failure before the env's acquire - // must surface as itself, not as the registry's missing-store - // throw. Flushes the currently bound workbench prefix. - await durableConversation.peek(bodyStepId)?.mirrorToSubstrate(); - } - } - }; - - // Adapt the workflow-runtime `StepInvoker` shape onto the host's - // `ChildStepInvoker` shape. The host's `onEvent` is the child's - // per-run event-channel sink: the runtime body passes it per step, - // and the chain from here is `onEvent -> child event-channel sender - // -> supervisor -> publishWorkflowInferenceEvent -> hub timeline`. - // - // The `authorize` argument is the child's credentials-backed - // authorize closure (`createCredentialsBackedAuthorize`), threaded - // in from `run-child.ts`'s runtime env. The step agent's runtime - // gates EVERY tool call through `env.authorize` with - // `resource = tool:`, `action = "invoke"` (the inference - // layer's authz before-tool extension); using the credentials-backed - // authorize here means each tool call resolves against the per-step - // grant snapshot the supervisor assembled from the agent's - // `state/grants.json` and pushed over the control IPC. A tool the - // agent's grants do not allow is blocked; a granted tool runs. The - // operator gate at deploy time (the capability walk's `tool:` - // approval) and this runtime grant check are complementary: the walk - // bounds the toolset the deploy may carry, the grant snapshot decides - // which of those the agent may invoke at run time. - // - // A fresh `createWorkflowStepInvoker` is built per invocation so the - // adapter subscribes the step agent's event stream to THIS step's - // `onEvent`. The per-step env builder and the tool-bearing agent - // factory are pinned (closed over above); the event sink and the - // authorize closure vary per step. - // - // The `warmCache` is the run-loop's per-deployment - // warm-agent cache, present only for the single-step long-lived - // deployment the deploy projection marked a warm candidate. When - // supplied, the adapter builds the agent once and reuses it across - // messages; when absent, it keeps instantiate-send-teardown per - // step. Forwarding it here is the only warm-keep wiring this binding - // needs -- the adapter and the run-loop own the rest of the - // lifecycle. - // Run-boundary durability flush. When the deployment is - // warm-kept, mirror the warm agent's conversation snapshot to the - // currently bound originating-workbench prefix after each message's - // send settles. The registry key is still the step identity (one warm - // agent); the store flushes `agent-state///`. - // Absent for a multi-step deploy (no durable registry). - const onRunBoundary: ((key: string) => Promise) | undefined = - durableConversation !== undefined - ? async (key: string) => { - await durableConversation.get(key).mirrorToSubstrate(); - } - : undefined; - - // Connector-thread seed: when the deployment is warm-kept, route each - // mail-derived inbound message onto the warm agent's connector thread - // before its send, so the reply path has thread state. Keyed by the step - // identity the durable store is filed under. - const seedInbound = - durableConversation !== undefined - ? async (key: string, message: InboundMessage) => { - await durableConversation.get(key).seedInbound(message); - } - : undefined; - - // Connector reply drain: on each `connector.reply` the warm agent emits, - // compose a threaded reply from the durable store's connector thread, - // send it through the outbound bridge (the same signed-send path the - // agent's own transport uses) and advance the thread from the receipt. - // The References chain comes from the committed mailbox: the parent's - // own References plus its Message-Id; a parent miss (first reply on a - // fresh thread) leaves the transport to derive [inReplyTo]. - const driveReplies = - durableConversation !== undefined - ? (key: string, stream: AgentEventStream): ConnectorReplyDrain => - driveConnectorReplies({ - stream, - composeReply: () => durableConversation.get(key).composeReply(), - send: (message) => env.outboundMailBridge.submit(env.spawn.mailboxAddress, message), - resolveReferences: async (inReplyTo) => { - const store = await transportInbound.reader.open(); - const parent = store.messages.find((m) => m.envelope.messageId === inReplyTo); - return parent === undefined - ? undefined - : [...parent.envelope.references, parent.envelope.messageId]; - }, - onReplySent: (receipt) => durableConversation.get(key).onReplySent(receipt), - }) - : undefined; - - const invokeStep: RunWorkflowChildBindings["invokeStep"] = async ( - req, - onEvent, - authorize, - warmCache, - sourcesRef, - credentialWiring, - mailPartReader, - ) => { - const stepId = req.authzContext.stepId; - if (durableConversation !== undefined && stepId !== undefined) { - await prepareConversationForOriginatingWorkbench({ - registry: durableConversation, - agentKey: stepId, - originatingWorkbenchId: originatingWorkbenchIdFromRequest(req), - ...(warmCache !== undefined ? { warmCache } : {}), - }); - } - return createWorkflowStepInvoker({ - workflowAuthorize: authorize, - buildEnv: (buildReq: Parameters[0]) => - buildStepEnv(buildReq, sourcesRef, credentialWiring), - agentFactory: stepAgentFactory, - onEvent, - sourcesRef, - mailPartReader, - ...(warmCache !== undefined ? { warmCache } : {}), - ...(onRunBoundary !== undefined ? { onRunBoundary } : {}), - ...(seedInbound !== undefined ? { seedInbound } : {}), - ...(driveReplies !== undefined ? { driveReplies } : {}), - })(req); - }; - - const evaluateGrantsAdapter: GrantEvaluator = async ({ resource, action, grants }) => { - const result = await evaluateGrants( - // The credentialsSnapshot's grants are typed as - // `readonly unknown[]` so the workflow-host package does not - // depend on the sidecar's grant-rule grammar. The sidecar owns - // that grammar; the cast surfaces here at the boundary where - // the typed grant shape is known. - // Boundary type assertion: credentialsSnapshot.steps[*].grants is typed unknown[] at the workflow-host boundary; the sidecar owns the GrantRule grammar - [...(grants as readonly GrantRule[])], - resource, - action, - ); - return { - effect: result.effect, - matchingGrants: [], - resolvedBy: null, - }; - }; - - const childRunDeps: SidecarRunChildDeps = { - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - scheduler, - invokeStep: childInvokeStep, - // The onTrigger body path runs real agent steps; the childWorkflow - // path (and a body's childWorkflow grandchildren) stay on `invokeStep`. - bodyInvokeStep, - dataDir: validated.SIDECAR_DATA_DIR, - }; - // Terminal childWorkflow executor. `run-child` builds the in-memory - // resolver from this plus the lifted-body map it extracts after loading - // the parent's re-verified definition, so an owned inline child spawns - // with no on-disk asset read. - const runChild = createSidecarRunChild(childRunDeps); - - // An onTrigger section runs each event's body as a suspendable child. - // `run-child` builds the in-memory body resolver from this raw executor - // plus the lifted-body map it extracts after re-evaluating the parent's - // closure, so a body resolves in-process with no on-disk read and no - // separate per-body re-verify -- the parent's re-verify already covers - // every inline body. - const runSuspendableChild = createSidecarSpawnSuspendableChild(childRunDeps); - - // Per-run scratch reclamation for the cold (multi-step) path. The - // run-loop fires this once each run reaches its terminal status; it - // drops the run's whole `workflow-step-state//runs//` - // subtree (every step/attempt the run produced), which nothing - // reopens after terminal (resume reads the substrate run log, not - // local step state). Built only for the cold path: a warm deploy - // roots its single agent's scratch per agent under the disjoint - // `warm/` sub-root (reclaimed on undeploy), and the run-loop's own - // `warmKeep` gate already suppresses the per-run call there, so - // leaving this undefined for warm deploys keeps the path-owning - // module's intent explicit. `rm -rf` semantics via `recursive + - // force` so a run that never wrote scratch (no buildEnv reached) is - // a no-op rather than an ENOENT throw. - const cleanupRunStorage: ((runId: string) => Promise) | undefined = env.spawn.warmKeep - ? undefined - : (runId: string) => - fs.promises.rm( - runStepStorageRoot({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - }), - { recursive: true, force: true }, - ); - - // Recover a parked correlation's approval snapshot for the child's - // re-registration enumeration (ported from upstream Interchange's - // sidecar at the vendored pin). Wired unconditionally (unlike - // `cleanupRunStorage`, which is cold-only): a warm agent parks on - // approval just as a cold one does, and the branch on `warmKeep` - // selects the durable read — cold reads the per-attempt isogit - // store, warm reconstructs the agent's durable conversation state - // from the substrate. - const loadParkedApproval: LoadParkedApproval = ({ runId, stepId, attempt, correlationId }) => - env.spawn.warmKeep - ? readWarmParkedApprovalSnapshot({ - substrate, - workflowRunRepoId, - stepId, - correlationId, - }) - : readColdParkedApprovalSnapshot({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - stepId, - attempt, - correlationId, - }); - - // Enumerate a crashed step's durable pending approval operations for - // the resume classifier, off the same cold/warm durable read as - // `loadParkedApproval`. Where that binding is a lookup by a known - // correlationId (answering the supervisor's re-registration), this is - // the enumeration the classifier needs when the correlationId never - // reached the log — the crash-across-park case. - const readParkedApprovalOps: ReadParkedApprovalOps = async ({ runId, stepId, attempt }) => - toParkedApprovalOps( - env.spawn.warmKeep - ? await readWarmParkedPendingOperations({ - substrate, - workflowRunRepoId, - stepId, - }) - : await readColdParkedPendingOperations({ - dataDir: validated.SIDECAR_DATA_DIR, - workflowRunRepoId, - runId, - stepId, - attempt, - }), - ); - - const bindings: RunWorkflowChildBindings = { - substrate, - workflowRunRepoId, - workflowRunRef: validated.WORKFLOW_RUN_REF, - principal, - invokeStep, - initialSources: stepInferenceSources, - runChild, - runSuspendableChild, - scheduler, - evaluateGrants: evaluateGrantsAdapter, - loadParkedApproval, - readParkedApprovalOps, - mailboxWatchRegistry, - ...(cleanupRunStorage !== undefined ? { cleanupRunStorage } : {}), - }; - return bindings; - }; -} - -/** - * Production substrate factory. The sidecar's - * `bin/workflow-child` binary calls - * `runWorkflowChildFromProcessEnv(createSubstrate, { substrateConfigKeys: SIDECAR_SUBSTRATE_CONFIG_KEYS })` - * and the helper invokes this factory with the parsed env. The - * factory is the default-deps variant of - * `createSidecarSubstrateFactory`; deployments that need a recording - * hub sink (tests, alternate hosts) construct their own via - * `createSidecarSubstrateFactory`. - */ -export const createSubstrate: SubstrateFactory = createSidecarSubstrateFactory(); diff --git a/apps/sidecar/src/workflow-substrate-factory/parked-approvals.test.ts b/apps/sidecar/src/workflow-substrate-factory/parked-approvals.test.ts deleted file mode 100644 index 5a05604d1e..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/parked-approvals.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -// Regression tests for the parked-approval recovery bindings (ported from -// upstream Interchange's sidecar; see parked-approvals.ts). Before the -// port, a sidecar restart crashed every restored workflow-child whose run -// was parked on an ask-approval: the child's re-registration enumeration -// found the park but had no `loadParkedApproval` binding wired and threw, -// killing the deployment at boot. -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, test } from "bun:test"; -import type { RepoStore } from "@intx/hub-sessions"; -import type { PendingOperation } from "@intx/types/runtime"; - -import { - findApprovalSnapshot, - readColdParkedApprovalSnapshot, - readWarmParkedPendingOperations, - toParkedApprovalOps, -} from "./parked-approvals"; - -const tempDirs: string[] = []; -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -const REPO_ID = { kind: "workflow-run" as const, id: "dep-parked-1" }; - -function approvalOp(correlationId: string, timeoutAt?: number) { - return { - kind: "approval", - correlationId, - approvalSnapshot: { question: `q-${correlationId}` }, - ...(timeoutAt !== undefined ? { timeoutAt } : {}), - } as unknown as PendingOperation; -} - -describe("findApprovalSnapshot", () => { - test("returns the snapshot for the matching correlation and undefined otherwise", () => { - const ops = [approvalOp("corr-a"), approvalOp("corr-b")]; - expect(findApprovalSnapshot(ops, "corr-b")).toEqual({ - question: "q-corr-b", - } as never); - expect(findApprovalSnapshot(ops, "corr-missing")).toBeUndefined(); - }); -}); - -describe("toParkedApprovalOps", () => { - test("keeps only approval ops, projecting correlationId and deadline", () => { - const ops = [ - approvalOp("corr-a", 1234), - approvalOp("corr-b"), - { kind: "input", correlationId: "corr-c" } as unknown as PendingOperation, - ]; - expect(toParkedApprovalOps(ops)).toEqual([ - { correlationId: "corr-a", timeoutAtMs: 1234 }, - { correlationId: "corr-b" }, - ]); - }); -}); - -describe("readColdParkedApprovalSnapshot", () => { - test("an absent step store reads as no snapshot, never an init or a throw", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "parked-cold-")); - tempDirs.push(dataDir); - const snapshot = await readColdParkedApprovalSnapshot({ - dataDir, - workflowRunRepoId: REPO_ID, - runId: "run-x", - stepId: "default", - attempt: 1, - correlationId: "corr-never", - }); - expect(snapshot).toBeUndefined(); - }); -}); - -const EMPTY_TOKEN_USAGE = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - thinking: 0, -}; - -async function writeRoomCheckpoint( - repoDir: string, - stepId: string, - workbenchId: string, - pendingOperations: unknown[], -): Promise { - const dir = path.join(repoDir, "agent-state", stepId, workbenchId); - await mkdir(dir, { recursive: true }); - const snapshot = { - turns: [], - pendingOperations, - tokenUsage: EMPTY_TOKEN_USAGE, - connectorState: null, - }; - const meta = { - checkpointSeq: 1, - turnCount: 0, - pendingOperations, - tokenUsage: EMPTY_TOKEN_USAGE, - connectorState: null, - }; - await writeFile(path.join(dir, "checkpoint.json"), JSON.stringify(snapshot)); - await writeFile(path.join(dir, "checkpoint.meta.json"), JSON.stringify(meta)); -} - -describe("readWarmParkedPendingOperations", () => { - test("collects pending ops from nested rooms and ignores a mixed agent-state blob", async () => { - const repoDir = await mkdtemp(path.join(tmpdir(), "parked-warm-")); - tempDirs.push(repoDir); - await writeRoomCheckpoint(repoDir, "default", "chan_a", [approvalOp("corr-a")]); - await writeRoomCheckpoint(repoDir, "default", "chan_b", [approvalOp("corr-b")]); - await writeFile( - path.join(repoDir, "agent-state", "default", "checkpoint.json"), - JSON.stringify({ pendingOperations: [approvalOp("corr-legacy")] }), - ); - - const substrate = { - getRepoDir: () => repoDir, - } as unknown as RepoStore; - - const pending = await readWarmParkedPendingOperations({ - substrate, - workflowRunRepoId: REPO_ID, - stepId: "default", - }); - const ids = pending.map((op) => op.correlationId).sort(); - expect(ids).toEqual(["corr-a", "corr-b"]); - }); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/parked-approvals.ts b/apps/sidecar/src/workflow-substrate-factory/parked-approvals.ts deleted file mode 100644 index 615cbd5ade..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/parked-approvals.ts +++ /dev/null @@ -1,157 +0,0 @@ -// Parked-approval recovery reads, ported from upstream Interchange's -// sidecar (apps/sidecar/src/workflow-substrate-factory.ts) — the -// durable-storage half of the child's -// `loadParkedApproval` / `readParkedApprovalOps` bindings. The child owns -// enumeration (it walks its reduced run state for `awaiting-signal` steps -// on control-plane channels); the snapshot lives in per-step durable -// storage whose on-disk layout (cold per-attempt isogit store vs warm -// substrate-mirrored conversation state) this host owns, so the reads -// live here. Unwired, every sidecar restart crashed each restored -// workflow-child whose run was parked on an ask-approval. -import fs from "node:fs"; -import path from "node:path"; -import type { RepoStore } from "@intx/hub-sessions"; -import { WORKFLOW_RUN_AGENT_STATE_PREFIX } from "@intx/hub-sessions"; -import type { RepoId } from "@intx/hub-sessions"; -import { createIsogitStore } from "@intx/storage-isogit/node"; -import type { ApprovalSnapshot, PendingOperation } from "@intx/types/runtime"; -import type { ParkedApprovalOp } from "@intx/workflow"; - -import { isErrnoNotFound, reconstructDurableConversation } from "../conversation-state"; -import { stepStorageRoot } from "./storage-paths"; - -async function directoryExists(dir: string): Promise { - try { - return (await fs.promises.stat(dir)).isDirectory(); - } catch (cause) { - if (isErrnoNotFound(cause)) return false; - throw cause; - } -} - -export function findApprovalSnapshot( - pendingOperations: readonly PendingOperation[], - correlationId: string, -): ApprovalSnapshot | undefined { - return pendingOperations.find((op) => op.correlationId === correlationId)?.approvalSnapshot; -} - -/** - * Read a cold (multi-step) parked step's durable pending operations from - * its on-disk per-attempt isogit store. The store is written at suspend - * and survives while the run is non-terminal — a parked step keeps the - * run in-flight, so the run-completion reclamation (`cleanupRunStorage`) - * never fires against it. - * - * Returns an empty list when the store directory is absent rather than - * manufacturing an empty repo on the read path: `createIsogitStore` calls - * `initAgentRepo`, which would `mkdir` and init a fresh repo for a - * non-existent dir. The `directoryExists` guard keeps the read a read — - * on an existing store `initAgentRepo` finds a repo and commits nothing, - * so no signer is needed (`load()` never signs). - */ -export async function readColdParkedPendingOperations(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; -}): Promise { - const storeDir = stepStorageRoot({ - dataDir: args.dataDir, - workflowRunRepoId: args.workflowRunRepoId, - runId: args.runId, - stepId: args.stepId, - attempt: args.attempt, - }); - if (!(await directoryExists(storeDir))) return []; - const store = await createIsogitStore(storeDir); - const { pendingOperations } = await store.load(); - return pendingOperations; -} - -export async function readColdParkedApprovalSnapshot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; - correlationId: string; -}): Promise { - return findApprovalSnapshot(await readColdParkedPendingOperations(args), args.correlationId); -} - -/** - * Read a warm (single-step) parked agent's durable pending operations - * from substrate state. A warm agent's pending operations live in its - * durable conversation store, mirrored to the workflow-run substrate - * under `agent-state///`. - * - * Reconstructs that state read-only — deliberately NOT through - * `DurableConversationRegistry.acquire`, whose first acquire writes and - * commits a substrate restore into the live store and would front-run - * the warm agent's own restore ordering. A respawned child has not - * rebuilt the live store when re-registration runs (resume re-parks - * without re-invoking the step), so the substrate is the only place the - * pending operations live at that moment. Returns an empty list when no - * durable state exists for the agent. Walks every nested workbench dir - * so a park in any room is visible. - */ -export async function readWarmParkedPendingOperations(args: { - substrate: RepoStore; - workflowRunRepoId: RepoId; - stepId: string; -}): Promise { - const agentStateDir = path.join( - args.substrate.getRepoDir(args.workflowRunRepoId), - WORKFLOW_RUN_AGENT_STATE_PREFIX, - encodeURIComponent(args.stepId), - ); - let children: fs.Dirent[]; - try { - children = await fs.promises.readdir(agentStateDir, { - withFileTypes: true, - }); - } catch (cause) { - if (isErrnoNotFound(cause)) return []; - throw cause; - } - const pending: PendingOperation[] = []; - for (const child of children) { - if (!child.isDirectory()) continue; - const reconstructed = await reconstructDurableConversation( - path.join(agentStateDir, child.name), - `${args.stepId}/${child.name}`, - ); - if (reconstructed !== null) { - pending.push(...reconstructed.pendingOperations); - } - } - return pending; -} - -export async function readWarmParkedApprovalSnapshot(args: { - substrate: RepoStore; - workflowRunRepoId: RepoId; - stepId: string; - correlationId: string; -}): Promise { - return findApprovalSnapshot(await readWarmParkedPendingOperations(args), args.correlationId); -} - -/** - * Project a parked step's durable pending operations down to the minimal - * approval records the resume classifier needs. Filters to `approval` - * (the only control-plane kind today) and keeps only the correlationId - * and the optional epoch-ms deadline; the runtime reconstructs the lost - * `SignalAwaited` from those alone, and must not see the reactor's - * pending-operation internals. - */ -export function toParkedApprovalOps(pendingOperations: PendingOperation[]): ParkedApprovalOp[] { - return pendingOperations - .filter((op) => op.kind === "approval") - .map((op) => ({ - correlationId: op.correlationId, - ...(op.timeoutAt !== undefined ? { timeoutAtMs: op.timeoutAt } : {}), - })); -} diff --git a/apps/sidecar/src/workflow-substrate-factory/step-env.test.ts b/apps/sidecar/src/workflow-substrate-factory/step-env.test.ts deleted file mode 100644 index 7f8b65f836..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/step-env.test.ts +++ /dev/null @@ -1,190 +0,0 @@ -// Proves the CL-6086 binding: `createSidecarStepBuildEnv` threads the -// deploying definition's own `definitionId` (and the derived -// `hubCapabilitiesUrl`) onto the built step env, following exactly the -// binding pattern `hubSkillsUrl`/`sidecarToken`/`address` already use. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, expect, test } from "bun:test"; - -import type { AgentDefinition, BaseEnv } from "@intx/agent"; -import type { RepoId } from "@intx/hub-sessions/substrate"; -import type { StepInvokeRequest } from "@intx/workflow"; -import type { ChildOutboundMailBridge, SourcesSnapshotRef } from "@intx/workflow-host"; - -import { SUMMARIZE_BUDGETED_TURNS_NAME, SUMMARIZE_OLDER_TURNS_NAME } from "./compactors"; -import { createSidecarStepBuildEnv } from "./step-env"; - -const tmpDirs: string[] = []; - -afterEach(async () => { - await Promise.all( - tmpDirs.splice(0).map((dir) => fs.promises.rm(dir, { recursive: true, force: true })), - ); -}); - -function makeTmpDataDir(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "step-env-test-")); - tmpDirs.push(dir); - return dir; -} - -const WORKFLOW_RUN_REPO_ID: RepoId = { kind: "workflow-run", id: "wfr_1" }; - -function buildEnvDeps(dataDir: string) { - return { - dataDir, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - signer: () => Promise.resolve("fake-signature"), - registries: new Map(), - mailboxAddress: "run_1@example.com", - stepCount: 1, - outboundMailBridge: {} as ChildOutboundMailBridge, - cache: { cacheMaxBytes: 1_000_000, registryMaxTarballBytes: 1_000_000 }, - hubArtifactsUrl: "https://hub.example.com", - sidecarToken: "sc-token", - adapters: { resolve: () => undefined } as never, - definitionId: "wfd_capability_owner", - tenantId: "ten_test", - principalId: "prin_test", - }; -} - -function stepInvokeRequest(): StepInvokeRequest { - return { - agent: {} as AgentDefinition, - input: "hello", - authzContext: { stepId: "step_1", runId: "run_1", attempt: 1 }, - signal: new AbortController().signal, - }; -} - -test("the built step env carries the deploying definition's own definitionId and a derived hubCapabilitiesUrl", async () => { - const dataDir = makeTmpDataDir(); - const buildEnv = createSidecarStepBuildEnv(buildEnvDeps(dataDir)); - const sourcesRef: SourcesSnapshotRef = { - current: { - step_1: [{ id: "src_1", provider: "anthropic", model: "claude" }], - }, - } as unknown as SourcesSnapshotRef; - - const env = await buildEnv(stepInvokeRequest(), sourcesRef); - - expect((env as unknown as { definitionId: string }).definitionId).toBe("wfd_capability_owner"); - expect((env as unknown as { hubCapabilitiesUrl: string }).hubCapabilitiesUrl).toBe( - "https://hub.example.com", - ); - // Same hub origin as the sibling artifacts/skills bindings — one - // origin, one name per tool-bundle surface, never overloaded. - expect((env as unknown as { hubArtifactsUrl: string }).hubArtifactsUrl).toBe( - "https://hub.example.com", - ); - expect((env as unknown as { memoryBaseUrl: string }).memoryBaseUrl).toBe( - "https://hub.example.com", - ); - expect((env as unknown as { memoryTenantId: string }).memoryTenantId).toBe("ten_test"); - expect((env as unknown as { address: string }).address).toBe("run_1@example.com"); -}); - -test("a body-step env with no staged deploy tree still carries definitionId, so the binding is not tool-materialization-gated", async () => { - const dataDir = makeTmpDataDir(); - const buildEnv = createSidecarStepBuildEnv({ - ...buildEnvDeps(dataDir), - definitionId: "wfd_parent_definition", - tenantId: "ten_test", - principalId: "prin_test", - }); - const sourcesRef: SourcesSnapshotRef = { - current: { - step_1: [{ id: "src_1", provider: "anthropic", model: "claude" }], - }, - } as unknown as SourcesSnapshotRef; - - const env = await buildEnv(stepInvokeRequest(), sourcesRef); - expect((env as unknown as { definitionId: string }).definitionId).toBe("wfd_parent_definition"); -}); - -// CL-6448: the body-turn history seam. A section body runs each message as -// its own child run (`turn__`), so conversation continuity depends on -// the env builder resolving `storage` through the per-agent durable -// registry keyed by the STABLE stepId — never the per-run isogit store a -// changing runId would reset every turn. -test("with a durable-conversation registry, envs built for different runIds share one storage keyed by stepId", async () => { - const dataDir = makeTmpDataDir(); - const acquired: string[] = []; - const sharedStorage = { marker: "durable-store" }; - const registry = { - acquire: (key: string) => { - acquired.push(key); - return Promise.resolve({ storage: sharedStorage } as never); - }, - get: () => { - throw new Error("unused"); - }, - peek: () => undefined, - }; - const buildEnv = createSidecarStepBuildEnv({ - ...buildEnvDeps(dataDir), - durableConversation: registry as never, - }); - const sourcesRef: SourcesSnapshotRef = { - current: { - step_1: [{ id: "src_1", provider: "anthropic", model: "claude" }], - }, - } as unknown as SourcesSnapshotRef; - - const turn1 = stepInvokeRequest(); - turn1.authzContext.runId = "turn__0"; - const turn2 = stepInvokeRequest(); - turn2.authzContext.runId = "turn__1"; - - const env1 = await buildEnv(turn1, sourcesRef); - const env2 = await buildEnv(turn2, sourcesRef); - - expect(acquired).toEqual(["step_1", "step_1"]); - expect(env1.storage).toBe(sharedStorage as never); - expect(env2.storage).toBe(env1.storage); -}); - -// CL-6448: without the registry, per-run isogit stores stay per-run — the -// multi-step cold path's behavior is unchanged. -test("without a durable-conversation registry, envs built for different runIds get distinct storage", async () => { - const dataDir = makeTmpDataDir(); - const buildEnv = createSidecarStepBuildEnv(buildEnvDeps(dataDir)); - const sourcesRef: SourcesSnapshotRef = { - current: { - step_1: [{ id: "src_1", provider: "anthropic", model: "claude" }], - }, - } as unknown as SourcesSnapshotRef; - - const turn1 = stepInvokeRequest(); - turn1.authzContext.runId = "turn__0"; - const turn2 = stepInvokeRequest(); - turn2.authzContext.runId = "turn__1"; - - const env1 = await buildEnv(turn1, sourcesRef); - const env2 = await buildEnv(turn2, sourcesRef); - - expect(env1.storage).not.toBe(env2.storage); -}); - -test("the built step env forwards the summarize-older-turns compactor (CL-6204) like the other env fields above", async () => { - const dataDir = makeTmpDataDir(); - const buildEnv = createSidecarStepBuildEnv(buildEnvDeps(dataDir)); - const sourcesRef: SourcesSnapshotRef = { - current: { - step_1: [{ id: "src_1", provider: "anthropic", model: "claude" }], - }, - } as unknown as SourcesSnapshotRef; - - const env = await buildEnv(stepInvokeRequest(), sourcesRef); - - const compactors = (env as unknown as { compactors: Record }) - .compactors; - expect(Object.keys(compactors).sort()).toEqual( - [SUMMARIZE_OLDER_TURNS_NAME, SUMMARIZE_BUDGETED_TURNS_NAME].sort(), - ); - expect(compactors[SUMMARIZE_OLDER_TURNS_NAME]?.name).toBe(SUMMARIZE_OLDER_TURNS_NAME); - expect(compactors[SUMMARIZE_BUDGETED_TURNS_NAME]?.name).toBe(SUMMARIZE_BUDGETED_TURNS_NAME); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/step-env.ts b/apps/sidecar/src/workflow-substrate-factory/step-env.ts deleted file mode 100644 index 511eb991b0..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/step-env.ts +++ /dev/null @@ -1,489 +0,0 @@ -// Per-step env builder for the workflow-process child: stands up each -// step invocation's storage/workspace/audit/directors, resolves the -// step's ordered inference-source failover chain, materializes the -// step's pinned tool-package closure, and wires the supervisor-backed -// outbound mail transport. - -import fs from "node:fs"; -import path from "node:path"; - -import type { RepoId } from "@intx/hub-sessions/substrate"; -import { createDependencies, type AdapterRegistry } from "@intx/inference"; -import { createIsogitStorage, createNodeIsogitRuntime } from "@intx/storage-isogit/node"; -import type { RegistryConfig } from "@intx/tool-packaging"; -import type { AuditStore, ContextStore, MessageTransport } from "@intx/types/runtime"; -import type { StepInvokeRequest } from "@intx/workflow"; -import { - createSupervisorBackedTransport, - type ChildOutboundMailBridge, - type CredentialWiring, - type SourcesSnapshotRef, - type StepEnvBase, - type SupervisorBackedTransportInbound, -} from "@intx/workflow-host"; - -import type { DurableConversationRegistry } from "../conversation-state"; -import { - attachStepCredentials, - stepInferenceMaterialResolver, - attachStepTools, - materializeStepTools, - type StepToolCacheConfig, - type StepToolMaterialization, -} from "../step-agent-tools"; -import { - SUMMARIZE_BUDGETED_TURNS_NAME, - SUMMARIZE_OLDER_TURNS_NAME, - createBudgetedContextCompactor, - createSummarizeOlderTurnsCompactor, -} from "./compactors"; -import { createStepInferenceSourceResolver } from "./config"; -import { resolveContextBudgetChars, resolveHardContextLimitChars } from "./context-budget"; -import { stepStorageRoot, warmStepStorageRoot } from "./storage-paths"; -import { createWorkbenchDirectorRegistry } from "./workbench-director"; - -// Registered once and reused across every step env this builder produces: -// the compactor is a pure, stateless `Compactor` (see `./compactors`), so -// one instance can serve every step's `env.compactors` map. Registering it -// here (CL-6204) makes the name resolvable to any director that names it -// via `caps.compact("summarize-older-turns", reason)`. -const stepCompactors = { - [SUMMARIZE_OLDER_TURNS_NAME]: createSummarizeOlderTurnsCompactor(), -}; - -const isogitStorage = createIsogitStorage(createNodeIsogitRuntime()); - -export interface SidecarStepBuildEnvDeps { - dataDir: string; - workflowRunRepoId: RepoId; - signer: (payload: string) => Promise; - /** - * Tool-package registries per-step materialization resolves against, - * parsed from the boot-edge-threaded `SIDECAR_TOOL_REGISTRIES` - * substrate-config entry. - */ - registries: ReadonlyMap; - /** - * Deployment mailbox address the supervisor threaded into the child - * (`MAILBOX_ADDRESS`). Used to locate each step's on-disk deploy tree - * for tool materialization (see `stepDeployTreeDir`) AND as the step - * agent's outbound mail `address`: the supervisor signs the agent's - * outbound mail as this address through the host transport (OUTBOUND - * half of mailbox ownership). For the single-step launched-agent - * deploy this is the legacy `ins_` identity the host registered - * the agent's `CryptoProvider` against. - */ - mailboxAddress: string; - /** - * Step count of the deployed `WorkflowDefinition` (`stepOrder.length`), - * threaded from the host through the spawn-time env. Selects the - * head/step collapse when locating a step's deploy tree - * (`stepDeployTreeDir` -> `resolveStepAddress`): a single-step - * deployment reads at the head, a multi-step deployment at the per-step - * address, matching the host's producer push. - */ - stepCount: number; - /** - * Child-side outbound-mail bridge over the upstream control channel - * (OUTBOUND half of mailbox ownership). The per-step env builder - * wraps it in a supervisor-backed `MessageTransport` it supplies as - * the step agent's `env.transport`; the agent's mail tools call - * `transport.send`, which routes through the bridge to the supervisor - * for the actual signed send. The step agent never holds the signing - * key. - */ - outboundMailBridge: ChildOutboundMailBridge; - /** - * Inbound local IMAP surface for the step agent's supervisor-backed - * transport (INBOUND half of mailbox ownership): the child mailbox reader - * (a fresh committed snapshot of the deployment's substrate `INBOX` per - * read), the shared watch registry the child's control loop fires - * `mailbox.notify` into, and the mutation bridge flag/expunge writes ride - * to the supervisor on. Present, `mail_read` / `mail_search` / `mail_wait` - * resolve against the committed mailbox; absent, the inbound methods stay - * inert. - */ - inbound?: SupervisorBackedTransportInbound; - /** Per-step tool-loader caps (cache + registry tarball size). */ - cache: StepToolCacheConfig; - /** - * The hub's plain HTTP origin (derived from `HUB_WS_URL` via - * `deriveHubHttpUrl`) and the same `SIDECAR_TOKEN` bearer the child - * already carries for pack-push. Carried on `env` beyond `BaseEnv` so - * a workflow-artifacts tool bundle (`@corbits/artifact-tools`, - * `requires: ["hubArtifactsUrl", "sidecarToken", "address"]`) can call - * the sanctioned workflow-artifacts HTTP surface - * (`@corbits/artifacts-hub`'s `createWorkflowArtifactRoutes`, CL-6000) - * without ever holding a database handle. `address` is already on - * `env` via `mailboxAddress` below; these two widen the same surface. - * The same origin is re-exposed on the step env as `memoryBaseUrl` for - * `@corbits/memory/tools` (CL-8186), which calls the STOCK - * `/api/tenants/:tenantId/memory/*` routes `@corbits/memory` mounts on - * the hub directly (see `memoryFetch` below), and again as - * `hubSkillsUrl` for `@corbits/tools-skills`, which calls - * `@corbits/skills`' `createWorkflowSkillRoutes`. - */ - hubArtifactsUrl: string; - sidecarToken: string; - /** - * The deploying `WorkflowDefinition`'s own id, threaded from the - * substrate factory's `WORKFLOW_DEFINITION_REPO_ID` substrate-config - * entry (`spec.definition.id` at deploy time, see - * `apps/sidecar/src/workflow-host-wiring/index.ts`'s - * `buildDeploymentRecord`). Carried on the step env beyond `BaseEnv` - * exactly like `hubArtifactsUrl`/`sidecarToken`/`address` above so - * `@corbits/capability-tools` (`requires: ["hubCapabilitiesUrl", - * "sidecarToken", "address", "definitionId"]`) can name its OWN - * definition when calling the workflow-run-authenticated capabilities - * route — the run's definitionId has no other sanctioned way to reach - * a tool execution (see the [Intx gap] note in - * `@corbits/capability-tools`'s `tool.ts`, now closed on the workbench - * side by this field). - */ - definitionId: string; - /** - * The run's own tenant and principal, threaded from the hub's signed - * deploy frame (`HarnessConfig.tenantId` / `.principalId`) through the - * `WORKFLOW_TENANT_ID` / `WORKFLOW_PRINCIPAL_ID` substrate-config - * entries. Carried on the step env beyond `BaseEnv` exactly like - * `definitionId` above so a tool package can address stock - * `/api/tenants/:tenantId/*` with the run bearer — the tenant is a path - * segment on every stock tenant route, and a step tool has no other - * sanctioned way to learn it. - */ - tenantId: string; - principalId: string; - /** - * Adapter registry the step agent resolves inference adapters through. - * The child builds this eagerly at boot from the validated - * `SIDECAR_ADAPTER_MANIFEST` (built-ins merged with operator custom - * adapters) and the env builder sets it on `env.deps`, so a step whose - * source names a custom provider resolves in the child exactly as it - * does on the sidecar main path. Without it the step agent would fall - * back to `createAgent`'s built-ins-only default and a custom-provider - * source would fail to resolve at run time. - */ - adapters: AdapterRegistry; - /** - * Durable-conversation registry for the warm single-step agent. - * When present, the env builder swaps the per-run isogit - * `ContextStore` for a per-agent durable store whose conversation is - * mirrored to the workflow-run substrate, and restores the prior - * conversation from the substrate before returning the env (so the - * agent's reactor `load()` and the warm cache's lazy build see the - * restored turns -- including the respawn-rebuild path). Absent for a - * multi-step deploy, whose per-step agents are not warm/long-lived and - * need no cross-run conversation durability. - */ - durableConversation?: DurableConversationRegistry; -} - -/** - * Build the step-invoker `buildEnv` callback the workflow-host's - * adapter consumes. Pulled out of `createSidecarSubstrateFactory` so - * the per-step env construction is observable without standing up the - * full substrate. - * - * The closure reads the per-step source table from the mutable - * reference passed per build, derives the `stepId` / `runId` / `attempt` - * from the runtime's `AuthorizeContext`, resolves the per-step - * `InferenceSource` from that table, and stands up - * a per-step isogit `ContextStore` (also serving as the audit store) - * plus a per-step workspace directory rooted under the run. A - * construction failure (mkdir, isogit init) surfaces here rather than - * being papered over with a stub: the single-step path now always runs - * a real agent against real storage. - */ -export function createSidecarStepBuildEnv( - deps: SidecarStepBuildEnvDeps, -): ( - req: StepInvokeRequest, - sourcesRef: SourcesSnapshotRef, - credentialWiring?: CredentialWiring, -) => Promise { - return async ( - req: StepInvokeRequest, - sourcesRef: SourcesSnapshotRef, - credentialWiring?: CredentialWiring, - ): Promise => { - // Resolve against the live table each build so a source rotation that - // wrote `sourcesRef.current` before this build is reflected in the - // agent this build constructs. A warm agent that is already built does - // not pass through here again, so a rotation does not reach it through - // this path -- this ref covers only a build that has not happened yet. - const resolveStepInferenceSource = createStepInferenceSourceResolver(sourcesRef.current); - const { stepId, runId, attempt } = req.authzContext; - if (stepId === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.stepId is required for per-step InferenceSource resolution; the workflow runtime must populate stepId on every step-originated invocation", - ); - } - if (runId === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.runId is required to root per-step storage under the run; the workflow runtime must populate runId on every step-originated invocation", - ); - } - if (attempt === undefined) { - throw new Error( - "sidecar workflow-child step invoker buildEnv: AuthorizeContext.attempt is required to root per-step storage per attempt; the workflow runtime must populate attempt on every step-originated invocation", - ); - } - const sources = resolveStepInferenceSource(stepId); - // The resolver's arktype guarantees a non-empty chain; assert it here so - // the reactor's initial-source pin (element 0) is a checked fact rather - // than an unchecked index. - const activeSource = sources[0]; - if (activeSource === undefined) { - throw new Error( - `sidecar workflow-child step invoker buildEnv: empty InferenceSource chain pinned for stepId ${JSON.stringify(stepId)}`, - ); - } - - // Context-window budget for this step's active source (CL-6204): - // sized from `activeSource.quirks`/`activeSource.model` so a 128K - // model and a 32K model compact at different points rather than - // sharing one constant. Built per invocation (not module scope) - // because the budget depends on the step's own resolved source. - const contextBudgetChars = resolveContextBudgetChars(activeSource.quirks, activeSource.model); - const contextHardLimitChars = resolveHardContextLimitChars( - activeSource.quirks, - activeSource.model, - ); - const stepDirectors = createWorkbenchDirectorRegistry({ - budgetChars: contextBudgetChars, - hardLimitChars: contextHardLimitChars, - compactorName: SUMMARIZE_BUDGETED_TURNS_NAME, - }); - const compactors = { - ...stepCompactors, - [SUMMARIZE_BUDGETED_TURNS_NAME]: createBudgetedContextCompactor(contextBudgetChars), - }; - - // Root the per-step scratch (workspace + tool tarball-cache + - // apply-state). The cold (multi-step) path keys it per - // run/step/attempt: each run rebuilds the agent and its scratch, and - // the run's whole `runs//` subtree is reclaimed on run - // completion. The warm single-step path (`durableConversation` - // present) keys it STABLY per agent so the cached agent reuses one - // workspace across every message -- bounding the warm case to one - // dir per agent and letting that workspace survive child respawn -- - // and the subtree is reclaimed on undeploy. The two keyings live - // under disjoint `runs/` and `warm/` sub-roots so neither sweep - // touches the other's tree, and the durable conversation under - // `agent-conversation-state/` is a different root that neither - // sweep touches. - const storeDir = - deps.durableConversation !== undefined - ? warmStepStorageRoot({ - dataDir: deps.dataDir, - workflowRunRepoId: deps.workflowRunRepoId, - stepId, - }) - : stepStorageRoot({ - dataDir: deps.dataDir, - workflowRunRepoId: deps.workflowRunRepoId, - runId, - stepId, - attempt, - }); - // Conversation storage. For the warm single-step agent the - // conversation must survive child respawn, so it is backed by a - // per-agent durable store (keyed by stepId) whose content is mirrored - // to the workflow-run substrate under - // `agent-state///`. Bind to the originating - // workbench happens before this builder runs so restore is already - // applied. A multi-step deploy (no durable registry) keeps the - // per-run isogit store. - const storage: ContextStore & AuditStore = - deps.durableConversation !== undefined - ? (await deps.durableConversation.acquire(stepId)).storage - : await isogitStorage.createIsogitStore(storeDir, deps.signer); - const workdir = path.join(storeDir, "workspace"); - await fs.promises.mkdir(workdir, { recursive: true }); - - // Materialize the step's pinned tool-package closure (posix, LSP, - // mail, ...) from its on-disk deploy tree, rooted per step under - // `storeDir` so concurrent steps in one child never collide on the - // tarball cache or the apply-state tree. A deploy with no manifest - // yields empty tools (the legitimate `rawManifestBytes === undefined` - // case); a present-but-broken manifest surfaces loudly through - // `materializeStepTools` rather than degrading to empty tools. - // - // An onTrigger body step (CL-6448) materializes through the same call: - // for the single-step section deployment the head/step collapse in - // `stepDeployTreeDir` reads the deployment's own staged manifest -- - // exactly the body agent's pins the folded launch staged -- and a - // deployment that staged no tree for the body's stepId reads ENOENT - // into the legitimate empty-tools case. - const materialization: StepToolMaterialization = await materializeStepTools({ - dataDir: deps.dataDir, - mailboxAddress: deps.mailboxAddress, - stepId, - stepCount: deps.stepCount, - storeDir, - cache: deps.cache, - registries: deps.registries, - }); - - // Supervisor-backed transport for the step agent's mail tools (both - // halves of mailbox ownership). Outbound (`send`) routes over the - // control IPC to the supervisor, which performs the actual signed send - // through the host transport as `address`: the deployment mailbox - // address, the same identity the host registered the agent's - // `CryptoProvider` against, so the outbound mail carries the agent's - // signature with parity to the in-process path. Inbound - // (`deps.inbound`) makes `mail_read` / `mail_search` / `mail_wait` - // resolve locally against a fresh committed snapshot of the - // deployment's substrate `INBOX`. Both `transport` and `address` are - // the env keys `@intx/tools-mail`'s sidecar bundle declares in its - // `requires`. - const transport = createSupervisorBackedTransport( - deps.outboundMailBridge, - deps.mailboxAddress, - deps.inbound, - ); - - // The step env carries `transport` + `address` beyond `BaseEnv` so - // the mail-tool bundle (`@intx/tools-mail`, `requires: ["transport", - // "address"]`) resolves its handles. The two keys are extra env - // surface the tool factory reads at handler-init; they widen the - // returned `StepEnvBase` structurally, which the buildEnv return - // type (`StepEnvBase`) accepts (a wider object is assignable to the - // narrower type). - const env: StepEnvBase & { - transport: MessageTransport; - address: string; - hubArtifactsUrl: string; - memoryBaseUrl: string; - memoryTenantId: string; - memoryAuthToken: string; - memoryFetch: ( - input: Parameters[0], - init?: Parameters[1], - ) => ReturnType; - hubSkillsUrl: string; - hubCapabilitiesUrl: string; - hubRoutinesUrl: string; - hubConnectionsUrl: string; - hubCatalogUrl: string; - hubAgentDirectoryUrl: string; - hubChatUrl: string; - hubWorkflowAuthoringUrl: string; - hubAccessUrl: string; - sidecarToken: string; - definitionId: string; - tenantId: string; - principalId: string; - } = { - // An `InferenceSource` names a `credentialId`, not an inline secret: - // the reactor fills the request's credential at send time through - // this resolver, off the same live cell the step's tool credentials - // resolve from. Omitted for a step with no credential wiring, which - // leaves the platform's fail-closed default in place. - ...(credentialWiring !== undefined - ? { - readCurrentMaterial: stepInferenceMaterialResolver({ - wiring: credentialWiring, - stepId, - }), - } - : {}), - // Feed the reactor the step's full ordered failover chain and pin - // its initial source to element 0. The reactor resolves the initial - // source by id and fails over forward through `sources`, so this - // restores cross-source failover inside the workflow-child. - sources, - defaultSource: activeSource.id, - storage, - workdir, - audit: storage, - directors: stepDirectors, - compactors, - // Resolve inference adapters through the child's boot-built - // registry (built-ins + operator custom adapters), so a - // custom-provider step source resolves in the child the same way - // it does on the sidecar main path rather than hitting - // `createAgent`'s built-ins-only default. - deps: createDependencies(deps.adapters), - transport, - address: deps.mailboxAddress, - hubArtifactsUrl: deps.hubArtifactsUrl, - // `@corbits/memory/tools` (CL-8186) calls the STOCK - // `/api/tenants/:tenantId/memory/*` routes `@corbits/memory` mounts - // on the hub, the same run-bearer scheme `@corbits/connections-tools` - // and friends use: sidecar token as the bearer, `tenantId` as the - // path segment. Its own HTTP client sends only an `Authorization` - // header, so `memoryFetch` (the client's documented fetch override, - // never part of `requires`) is what actually attaches the - // `x-workflow-run-address` header the hub's `withWorkflowRunTenantAuth` - // needs to resolve the bearer to this run's principal. - memoryBaseUrl: deps.hubArtifactsUrl, - memoryTenantId: deps.tenantId, - memoryAuthToken: deps.sidecarToken, - memoryFetch: (input, init) => - fetch(input, { - ...init, - headers: { - ...init?.headers, - "x-workflow-run-address": deps.mailboxAddress, - }, - }), - // And once more under the key `@corbits/tools-skills` declares - // (`requires: ["hubSkillsUrl", "sidecarToken", "address"]`) for the - // skill registry's own run-authenticated surface. - hubSkillsUrl: deps.hubArtifactsUrl, - // Same hub HTTP origin again, under the key - // `@corbits/capability-tools` declares (`requires: - // ["hubCapabilitiesUrl", "sidecarToken", "address", - // "definitionId"]`) for its workflow-run-authenticated capabilities - // surface. - hubCapabilitiesUrl: deps.hubArtifactsUrl, - // Same hub HTTP origin again, under the keys the Myra manager-tools - // bundles declare (CL-5879 follow-up: connections-tools, - // agent-directory-tools) for their own - // workflow-run-authenticated surfaces — one hub origin, one env key - // per tool-bundle surface, matching every precedent above rather - // than overloading an existing name for an unrelated surface. - hubRoutinesUrl: deps.hubArtifactsUrl, - hubConnectionsUrl: deps.hubArtifactsUrl, - // And once more under the key `@corbits/catalog-tools` declares - // (`requires: ["hubCatalogUrl", "tenantId", "sidecarToken", - // "address"]`) for the stock tenant model-discovery and tenant reads - // it resolves a model chain from. - hubCatalogUrl: deps.hubArtifactsUrl, - hubAgentDirectoryUrl: deps.hubArtifactsUrl, - hubChatUrl: deps.hubArtifactsUrl, - // And under the key `@corbits/workflow-authoring-tools` declares - // (`requires: ["hubWorkflowAuthoringUrl", "tenantId", "sidecarToken", - // "address"]`) for `@corbits/workflows`'s `./authoring`'s - // run-authenticated routes and for the stock deploy route it calls - // alongside them. - hubWorkflowAuthoringUrl: deps.hubArtifactsUrl, - // And once more under the key `@corbits/access-tools` declares - // (`requires: ["hubAccessUrl", "sidecarToken", "address"]`) for its - // own workflow-run-authenticated principal/grant surface. - hubAccessUrl: deps.hubArtifactsUrl, - sidecarToken: deps.sidecarToken, - definitionId: deps.definitionId, - // The run's own tenant/principal, for the tool bundles that address - // stock `/api/tenants/:tenantId/*` routes with the run bearer - // (`@corbits/access-tools`, `@corbits/connections-tools`, - // `@corbits/workflow-authoring-tools`, `@corbits/catalog-tools`). - tenantId: deps.tenantId, - principalId: deps.principalId, - }; - // Carry the materialized tool runtime to the tool-bearing - // `agentFactory` via the env's symbol-keyed slot. The step-invoker - // adapter spreads this env (`{ ...envBase, authorize }`) before - // handing it to `agentFactory`; object spread preserves own - // symbol-keyed properties, so the slot survives the spread. - attachStepTools(env, materialization); - // Carry this step's live credential wiring the same way, so the - // tool-bearing `agentFactory` can shape a consumer-scoped - // `credentials` capability for any tool package that declares one. - if (credentialWiring !== undefined) { - attachStepCredentials(env, { wiring: credentialWiring, stepId }); - } - return env; - }; -} diff --git a/apps/sidecar/src/workflow-substrate-factory/storage-paths.ts b/apps/sidecar/src/workflow-substrate-factory/storage-paths.ts deleted file mode 100644 index 05ee249753..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/storage-paths.ts +++ /dev/null @@ -1,120 +0,0 @@ -// Per-step storage path derivation for the workflow-process child: -// where a step invocation's agent-state storage and workspace root on -// the sidecar's local disk, for the cold per-run/step/attempt keying, -// the per-run reclamation root, and the warm single-step agent's -// stable per-agent keying. - -import path from "node:path"; - -import type { RepoId } from "@intx/hub-sessions/substrate"; - -/** - * Root directory for a single step invocation's agent-state storage and - * workspace, derived from the sidecar data dir and the run/step/attempt - * coordinates the workflow runtime owns. - * - * The per-step agent storage is a distinct isogit repo, deliberately - * rooted OUTSIDE the workflow-run repo's working tree. The workflow-run - * repo's single writer is the supervisor, and its working tree carries - * the run-event log under `runs//events/...`; nesting a second - * git repo inside that tree would collide with the event subtree and - * with the supervisor's write contract. Rooting the per-step store under - * a dedicated `workflow-step-state/` sibling subtree keyed by the - * workflow-run repo id keeps every step's storage isolated per run and - * per step while never touching the run-event tree. - */ -export function stepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; - stepId: string; - attempt: number; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "runs", - args.runId, - "steps", - args.stepId, - `attempt-${String(args.attempt)}`, - ); -} - -/** - * Root directory for a single workflow-run subtree's per-step scratch: - * `/workflow-step-state//runs//`. The cold - * (multi-step) path's per-step `stepStorageRoot` nests under this, so - * reclaiming this subtree on run completion drops every step/attempt the - * run produced in one `rm -rf`. Kept distinct from `stepStorageRoot` so - * the deletion granularity (a whole run, not a single step/attempt) is - * expressed at the call site that owns run-completion cleanup. - */ -export function runStepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - runId: string; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "runs", - args.runId, - ); -} - -/** - * Stable per-agent scratch root for the WARM single-step agent's - * workspace + tool materialization (tarball-cache + apply-state). Keyed - * by the step identity exactly like the durable conversation store's - * `agent-conversation-state///` (conversation-state.ts), - * NOT by the arbitrary first-message runId. Keying it stably is what - * bounds the warm case: the cached agent reuses ONE workspace across - * every message in the child's lifetime, and that same workspace is - * re-derived (and so survives) across a child respawn, instead of - * stranding a fresh per-runId subtree each time. The whole subtree is - * reclaimed on undeploy, when the deployment's supervisor + child are - * already torn down. Rooted under a `warm/` sibling of the cold `runs/` - * subtree so the undeploy sweep of `workflow-step-state//` - * reclaims both with one removal and the two keyings never collide. - */ -/** - * Stable per-action-step scratch root for the action-handler registry's - * tool materialization (tarball-cache + apply-state + workspace). An - * action step's tool closure is materialized ONCE, at deployment - * establish (registry construction), and its handler dispatches against - * that closure across every run -- so the keying is per step, not per - * run/attempt. Rooted under an `action/` sibling of the cold `runs/` - * and warm `warm/` sub-roots so no reclamation sweep collides with it; - * the undeploy sweep of `workflow-step-state//` reclaims all - * three with one removal. - */ -export function actionStepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - stepId: string; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "action", - encodeURIComponent(args.stepId), - ); -} - -export function warmStepStorageRoot(args: { - dataDir: string; - workflowRunRepoId: RepoId; - stepId: string; -}): string { - return path.join( - args.dataDir, - "workflow-step-state", - args.workflowRunRepoId.id, - "warm", - encodeURIComponent(args.stepId), - ); -} diff --git a/apps/sidecar/src/workflow-substrate-factory/workbench-director.test.ts b/apps/sidecar/src/workflow-substrate-factory/workbench-director.test.ts deleted file mode 100644 index 3b4cc337d8..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/workbench-director.test.ts +++ /dev/null @@ -1,382 +0,0 @@ -// Workbench director: empty-turn retry + honest reply, composed over -// DefaultDirector so tool.done (including errors) still re-infers. -import { expect, test } from "bun:test"; - -import { defaultDirectorFactory } from "@intx/agent"; -import { createCapabilities, createDefaultDirector } from "@intx/inference"; -import type { - AssistantTurn, - ConversationTurn, - ReactorAction, - ReactorDirector, - ReactorInboundEvent, - ReactorState, - ToolResult, -} from "@intx/types/runtime"; - -import { - CONTEXT_OVERFLOW_MESSAGE, - EMPTY_TURN_REPLY, - WORKBENCH_DIRECTOR_ID, - createWorkbenchDirector, - createWorkbenchDirectorRegistry, -} from "./workbench-director"; - -const caps = createCapabilities(); - -const usage = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - thinking: 0, -}; - -const source = { sourceId: "src_1", provider: "test", model: "test" }; - -function state(): ReactorState { - return { - turns: [], - activeForks: [], - pendingOperations: [], - activeGates: [], - tokenUsage: usage, - lastCycleUsage: usage, - lastCycleSource: source, - sessionId: "sess_1", - }; -} - -function emptyTurn(): AssistantTurn { - return { role: "assistant", content: [], model: "test", timestamp: 0 }; -} - -function textTurn(text: string): AssistantTurn { - return { - role: "assistant", - content: [{ type: "text", text }], - model: "test", - timestamp: 0, - }; -} - -function toolTurn(name: string): AssistantTurn { - return { - role: "assistant", - content: [{ type: "tool_call", id: "call_1", name, arguments: {} }], - model: "test", - timestamp: 0, - }; -} - -function twoCallToolTurn(): AssistantTurn { - return { - role: "assistant", - content: [ - { type: "tool_call", id: "call_1", name: "look_up", arguments: {} }, - { type: "tool_call", id: "call_2", name: "look_up", arguments: {} }, - ], - model: "test", - timestamp: 0, - }; -} - -function conversationTurn(text: string): ConversationTurn { - return { role: "user", content: [{ type: "text", text }], timestamp: 0 }; -} - -function toolResultTurn(payload: string, callId: string): ConversationTurn { - return { - role: "user", - content: [ - { - type: "tool_result", - callId, - content: [{ type: "text", text: payload }], - }, - ], - timestamp: 0, - }; -} - -function stateWithTurns(turns: ConversationTurn[]): ReactorState { - return { ...state(), turns }; -} - -function inferenceDone(turn: AssistantTurn): ReactorInboundEvent { - return { type: "inference.done", turn, usage, source }; -} - -function asList(action: ReactorAction | ReactorAction[]): ReactorAction[] { - return Array.isArray(action) ? action : [action]; -} - -function typesOf(action: ReactorAction | ReactorAction[]): string[] { - return asList(action).map((a) => a.type); -} - -function replyOf(action: ReactorAction | ReactorAction[]): string | undefined { - return asList(action).find((a) => a.type === "reply")?.content; -} - -async function seedToolBatch(director: ReactorDirector): Promise { - await director.decide(inferenceDone(toolTurn("look_up")), state(), caps); -} - -test("empty inference.done retries infer once", async () => { - const director = createWorkbenchDirector("you are a test agent"); - const actions = await director.decide(inferenceDone(emptyTurn()), state(), caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "infer"]); -}); - -test("a second empty inference.done replies honestly instead of waiting", async () => { - const director = createWorkbenchDirector("you are a test agent"); - await director.decide(inferenceDone(emptyTurn()), state(), caps); - const actions = await director.decide(inferenceDone(emptyTurn()), state(), caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "reply"]); - expect(replyOf(actions)).toBe(EMPTY_TURN_REPLY); - expect(replyOf(actions)).toContain("empty model turn"); -}); - -test("empty then a real text turn replies with the model text", async () => { - const director = createWorkbenchDirector("you are a test agent"); - await director.decide(inferenceDone(emptyTurn()), state(), caps); - const actions = await director.decide( - inferenceDone(textTurn("here is the answer")), - state(), - caps, - ); - - expect(typesOf(actions)).toEqual(["checkpoint", "reply"]); - expect(replyOf(actions)).toBe("here is the answer"); -}); - -test("a text turn on the first inference.done replies without retrying", async () => { - const director = createWorkbenchDirector("you are a test agent"); - const actions = await director.decide(inferenceDone(textTurn("hello")), state(), caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "reply"]); - expect(replyOf(actions)).toBe("hello"); -}); - -test("tool calls still execute; DefaultDirector already re-infers on errored tool.done", async () => { - const vendor = createDefaultDirector("you are a test agent"); - const workbench = createWorkbenchDirector("you are a test agent"); - await seedToolBatch(vendor); - await seedToolBatch(workbench); - - const errored: ToolResult = { - callId: "call_1", - content: "look_up failed", - isError: true, - }; - const event: ReactorInboundEvent = { type: "tool.done", result: errored }; - - const vendorActions = await vendor.decide(event, state(), caps); - const workbenchActions = await workbench.decide(event, state(), caps); - - expect(typesOf(vendorActions)).toEqual(["checkpoint", "infer"]); - expect(typesOf(workbenchActions)).toEqual(["checkpoint", "infer"]); -}); - -test("reactive mode still waits on an empty turn (no retry)", async () => { - const director = createWorkbenchDirector("you are a test agent", [], { - mode: "reactive", - }); - const actions = await director.decide(inferenceDone(emptyTurn()), state(), caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "wait"]); -}); - -test("a new message.received resets the empty-turn retry budget", async () => { - const director = createWorkbenchDirector("you are a test agent"); - await director.decide(inferenceDone(emptyTurn()), state(), caps); - await director.decide( - { - type: "message.received", - message: { id: "m1", content: "hi" } as never, - }, - state(), - caps, - ); - const actions = await director.decide(inferenceDone(emptyTurn()), state(), caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "infer"]); -}); - -test("context budget: a short conversation under budget is untouched (infers normally)", async () => { - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 10_000, - hardLimitChars: 20_000, - compactorName: "summarize-budgeted-turns", - }, - ); - const shortState = stateWithTurns([conversationTurn("hi")]); - - const actions = await director.decide( - { type: "message.received", message: { id: "m1", content: "hi" } as never }, - shortState, - caps, - ); - - expect(typesOf(actions)).toEqual(["infer"]); -}); - -test("context budget: message.received over budget but under the hard limit still infers", async () => { - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 100, - hardLimitChars: 1_000_000, - compactorName: "summarize-budgeted-turns", - }, - ); - const overBudget = stateWithTurns([ - conversationTurn("a".repeat(200)), - conversationTurn("b".repeat(200)), - ]); - - const actions = await director.decide( - { type: "message.received", message: { id: "m1", content: "hi" } as never }, - overBudget, - caps, - ); - - expect(typesOf(actions)).toEqual(["infer"]); - expect(typesOf(actions)).not.toContain("compact"); -}); - -test("context budget: tool-heavy history past the hard limit is caught even though every turn's text excerpt is short", async () => { - // The reviewer's exact repro: 10 turns each carrying a 20,000-char - // tool_result. Measured by placeholder length this was ~160 chars - // total (invisible to a 32,000-char hard limit); measured by real - // payload size it is 200,000 chars, well past it. - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 16_000, - hardLimitChars: 32_000, - compactorName: "summarize-budgeted-turns", - }, - ); - const bigState = stateWithTurns( - Array.from({ length: 10 }, (_, i) => toolResultTurn("x".repeat(20_000), `call_${String(i)}`)), - ); - - const actions = await director.decide( - { type: "message.received", message: { id: "m1", content: "hi" } as never }, - bigState, - caps, - ); - - expect(typesOf(actions)).toEqual(["checkpoint", "reply"]); - expect(replyOf(actions)).toBe(CONTEXT_OVERFLOW_MESSAGE); -}); - -test("context budget: history past the hard limit replies with the honest overflow message instead of inferring", async () => { - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 10, - hardLimitChars: 20, - compactorName: "summarize-budgeted-turns", - }, - ); - const bigState = stateWithTurns([ - conversationTurn("a".repeat(1_000)), - conversationTurn("more content that pushes well past the hard limit"), - ]); - - const actions = await director.decide( - { type: "message.received", message: { id: "m1", content: "hi" } as never }, - bigState, - caps, - ); - - expect(typesOf(actions)).toEqual(["checkpoint", "reply"]); - expect(replyOf(actions)).toBe(CONTEXT_OVERFLOW_MESSAGE); -}); - -test("context budget: a non-final tool.done in a multi-call batch compacts instead of no-op when over budget", async () => { - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 10, - hardLimitChars: 1_000_000, - compactorName: "summarize-budgeted-turns", - }, - ); - await director.decide(inferenceDone(twoCallToolTurn()), state(), caps); - - const bigState = stateWithTurns([ - conversationTurn("a".repeat(200)), - conversationTurn("b".repeat(200)), - ]); - const firstDone: ReactorInboundEvent = { - type: "tool.done", - result: { callId: "call_1", content: "ok", isError: false }, - }; - - const actions = await director.decide(firstDone, bigState, caps); - - expect(typesOf(actions)).toEqual(["checkpoint", "compact"]); -}); - -test("context budget: a non-final tool.done under budget stays a no-op (regression)", async () => { - const director = createWorkbenchDirector( - "you are a test agent", - [], - {}, - { - budgetChars: 1_000_000, - hardLimitChars: 2_000_000, - compactorName: "summarize-budgeted-turns", - }, - ); - await director.decide(inferenceDone(twoCallToolTurn()), state(), caps); - - const smallState = stateWithTurns([conversationTurn("hi")]); - const firstDone: ReactorInboundEvent = { - type: "tool.done", - result: { callId: "call_1", content: "ok", isError: false }, - }; - - const actions = await director.decide(firstDone, smallState, caps); - - expect(typesOf(actions)).toEqual([]); -}); - -test("context budget: with no contextBudget configured, behavior is unchanged", async () => { - const director = createWorkbenchDirector("you are a test agent"); - const bigState = stateWithTurns([conversationTurn("a".repeat(100_000))]); - - const actions = await director.decide( - { type: "message.received", message: { id: "m1", content: "hi" } as never }, - bigState, - caps, - ); - - expect(typesOf(actions)).toEqual(["infer"]); -}); - -test("the factory is namespaced and is the sidecar registry default", () => { - const registry = createWorkbenchDirectorRegistry(); - expect(registry.defaultFactory().id).toBe(WORKBENCH_DIRECTOR_ID); - expect(registry.resolve({ id: defaultDirectorFactory.id, config: {} }).id).toBe( - defaultDirectorFactory.id, - ); -}); diff --git a/apps/sidecar/src/workflow-substrate-factory/workbench-director.ts b/apps/sidecar/src/workflow-substrate-factory/workbench-director.ts deleted file mode 100644 index 23e091a948..0000000000 --- a/apps/sidecar/src/workflow-substrate-factory/workbench-director.ts +++ /dev/null @@ -1,285 +0,0 @@ -// Workbench director: DefaultDirector plus an empty-turn retry and a -// context-budget gate. -// -// `@intx/inference`'s DefaultDirector checkpoints and waits when -// inference.done has no text and no tool calls. The human then sits in a -// silent wait. This director retries infer once; a second empty turn -// replies with a short honest message instead of waiting. -// -// tool.done is unchanged: DefaultDirector already re-infers once the -// outstanding batch is complete, including when the result is an error. -// We compose that path rather than reimplement it. -// -// Context budget (`contextBudget`, optional -- absent means unbudgeted, -// today's pre-CL-6204 behavior): checks the turn history against the -// model's real context window (`resolveContextBudgetChars` / -// `resolveHardContextLimitChars` in `./context-budget`, sized from -// `InferenceSource.quirks` and the advertised catalog window). -// -// - Over the hard limit (no headroom left at all): this is Ollama's -// silent-truncation case made honest -- reply with the same -// "exceeded the model's context limit" message hosted providers -// produce via `inference.error`'s `context_overflow` category -// (`@intx/inference`'s `default-director.js`), instead of sending a -// request that would truncate server-side with no error. Compact -// cannot help here, so `CONTEXT_OVERFLOW_MESSAGE` is the only reply. -// - Over the (headroomed) budget but under the hard limit, on a -// non-final `tool.done` in a multi-call batch (DefaultDirector -// returns `[]` for every `tool.done` before the batch's last): fire -// `caps.compact` instead of the no-op. The batch's remaining -// `tool.done` events are already enqueued (`@intx/inference`'s -// `reactor.js` `executeTools` enqueues every result from one -// `Promise.all` before the reactor dequeues any of them) and will -// still drive the eventual re-infer. Do not compact as the sole -// terminal action on `message.received`: the reactor forbids -// compact+infer in one cycle and does not re-enter the director -// after compact, so compact-only would leave the inbound message -// unanswered. Over-budget inbound still infers; compaction waits -// for that next safe point. -// - Otherwise: let the inner decision through unchanged. Compaction is -// deferred to the next safe point rather than forced here. -// -// Registered as the sidecar step-env default via -// `createWorkbenchDirectorRegistry` (see `./step-env`). Id is -// `@workbench/sidecar/workbench`; `@intx/agent/default` stays resolvable -// for definitions that name it explicitly. - -import { type } from "arktype"; - -import { - createDirectorRegistry, - defaultDirectorFactory, - defineDirector, - type DirectorRegistry, -} from "@intx/agent"; -import { createDefaultDirector, type DefaultDirectorPolicy } from "@intx/inference"; -import { - formatSafetyRatingText, - type AssistantTurn, - type ReactorAction, - type ReactorCapabilities, - type ReactorDirector, - type ReactorInboundEvent, - type ReactorState, - type ToolCall, - type ToolDefinition, -} from "@intx/types/runtime"; - -import { estimateTurnsChars } from "./compactors"; - -export const WORKBENCH_DIRECTOR_ID = "@workbench/sidecar/workbench"; -export const EMPTY_TURN_REPLY = "I got an empty model turn"; -export const CONTEXT_OVERFLOW_MESSAGE = - "This agent could not complete your request because the conversation exceeded the model's context limit"; - -export type ContextBudgetOptions = { - /** Headroomed char budget past which a safe compaction point fires. */ - budgetChars: number; - /** Raw char limit past which sending would overflow the model's window. */ - hardLimitChars: number; - /** Name the compactor is registered under in `env.compactors`. */ - compactorName: string; -}; - -function extractToolCalls(turn: AssistantTurn): ToolCall[] { - const calls: ToolCall[] = []; - for (const block of turn.content) { - if (block.type === "tool_call") { - calls.push({ - id: block.id, - name: block.name, - arguments: block.arguments, - }); - } - } - return calls; -} - -function extractTextContent(turn: AssistantTurn): string { - const parts: string[] = []; - for (const block of turn.content) { - if (block.type === "text") { - parts.push(block.text); - } else if (block.type === "refusal") { - parts.push(block.reason); - } else if (block.type === "safety_rating") { - parts.push(formatSafetyRatingText(block)); - } - } - return parts.join("\n").trim(); -} - -function actionsIncludeWait(actions: ReactorAction | ReactorAction[]): boolean { - const list = Array.isArray(actions) ? actions : [actions]; - return list.some((action) => action.type === "wait"); -} - -export class WorkbenchDirector implements ReactorDirector { - private readonly inner: ReactorDirector; - private readonly systemPrompt: string; - private readonly toolDefinitions: ToolDefinition[]; - private readonly conversational: boolean; - private readonly contextBudget: ContextBudgetOptions | undefined; - private emptyTurnRetried = false; - - constructor( - systemPrompt: string, - toolDefinitions: ToolDefinition[] = [], - policy: DefaultDirectorPolicy = {}, - contextBudget?: ContextBudgetOptions, - ) { - this.inner = createDefaultDirector(systemPrompt, toolDefinitions, policy); - this.systemPrompt = systemPrompt; - this.toolDefinitions = toolDefinitions; - this.conversational = policy.mode !== "reactive"; - this.contextBudget = contextBudget; - } - - async decide( - event: ReactorInboundEvent, - state: ReactorState, - capabilities: ReactorCapabilities, - ): Promise { - if (event.type === "message.received") { - this.emptyTurnRetried = false; - } - - const actions = await this.inner.decide(event, state, capabilities); - - const budgeted = this.applyContextBudget(event, state, capabilities, actions); - if (budgeted !== undefined) { - return budgeted; - } - - if (event.type !== "inference.done") { - return actions; - } - - if (!this.isEmptyConversationalTurn(event.turn) || !actionsIncludeWait(actions)) { - this.emptyTurnRetried = false; - return actions; - } - - if (!this.emptyTurnRetried) { - this.emptyTurnRetried = true; - return [ - capabilities.checkpoint("empty-turn-retry"), - capabilities.infer({ - systemPrompt: this.systemPrompt, - tools: this.toolDefinitions, - }), - ]; - } - - this.emptyTurnRetried = false; - return [capabilities.checkpoint("empty-turn-reply"), capabilities.reply(EMPTY_TURN_REPLY)]; - } - - private isEmptyConversationalTurn(turn: AssistantTurn): boolean { - if (!this.conversational) { - return false; - } - return extractToolCalls(turn).length === 0 && extractTextContent(turn).length === 0; - } - - /** - * Returns a replacement action set when the context budget overrides - * the inner director's decision, `undefined` to let it through - * unchanged. See this file's header comment for the two cases this - * covers (honest overflow, safe-point compaction) and why every other - * case passes through untouched. - */ - private applyContextBudget( - event: ReactorInboundEvent, - state: ReactorState, - capabilities: ReactorCapabilities, - actions: ReactorAction | ReactorAction[], - ): ReactorAction[] | undefined { - if (this.contextBudget === undefined) { - return undefined; - } - const list = Array.isArray(actions) ? actions : [actions]; - const chars = estimateTurnsChars(state.turns); - - if (chars > this.contextBudget.hardLimitChars) { - if (list.some((action) => action.type === "infer") || event.type === "message.received") { - return [ - capabilities.checkpoint("context-overflow"), - capabilities.reply(CONTEXT_OVERFLOW_MESSAGE), - ]; - } - } - - if (list.some((action) => action.type === "infer")) { - return undefined; - } - - if (event.type === "tool.done" && list.length === 0 && chars > this.contextBudget.budgetChars) { - return [ - capabilities.checkpoint("context-budget-compact"), - capabilities.compact(this.contextBudget.compactorName, "context-budget"), - ]; - } - - return undefined; - } -} - -export function createWorkbenchDirector( - systemPrompt: string, - toolDefinitions: ToolDefinition[] = [], - policy: DefaultDirectorPolicy = {}, - contextBudget?: ContextBudgetOptions, -): ReactorDirector { - return new WorkbenchDirector(systemPrompt, toolDefinitions, policy, contextBudget); -} - -const WorkbenchDirectorConfigSchema = type({ - "mode?": '"conversational" | "reactive"', -}); - -export type WorkbenchDirectorConfig = { - mode?: "conversational" | "reactive"; -}; - -function buildWorkbenchFactory(contextBudget: ContextBudgetOptions | undefined) { - return defineDirector({ - id: WORKBENCH_DIRECTOR_ID, - configSchema: WorkbenchDirectorConfigSchema, - factory: (config, _env, agent) => { - const policy: DefaultDirectorPolicy = {}; - if (config.mode !== undefined) { - policy.mode = config.mode; - } - return createWorkbenchDirector( - agent.systemPrompt, - [...agent.toolDefinitions], - policy, - contextBudget, - ); - }, - }).factory; -} - -/** - * Sidecar step-env director registry: workbench is the default so - * unspecified AgentDefinitions get empty-turn retry. The built-in - * `@intx/agent/default` stays resolvable for definitions that name it. - * - * `contextBudget`, when supplied, bakes a per-step context-window budget - * (sized from the step's active `InferenceSource`, which varies per - * step/model) into the workbench factory this registry resolves -- - * `createSidecarStepBuildEnv` builds a fresh registry per step build - * rather than reusing one shared instance so each step's budget matches - * its own model. See `WorkbenchDirector`'s header comment for what the - * budget gates. - */ -export function createWorkbenchDirectorRegistry( - contextBudget?: ContextBudgetOptions, -): DirectorRegistry { - const workbenchFactory = buildWorkbenchFactory(contextBudget); - return createDirectorRegistry({ - factories: [workbenchFactory, defaultDirectorFactory], - defaultId: workbenchFactory.id, - }); -} diff --git a/apps/sidecar/test/adapter-registry.test.ts b/apps/sidecar/test/adapter-registry.test.ts deleted file mode 100644 index 791aba9e9e..0000000000 --- a/apps/sidecar/test/adapter-registry.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Proves the default sidecar boot -- no `SIDECAR_ADAPTER_MANIFEST` set -- -// actually registers `@corbits/ollama-adapter` for the `ollama` provider -// key, and that a seeded Ollama model's `quirks.numCtx` reaches the built -// request as `options.num_ctx`. Before this fix, `SIDECAR_ADAPTER_MANIFEST` -// defaulted to `[]`, so a default deployment resolved the built-in OpenAI -// adapter for `ollama` sources and never sent `num_ctx` at all. -import { expect, test } from "bun:test"; -import { loadAdapterRegistry } from "@intx/inference/providers"; -import type { ConversationTurn, LastCycleSource } from "@intx/types/runtime"; - -import { readSidecarConfig } from "../src/config"; - -const VALID_ENV = { - SIDECAR_DATA_DIR: "/var/lib/sidecar", - HUB_WS_URL: "wss://hub.example.com/api/sidecars/ws", - SIDECAR_ID: "sidecar-1", - SIDECAR_TOKEN: "secret-token", - PATH: "/usr/local/bin:/usr/bin", -}; - -test("a default boot (no SIDECAR_ADAPTER_MANIFEST) registers the ollama provider", async () => { - const config = readSidecarConfig(VALID_ENV); - const registry = await loadAdapterRegistry(config.adapterManifest); - - expect(registry.has("ollama")).toBe(true); -}); - -test("a seeded Ollama model's quirks.numCtx reaches the built request as options.num_ctx, with no operator configuration", async () => { - const config = readSidecarConfig(VALID_ENV); - const registry = await loadAdapterRegistry(config.adapterManifest); - - const source: LastCycleSource = { - sourceId: "src_1", - provider: "ollama", - model: "gpt-oss:20b", - }; - // Shaped exactly like `@corbits/inference-catalog`'s - // `quirksForDeployment` writes onto a catalog offering's `quirks` column. - const quirks = { default: { numCtx: 32_768 } }; - const adapter = registry.resolve(source, quirks); - - const messages: ConversationTurn[] = [ - { role: "user", content: [{ type: "text", text: "hi" }], timestamp: 0 }, - ]; - const built = adapter.buildRequest(messages, "gpt-oss:20b", {}); - const body = JSON.parse(built.body) as { options?: { num_ctx?: number } }; - - expect(body.options?.num_ctx).toBe(32_768); -}); diff --git a/apps/sidecar/test/atomic-write.test.ts b/apps/sidecar/test/atomic-write.test.ts deleted file mode 100644 index 11bdd50405..0000000000 --- a/apps/sidecar/test/atomic-write.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { writeFileAtomicDurable } from "../src/atomic-write"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-atomic-")); - tempDirs.push(dir); - return dir; -} - -test("writes the contents and applies the mode", async () => { - const dir = await makeDir(); - const target = path.join(dir, "record.json"); - await writeFileAtomicDurable(target, "{}", { mode: 0o600 }); - expect(await readFile(target, "utf8")).toBe("{}"); - const info = await stat(target); - expect(info.mode & 0o777).toBe(0o600); -}); - -test("replaces an existing file completely", async () => { - const dir = await makeDir(); - const target = path.join(dir, "record.json"); - await writeFileAtomicDurable(target, "first", { mode: 0o600 }); - await writeFileAtomicDurable(target, "second", { mode: 0o600 }); - expect(await readFile(target, "utf8")).toBe("second"); -}); - -test("leaves no temp files behind after a write", async () => { - const dir = await makeDir(); - await writeFileAtomicDurable(path.join(dir, "record.json"), "{}", { - mode: 0o600, - }); - expect(await readdir(dir)).toEqual(["record.json"]); -}); diff --git a/apps/sidecar/test/attachments-only-mail-step.test.ts b/apps/sidecar/test/attachments-only-mail-step.test.ts deleted file mode 100644 index b047355f5b..0000000000 --- a/apps/sidecar/test/attachments-only-mail-step.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -// CL-6164 pristine repro at the step-invoker seam. An event-only mail -// (`@corbits/chat`'s `workbench.agent-joined`: an empty text/plain part plus -// a JSON attachment) used to reach `agent.send` as `""` and kill the run -// with `retriesExhausted`. Upstream now decodes the mail and omits empty -// content, and the fork threads the run's `MailPartReader` to onTrigger -// bodies, so a body step receives the attachment and no `content`. -import { expect, test } from "bun:test"; - -import type { Agent, SendResult } from "@intx/agent"; -import type { AuthzCallResult } from "@intx/inference"; -import type { StepInvokeRequest } from "@intx/workflow"; -import { createWorkflowStepInvoker, type StepEnvBase } from "@intx/workflow-host"; -import type { InboundMessage, Mail } from "@intx/types/runtime"; - -const ALLOW: AuthzCallResult = { - effect: "allow", - matchingGrants: [], - resolvedBy: null, -}; - -const AGENT_JOINED = new TextEncoder().encode( - JSON.stringify({ type: "workbench.agent-joined", agentId: "agt_1" }), -); - -function eventOnlyMail(): Mail { - return { - headers: { - from: "prn_member@bench.example.test", - to: ["run_anchor@runs.example.test"], - messageId: "", - } as Mail["headers"], - rawHeaders: {}, - parts: [ - { contentType: "text/plain", ref: "part:text", text: "" }, - { - contentType: "application/json", - filename: "workbench.agent-joined.json", - disposition: "attachment", - ref: "part:event", - }, - ], - }; -} - -function stubAgent(sent: (string | InboundMessage)[]): Agent { - const result: SendResult = { - type: "reply", - reply: "ok", - turn: { role: "assistant", content: [] } as never, - }; - return { - send: async (content: string | InboundMessage) => { - sent.push(content); - return result; - }, - stream: async function* () {}, - deliver: () => undefined, - close: async () => undefined, - } as unknown as Agent; -} - -test("an attachments-only mail reaches a body step as attachments with no empty content", async () => { - const sent: (string | InboundMessage)[] = []; - const read: string[] = []; - const invoke = createWorkflowStepInvoker({ - workflowAuthorize: () => Promise.resolve(ALLOW), - buildEnv: () => Promise.resolve({} as unknown as StepEnvBase), - agentFactory: () => Promise.resolve(stubAgent(sent)), - sourcesRef: { current: {} }, - mailPartReader: { - read: (ref) => { - read.push(ref); - return Promise.resolve(AGENT_JOINED); - }, - }, - }); - - const req = { - agent: { id: "reply", instructions: "reply", tools: [] }, - input: eventOnlyMail(), - authzContext: { stepId: "reply", runId: "run_1" }, - signal: new AbortController().signal, - } as unknown as StepInvokeRequest; - - await expect(invoke(req)).resolves.toMatchObject({ - output: { reply: "ok" }, - }); - expect(read).toEqual(["part:event"]); - const message = sent[0]; - expect(typeof message).toBe("object"); - const inbound = message as InboundMessage; - expect(inbound.content).toBeUndefined(); - expect(inbound.attachments).toHaveLength(1); - expect(inbound.attachments?.[0]?.contentType).toBe("application/json"); -}); - -test("without a mail-part reader the same mail is refused loudly, never flattened to an empty string", async () => { - const sent: (string | InboundMessage)[] = []; - const invoke = createWorkflowStepInvoker({ - workflowAuthorize: () => Promise.resolve(ALLOW), - buildEnv: () => Promise.resolve({} as unknown as StepEnvBase), - agentFactory: () => Promise.resolve(stubAgent(sent)), - sourcesRef: { current: {} }, - }); - const req = { - agent: { id: "reply", instructions: "reply", tools: [] }, - input: eventOnlyMail(), - authzContext: { stepId: "reply", runId: "run_1" }, - signal: new AbortController().signal, - } as unknown as StepInvokeRequest; - - await expect(invoke(req)).rejects.toThrow(/no mail-part reader wired/); - expect(sent).toHaveLength(0); -}); diff --git a/apps/sidecar/test/body-step-grants-collapse.test.ts b/apps/sidecar/test/body-step-grants-collapse.test.ts deleted file mode 100644 index 141588d7af..0000000000 --- a/apps/sidecar/test/body-step-grants-collapse.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -// CL-6448: an onTrigger body step's tool calls authorize against the -// deployment's grants. The credentials snapshot is keyed by the PARENT -// stepOrder, so the body's own stepId (`reply`) never appears in it; -// for a single-step deployment the sole entry IS the deployment's -// grant set and the lookup collapses to it (mirroring the head/step -// collapse the body's tool materialization already uses). A multi-step -// snapshot stays strict — an unknown stepId is ambiguous and throws. -import { expect, test } from "bun:test"; - -import { createCredentialsBackedAuthorize } from "@intx/workflow-host"; - -const HEAD_GRANTS = [{ resource: "tool:@corbits/x/t", effect: "allow" }]; - -function evaluatorRecorder(seen: { grants?: readonly unknown[] }) { - return (call: { grants: readonly unknown[] }) => { - seen.grants = call.grants; - return Promise.resolve({ effect: "allow" as const }); - }; -} - -test("a body stepId absent from a single-step snapshot collapses to the sole entry's grants", async () => { - const seen: { grants?: readonly unknown[] } = {}; - const authorize = createCredentialsBackedAuthorize( - { - current: { - steps: [ - { - stepId: "turn", - address: "run_1@local", - grants: HEAD_GRANTS, - contentHash: "h", - }, - ], - }, - } as never, - evaluatorRecorder(seen) as never, - ); - - const result = await authorize("tool:@corbits/x/t", "invoke", { - stepId: "reply", - runId: "turn__3", - attempt: 1, - } as never); - - expect(result.effect).toBe("allow"); - expect(seen.grants).toBe(HEAD_GRANTS); -}); - -test("an unknown stepId against a multi-step snapshot stays a loud miss", async () => { - const authorize = createCredentialsBackedAuthorize( - { - current: { - steps: [ - { - stepId: "a", - address: "run_1-a@local", - grants: [], - contentHash: "h", - }, - { - stepId: "b", - address: "run_1-b@local", - grants: [], - contentHash: "h", - }, - ], - }, - } as never, - evaluatorRecorder({}) as never, - ); - - await expect( - authorize("tool:@corbits/x/t", "invoke", { - stepId: "reply", - runId: "r", - attempt: 1, - } as never), - ).rejects.toThrow("credentialsSnapshot has no entry for stepId reply"); -}); diff --git a/apps/sidecar/test/body-turn-request-assembly.test.ts b/apps/sidecar/test/body-turn-request-assembly.test.ts deleted file mode 100644 index e58e0ce105..0000000000 --- a/apps/sidecar/test/body-turn-request-assembly.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -// CL-6448 regression guard at the request-assembly layer. -// -// The section-body (chat turn) defect was invisible to every unit above -// the wire: the agent ran, the reply streamed, and only the outbound -// inference request showed the loss — `tools` absent and `messages` -// holding just the system prompt plus the latest user message. This -// suite pins the contract at exactly that layer: a real `createAgent` -// send with a captured `deps.fetch` asserts the literal HTTP body an -// OpenAI-compatible provider receives carries BOTH the prior -// conversation turns the context store restored AND the agent's -// declared tools. -import { expect, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { createAgent, createDefaultDirectorRegistry, defineAgent, defineTool } from "@intx/agent"; -import { createDependencies } from "@intx/inference"; -import { createBuiltinRegistry } from "@intx/inference/providers"; - -const PRIOR_TURNS = [ - { - role: "user", - content: [{ type: "text", text: "My favorite color is teal." }], - }, - { - role: "assistant", - content: [{ type: "text", text: "Noted: teal." }], - }, -]; - -function contextStoreStub() { - return { - load: () => - Promise.resolve({ - turns: PRIOR_TURNS, - pendingOperations: [], - tokenUsage: undefined, - }), - writeTurns: () => Promise.resolve(), - writePrompt: () => Promise.resolve(), - writeResponse: () => Promise.resolve(), - writeManifest: () => Promise.resolve(), - writeMetadata: () => Promise.resolve(), - writeBlob: () => Promise.resolve(), - commit: () => Promise.resolve({ commitId: "stub" }), - record: () => Promise.resolve(), - }; -} - -function sseResponse(): Response { - const chunk = { - id: "c1", - object: "chat.completion.chunk", - created: 1, - model: "stub-model", - choices: [ - { - index: 0, - delta: { role: "assistant", content: "ok" }, - finish_reason: null, - }, - ], - }; - const stop = { - id: "c1", - object: "chat.completion.chunk", - created: 1, - model: "stub-model", - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - }; - const body = [ - `data: ${JSON.stringify(chunk)}`, - "", - `data: ${JSON.stringify(stop)}`, - "", - "data: [DONE]", - "", - "", - ].join("\n"); - return new Response(body, { - status: 200, - headers: { "content-type": "text/event-stream" }, - }); -} - -test("an openai-compatible turn's wire request carries the restored history and the declared tools", async () => { - const captured: { url?: string; body?: unknown } = {}; - const fetchStub = (url: string | URL | Request, init?: RequestInit) => { - captured.url = String(url); - captured.body = JSON.parse(String(init?.body)); - return Promise.resolve(sseResponse()); - }; - - const echoDefinition = { - name: "@corbits/test-tools/echo", - description: "Echo the input back.", - inputSchema: { - type: "object", - properties: { text: { type: "string" } }, - }, - }; - const echoFactory = defineTool({ - id: "@corbits/test-tools/echo", - definitions: [echoDefinition], - factory: () => ({ - definitions: [echoDefinition], - run: (call: { callId: string }) => - Promise.resolve({ - callId: call.callId, - content: [{ type: "text" as const, text: "ok" }], - isError: false, - }), - }), - } as never); - - const definition = defineAgent({ - id: "body-turn-assembly-test", - systemPrompt: "You are the assembly-regression fixture.", - tools: [echoFactory], - capabilities: [], - inference: { sources: [{ provider: "openai", model: "stub-model" }] }, - }); - - const workdir = fs.mkdtempSync(path.join(os.tmpdir(), "cl6448-assembly-")); - const storage = contextStoreStub(); - const agent = await createAgent( - definition as never, - { - sources: [ - { - id: "src-openai-stub", - provider: "openai", - baseURL: "http://inference.stub.invalid/v1", - credentialId: "stub-key", - model: "stub-model", - }, - ], - defaultSource: "src-openai-stub", - // An `InferenceSource` names a `credentialId`; the reactor fills the - // request's secret through this resolver at send time. - readCurrentMaterial: () => ({ secret: "stub-secret" }), - storage, - audit: storage, - workdir, - directors: createDefaultDirectorRegistry(), - authorize: () => Promise.resolve({ effect: "allow" }), - deps: { - ...createDependencies(createBuiltinRegistry()), - fetch: fetchStub, - }, - } as never, - ); - - try { - await agent.send("What is my favorite color?"); - } finally { - await agent.close(); - } - - expect(captured.url).toBe("http://inference.stub.invalid/v1/chat/completions"); - const request = captured.body as { - messages: { role: string; content: unknown }[]; - tools?: { function: { name: string } }[]; - }; - - // History: every prior turn the context store restored precedes the - // new user message on the wire. - const serialized = JSON.stringify(request.messages); - expect(serialized).toContain("My favorite color is teal."); - expect(serialized).toContain("Noted: teal."); - expect(serialized).toContain("What is my favorite color?"); - const roles = request.messages.map((m) => m.role); - expect(roles[0]).toBe("system"); - expect(roles).toContain("assistant"); - - // Tools: the declared tool rides the request (name is - // provider-encoded, so match on the stable suffix). - expect(request.tools).toBeDefined(); - expect(request.tools?.length).toBe(1); - expect(request.tools?.[0]?.function.name).toContain("echo"); -}); diff --git a/apps/sidecar/test/config.test.ts b/apps/sidecar/test/config.test.ts deleted file mode 100644 index bacfebad49..0000000000 --- a/apps/sidecar/test/config.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { expect, test } from "bun:test"; -import { readSidecarConfig } from "../src/config"; - -const VALID_ENV = { - SIDECAR_DATA_DIR: "/var/lib/sidecar", - HUB_WS_URL: "wss://hub.example.com/api/sidecars/ws", - SIDECAR_ID: "sidecar-1", - SIDECAR_TOKEN: "secret-token", - PATH: "/usr/local/bin:/usr/bin", -}; - -const DEFAULT_MANIFEST = [ - { - provider: "ollama", - specifier: "@corbits/ollama-adapter", - export: "createOllamaAdapter", - }, - { - provider: "codex", - specifier: "@corbits/codex-provider", - export: "createCodexResponsesAdapter", - }, - { - provider: "xai-oauth", - specifier: "@corbits/xai-provider", - export: "createXaiResponsesAdapter", - }, -]; - -test("parses a complete environment into config", () => { - const config = readSidecarConfig(VALID_ENV); - expect(config).toEqual({ - dataDir: "/var/lib/sidecar", - hubURL: "wss://hub.example.com/api/sidecars/ws", - sidecarId: "sidecar-1", - token: "secret-token", - path: "/usr/local/bin:/usr/bin", - home: undefined, - tmpdir: undefined, - toolRegistries: undefined, - adapterManifest: DEFAULT_MANIFEST, - consumedRetentionMs: undefined, - readyTimeoutMs: undefined, - idleHibernateMs: undefined, - }); -}); - -test("an unset SIDECAR_ADAPTER_MANIFEST defaults to the shipped Ollama adapter, not an empty registry", () => { - const config = readSidecarConfig(VALID_ENV); - expect(config.adapterManifest).toEqual(DEFAULT_MANIFEST); -}); - -test("the shipped default manifest registers the loopback-OAuth Responses adapters (CL-7510)", () => { - const config = readSidecarConfig(VALID_ENV); - const byProvider = new Map(config.adapterManifest.map((entry) => [entry.provider, entry])); - expect(byProvider.get("codex")).toEqual({ - provider: "codex", - specifier: "@corbits/codex-provider", - export: "createCodexResponsesAdapter", - }); - expect(byProvider.get("xai-oauth")).toEqual({ - provider: "xai-oauth", - specifier: "@corbits/xai-provider", - export: "createXaiResponsesAdapter", - }); -}); - -test("an operator-set adapter manifest fully replaces the default rather than merging with it", () => { - const manifest = [ - { - provider: "anthropic", - specifier: "@acme/custom-anthropic-adapter", - export: "createCustomAdapter", - }, - ]; - const config = readSidecarConfig({ - ...VALID_ENV, - SIDECAR_ADAPTER_MANIFEST: JSON.stringify(manifest), - }); - expect(config.adapterManifest).toEqual(manifest); -}); - -test("carries a valid adapter manifest through as its parsed form", () => { - const manifest = [ - { - provider: "ollama", - specifier: "@corbits/ollama-adapter", - export: "createOllamaAdapter", - }, - ]; - const config = readSidecarConfig({ - ...VALID_ENV, - SIDECAR_ADAPTER_MANIFEST: JSON.stringify(manifest), - }); - expect(config.adapterManifest).toEqual(manifest); -}); - -test("a malformed adapter manifest fails boot naming the variable", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, SIDECAR_ADAPTER_MANIFEST: "{not json" })).toThrow( - /SIDECAR_ADAPTER_MANIFEST/, - ); -}); - -test("an adapter manifest entry missing a required field fails boot", () => { - expect(() => - readSidecarConfig({ - ...VALID_ENV, - SIDECAR_ADAPTER_MANIFEST: JSON.stringify([{ provider: "ollama" }]), - }), - ).toThrow(/SIDECAR_ADAPTER_MANIFEST/); -}); - -test("carries operator overrides for consumedRetentionMs/readyTimeoutMs through when present", () => { - const config = readSidecarConfig({ - ...VALID_ENV, - CONSUMED_RETENTION_MS: "3600000", - CHILD_READY_TIMEOUT_MS: "5000", - }); - expect(config.consumedRetentionMs).toBe(3_600_000); - expect(config.readyTimeoutMs).toBe(5_000); -}); - -test("a non-numeric CONSUMED_RETENTION_MS fails boot naming the variable", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, CONSUMED_RETENTION_MS: "not-a-number" })).toThrow( - /CONSUMED_RETENTION_MS/, - ); -}); - -test("a non-positive CHILD_READY_TIMEOUT_MS fails boot naming the variable", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, CHILD_READY_TIMEOUT_MS: "0" })).toThrow( - /CHILD_READY_TIMEOUT_MS/, - ); -}); - -test("carries a valid tool-registry pin through as the raw JSON", () => { - const pin = JSON.stringify([{ name: "internal", url: "https://npm.example.com" }]); - const config = readSidecarConfig({ - ...VALID_ENV, - SIDECAR_TOOL_REGISTRIES: pin, - }); - expect(config.toolRegistries).toBe(pin); -}); - -test("a malformed tool-registry pin fails boot naming the variable", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, SIDECAR_TOOL_REGISTRIES: "{not json" })).toThrow( - /SIDECAR_TOOL_REGISTRIES/, - ); -}); - -test("an empty tool-registry pin fails boot instead of defaulting to npmjs", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, SIDECAR_TOOL_REGISTRIES: "" })).toThrow( - /SIDECAR_TOOL_REGISTRIES/, - ); -}); - -test("carries the optional child-env OS facts through when present", () => { - const config = readSidecarConfig({ - ...VALID_ENV, - HOME: "/home/sidecar", - TMPDIR: "/var/tmp", - }); - expect(config.home).toBe("/home/sidecar"); - expect(config.tmpdir).toBe("/var/tmp"); -}); - -test("a missing PATH errors naming it", () => { - const env: Record = { ...VALID_ENV }; - delete env["PATH"]; - expect(() => readSidecarConfig(env)).toThrow(/PATH/); -}); - -test("ignores unrelated environment variables", () => { - const config = readSidecarConfig({ ...VALID_ENV, UNRELATED: "value" }); - expect(config.sidecarId).toBe("sidecar-1"); -}); - -test("a missing variable errors naming it", () => { - const env: Record = { ...VALID_ENV }; - delete env["SIDECAR_TOKEN"]; - expect(() => readSidecarConfig(env)).toThrow(/SIDECAR_TOKEN/); -}); - -test("an empty variable errors naming it", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, SIDECAR_ID: "" })).toThrow(/SIDECAR_ID/); -}); - -test("a non-websocket hub URL errors naming the expected shape", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, HUB_WS_URL: "https://hub.example.com" })).toThrow( - /ws:\/\/ or wss:\/\//, - ); -}); - -test("an unparseable hub URL errors", () => { - expect(() => readSidecarConfig({ ...VALID_ENV, HUB_WS_URL: "ws://" })).toThrow(/HUB_WS_URL/); -}); diff --git a/apps/sidecar/test/credential-bindings-parity.test.ts b/apps/sidecar/test/credential-bindings-parity.test.ts deleted file mode 100644 index b192b23e81..0000000000 --- a/apps/sidecar/test/credential-bindings-parity.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -// Drift guard (review follow-up, CL-6032): `apps/sidecar/src/ -// step-agent-tools.ts`'s `consumerBindings` derives a consumer's -// ResolvedCredentialBinding map from a step's live `CredentialWiring` -// (a ref-holding indirection this app owns); `@corbits/credential-providers`'s -// `deriveResolvedBindings` performs the same derivation over a plain -// `CredentialDelivery` value so tool-package tests (which must never -// import an app -- "apps stay generic; packages own the domain" runs one -// direction only) can compose the real capability without duplicating -// the loop. Both exist because the two callers hold different inputs -// (a live ref vs. a resolved value); this test proves they agree on -// every input shape rather than trusting that by inspection. -import { describe, expect, test } from "bun:test"; -import type { CredentialDelivery } from "@intx/types/sidecar"; -import { deriveResolvedBindings } from "@corbits/credential-providers"; - -import { consumerBindings, type StepCredentialContext } from "../src/step-agent-tools"; - -const CONSUMER = "tool:@corbits/granola-tools"; -const OTHER_CONSUMER = "tool:@corbits/linear-tools"; -const CREDENTIAL_ID = "cred_parity_1"; - -function delivery(secret: string): CredentialDelivery { - return { - bindings: [{ handle: "granola", credentialId: CREDENTIAL_ID, consumer: CONSUMER }], - materials: [ - { - credentialId: CREDENTIAL_ID, - providerKey: "http", - origin: "https://api.granola.ai", - secret, - }, - ], - }; -} - -function contextFor(current: CredentialDelivery | null): StepCredentialContext { - return { - wiring: { - materialRef: { current }, - resolveStepGrants: () => [], - }, - stepId: "step_parity", - }; -} - -describe("consumerBindings (sidecar) vs deriveResolvedBindings (@corbits/credential-providers)", () => { - test("agree on the bound handle's static fields for a matching consumer", () => { - const context = contextFor(delivery("s3cr3t")); - const fromSidecar = consumerBindings(context, CONSUMER); - const fromGranolaCopy = deriveResolvedBindings(delivery("s3cr3t"), CONSUMER); - - expect([...fromSidecar.keys()]).toEqual([...fromGranolaCopy.keys()]); - const sidecarBinding = fromSidecar.get("granola"); - const copyBinding = fromGranolaCopy.get("granola"); - expect(sidecarBinding?.credentialId).toBe(copyBinding?.credentialId); - expect(sidecarBinding?.providerKey).toBe(copyBinding?.providerKey); - expect(sidecarBinding?.origin).toBe(copyBinding?.origin); - }); - - test("agree on the secret readCurrentMaterial resolves, including after a rotation", () => { - const originalDelivery = delivery("original-secret"); - const ref: { current: CredentialDelivery | null } = { - current: originalDelivery, - }; - const context: StepCredentialContext = { - wiring: { materialRef: ref, resolveStepGrants: () => [] }, - stepId: "step_parity", - }; - const sidecarBinding = consumerBindings(context, CONSUMER).get("granola"); - const copyBinding = deriveResolvedBindings(originalDelivery, CONSUMER).get("granola"); - if (sidecarBinding === undefined || copyBinding === undefined) { - throw new Error("expected both derivations to bind the granola handle"); - } - expect(sidecarBinding.readCurrentMaterial().secret).toBe( - copyBinding.readCurrentMaterial().secret, - ); - - // The sidecar's binding re-reads the live ref (rotation-safe); the - // granola-tools copy captures its `delivery` argument once and is - // NOT expected to reflect a later rotation of a ref it was never - // given -- this only re-derives it from the rotated value directly, - // matching how the e2e test always re-derives per call. - const rotatedDelivery = delivery("rotated-secret"); - ref.current = rotatedDelivery; - expect(sidecarBinding.readCurrentMaterial().secret).toBe("rotated-secret"); - const copyBindingAfterRotation = deriveResolvedBindings(rotatedDelivery, CONSUMER).get( - "granola", - ); - expect(copyBindingAfterRotation?.readCurrentMaterial().secret).toBe("rotated-secret"); - }); - - test("agree on filtering out a binding for a different consumer", () => { - const d = delivery("s3cr3t"); - const context = contextFor(d); - expect(consumerBindings(context, OTHER_CONSUMER).size).toBe(0); - expect(deriveResolvedBindings(d, OTHER_CONSUMER).size).toBe(0); - }); - - test("agree on an empty delivery yielding no bindings", () => { - const context = contextFor(null); - expect(consumerBindings(context, CONSUMER).size).toBe(0); - expect(deriveResolvedBindings({ bindings: [], materials: [] }, CONSUMER).size).toBe(0); - }); -}); diff --git a/apps/sidecar/test/deploy-router.test.ts b/apps/sidecar/test/deploy-router.test.ts deleted file mode 100644 index 4bc7c10f8b..0000000000 --- a/apps/sidecar/test/deploy-router.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -// Deploy-router surface wiring: the provision-step staging branch, the -// unsupported-frame refusal, and the source-admission gate ordering -// (a deploy with an unbuildable provider is rejected before any child -// is spawned). Supervisor scheduling and child IPC are the published -// packages' concern and are not re-proven here. - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { createEd25519Crypto, generateKeyPair, verifySSHSignature } from "@intx/crypto"; -import { - createAgentKeyStore, - createAgentRepoStore as createSidecarSideRepoStore, - createSessionManager, -} from "@intx/hub-agent"; -import { createAgentRepoStore } from "@intx/hub-sessions"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import { hexEncode } from "@intx/types"; -import type { HarnessConfig, InferenceSource } from "@intx/types/runtime"; -import type { AgentDeployFrame } from "@intx/types/sidecar"; -import type { SubprocessSpawner } from "@intx/workflow-host"; -import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; -import { buildSingleStepAgentDefinition } from "@intx/workflow-deploy"; -import { - createSidecarDeployRouter, - deriveDeploymentId, - type SidecarDeployRouter, -} from "../src/workflow-host-wiring"; -import { assembleRunCredentialsSnapshot } from "../src/workflow-host-wiring/supervisor"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeDataDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-router-")); - tempDirs.push(dir); - return dir; -} - -type RouterFixture = { - router: SidecarDeployRouter; - spawnedBinaries: string[]; - rejectedSources: InferenceSource[]; -}; - -/** - * The closure a stubbed materializer evaluates to, keyed by the deployment id - * the router derives. A source-ref deploy has no inline definition on the wire, - * so a test that wants a specific definition registers it here rather than - * publishing a real package. - */ -const closureDefinitions = new Map(); - -async function makeRouter(dataDir: string): Promise { - const signingKey = await generateKeyPair(); - const substrate = createAgentRepoStore({ dataDir, signingKey }); - const repoStore = createSidecarSideRepoStore({ dataDir }); - const sessions = createSessionManager({ repoStore }); - const keyStore = createAgentKeyStore({ - dataDir, - generateKeyPair, - verifySSHSig: verifySSHSignature, - }); - const spawnedBinaries: string[] = []; - const recordingSpawner: SubprocessSpawner = ({ binaryPath }) => { - spawnedBinaries.push(binaryPath); - throw new Error("test spawner refuses to launch a real child"); - }; - const rejectedSources: InferenceSource[] = []; - const router = createSidecarDeployRouter({ - sessions, - keyStore, - transport: createInMemoryTransport(), - repoStore: substrate.repoStore, - signingKeySeed: signingKey.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: (source) => { - if (source.provider === "unbuildable") { - rejectedSources.push(source); - throw new Error(`provider ${source.provider} is not registered`); - } - }, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, - // Stand in for the real fetch + SRI-verify + layout + evaluate pass: a - // test cannot publish a package, so the pinned code's evaluation result is - // registered by deployment id instead. - materializeDeploymentClosure: ({ deploymentId }) => { - const definition = closureDefinitions.get(deploymentId); - if (definition === undefined) { - throw new Error(`test closure materializer: no definition registered for ${deploymentId}`); - } - return Promise.resolve({ - definition, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - }); - }, - multistepSubprocessSpawner: recordingSpawner, - multistepBinaryPath: path.join(dataDir, "workflow-child-sentinel"), - }); - return { router, spawnedBinaries, rejectedSources }; -} - -function makeHarnessConfig(agentAddress: string): HarnessConfig { - return { - sessionId: "session-1", - agentId: "agent-1", - tenantId: "org-1", - principalId: "principal-1", - agentAddress, - systemPrompt: "", - tools: [], - grants: [], - sources: [], - defaultSource: "source-1", - }; -} - -/** - * The source-ref pin every workflow frame now carries. Its `closure` is never - * fetched in these tests -- the injected materializer answers from - * `closureDefinitions` -- so an empty frozen manifest is the honest fixture. - */ -const SOURCE_REF: NonNullable["sourceRef"] = { - source: { kind: "registry", registry: "npm" }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, -}; - -/** - * Register the definition the pinned closure evaluates to for `agentAddress` - * and return the live shape the router's projection gate runs against. - */ -function stageClosureDefinition(agentAddress: string, stepOrder: string[]): void { - const steps: Record> = {}; - for (const stepId of stepOrder) { - steps[stepId] = step({ - agent: buildSingleStepAgentDefinition({ - id: stepId, - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }); - } - closureDefinitions.set( - deriveDeploymentId(agentAddress), - defineWorkflow({ - id: "definition-1", - trigger: { type: "mail", to: "definition-1@example.com" }, - steps, - }), - ); -} - -function makeSource(provider: string): InferenceSource { - return { - id: `source-${provider}`, - provider, - baseURL: "https://inference.example.com", - credentialId: "key", - model: "model-1", - }; -} - -test("provision-step frame stages the step repo and acks a public key", async () => { - const dataDir = await makeDataDir(); - const { router, spawnedBinaries } = await makeRouter(dataDir); - const hubKey = await generateKeyPair(); - const frame: AgentDeployFrame = { - type: "agent.deploy", - agentAddress: "ins_dep_1-step-1@example.com", - agentId: "agent-1", - config: makeHarnessConfig("ins_dep_1-step-1@example.com"), - hubPublicKey: hexEncode(hubKey.publicKey), - provisionStep: true, - }; - - const result = await router.deploy(frame); - - expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - // Provisioning spawns nothing; the deployment-level workflow frame does. - expect(spawnedBinaries).toEqual([]); - expect(router.activeAddresses()).toEqual([]); -}); - -test("a frame with neither provisionStep nor workflow is refused", async () => { - const dataDir = await makeDataDir(); - const { router } = await makeRouter(dataDir); - const hubKey = await generateKeyPair(); - const frame: AgentDeployFrame = { - type: "agent.deploy", - agentAddress: "ins_dep_1@example.com", - agentId: "agent-1", - config: makeHarnessConfig("ins_dep_1@example.com"), - hubPublicKey: hexEncode(hubKey.publicKey), - }; - - await expect(router.deploy(frame)).rejects.toThrow(/provisionStep or a workflow definition/); -}); - -test("an unbuildable inference provider rejects the deploy before any spawn", async () => { - const dataDir = await makeDataDir(); - const { router, spawnedBinaries, rejectedSources } = await makeRouter(dataDir); - const hubKey = await generateKeyPair(); - const frame: AgentDeployFrame = { - type: "agent.deploy", - agentAddress: "ins_dep_1@example.com", - agentId: "agent-1", - config: makeHarnessConfig("ins_dep_1@example.com"), - hubPublicKey: hexEncode(hubKey.publicKey), - workflow: { - sources: { "step-1": [makeSource("unbuildable")] }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }, - }; - - stageClosureDefinition("ins_dep_1@example.com", ["step-1"]); - - await expect(router.deploy(frame)).rejects.toThrow(/not registered/); - expect(rejectedSources).toHaveLength(1); - expect(spawnedBinaries).toEqual([]); - expect(router.activeAddresses()).toEqual([]); -}); - -// The deploy frame no longer carries a definition, so its arktype `narrow` -// cannot check that the sources table covers every step. That coverage is now -// checked against the CLOSURE-derived definition, after the apply. -test("a closure-derived definition whose step has no sources entry is refused", async () => { - const dataDir = await makeDataDir(); - const { router, spawnedBinaries } = await makeRouter(dataDir); - const hubKey = await generateKeyPair(); - const frame: AgentDeployFrame = { - type: "agent.deploy", - agentAddress: "ins_dep_1@example.com", - agentId: "agent-1", - config: makeHarnessConfig("ins_dep_1@example.com"), - hubPublicKey: hexEncode(hubKey.publicKey), - workflow: { - sources: {}, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }, - }; - - stageClosureDefinition("ins_dep_1@example.com", ["step-1"]); - - await expect(router.deploy(frame)).rejects.toThrow(/sources/); - expect(spawnedBinaries).toEqual([]); -}); - -// `spawn` replays any mail already sitting in the deployment's inbox, which -// births the self-anchored run immediately -- before the hub's own -// `run.grants` frame can arrive, since that is only sent once the deploy -// acks. The deploy must therefore put the run's grants on disk itself, or -// every wake with mail pending fails its `onRunStart` barrier closed and the -// agent goes silent. The spawner here throws, so reaching the assertion at -// all proves the write lands BEFORE the spawn. -test("a single-step deploy writes the self-anchored run's grants before spawning", async () => { - const dataDir = await makeDataDir(); - const { router } = await makeRouter(dataDir); - const hubKey = await generateKeyPair(); - const agentAddress = `run_${"a1".repeat(16)}@example.com`; - const runId = `run_${"a1".repeat(16)}`; - const grant = { - id: "grant_1", - resource: "tool:@corbits/catalog-tools:models_list", - action: "invoke", - effect: "allow" as const, - origin: "system" as const, - conditions: null, - expiresAt: null, - roleId: null, - principalId: "principal-1", - }; - const frame: AgentDeployFrame = { - type: "agent.deploy", - agentAddress, - agentId: "agent-1", - config: { ...makeHarnessConfig(agentAddress), grants: [grant] }, - hubPublicKey: hexEncode(hubKey.publicKey), - workflow: { - sources: { "step-1": [makeSource("openai")] }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }, - }; - - stageClosureDefinition(agentAddress, ["step-1"]); - - await expect(router.deploy(frame)).rejects.toThrow(/refuses to launch a real child/); - - const substrate = createAgentRepoStore({ - dataDir, - signingKey: await generateKeyPair(), - }); - const snapshot = await assembleRunCredentialsSnapshot({ - repoStore: substrate.repoStore, - deploymentId: deriveDeploymentId(agentAddress), - runId, - stepOrder: ["step-1"], - deriveStepAddress: () => agentAddress, - }); - expect(snapshot.steps).toHaveLength(1); - expect(snapshot.steps[0]?.grants).toEqual([grant]); -}); diff --git a/apps/sidecar/test/host-boot.test.ts b/apps/sidecar/test/host-boot.test.ts deleted file mode 100644 index 45cb92ec47..0000000000 --- a/apps/sidecar/test/host-boot.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -// End-to-end boot check: the host process starts against a stub hub, -// dials in over WebSocket, and exits cleanly on SIGTERM. The stub is a -// bare WebSocket acceptor -- the host's first register frame proves the -// dial-in reached the application layer, not just the socket. - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { expect, test } from "bun:test"; - -const APP_ROOT = path.join(import.meta.dir, ".."); - -function requireTestPath(): string { - const value = process.env["PATH"]; - if (value === undefined) { - throw new Error("PATH missing from the test environment"); - } - return value; -} - -test("host dials in against a stub hub and exits cleanly on SIGTERM", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "sidecar-boot-")); - const registered = Promise.withResolvers(); - const server = Bun.serve({ - port: 0, - fetch(request, srv) { - if (srv.upgrade(request)) return undefined; - return new Response("expected a websocket upgrade", { status: 400 }); - }, - websocket: { - open() {}, - message(_ws, message) { - registered.resolve(typeof message === "string" ? message : message.toString()); - }, - }, - }); - try { - const proc = Bun.spawn(["bun", "src/index.ts"], { - cwd: APP_ROOT, - env: { - PATH: requireTestPath(), - SIDECAR_DATA_DIR: dataDir, - HUB_WS_URL: `ws://localhost:${String(server.port)}/api/sidecars/ws`, - SIDECAR_ID: "sidecar-under-test", - SIDECAR_TOKEN: "token-under-test", - }, - stdout: "pipe", - stderr: "pipe", - }); - - const firstFrame = await registered.promise; - expect(firstFrame).toContain("register"); - - proc.kill("SIGTERM"); - const exitCode = await proc.exited; - expect(exitCode).toBe(0); - } finally { - server.stop(true); - await rm(dataDir, { recursive: true, force: true }); - } -}, 20_000); - -test("host refuses to boot when the environment is incomplete, naming the variable", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "sidecar-boot-")); - try { - const proc = Bun.spawn(["bun", "src/index.ts"], { - cwd: APP_ROOT, - env: { - PATH: requireTestPath(), - SIDECAR_DATA_DIR: dataDir, - HUB_WS_URL: "ws://localhost:9/api/sidecars/ws", - SIDECAR_ID: "sidecar-under-test", - }, - stdout: "pipe", - stderr: "pipe", - }); - const exitCode = await proc.exited; - const stderr = await new Response(proc.stderr).text(); - expect(exitCode).not.toBe(0); - expect(stderr).toContain("SIDECAR_TOKEN"); - } finally { - await rm(dataDir, { recursive: true, force: true }); - } -}, 20_000); diff --git a/apps/sidecar/test/hub-link-watchdog.test.ts b/apps/sidecar/test/hub-link-watchdog.test.ts deleted file mode 100644 index 9afe26f174..0000000000 --- a/apps/sidecar/test/hub-link-watchdog.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { expect, test } from "bun:test"; -import { createHubLinkWatchdog } from "../src/hub-link-watchdog"; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -test("fires onStall when an armed deadline expires without progress", async () => { - let stalled = 0; - const watchdog = createHubLinkWatchdog({ - stallDeadlineMs: 20, - onStall: () => { - stalled += 1; - }, - }); - watchdog.armForBoot(); - await sleep(40); - expect(stalled).toBe(1); -}); - -test("markAlive disarms the deadline", async () => { - let stalled = 0; - const watchdog = createHubLinkWatchdog({ - stallDeadlineMs: 20, - onStall: () => { - stalled += 1; - }, - }); - watchdog.armForBoot(); - watchdog.markAlive(); - await sleep(40); - expect(stalled).toBe(0); -}); - -test("a scheduled reconnect re-arms the deadline before running", async () => { - let stalled = 0; - let ran = 0; - const watchdog = createHubLinkWatchdog({ - stallDeadlineMs: 20, - onStall: () => { - stalled += 1; - }, - }); - watchdog.scheduleReconnect(() => { - ran += 1; - }, 5); - await sleep(50); - expect(ran).toBe(1); - expect(stalled).toBe(1); -}); - -test("a cancelled reconnect never runs and never arms", async () => { - let stalled = 0; - let ran = 0; - const watchdog = createHubLinkWatchdog({ - stallDeadlineMs: 20, - onStall: () => { - stalled += 1; - }, - }); - const cancel = watchdog.scheduleReconnect(() => { - ran += 1; - }, 5); - cancel(); - await sleep(50); - expect(ran).toBe(0); - expect(stalled).toBe(0); -}); diff --git a/apps/sidecar/test/shutdown.test.ts b/apps/sidecar/test/shutdown.test.ts deleted file mode 100644 index cbe5424b3e..0000000000 --- a/apps/sidecar/test/shutdown.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { expect, test } from "bun:test"; -import { - attachShutdownRejectionHandler, - drainWithTimeout, - runSidecarShutdown, -} from "../src/shutdown"; - -function capturingLog() { - const errors: string[] = []; - return { - errors, - info(_strings: TemplateStringsArray, ..._values: unknown[]): void {}, - error(strings: TemplateStringsArray, ...values: unknown[]): void { - errors.push(String.raw({ raw: strings }, ...values)); - }, - }; -} - -test("resolves drained when the drain completes inside the bound", async () => { - const outcome = await drainWithTimeout(() => Promise.resolve(), 1_000); - expect(outcome).toEqual({ kind: "drained" }); -}); - -test("resolves timed-out when the drain outlives the bound", async () => { - const outcome = await drainWithTimeout(() => new Promise(() => undefined), 10); - expect(outcome).toEqual({ kind: "timed-out" }); -}); - -test("resolves failed with the thrown error when the drain throws", async () => { - const error = new Error("drain fault"); - const outcome = await drainWithTimeout(() => Promise.reject(error), 1_000); - expect(outcome).toEqual({ kind: "failed", error }); -}); - -test("a throw from close exits 1 and does not reject", async () => { - const exits: number[] = []; - const log = capturingLog(); - let drained = false; - const promise = runSidecarShutdown({ - signal: "SIGTERM", - close: () => { - throw new Error("close failed"); - }, - drain: async () => { - drained = true; - }, - drainTimeoutMs: 1_000, - exit: (code) => { - exits.push(code); - }, - log, - }); - await expect(promise).resolves.toBeUndefined(); - expect(exits).toEqual([1]); - expect(drained).toBe(false); - expect(log.errors.some((line) => line.includes("close failed"))).toBe(true); -}); - -test("a drain failure reports through reportError and exits 1 without rejecting", async () => { - const exits: number[] = []; - const error = new Error("drain fault"); - const reported: unknown[] = []; - const contexts: unknown[] = []; - const promise = runSidecarShutdown({ - signal: "SIGINT", - close: () => undefined, - drain: async () => { - throw error; - }, - drainTimeoutMs: 1_000, - exit: (code) => { - exits.push(code); - }, - log: capturingLog(), - report: (reportedError, context) => { - reported.push(reportedError); - contexts.push(context); - return "ref-test"; - }, - }); - await expect(promise).resolves.toBeUndefined(); - expect(exits).toEqual([1]); - expect(reported).toEqual([error]); - expect(contexts[0]).toMatchObject({ - operation: "sidecar.shutdown", - }); -}); - -test("a clean drain exits 0", async () => { - const exits: number[] = []; - const promise = runSidecarShutdown({ - signal: "SIGTERM", - close: () => undefined, - drain: async () => undefined, - drainTimeoutMs: 1_000, - exit: (code) => { - exits.push(code); - }, - log: capturingLog(), - }); - await expect(promise).resolves.toBeUndefined(); - expect(exits).toEqual([0]); -}); - -test("a rejected shutdown promise is caught by the signal handler binding", async () => { - const unhandled: unknown[] = []; - const onUnhandled = (reason: unknown) => { - unhandled.push(reason); - }; - process.on("unhandledRejection", onUnhandled); - const reported: unknown[] = []; - try { - attachShutdownRejectionHandler(Promise.reject(new Error("escaped")), (error) => { - reported.push(error); - }); - await Bun.sleep(20); - expect(reported).toHaveLength(1); - expect(unhandled).toEqual([]); - } finally { - process.off("unhandledRejection", onUnhandled); - } -}); diff --git a/apps/sidecar/test/signing-keypair.test.ts b/apps/sidecar/test/signing-keypair.test.ts deleted file mode 100644 index 0758a68692..0000000000 --- a/apps/sidecar/test/signing-keypair.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { loadOrMintSidecarKeypair } from "../src/signing-keypair"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeSigningDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-signing-")); - tempDirs.push(dir); - return path.join(dir, ".sidecar-signing"); -} - -test("mints and persists a keypair on first boot", async () => { - const signingDir = await makeSigningDir(); - const keyPair = await loadOrMintSidecarKeypair(signingDir); - expect(keyPair.privateKey.length).toBe(32); - expect(keyPair.publicKey.length).toBe(32); - const persisted = await readFile(path.join(signingDir, "ed25519.private")); - expect(new Uint8Array(persisted)).toEqual(new Uint8Array(keyPair.privateKey)); -}); - -test("loads the same identity on a second boot", async () => { - const signingDir = await makeSigningDir(); - const first = await loadOrMintSidecarKeypair(signingDir); - const second = await loadOrMintSidecarKeypair(signingDir); - expect(second.publicKey).toEqual(first.publicKey); - expect(second.privateKey).toEqual(first.privateKey); -}); - -test("halts when the public anchor does not match the seed", async () => { - const signingDir = await makeSigningDir(); - await loadOrMintSidecarKeypair(signingDir); - await writeFile(path.join(signingDir, "ed25519.public"), new Uint8Array(32).fill(7)); - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/does not match/); -}); - -test("halts on a partial keypair directory", async () => { - const signingDir = await makeSigningDir(); - await loadOrMintSidecarKeypair(signingDir); - await rm(path.join(signingDir, "ed25519.public")); - await expect(loadOrMintSidecarKeypair(signingDir)).rejects.toThrow(/partial/); -}); diff --git a/apps/sidecar/test/support/workflow-lifecycle-fixture.ts b/apps/sidecar/test/support/workflow-lifecycle-fixture.ts deleted file mode 100644 index 9fb3c5b881..0000000000 --- a/apps/sidecar/test/support/workflow-lifecycle-fixture.ts +++ /dev/null @@ -1,470 +0,0 @@ -// Shared test scaffolding for exercising `createSidecarDeployRouter` -// end-to-end against a mock workflow-process child: an in-memory -// NDJSON control channel, an in-memory event-channel frame stream, and a -// minimal `RepoStore` stub that answers only the calls the deploy router's -// grants bridge and credentials snapshot assembly make. Shared by every -// test that exercises `createSidecarDeployRouter` end-to-end so none of -// them reinvent the mock child handshake. - -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { createEd25519Crypto, generateKeyPair } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; -import { agentDir } from "@intx/hub-agent"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import type { RepoId, RepoStore } from "@intx/hub-sessions"; -import { - createControlChannelSender, - type FrameReader, - type NdjsonReader, - type NdjsonWriter, - type SubprocessHandle, - type SubprocessSpawner, -} from "@intx/workflow-host"; -import type { AgentDeployFrame } from "@intx/types/sidecar"; -import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; -import { buildSingleStepAgentDefinition } from "@intx/workflow-deploy"; - -import { - createSidecarDeployRouter, - type SidecarDeployRouter, -} from "../../src/workflow-host-wiring"; -import { - createMultistepCredentialsRouter, - createMultistepDrainRouter, - createMultistepMailRouter, - createMultistepSignalRouter, - createMultistepSourcesRouter, - type MultistepCredentialsRouter, - type MultistepDrainRouter, - type MultistepMailRouter, - type MultistepSignalRouter, - type MultistepSourcesRouter, -} from "../../src/workflow-run-pack-client"; - -export function createMemoryNdjsonStream() { - const buffer: string[] = []; - let waiter: (() => void) | null = null; - let done = false; - function wake() { - const w = waiter; - waiter = null; - if (w) w(); - } - const reader: NdjsonReader = { - read(): AsyncIterableIterator { - return (async function* () { - while (true) { - if (buffer.length > 0) { - const next = buffer.shift(); - if (next === undefined) { - throw new Error("buffer shift returned undefined"); - } - yield next; - continue; - } - if (done) return; - await new Promise((resolve) => { - waiter = resolve; - }); - } - })(); - }, - }; - const writer: NdjsonWriter = { - write(line: string) { - buffer.push(line.replace(/\n$/, "")); - wake(); - return Promise.resolve(); - }, - }; - return { - writer, - reader, - inject(line: string) { - buffer.push(line.replace(/\n$/, "")); - wake(); - }, - close() { - done = true; - wake(); - }, - }; -} - -export function createMemoryFrameStream() { - const buffer: Uint8Array[] = []; - let waiter: (() => void) | null = null; - let done = false; - function wake() { - const w = waiter; - waiter = null; - if (w) w(); - } - const reader: FrameReader = { - read(): AsyncIterableIterator { - return (async function* () { - while (true) { - if (buffer.length > 0) { - const next = buffer.shift(); - if (next === undefined) { - throw new Error("frame buffer shift returned undefined"); - } - yield next; - continue; - } - if (done) return; - await new Promise((resolve) => { - waiter = resolve; - }); - } - })(); - }, - }; - return { - reader, - close() { - done = true; - wake(); - }, - }; -} - -export function createSpawnTestRepoStore(tempBase: string): RepoStore { - const stub: Partial = { - getRepoDir(repoId: RepoId): string { - return path.join(tempBase, repoId.kind, repoId.id); - }, - async writeTreePreservingPrefix(_p, _id, _ref, args) { - await args.merge(new Map()); - return { commitSha: "stub-sha", newlyTerminalRuns: [] }; - }, - // The deploy router's grants bridge writes `state/grants.json` to - // each step's agent-state repo before `spawn()`. Mirror the - // `getRepoDir` layout so the write lands where the subsequent - // `assembleCredentialsSnapshot` working-tree read looks for it. - async writeTree(_p, repoId, _ref, content) { - const dir = path.join(tempBase, repoId.kind, repoId.id); - for (const [relPath, contents] of Object.entries(content.files)) { - const full = path.join(dir, relPath); - await fs.mkdir(path.dirname(full), { recursive: true }); - await fs.writeFile(full, contents); - } - return { commitSha: "stub-sha", newlyTerminalRuns: [] }; - }, - }; - // Boundary type assertion: test stub - return new Proxy(stub as RepoStore, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); - if (value !== undefined) return value; - return () => { - throw new Error(`stub RepoStore: ${String(prop)} not implemented for this test`); - }; - }, - }); -} - -// Per-spawn tracking the mock spawner records. -export type Spawn = { - handle: SubprocessHandle; - childToSupervisor: ReturnType; - supervisorToChild: ReturnType; - eventChildToSupervisor: ReturnType; - /** Every line the supervisor wrote to the child's control channel, in order. */ - sentControlLines: string[]; - env: Record; - killed: boolean; - exitedResolved: boolean; - resolveExited: (code: number) => void; -}; - -export type Fixture = { - router: SidecarDeployRouter; - spawns: Spawn[]; - dataDir: string; - repoStore: RepoStore; - multistepMailRouter: MultistepMailRouter; - multistepSignalRouter: MultistepSignalRouter; - multistepDrainRouter: MultistepDrainRouter; - multistepSourcesRouter: MultistepSourcesRouter; - multistepCredentialsRouter: MultistepCredentialsRouter; -}; - -/** - * A `SubprocessSpawner` whose child never signals `ready` on its own -- - * `answerReadyHandshake` drives that half explicitly -- so a caller - * controls exactly when a mocked `wired.supervisor.spawn()` call resolves. - * Exported (not just used internally by `makeLifecycleFixture`) so a test - * that builds its own router directly (rather than through the fixture) - * can still exercise a real spawn/ready handshake instead of a spawner - * that throws or resolves synchronously. - */ -export function createHandshakeSpawner(spawns: Spawn[]): SubprocessSpawner { - return ({ env }) => { - const supervisorToChild = createMemoryNdjsonStream(); - const childToSupervisor = createMemoryNdjsonStream(); - const eventChildToSupervisor = createMemoryFrameStream(); - let resolveExit: ((code: number) => void) | undefined; - const exited = new Promise((resolve) => { - resolveExit = resolve; - }); - const entry: Spawn = { - // Boundary type assertion: assigned below - handle: undefined as unknown as SubprocessHandle, - supervisorToChild, - childToSupervisor, - eventChildToSupervisor, - sentControlLines: [], - env, - killed: false, - exitedResolved: false, - resolveExited: (code) => { - entry.exitedResolved = true; - resolveExit?.(code); - }, - }; - const handle: SubprocessHandle = { - pid: 5100 + spawns.length, - controlWriter: { - async write(line: string) { - entry.sentControlLines.push(line); - await supervisorToChild.writer.write(line); - }, - }, - controlReader: childToSupervisor.reader, - eventReader: eventChildToSupervisor.reader, - kill: () => { - entry.killed = true; - childToSupervisor.close(); - eventChildToSupervisor.close(); - entry.resolveExited(0); - }, - exited, - }; - entry.handle = handle; - spawns.push(entry); - return handle; - }; -} - -export async function makeLifecycleFixture(opts?: { - /** - * Reuse a prior fixture's data dir, so a test can construct a SECOND - * router against the SAME on-disk state to model a sidecar process - * restart (boot-time restore). - */ - dataDir?: string; - /** - * Override the default in-memory fake key store with a real one (e.g. - * `@intx/hub-agent`'s `createAgentKeyStore` bound to the same - * `dataDir`), for a test that needs the actual on-disk key-persistence - * behavior the fake bypasses entirely. - */ - keyStore?: Parameters[0]["keyStore"]; - /** - * Override the default single-step `LIFECYCLE_CLOSURE_DEFINITION` closure - * materializer -- e.g. with a multi-step definition, for a test that needs - * the eager (non-deferred-to-wake) boot-restore path. - */ - materializeDeploymentClosure?: Parameters< - typeof createSidecarDeployRouter - >[0]["materializeDeploymentClosure"]; - /** - * Override the boot-restore attempt's timeout ceiling, so a test can - * exercise the timeout path in milliseconds instead of the real 30s - * default. - */ - restoreAttemptTimeoutMs?: number; -}): Promise { - const spawns: Spawn[] = []; - const spawner = createHandshakeSpawner(spawns); - - const transport = createInMemoryTransport(); - const keyPair = await generateKeyPair(); - const tempBase = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-lifecycle-repos-")); - const dataDir = - opts?.dataDir ?? (await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-lifecycle-data-"))); - const repoStore = createSpawnTestRepoStore(tempBase); - - const multistepMailRouter = createMultistepMailRouter(); - const multistepSignalRouter = createMultistepSignalRouter(); - const multistepDrainRouter = createMultistepDrainRouter(); - const multistepSourcesRouter = createMultistepSourcesRouter(); - const multistepCredentialsRouter = createMultistepCredentialsRouter(); - - const router = createSidecarDeployRouter({ - // Boundary type assertion: the single-step branch invokes only initRepo (head deploy-tree repo); provisionAgent/persistHubPublicKey stay unused (the supervised child mints its own key and persists no hub-agent config) - sessions: { - provisionAgent: async () => { - throw new Error("single-step branch must not invoke provisionAgent"); - }, - persistHubPublicKey: async () => { - throw new Error("single-step branch must not invoke persistHubPublicKey"); - }, - initRepo: async () => undefined, - } as unknown as Parameters[0]["sessions"], - // Boundary type assertion: the single-step branch registers the agent's signing key (loadOrGenerateKey) and records the hub key (recordHubKey) at the head before spawn - keyStore: - opts?.keyStore ?? - ({ - recordHubKey: () => undefined, - forgetAgent: () => undefined, - // Mirrors the real `AgentKeyStore`'s on-disk contract just enough - // for `hibernated-agent-identity-vault.ts`'s snapshot step to find - // a real `agentDir` to preserve: a fixture-driven hibernate - // teardown must not spuriously trip its "ordering broke" report. - loadOrGenerateKey: async (address: string) => { - await fs.mkdir(agentDir(dataDir, address), { recursive: true }); - return { keyPair: await generateKeyPair(), isNew: false }; - }, - } as unknown as Parameters[0]["keyStore"]), - transport, - repoStore, - signingKeySeed: keyPair.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: () => undefined, - registerDeployment: () => { - /* no-op */ - }, - unregisterDeployment: () => { - /* no-op */ - }, - multistepSubprocessSpawner: spawner, - multistepSubstrateEnv: { - SIDECAR_DATA_DIR: dataDir, - }, - // Stand in for the real fetch + SRI-verify + layout + evaluate pass: a - // fixture cannot publish a package, so every deploy evaluates to the one - // lifecycle definition below. - materializeDeploymentClosure: - opts?.materializeDeploymentClosure ?? - (({ deploymentId }) => - Promise.resolve({ - definition: LIFECYCLE_CLOSURE_DEFINITION, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - })), - multistepMailRouter, - multistepSignalRouter, - multistepDrainRouter, - multistepSourcesRouter, - multistepCredentialsRouter, - ...(opts?.restoreAttemptTimeoutMs !== undefined - ? { restoreAttemptTimeoutMs: opts.restoreAttemptTimeoutMs } - : {}), - }); - return { - router, - spawns, - dataDir, - repoStore, - multistepMailRouter, - multistepSignalRouter, - multistepDrainRouter, - multistepSourcesRouter, - multistepCredentialsRouter, - }; -} - -/** - * The definition the fixture's stubbed closure evaluates to. Source-ref is the - * only deploy lineage, so a frame carries no inline definition and this is the - * single source of the deployment's shape. - */ -const LIFECYCLE_CLOSURE_DEFINITION: WorkflowDefinition = defineWorkflow({ - id: "wf-lifecycle", - trigger: { type: "mail", to: "wf-lifecycle@example.com" }, - steps: { - "step-1": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-1", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - }, -}); - -/** The wire hash the hub approved for `LIFECYCLE_CLOSURE_DEFINITION`. */ -export const LIFECYCLE_APPROVED_WIRE_HASH = "d".repeat(64); - -export function makeWorkflowFrame(agentAddress: string): AgentDeployFrame { - return { - type: "agent.deploy", - // Single-step projection: the deploy router derives the sole - // step's agent-state repo from `parseAgentId(agentAddress)`, which - // requires the canonical `ins_@` instance shape. - agentAddress, - agentId: "lifecycle-agent", - hubPublicKey: "hub-pk", - // Boundary type assertion: the multi-step branch does not read most of - // `config` -- but `tenantId`/`principalId` ride the deploy frame's - // config into the v2 deployment record (CL-8184), so a record written - // from this fixture must carry real values or every subsequent - // `readWorkflowDeploymentRecord` fails schema validation and silently - // returns undefined. - config: { - tenantId: "tenant-lifecycle-fixture", - principalId: "principal-lifecycle-fixture", - } as AgentDeployFrame["config"], - workflow: { - approvedWireHash: LIFECYCLE_APPROVED_WIRE_HASH, - sourceRef: { - source: { kind: "registry", registry: "npm" }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, - }, - sources: { - "step-1": [ - { - id: "step-1", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - credentialId: "sk-step-1", - model: "claude-3-5", - }, - ], - }, - }, - }; -} - -/** - * Drive the mock child's half of the supervisor's spawn handshake: wait - * for the spawner to fire, then send a signed `ready` envelope over the - * recorded control channel so `supervisor.spawn()` resolves. - */ -export async function answerReadyHandshake(spawns: Spawn[], at: number) { - while (spawns.length <= at) { - await new Promise((r) => setTimeout(r, 1)); - } - const spawn = spawns[at]; - if (spawn === undefined) throw new Error("unreachable"); - const channelId = spawn.env.IPC_CHANNEL_ID; - if (channelId === undefined) { - throw new Error("IPC_CHANNEL_ID missing from spawn env"); - } - const childIpcKeyPair = await generateKeyPair(); - const childSender = createControlChannelSender({ - privateKeySeed: childIpcKeyPair.privateKey, - channelId, - writer: { - write(line: string) { - spawn.childToSupervisor.inject(line); - return Promise.resolve(); - }, - }, - }); - await childSender.send({ - type: "ready", - data: { - childPid: spawn.handle.pid, - childPublicKey: hexEncode(childIpcKeyPair.publicKey), - }, - }); - return spawn; -} diff --git a/apps/sidecar/test/tool-materialization.test.ts b/apps/sidecar/test/tool-materialization.test.ts deleted file mode 100644 index d3c8076b29..0000000000 --- a/apps/sidecar/test/tool-materialization.test.ts +++ /dev/null @@ -1,365 +0,0 @@ -import { describe, test, expect, afterEach } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { - DEFAULT_TOOL_REGISTRIES_JSON, - clearDirtyMarker, - materializeToolPackages, - parseActiveDeployId, - parseToolRegistries, - persistActiveDeployIdWithFallback, -} from "../src/tool-materialization"; -import { SIDECAR_SUBSTRATE_CONFIG_KEYS } from "../src/workflow-substrate-factory"; - -const tempDirs: string[] = []; - -async function tempDir(): Promise { - const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), "default-harness-test-")); - tempDirs.push(d); - return d; -} - -afterEach(async () => { - const dirs = tempDirs.splice(0); - await Promise.all(dirs.map((d) => fs.promises.rm(d, { recursive: true, force: true }))); -}); - -describe("materializeToolPackages — manifest.invalid gate", () => { - test("returns empty factories when no manifest bytes supplied", async () => { - const storeDir = await tempDir(); - const result = await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: undefined, - storeDir, - agentAddress: "agent-no-manifest", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - }); - expect(result.factories).toEqual([]); - expect(result.pluginFactories).toEqual([]); - }); - - test("JSON.parse failure throws and persists the raw bytes verbatim", async () => { - const storeDir = await tempDir(); - const corrupt = "{this is not valid json"; - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: corrupt, - storeDir, - agentAddress: "agent-corrupt", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - expect(String(caught)).toMatch(/JSON\.parse failed/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const attemptDir = path.join(auditRoot, attemptDirName); - - // The manifest file must contain the exact corrupt bytes, not a - // JSON-encoded string literal wrapping them — the audit-trail - // writer must not push the bytes through JSON.stringify, which - // would double-encode the string and erase the original-input - // evidence the audit file exists to preserve. - const persisted = await fs.promises.readFile(path.join(attemptDir, "manifest.json"), "utf-8"); - expect(persisted).toBe(corrupt); - - const errorJson = JSON.parse( - await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), - ); - expect(errorJson.category).toBe("manifest.invalid"); - expect(errorJson.message).toMatch(/JSON\.parse failed/); - expect(errorJson.previousDeployId).toBe("none"); - expect(typeof errorJson.attemptId).toBe("string"); - expect(errorJson.attemptId.length).toBeGreaterThan(0); - }); - - test("schema failure throws and persists the raw bytes verbatim", async () => { - const storeDir = await tempDir(); - // Valid JSON but wrong shape — arktype rejects it. - const wrongShape = JSON.stringify({ schemaVersion: 99 }); - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: wrongShape, - storeDir, - agentAddress: "agent-schema", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - expect(String(caught)).toMatch(/schema validation failed/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const attemptDir = path.join(auditRoot, attemptDirName); - - // The audit trail records the bytes-as-supplied for both - // failure modes, so a future investigator can replay the - // same input against a newer validator. - const persisted = await fs.promises.readFile(path.join(attemptDir, "manifest.json"), "utf-8"); - expect(persisted).toBe(wrongShape); - - const errorJson = JSON.parse( - await fs.promises.readFile(path.join(attemptDir, "error.json"), "utf-8"), - ); - expect(errorJson.category).toBe("manifest.invalid"); - }); - - describe("parseActiveDeployId", () => { - test("accepts a v1-prefixed id", () => { - expect(parseActiveDeployId("v1:abc-123", "/dummy")).toBe("abc-123"); - }); - - test("accepts a bare id for backward compatibility with pre-versioned files", () => { - expect(parseActiveDeployId("legacy-deploy-id", "/dummy")).toBe("legacy-deploy-id"); - }); - - test("rejects an unknown version prefix loudly", () => { - expect(() => parseActiveDeployId("v2:abc", "/dummy")).toThrow(/unknown version prefix/); - }); - - test("rejects a v1-prefixed but empty id", () => { - expect(() => parseActiveDeployId("v1:", "/dummy")).toThrow(/carries the v1 prefix but no id/); - }); - - test("rejects an empty file", () => { - expect(() => parseActiveDeployId("", "/dummy")).toThrow(/is empty/); - }); - }); - - describe("persistActiveDeployIdWithFallback", () => { - test("returns degraded=false and removes a stale dirty marker on full success", async () => { - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const dirtyFile = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyFile, "v1:stale-id"); - - const outcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - "fresh-id", - ); - expect(outcome.degraded).toBe(false); - const recorded = await fs.promises.readFile(activeIdFile, "utf-8"); - expect(recorded).toBe("v1:fresh-id"); - // The dirty marker must be cleared so the next boot reads the - // freshly-recorded id directly, not the stale marker that - // pre-dated the successful persist. - let dirtyExists: boolean; - try { - await fs.promises.access(dirtyFile); - dirtyExists = true; - } catch { - dirtyExists = false; - } - expect(dirtyExists).toBe(false); - }); - - test("the degraded-persist code path clears a stale dirty marker via the shared helper", async () => { - // The fsync'd primary persist and the no-fsync fallback persist - // both delegate to `clearDirtyMarker` after writing the recorded - // id. The fallback branch is hard to drive end-to-end without - // injecting an `fs` failure mode that distinguishes - // `fs.promises.open(path, "w")` from `fs.promises.writeFile(path)` - // — they share flags and permissions on every supported FS — so - // the contract is pinned at the helper level: when called against - // an instance dir that holds a stale marker, the marker is gone - // afterward. The wrapper's no-fsync branch routes through this - // helper, which gives the boot reader the freshly-recorded id on - // the next boot. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - const dirtyFile = `${activeIdFile}.dirty`; - await fs.promises.writeFile(dirtyFile, "v1:stale-from-prior-apply"); - await clearDirtyMarker(activeIdFile, "test"); - let dirtyExists: boolean; - try { - await fs.promises.access(dirtyFile); - dirtyExists = true; - } catch { - dirtyExists = false; - } - expect(dirtyExists).toBe(false); - }); - - test("clearing a missing dirty marker is a no-op", async () => { - // ENOENT is the normal case when no prior apply wrote a marker. - // The helper must not throw or log when there is nothing to - // remove; the wrapper relies on the helper being safe to call - // unconditionally on every successful persist. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - await clearDirtyMarker(activeIdFile, "test"); - }); - - test("writes the dirty marker when the primary persist cannot open the file", async () => { - // Simulate a primary persist failure by making activeIdFile itself - // a directory. Both the fsync'd open() and the no-fsync - // writeFile() will then fail with EISDIR, exercising the dirty- - // marker path. - const instanceDir = await tempDir(); - const activeIdFile = path.join(instanceDir, "active-deploy-id"); - await fs.promises.mkdir(activeIdFile); - - const outcome = await persistActiveDeployIdWithFallback( - instanceDir, - activeIdFile, - "new-id-42", - ); - expect(outcome.degraded).toBe(true); - expect(outcome.error).toBeInstanceOf(Error); - - const dirtyContents = await fs.promises.readFile(`${activeIdFile}.dirty`, "utf-8"); - expect(dirtyContents).toBe("v1:new-id-42"); - }); - }); - - test("boot reconciliation records the dirty marker as previousDeployId", async () => { - // Seed an instance where the prior apply could not durably record - // the committed deploy id and a dirty marker carries the truth. - // The next apply (any apply — here, a manifest-invalid one) must - // surface the marker's id as `previousDeployId` in the audit - // record, not the stale recorded id and not "none". - const storeDir = await tempDir(); - const instanceDir = path.join(storeDir, "tool-packages"); - await fs.promises.mkdir(instanceDir, { recursive: true }); - await fs.promises.writeFile(path.join(instanceDir, "active-deploy-id"), "v1:stale-recorded"); - await fs.promises.writeFile( - path.join(instanceDir, "active-deploy-id.dirty"), - "v1:truth-from-marker", - ); - - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: "{ not json", - storeDir, - agentAddress: "agent-dirty-marker", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - const attemptDirName = attemptDirs[0]; - if (attemptDirName === undefined) { - throw new Error("no attempt dir was created"); - } - const errorJson = JSON.parse( - await fs.promises.readFile(path.join(auditRoot, attemptDirName, "error.json"), "utf-8"), - ); - expect(errorJson.previousDeployId).toBe("truth-from-marker"); - }); - - test("manifest.invalid throws and writes an audit entry", async () => { - const storeDir = await tempDir(); - let caught: unknown; - try { - await materializeToolPackages({ - assetMounts: new Map(), - rawManifestBytes: "{ not json", - storeDir, - agentAddress: "agent-throws", - cacheRoot: "/unused-by-manifest-invalid-gate", - cacheMaxBytes: 1024 * 1024, - registryMaxTarballBytes: 10 * 1024 * 1024, - registries: new Map([["npmjs", { url: "https://registry.npmjs.org" }]]), - }); - } catch (err) { - caught = err; - } - expect(caught).toBeInstanceOf(Error); - expect(String(caught)).toMatch(/manifest\.invalid/); - - const auditRoot = path.join(storeDir, "audit", "rejected-applies"); - const attemptDirs = await fs.promises.readdir(auditRoot); - expect(attemptDirs).toHaveLength(1); - }); -}); - -describe("tool-registry threading", () => { - test("SIDECAR_TOOL_REGISTRIES is a child substrate-config key, so the boot edge's registries reach per-step materialization", () => { - expect(SIDECAR_SUBSTRATE_CONFIG_KEYS).toContain("SIDECAR_TOOL_REGISTRIES"); - }); - - test("the default payload the boot edge serializes parses to the public npmjs registry", () => { - const registries = parseToolRegistries(DEFAULT_TOOL_REGISTRIES_JSON); - expect([...registries.entries()]).toEqual([["npmjs", { url: "https://registry.npmjs.org" }]]); - }); - - test("parseToolRegistries collapses the wire array into a map keyed by name", () => { - const registries = parseToolRegistries( - JSON.stringify([ - { - name: "internal", - url: "https://npm.example.com", - auth: { token: "t" }, - }, - { name: "npmjs", url: "https://registry.npmjs.org" }, - ]), - ); - expect(registries.get("internal")).toEqual({ - url: "https://npm.example.com", - auth: { token: "t" }, - }); - expect(registries.get("npmjs")).toEqual({ - url: "https://registry.npmjs.org", - }); - }); - - test("an empty value fails loud instead of falling back to public npmjs", () => { - expect(() => parseToolRegistries("")).toThrow(/SIDECAR_TOOL_REGISTRIES/); - }); - - test("malformed JSON, wrong shapes, and duplicate names fail loud", () => { - expect(() => parseToolRegistries("{not json")).toThrow(/not valid JSON/); - expect(() => parseToolRegistries(JSON.stringify({ npmjs: {} }))).toThrow(/failed validation/); - expect(() => - parseToolRegistries( - JSON.stringify([ - { name: "dup", url: "https://a.example.com" }, - { name: "dup", url: "https://b.example.com" }, - ]), - ), - ).toThrow(/duplicate registry name/); - }); -}); diff --git a/apps/sidecar/test/workflow-deploy-lifecycle.test.ts b/apps/sidecar/test/workflow-deploy-lifecycle.test.ts deleted file mode 100644 index 067c53ed5c..0000000000 --- a/apps/sidecar/test/workflow-deploy-lifecycle.test.ts +++ /dev/null @@ -1,299 +0,0 @@ -// Workflow deployment lifecycle through OUR wiring: an accepted -// workflow frame spawns a supervised child and becomes addressable; the -// process-exit drain (`shutdownAll`) kills the child but preserves every -// piece of durable state a later boot restores from; `undeploy` reclaims -// the deployment's records and per-step scratch while the durable -// conversation root survives BOTH -- it is never deleted, so a re-deploy -// on the same address resumes the prior conversation. Supervisor -// scheduling and child IPC internals are the published packages' -// concern and are not re-proven here; the mock spawner below stands in -// for the child, completing only the signed ready handshake the -// supervisor requires. - -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import path from "node:path"; - -import { deriveDeploymentId } from "../src/workflow-host-wiring"; -import { - answerReadyHandshake, - makeLifecycleFixture, - makeWorkflowFrame, - LIFECYCLE_APPROVED_WIRE_HASH, -} from "./support/workflow-lifecycle-fixture"; - -describe("workflow deployment lifecycle through the deploy router", () => { - test("a deploy frame carrying referencedDefinitions stages each body's sources.json and no body definition", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-bodies@example.com"); - if (frame.workflow === undefined) throw new Error("unreachable"); - const bodySources = { - "body-step": [ - { - id: "body-step", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - credentialId: "sk-body", - model: "claude-3-5", - }, - ], - }; - const bodyDefinition = { - id: "wf-lifecycle-body", - triggers: [{ type: "manual" }], - stepOrder: ["body-step"], - steps: { "body-step": { kind: "step" } }, - }; - frame.workflow.referencedDefinitions = [{ definition: bodyDefinition, sources: bodySources }]; - - const deployPromise = router.deploy(frame); - await answerReadyHandshake(spawns, 0); - await deployPromise; - - // A body child runs in-process and loses its env across a restart, so its - // per-step source pins must be durable on disk. - const bodyDir = path.join(dataDir, "assets", "workflow", bodyDefinition.id); - expect(JSON.parse(await fs.readFile(path.join(bodyDir, "sources.json"), "utf8"))).toEqual( - bodySources, - ); - - // The body DEFINITION is never staged: the run child resolves each body - // in-memory from the parent's re-verified closure. A staged copy would be - // a second, un-verified source of the body's bytes. - await expect(fs.stat(path.join(bodyDir, "workflow.json"))).rejects.toThrow(); - await expect( - fs.stat(path.join(dataDir, "assets", "workflow", "wf-lifecycle")), - ).rejects.toThrow(); - }); - - test("a deploy threads the materialized closure dir and the hub-approved hash to the child, and persists the pin for restore", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-closure@example.com"); - - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - const deploymentId = deriveDeploymentId(frame.agentAddress); - // The child EVALUATES the pinned code from this dir rather than reading an - // inert definition off disk, and re-verifies its projection against the - // hub-approved hash -- never a sidecar recompute. - expect(spawn.env.CLOSURE_PACKAGE_DIR).toBe(path.join(dataDir, "closure-package", deploymentId)); - expect(spawn.env.DEFINITION_HASH).toBe(LIFECYCLE_APPROVED_WIRE_HASH); - expect(spawn.env.WORKFLOW_DEFINITION_REF).toBeUndefined(); - expect(spawn.env.REFERENCED_DEFINITION_HASHES).toBeUndefined(); - - // The record carries what a boot-time restore needs to re-materialize the - // same closure and re-verify it against the same anchor. - const record = JSON.parse( - await fs.readFile( - path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"), - "utf8", - ), - ); - expect(record.approvedWireHash).toBe(LIFECYCLE_APPROVED_WIRE_HASH); - expect(record.sourceRef.source).toEqual({ - kind: "registry", - registry: "npm", - }); - }); - - test("a workflow frame is accepted: the child spawns, the address goes live, and a durable record lands", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-accept@example.com"); - - const deployPromise = router.deploy(frame); - await answerReadyHandshake(spawns, 0); - const result = await deployPromise; - - // The ack carries the supervisor principal's verifying key. - expect(result.publicKey).toMatch(/^[0-9a-f]{64}$/); - // The deployment is addressable: the boot edge announces exactly - // this set to the hub on every (re)connect. - expect(router.activeAddresses()).toEqual([frame.agentAddress]); - // The durable deployment record the next boot restores from. - const deploymentId = deriveDeploymentId(frame.agentAddress); - const recordFile = path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); - await expect(fs.stat(recordFile)).resolves.toBeDefined(); - }); - - test("shutdownAll drains the child but preserves the deployment record and conversation root", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-drain@example.com"); - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - const deploymentId = deriveDeploymentId(frame.agentAddress); - const recordFile = path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); - const durableConversationFile = path.join( - dataDir, - "agent-conversation-state", - deploymentId, - encodeURIComponent("step-1"), - "checkpoint.json", - ); - await fs.mkdir(path.dirname(durableConversationFile), { recursive: true }); - await fs.writeFile(durableConversationFile, "x"); - - await router.shutdownAll(); - - // The child is gone and the address is no longer announced... - expect(spawn.killed).toBe(true); - expect(spawn.exitedResolved).toBe(true); - expect(router.activeAddresses()).toEqual([]); - // ...but every durable restore source survives: the record the next - // boot's restore re-spawns from, and the conversation the respawned - // warm agent resumes. - await expect(fs.stat(recordFile)).resolves.toBeDefined(); - expect(await fs.readFile(durableConversationFile, "utf8")).toBe("x"); - }); - - test("undeploy kills the child, reclaims scratch and record, and never deletes the conversation root", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-undeploy@example.com"); - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - expect(spawn.killed).toBe(false); - expect(spawn.exitedResolved).toBe(false); - - const undeploy = router.undeploy; - if (undeploy === undefined) { - throw new Error("router.undeploy is undefined"); - } - - // Pre-seed the on-disk per-step scratch the child roots under - // `/workflow-step-state//` and the durable - // conversation under `/agent-conversation-state//`. - // The warm subtree is the stable per-agent workspace (one dir, not - // one-per-message); a stale cold `runs//` subtree models a - // multi-step leftover the per-run cleanup did not drop. An unrelated - // deployment's step-state subtree must survive the undeploy sweep. - const deploymentId = deriveDeploymentId(frame.agentAddress); - const stepStateRoot = path.join(dataDir, "workflow-step-state"); - const warmWorkspaceFile = path.join( - stepStateRoot, - deploymentId, - "warm", - encodeURIComponent("step-1"), - "workspace", - "notes.txt", - ); - const coldLeftoverFile = path.join( - stepStateRoot, - deploymentId, - "runs", - "run-stale", - "steps", - "step-1", - "attempt-1", - "workspace", - "scratch.txt", - ); - const otherDeploymentFile = path.join( - stepStateRoot, - "other-deployment", - "warm", - "step-1", - "workspace", - "keep.txt", - ); - const durableConversationFile = path.join( - dataDir, - "agent-conversation-state", - deploymentId, - encodeURIComponent("step-1"), - "checkpoint.json", - ); - for (const file of [ - warmWorkspaceFile, - coldLeftoverFile, - otherDeploymentFile, - durableConversationFile, - ]) { - await fs.mkdir(path.dirname(file), { recursive: true }); - await fs.writeFile(file, "x"); - } - - await undeploy({ - type: "agent.undeploy", - agentAddress: frame.agentAddress, - reason: "test undeploy", - }); - - expect(spawn.killed).toBe(true); - expect(spawn.exitedResolved).toBe(true); - expect(router.activeAddresses()).toEqual([]); - - // The deployment's whole step-state subtree is reclaimed -- warm - // stable workspace AND any cold leftover -- now that its supervisor - // and child are torn down. - await expect(fs.stat(path.join(stepStateRoot, deploymentId))).rejects.toThrow(); - // So is the deployment record: an undeployed deployment must not be - // restored on the next boot. - await expect( - fs.stat(path.join(dataDir, "workflow-runs", deploymentId, "deployment.json")), - ).rejects.toThrow(); - // A different deployment's scratch is untouched: the sweep is scoped - // to this deployment's `` subtree only. - expect(await fs.readFile(otherDeploymentFile, "utf8")).toBe("x"); - // The durable conversation lives under a DIFFERENT root and must - // survive so a re-deploy restores the prior conversation. - expect(await fs.readFile(durableConversationFile, "utf8")).toBe("x"); - }); - - // CL-6192: the hub's `credentials.update` frame (a rotation, or a - // revocation) must reach the resident workflow-process child. Before this - // port, no `MultistepCredentialsRouter` handler was ever installed, so an - // inbound `credentials.update` frame was unrouted for every deployment. - test("a hub credentials.update frame reaches the child as a credentials-updated control frame", async () => { - const { router, spawns, multistepCredentialsRouter } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_lifecycle-credentials@example.com"); - - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - const delivery = { - bindings: [ - { - handle: "mcp.exa", - credentialId: "cred_1", - consumer: "tool:@corbits/mcp-tools", - }, - ], - materials: [ - { - credentialId: "cred_1", - providerKey: "http", - origin: "https://mcp.exa.ai/mcp", - secret: "rotated-secret", - }, - ], - }; - - const routed = await multistepCredentialsRouter.tryRoute({ - type: "credentials.update", - agentAddress: frame.agentAddress, - delivery, - }); - - expect(routed).toBe(true); - const credentialsUpdatedLine = spawn.sentControlLines.find( - (line) => - (JSON.parse(line) as { envelope: { payload: { type: string } } }).envelope.payload.type === - "credentials-updated", - ); - if (credentialsUpdatedLine === undefined) { - throw new Error( - "expected a credentials-updated control frame after routing credentials.update", - ); - } - const parsed = JSON.parse(credentialsUpdatedLine) as { - envelope: { payload: { type: string; data: { delivery: unknown } } }; - }; - expect(parsed.envelope.payload.data.delivery).toEqual(delivery); - }); -}); diff --git a/apps/sidecar/test/workflow-grants-barrier.test.ts b/apps/sidecar/test/workflow-grants-barrier.test.ts deleted file mode 100644 index 0002950661..0000000000 --- a/apps/sidecar/test/workflow-grants-barrier.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -// The per-run grants barrier fails closed. Every run birth path produces a -// `run.grants` frame (the hub's `deployAtHead` sends it once the deploy acks), -// so a run reaching `assembleRunCredentialsSnapshot` with no -// `runs//grants.json` is a run that would start under-authorized, not a -// run legitimately inheriting deploy-time grants. This pins that a missing -// file throws rather than resolving to an empty snapshot, and that a present -// file is applied uniformly across the deployment's steps. - -import { describe, expect, test } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { assembleRunCredentialsSnapshot } from "../src/workflow-host-wiring/supervisor"; -import { writeStepGrants } from "../src/workflow-host-wiring/step-strategy"; -import { createSpawnTestRepoStore } from "./support/workflow-lifecycle-fixture"; - -const DEPLOYMENT_ID = "dep-barrier"; -const STEP_ORDER = ["step-a", "step-b"]; -const deriveStepAddress = ({ stepId }: { stepId: string }) => `${DEPLOYMENT_ID}-${stepId}@local`; - -// Wrapped in an object: the stub store is a Proxy that throws on any -// unknown property, so returning it bare from an async function makes the -// promise machinery probe `then` and blow up. -async function makeRepoStore() { - const tempBase = await fs.mkdtemp(path.join(os.tmpdir(), "grants-barrier-")); - return { store: createSpawnTestRepoStore(tempBase) }; -} - -describe("assembleRunCredentialsSnapshot", () => { - test("throws when the run has no grants file", async () => { - const { store: repoStore } = await makeRepoStore(); - - await expect( - assembleRunCredentialsSnapshot({ - repoStore, - deploymentId: DEPLOYMENT_ID, - runId: "run-missing", - stepOrder: STEP_ORDER, - deriveStepAddress, - }), - ).rejects.toThrow(/has no grants file; failing closed/); - }); - - test("applies a written run's grants across every step", async () => { - const { store: repoStore } = await makeRepoStore(); - const grants = [ - { - id: "grant_1", - resource: "tool:@corbits/mcp-tools:search", - action: "invoke", - effect: "allow" as const, - origin: "system" as const, - conditions: null, - expiresAt: null, - roleId: null, - principalId: "prn_1", - }, - ]; - - await writeStepGrants({ - repoStore, - deploymentId: DEPLOYMENT_ID, - stepOrder: STEP_ORDER, - // Inert in per-run mode: `writeStepGrants` with a `runId` writes the - // single `runs//grants.json` into the deployment's workflow-run - // repo rather than fanning out to per-step repos. - deriveStepRepoId: ({ stepId }) => ({ - kind: "agent-state" as const, - id: `${DEPLOYMENT_ID}-${stepId}`, - }), - grants, - runId: "run-present", - }); - - const snapshot = await assembleRunCredentialsSnapshot({ - repoStore, - deploymentId: DEPLOYMENT_ID, - runId: "run-present", - stepOrder: STEP_ORDER, - deriveStepAddress, - }); - - expect(snapshot.steps.map((step) => step.stepId)).toEqual(STEP_ORDER); - for (const step of snapshot.steps) { - expect(step.grants).toEqual(grants); - expect(step.address).toBe(deriveStepAddress({ stepId: step.stepId })); - } - expect(new Set(snapshot.steps.map((step) => step.contentHash)).size).toBe(1); - }); -}); diff --git a/apps/sidecar/test/workflow-restore-defer-to-wake.test.ts b/apps/sidecar/test/workflow-restore-defer-to-wake.test.ts deleted file mode 100644 index 285fe5bca9..0000000000 --- a/apps/sidecar/test/workflow-restore-defer-to-wake.test.ts +++ /dev/null @@ -1,268 +0,0 @@ -// CL-6648: a single-step ("warm-keep") deployment's persisted `sources` -// is only ever a snapshot of what resolved against the tenant catalog at -// its last deploy or rotation. Restoring one eagerly from that snapshot -// at boot would replay a chain whose credential died after the freeze -// forever -- the deployment reads as "already live" to every later wake -// check, so the lazy-wake path (`ensureAwake` -> `@corbits/chat`'s -// `wakeByAddress`, which DOES re-resolve fresh against the live catalog -// on every call) never gets a chance to heal it. -// -// The fix: boot-time restore defers a single-step deployment to that -// wake path instead of restoring it from frozen sources -- proven here by -// asserting the mock spawner is never invoked and the dead source is -// never even reaches `assertSourceBuildable`. A true multi-step -// deployment has no such wake port, so it must keep restoring eagerly -// from its frozen sources exactly as before -- proven by the sibling -// test below with an unchanged buildable-source multi-step record. - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { createEd25519Crypto, generateKeyPair, verifySSHSignature } from "@intx/crypto"; -import { - createAgentKeyStore, - createAgentRepoStore as createSidecarSideRepoStore, - createSessionManager, -} from "@intx/hub-agent"; -import { createAgentRepoStore } from "@intx/hub-sessions"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import type { InferenceSource } from "@intx/types/runtime"; -import type { SubprocessSpawner } from "@intx/workflow-host"; -import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; -import { buildSingleStepAgentDefinition } from "@intx/workflow-deploy"; -import { - createSidecarDeployRouter, - deriveDeploymentId, - type SidecarDeployRouter, -} from "../src/workflow-host-wiring"; -import { - readWorkflowDeploymentRecord, - writeWorkflowDeploymentRecord, - type WorkflowDeploymentRecord, -} from "../src/workflow-deployment-record"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeDataDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-restore-defer-")); - tempDirs.push(dir); - return dir; -} - -const closureDefinitions = new Map(); - -async function makeRouter( - dataDir: string, - opts: { - assertSourceBuildable: (source: InferenceSource) => void; - onSpawn: () => void; - }, -): Promise { - const signingKey = await generateKeyPair(); - const substrate = createAgentRepoStore({ dataDir, signingKey }); - const repoStore = createSidecarSideRepoStore({ dataDir }); - const sessions = createSessionManager({ repoStore }); - const keyStore = createAgentKeyStore({ - dataDir, - generateKeyPair, - verifySSHSig: verifySSHSignature, - }); - const recordingSpawner: SubprocessSpawner = () => { - opts.onSpawn(); - throw new Error("test spawner refuses to launch a real child"); - }; - return createSidecarDeployRouter({ - sessions, - keyStore, - transport: createInMemoryTransport(), - repoStore: substrate.repoStore, - signingKeySeed: signingKey.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: opts.assertSourceBuildable, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, - materializeDeploymentClosure: ({ deploymentId }) => { - const definition = closureDefinitions.get(deploymentId); - if (definition === undefined) { - throw new Error(`test closure materializer: no definition registered for ${deploymentId}`); - } - return Promise.resolve({ - definition, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - }); - }, - multistepSubprocessSpawner: recordingSpawner, - multistepBinaryPath: path.join(dataDir, "workflow-child-sentinel"), - }); -} - -function stageSingleStepClosureDefinition(deploymentId: string): void { - closureDefinitions.set( - deploymentId, - defineWorkflow({ - id: "definition-1", - trigger: { type: "mail", to: "definition-1@example.com" }, - steps: { - "step-1": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-1", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - }, - }), - ); -} - -function stageMultiStepClosureDefinition(deploymentId: string): void { - closureDefinitions.set( - deploymentId, - defineWorkflow({ - id: "definition-multi", - trigger: { type: "mail", to: "definition-multi@example.com" }, - steps: { - "step-1": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-1", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - "step-2": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-2", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - }, - }), - ); -} - -function makeDeadSource(provider: string): InferenceSource { - return { - id: `source-${provider}`, - provider, - baseURL: "https://api.example.com", - credentialId: "sk-dead-key-from-before-a-provider-reconfigure", - model: "model-1", - }; -} - -const SOURCE_REF: WorkflowDeploymentRecord["sourceRef"] = { - source: { kind: "registry", registry: "npm" }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, -}; - -test("a single-step deployment's frozen dead source is never replayed at restore -- it defers to the wake path instead", async () => { - const dataDir = await makeDataDir(); - let assertSourceBuildableCalls = 0; - let spawnCalls = 0; - const agentAddress = "run_dana-myra-frozen-anthropic@example.com"; - const deploymentId = deriveDeploymentId(agentAddress); - stageSingleStepClosureDefinition(deploymentId); - - const router = await makeRouter(dataDir, { - assertSourceBuildable: () => { - assertSourceBuildableCalls += 1; - }, - onSpawn: () => { - spawnCalls += 1; - }, - }); - - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress, - definitionId: "def_1", - // A single-step deployment's `sources` table has exactly one entry, - // per `validateWorkflowProjection`'s own wire invariant -- this is - // the dead, pre-reconfigure Anthropic chain the ticket describes. - sources: { "step-1": [makeDeadSource("anthropic")] }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); - - await router.restoreWorkflowDeployments(); - - // Never even reached the source-admission gate or a spawn attempt -- - // the dead source's buildability was never checked because the - // deployment was deferred before it was consulted at all. - expect(assertSourceBuildableCalls).toBe(0); - expect(spawnCalls).toBe(0); - expect(router.activeAddresses()).not.toContain(agentAddress); - - // The record is untouched: no restore failure was recorded (this was - // never attempted, let alone failed), and the frozen sources are left - // exactly as they were for the next wake to overwrite once it - // re-resolves against the live catalog. - const onDisk = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(onDisk?.restoreFailure).toBeUndefined(); - expect(onDisk?.sources).toEqual(record.sources); -}); - -test("a multi-step deployment still restores eagerly from its frozen sources, unchanged", async () => { - const dataDir = await makeDataDir(); - let assertSourceBuildableCalls = 0; - let spawnCalls = 0; - const agentAddress = "run_multi-step-unchanged@example.com"; - const deploymentId = deriveDeploymentId(agentAddress); - stageMultiStepClosureDefinition(deploymentId); - - const router = await makeRouter(dataDir, { - assertSourceBuildable: () => { - assertSourceBuildableCalls += 1; - }, - onSpawn: () => { - spawnCalls += 1; - }, - }); - - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress, - definitionId: "def_multi", - sources: { - "step-1": [makeDeadSource("buildable-1")], - "step-2": [makeDeadSource("buildable-2")], - }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); - - await router.restoreWorkflowDeployments(); - - // Every source in the frozen chain was gated for buildability, and the - // (mock) spawn was attempted -- multi-step restore behavior is - // unaffected by CL-6648's single-step deferral. - expect(assertSourceBuildableCalls).toBe(2); - expect(spawnCalls).toBe(1); - - // The mock spawner throws, so the restore attempt fails (as it did - // before this change) -- transient, since a plain thrown `Error` from - // the spawn core is the default classification. - const onDisk = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(onDisk?.restoreFailure?.kind).toBe("transient"); - expect(onDisk?.sources).toEqual(record.sources); -}); diff --git a/apps/sidecar/test/workflow-restore-quarantine.test.ts b/apps/sidecar/test/workflow-restore-quarantine.test.ts deleted file mode 100644 index 35e9c49efb..0000000000 --- a/apps/sidecar/test/workflow-restore-quarantine.test.ts +++ /dev/null @@ -1,333 +0,0 @@ -// Boot-time restore's failure accounting: a record whose failure is -// PERMANENT (deterministic, intrinsic to the record's own persisted -// bytes -- here, an address that derives a different slug than its -// on-disk directory) stops being attempted after -// RESTORE_QUARANTINE_THRESHOLD consecutive boots and collapses into a -// one-line summary; a record whose failure is TRANSIENT (this boot's -// environment -- here, an inference provider the sidecar cannot build) -// is retried every boot forever, however many times it has failed. Real -// child spawning is out of scope here (see workflow-deploy-lifecycle.test.ts); -// this only exercises the restore loop's classification and skip logic. - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { createEd25519Crypto, generateKeyPair, verifySSHSignature } from "@intx/crypto"; -import { - createAgentKeyStore, - createAgentRepoStore as createSidecarSideRepoStore, - createSessionManager, -} from "@intx/hub-agent"; -import { createAgentRepoStore } from "@intx/hub-sessions"; -import { createInMemoryTransport } from "@intx/mail-memory"; -import type { InferenceSource } from "@intx/types/runtime"; -import type { SubprocessSpawner } from "@intx/workflow-host"; -import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; -import { buildSingleStepAgentDefinition } from "@intx/workflow-deploy"; -import { - createSidecarDeployRouter, - deriveDeploymentId, - type SidecarDeployRouter, -} from "../src/workflow-host-wiring"; -import { - RESTORE_QUARANTINE_THRESHOLD, - readWorkflowDeploymentRecord, - writeWorkflowDeploymentRecord, - type WorkflowDeploymentRecord, -} from "../src/workflow-deployment-record"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeDataDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-restore-")); - tempDirs.push(dir); - return dir; -} - -const closureDefinitions = new Map(); -// CL-6640 fixture: deployment ids in this set fail materialization with -// the exact wrapped-ENOENT shape `readPackageJSON` (`@intx/workflow-host`) -// throws when a closure's staged `package.json` is missing. -const closureMissingStagingDir = new Set(); - -async function makeRouter( - dataDir: string, - opts: { assertSourceBuildable: (source: InferenceSource) => void }, -): Promise { - const signingKey = await generateKeyPair(); - const substrate = createAgentRepoStore({ dataDir, signingKey }); - const repoStore = createSidecarSideRepoStore({ dataDir }); - const sessions = createSessionManager({ repoStore }); - const keyStore = createAgentKeyStore({ - dataDir, - generateKeyPair, - verifySSHSig: verifySSHSignature, - }); - const recordingSpawner: SubprocessSpawner = () => { - throw new Error("test spawner refuses to launch a real child"); - }; - return createSidecarDeployRouter({ - sessions, - keyStore, - transport: createInMemoryTransport(), - repoStore: substrate.repoStore, - signingKeySeed: signingKey.privateKey, - createAgentCrypto: createEd25519Crypto, - assertSourceBuildable: opts.assertSourceBuildable, - registerDeployment: () => undefined, - unregisterDeployment: () => undefined, - multistepSubstrateEnv: { SIDECAR_DATA_DIR: dataDir }, - materializeDeploymentClosure: ({ deploymentId }) => { - if (closureMissingStagingDir.has(deploymentId)) { - const rawEnoent = new Error( - "ENOENT: no such file or directory, open '.../store/@workbench-seed/last-30-days-research/0.0.0/package.json'", - ) as Error & { code: string }; - rawEnoent.code = "ENOENT"; - throw new Error( - `cannot read package.json for workflow package at ${dataDir}/workflow-definition-closures/${deploymentId}/packages/some-fresh-uuid/store/@workbench-seed/last-30-days-research/0.0.0`, - { cause: rawEnoent }, - ); - } - const definition = closureDefinitions.get(deploymentId); - if (definition === undefined) { - throw new Error(`test closure materializer: no definition registered for ${deploymentId}`); - } - return Promise.resolve({ - definition, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - }); - }, - multistepSubprocessSpawner: recordingSpawner, - multistepBinaryPath: path.join(dataDir, "workflow-child-sentinel"), - }); -} - -// Two steps, deliberately: CL-6648 defers a SINGLE-step ("warm-keep") -// deployment's boot-time restore to its wake path instead of restoring it -// from a frozen `sources` snapshot (see `workflow-restore-defer-to-wake.test.ts`), -// so this file's transient/permanent classification scenarios below need a -// record shape that still goes through the (unchanged) eager restore path -// to actually exercise it. -function stageClosureDefinition(deploymentId: string): void { - closureDefinitions.set( - deploymentId, - defineWorkflow({ - id: "definition-1", - trigger: { type: "mail", to: "definition-1@example.com" }, - steps: { - "step-1": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-1", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - "step-2": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-2", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - }, - }), - ); -} - -function makeSource(provider: string): InferenceSource { - return { - id: `source-${provider}`, - provider, - baseURL: "https://inference.example.com", - credentialId: "key", - model: "model-1", - }; -} - -const SOURCE_REF: WorkflowDeploymentRecord["sourceRef"] = { - source: { kind: "registry", registry: "npm" }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, -}; - -test("a permanently unrestorable record is quarantined after RESTORE_QUARANTINE_THRESHOLD boots and stops being attempted", async () => { - const dataDir = await makeDataDir(); - const router = await makeRouter(dataDir, { - assertSourceBuildable: () => undefined, - }); - - // A record filed under a directory that does not match what its own - // agentAddress derives to -- corrupt/misplaced, and deterministically so: - // no future boot ever makes `deriveDeploymentId` agree with the directory. - const mismatchedDeploymentId = "dep_mismatched-directory"; - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress: "run_permanent-case@example.com", - definitionId: "def_1", - sources: {}, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, mismatchedDeploymentId, record); - - for (let boot = 1; boot <= RESTORE_QUARANTINE_THRESHOLD; boot++) { - await router.restoreWorkflowDeployments(); - const onDisk = await readWorkflowDeploymentRecord(dataDir, mismatchedDeploymentId); - expect(onDisk?.restoreFailure?.kind).toBe("permanent"); - expect(onDisk?.restoreFailure?.attempts).toBe(boot); - } - - const quarantined = await readWorkflowDeploymentRecord(dataDir, mismatchedDeploymentId); - expect(quarantined?.restoreFailure?.attempts).toBe(RESTORE_QUARANTINE_THRESHOLD); - - // One more boot: quarantine means the attempt count does NOT move -- - // the record is skipped entirely rather than attempted and re-failed. - await router.restoreWorkflowDeployments(); - const afterQuarantineBoot = await readWorkflowDeploymentRecord(dataDir, mismatchedDeploymentId); - expect(afterQuarantineBoot?.restoreFailure?.attempts).toBe(RESTORE_QUARANTINE_THRESHOLD); - - // The record itself is never deleted -- an operator can still undeploy - // the address to reclaim it. - expect(afterQuarantineBoot?.agentAddress).toBe(record.agentAddress); -}); - -test("a transiently unbuildable provider is retried every boot and never quarantines", async () => { - const dataDir = await makeDataDir(); - let assertCalls = 0; - const router = await makeRouter(dataDir, { - assertSourceBuildable: (source) => { - assertCalls += 1; - if (source.provider === "unbuildable") { - throw new Error(`Source provider "${source.provider}" is not registered`); - } - }, - }); - - const agentAddress = "run_transient-case@example.com"; - const deploymentId = deriveDeploymentId(agentAddress); - stageClosureDefinition(deploymentId); - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress, - definitionId: "def_1", - // `step-1`'s unbuildable source throws before `step-2` is ever - // consulted -- `step-2` only needs to be present to satisfy - // `validateWorkflowProjection`'s "sources covers every stepOrder - // entry" invariant for this two-step definition. - sources: { - "step-1": [makeSource("unbuildable")], - "step-2": [makeSource("buildable")], - }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); - - const bootCount = RESTORE_QUARANTINE_THRESHOLD + 2; - for (let boot = 1; boot <= bootCount; boot++) { - await router.restoreWorkflowDeployments(); - } - - // Every boot actually reached the source-admission gate -- nothing was - // ever skipped, unlike the permanent case above. - expect(assertCalls).toBe(bootCount); - - const onDisk = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(onDisk?.restoreFailure?.kind).toBe("transient"); - expect(onDisk?.restoreFailure?.attempts).toBe(bootCount); -}); - -// CL-6640: the sidecar crash-looped on boot because an ENOENT reading a -// closure's `package.json` -- observed for a UUID staging directory -// `applyFrozenWorkflowClosure` had just minted and populated moments -// earlier -- was falling through the boot loop's default "transient" -// classification, which never quarantines, so the record was retried -// forever and (by a mechanism outside this test's scope) the process -// itself died and was relaunched. This fabricates that exact failure -// shape -- `readPackageJSON`'s wrapped-ENOENT `Error` -- from the -// closure-materialization seam and proves: (a) the boot loop classifies -// it PERMANENT and quarantines it rather than retrying forever, (b) a -// sibling record's restore is still attempted every boot, unaffected, and -// (c) `restoreWorkflowDeployments()` itself never throws -- boot -// completes either way. -test("a missing/incomplete closure staging directory quarantines as a permanent failure without stopping the boot restore loop", async () => { - const dataDir = await makeDataDir(); - const brokenAgentAddress = "run_missing-staging-dir@example.com"; - const brokenDeploymentId = deriveDeploymentId(brokenAgentAddress); - closureMissingStagingDir.add(brokenDeploymentId); - - const healthyAgentAddress = "run_sibling-case@example.com"; - const healthyDeploymentId = deriveDeploymentId(healthyAgentAddress); - stageClosureDefinition(healthyDeploymentId); - - const router = await makeRouter(dataDir, { - assertSourceBuildable: () => undefined, - }); - - const brokenRecord: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress: brokenAgentAddress, - definitionId: "def_1", - sources: {}, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, brokenDeploymentId, brokenRecord); - - const healthyRecord: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress: healthyAgentAddress, - definitionId: "def_1", - // A buildable provider clears the source-admission gate and reaches - // `spawnWorkflowDeployment`, where the test's `recordingSpawner` - // refuses to launch a real child -- a plain `Error`, classified - // "transient" by the boot loop's default. What matters for this test - // is only that this record's own restore is attempted every boot, - // independent of the broken sibling record. - sources: { - "step-1": [makeSource("buildable")], - "step-2": [makeSource("buildable")], - }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, healthyDeploymentId, healthyRecord); - - for (let boot = 1; boot <= RESTORE_QUARANTINE_THRESHOLD; boot++) { - // Boot completes without throwing -- the ENOENT never escapes the - // restore loop, however it is classified. - await expect(router.restoreWorkflowDeployments()).resolves.toBeUndefined(); - const broken = await readWorkflowDeploymentRecord(dataDir, brokenDeploymentId); - expect(broken?.restoreFailure?.kind).toBe("permanent"); - expect(broken?.restoreFailure?.attempts).toBe(boot); - - // The sibling deployment's own restore was attempted on every boot, - // unaffected by the broken record ahead of (or behind) it in the scan. - const healthy = await readWorkflowDeploymentRecord(dataDir, healthyDeploymentId); - expect(healthy?.restoreFailure?.kind).toBe("transient"); - expect(healthy?.restoreFailure?.attempts).toBe(boot); - } - - // One more boot: quarantined now, so the attempt count stops moving -- - // the record is skipped rather than re-attempted and re-failed. - await router.restoreWorkflowDeployments(); - const afterQuarantine = await readWorkflowDeploymentRecord(dataDir, brokenDeploymentId); - expect(afterQuarantine?.restoreFailure?.attempts).toBe(RESTORE_QUARANTINE_THRESHOLD); -}); diff --git a/apps/sidecar/test/workflow-restore-timeout-cancel.test.ts b/apps/sidecar/test/workflow-restore-timeout-cancel.test.ts deleted file mode 100644 index 93f248e191..0000000000 --- a/apps/sidecar/test/workflow-restore-timeout-cancel.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -// CL-7215: `withRestoreTimeout` used to race a boot-restore attempt -// against a timer and, on timeout, simply abandon the still-running -// attempt -- if it later succeeded, `spawnWorkflowDeployment` registered -// a live supervisor for an address the boot loop had already recorded as -// a restore FAILURE. This exercises the fix: a restore that overshoots -// its timeout still gets an `AbortSignal`, and once it actually finishes -// (late), it corrects its own durable record instead of leaving a live -// deployment marked failed. - -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { afterEach, expect, test } from "bun:test"; -import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; -import { buildSingleStepAgentDefinition } from "@intx/workflow-deploy"; -import type { InferenceSource } from "@intx/types/runtime"; - -import { deriveDeploymentId } from "../src/workflow-host-wiring"; -import { - readWorkflowDeploymentRecord, - writeWorkflowDeploymentRecord, - type WorkflowDeploymentRecord, -} from "../src/workflow-deployment-record"; -import { answerReadyHandshake, makeLifecycleFixture } from "./support/workflow-lifecycle-fixture"; - -const tempDirs: string[] = []; - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); -}); - -async function makeDataDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "sidecar-restore-timeout-")); - tempDirs.push(dir); - return dir; -} - -// Two steps, deliberately: a single-step ("warm-keep") record defers to its -// wake path instead of eagerly restoring (CL-6648), so it never reaches -// `spawnWorkflowDeployment` and could never exercise this timeout path. -const TWO_STEP_DEFINITION: WorkflowDefinition = defineWorkflow({ - id: "wf-restore-timeout", - trigger: { type: "mail", to: "wf-restore-timeout@example.com" }, - steps: { - "step-1": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-1", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - "step-2": step({ - agent: buildSingleStepAgentDefinition({ - id: "step-2", - systemPrompt: "", - inferencePreferences: [], - toolFactories: [], - }), - triggers: "unbounded", - }), - }, -}); - -function makeSource(): InferenceSource { - return { - id: "source-1", - provider: "buildable", - baseURL: "https://inference.example.com", - credentialId: "key", - model: "model-1", - }; -} - -const SOURCE_REF: WorkflowDeploymentRecord["sourceRef"] = { - source: { kind: "registry", registry: "npm" }, - closure: { schemaVersion: "1", topLevel: [], entries: [] }, -}; - -test("a restore that outlasts its timeout corrects its record once the spawn actually finishes, instead of leaving a live deployment marked failed", async () => { - const dataDir = await makeDataDir(); - const restoreAttemptTimeoutMs = 25; - - const { router, spawns } = await makeLifecycleFixture({ - dataDir, - restoreAttemptTimeoutMs, - materializeDeploymentClosure: ({ deploymentId }) => - Promise.resolve({ - definition: TWO_STEP_DEFINITION, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - }), - }); - - const agentAddress = "run_late-restore@example.com"; - const deploymentId = deriveDeploymentId(agentAddress); - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress, - definitionId: "def_1", - sources: { - "step-1": [makeSource()], - "step-2": [makeSource()], - }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); - - // Don't answer the ready handshake yet: `wired.supervisor.spawn()` stays - // pending well past `restoreAttemptTimeoutMs`, so the boot loop's own - // timeout fires and records this attempt as a boot failure first. - const restorePromise = router.restoreWorkflowDeployments(); - await restorePromise; - - const afterTimeout = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(afterTimeout?.restoreFailure).toBeDefined(); - expect(router.activeAddresses()).toEqual([]); - - // The underlying restore was never abandoned: answering the handshake now - // lets `spawnWorkflowDeployment` actually finish, well after the boot - // loop already gave up on it. - await answerReadyHandshake(spawns, 0); - - // Both the live registration AND the record correction are background - // continuations of the restore attempt, not something - // `restoreWorkflowDeployments()` awaits -- and the registration lands - // strictly before the correction (the correction only runs once - // `spawnWorkflowDeployment` has already resolved). Poll on the record - // correction specifically, since it's the last of the two to land. - let afterLateRestore: WorkflowDeploymentRecord | undefined; - const deadline = Date.now() + 5000; - while (Date.now() < deadline) { - afterLateRestore = await readWorkflowDeploymentRecord(dataDir, deploymentId); - if (afterLateRestore?.restoreFailure === undefined) break; - await new Promise((r) => setTimeout(r, 5)); - } - - // The deployment is genuinely live now -- the record must no longer - // claim it failed to restore. - expect(afterLateRestore?.restoreFailure).toBeUndefined(); - expect(router.activeAddresses()).toEqual([agentAddress]); -}); - -test("a reclaiming teardown racing a dangling restore's boot-failure write is not resurrected with a false restoreFailure, and the restore's late spawn does not orphan a live supervisor with no record", async () => { - // The unwind this test exercises routes through `teardownDeployment`'s - // real `drain`/kill-escalation timers (`TEARDOWN_DRAIN_DEADLINE_MS` = - // 5000ms, `CHILD_KILL_ESCALATION_MS` = 3000ms). Bun's 5000ms default test - // timeout leaves no margin against that combined worst case, so both this - // timeout and the poll deadline below give it headroom past it. - const dataDir = await makeDataDir(); - const restoreAttemptTimeoutMs = 25; - - const { router, spawns } = await makeLifecycleFixture({ - dataDir, - restoreAttemptTimeoutMs, - materializeDeploymentClosure: ({ deploymentId }) => - Promise.resolve({ - definition: TWO_STEP_DEFINITION, - packageDir: path.join(dataDir, "closure-package", deploymentId), - deployDir: path.join(dataDir, "closure-deploy", deploymentId), - }), - }); - - const agentAddress = "run_teardown-race@example.com"; - const deploymentId = deriveDeploymentId(agentAddress); - const record: WorkflowDeploymentRecord = { - version: 2, - tenantId: "ten_test", - principalId: "prin_test", - agentAddress, - definitionId: "def_1", - sources: { - "step-1": [makeSource()], - "step-2": [makeSource()], - }, - approvedWireHash: "d".repeat(64), - sourceRef: SOURCE_REF, - }; - await writeWorkflowDeploymentRecord(dataDir, deploymentId, record); - - // Don't answer the ready handshake yet: the restore's spawn stays pending - // past `restoreAttemptTimeoutMs`, leaving a window for a concurrent - // teardown to race it. - const restorePromise = router.restoreWorkflowDeployments(); - - // Wait for the restore to actually reach the spawn (i.e. the mock - // spawner has been invoked) before racing a teardown against it. - // Racing it any earlier collides with a DIFFERENT, unrelated race: the - // boot scan's own directory read of this same record file, which the - // scan already handles by skipping a record it can no longer read -- - // that would starve this test of the spawn-in-flight race it exists to - // exercise. - const spawnDeadline = Date.now() + 2000; - while (Date.now() < spawnDeadline && spawns.length === 0) { - await new Promise((r) => setTimeout(r, 1)); - } - expect(spawns).toHaveLength(1); - - // Race a reclaiming teardown against the still-in-flight restore, before - // its boot-restore timeout fires. `activeSupervisors` has no entry for - // this address yet (the spawn handshake is unanswered), so this exercises - // the ordinary "operator undeploys mid-restore" path, not the - // already-live-supervisor guard the other test above covers. - await router.teardownDeployment(agentAddress, { reclaimDirs: true }); - - await restorePromise; - - // The teardown deleted the record before the boot loop's own timeout - // catch could write to it. `recordWorkflowDeploymentRestoreFailure` - // re-reads from disk and must no-op on a missing record rather than - // resurrecting `deployment.json` with a false restoreFailure for a - // deployment that was fully reclaimed (CL-7215). - const afterRace = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(afterRace).toBeUndefined(); - - // The underlying restore was never abandoned: answering the handshake now - // lets `spawnWorkflowDeployment` finish, well after the teardown already - // reclaimed this address. Without reconciling against the now-missing - // record, this would register a live supervisor with no durable record - // behind it -- an orphaned deployment, invisible to any future boot scan. - await answerReadyHandshake(spawns, 0); - - // Poll for the unwind: it is a background continuation of the restore - // attempt, not something `restoreWorkflowDeployments()` awaits. - const deadline = Date.now() + 10_000; - while (Date.now() < deadline && router.activeAddresses().includes(agentAddress)) { - await new Promise((r) => setTimeout(r, 5)); - } - - expect(router.activeAddresses()).toEqual([]); - const afterUnwind = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(afterUnwind).toBeUndefined(); -}, 15_000); diff --git a/apps/sidecar/test/workflow-substrate-factory-child-cancel-report.test.ts b/apps/sidecar/test/workflow-substrate-factory-child-cancel-report.test.ts deleted file mode 100644 index 3856f34238..0000000000 --- a/apps/sidecar/test/workflow-substrate-factory-child-cancel-report.test.ts +++ /dev/null @@ -1,324 +0,0 @@ -// Force handle.cancel to reject during parent abort and prove the -// fire-and-forget call reports through reportError instead of becoming -// an unhandled rejection. Injected via SidecarRunChildDeps test seams -// rather than bun's `mock.module`, whose process-wide registry swap -// cannot be undone for later files in the same `bun test` process. - -import { afterAll, beforeAll, expect, mock, test } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { defineAgent } from "@intx/agent"; -import { generateKeyPair } from "@intx/crypto"; -import { - createRepoStore, - workflowRunKindHandler, - WORKFLOW_RUN_GITIGNORE_PATH, -} from "@intx/hub-sessions"; -import type { AuthorizeFn } from "@intx/hub-sessions"; -import type { RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions/substrate"; -import type { KeyPair } from "@intx/types/runtime"; -import { - createInMemoryRepoStore, - createInMemoryScheduler, - defineWorkflow, - step, -} from "@intx/workflow"; -import { createWorkflowHostSignalChannel, type RunChildWorkflow } from "@intx/workflow-host"; - -import { - createSidecarRunChild, - createSidecarSpawnSuspendableChild, -} from "../src/workflow-substrate-factory/child-runtime"; - -const REF = "refs/heads/main"; -const DEPLOYMENT_ID = "deployment-cancel-report"; -const WORKFLOW_RUN_REPO_ID: RepoId = { - kind: "workflow-run", - id: DEPLOYMENT_ID, -}; -const allowAll: AuthorizeFn = () => ({ allowed: true }); -const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { - kind: "workflow-process", - anchorRunId: DEPLOYMENT_ID, -}; - -const tempDirs: string[] = []; -let signingKey: KeyPair; - -beforeAll(async () => { - signingKey = await generateKeyPair(); -}); - -afterAll(async () => { - for (const d of tempDirs.splice(0)) { - await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { - /* best effort */ - }); - } -}); - -test("a cancel rejection during abort is reported and is not unhandled", async () => { - const forced = new Error("forced cancel reject"); - const cancelMock = mock(() => Promise.reject(forced)); - const reportErrorMock = mock((_error: unknown, _context: unknown) => "ref-test"); - - const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "run-child-cancel-report-")); - tempDirs.push(dataDir); - const substrate = createRepoStore({ - dataDir, - signingKey, - handlers: { "workflow-run": workflowRunKindHandler }, - authorize: allowAll, - }); - await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { - files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, - message: "genesis", - }); - - const runChild: RunChildWorkflow = createSidecarRunChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep: async () => ({ output: { done: true } }), - runtimeRun: () => ({ - runId: "run-child-forced-cancel", - complete: new Promise(() => undefined), - cancel: cancelMock, - signal: async () => undefined, - }), - reportError: reportErrorMock, - }); - - const agent = defineAgent({ - id: "child-step", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m" }] }, - }); - const definition = defineWorkflow({ - id: "child-wf-cancel-report", - trigger: { type: "manual" }, - steps: { s: step({ agent }) }, - }); - - const unhandled: unknown[] = []; - const onUnhandled = (reason: unknown) => { - unhandled.push(reason); - }; - process.on("unhandledRejection", onUnhandled); - try { - const abort = new AbortController(); - abort.abort(); - void runChild( - { - definition, - definitionRef: REF, - childRunId: "run-child-cancel-report", - input: { text: "event" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: abort.signal, - depth: 1, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - await Bun.sleep(50); - expect(cancelMock).toHaveBeenCalled(); - expect(reportErrorMock).toHaveBeenCalledTimes(1); - expect(reportErrorMock.mock.calls[0]?.[0]).toBe(forced); - expect(reportErrorMock.mock.calls[0]?.[1]).toMatchObject({ - operation: "sidecar.child-runtime.cancel", - }); - expect(unhandled).toEqual([]); - } finally { - process.off("unhandledRejection", onUnhandled); - } -}); - -test("a signalChannel.stop rejection is reported and is not unhandled", async () => { - const forced = new Error("forced stop reject"); - const stopMock = mock(() => Promise.reject(forced)); - const reportErrorMock = mock((_error: unknown, _context: unknown) => "ref-test"); - - const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "run-child-stop-report-")); - tempDirs.push(dataDir); - const substrate = createRepoStore({ - dataDir, - signingKey, - handlers: { "workflow-run": workflowRunKindHandler }, - authorize: allowAll, - }); - await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { - files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, - message: "genesis", - }); - - const spawn = createSidecarSpawnSuspendableChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep: async () => ({ output: { done: true } }), - runtimeRun: () => ({ - runId: "run-child-forced-stop", - complete: Promise.resolve({ - runId: "run-child-forced-stop", - terminalStatus: "completed" as const, - outputs: {}, - events: [], - }), - cancel: async () => undefined, - signal: async () => undefined, - }), - reportError: reportErrorMock, - createSignalChannel: (opts) => { - const channel = createWorkflowHostSignalChannel(opts); - return { ...channel, stop: stopMock }; - }, - }); - - const agent = defineAgent({ - id: "child-step", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m" }] }, - }); - const definition = defineWorkflow({ - id: "child-wf-stop-report", - trigger: { type: "manual" }, - steps: { s: step({ agent }) }, - }); - - const unhandled: unknown[] = []; - const onUnhandled = (reason: unknown) => { - unhandled.push(reason); - }; - process.on("unhandledRejection", onUnhandled); - try { - await spawn( - { - definition, - definitionRef: REF, - childRunId: "run-child-stop-report", - input: { text: "event" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: new AbortController().signal, - depth: 0, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - await Bun.sleep(50); - expect(stopMock).toHaveBeenCalled(); - expect(reportErrorMock).toHaveBeenCalledTimes(1); - expect(reportErrorMock.mock.calls[0]?.[0]).toBe(forced); - expect(reportErrorMock.mock.calls[0]?.[1]).toMatchObject({ - operation: "sidecar.child-runtime.signal-channel-stop", - }); - expect(unhandled).toEqual([]); - } finally { - process.off("unhandledRejection", onUnhandled); - } -}); - -test("a spawn-suspendable abort cancel rejection is reported and is not unhandled", async () => { - const forced = new Error("forced suspendable cancel reject"); - const cancelMock = mock(() => Promise.reject(forced)); - const reportErrorMock = mock((_error: unknown, _context: unknown) => "ref-test"); - - const dataDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), "run-suspendable-cancel-report-"), - ); - tempDirs.push(dataDir); - const substrate = createRepoStore({ - dataDir, - signingKey, - handlers: { "workflow-run": workflowRunKindHandler }, - authorize: allowAll, - }); - await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { - files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, - message: "genesis", - }); - - const spawn = createSidecarSpawnSuspendableChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep: async () => ({ output: { done: true } }), - runtimeRun: () => ({ - runId: "run-suspendable-forced-cancel", - complete: new Promise(() => undefined), - cancel: cancelMock, - signal: async () => undefined, - }), - reportError: reportErrorMock, - }); - - const agent = defineAgent({ - id: "child-step", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m" }] }, - }); - const definition = defineWorkflow({ - id: "child-wf-suspendable-cancel-report", - trigger: { type: "manual" }, - steps: { s: step({ agent }) }, - }); - - const unhandled: unknown[] = []; - const onUnhandled = (reason: unknown) => { - unhandled.push(reason); - }; - process.on("unhandledRejection", onUnhandled); - try { - const abort = new AbortController(); - abort.abort(); - await spawn( - { - definition, - definitionRef: REF, - childRunId: "run-suspendable-cancel-report", - input: { text: "event" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: abort.signal, - depth: 0, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - await Bun.sleep(50); - expect(cancelMock).toHaveBeenCalled(); - expect(reportErrorMock).toHaveBeenCalledTimes(1); - expect(reportErrorMock.mock.calls[0]?.[0]).toBe(forced); - expect(reportErrorMock.mock.calls[0]?.[1]).toMatchObject({ - operation: "sidecar.child-runtime.cancel", - }); - expect(unhandled).toEqual([]); - } finally { - process.off("unhandledRejection", onUnhandled); - } -}); diff --git a/apps/sidecar/test/workflow-substrate-factory-run-child.test.ts b/apps/sidecar/test/workflow-substrate-factory-run-child.test.ts deleted file mode 100644 index 75d3cb70f9..0000000000 --- a/apps/sidecar/test/workflow-substrate-factory-run-child.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -// A terminal-only child workflow, run through `createSidecarRunChild` -- -// the independently constructed seam `spawnChild` delegates to, distinct -// from the suspendable-child path. The callback awaits a single terminal, -// so what is proven here is OUR abort wiring on that path: the -// parent-supplied `AbortSignal` reaches the child's cancel cascade and the -// callback resolves with `terminalStatus: "cancelled"` -- both for an -// abort that fires mid-run (while the child is parked with nothing to -// resume it) and for a signal that is already aborted when the callback -// observes it. The completion control keeps those cancels honest: the -// same seam resolves "completed" when nothing aborts. - -import { describe, test, expect, afterAll, beforeAll } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { generateKeyPair } from "@intx/crypto"; -import type { KeyPair } from "@intx/types/runtime"; -import { defineAgent } from "@intx/agent"; -import { - createRepoStore, - workflowRunKindHandler, - WORKFLOW_RUN_GITIGNORE_PATH, -} from "@intx/hub-sessions"; -import type { AuthorizeFn } from "@intx/hub-sessions"; -import type { RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions/substrate"; -import { - createInMemoryRepoStore, - createInMemoryScheduler, - defineWorkflow, - step, - type StepInvoker, - type WorkflowDefinition, -} from "@intx/workflow"; -import type { RunChildWorkflow } from "@intx/workflow-host"; - -import { createSidecarRunChild } from "../src/workflow-substrate-factory"; - -const REF = "refs/heads/main"; -const DEPLOYMENT_ID = "deployment-run-child"; -const WORKFLOW_RUN_REPO_ID: RepoId = { - kind: "workflow-run", - id: DEPLOYMENT_ID, -}; -const allowAll: AuthorizeFn = () => ({ allowed: true }); -const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { - kind: "workflow-process", - anchorRunId: DEPLOYMENT_ID, -}; - -const tempDirs: string[] = []; -let signingKey: KeyPair; - -beforeAll(async () => { - signingKey = await generateKeyPair(); -}); - -afterAll(async () => { - for (const d of tempDirs.splice(0)) { - await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { - /* best effort */ - }); - } -}); - -async function makeSubstrate(prefix: string): Promise> { - const dataDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); - tempDirs.push(dataDir); - const substrate = createRepoStore({ - dataDir, - signingKey, - handlers: { "workflow-run": workflowRunKindHandler }, - authorize: allowAll, - }); - await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { - files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, - message: "genesis", - }); - return substrate; -} - -// The one-step child definition every run uses; its injected invoker -// decides whether the step completes or parks forever. -function childDefinition(id: string): WorkflowDefinition { - const agent = defineAgent({ - id: "child-step", - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m" }] }, - }); - return defineWorkflow({ - id, - trigger: { type: "manual" }, - steps: { s: step({ agent }) }, - }); -} - -// Parks on a correlation nothing will ever resume: the run can only end -// through the cancel cascade, so a resolved terminal proves the abort -// wiring rather than a step that happened to finish first. -const parkForever: StepInvoker = async () => ({ - suspend: { - correlationId: "corr-never-resumed", - kind: "approval" as const, - approvalSnapshot: { - name: "child-step", - description: "never granted", - inputSchema: {}, - arguments: {}, - }, - }, -}); - -const completeImmediately: StepInvoker = async () => ({ - output: { done: true }, -}); - -async function makeRunChild(prefix: string, invokeStep: StepInvoker): Promise { - const substrate = await makeSubstrate(prefix); - return createSidecarRunChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep, - }); -} - -function childInput(id: string, signal: AbortSignal): Parameters[0] { - return { - definition: childDefinition(`child-wf-${id}`), - definitionRef: REF, - childRunId: `run-child-${id}`, - input: { text: "event" }, - parentRunId: "run-parent", - parentStepId: "section", - signal, - depth: 1, - maxChildSpawnDepth: 32, - }; -} - -describe("createSidecarRunChild", () => { - test("an unaborted child runs to a completed terminal", async () => { - const runChild = await makeRunChild("run-child-complete-", completeImmediately); - const result = await runChild( - childInput("complete", new AbortController().signal), - () => undefined, - ); - expect(result.terminalStatus).toBe("completed"); - }); - - test("a parent abort mid-run cancels the child and resolves cancelled", async () => { - let announceStepEntered!: () => void; - const stepEntered = new Promise((resolve) => { - announceStepEntered = resolve; - }); - const parkAndAnnounce: StepInvoker = async (...args) => { - announceStepEntered(); - return parkForever(...args); - }; - - const runChild = await makeRunChild("run-child-abort-", parkAndAnnounce); - const abort = new AbortController(); - const pending = runChild(childInput("abort", abort.signal), () => undefined); - - // Wait on the step actually being entered, not on a fixed delay: the - // cancel has to interrupt a run in flight, and a machine slow enough - // to still be starting the child when a timer fires would abort a run - // that has nothing to cancel — which then never reaches a terminal. - // The short settle after it covers the park's own commit. - await stepEntered; - await Bun.sleep(50); - abort.abort(); - - const result = await pending; - expect(result.terminalStatus).toBe("cancelled"); - }); - - test("a signal already aborted at invocation resolves cancelled", async () => { - const runChild = await makeRunChild("run-child-preabort-", parkForever); - const abort = new AbortController(); - abort.abort(); - const result = await runChild(childInput("preabort", abort.signal), () => undefined); - expect(result.terminalStatus).toBe("cancelled"); - }); -}); diff --git a/apps/sidecar/test/workflow-substrate-factory-suspendable-child.test.ts b/apps/sidecar/test/workflow-substrate-factory-suspendable-child.test.ts deleted file mode 100644 index c068986c4a..0000000000 --- a/apps/sidecar/test/workflow-substrate-factory-suspendable-child.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -// An onTrigger body child, spawned through the real sidecar seam, parks on -// an approval and resumes on the correlated grant. -// -// `createSidecarSpawnSuspendableChild` runs the body definition against a -// real on-disk workflow-run substrate and hands back the live handle -// `runOnTrigger` drives. This test exercises that handle end to end: the -// body step's injected `invokeStep` suspends as an approval, so the child -// runtime parks on the reserved correlation channel; `handle.next()` -// surfaces the park with the step's snapshot; `handle.resume` delivers the -// grant on the child's own signal channel, and the re-invoked step -// completes the run. Runtime park/resume mechanics themselves are the -// published packages' concern -- what is proven here is OUR handle wiring: -// the park FIFO, the resume delivery, and the terminal surfacing. - -import { describe, test, expect, afterAll, beforeAll } from "bun:test"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { generateKeyPair } from "@intx/crypto"; -import type { AuthzCallResult } from "@intx/inference"; -import type { ApprovalSnapshot, KeyPair, MailPartReader } from "@intx/types/runtime"; -import { defineAgent } from "@intx/agent"; -import { - createRepoStore, - workflowRunKindHandler, - WORKFLOW_RUN_GITIGNORE_PATH, -} from "@intx/hub-sessions"; -import type { AuthorizeFn } from "@intx/hub-sessions"; -import type { RepoId, WorkflowRunWorkflowProcessPrincipal } from "@intx/hub-sessions/substrate"; -import { - createInMemoryRepoStore, - createInMemoryScheduler, - defineWorkflow, - step, - type StepInvoker, - type WorkflowAuthorizeFn, - type WorkflowDefinition, -} from "@intx/workflow"; -import type { CredentialWiring } from "@intx/workflow-host"; - -import { - createSidecarSpawnSuspendableChild, - type SidecarBodyStepInvoker, -} from "../src/workflow-substrate-factory"; - -const REF = "refs/heads/main"; -const DEPLOYMENT_ID = "deployment-suspendable-child"; -const WORKFLOW_RUN_REPO_ID: RepoId = { - kind: "workflow-run", - id: DEPLOYMENT_ID, -}; -const allowAll: AuthorizeFn = () => ({ allowed: true }); -const ALLOW_AUTHZ: AuthzCallResult = { - effect: "allow", - matchingGrants: [], - resolvedBy: null, -}; -const PRINCIPAL: WorkflowRunWorkflowProcessPrincipal = { - kind: "workflow-process", - anchorRunId: DEPLOYMENT_ID, -}; - -const BODY_STEP_AGENT_ID = "wallet-spend"; -const CORRELATION_ID = "corr-approval-1"; -const SNAPSHOT: ApprovalSnapshot = { - name: BODY_STEP_AGENT_ID, - description: "spend from the shared wallet", - inputSchema: { amount: "number" }, - arguments: { amount: 100 }, -}; - -const tempDirs: string[] = []; -let signingKey: KeyPair; - -beforeAll(async () => { - signingKey = await generateKeyPair(); -}); - -afterAll(async () => { - for (const d of tempDirs.splice(0)) { - await fs.promises.rm(d, { recursive: true, force: true }).catch(() => { - /* best effort */ - }); - } -}); - -async function makeTempDir(prefix: string): Promise { - const d = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix)); - tempDirs.push(d); - return d; -} - -// The one-step body definition every spawn runs. Its single step's injected -// invoker suspends as an approval on the first invocation and completes on -// the resume re-invocation. -function bodyDefinition(id: string): WorkflowDefinition { - const agent = defineAgent({ - id: BODY_STEP_AGENT_ID, - systemPrompt: "s", - tools: [], - capabilities: [], - inference: { sources: [{ provider: "anthropic", model: "m" }] }, - }); - return defineWorkflow({ - id, - trigger: { type: "manual" }, - steps: { s: step({ agent }) }, - }); -} - -async function makeSubstrate(prefix: string): Promise> { - const dataDir = await makeTempDir(prefix); - const substrate = createRepoStore({ - dataDir, - signingKey, - handlers: { "workflow-run": workflowRunKindHandler }, - authorize: allowAll, - }); - await substrate.writeTree({ kind: "hub" }, WORKFLOW_RUN_REPO_ID, REF, { - files: { [WORKFLOW_RUN_GITIGNORE_PATH]: "" }, - message: "genesis", - }); - return substrate; -} - -// An invoker that suspends as an approval on its first invocation and, on -// the resume re-invocation, records the delivered decision and completes. -function suspendThenComplete(record: { resumeDecision?: unknown }): StepInvoker { - return async (req) => { - if (req.resume === undefined) { - return { - suspend: { - correlationId: CORRELATION_ID, - kind: "approval", - approvalSnapshot: SNAPSHOT, - }, - }; - } - record.resumeDecision = req.resume.decision; - return { output: { echoed: req.resume.decision } }; - }; -} - -function makeSpawner( - substrate: ReturnType, - invokeStep: StepInvoker, -): ReturnType { - return createSidecarSpawnSuspendableChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep, - }); -} - -describe("createSidecarSpawnSuspendableChild", () => { - test("surfaces a body approval park, resumes on the grant, and completes", async () => { - const substrate = await makeSubstrate("suspendable-approval-"); - - const record: { resumeDecision?: unknown } = {}; - const spawn = makeSpawner(substrate, suspendThenComplete(record)); - - const handle = await spawn( - { - definition: bodyDefinition("body-wf"), - definitionRef: REF, - childRunId: "run-body-0", - input: { text: "event-0" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: new AbortController().signal, - depth: 0, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - - // The body step suspended -> the handle surfaces the approval park on - // the reserved correlation, carrying the step's snapshot. - const parked = await handle.next(); - expect(parked.kind).toBe("park"); - if (parked.kind !== "park") throw new Error("expected a park"); - expect(parked.park.correlationId).toBe(CORRELATION_ID); - expect(parked.park.approvalSnapshot).toEqual(SNAPSHOT); - - // Grant it: resume delivers the decision on the child's own signal - // channel, unblocking the parked step. - const decision = { outcome: "approved" as const, note: "ok" }; - await handle.resume(CORRELATION_ID, decision); - - // The re-invoked step ran with the delivered decision and the run - // completed. - const terminal = await handle.next(); - expect(terminal.kind).toBe("terminal"); - if (terminal.kind !== "terminal") throw new Error("expected a terminal"); - expect(terminal.terminalStatus).toBe("completed"); - expect(record.resumeDecision).toEqual(decision); - }); - - test("a wired body invoker runs the body step with the on-disk sources", async () => { - const substrate = await makeSubstrate("suspendable-body-invoker-"); - - // The per-body pins the deploy router materializes beside the body's - // workflow.json; the spawn must read exactly this file to build the - // body's sources ref. - const bodySources = { - s: [ - { - id: "s", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - credentialId: "sk-body", - model: "claude-3-5", - }, - ], - }; - const dataDir = await makeTempDir("suspendable-body-datadir-"); - const sourcesDir = path.join(dataDir, "assets", "workflow", "body-wf-real"); - await fs.promises.mkdir(sourcesDir, { recursive: true }); - await fs.promises.writeFile(path.join(sourcesDir, "sources.json"), JSON.stringify(bodySources)); - - const seen: { sources?: unknown; agentId?: string } = {}; - const bodyInvokeStep: SidecarBodyStepInvoker = async (req, _authorize, sourcesRef) => { - seen.sources = sourcesRef.current; - seen.agentId = req.agent.id; - return { output: { done: true } }; - }; - const spawn = createSidecarSpawnSuspendableChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep: () => { - throw new Error( - "the body step must route through bodyInvokeStep, not the childWorkflow stub", - ); - }, - bodyInvokeStep, - dataDir, - }); - - const handle = await spawn( - { - definition: bodyDefinition("body-wf-real"), - definitionRef: REF, - childRunId: "run-body-2", - input: { text: "event-2" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: new AbortController().signal, - depth: 0, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - - const terminal = await handle.next(); - expect(terminal.kind).toBe("terminal"); - if (terminal.kind !== "terminal") throw new Error("expected a terminal"); - expect(terminal.terminalStatus).toBe("completed"); - // The invoker ran the body's own step and saw the sources read off - // disk -- readBodyStepInferenceSources fed the ref, not the parent's - // pinned table. - expect(seen.agentId).toBe(BODY_STEP_AGENT_ID); - expect(seen.sources).toEqual(bodySources); - }); - - // The host threads optional authorize / credentialWiring / mailPartReader - // on the spawn input; the body invoker must receive those, not the env - // throwing stub and unset extras. - test("the body invoker receives spawn-input authorize, credentialWiring, and mailPartReader", async () => { - const substrate = await makeSubstrate("suspendable-body-authorize-"); - const dataDir = await makeTempDir("suspendable-body-authz-datadir-"); - const sourcesDir = path.join(dataDir, "assets", "workflow", "body-wf-authz"); - await fs.promises.mkdir(sourcesDir, { recursive: true }); - await fs.promises.writeFile( - path.join(sourcesDir, "sources.json"), - JSON.stringify({ - s: [ - { - id: "s", - provider: "anthropic", - baseURL: "https://api.anthropic.com", - credentialId: "sk-body", - model: "claude-3-5", - }, - ], - }), - ); - - const parentAuthorize: WorkflowAuthorizeFn = async () => ALLOW_AUTHZ; - const parentCredentialWiring: CredentialWiring = { - materialRef: { current: null }, - resolveStepGrants: () => [], - }; - const parentMailPartReader: MailPartReader = { - read: async () => new Uint8Array(), - }; - - const seen: { - authorize?: unknown; - credentialWiring?: unknown; - mailPartReader?: unknown; - } = {}; - const bodyInvokeStep: SidecarBodyStepInvoker = async ( - _req, - authorize, - _sourcesRef, - _onEvent, - credentialWiring, - mailPartReader, - ) => { - seen.authorize = authorize; - seen.credentialWiring = credentialWiring; - seen.mailPartReader = mailPartReader; - return { output: { done: true } }; - }; - const spawn = createSidecarSpawnSuspendableChild({ - substrate, - workflowRunRepoId: WORKFLOW_RUN_REPO_ID, - workflowRunRef: REF, - principal: PRINCIPAL, - scheduler: createInMemoryScheduler({ - repoStore: createInMemoryRepoStore(), - clock: () => new Date(), - }), - invokeStep: () => { - throw new Error("must route through bodyInvokeStep"); - }, - bodyInvokeStep, - dataDir, - }); - - const handle = await spawn( - { - definition: bodyDefinition("body-wf-authz"), - definitionRef: REF, - childRunId: "run-body-3", - input: { text: "event-3" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: new AbortController().signal, - depth: 0, - maxChildSpawnDepth: 32, - authorize: parentAuthorize, - credentialWiring: parentCredentialWiring, - mailPartReader: parentMailPartReader, - }, - () => undefined, - ); - - const terminal = await handle.next(); - expect(terminal.kind).toBe("terminal"); - expect(seen.authorize).toBe(parentAuthorize); - expect(seen.credentialWiring).toBe(parentCredentialWiring); - expect(seen.mailPartReader).toBe(parentMailPartReader); - }); - - test("a parent abort while parked cancels the child and surfaces a terminal", async () => { - const substrate = await makeSubstrate("suspendable-abort-"); - const spawn = makeSpawner(substrate, suspendThenComplete({})); - - const abort = new AbortController(); - const handle = await spawn( - { - definition: bodyDefinition("body-wf-abort"), - definitionRef: REF, - childRunId: "run-body-1", - input: { text: "event-1" }, - parentRunId: "run-parent", - parentStepId: "section", - signal: abort.signal, - depth: 0, - maxChildSpawnDepth: 32, - }, - () => undefined, - ); - - const parked = await handle.next(); - expect(parked.kind).toBe("park"); - - abort.abort(); - - const terminal = await handle.next(); - expect(terminal.kind).toBe("terminal"); - if (terminal.kind !== "terminal") throw new Error("expected a terminal"); - expect(terminal.terminalStatus).toBe("cancelled"); - }); -}); diff --git a/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts b/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts deleted file mode 100644 index 8ae09c0134..0000000000 --- a/apps/sidecar/test/workflow-suspend-identity-preservation.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -// End-to-end proof that the real `createSidecarDeployRouter` wiring -- -// not just the standalone `hibernated-agent-identity-vault.ts` module -- -// preserves an agent's reconnect-challenge keypair across a -// state-preserving "hibernate" teardown (reclaimDirs: false), and still -// lets a reclaiming (non-hibernate) teardown destroy it. Uses a REAL -// `@intx/hub-agent` key store bound to the fixture's data dir (via -// `makeLifecycleFixture`'s `keyStore` override) rather than the shared -// fixture's default in-memory fake, which never touches disk and so -// cannot exercise this path. -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; - -import { createAgentKeyStore, createAgentRepoStore, agentDir } from "@intx/hub-agent"; -import { generateKeyPair, verifySSHSignature } from "@intx/crypto"; -import { hexEncode } from "@intx/types"; - -import { - answerReadyHandshake, - makeLifecycleFixture, - makeWorkflowFrame, -} from "./support/workflow-lifecycle-fixture"; - -const cryptoOps = { - generateKeyPair, - verifySSHSig: verifySSHSignature, -}; - -// `makeWorkflowFrame`'s default `hubPublicKey` ("hub-pk") is a placeholder -// the shared fixture's FAKE key store never validates. The real -// `AgentKeyStore.recordHubKey` this suite exercises hex-decodes it, so -// every frame here carries a real hex-encoded key instead. -async function makeFrameWithRealHubKey(agentAddress: string) { - const hubKeyPair = await generateKeyPair(); - return { - ...makeWorkflowFrame(agentAddress), - hubPublicKey: hexEncode(hubKeyPair.publicKey), - }; -} - -describe("hibernate/wake preserves the real deployed agent's identity", () => { - test("hibernate teardown, then redeploy: the same keypair comes back (isNew: false)", async () => { - const dataDir = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-suspend-identity-")); - const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); - // The real, unmodified package's own repo store -- its `remove` is - // exactly the destructive call `@intx/hub-agent`'s `ws/hub-link.js` - // issues AFTER our `undeploy` hook returns for every undeploy, - // hibernate or not. - const realRepoStore = createAgentRepoStore({ dataDir }); - - const { router, spawns } = await makeLifecycleFixture({ - dataDir, - keyStore, - }); - const address = "run_suspend-hibernate@example.com"; - - const frame1 = await makeFrameWithRealHubKey(address); - const deploy1 = router.deploy(frame1); - await answerReadyHandshake(spawns, 0); - await deploy1; - - const { keyPair: original } = await keyStore.loadOrGenerateKey(address); - - await router.teardownDeployment(address, { reclaimDirs: false }); - // Models the published package's own post-hook delete. - await realRepoStore.remove(address); - expect(await fs.readdir(dataDir)).not.toContain(path.basename(agentDir(dataDir, address))); - - const frame2 = await makeFrameWithRealHubKey(address); - const deploy2 = router.deploy(frame2); - await answerReadyHandshake(spawns, 1); - await deploy2; - - // The real proof: deploy2's redeploy loaded back the SAME key bytes - // `original` held before the hibernate teardown + destructive remove, - // not a freshly minted keypair. - const { keyPair: restored } = await keyStore.loadOrGenerateKey(address); - expect(restored.privateKey).toEqual(original.privateKey); - expect(restored.publicKey).toEqual(original.publicKey); - }); - - test("reclaiming (non-hibernate) teardown, then redeploy: a fresh keypair comes back", async () => { - const dataDir = await fs.mkdtemp(path.join(os.tmpdir(), "sidecar-suspend-identity-")); - const keyStore = createAgentKeyStore({ dataDir, ...cryptoOps }); - const realRepoStore = createAgentRepoStore({ dataDir }); - - const { router, spawns } = await makeLifecycleFixture({ - dataDir, - keyStore, - }); - const address = "run_suspend-reclaim@example.com"; - - const frame1 = await makeFrameWithRealHubKey(address); - const deploy1 = router.deploy(frame1); - await answerReadyHandshake(spawns, 0); - await deploy1; - - const { keyPair: original } = await keyStore.loadOrGenerateKey(address); - - await router.teardownDeployment(address, { reclaimDirs: true }); - await realRepoStore.remove(address); - - const frame2 = await makeFrameWithRealHubKey(address); - const deploy2 = router.deploy(frame2); - await answerReadyHandshake(spawns, 1); - await deploy2; - - // A THIRD `loadOrGenerateKey` call for the same address always reads - // back `isNew: false` regardless of whether deploy2's own internal - // call (inside `spawnWorkflowDeployment`) minted a fresh key or - // restored an old one -- some key now exists on disk either way, so - // `isNew` here would not distinguish the two. The actual proof is the - // key material itself: a destructive (reclaimDirs: true) teardown - // must leave deploy2 minting a DIFFERENT keypair than `original`. - const { keyPair: regenerated } = await keyStore.loadOrGenerateKey(address); - expect(regenerated.privateKey).not.toEqual(original.privateKey); - expect(regenerated.publicKey).not.toEqual(original.publicKey); - }); -}); diff --git a/apps/sidecar/test/workflow-teardown-flavor.test.ts b/apps/sidecar/test/workflow-teardown-flavor.test.ts deleted file mode 100644 index 905795710d..0000000000 --- a/apps/sidecar/test/workflow-teardown-flavor.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -// `teardownDeployment` is the shared body behind `undeploy` (`reclaimDirs: -// true` -- forget the deployment entirely) and the state-preserving -// "hibernate" flavor (`reclaimDirs: false` -- tear the child down while -// keeping the deployment record and on-disk step state so a later `deploy` -// call for the same address resumes rather than starts fresh). No caller in -// this lane invokes the hibernate flavor yet -- the hub-side reap-and-relaunch -// flow that will is a separate lane -- so this suite drives it directly -// through the router's `teardownDeployment` method. - -import { describe, test, expect } from "bun:test"; -import fs from "node:fs/promises"; -import path from "node:path"; - -import { deriveDeploymentId, IDLE_HIBERNATE_UNDEPLOY_REASON } from "../src/workflow-host-wiring"; -import { readWorkflowDeploymentRecord } from "../src/workflow-deployment-record"; -import { - answerReadyHandshake, - makeLifecycleFixture, - makeWorkflowFrame, -} from "./support/workflow-lifecycle-fixture"; - -describe("teardownDeployment reclaimDirs flavors", () => { - test("reclaimDirs: true (undeploy) deletes the step-state dir and the deployment record", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_teardown-reclaim@example.com"); - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - const deploymentId = deriveDeploymentId(frame.agentAddress); - const recordFile = path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); - const stepStateDir = path.join(dataDir, "workflow-step-state", deploymentId); - await fs.mkdir(stepStateDir, { recursive: true }); - await fs.writeFile(path.join(stepStateDir, "marker.txt"), "x"); - - await router.teardownDeployment(frame.agentAddress, { - reclaimDirs: true, - }); - - expect(spawn.killed).toBe(true); - await expect(fs.stat(recordFile)).rejects.toThrow(); - await expect(fs.stat(stepStateDir)).rejects.toThrow(); - expect(router.activeAddresses()).toEqual([]); - }); - - test("reclaimDirs: false (hibernate) preserves the step-state dir and the deployment record", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_teardown-hibernate@example.com"); - const deployPromise = router.deploy(frame); - const spawn = await answerReadyHandshake(spawns, 0); - await deployPromise; - - const deploymentId = deriveDeploymentId(frame.agentAddress); - const recordFile = path.join(dataDir, "workflow-runs", deploymentId, "deployment.json"); - const stepStateDir = path.join(dataDir, "workflow-step-state", deploymentId); - await fs.mkdir(stepStateDir, { recursive: true }); - await fs.writeFile(path.join(stepStateDir, "marker.txt"), "x"); - - await router.teardownDeployment(frame.agentAddress, { - reclaimDirs: false, - }); - - // The child is gone -- a hibernate still kills the process... - expect(spawn.killed).toBe(true); - // ...but every piece of durable state a relaunch resumes from survives. - await expect(fs.stat(recordFile)).resolves.toBeDefined(); - await expect(fs.stat(stepStateDir)).resolves.toBeDefined(); - expect(await fs.readFile(path.join(stepStateDir, "marker.txt"), "utf8")).toBe("x"); - // The address is no longer live/routable: a relaunch re-establishes it - // through the ordinary deploy path, not a resumed registration. - expect(router.activeAddresses()).toEqual([]); - - // The kept record is stamped as parked -- the durable marker a boot - // scan reads to tell "the hub parked this" from "this was live when - // the process stopped". - const record = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(record?.parkedAt).toBeDefined(); - }); - - test("reclaimDirs: true (undeploy) never stamps a parked marker (the record is gone)", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_teardown-reclaim-no-mark@example.com"); - const deployPromise = router.deploy(frame); - await answerReadyHandshake(spawns, 0); - await deployPromise; - - const deploymentId = deriveDeploymentId(frame.agentAddress); - await router.teardownDeployment(frame.agentAddress, { - reclaimDirs: true, - }); - - await expect(readWorkflowDeploymentRecord(dataDir, deploymentId)).resolves.toBeUndefined(); - }); - - test("both flavors unregister the transport, routers, and deployment-address mapping", async () => { - const { router, spawns, multistepMailRouter, multistepCredentialsRouter } = - await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_teardown-unregister@example.com"); - const deployPromise = router.deploy(frame); - await answerReadyHandshake(spawns, 0); - await deployPromise; - - await router.teardownDeployment(frame.agentAddress, { - reclaimDirs: false, - }); - - // No live mail/credentials handler remains registered for the address: - // routing a frame against it fails to find a handler. - expect( - multistepMailRouter.tryRoute(frame.agentAddress, new TextEncoder().encode("post-hibernate")), - ).toBeNull(); - expect( - await multistepCredentialsRouter.tryRoute({ - type: "credentials.update", - agentAddress: frame.agentAddress, - delivery: { bindings: [], materials: [] }, - }), - ).toBe(false); - }); - - test("undeploy(frame) picks the flavor from frame.reason", async () => { - const hibernating = await makeLifecycleFixture(); - const hibernateFrame = makeWorkflowFrame("run_undeploy-reason-hibernate@example.com"); - const hibernateDeploy = hibernating.router.deploy(hibernateFrame); - await answerReadyHandshake(hibernating.spawns, 0); - await hibernateDeploy; - const hibernateDeploymentId = deriveDeploymentId(hibernateFrame.agentAddress); - const hibernateStepStateDir = path.join( - hibernating.dataDir, - "workflow-step-state", - hibernateDeploymentId, - ); - await fs.mkdir(hibernateStepStateDir, { recursive: true }); - - await hibernating.router.undeploy?.({ - type: "agent.undeploy", - agentAddress: hibernateFrame.agentAddress, - reason: IDLE_HIBERNATE_UNDEPLOY_REASON, - }); - - // Tagged with the hub idle-reap's reason: preserved, not reclaimed. - await expect(fs.stat(hibernateStepStateDir)).resolves.toBeDefined(); - - const reclaiming = await makeLifecycleFixture(); - const reclaimFrame = makeWorkflowFrame("run_undeploy-reason-reclaim@example.com"); - const reclaimDeploy = reclaiming.router.deploy(reclaimFrame); - await answerReadyHandshake(reclaiming.spawns, 0); - await reclaimDeploy; - const reclaimDeploymentId = deriveDeploymentId(reclaimFrame.agentAddress); - const reclaimStepStateDir = path.join( - reclaiming.dataDir, - "workflow-step-state", - reclaimDeploymentId, - ); - await fs.mkdir(reclaimStepStateDir, { recursive: true }); - - await reclaiming.router.undeploy?.({ - type: "agent.undeploy", - agentAddress: reclaimFrame.agentAddress, - reason: "channel-deleted", - }); - - // Any other reason still gets the destructive default. - await expect(fs.stat(reclaimStepStateDir)).rejects.toThrow(); - }); - - // CL-6644 part B: a hub-driven wake for a parked deployment is a fresh - // `agent.deploy` for the same address, not a resume in place -- `0fd3fbc8` - // deleted the sidecar-side in-place park/wake handler this used to be, and - // a boot scan (CL-6282) skips a parked record entirely rather than - // restoring it. Nothing sidecar-side ever resumes a parked deployment on - // its own, so the wake's redeploy must complete cleanly against whatever - // residue the parked record and its hibernate teardown left behind: the - // deployment record still on disk (parked, not deleted), the slug - // (`releaseSlug` only runs for a reclaiming teardown), and the step-state - // dir the hibernate deliberately preserved. This proves that residue - // never blocks the redeploy. - test("deploy() after a hibernate teardown completes (a wake's redeploy for a parked address)", async () => { - const { router, spawns, dataDir } = await makeLifecycleFixture(); - const frame = makeWorkflowFrame("run_teardown-wake-redeploy@example.com"); - const firstDeploy = router.deploy(frame); - await answerReadyHandshake(spawns, 0); - await firstDeploy; - - await router.teardownDeployment(frame.agentAddress, { - reclaimDirs: false, - }); - - const deploymentId = deriveDeploymentId(frame.agentAddress); - const parkedBeforeRedeploy = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(parkedBeforeRedeploy?.parkedAt).toBeDefined(); - - const secondDeploy = router.deploy(frame); - await answerReadyHandshake(spawns, 1); - await expect(secondDeploy).resolves.toBeDefined(); - - expect(router.activeAddresses()).toEqual([frame.agentAddress]); - const redeployedRecord = await readWorkflowDeploymentRecord(dataDir, deploymentId); - expect(redeployedRecord?.parkedAt).toBeUndefined(); - }); -}); diff --git a/apps/sidecar/tsconfig.json b/apps/sidecar/tsconfig.json index 5ce2eadb7a..b8731c2847 100644 --- a/apps/sidecar/tsconfig.json +++ b/apps/sidecar/tsconfig.json @@ -10,24 +10,9 @@ "tsBuildInfoFile": "tsconfig.tsbuildinfo", "rootDir": "../.." }, - "include": ["src", "test"], + "include": ["src"], "exclude": [], "references": [ - { - "path": "../../packages/codex-provider/tsconfig.src.json" - }, - { - "path": "../../packages/credential-providers/tsconfig.src.json" - }, - { - "path": "../../packages/error-sink/tsconfig.src.json" - }, - { - "path": "../../packages/ollama-adapter/tsconfig.src.json" - }, - { - "path": "../../packages/xai-provider/tsconfig.src.json" - }, { "path": "../../vendor/intx/agent/tsconfig.src.json" }, @@ -46,9 +31,6 @@ { "path": "../../vendor/intx/mail-memory/tsconfig.src.json" }, - { - "path": "../../vendor/intx/mime/tsconfig.src.json" - }, { "path": "../../vendor/intx/types/tsconfig.src.json" }, diff --git a/apps/sidecar/tsconfig.src.json b/apps/sidecar/tsconfig.src.json index a79b093994..097f1dee8d 100644 --- a/apps/sidecar/tsconfig.src.json +++ b/apps/sidecar/tsconfig.src.json @@ -10,21 +10,6 @@ "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo" }, "references": [ - { - "path": "../../packages/codex-provider/tsconfig.src.json" - }, - { - "path": "../../packages/credential-providers/tsconfig.src.json" - }, - { - "path": "../../packages/error-sink/tsconfig.src.json" - }, - { - "path": "../../packages/ollama-adapter/tsconfig.src.json" - }, - { - "path": "../../packages/xai-provider/tsconfig.src.json" - }, { "path": "../../vendor/intx/agent/tsconfig.src.json" }, @@ -43,9 +28,6 @@ { "path": "../../vendor/intx/mail-memory/tsconfig.src.json" }, - { - "path": "../../vendor/intx/mime/tsconfig.src.json" - }, { "path": "../../vendor/intx/types/tsconfig.src.json" }, diff --git a/bun.lock b/bun.lock index 03821bd460..69552cd6d8 100644 --- a/bun.lock +++ b/bun.lock @@ -106,17 +106,11 @@ }, "apps/sidecar": { "name": "@workbench/sidecar", - "version": "0.0.1", + "version": "0.3.0", "bin": { "workflow-child": "./bin/workflow-child", }, "dependencies": { - "@corbits/codex-provider": "workspace:*", - "@corbits/credential-providers": "workspace:*", - "@corbits/error-sink": "workspace:*", - "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core#v0.1.1", - "@corbits/ollama-adapter": "workspace:*", - "@corbits/xai-provider": "workspace:*", "@intx/agent": "workspace:*", "@intx/authz": "0.3.0", "@intx/crypto": "0.3.0", @@ -126,9 +120,12 @@ "@intx/inference": "workspace:*", "@intx/log": "0.3.0", "@intx/mail-memory": "workspace:*", - "@intx/mime": "workspace:*", + "@intx/pack-transport": "0.3.0", "@intx/storage-isogit": "0.3.0", "@intx/tool-packaging": "0.3.0", + "@intx/tools-lsp": "0.3.0", + "@intx/tools-mail": "0.3.0", + "@intx/tools-posix": "0.3.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -137,6 +134,7 @@ }, "devDependencies": { "@types/bun": "catalog:", + "isomorphic-git": "catalog:", "typescript": "catalog:", }, }, @@ -1766,6 +1764,12 @@ "@intx/tool-packaging": ["@intx/tool-packaging@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/log": "0.3.0", "@intx/storage-isogit": "0.3.0", "@intx/types": "0.3.0", "arktype": "^2.1.29", "isomorphic-git": "^1.27.2", "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" } }, "sha512-Lr/xdEjXTindJRm+eITvfeqpW/yl71thYjr3HXl41GJ9wpuOZPCjm9vBoRVKhUSwmIUKBYbm2VplXCCXaS7f4g=="], + "@intx/tools-lsp": ["@intx/tools-lsp@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/log": "0.3.0", "@intx/tools-posix": "0.3.0", "@intx/types": "0.3.0", "vscode-jsonrpc": "^9.0.1", "vscode-languageserver-types": "^3.17.5", "which": "^4.0.0" } }, "sha512-Ywnm9NNfCLlkzXTL1UoHyVvee4eaQ9rqKkkHM8zmf4l8L+ZqEPIW4IG5nERJG9F9XxOaf4gboqbFgqMe+AF5Dg=="], + + "@intx/tools-mail": ["@intx/tools-mail@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/types": "0.3.0", "arktype": "^2.1.29" } }, "sha512-DLI3+5mVhqdiW1VtSt4/lVRI8UFDp1Li7z/0Dfxqe44rdP2HvPRiZ8LmoSoZ8DdnGuC7dXT1MwNIN5kr9cCQCw=="], + + "@intx/tools-posix": ["@intx/tools-posix@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/types": "0.3.0" } }, "sha512-0cQt5d7POpe/Wbee4pmnwTSux9JeyEy115X/96Ok0enNSIpAaPmHB61iRMuMHStS0hNF1e7DCpA7OH5Bi08Dbg=="], + "@intx/types": ["@intx/types@workspace:vendor/intx/types"], "@intx/workflow": ["@intx/workflow@workspace:vendor/intx/workflow"], @@ -2370,7 +2374,7 @@ "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], - "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "isexe": ["isexe@3.1.5", "", {}, "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w=="], "isomorphic-git": ["isomorphic-git@1.42.2", "", { "dependencies": { "async-lock": "^1.4.1", "clean-git-ref": "^2.0.1", "crc-32": "^1.2.0", "diff3": "0.0.3", "ignore": "^5.1.4", "minimisted": "^2.0.0", "pako": "^1.0.10", "pify": "^4.0.1", "readable-stream": "^4.0.0", "sha.js": "^2.4.12", "simple-get": "^4.0.1" }, "bin": { "isogit": "cli.cjs" } }, "sha512-Rb7czWhLO4JYqmg4an01nNomuAgFbWWK2CVfkHGBOvK5t7hrGTKOXPx/csscJ1j9o3uXFzNcC6njgvCafSmIPw=="], @@ -2788,6 +2792,8 @@ "vite": ["vite@7.3.6", "", { "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg=="], + "vscode-jsonrpc": ["vscode-jsonrpc@9.0.2", "", {}, "sha512-SbQSV9yRemARxeXw6LU5sS6Zq0e9/DgCCX5yelH263ZQWukbTk8EF8fjTrr1dziasf4GwlJbvTwFnTrnQFWZXQ=="], + "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.14", "", {}, "sha512-EQyqJMi552E4ZTf46izQ4Fj6XquqxCySR3J5ZSD1SisMf6RfpeOWHxGBE8Gr6V0/3GHIGdAzDn8F8+1nTGCnoQ=="], "vscode-languageserver-types": ["vscode-languageserver-types@3.18.3", "", {}, "sha512-XIlzJ7Qp/jzSI1ds7/FwPAWrPeTZA7pAtlW4hdJ1J6xXWJL6dR9QYnDhJOdLzdKhUQ5Mm6mvUMw+3DcOQQasPw=="], @@ -2796,7 +2802,7 @@ "whatwg-mimetype": ["whatwg-mimetype@3.0.0", "", {}, "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="], "which-typed-array": ["which-typed-array@1.1.23", "", { "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.9", "call-bound": "^1.0.4", "for-each": "^0.3.5", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" } }, "sha512-JMh8aK+1B/0bk/YNupICmH5MgCq6yNLKYQUfsZtSDLLCCmxUkHjUY+oOlIa90lO7lHN1woAfpenLdlKpXy9o+A=="], @@ -2854,6 +2860,8 @@ "cliui/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + "cross-spawn/which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "hono-openapi/@hono/standard-validator": ["@hono/standard-validator@0.4.0", "", { "peerDependencies": { "@standard-schema/spec": "^1.0.0", "hono": ">=4.11.2" } }, "sha512-MxOm1asDh9j7c1D0KiWwppSbFgAQxTRO4YEhjMrJn3lF1BIcXenPdgjSGKDRfQmZdaTaMA8slQLETgVOKyJxFw=="], "hono-openapi/@standard-community/standard-json": ["@standard-community/standard-json@0.3.6", "", { "dependencies": { "quansync": "^0.2.11" }, "peerDependencies": { "@standard-schema/spec": "^1.0.0", "@types/json-schema": "^7.0.15", "@valibot/to-json-schema": "^1.3.0", "arktype": "^2.1.20", "effect": "^3.20.0", "sury": "^10.0.0", "typebox": "^1.0.17", "valibot": "^1.4.2", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.24.5" }, "optionalPeers": ["@valibot/to-json-schema", "arktype", "effect", "sury", "typebox", "valibot", "zod", "zod-to-json-schema"] }, "sha512-tuXKz1Ps4al0W4xhfwii7v4lskKtOv3QE7mcouRVThrBdOR6JzY4Vea3nsxcEnte84bOJvMvXGmOMWE3IBbHyA=="], @@ -2930,6 +2938,8 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], + "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], "minipass-pipeline/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="],