diff --git a/VENDORED.md b/VENDORED.md index 057655fbb2..2e74db3680 100644 --- a/VENDORED.md +++ b/VENDORED.md @@ -23,7 +23,7 @@ never a convenience. ## Ledger Pinned to [faremeter/interchange](https://github.com/faremeter/interchange) -@ `79adc433` (origin/main). +@ `5453d0b0` (origin/main, v0.4.0). | Vendored path | What it is | Owner | Kill date | | ------------------------------ | ------------------------------------------------------------------------ | ------ | ---------- | @@ -44,8 +44,9 @@ Pinned to [faremeter/interchange](https://github.com/faremeter/interchange) | `vendor/intx/workflow-deploy` | `@intx/workflow-deploy` at a newer commit than npm has published | maintainers | 2026-11-03 | | `vendor/intx/workflow-host` | `@intx/workflow-host`, with a local step-grants-collapse delta and a temporary delta omitting an empty inbound `Subject` header instead of passing it through empty (kill condition: upstream tolerates an empty Subject, INTR-577) | maintainers | 2026-11-03 | -npm's published `0.3.0` predates this pin for every row above, so each is -re-vendored at the same commit rather than mixed pins; the root +Every row above is re-vendored at the same commit rather than mixed pins +(npm's `0.4.0` is this commit; rows with no local delta are un-vendoring +candidates); the root `package.json` `overrides` point each vendored name at `workspace:*` so their own `@intx/*` dependencies resolve onto the vendored copies too. Local deltas are restated in each package's own `VENDORED-FROM` file. diff --git a/agents/myra/package.json b/agents/myra/package.json index 08ab0fa875..16197b5021 100644 --- a/agents/myra/package.json +++ b/agents/myra/package.json @@ -37,17 +37,17 @@ "@corbits/deferred-tools": "workspace:*", "@corbits/mcp": "github:corbitsdev/corbits-mcp#9df1ab81d2f50d798bca2d2f44eb4759c92fac0e", "@corbits/memory": "github:corbitsdev/corbits-memory#8be608becc749fa0795a899826277823dca6db3e", - "@intx/tools-mail": "0.3.0", - "@intx/tools-posix": "0.3.0" + "@intx/tools-mail": "0.4.0", + "@intx/tools-posix": "0.4.0" }, "devDependencies": { "@types/bun": "catalog:", "typescript": "catalog:" }, "peerDependencies": { - "@intx/agent": "0.3.0", - "@intx/types": "0.3.0", - "@intx/workflow": "0.3.0" + "@intx/agent": "0.4.0", + "@intx/types": "0.4.0", + "@intx/workflow": "0.4.0" }, "corbits": { "workflow": { diff --git a/apps/hub/package.json b/apps/hub/package.json index 2b1b724718..c424488706 100644 --- a/apps/hub/package.json +++ b/apps/hub/package.json @@ -27,13 +27,13 @@ "@corbits/webhooks": "github:corbitsdev/webhooks#3f4f83147fd826d3aec5bcac7ef4b0ae7f75c00d", "@corbits/workflows": "workspace:*", "@corbits/xai-provider": "github:corbitsdev/corbits-xai-provider#9f2d4bac40ea075df092fef807404a13c638cf14", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", diff --git a/apps/sidecar/package.json b/apps/sidecar/package.json index 7472de6abb..de7ae399e4 100644 --- a/apps/sidecar/package.json +++ b/apps/sidecar/package.json @@ -1,6 +1,6 @@ { "name": "@workbench/sidecar", - "version": "0.3.0", + "version": "0.4.0", "private": true, "description": "Host process running a fleet of hub-orchestrated agent runtimes in one Bun process", "license": "SEE LICENSE IN ../../LICENSE.md", @@ -18,20 +18,20 @@ "@corbits/credential-header": "github:corbitsdev/corbits-credential-header#eee7b7eb79755fd1ddc3eb2b9a57c4e50982e293", "@corbits/credential-mcp": "github:corbitsdev/corbits-credential-mcp#fb9ec3a6754348cd33c388c70c0830b8844c83de", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", - "@intx/tools-lsp": "0.3.0", - "@intx/tools-mail": "0.3.0", - "@intx/tools-posix": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", + "@intx/tools-lsp": "0.4.0", + "@intx/tools-mail": "0.4.0", + "@intx/tools-posix": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/apps/sidecar/src/config.ts b/apps/sidecar/src/config.ts index e767d90351..79a9f47ef1 100644 --- a/apps/sidecar/src/config.ts +++ b/apps/sidecar/src/config.ts @@ -1,13 +1,27 @@ -// Boundary readers for the sidecar's env-config inputs, centralized so -// validation rules live in one place at the boot edge. +// Boundary readers for the sidecar's env-config inputs. +// +// Centralizing the readers here keeps env-validation rules in one +// place — `SIDECAR_CACHE_MAX_BYTES` must be a positive finite +// number, with a 10 GiB default — even though the sidecar only +// invokes the reader at one site today (the orchestrator boot in +// `apps/sidecar/src/index.ts`). The harness builder receives the +// resolved value through `DefaultHarnessBuilderConfig` rather than +// re-reading env, so the boundary stays at the boot edge. import { AdapterManifest } from "@intx/inference"; import { hexDecode } from "@intx/types"; const DEFAULT_CACHE_MAX_BYTES = 10 * 1024 * 1024 * 1024; -// Separate from the hub's CREDENTIAL_ENCRYPTION_KEY, so a sidecar -// disk-plus-key compromise can't also decrypt the hub's database. +// The operator key the sidecar seals its at-rest credential material under +// (inference-source apiKeys, and the tool credential material store). A +// SEPARATE key from the hub's `CREDENTIAL_ENCRYPTION_KEY`: the sidecar seals a +// local store on a host it does not control as tightly, so a sidecar +// disk-plus-key compromise must not also decrypt the hub's credential +// database, and the two keys rotate independently. Required at boot: a missing +// key fails loudly here rather than letting the sidecar run and persist secrets +// it cannot protect. 32 bytes, hex -- e.g. `openssl rand -hex 32`, the same +// shape as the hub's key. export function readCredentialEncryptionKey(): Uint8Array { const raw = process.env["SIDECAR_CREDENTIAL_ENCRYPTION_KEY"]; if (raw === undefined || raw.trim() === "") { @@ -26,7 +40,11 @@ export function readCacheMaxBytes(): number { return n; } -// Mirrors the hub's DEFAULT_HUB_MAX_TARBALL_BYTES. +// Mirrors the hub's `DEFAULT_HUB_MAX_TARBALL_BYTES`. The sidecar's +// HTTP-registry fetcher enforces this cap on every upstream registry +// tarball pull. An operator pointing the sidecar at a third-party +// registry whose curated tarballs run larger should raise the cap +// explicitly via `SIDECAR_REGISTRY_MAX_TARBALL_BYTES`. const DEFAULT_REGISTRY_MAX_TARBALL_BYTES = 10 * 1024 * 1024; export function readRegistryMaxTarballBytes(): number { @@ -39,8 +57,49 @@ export function readRegistryMaxTarballBytes(): number { return n; } -// Trusted operator input only: import(specifier) is arbitrary code -// execution, so a specifier must never originate from deploy or tenant data. +// Hub-link reconnect backoff, in milliseconds. Absent or whitespace-only +// yields `undefined`, which leaves the delay to the hub link's +// `DEFAULT_RECONNECT_DELAY_MS`; the constant therefore lives at one site +// instead of being restated here. A present value must be a positive +// integer, and anything else throws, so a typo fails the boot rather than +// silently reverting to the link's default: `Number` rejects the whole +// string rather than reading a leading numeric prefix, so a unit suffix +// like "3000ms" throws instead of arriving as 3000, and `Number.isInteger` +// rejects a fractional value rather than truncating it. +// +// Production never sets this. The deploy-flow test harness sets a short +// value so reconnect-survival tests that do not assert the delay itself +// (they assert the reconnect's recovery semantics, not the backoff +// duration) do not burn 3s of wall clock per dropped link. +// +// Takes the raw string rather than reading `process.env` itself: the +// sidecar's only caller is its module-top-level boot, which cannot be +// imported without booting a sidecar, so the validation rule is reachable +// from a test only as a pure function of its input. +export function parseReconnectDelayMs(raw: string | undefined): number | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + const n = Number(raw); + if (!Number.isInteger(n) || n <= 0) { + throw new Error( + `SIDECAR_RECONNECT_DELAY_MS must be a positive integer (milliseconds), got ${raw}`, + ); + } + return n; +} + +// Operator-configured custom inference adapter manifest. The value is +// TRUSTED operator input read only from this process's environment; +// `import(specifier)` is arbitrary code execution, so a specifier must +// never originate from deploy or tenant data — the agent deploy tree +// carries only a `provider` key, never a specifier. The shape is +// arktype-validated here (and re-validated at the workflow-child spawn +// boundary as defense in depth), but the value itself is trusted. +// +// Unset or whitespace-only means "no custom adapters", a valid +// configuration — the sidecar then resolves only the statically-linked +// built-ins. A present-but-malformed value fails loud at boot: an +// opaque `JSON.parse` SyntaxError is rethrown naming the env key, and +// the parsed value is asserted against `AdapterManifest`. export function readAdapterManifest(): AdapterManifest { const raw = process.env["SIDECAR_ADAPTER_MANIFEST"]; if (raw === undefined || raw.trim() === "") return []; diff --git a/apps/sidecar/src/index.ts b/apps/sidecar/src/index.ts index 82708336c9..f13ba28c03 100644 --- a/apps/sidecar/src/index.ts +++ b/apps/sidecar/src/index.ts @@ -23,6 +23,7 @@ import { createTarballCache } from "@intx/tool-packaging"; import { loadAdapterRegistry } from "@intx/inference/providers"; import { + parseReconnectDelayMs, readAdapterManifest, readCacheMaxBytes, readCredentialEncryptionKey, @@ -84,15 +85,45 @@ const cacheRoot = const cacheMaxBytes = readCacheMaxBytes(); const registryMaxTarballBytes = readRegistryMaxTarballBytes(); -// Custom modules import eagerly here so a bad specifier fails at boot, -// not first inference. The child rebuilds an equivalent registry from -// the serialized manifest since it can't receive this object across the fork. +// Operator-configured custom inference adapters, resolved once at the +// boot edge. `loadAdapterRegistry` merges the statically-linked +// built-ins with any custom adapters the manifest names, importing each +// custom module eagerly here so a bad specifier fails the sidecar at +// boot rather than at first inference. The SAME registry backs the +// deploy router's source-admission check below; the workflow child +// cannot receive this object across the fork, so the validated +// manifest is serialized into the child's spawn env (see +// `multistepSubstrateEnv`) and the child rebuilds an equivalent +// registry from it. Specifiers are operator-config-only — the agent +// deploy tree never contributes one. const adapterManifest = readAdapterManifest(); const adapters = await loadAdapterRegistry(adapterManifest); -// A file, not stdout, since the spawn fixture caps its stdout drain buffer -// below what a few-hundred-message run emits. Observability-only; a failed -// append surfaces rather than silently yielding an empty result set. +// Phase 4.7 latency-gate hook. When `SIDECAR_LATENCY_BENCH_FILE` names a +// path, the deploy router wires the supervisor's `onDispatchTiming` +// observer to append a parseable line per per-message dispatch boundary +// to that file, which the benchmark harness reads after the run. A file +// (rather than stdout) is the channel because the spawn fixture caps its +// stdout drain buffer, and a few-hundred-message run emits more lines +// than that cap holds. Resolved here at the boot edge (the one layer +// that reads env) so no non-boundary site re-decides what the absent +// value means; an unset path leaves the observer unwired and the +// supervisor's dispatch path untouched. This is observability-only -- no +// control-flow effect -- and is the single benchmark-side instrumentation +// hook the 4.7 gate adds. The synchronous append (not the logger) is the +// raw measurement channel: it is not application logging and must not be +// formatted or level-gated. A failed append surfaces (it is not +// swallowed) so a broken benchmark channel does not silently yield an +// empty result set. +// Two line shapes share the channel, discriminated by `mark.kind`. Both +// lead with the per-message id (the mail's Message-ID), the key the +// per-message fits group on: +// roundtrip: ` ` (4.7 latency gate) +// leg: ` leg [runsFanOut consumedFanOut looseObjects gitBytes]` +// (D2 per-leg attribution; the trailing four counters are +// present only on the `end` phase). The leg shape is a +// strict superset prefixed with the literal `leg` token, so +// a reader can split on whitespace and branch on field 2. const latencyBenchFile = process.env["SIDECAR_LATENCY_BENCH_FILE"]; const onDispatchTiming: ((mark: DispatchTimingMark) => void) | undefined = latencyBenchFile !== undefined && latencyBenchFile.trim() !== "" @@ -111,8 +142,14 @@ const onDispatchTiming: ((mark: DispatchTimingMark) => void) | undefined = } : undefined; -// Measurement-only: forces a repack every Nth message to discriminate -// pack-growth from tree-fan-out, never to run in production. +// D2 §10c forced-repack A/B toggle. When `SIDECAR_REPACK_EVERY_MESSAGES` +// names a positive integer, the supervisor forces a `git gc`/repack of +// the workflow-run repo every Nth dispatched message (under the +// single-writer discipline). Resolved here at the boot edge alongside the +// timing channel; absent or non-positive => the supervisor forks no +// `git gc` and the dispatch path is untouched. Measurement-only: this +// exists to discriminate pack-growth from tree-fan-out, never to run in +// production. const repackEveryRaw = process.env["SIDECAR_REPACK_EVERY_MESSAGES"]; let repackEveryMessages: { everyMessages: number } | undefined; if (repackEveryRaw !== undefined && repackEveryRaw.trim() !== "") { @@ -125,8 +162,15 @@ if (repackEveryRaw !== undefined && repackEveryRaw.trim() !== "") { repackEveryMessages = { everyMessages: parsed }; } -// The longest window a re-submitted message must still be caught as a -// duplicate; must be >= the max redelivery window of any at-least-once source. +// Consumed-dedup retention horizon (ms). Resolved here at the boot edge +// -- the single layer that owns operator config -- and threaded into +// every supervisor the deploy router constructs. Absent or empty => +// the supervisor applies its 24h default. This is the OPERATOR-policy +// value: the longest window in which the same message could +// legitimately be re-submitted and still must be caught as a duplicate +// (the consumed/ dedup index retains at least this long before the +// retention watermark prunes it). It must be >= the maximum redelivery +// window of any at-least-once mail source if one is ever added. const consumedRetentionRaw = process.env["CONSUMED_RETENTION_MS"]; let consumedRetentionMs: number | undefined; if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { @@ -139,8 +183,11 @@ if (consumedRetentionRaw !== undefined && consumedRetentionRaw.trim() !== "") { consumedRetentionMs = parsed; } -// On expiry the supervisor kills the child and rejects the spawn, so a -// child that never signals ready fails the deploy instead of hanging it. +// Bound on the child's spawn-time `ready` handshake. Threaded to every +// per-deployment supervisor; on expiry the supervisor kills the child and +// rejects the spawn, so a child that spawns but never signals ready fails +// the deploy (or is skipped by boot-time restore) instead of hanging it. +// Absent, the supervisor applies its 30s default. const readyTimeoutRaw = process.env["CHILD_READY_TIMEOUT_MS"]; let readyTimeoutMs: number | undefined; if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { @@ -153,43 +200,50 @@ if (readyTimeoutRaw !== undefined && readyTimeoutRaw.trim() !== "") { readyTimeoutMs = parsed; } -// Production never sets this; test harnesses set a short value so -// reconnect tests don't burn 3s of wall clock per dropped link. -const reconnectDelayRaw = process.env["SIDECAR_RECONNECT_DELAY_MS"]; -let reconnectDelayMs: number | undefined; -if (reconnectDelayRaw !== undefined && reconnectDelayRaw.trim() !== "") { - const parsed = Number.parseInt(reconnectDelayRaw, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error( - `SIDECAR_RECONNECT_DELAY_MS must be a positive integer (milliseconds), got ${reconnectDelayRaw}`, - ); - } - reconnectDelayMs = parsed; -} - -// Sweeps orphaned tmp staging dirs left by a crash between staging and -// rename on a previous boot. +// Hub-link reconnect backoff. Resolved here at the boot edge -- the single +// layer that owns operator config -- and forwarded to the orchestrator's hub +// link below. `parseReconnectDelayMs` owns the validation rule and the +// absent-means-link-default semantics. +const reconnectDelayMs = parseReconnectDelayMs(process.env["SIDECAR_RECONNECT_DELAY_MS"]); + +// Sweep any tmp staging directories left behind by a `put` or +// `extractTarball` that crashed between staging and the final rename +// on a previous boot. Running here, before the orchestrator starts +// accepting apply work, keeps the cache root from accumulating +// orphans for the lifetime of the sidecar's data directory. await createTarballCache({ rootDir: cacheRoot, maxBytes: cacheMaxBytes, }).sweepOrphans(); -// One key, one identity: signs every workflow-run commit and every -// SSH-signed commit, and is re-used by the child via spawn-time env vars. +// Load or mint the sidecar's local Ed25519 keypair. The supervisor +// principal signs every workflow-run commit with this key; the +// substrate's `signingCallback` signs every SSH-signed commit with +// it; the workflow-host child's substrate factory re-uses it via the +// `SIDECAR_SIGNING_*` spawn-time env vars. One key, one identity for +// the sidecar process. const SIDECAR_SIGNING_DIR = path.join(dataDir, ".sidecar-signing"); const sidecarSigningKey = await loadOrMintSidecarKeypair(SIDECAR_SIGNING_DIR); -// Wrapped below by the boot-edge facade so a successful workflow-run -// write fires the pack push hook before its Promise resolves. +// Construct the substrate-backed RepoStore at the boot edge. The +// supervisor consumes this through the deploy router; the supervisor's +// conversation-state writes reach `writeTreePreservingPrefix` on this +// store. The boot-edge facade below wraps the store so a successful +// workflow-run write fires the pack push hook before its Promise +// resolves. const agentRepoStore = createAgentRepoStore({ dataDir, signingKey: sidecarSigningKey, }); -// Loaded here at boot so a restart keeps every previously-cached key; the -// durable-write primitive is injected so the write stays at least as -// durable as the run-grants write it gates. +// Cache of the hub-vouched public keys of senders this sidecar's deployments +// are authorized to receive mail from. The grants handler writes each key the +// hub co-delivers on a `run.grants` frame; the recipient's inbound-mail verify +// reads it back. The keyring is loaded here at boot so a restart keeps every +// previously-cached key. The durable-write primitive is injected so the cache +// write is atomic and fsynced -- at least as durable as the run-grants write it +// gates -- while the cache itself stays in @intx/hub-agent. const senderKeyCache = await createSenderKeyCache({ dataDir, writeFileDurable: (filePath, contents) => @@ -197,33 +251,74 @@ const senderKeyCache = await createSenderKeyCache({ removeFileDurable: (filePath) => removeFileAtomicDurable(filePath), }); -// Built here so the hub link stays source-opaque, resolving address -> crypto -// without holding the cache or knowing where the key came from. +// The read side of the same cache: the inbound signature verify resolves a +// sender address to the crypto that verifies its mail. Built here at the edge +// so the hub link stays source-opaque -- it resolves address -> crypto without +// holding the cache or knowing where the key came from. const resolveSenderCrypto = createSenderCryptoResolver(senderKeyCache); -// Mirrors resolveSenderCrypto's opacity for inbound-mail admission policies. +// Per-recipient-address registry of resolved inbound-mail admission policies. +// The workflow-host wiring resolves each hydrated deployment's authored +// `inboundMailPolicy` into this registry beside its mail-router registration +// and removes it on teardown; the read side below hands the hub-link's +// `mail.inbound` seam a total policy for every address. Built here at the edge +// so the hub link stays policy-source-opaque, mirroring `resolveSenderCrypto`. const inboundMailPolicyRegistry = createInboundMailPolicyRegistry(); const lookupInboundMailPolicy = createInboundMailPolicyLookup(inboundMailPolicyRegistry); -// The facade resolves this mapping when firing the pack push so outbound -// frames carry the right agentAddress for hub-side routing. +// The deploy router records `(runId -> agentAddress)` here on +// every inbound `agent.deploy`; the facade resolves the mapping when +// firing the pack push so the outbound frames carry the right +// agentAddress for hub-side routing. const deploymentAddressRegistry = createDeploymentAddressRegistry(); -// Mail for an address with no registered handler is logged-and-dropped. +// Per-deployment-address mail handler registry the hub-link consults on +// every inbound `mail.inbound` frame. The deploy router's multi-step +// branch registers `wired.routeInbound` against the deployment's mail +// address once its supervisor spawns; the hub-link's `mail.inbound` +// handler calls `tryRoute` so an inbound deployment-address message +// lands on the supervisor's mail-bus subscription. Mail for an address +// with no registered handler has no receiver and is logged-and-dropped. const multistepMailRouter = createMultistepMailRouter(); -// The child commits the resulting SignalReceived through its own -// substrate, the single writer of the workflow-run repo sidecar-side. +// Per-deployment-address signal handler registry the hub-link consults +// on every inbound `signal.deliver` frame. The multi-step deploy +// router registers a handler against the deployment's mail address +// once its supervisor spawns; the handler forwards the signal into the +// supervisor's `deliverSignal`, which sends a `signal.deliver` control +// IPC frame to the workflow-process child. The child commits the +// resulting `SignalReceived` event through its own substrate -- the +// single writer of the workflow-run repo on the sidecar side. const multistepSignalRouter = createMultistepSignalRouter(); -// Cancel-mode in-flight steps abort child-side; wait-mode steps continue. +// Per-deployment-address drain handler registry the hub-link consults +// on every inbound `drain.deliver` frame. The multi-step deploy router +// registers a handler against the deployment's mail address once its +// supervisor spawns; the handler forwards the drain into the +// supervisor's `drain`, which sends a `drain` control IPC frame to the +// workflow-process child and arms one drainTimeout accumulator per +// in-flight run. Cancel-mode in-flight steps abort on the child side; +// wait-mode steps continue. Accumulators commit a signed +// `CancelRequested{origin: "supervisor-drain"}` against the +// workflow-run repo when the deadline expires. const multistepDrainRouter = createMultistepDrainRouter(); -// The write is awaited so the frame's FIFO completion means grants are durable. +// Per-deployment-address grants handler registry the hub-link consults +// on every inbound `run.grants` frame. The deploy router registers a +// handler against the deployment's mail address once its supervisor +// spawns; the handler writes the run's grants to `runs//grants.json` +// inside the deployment's workflow-run repo, sibling to that run's +// `runs//events/` subtree. The write is awaited so the frame's +// FIFO completion means the grants are durable on disk. const multistepGrantsRouter = createMultistepGrantsRouter(); -// Only a single-step warm deployment registers a handler; a multi-step -// deployment registers none, so a rotation against its address is unrouted. +// Per-deployment-address sources-rotation handler registry. Only a +// single-step warm deployment registers a handler once its supervisor +// spawns; the handler forwards the rotated list into the supervisor's +// `deliverSources`, which sends a `sources-updated` control IPC frame to +// the child, where the warm agent's live sources are swapped in place. A +// multi-step deployment registers none, so a rotation resolved against +// its address is unrouted. const multistepSourcesRouter = createMultistepSourcesRouter(); // Per-deployment credential-delivery handler registry. A deployment registers // its handler after `spawn` (any deployment with a supervisor, not only warm @@ -233,7 +328,11 @@ const multistepCredentialsRouter = createMultistepCredentialsRouter(); const transport = createInMemoryTransport(); -// Consulted lazily since createSidecarOrchestrator calls createDeployRouter +// The pack-push client closes over the substrate (for `createPack`) +// and a lazy hub-link binding (for `pushWorkflowRunPack`). The link +// reference is set once the orchestrator is constructed below; the +// closure here is consulted lazily because the +// `createSidecarOrchestrator` factory calls `createDeployRouter` // during its constructor, before the orchestrator handle is bound. let resolvedHubLink: HubLink | null = null; const workflowRunPackClient = createWorkflowRunPackClient({ @@ -257,7 +356,11 @@ const restoreWorkflowRunPack = createWorkflowRunPackRestorer({ markRestored: workflowRunPackClient.markRestored, }); -// Non-workflow-run writes (today, the agent-state deploy-applier path) flow through unchanged. +// Wrap the substrate's RepoStore with the boot-edge facade so a +// successful supervisor write against a workflow-run repo fires the +// pack push hook before its Promise resolves. Non-workflow-run writes +// (today, the agent-state deploy-applier path) flow through +// unchanged. const wrappedRepoStore = createWorkflowRunPackPushingRepoStore({ underlying: agentRepoStore.repoStore, packClient: workflowRunPackClient, @@ -268,9 +371,20 @@ const hubWsUrl = requireEnv("HUB_WS_URL"); const sidecarId = requireEnv("SIDECAR_ID"); const sidecarToken = requireEnv("SIDECAR_TOKEN"); -// PATH/HOME/TMPDIR propagate from the boot edge so the child's shebang can -// resolve bun and tmp-file APIs land on the host's temp root; the -// SubstrateConfig validator ignores these as undeclared keys. +// Multi-step substrate-config the deploy router threads into the +// workflow-process child's spawn-time env. The child's substrate +// factory consumes these via the typed `SubstrateConfig` validator so +// the per-step pack-push wrap can identify the deployment's hub-side +// trust anchors. Today the IPC bridge in `pack.push.request` carries +// the pack; the WebSocket-connection keys are reserved for a future +// child-local hub link without redoing the boot-edge wiring. +// +// `PATH`, `HOME`, and `TMPDIR` are propagated from the boot edge's own +// environment so the child's `#!/usr/bin/env bun` shebang can resolve +// `bun`, agent code can find a writable home, and tmp-file APIs land +// on the same temp root the host uses. The substrate-config validator +// ignores undeclared keys, so these are visible to the OS for binary +// lookup but invisible to the typed `SubstrateConfig` shape. const multistepSubstrateEnv: Record = { SIDECAR_DATA_DIR: dataDir, SIDECAR_SIGNING_PUBLIC_KEY: hexEncode(sidecarSigningKey.publicKey), @@ -279,10 +393,20 @@ const multistepSubstrateEnv: Record = { SIDECAR_ID: sidecarId, SIDECAR_TOKEN: sidecarToken, PATH: requireEnv("PATH"), + // Tool-loader caps the child's per-step tool materialization uses. + // Resolved once at the boot edge and threaded into the child through + // the substrate config so the child does not re-read env at a + // non-boundary site. SIDECAR_CACHE_MAX_BYTES: String(cacheMaxBytes), SIDECAR_REGISTRY_MAX_TARBALL_BYTES: String(registryMaxTarballBytes), - // The already-validated manifest, not the raw env string, so the child - // rebuilds the same set; the child re-validates before importing any module. + // Serialize the parent's ALREADY-VALIDATED manifest (the object + // `readAdapterManifest` returned), not the raw env string, so the + // child rebuilds the same custom-adapter set. Always present (defaults + // to "[]" when no custom adapters are configured); the child treats a + // missing key as a serialization bug and fails loud. The child + // re-validates the shape before importing any module — defense in + // depth at the deserialization boundary, since the child env is + // operator-controlled via Bun.spawn. SIDECAR_ADAPTER_MANIFEST: JSON.stringify(adapterManifest), }; const hostHome = process.env["HOME"]; @@ -299,9 +423,19 @@ if (hostTmpdir !== undefined) { // than a second copy of the check. const buildHarness = createDefaultHarnessBuilder({ adapters }); -// Answers workflow.probe.request with a real inert projection instead of -// the hub-link's rejecting placeholder; scratch dir rooted here to share -// the sidecar data dir's lifecycle. +// Airlocked workflow-probe executor, assembled at the boot edge and +// injected through the orchestrator so the live sidecar answers +// `workflow.probe.request` with a real inert projection instead of the +// hub-link's rejecting placeholder. The host-side `MaterializeWorkflowClosure` +// lays out a probe frame's frozen closure under a per-probe scratch dir +// (rooted here so it shares the sidecar data dir's lifecycle); the executor +// spawns the one-shot child that evaluates the workflow entry against it. +// The cache/registry caps are the same boot-edge-resolved values the tool +// loader uses. +// The probe delivers source assets inline in one frame (see +// `WorkflowProbeRequestFrame`). A git-sourced asset that grows past the shared +// inline-payload cap is the signal to move the probe's asset delivery to the +// streamed transfer the deploy path uses. const workflowProbeExecutor = createWorkflowProbeExecutor({ materialize: createWorkflowClosureMaterializer({ cacheRoot, @@ -371,10 +505,19 @@ const orchestrator = createSidecarOrchestrator({ // workflow_run.public_key); the link stays source-opaque and reports whatever // this returns. getCachedSenderAddresses: () => senderKeyCache.rotatableAddresses(), - // Re-drives a pack the disconnect cancelled; fires after the reconnect frame - // since both frame families queue on the hub's per-connection chain. + // When the hub-link re-announces a deployment address in an authenticated + // reconnect, re-drive any workflow-run pack the disconnect cancelled. The + // link fires this AFTER sending the reconnect frame, so the hub routes the + // address before it sees the re-shipped pack (both frame families queue on + // the hub's per-connection chain). This is the liveness half of + // reconnect recovery: without it, a synchronous single-step run whose only + // pack was interrupted mid-transfer never re-ships, because it has no later + // local write to re-arm the coalescing loop. onWorkflowAddressesRoutable: (addresses) => { - // Assert rather than optional-chain so a wiring regression fails loud. + // The deploy router is captured synchronously during orchestrator + // construction, before the link ever connects, so a routable callback can + // only fire once it exists; assert (like `getWorkflowAddresses`) rather + // than optional-chain so a wiring regression fails loud. if (sidecarDeployRouter === undefined) { throw new Error( "sidecar boot: deploy router was not constructed before a reconnect made workflow addresses routable", @@ -382,8 +525,25 @@ const orchestrator = createSidecarOrchestrator({ } for (const address of addresses) { wrappedRepoStore.notifyAddressRoutable(address); - // Re-registers parked correlations a long outage may have evicted from - // the link's send queue; dedups hub-side on the correlationId constraint. + // Trigger B: re-register the deployment's parked correlations. A long hub + // outage can evict a `signal.correlation.register` frame from the link's + // bounded send queue; re-driving it on reconnect recovers the parked + // run's approvability. Two independent facts make this safe: + // - The hub-side co-write's `registerSignalCorrelation` throws only when + // the deployment has no `status = "deployed"` row. That row is written + // at DEPLOY time and persists across the outage -- the reconnect + // re-routes the address in the hub's in-memory index but writes no DB + // status, so the lookup already resolves. (The frame also lands after + // the address is wire-routable: the link fires this after the + // reconnect frame, and both frames queue on the hub's per-connection + // chain -- see the pack re-ship note above.) + // - A sidecar restart re-emits the parked set twice (child + // re-establishment fires Trigger A too), but the co-write dedups on + // the `correlationId` PK/unique constraints via `onConflictDoNothing`, + // so the duplicate is a no-op. The pre-existing non-transactional + // status-check window in `registerSignalCorrelation` that this + // widened concurrency exercises is tracked in INTR-338, out of scope + // here. sidecarDeployRouter.reEmitParkedCorrelations(address); } }, diff --git a/apps/sidecar/src/workflow-substrate-factory.ts b/apps/sidecar/src/workflow-substrate-factory.ts index bb719e39b5..2270798427 100644 --- a/apps/sidecar/src/workflow-substrate-factory.ts +++ b/apps/sidecar/src/workflow-substrate-factory.ts @@ -842,6 +842,10 @@ export function createSidecarRunChild(deps: SidecarRunChildDeps): RunChildWorkfl childRunId, parentRunId, onEvent, + // Terminal: the caller awaits this child's terminal rather than driving + // it across parks, and no signal can be addressed to a child run, so a + // park inside it has nothing that could answer it. + hasUpstreamSignalResolver: false, ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), }); try { @@ -918,7 +922,14 @@ export function createSidecarSpawnSuspendableChild(deps: SidecarRunChildDeps): R childRunId, parentRunId, onEvent, - // A grandchild spawned from the body inherits this through the recursive spawnChild. + // The run's live credential-material cell so the body's inference resolves + // its source secret against the parent's current delivery; a grandchild + // spawned from the body inherits it through the recursive spawnChild. + // + // Park-aware: the container drives this body across its parks and relays + // a decision back down onto the body's own channel, so a park here is + // answerable even though the body run carries no address of its own. + hasUpstreamSignalResolver: true, ...(credentialMaterial !== undefined ? { materialCell: credentialMaterial } : {}), }); @@ -1001,6 +1012,14 @@ async function buildChildRunEnv(args: { onEvent: (event: InferenceEvent) => void; /** Absent when a non-sidecar executor carries no credential material, leaving the child's inference reader unset. */ materialCell?: CredentialMaterialCell; + /** + * Whether a park in this child can be answered from outside it. The two + * seams that share this builder differ precisely here, so neither gets a + * default: a suspendable body is driven across its parks by a container + * that relays a decision back down, while a terminal child has no address + * and no relay above it. + */ + hasUpstreamSignalResolver: boolean; }): Promise<{ env: WorkflowRuntimeEnv; signalChannel: ReturnType; @@ -1165,6 +1184,7 @@ async function buildChildRunEnv(args: { clock, newId, drain, + hasUpstreamSignalResolver: args.hasUpstreamSignalResolver, ...(loopFns !== undefined ? { loopFns } : {}), }; // Assigned after the env literal since the iteration host closes over env. diff --git a/bun.lock b/bun.lock index ffcfbcfd59..6082af939f 100644 --- a/bun.lock +++ b/bun.lock @@ -16,11 +16,11 @@ "@intx/crypto": "workspace:*", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/log": "workspace:*", - "@intx/mime": "0.3.0", - "@intx/types": "0.3.0", + "@intx/mime": "0.4.0", + "@intx/types": "0.4.0", "@types/bun": "1.3.14", "@typescript/native-preview": "7.0.0-dev.20260707.2", "drizzle-orm": "catalog:", @@ -40,17 +40,17 @@ "@corbits/deferred-tools": "workspace:*", "@corbits/mcp": "github:corbitsdev/corbits-mcp#9df1ab81d2f50d798bca2d2f44eb4759c92fac0e", "@corbits/memory": "github:corbitsdev/corbits-memory#8be608becc749fa0795a899826277823dca6db3e", - "@intx/tools-mail": "0.3.0", - "@intx/tools-posix": "0.3.0", + "@intx/tools-mail": "0.4.0", + "@intx/tools-posix": "0.4.0", }, "devDependencies": { "@types/bun": "catalog:", "typescript": "catalog:", }, "peerDependencies": { - "@intx/agent": "0.3.0", - "@intx/types": "0.3.0", - "@intx/workflow": "0.3.0", + "@intx/agent": "0.4.0", + "@intx/types": "0.4.0", + "@intx/workflow": "0.4.0", }, }, "apps/hub": { @@ -72,13 +72,13 @@ "@corbits/webhooks": "github:corbitsdev/webhooks#3f4f83147fd826d3aec5bcac7ef4b0ae7f75c00d", "@corbits/workflows": "workspace:*", "@corbits/xai-provider": "github:corbitsdev/corbits-xai-provider#9f2d4bac40ea075df092fef807404a13c638cf14", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", @@ -98,7 +98,7 @@ }, "apps/sidecar": { "name": "@workbench/sidecar", - "version": "0.3.0", + "version": "0.4.0", "bin": { "workflow-child": "./bin/workflow-child", }, @@ -106,20 +106,20 @@ "@corbits/credential-header": "github:corbitsdev/corbits-credential-header#eee7b7eb79755fd1ddc3eb2b9a57c4e50982e293", "@corbits/credential-mcp": "github:corbitsdev/corbits-credential-mcp#fb9ec3a6754348cd33c388c70c0830b8844c83de", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", - "@intx/tools-lsp": "0.3.0", - "@intx/tools-mail": "0.3.0", - "@intx/tools-posix": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", + "@intx/tools-lsp": "0.4.0", + "@intx/tools-mail": "0.4.0", + "@intx/tools-posix": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -174,7 +174,7 @@ "name": "@corbits/approvals", "version": "0.0.1", "dependencies": { - "@intx/authz": "0.3.0", + "@intx/authz": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/types": "workspace:*", @@ -183,7 +183,7 @@ "hono": "catalog:", }, "devDependencies": { - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@types/bun": "catalog:", "postgres": "catalog:", "typescript": "catalog:", @@ -201,10 +201,10 @@ "typescript": "catalog:", }, "peerDependencies": { - "@intx/agent": "0.3.0", - "@intx/inference": "0.3.0", - "@intx/log": "0.3.0", - "@intx/types": "0.3.0", + "@intx/agent": "0.4.0", + "@intx/inference": "0.4.0", + "@intx/log": "0.4.0", + "@intx/types": "0.4.0", }, }, "packages/error-sink": { @@ -219,7 +219,7 @@ "typescript": "catalog:", }, "peerDependencies": { - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", }, }, "packages/url-path": { @@ -235,10 +235,10 @@ "version": "0.0.1", "dependencies": { "@corbits/error-sink": "workspace:*", - "@intx/authz": "0.3.0", + "@intx/authz": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -254,10 +254,10 @@ }, "vendor/intx/agent": { "name": "@intx/agent", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:", @@ -269,12 +269,12 @@ }, "vendor/intx/db": { "name": "@intx/db", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", - "@intx/hub-common": "0.3.0", - "@intx/log": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", + "@intx/hub-common": "0.4.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", @@ -288,11 +288,11 @@ }, "vendor/intx/harness": { "name": "@intx/harness", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/log": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*", }, "devDependencies": { @@ -304,14 +304,14 @@ }, "vendor/intx/hub-agent": { "name": "@intx/hub-agent", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/harness": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", "isomorphic-git": "catalog:", @@ -324,18 +324,18 @@ }, "vendor/intx/hub-api": { "name": "@intx/hub-api", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@hono/standard-validator": "^0.2.2", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.3.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:", @@ -357,7 +357,7 @@ }, "vendor/intx/hub-common": { "name": "@intx/hub-common", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/types": "workspace:*", }, @@ -368,18 +368,18 @@ }, "vendor/intx/hub-sessions": { "name": "@intx/hub-sessions", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -395,9 +395,9 @@ }, "vendor/intx/inference": { "name": "@intx/inference", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", }, @@ -408,10 +408,10 @@ }, "vendor/intx/mail-memory": { "name": "@intx/mail-memory", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { - "@intx/crypto": "0.3.0", - "@intx/log": "0.3.0", + "@intx/crypto": "0.4.0", + "@intx/log": "0.4.0", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", @@ -424,9 +424,9 @@ }, "vendor/intx/mailbox": { "name": "@intx/mailbox", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:", @@ -438,9 +438,9 @@ }, "vendor/intx/mime": { "name": "@intx/mime", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", }, @@ -451,7 +451,7 @@ }, "vendor/intx/types": { "name": "@intx/types", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "arktype": "catalog:", "semver": "catalog:", @@ -464,7 +464,7 @@ }, "vendor/intx/workflow": { "name": "@intx/workflow", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/agent": "workspace:*", "@intx/inference": "workspace:*", @@ -478,10 +478,10 @@ }, "vendor/intx/workflow-deploy": { "name": "@intx/workflow-deploy", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/tool-packaging": "0.3.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", }, @@ -493,17 +493,17 @@ }, "vendor/intx/workflow-host": { "name": "@intx/workflow-host", - "version": "0.3.0", + "version": "0.4.0", "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.3.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "arktype": "catalog:", @@ -519,24 +519,26 @@ "@corbits/mailbox", ], "overrides": { + "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core#3028f8604155eee63f5a1711f0d4022d7984e3f7", + "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses#7efe0006da9d94ac9037c1389f23d3d6163df4a5", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/inference-catalog": "0.3.0", - "@intx/log": "0.3.0", + "@intx/inference-catalog": "0.4.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", @@ -744,9 +746,9 @@ "@intx/agent": ["@intx/agent@workspace:vendor/intx/agent"], - "@intx/authz": ["@intx/authz@0.3.0", "", { "dependencies": { "@intx/types": "0.3.0" } }, "sha512-kyIKQAVjfB8H84mES0QA4cNnYfpPoyLmQITVBxiuXv1VO/K74TQZnCDpoo0IL7oFk2q7bbDXSAEE4ubMBVTHzA=="], + "@intx/authz": ["@intx/authz@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-A9x4EzcECnFjhDoqj0XjaOb1GaxRyJucXOuIbY+fcPluUuMCGFvExD6DJZwjJmIYdeCvL8i96gPkncu/fAR9qA=="], - "@intx/crypto": ["@intx/crypto@0.3.0", "", { "dependencies": { "@intx/types": "0.3.0" } }, "sha512-NsRzvkFGb0Pcsm9uLFFSXNl6Cbu+ii6VkYAw81qs9IbZzbAFfsZSUVQaX6EZuPNyJ8KK4KmBtqmioq1mhXsJww=="], + "@intx/crypto": ["@intx/crypto@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-4/qWypC90GevbV9u1i/93tn46jsXhj8wE1Pn+awg8/ECpwhHnhPLT5Uy423Arr9eGmRmF6HiMyXcEgH1Z9/Rkw=="], "@intx/db": ["@intx/db@workspace:vendor/intx/db"], @@ -762,7 +764,7 @@ "@intx/inference": ["@intx/inference@workspace:vendor/intx/inference"], - "@intx/log": ["@intx/log@0.3.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-iooPSZjiEUO1A91X5STRhOAk+2RprliB4nbriV/dOOz86kuxZ60y5T8ISQ5Jaj1US870gntN6QwnefY38aWi2Q=="], + "@intx/log": ["@intx/log@0.4.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-f1IPZ/9YpEqlA0I2UZPOlRJOTDoOFEZRwZoUVuw8DpgR1WatOMm4hA0E8o6Mtz+pA0ux7gVdls4y/JZcgypsfQ=="], "@intx/mail-memory": ["@intx/mail-memory@workspace:vendor/intx/mail-memory"], @@ -770,17 +772,17 @@ "@intx/mime": ["@intx/mime@workspace:vendor/intx/mime"], - "@intx/pack-transport": ["@intx/pack-transport@0.3.0", "", { "dependencies": { "@intx/types": "0.3.0" } }, "sha512-EQ1wM16323aIFhw3Ak6HwB8YqzKSAfRmJUwjIfqyXk2t6TGv4grws6dIs7Yau1dc372s/SO+GhvFaKgvI6XTWA=="], + "@intx/pack-transport": ["@intx/pack-transport@0.4.0", "", { "dependencies": { "@intx/types": "0.4.0" } }, "sha512-I+axd067ZFXaTRtr2zmBXl6/yU253wLc/YWEdXiPiqz06mIspR2BFJMbSXPzV4pM26n+1tv/9E5VNBQtdiYmSA=="], - "@intx/storage-isogit": ["@intx/storage-isogit@0.3.0", "", { "dependencies": { "@intx/log": "0.3.0", "@intx/mime": "0.3.0", "@intx/types": "0.3.0", "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "^2.1.29", "buffer": "^6.0.3", "isomorphic-git": "^1.27.2" } }, "sha512-nzhl9WvxV66J1jjBfuAOjvwgiuS1DScnYomCPQrBxZNG35M5CYR3xyqR0RN0GfTj/7JlKWk6rF6Lu44aZDfzuw=="], + "@intx/storage-isogit": ["@intx/storage-isogit@0.4.0", "", { "dependencies": { "@intx/log": "0.4.0", "@intx/mime": "0.4.0", "@intx/types": "0.4.0", "@isomorphic-git/lightning-fs": "4.7.0", "arktype": "^2.1.29", "buffer": "^6.0.3", "isomorphic-git": "^1.27.2" } }, "sha512-3H4dhUsWmAixLEBp79vq9NVUf6Z4++pKzaODWFEgNfhLX2WeV5J/t6soUamL6T/8eok/SDgQ/sZiZMiJ5kDKOw=="], - "@intx/tool-packaging": ["@intx/tool-packaging@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/log": "0.3.0", "@intx/storage-isogit": "0.3.0", "@intx/types": "0.3.0", "arktype": "^2.1.29", "isomorphic-git": "^1.27.2", "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" } }, "sha512-Lr/xdEjXTindJRm+eITvfeqpW/yl71thYjr3HXl41GJ9wpuOZPCjm9vBoRVKhUSwmIUKBYbm2VplXCCXaS7f4g=="], + "@intx/tool-packaging": ["@intx/tool-packaging@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/log": "0.4.0", "@intx/storage-isogit": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29", "isomorphic-git": "^1.27.2", "npm-package-arg": "^12.0.2", "npm-pick-manifest": "^10.0.0", "npm-registry-fetch": "^19.0.0", "semver": "^7.7.2", "ssri": "^12.0.0", "tar": "^7.5.1" } }, "sha512-eBZIbGUcEBp0ppMCceA1PPd75LvXpUyWitqTIu7AsLxRNlU9gbE1hymWPXnhUKnt3LgSlHUi959Ehd6jG6mR7g=="], - "@intx/tools-lsp": ["@intx/tools-lsp@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/log": "0.3.0", "@intx/tools-posix": "0.3.0", "@intx/types": "0.3.0", "vscode-jsonrpc": "^9.0.1", "vscode-languageserver-types": "^3.17.5", "which": "^4.0.0" } }, "sha512-Ywnm9NNfCLlkzXTL1UoHyVvee4eaQ9rqKkkHM8zmf4l8L+ZqEPIW4IG5nERJG9F9XxOaf4gboqbFgqMe+AF5Dg=="], + "@intx/tools-lsp": ["@intx/tools-lsp@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/log": "0.4.0", "@intx/tools-posix": "0.4.0", "@intx/types": "0.4.0", "vscode-jsonrpc": "^9.0.1", "vscode-languageserver-types": "^3.17.5", "which": "^4.0.0" } }, "sha512-qkrO3dpnPBtyLEEPBrtEzTRhXiKI1mfrQlbehmnq5D7ZcdJ99/FiTlLWzb8WHuzsh7J26xdWOYKzwTp3V/U2Mw=="], - "@intx/tools-mail": ["@intx/tools-mail@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/types": "0.3.0", "arktype": "^2.1.29" } }, "sha512-DLI3+5mVhqdiW1VtSt4/lVRI8UFDp1Li7z/0Dfxqe44rdP2HvPRiZ8LmoSoZ8DdnGuC7dXT1MwNIN5kr9cCQCw=="], + "@intx/tools-mail": ["@intx/tools-mail@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29" } }, "sha512-z7AAXlGEi9Tivf3PPvw8h8Xr9lY4vYidXKX17qQG95iLV/OW8FCmaWXxuUVLBMtlxfXQKpFWKCicdSRbJsbvLQ=="], - "@intx/tools-posix": ["@intx/tools-posix@0.3.0", "", { "dependencies": { "@intx/agent": "0.3.0", "@intx/types": "0.3.0" } }, "sha512-0cQt5d7POpe/Wbee4pmnwTSux9JeyEy115X/96Ok0enNSIpAaPmHB61iRMuMHStS0hNF1e7DCpA7OH5Bi08Dbg=="], + "@intx/tools-posix": ["@intx/tools-posix@0.4.0", "", { "dependencies": { "@intx/agent": "0.4.0", "@intx/types": "0.4.0" } }, "sha512-ZqC3nEDBk5ix+lW9nWAyTynkXOGycsTfCXS24uSqwqLcivucSmGDk6kJ9zs+J0Ybz3tpq7rMmoQcvM3RuZprgQ=="], "@intx/types": ["@intx/types@workspace:vendor/intx/types"], @@ -1840,8 +1842,6 @@ "@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], - "@corbits/mcp/@corbits/credential-mcp": ["@corbits/credential-mcp@github:corbitsdev/corbits-credential-mcp#fb9ec3a", { "peerDependencies": { "@intx/harness": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-credential-mcp-fb9ec3a", "sha512-Co4jR+L3pfBbVYY7I6jRftoHeSb5+LCxzEEaAhVLYA3aY2PrPHZV37+DWxj3fcK/KdzboqbT3XSnqU8DyHRU9Q=="], - "@corbits/myra/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#4a2a3c2", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/agent": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-4a2a3c2", "sha512-UY5nkQ31kGmYgqgIOSlbigwvm7c/s2kpqEbF6Oqo0rzoS1RejpPp+w/Yim6h2MRpYCl5CXgAET1MhkDboHuWBA=="], "@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="], @@ -1860,11 +1860,13 @@ "@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "@workbench/hub/@corbits/mcp": ["@corbits/mcp@github:corbitsdev/corbits-mcp#9df1ab8", { "dependencies": { "@corbits/credential-mcp": "github:corbitsdev/corbits-credential-mcp#fb9ec3a6754348cd33c388c70c0830b8844c83de", "@intx/db": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "2.1.20", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" }, "peerDependencies": { "@intx/agent": ">=0.3.0" } }, "corbitsdev-corbits-mcp-9df1ab8", "sha512-5ILvgNI4J/U2kWitFKZmIJO1qybCdIkKZ73ZELu6PWrbw6tARxO8/ffaGoK0nIMrT4EcBZaMd6YkR0vLdPOWnw=="], + "@workbench/hub/@corbits/artifacts": ["@corbits/artifacts@github:corbitsdev/corbits-artifacts#4a2a3c2", { "dependencies": { "@hono/standard-validator": "^0.2.3" }, "peerDependencies": { "@intx/agent": "^0.3.0", "@intx/hub-api": "^0.3.0", "@intx/types": "^0.3.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.2", "hono": "^4.12.32", "hono-openapi": "^1.2.0", "postgres": "^3.4.9" } }, "corbitsdev-corbits-artifacts-4a2a3c2", "sha512-UY5nkQ31kGmYgqgIOSlbigwvm7c/s2kpqEbF6Oqo0rzoS1RejpPp+w/Yim6h2MRpYCl5CXgAET1MhkDboHuWBA=="], + + "@workbench/hub/@corbits/codex-provider": ["@corbits/codex-provider@github:corbitsdev/corbits-codex-provider#a1f475f", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-codex-provider-a1f475f", "sha512-D6d0lblWx/4KudfLwQoQGrLB8GKRLP/HVp1cx08L2/hOGZAgH6txLPpeEtBqjl+R520qXSSPfM0kDC+3XNaAfw=="], - "@workbench/hub/@corbits/memory": ["@corbits/memory@github:corbitsdev/corbits-memory#8be608b", { "dependencies": { "@intx/agent": "0.2.2", "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", "@intx/types": "^0.3.0", "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "corbitsdev-corbits-memory-8be608b", "sha512-7NcVcz3IHFTfx6ICQT+4gjX1PBNLqjlCqoEOPDFVW7wc4e2+Ueg9wOZQ+0MVttpQCaDg6MObl4kie410UQm/sQ=="], + "@workbench/hub/@corbits/mailbox": ["@corbits/mailbox@github:corbitsdev/corbits-mailbox#65590a8", { "dependencies": { "@hono/standard-validator": "0.2.3", "@intx/crypto": "0.3.0", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", "arktype": "2.1.29", "hono-openapi": "1.3.1" }, "peerDependencies": { "@intx/log": "^0.2.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.0", "postgres": "^3.4.0" } }, "corbitsdev-corbits-mailbox-65590a8", "sha512-tK7xXcL780iX3I+5R68gmyqgEA0uwYO1qdTDNn/LF6tOiFIm0lt2szjspJ8JMKMTzKC6SSai03p723rEGM1jOg=="], - "@workbench/web/@corbits/credential-mcp": ["@corbits/credential-mcp@github:corbitsdev/corbits-credential-mcp#fb9ec3a", { "peerDependencies": { "@intx/harness": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-credential-mcp-fb9ec3a", "sha512-Co4jR+L3pfBbVYY7I6jRftoHeSb5+LCxzEEaAhVLYA3aY2PrPHZV37+DWxj3fcK/KdzboqbT3XSnqU8DyHRU9Q=="], + "@workbench/hub/@corbits/xai-provider": ["@corbits/xai-provider@github:corbitsdev/corbits-xai-provider#9f2d4ba", { "dependencies": { "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core", "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses", "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-xai-provider-9f2d4ba", "sha512-ZjvKxGcp+QzCh8zeXHg7kNbtkpR/itZkmQAcdG1aOAOjHaNwfGVpkPc/c/XtwPNsQyaDjF8W9CHb4rH1xI2uCA=="], "better-call/@better-auth/utils": ["@better-auth/utils@0.5.0", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-BL8W4EfIZFwlu0r54m3v1ztjDhu6dDe/amLTm0xybmbZaNgYUqhD3SjpAsnq0q8YD6/ki4iwIgxJNLP/N3TxiA=="], @@ -1960,6 +1962,9 @@ "@esbuild-kit/core-utils/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.18.20", "", { "os": "win32", "cpu": "x64" }, "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ=="], + "@workbench/hub/@corbits/mailbox/hono-openapi": ["hono-openapi@1.3.1", "", { "peerDependencies": { "@hono/standard-validator": "^0.2.0", "@standard-community/standard-json": "^0.3.5", "@standard-community/standard-openapi": "^0.2.9", "@types/json-schema": "^7.0.15", "hono": "^4.11.2", "openapi-types": "^12.1.3" }, "optionalPeers": ["@hono/standard-validator", "hono"] }, "sha512-NLVeVkhKZ3drmQNEIPac8HX8Y54uf1hJAgIM/7MfDsaeVVmB+QILWQxx5x3R3NvRHgedcbEbOCGY2uR7WQYyMw=="], + + "cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], "minipass-flush/minipass/yallist": ["yallist@4.0.0", "", {}, "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="], diff --git a/package.json b/package.json index 652279feea..99a95dac05 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,8 @@ "@intx/crypto": "workspace:*", "@intx/log": "workspace:*", "@corbits/mailbox": "github:corbitsdev/corbits-mailbox#65590a85fa143251b3ac20ba2eca92fc35e70e51", - "@intx/mime": "0.3.0", - "@intx/types": "0.3.0", + "@intx/mime": "0.4.0", + "@intx/types": "0.4.0", "@types/bun": "1.3.14", "@typescript/native-preview": "7.0.0-dev.20260707.2", "jiti": "^2.7.0", @@ -43,7 +43,7 @@ "puppeteer-core": "^25.7.0", "typescript": "6.0.3", "@intx/hub-sessions": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-api": "workspace:*", "hono": "catalog:", "drizzle-orm": "catalog:" @@ -79,28 +79,30 @@ "overrides": { "arktype": "^2.2.0", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", "@intx/harness": "workspace:*", "@intx/hub-agent": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/inference-catalog": "0.3.0", - "@intx/log": "0.3.0", + "@intx/inference-catalog": "0.4.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", "@intx/workflow-host": "workspace:*", "better-auth": "1.6.29", "hono": "4.13.3", - "@intx/mailbox": "workspace:*" + "@intx/mailbox": "workspace:*", + "@corbits/oauth-core": "github:corbitsdev/corbits-oauth-core#3028f8604155eee63f5a1711f0d4022d7984e3f7", + "@corbits/openai-responses": "github:corbitsdev/corbits-openai-responses#7efe0006da9d94ac9037c1389f23d3d6163df4a5" } } diff --git a/packages/approvals/package.json b/packages/approvals/package.json index 7b7d22e7fc..3bb800c805 100644 --- a/packages/approvals/package.json +++ b/packages/approvals/package.json @@ -14,7 +14,7 @@ "test": "bun test" }, "dependencies": { - "@intx/authz": "0.3.0", + "@intx/authz": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", "@intx/types": "workspace:*", @@ -23,7 +23,7 @@ "hono": "catalog:" }, "devDependencies": { - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@types/bun": "catalog:", "postgres": "catalog:", "typescript": "catalog:" diff --git a/packages/deferred-tools/package.json b/packages/deferred-tools/package.json index fe2ba0e4cb..fcc5cab304 100644 --- a/packages/deferred-tools/package.json +++ b/packages/deferred-tools/package.json @@ -32,9 +32,9 @@ "typescript": "catalog:" }, "peerDependencies": { - "@intx/agent": "0.3.0", - "@intx/inference": "0.3.0", - "@intx/log": "0.3.0", - "@intx/types": "0.3.0" + "@intx/agent": "0.4.0", + "@intx/inference": "0.4.0", + "@intx/log": "0.4.0", + "@intx/types": "0.4.0" } } diff --git a/packages/error-sink/package.json b/packages/error-sink/package.json index b3dceb878a..748a95c563 100644 --- a/packages/error-sink/package.json +++ b/packages/error-sink/package.json @@ -33,6 +33,6 @@ "typescript": "catalog:" }, "peerDependencies": { - "@intx/log": "0.3.0" + "@intx/log": "0.4.0" } } diff --git a/packages/workflows/package.json b/packages/workflows/package.json index 81f7bc5007..25f0d8267e 100644 --- a/packages/workflows/package.json +++ b/packages/workflows/package.json @@ -16,10 +16,10 @@ }, "dependencies": { "@corbits/error-sink": "workspace:*", - "@intx/authz": "0.3.0", + "@intx/authz": "0.4.0", "@intx/db": "workspace:*", "@intx/hub-api": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/vendor/intx/agent/VENDORED-FROM b/vendor/intx/agent/VENDORED-FROM index 0d203c5e4d..ec8336b8aa 100644 --- a/vendor/intx/agent/VENDORED-FROM +++ b/vendor/intx/agent/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/agent) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-7190: ToolBundle.beforeToolExtension (tool.ts) and its composition into ResolvedTools.beforeToolExtensions (agent.ts), so a tool package can contribute its own suspend-capable extension without the reactor or director special-casing it by name. diff --git a/vendor/intx/agent/package.json b/vendor/intx/agent/package.json index 867f157178..72c233aec1 100644 --- a/vendor/intx/agent/package.json +++ b/vendor/intx/agent/package.json @@ -1,7 +1,7 @@ { "name": "@intx/agent", "description": "In-process agent runtime: construct an agent, send it a message, get a reply", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -19,7 +19,7 @@ }, "dependencies": { "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx/agent/src/testing/index.ts b/vendor/intx/agent/src/testing/index.ts index 1819854b4b..36faffa274 100644 --- a/vendor/intx/agent/src/testing/index.ts +++ b/vendor/intx/agent/src/testing/index.ts @@ -16,3 +16,4 @@ export { noopAuditStore } from "./audit-noop"; export { permissiveAuthorize } from "./authorize-allow"; +export { waitForReactorDone } from "./reactor-waiters"; diff --git a/vendor/intx/agent/src/testing/reactor-waiters.ts b/vendor/intx/agent/src/testing/reactor-waiters.ts new file mode 100644 index 0000000000..8cb69a5602 --- /dev/null +++ b/vendor/intx/agent/src/testing/reactor-waiters.ts @@ -0,0 +1,25 @@ +// Waiter over a reactor's emitted-event stream. +// +// A reactor run ends with a `reactor.done` event, including on the fatal +// error path, so a test that needs the run finished waits for that event +// rather than for an interval. CONVENTIONS.md names this shape as the one to +// copy; it lives here so it is imported rather than copied again. + +/** + * Resolve once the stream emits `reactor.done`. + * + * Throws if the stream ends without it: returning would resolve as though the + * run had completed, and the caller would then fail on a later assertion + * instead of on the real cause. + * + * Typed structurally so a caller need not import the reactor's event union to + * await its terminal event. + */ +export async function waitForReactorDone( + stream: AsyncIterable<{ type: string }>, +): Promise { + for await (const event of stream) { + if (event.type === "reactor.done") return; + } + throw new Error("reactor event stream ended before reactor.done"); +} diff --git a/vendor/intx/db/README.md b/vendor/intx/db/README.md index 71ca470e64..d10d719dfe 100644 --- a/vendor/intx/db/README.md +++ b/vendor/intx/db/README.md @@ -37,6 +37,17 @@ typed values; downstream code uses those values without re-casting. See `CONVENTIONS.md` for the project-wide rule against cast-at-callsite in DB consumers. +Connections created by `createDB` set PostgreSQL's `statement_timeout` to +60,000 milliseconds. Override it with `statementTimeoutMs` in `DBConfig` +(a positive integer up to 2,147,483,647 milliseconds). The Hub and the +`bin/` database scripts read this override from `DB_STATEMENT_TIMEOUT_MS`. + +PostgreSQL cancels statements that exceed the deadline, including time +waiting for locks. An uncaught timeout rolls back the enclosing transaction. +This bounds individual statements once they reach PostgreSQL; pool waits, +network stalls, and time between statements in an open transaction remain +outside that deadline. Migration clients use their own connection settings. + ## Model catalog The `model`, `model_provider`, `model_offering`, and `model_pricing` diff --git a/vendor/intx/db/VENDORED-FROM b/vendor/intx/db/VENDORED-FROM index 53d628caf9..44aa6c3113 100644 --- a/vendor/intx/db/VENDORED-FROM +++ b/vendor/intx/db/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/db) -Commit: 79adc433 (origin/main, 2026-09-17; src re-pinned in CL-8206, migrations completed in CL-8425) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) -Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-6324: workflow_definition_version gains a wire_projection jsonb column (migration 0089_workflow_definition_version_wire_projection.sql -- renumbered from 0086 as upstream took 0086/0087/0088; the journal keeps our original `when` values, which still sort after upstream's), schema/workflow-definitions.ts declares it, and workflow-definition-store.ts adds loadFrozenWireProjection (exported from src/index.ts) to read it back beside the approved wire hash. CL-6452: workflow_definition gains an origin column (migration 0090_workflow_definition_origin.sql; declared in schema/workflow-definitions.ts) marking the per-run record a folded run's deploy mints for itself, so launch and refresh resolve the hub-authored definition instead of the newest same-named row. Migration 0091_workflow_definition_version_tolerate_body_failure.sql rewrites the retired `onBodyFailure: "continue"` literal to upstream's `"tolerate"` inside stored wire projections. CL-4455: workflow_definition gains schedule_claimed_minute (migration 0092_workflow_definition_schedule_claimed_minute.sql; declared in schema/workflow-definitions.ts) so the hub's native ScheduleTrigger poller can CAS each UTC minute. CL-8162: the hub-side cron tick is deleted, so schedule_claimed_minute is dropped again (migration 0094_drop_workflow_definition_schedule_claimed_minute.sql, journal idx 94) and schema/workflow-definitions.ts no longer declares it. Workbench applies these files by sorted filename through scripts/db-setup.ts (replay-from-scratch with a filename ledger); the drizzle journal exists for drizzle-kit only. CL-7495 re-pin to c1b5bb73: upstream's 0089_tense_selene.sql (principal_key table) is renumbered to 0093_principal_keys.sql behind our four delta migrations, with journal idx 93 and a 0093_snapshot.json built from upstream's 0089 snapshot reconciled with the wire_projection/origin/schedule_claimed_minute columns; upstream 447ebb08's snapshot-chain repair (relinked 0087 prevId, workflow_run.credential_refs restored) is applied to 0087/0088. Our fork migrations 0089-0092 ship no drizzle snapshots (hand-written SQL, journal only), so there was no fork snapshot chain to repair. CL-8425: the CL-8206 re-pin to 79adc433 moved src/schema (tenant domain lower-unique index, workflow_run_dispatch.sender_address, sidecar_allocation destroy_failed status) without its SQL; upstream's 0090_tenant_domain_lower_unique, 0091_workflow_run_dispatch_sender_address and 0092_sidecar_destroy_failed are vendored verbatim as 0095/0096/0097 (journal idx 95-97, no snapshots). +Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-6324: workflow_definition_version gains a wire_projection jsonb column (migration 0089_workflow_definition_version_wire_projection.sql -- renumbered from 0086 as upstream took 0086/0087/0088; the journal keeps our original `when` values, which still sort after upstream's), schema/workflow-definitions.ts declares it, and workflow-definition-store.ts adds loadFrozenWireProjection (exported from src/index.ts) to read it back beside the approved wire hash. CL-6452: workflow_definition gains an origin column (migration 0090_workflow_definition_origin.sql; declared in schema/workflow-definitions.ts) marking the per-run record a folded run's deploy mints for itself, so launch and refresh resolve the hub-authored definition instead of the newest same-named row. Migration 0091_workflow_definition_version_tolerate_body_failure.sql rewrites the retired `onBodyFailure: "continue"` literal to upstream's `"tolerate"` inside stored wire projections. CL-4455: workflow_definition gains schedule_claimed_minute (migration 0092_workflow_definition_schedule_claimed_minute.sql; declared in schema/workflow-definitions.ts) so the hub's native ScheduleTrigger poller can CAS each UTC minute. CL-8162: the hub-side cron tick is deleted, so schedule_claimed_minute is dropped again (migration 0094_drop_workflow_definition_schedule_claimed_minute.sql, journal idx 94) and schema/workflow-definitions.ts no longer declares it. Workbench applies these files by sorted filename through scripts/db-setup.ts (replay-from-scratch with a filename ledger); the drizzle journal exists for drizzle-kit only. CL-7495 re-pin to c1b5bb73: upstream's 0089_tense_selene.sql (principal_key table) is renumbered to 0093_principal_keys.sql behind our four delta migrations, with journal idx 93 and a 0093_snapshot.json built from upstream's 0089 snapshot reconciled with the wire_projection/origin/schedule_claimed_minute columns; upstream 447ebb08's snapshot-chain repair (relinked 0087 prevId, workflow_run.credential_refs restored) is applied to 0087/0088. Our fork migrations 0089-0092 ship no drizzle snapshots (hand-written SQL, journal only), so there was no fork snapshot chain to repair. CL-8425: the CL-8206 re-pin to 79adc433 moved src/schema (tenant domain lower-unique index, workflow_run_dispatch.sender_address, sidecar_allocation destroy_failed status) without its SQL; upstream's 0090_tenant_domain_lower_unique, 0091_workflow_run_dispatch_sender_address and 0092_sidecar_destroy_failed are vendored verbatim as 0095/0096/0097 (journal idx 95-97, no snapshots). Re-vendor to 5453d0b0: upstream's 0093_sidecar_initialization.sql is vendored verbatim as 0098 (journal idx 98, no snapshot). diff --git a/vendor/intx/db/migrations/0098_sidecar_initialization.sql b/vendor/intx/db/migrations/0098_sidecar_initialization.sql new file mode 100644 index 0000000000..857250ecfb --- /dev/null +++ b/vendor/intx/db/migrations/0098_sidecar_initialization.sql @@ -0,0 +1,2 @@ +ALTER TABLE "sidecar_allocation" ADD COLUMN "initialization_lease_id" text;--> statement-breakpoint +ALTER TABLE "sidecar_allocation" ADD CONSTRAINT "sidecar_allocation_initialization_check" CHECK ("sidecar_allocation"."initialization_lease_id" is null or ("sidecar_allocation"."status" = 'allocated' and "sidecar_allocation"."ensure_accepted_generation" is not null and "sidecar_allocation"."ensure_accepted_generation" = "sidecar_allocation"."generation")); \ No newline at end of file diff --git a/vendor/intx/db/migrations/meta/_journal.json b/vendor/intx/db/migrations/meta/_journal.json index 175560d2bf..e7157a2d3b 100644 --- a/vendor/intx/db/migrations/meta/_journal.json +++ b/vendor/intx/db/migrations/meta/_journal.json @@ -680,6 +680,13 @@ "when": 1789100000002, "tag": "0097_sidecar_destroy_failed", "breakpoints": true + }, + { + "idx": 98, + "version": "7", + "when": 1789466440782, + "tag": "0098_sidecar_initialization", + "breakpoints": true } ] } diff --git a/vendor/intx/db/package.json b/vendor/intx/db/package.json index 6cb0a4723b..78259ae748 100644 --- a/vendor/intx/db/package.json +++ b/vendor/intx/db/package.json @@ -1,7 +1,7 @@ { "name": "@intx/db", "description": "Client, schema, migrations, and row parsers for the hub's PostgreSQL database", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -18,10 +18,10 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", - "@intx/hub-common": "0.3.0", - "@intx/log": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", + "@intx/hub-common": "0.4.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", "drizzle-orm": "catalog:", diff --git a/vendor/intx/db/src/config.ts b/vendor/intx/db/src/config.ts index 04f6be8b27..1448dd6dc8 100644 --- a/vendor/intx/db/src/config.ts +++ b/vendor/intx/db/src/config.ts @@ -8,6 +8,8 @@ export const DBConfig = type({ database: "string", "ssl?": "boolean", "max?": "number.integer > 0", + // PostgreSQL statement deadline in milliseconds. Defaults to 60 seconds. + "statementTimeoutMs?": "0 < number.integer <= 2147483647", // Postgres schema name. When set, the connection's `search_path` is // pinned to this schema and migrations apply into it. This is the // mechanism the integration-test harness uses to give each spawned diff --git a/vendor/intx/db/src/connection.ts b/vendor/intx/db/src/connection.ts index b85c7090e6..0d818d1afa 100644 --- a/vendor/intx/db/src/connection.ts +++ b/vendor/intx/db/src/connection.ts @@ -17,6 +17,7 @@ export function createConnection(config: DBConfig) { ...(config.ssl !== undefined && { ssl: config.ssl }), connection: { TimeZone: "UTC", + statement_timeout: config.statementTimeoutMs ?? 60_000, ...(config.schema !== undefined && { // Pin the connection's search_path so unqualified table // references resolve to the caller's schema. The migration diff --git a/vendor/intx/db/src/schema/sidecar-allocation.ts b/vendor/intx/db/src/schema/sidecar-allocation.ts index fb1ead6d6a..7e9247695e 100644 --- a/vendor/intx/db/src/schema/sidecar-allocation.ts +++ b/vendor/intx/db/src/schema/sidecar-allocation.ts @@ -45,6 +45,7 @@ export const sidecarAllocation = pgTable( nextAttemptAt: timestamp("next_attempt_at"), reconciliationLeaseId: text("reconciliation_lease_id"), reconciliationLeaseExpiresAt: timestamp("reconciliation_lease_expires_at"), + initializationLeaseId: text("initialization_lease_id"), ensureAttempts: integer("ensure_attempts").notNull().default(0), destroyAttempts: integer("destroy_attempts").notNull().default(0), connectDeadline: timestamp("connect_deadline"), @@ -75,5 +76,9 @@ export const sidecarAllocation = pgTable( "sidecar_allocation_accepted_generation_check", sql`${t.ensureAcceptedGeneration} is null or ${t.ensureAcceptedGeneration} <= ${t.generation}`, ), + check( + "sidecar_allocation_initialization_check", + sql`${t.initializationLeaseId} is null or (${t.status} = 'allocated' and ${t.ensureAcceptedGeneration} is not null and ${t.ensureAcceptedGeneration} = ${t.generation})`, + ), ], ); diff --git a/vendor/intx/db/src/sidecar-allocation-store.ts b/vendor/intx/db/src/sidecar-allocation-store.ts index 9d2fb946b4..d95d594493 100644 --- a/vendor/intx/db/src/sidecar-allocation-store.ts +++ b/vendor/intx/db/src/sidecar-allocation-store.ts @@ -27,6 +27,7 @@ import { sidecarAllocation, workflowRun, workflowRunLaunchSpec, + type WorkflowRunCredentialRefs, } from "./schema"; type DBHandle = DB["db"]; @@ -60,6 +61,8 @@ export type SidecarAllocation = { readonly nextAttemptAt?: Date; readonly reconciliationLeaseId?: string; readonly reconciliationLeaseExpiresAt?: Date; + /** Outstanding deploy attempt, retained after its reconciliation lease ends. */ + readonly initializationLeaseId?: string; readonly ensureAttempts: number; readonly destroyAttempts: number; readonly connectDeadline?: Date; @@ -150,6 +153,8 @@ export type BeginSidecarReplacementArgs = { readonly expectedStatus: "provisioning" | "allocated"; readonly expectedGeneration: number; readonly expectedLeaseId: string; + readonly onlyIfInitializationIncomplete?: boolean; + readonly expectedInitializationLeaseId?: string; readonly nextAttemptAt: Date; readonly failureCode: string; readonly failureMessage: string; @@ -166,6 +171,7 @@ export type BeginSidecarReleaseArgs = { readonly failureCode?: string; readonly failureMessage?: string; readonly expectedLeaseId?: string; + readonly expectedInitializationLeaseId?: string; readonly now?: Date; }; @@ -173,6 +179,8 @@ export type BeginUnrecoverableSidecarReleaseArgs = { readonly allocationId: string; readonly expectedGeneration: number; readonly expectedLeaseId: string; + readonly onlyIfInitializationIncomplete?: boolean; + readonly expectedInitializationLeaseId?: string; readonly failureCode: string; readonly failureMessage: string; readonly now?: Date; @@ -221,6 +229,15 @@ export type MarkSidecarDestroyFailedArgs = Omit< "expectedStatus" >; +type InitializationArgs = { + readonly allocationId: string; + readonly generation: number; + readonly anchorRunId: string; + readonly tenantId: string; + readonly leaseId: string; + readonly signal: AbortSignal; +}; + function parseSidecarAllocationRow( row: SidecarAllocationRow, ): SidecarAllocation { @@ -247,6 +264,9 @@ function parseSidecarAllocationRow( ...(row.reconciliationLeaseExpiresAt !== null ? { reconciliationLeaseExpiresAt: row.reconciliationLeaseExpiresAt } : {}), + ...(row.initializationLeaseId !== null + ? { initializationLeaseId: row.initializationLeaseId } + : {}), ensureAttempts: row.ensureAttempts, destroyAttempts: row.destroyAttempts, ...(row.connectDeadline !== null @@ -280,6 +300,125 @@ function leaseCondition(expectedLeaseId?: string) { export function createSidecarAllocationStore(db: DBHandle) { const workflowRunDispatchStore = createWorkflowRunDispatchStore(db); + function initializationConditions( + args: InitializationArgs, + expectedMarker: string | null, + { requireCurrentLease = true }: { requireCurrentLease?: boolean } = {}, + ) { + return and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.anchorRunId, args.anchorRunId), + eq(sidecarAllocation.tenantId, args.tenantId), + eq(sidecarAllocation.status, "allocated"), + eq(sidecarAllocation.generation, args.generation), + eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), + ...(requireCurrentLease ? leaseCondition(args.leaseId) : []), + expectedMarker === null + ? isNull(sidecarAllocation.initializationLeaseId) + : eq(sidecarAllocation.initializationLeaseId, expectedMarker), + ); + } + + async function writeInitialization( + args: InitializationArgs, + completion?: { + readonly publicKey: string; + readonly credentialRefs?: WorkflowRunCredentialRefs; + }, + ): Promise<{ previousPublicKey: string | null } | null> { + args.signal.throwIfAborted(); + return db.transaction(async (tx) => { + const condition = initializationConditions( + args, + completion === undefined ? null : args.leaseId, + ); + const [allocation] = await tx + .select({ id: sidecarAllocation.id }) + .from(sidecarAllocation) + .where(condition) + .for("update"); + args.signal.throwIfAborted(); + if (allocation === undefined) return null; + let previousPublicKey: string | null = null; + if (completion === undefined) { + const [previous] = await tx + .select({ publicKey: workflowRun.publicKey }) + .from(workflowRun) + .where( + and( + eq(workflowRun.id, args.anchorRunId), + eq(workflowRun.anchorRunId, args.anchorRunId), + eq(workflowRun.tenantId, args.tenantId), + ), + ) + .for("update"); + if (previous === undefined) + throw new Error("Initialization anchor is missing"); + previousPublicKey = previous.publicKey; + } + const [anchor] = await tx + .update(workflowRun) + .set({ + publicKey: completion?.publicKey ?? null, + ...(completion?.credentialRefs !== undefined + ? { credentialRefs: completion.credentialRefs } + : {}), + }) + .where( + and( + eq(workflowRun.id, args.anchorRunId), + eq(workflowRun.anchorRunId, args.anchorRunId), + eq(workflowRun.tenantId, args.tenantId), + ), + ) + .returning({ id: workflowRun.id }); + if (anchor === undefined) + throw new Error("Initialization anchor is missing"); + // Check the lease again after waiting on the anchor row. Both writes roll + // back if ownership expired while the transaction held the allocation lock. + const [updated] = await tx + .update(sidecarAllocation) + .set({ + initializationLeaseId: completion === undefined ? args.leaseId : null, + }) + .where(condition) + .returning({ id: sidecarAllocation.id }); + args.signal.throwIfAborted(); + if (updated === undefined) + throw new Error("Initialization lease expired"); + return { previousPublicKey }; + }); + } + + async function initializationCompleted( + tx: DBExecutor, + args: BeginSidecarReplacementArgs | BeginUnrecoverableSidecarReleaseArgs, + ): Promise { + const [allocation] = await tx + .select({ + anchorRunId: sidecarAllocation.anchorRunId, + initializationLeaseId: sidecarAllocation.initializationLeaseId, + }) + .from(sidecarAllocation) + .where( + and( + eq(sidecarAllocation.id, args.allocationId), + eq(sidecarAllocation.generation, args.expectedGeneration), + ...leaseCondition(args.expectedLeaseId), + ), + ) + .for("update"); + if (allocation === undefined || allocation.initializationLeaseId !== null) + return false; + // Read the key after acquiring the allocation lock, so a completion that + // committed while we waited is visible even if its response was lost. + const [anchor] = await tx + .select({ publicKey: workflowRun.publicKey }) + .from(workflowRun) + .where(eq(workflowRun.id, allocation.anchorRunId)); + return anchor !== undefined && anchor.publicKey !== null; + } + async function insertSidecarIdentity( tx: DBExecutor, args: { @@ -336,6 +475,7 @@ export function createSidecarAllocationStore(db: DBHandle) { .set({ status: "releasing", generation: args.expectedGeneration + 1, + initializationLeaseId: null, nextAttemptAt: now, reconciliationLeaseId: null, reconciliationLeaseExpiresAt: null, @@ -354,6 +494,14 @@ export function createSidecarAllocationStore(db: DBHandle) { eq(sidecarAllocation.status, args.expectedStatus), eq(sidecarAllocation.generation, args.expectedGeneration), ...leaseCondition(args.expectedLeaseId), + ...(args.expectedInitializationLeaseId !== undefined + ? [ + eq( + sidecarAllocation.initializationLeaseId, + args.expectedInitializationLeaseId, + ), + ] + : []), ), ) .returning(); @@ -361,6 +509,65 @@ export function createSidecarAllocationStore(db: DBHandle) { } return { + beginInitialization(args: InitializationArgs) { + return writeInitialization(args); + }, + + async completeInitialization( + args: InitializationArgs & { + readonly publicKey: string; + readonly credentialRefs?: WorkflowRunCredentialRefs; + }, + ): Promise { + return (await writeInitialization(args, args)) !== null; + }, + + async clearUnsentInitialization( + args: InitializationArgs & { readonly previousPublicKey: string | null }, + ): Promise { + // No signal gate: the sole caller invokes this only for a proven-unsent + // frame, which is reachable precisely when the attempt was cancelled. + // The WHERE clause is the guard. Marker equality proves no newer + // attempt began, and generation/status equality proves the allocation + // did not move on. The reconciliation lease is deliberately not + // required: a disconnect nulls the lease while leaving this attempt's + // marker behind, and only this attempt's own clear may remove it. Restore + // its previous key in the same transaction so absence of the marker cannot + // expose a live workflow as keyless or certify a different attempt. + return db.transaction(async (tx) => { + const condition = initializationConditions(args, args.leaseId, { + requireCurrentLease: false, + }); + const [allocation] = await tx + .select({ id: sidecarAllocation.id }) + .from(sidecarAllocation) + .where(condition) + .for("update"); + if (allocation === undefined) return false; + const [anchor] = await tx + .update(workflowRun) + .set({ publicKey: args.previousPublicKey }) + .where( + and( + eq(workflowRun.id, args.anchorRunId), + eq(workflowRun.anchorRunId, args.anchorRunId), + eq(workflowRun.tenantId, args.tenantId), + ), + ) + .returning({ id: workflowRun.id }); + if (anchor === undefined) + throw new Error("Initialization anchor is missing"); + const [updated] = await tx + .update(sidecarAllocation) + .set({ initializationLeaseId: null }) + .where(condition) + .returning({ id: sidecarAllocation.id }); + if (updated === undefined) + throw new Error("Initialization attempt changed before rollback"); + return true; + }); + }, + async createPending( args: CreatePendingSidecarAllocationArgs, tx?: DBExecutor, @@ -658,11 +865,17 @@ export function createSidecarAllocationStore(db: DBHandle) { ): Promise { const now = databaseTimestamp(args.now); return db.transaction(async (tx) => { + if ( + args.onlyIfInitializationIncomplete && + (await initializationCompleted(tx, args)) + ) + return null; const [updated] = await tx .update(sidecarAllocation) .set({ status: "replacing", generation: args.expectedGeneration + 1, + initializationLeaseId: null, ensureAcceptedGeneration: null, nextAttemptAt: args.nextAttemptAt, reconciliationLeaseId: null, @@ -678,6 +891,14 @@ export function createSidecarAllocationStore(db: DBHandle) { eq(sidecarAllocation.status, args.expectedStatus), eq(sidecarAllocation.generation, args.expectedGeneration), ...leaseCondition(args.expectedLeaseId), + ...(args.expectedInitializationLeaseId !== undefined + ? [ + eq( + sidecarAllocation.initializationLeaseId, + args.expectedInitializationLeaseId, + ), + ] + : []), ), ) .returning(); @@ -703,6 +924,11 @@ export function createSidecarAllocationStore(db: DBHandle) { ): Promise { const now = databaseTimestamp(args.now); return db.transaction(async (tx) => { + if ( + args.onlyIfInitializationIncomplete && + (await initializationCompleted(tx, args)) + ) + return null; const releasing = await beginRelease( { ...args, @@ -956,6 +1182,7 @@ export function createSidecarAllocationStore(db: DBHandle) { eq(sidecarAllocation.status, "allocated"), eq(sidecarAllocation.generation, args.generation), eq(sidecarAllocation.ensureAcceptedGeneration, args.generation), + isNull(sidecarAllocation.initializationLeaseId), ...leaseCondition(args.expectedLeaseId), ), ) diff --git a/vendor/intx/db/src/workflow-run-dispatch-store.ts b/vendor/intx/db/src/workflow-run-dispatch-store.ts index 0d0e058b00..22b3827d2c 100644 --- a/vendor/intx/db/src/workflow-run-dispatch-store.ts +++ b/vendor/intx/db/src/workflow-run-dispatch-store.ts @@ -7,6 +7,7 @@ import { inArray, isNull, lte, + notInArray, or, sql, type SQL, @@ -74,6 +75,7 @@ export class WorkflowRunDispatchPayloadConflictError extends Error { } export type ClaimWorkflowRunDispatchArgs = { + readonly excludedDispatchIds?: readonly string[]; readonly leaseId: string; readonly leaseDurationMs: number; }; @@ -285,6 +287,14 @@ export function createWorkflowRunDispatchStore(db: DBHandle) { and( eq(workflowRunDispatch.status, "pending"), lte(workflowRunDispatch.nextAttemptAt, sql`now()`), + ...(args.excludedDispatchIds !== undefined && + args.excludedDispatchIds.length > 0 + ? [ + notInArray(workflowRunDispatch.id, [ + ...args.excludedDispatchIds, + ]), + ] + : []), or( isNull(workflowRunDispatch.deliveryLeaseExpiresAt), lte(workflowRunDispatch.deliveryLeaseExpiresAt, sql`now()`), diff --git a/vendor/intx/harness/VENDORED-FROM b/vendor/intx/harness/VENDORED-FROM index a80ad0cd87..3473f00e64 100644 --- a/vendor/intx/harness/VENDORED-FROM +++ b/vendor/intx/harness/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/harness) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/harness/package.json b/vendor/intx/harness/package.json index 0c3bed1a94..8ead04dc26 100644 --- a/vendor/intx/harness/package.json +++ b/vendor/intx/harness/package.json @@ -1,7 +1,7 @@ { "name": "@intx/harness", "description": "Mail-transport composition layer over @intx/agent adding INBOX watch and connector routing", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -15,8 +15,8 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/log": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*" }, "devDependencies": { diff --git a/vendor/intx/hub-agent/VENDORED-FROM b/vendor/intx/hub-agent/VENDORED-FROM index bffda3ec79..02adf38a51 100644 --- a/vendor/intx/hub-agent/VENDORED-FROM +++ b/vendor/intx/hub-agent/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-agent) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-6396: SessionEventSink and createSidecarOrchestrator's dispatchEvent/publishWorkflowInferenceEvent take an optional childRunId, and createHubLink's sendEvent stamps it on agent.event so overlapping turns close the right occurrence row. diff --git a/vendor/intx/hub-agent/package.json b/vendor/intx/hub-agent/package.json index dc148483c6..864f9a1c08 100644 --- a/vendor/intx/hub-agent/package.json +++ b/vendor/intx/hub-agent/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-agent", "description": "Sidecar-side orchestrator linking per-agent stores and sessions to the hub over WebSocket", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -19,11 +19,11 @@ }, "dependencies": { "@intx/harness": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:", "isomorphic-git": "catalog:" diff --git a/vendor/intx/hub-api/VENDORED-FROM b/vendor/intx/hub-api/VENDORED-FROM index 2f66e0ef8a..1b0deed77b 100644 --- a/vendor/intx/hub-api/VENDORED-FROM +++ b/vendor/intx/hub-api/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-api) -Commit: c1b5bb73ff4815414af44e4662214e054e9e70e2 (origin/main, 2026-09-06) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-6345: resolveApproval's ResolveApprovalRequest accepts principalId: null for policy-resolved (grant-allowance) decisions, which skip the per-principal approval:/resolve gate the HTTP routes still enforce; readDurableWorkflowRunLifecycles is re-exported from the barrel for the hub's grant-allowance gate. CL-workflow-deploy-bearer: middleware/workflow-run-deploy-auth.ts (+ test) is a bearer-authenticated mirror onto the session-cookie POST/GET .../workflows/deployments route; MountHubRoutesDeps and CreateAppOpts gain an optional workflowRunAuthenticator that mounts it ahead of resolveTenant, and src/index.ts exports the middleware and its types. diff --git a/vendor/intx/hub-api/package.json b/vendor/intx/hub-api/package.json index 1a4d712a02..117a24be8d 100644 --- a/vendor/intx/hub-api/package.json +++ b/vendor/intx/hub-api/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-api", "description": "The hub's HTTP API: routes, middleware, authentication, and served OpenAPI spec", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -16,14 +16,14 @@ "dependencies": { "@hono/standard-validator": "^0.2.2", "@intx/agent": "workspace:*", - "@intx/authz": "0.3.0", - "@intx/crypto": "0.3.0", + "@intx/authz": "0.4.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/hub-sessions": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.3.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow-deploy": "workspace:*", "arktype": "catalog:", diff --git a/vendor/intx/hub-common/VENDORED-FROM b/vendor/intx/hub-common/VENDORED-FROM index b8b7b90bc1..34c53693b8 100644 --- a/vendor/intx/hub-common/VENDORED-FROM +++ b/vendor/intx/hub-common/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-common) -Commit: c1b5bb73ff4815414af44e4662214e054e9e70e2 (origin/main, 2026-09-06) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. First vendored in CL-7495: the hub's signing-key series needs generateId's pky_ principal-key prefix (npm 0.3.0 predates it), and @intx/db's principal-key store depends on this package. Only src/index.ts, ids.ts, action-mapping.ts, and glob.ts are consumed; the package depends on @intx/types via workspace: so the vendored types source joins the composite project-reference graph. diff --git a/vendor/intx/hub-common/package.json b/vendor/intx/hub-common/package.json index fbe28b7ded..0957c6ea0c 100644 --- a/vendor/intx/hub-common/package.json +++ b/vendor/intx/hub-common/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-common", "description": "Shared utilities for the Interchange hub packages", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { diff --git a/vendor/intx/hub-sessions/VENDORED-FROM b/vendor/intx/hub-sessions/VENDORED-FROM index 8b6012f84c..7e46f3b962 100644 --- a/vendor/intx/hub-sessions/VENDORED-FROM +++ b/vendor/intx/hub-sessions/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/hub-sessions) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. Terminal-anchor pack acceptance: hub-session-lookups.ts's receiveWorkflowRunPack gates the anchor lookup on the exported pure helper ownsWorkflowRunRepo (self-anchored row with a routable address, no liveness requirement) instead of upstream's live-status filter, so a terminal run can still land the inbox-enqueue and markConsumed-rejection packs that retire mail arriving in its teardown window; upstream's isNotNull(definitionId) tightening is kept. CL-6361 widens the same lookup to peel a per-step pack source address back to its base run's anchor via anchorAddressForPackSource; CL-6379 classifies an accepted pack's newly-terminal runs through decideTerminalRunFlip so a section occurrence's repo-local child run (turn__) is skipped quietly. CL-6324: workflow-probe-gate.ts's PersistFrozenApprovalFn carries the inert projection and createDbFrozenApprovalWriter stamps workflow_definition_version.wire_projection in the same transaction as approved_wire_hash. CL-6478: sanitize-tool-name.ts + event-collector.ts's tool_call case persist only a tool-call name encodeToolName can re-invert (anything else collapses to MALFORMED_TOOL_NAME), so one bad name fails its turn instead of wedging the room; @intx/inference is a dependency for this. CL-6595: workflow-run-kind.ts's validatePush also reports a run sealed from birth (combined events.jsonl with no per-event blobs) as newly terminal, and the new readCommittedWorkflowRunTerminalStatus export backs a same-push markTerminal backfill in hub-session-lookups.ts; both classify through upstream's classifyTerminalEvent. CL-7190: registerSignalCorrelation (hub-session-lookups.ts) now throws for any signal kind other than "approval", failing loud instead of silently mis-persisting a future SignalKind it has no co-write for. CL-6396: sidecar-events.ts's agent.event map and sidecar-handler.ts's agent.event dispatch forward an optional childRunId from the sidecar frame so the hub can close the matching occurrence row. CL-7508: sidecar-handler.ts's oauth.login result handling settles the caller's request promise as an error when a terminal outcome arrives before the started acknowledgement — previously a completed-before-started frame left the HTTP caller hanging until the whole-login timeout; the no-hang pin lives in sidecar-handler-oauth-login.test.ts (only tests covering a workbench delta are vendored). RETIRED AT THIS PIN: the CL-6324 session-service.ts deltas (deployAdoptedCodeSourcedWorkflow / deployAdoptedWorkflowFromSource / AdoptingWorkflowDeployer, the sourceRef threading, and CL-7191's UserMessageParams.correlationId) hung off upstream's deployWorkflowFromSource front, which this pin deletes in favour of the catalog-offering + provisioner-selection deploy path. corbitsdev/workbench#709: session-service.ts's prepared-deploy path threads `deployContent` + `toolPackagePins` through `DeployPreparedCodeSourcedWorkflowParams` and fans every top-level step out to `stageWorkflowStep` before the deployment frame (pure `preparedStepStageArgs` maps one step's address via `resolveStepAddress`, agent id via `deriveStepAgentId`, pins verbatim-or-absent); workflow-allocation-service.ts rehydrates both from the frozen launch spec (`preparedDeployContent` fails closed without a string prompt). The pins threading is pinned in session-service-pins.test.ts (only tests covering a workbench delta are vendored). diff --git a/vendor/intx/hub-sessions/package.json b/vendor/intx/hub-sessions/package.json index 7fa1ca8fee..c353e88509 100644 --- a/vendor/intx/hub-sessions/package.json +++ b/vendor/intx/hub-sessions/package.json @@ -1,7 +1,7 @@ { "name": "@intx/hub-sessions", "description": "Session-orchestration substrate for the hub: sidecar router, session, and asset services", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -19,15 +19,15 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/db": "workspace:*", - "@intx/hub-common": "0.3.0", + "@intx/hub-common": "0.4.0", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mime": "workspace:*", - "@intx/pack-transport": "0.3.0", - "@intx/storage-isogit": "0.3.0", - "@intx/tool-packaging": "0.3.0", + "@intx/pack-transport": "0.4.0", + "@intx/storage-isogit": "0.4.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "@intx/workflow-deploy": "workspace:*", diff --git a/vendor/intx/hub-sessions/src/index.ts b/vendor/intx/hub-sessions/src/index.ts index b3caf6f288..7944bcd64b 100644 --- a/vendor/intx/hub-sessions/src/index.ts +++ b/vendor/intx/hub-sessions/src/index.ts @@ -5,6 +5,7 @@ export { } from "./agent-repo"; export { createSessionService, + recoverSenderDeploy, SessionLaunchError, bridgeOrchestratorDeployContent, deployCodeSourcedWorkflow, @@ -40,10 +41,12 @@ export { type OAuthLoginGateOutcome, type OAuthLoginRequestOutcome, DEFAULT_OAUTH_LOGIN_TIMEOUT_MS, + type AllocatedSenderDeployAttempt, type SidecarAuthIdentity, type SidecarAuthenticator, type AllocatedSidecarTarget, type SidecarAllocationRouter, + SidecarIdentityValidationError, createSidecarCredentialResolver, createSidecarTokenAuthenticator, type CreateSidecarTokenAuthenticatorDeps, diff --git a/vendor/intx/hub-sessions/src/session-service.ts b/vendor/intx/hub-sessions/src/session-service.ts index dca48e3af8..ba8cf9b924 100644 --- a/vendor/intx/hub-sessions/src/session-service.ts +++ b/vendor/intx/hub-sessions/src/session-service.ts @@ -1,15 +1,16 @@ import { type } from "arktype"; -import { and, eq, gt, isNull, sql } from "drizzle-orm"; +import { and, eq, isNull } from "drizzle-orm"; import { getLogger } from "@intx/log"; import { buildCredentialDelivery, + createSidecarAllocationStore, listAssetsForTenant, resolveInferenceMaterials, type DB, + type SidecarAllocation, } from "@intx/db"; import { - sidecarAllocation as sidecarAllocationTable, workflowDefinition as workflowDefinitionTable, workflowRun as workflowRunTable, type WorkflowRunCredentialRefs, @@ -63,7 +64,6 @@ import { } from "./asset-service"; import type { AllocatedSidecarTarget, - SenderDeploySettledOutcome, SendProbeArgs, SidecarAllocationRouter, SidecarRouter, @@ -440,6 +440,7 @@ export type SendMultiStepDeployFrameArgs = SourceRefDeployFrameArgs; export async function sendMultiStepDeployFrame( args: SendMultiStepDeployFrameArgs, signal?: AbortSignal, + beforeSend?: () => Promise, ): Promise<{ publicKey: string }> { signal?.throwIfAborted(); const workflow = { @@ -466,6 +467,7 @@ export async function sendMultiStepDeployFrame( args.config, workflow, signal, + beforeSend, ); } @@ -813,18 +815,43 @@ async function emitSourceRefDeployFrame( args: DeployCodeSourcedWorkflowArgs & { allocationTarget?: AllocatedSidecarTarget; sidecarAllocationRouter?: SidecarAllocationRouter; + onUnsentInitializationCleared(publicKey: string | null): void; }, - signal?: AbortSignal, + reconciliation: SidecarReconciliationContext, ): Promise<{ publicKey: string; definitionId: string; credentialRefs?: WorkflowRunCredentialRefs; }> { - signal?.throwIfAborted(); + const { signal, leaseId } = reconciliation; + signal.throwIfAborted(); const { definitionId, sendArgs } = await prepareSourceRefDeploy(args); - signal?.throwIfAborted(); + signal.throwIfAborted(); + const allocationStore = createSidecarAllocationStore(args.db); + const initialization = { + allocationId: sendArgs.allocationTarget.allocationId, + generation: sendArgs.allocationTarget.generation, + anchorRunId: args.anchorRunId, + tenantId: args.tenantId, + leaseId, + signal, + }; + let previousPublicKey: string | null | undefined; try { - const result = await sendMultiStepDeployFrame(sendArgs, signal); + const result = await sendMultiStepDeployFrame( + sendArgs, + signal, + async () => { + const reserved = + await allocationStore.beginInitialization(initialization); + if (reserved === null) { + throw new Error( + "Allocation no longer permits this initialization attempt", + ); + } + previousPublicKey = reserved.previousPublicKey; + }, + ); return { publicKey: result.publicKey, definitionId, @@ -834,6 +861,20 @@ async function emitSourceRefDeployFrame( }; } catch (cause) { if (!isDeployFrameFailure(cause)) throw cause; + // Only a confirmed reservation gives us the key to restore. An ambiguous + // reservation response leaves its durable marker for conservative cleanup. + // A confirmed unsent attempt can roll back even after lease cancellation. + if (!cause.frameSent && previousPublicKey !== undefined) { + try { + const cleared = await allocationStore.clearUnsentInitialization({ + ...initialization, + previousPublicKey, + }); + if (cleared) args.onUnsentInitializationCleared(previousPublicKey); + } catch (error) { + logger.warn`Could not clear unsent initialization for ${initialization.allocationId}: ${error instanceof Error ? error.message : String(error)}`; + } + } throw new SessionLaunchError("start", cause, cause.frameSent); } } @@ -1029,6 +1070,36 @@ export function preparedStepStageArgs(params: { }; } +/** Resolve deferred sender mail after claiming the previous initializer's lease. */ +export async function recoverSenderDeploy(args: { + db: DB["db"]; + sidecarRouter: Pick; + allocation: SidecarAllocation; + reconciliation: SidecarReconciliationContext; +}): Promise { + const { allocation, reconciliation } = args; + reconciliation.signal.throwIfAborted(); + // A proven-unsent clear may still restore the previous key after this claim. + // Do not fail its mail using the claim's stale marker. Cleanup rechecks under + // the allocation lock; an advanced fence settles failure, while a rolled-back + // attempt is resolved by its caller or the next claim's completed key. + if (allocation.initializationLeaseId !== undefined) return; + // Claiming the lease prevents the previous attempt from publishing. Its + // marker and key now distinguish a committed initialization from failure, + // even before the worker reconnects or the old response arrives. + const anchor = await args.db.query.workflowRun.findFirst({ + where: eq(workflowRunTable.id, allocation.anchorRunId), + columns: { publicKey: true }, + }); + reconciliation.signal.throwIfAborted(); + args.sidecarRouter.noteSenderDeploySettled( + { allocationId: allocation.id, generation: allocation.generation }, + anchor !== undefined && anchor.publicKey !== null + ? { recorded: anchor.publicKey } + : { failed: "Previous deployment initialization did not complete" }, + ); +} + export function createSessionService( deps: SessionServiceDeps, ): SessionService & PreparedWorkflowDeployer { @@ -1554,64 +1625,21 @@ export function createSessionService( } const dbHandle = db; try { - args.reconciliation.signal.throwIfAborted(); - const updated = await dbHandle.transaction(async (tx) => { - const [allocation] = await tx - .select({ - id: sidecarAllocationTable.id, - anchorRunId: sidecarAllocationTable.anchorRunId, - status: sidecarAllocationTable.status, - generation: sidecarAllocationTable.generation, - ensureAcceptedGeneration: - sidecarAllocationTable.ensureAcceptedGeneration, - }) - .from(sidecarAllocationTable) - .where( - and( - eq(sidecarAllocationTable.id, args.allocationTarget.allocationId), - eq( - sidecarAllocationTable.reconciliationLeaseId, - args.reconciliation.leaseId, - ), - gt( - sidecarAllocationTable.reconciliationLeaseExpiresAt, - sql`clock_timestamp()`, - ), - ), - ) - .limit(1) - .for("update"); - args.reconciliation.signal.throwIfAborted(); - if ( - allocation === undefined || - allocation.anchorRunId !== args.anchorRunId || - allocation.status !== "allocated" || - allocation.generation !== args.allocationTarget.generation || - allocation.ensureAcceptedGeneration !== - args.allocationTarget.generation - ) { - return null; - } - const [anchor] = await tx - .update(workflowRunTable) - .set({ - publicKey: args.publicKey, - ...(args.credentialRefs !== undefined - ? { credentialRefs: args.credentialRefs } - : {}), - }) - .where( - and( - eq(workflowRunTable.id, args.anchorRunId), - eq(workflowRunTable.anchorRunId, args.anchorRunId), - eq(workflowRunTable.tenantId, args.tenantId), - ), - ) - .returning({ id: workflowRunTable.id }); - args.reconciliation.signal.throwIfAborted(); - return anchor ?? null; + const updated = await createSidecarAllocationStore( + dbHandle, + ).completeInitialization({ + allocationId: args.allocationTarget.allocationId, + generation: args.allocationTarget.generation, + anchorRunId: args.anchorRunId, + tenantId: args.tenantId, + leaseId: args.reconciliation.leaseId, + signal: args.reconciliation.signal, + publicKey: args.publicKey, + ...(args.credentialRefs !== undefined + ? { credentialRefs: args.credentialRefs } + : {}), }); - if (updated === null) { + if (!updated) { throw new Error( `Prepared anchor run ${args.anchorRunId} lost allocation ownership before initialization completed`, ); @@ -1701,7 +1729,11 @@ export function createSessionService( ); } + let restoredPublicKey: string | null | undefined; const commonEmit = { + onUnsentInitializationCleared(publicKey: string | null) { + restoredPublicKey = publicKey; + }, approved: params.approved, sidecarAllocationRouter: allocationRouter, allocationTarget: params.allocationTarget, @@ -1724,30 +1756,17 @@ export function createSessionService( definitionId: string; credentialRefs?: WorkflowRunCredentialRefs; }; - let senderDeploySettled = false; - function settleSenderDeploy(outcome: SenderDeploySettledOutcome): void { - if (senderDeploySettled) return; - senderDeploySettled = true; - sidecarRouter.noteSenderDeploySettled(params.agentAddress, outcome); - } - const cancelSenderDeploy = () => { - settleSenderDeploy({ - failed: - signal.reason instanceof Error - ? signal.reason.message - : "Initialization cancelled", - }); + const senderAttempt = { + ...params.allocationTarget, + leaseId: params.reconciliation.leaseId, }; + signal.throwIfAborted(); + sidecarRouter.noteSenderDeployStarted(params.agentAddress, senderAttempt); try { // Bracket the allocated pre-ack window: mark the sender's key-record as // mid-flight before the deploy emit so a run that sends mail before its // anchor key is committed parks rather than delivering keyless. The settle - // in both the success and catch paths below clears the marker. - sidecarRouter.noteSenderDeployStarted(params.agentAddress); - // Settle before replacement starts, so this attempt's late completion - // cannot clear the next attempt's sender-key marker at the same address. - signal.addEventListener("abort", cancelSenderDeploy, { once: true }); - signal.throwIfAborted(); + // follows durable completion; cancellation leaves the outcome to recovery. if (source.kind === "asset") { if (resolveAttachment === null) { throw new Error( @@ -1756,12 +1775,12 @@ export function createSessionService( } result = await emitSourceRefDeployFrame( { ...commonEmit, source, resolveAttachment }, - signal, + params.reconciliation, ); } else { result = await emitSourceRefDeployFrame( { ...commonEmit, source }, - signal, + params.reconciliation, ); } @@ -1783,7 +1802,7 @@ export function createSessionService( // `params.agentAddress` is the run's deploy address, byte-identical to the // sender address its mail was sent under (asserted against the anchor at // deploy time), so a settle matches the parked entries. - settleSenderDeploy({ + sidecarRouter.noteSenderDeploySettled(senderAttempt, { recorded: result.publicKey, }); @@ -1793,16 +1812,24 @@ export function createSessionService( publicKey: result.publicKey, }; } catch (error) { - // The deploy frame or the durable key write failed. Drain any mail the run - // parked while pre-ack so it surfaces as undelivered rather than waiting - // out the TTL. An allocated deploy's failure is owned here, not in the - // router's reject boundary. - settleSenderDeploy({ - failed: error instanceof Error ? error.message : String(error), - }); + if (restoredPublicKey !== undefined) { + sidecarRouter.noteSenderDeploySettled( + senderAttempt, + restoredPublicKey === null + ? { failed: error instanceof Error ? error.message : String(error) } + : { recorded: restoredPublicKey }, + ); + } + // A sent deploy or cancelled publication may have committed despite its + // lost response, as may an unsent rollback. Without a confirmed rollback, + // recovery owns transport failures too. Only an uncancelled preparation + // failure can settle definitively here. + else if (!signal.aborted && !(error instanceof SessionLaunchError)) { + sidecarRouter.noteSenderDeploySettled(senderAttempt, { + failed: error instanceof Error ? error.message : String(error), + }); + } throw error; - } finally { - signal.removeEventListener("abort", cancelSenderDeploy); } } diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts index a6d1c65c5d..9ee5a3c451 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/operation.ts @@ -50,6 +50,10 @@ export async function runSidecarOperation( new SidecarOperationTimeoutError(operation, timeoutMs), ); }, timeoutMs); + // A deadline must never keep the process alive on its own: in-flight work + // always holds its own handles, and a bare 120s timer would stall teardown + // and test runners after everything else settled. + timer?.unref?.(); try { return await Promise.race([ Promise.resolve().then(() => { diff --git a/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts b/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts index 5708cc725d..852e86e9ef 100644 --- a/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts +++ b/vendor/intx/hub-sessions/src/sidecar-allocation/reconciler.ts @@ -9,7 +9,9 @@ import type { AllocatedSidecarTarget, SidecarAllocationRouter, } from "../ws/sidecar-handler"; +import { SidecarIdentityValidationError } from "../ws/sidecar-handler"; import { SessionLaunchError } from "../session-service"; +import { DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY } from "../reconciliation-scheduler"; import { DestroySidecarResult, EnsureSidecarResult, @@ -19,6 +21,7 @@ import type { SidecarPluginRegistry } from "./plugin-registry"; import { DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, runSidecarOperation, + SidecarOperationTimeoutError, type SidecarReconciliationContext, } from "./operation"; @@ -57,6 +60,14 @@ export type SidecarAllocationReconcilerDeps = { | "waitForAllocatedSidecar" >; readonly hubWebSocketUrl: string; + /** + * Resolve the previous attempt's deferred mail under the newly claimed + * lease, before connection waits or cleanup. + */ + readonly onInitializationRecovery?: ( + allocation: SidecarAllocation, + reconciliation: SidecarReconciliationContext, + ) => Promise; /** Idempotently restores and deploys one connected allocation generation. */ readonly onReady?: ( allocation: SidecarAllocation, @@ -72,6 +83,11 @@ export type SidecarAllocationReconcilerDeps = { readonly leaseDurationMs?: number; readonly connectTimeoutMs?: number; readonly operationTimeoutMs?: number; + /** + * Bounds admitted claims, active reconciliation, and retained allocation + * queries or writes. + */ + readonly maxConcurrentClaims?: number; readonly retryDelayMs?: (attempt: number) => number; readonly now?: () => Date; readonly createSidecarId?: () => string; @@ -143,11 +159,13 @@ export function createSidecarAllocationReconciler({ plugins, router, hubWebSocketUrl, + onInitializationRecovery, onReady, enableAutomaticReplacementRecovery = false, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS, operationTimeoutMs = DEFAULT_SIDECAR_OPERATION_TIMEOUT_MS, + maxConcurrentClaims = DEFAULT_SIDECAR_ALLOCATION_CONCURRENCY, retryDelayMs = defaultRetryDelay, now = () => new Date(), createSidecarId = () => `sc_${randomHex(16)}`, @@ -161,6 +179,9 @@ export function createSidecarAllocationReconciler({ if (!Number.isSafeInteger(operationTimeoutMs) || operationTimeoutMs <= 0) { throw new Error("operationTimeoutMs must be a positive integer"); } + if (!Number.isSafeInteger(maxConcurrentClaims) || maxConcurrentClaims <= 0) { + throw new Error("maxConcurrentClaims must be a positive integer"); + } const activeAllocations = new Map< string, @@ -188,7 +209,7 @@ export function createSidecarAllocationReconciler({ if (active === undefined) throw new ReconciliationLeaseLostError(allocationId); const target = { allocationId, generation: active.allocation.generation }; - await queueConnectionEvent(target, async () => { + await queueReconciliationStep(target, async () => { active.controller.signal.throwIfAborted(); await apply( active.pendingConnect?.generation === target.generation && @@ -216,7 +237,7 @@ export function createSidecarAllocationReconciler({ return new Date(now().getTime() + retryDelayMs(attempt)); } - async function withLeaseHeartbeat( + async function withReconciliationLease( allocation: SidecarAllocation, leaseId: string, operationName: string, @@ -228,53 +249,6 @@ export function createSidecarAllocationReconciler({ if (active === undefined) throw new ReconciliationLeaseLostError(allocation.id); const controller = active.controller; - let finished = false; - let expiryTimer: ReturnType | undefined; - const checkLeaseExpiry = (): void => { - if (finished || controller.signal.aborted) return; - const remaining = active.leaseDeadline - performance.now(); - if (remaining <= 0) { - controller.abort(new ReconciliationLeaseLostError(allocation.id)); - } else { - expiryTimer = setTimeout(checkLeaseExpiry, Math.ceil(remaining)); - } - }; - let renewing = false; - const renew = async (): Promise => { - if (renewing || controller.signal.aborted) return; - renewing = true; - const startedAt = performance.now(); - try { - const renewed = await allocationStore.extendReconciliationLease( - allocation.id, - leaseId, - leaseDurationMs, - ); - if (finished || controller.signal.aborted) return; - if (!renewed || performance.now() >= active.leaseDeadline) { - controller.abort(new ReconciliationLeaseLostError(allocation.id)); - } else { - // The database grants the lease during the request. Counting from its - // start avoids extending ownership by the response's transit time. - active.leaseDeadline = startedAt + leaseDurationMs; - } - } catch (error) { - if (!finished) { - controller.abort( - new ReconciliationLeaseLostError(allocation.id, error), - ); - } - } finally { - renewing = false; - } - }; - const interval = setInterval( - () => { - void renew(); - }, - Math.max(1, Math.floor(leaseDurationMs / 3)), - ); - checkLeaseExpiry(); try { return await runSidecarOperation( operationName, @@ -284,12 +258,19 @@ export function createSidecarAllocationReconciler({ !(await runSidecarOperation( "Reconciliation lease validation", operationTimeoutMs, - () => - allocationStore.isReconciliationLeaseCurrent( - allocation.id, - allocation.generation, - leaseId, - ), + async () => { + try { + return await trackAllocationQuery(allocation.id, () => + allocationStore.isReconciliationLeaseCurrent( + allocation.id, + allocation.generation, + leaseId, + ), + ); + } catch (cause) { + throw new ReconciliationLeaseLostError(allocation.id, cause); + } + }, signal, )) ) { @@ -306,52 +287,129 @@ export function createSidecarAllocationReconciler({ } catch (error) { controller.signal.throwIfAborted(); throw error; + } + } + + // A timed-out lookup still occupies its database connection. Exclude its + // allocation until it settles so retries cannot accumulate duplicate reads. + const pendingAllocationQueries = new Map(); + + async function trackAllocationQuery( + allocationId: string, + query: () => Promise, + ): Promise { + const pending = pendingAllocationQueries.get(allocationId) ?? { count: 0 }; + pending.count += 1; + pendingAllocationQueries.set(allocationId, pending); + try { + return await query(); } finally { - finished = true; - clearInterval(interval); - clearTimeout(expiryTimer); + pending.count -= 1; + if (pending.count === 0) pendingAllocationQueries.delete(allocationId); } } + async function isSidecarReady( + allocation: SidecarAllocation, + ): Promise { + const active = activeAllocations.get(allocation.id); + if (active === undefined) + throw new ReconciliationLeaseLostError(allocation.id); + return runSidecarOperation( + "Sidecar readiness", + operationTimeoutMs, + () => + trackAllocationQuery(allocation.id, () => + router.isAllocatedSidecarReady({ + allocationId: allocation.id, + generation: allocation.generation, + }), + ), + active.controller.signal, + ); + } + async function replaceAfterFailure( allocation: SidecarAllocation, leaseId: string, code: string, message: string, + { + onlyIfInitializationIncomplete = false, + }: { onlyIfInitializationIncomplete?: boolean } = {}, ): Promise { - if ( - allocation.status === "allocated" && - !enableAutomaticReplacementRecovery - ) { - const releasing = await allocationStore.beginUnrecoverableRelease({ + const initializationCheck = onlyIfInitializationIncomplete + ? { + onlyIfInitializationIncomplete: true, + ...(allocation.initializationLeaseId !== undefined + ? { + expectedInitializationLeaseId: allocation.initializationLeaseId, + } + : {}), + } + : {}; + let shouldRetryInitialization = false; + await queueReconciliationStep( + { allocationId: allocation.id, generation: allocation.generation }, + async () => { + const updated = + allocation.status === "allocated" && + !enableAutomaticReplacementRecovery + ? await allocationStore.beginUnrecoverableRelease({ + ...initializationCheck, + allocationId: allocation.id, + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + failureCode: code, + failureMessage: `Automatic recovery is disabled: ${message}`, + now: now(), + }) + : await allocationStore.beginReplacement({ + ...initializationCheck, + allocationId: allocation.id, + expectedStatus: + allocation.status === "allocated" + ? "allocated" + : "provisioning", + expectedGeneration: allocation.generation, + expectedLeaseId: leaseId, + nextAttemptAt: retryAt( + allocation.ensureAttempts + allocation.destroyAttempts, + ), + failureCode: code, + failureMessage: message, + now: now(), + }); + if (updated !== null) { + // A late commit still advances the fence before this queued work + // settles and its allocation becomes eligible for another claim. + router.fenceAllocation(updated.id, updated.generation); + } else { + shouldRetryInitialization = onlyIfInitializationIncomplete; + } + }, + ); + if (shouldRetryInitialization) + await retryInitialization(allocation, leaseId); + } + + async function retryInitialization( + allocation: SidecarAllocation, + leaseId: string, + ): Promise { + // A publication or unsent rollback may have committed since the claim. + // Retry the ordinary callback, including initialization and dispatch requeue. + // If ownership was lost instead, this lease-guarded update changes nothing. + await finishReconciliation(allocation.id, () => + allocationStore.scheduleRetry({ allocationId: allocation.id, + expectedStatus: "allocated", expectedGeneration: allocation.generation, expectedLeaseId: leaseId, - failureCode: code, - failureMessage: `Automatic recovery is disabled: ${message}`, + nextAttemptAt: now(), now: now(), - }); - if (releasing !== null) { - router.fenceAllocation(releasing.id, releasing.generation); - } - return; - } - const replaced = await allocationStore.beginReplacement({ - allocationId: allocation.id, - expectedStatus: - allocation.status === "allocated" ? "allocated" : "provisioning", - expectedGeneration: allocation.generation, - expectedLeaseId: leaseId, - nextAttemptAt: retryAt( - allocation.ensureAttempts + allocation.destroyAttempts, - ), - failureCode: code, - failureMessage: message, - now: now(), - }); - if (replaced !== null) { - router.fenceAllocation(replaced.id, replaced.generation); - } + }), + ); } async function waitUntilReady( @@ -366,21 +424,45 @@ export function createSidecarAllocationReconciler({ const deadline = allocation.connectDeadline; const remaining = deadline === undefined ? 0 : deadline.getTime() - now().getTime(); + // A stalled identity query is not evidence that the worker missed its + // connection deadline. Let it retry without releasing the generation. + const connectionReady = + connectionAlreadyReady || (await isSidecarReady(allocation)); try { - if ( - !connectionAlreadyReady && - !(await router.isAllocatedSidecarReady(target)) - ) { - await withLeaseHeartbeat( + if (!connectionReady) { + await withReconciliationLease( allocation, leaseId, "Sidecar connection", - () => router.waitForAllocatedSidecar(target, Math.max(0, remaining)), + () => + trackAllocationQuery(allocation.id, () => + router.waitForAllocatedSidecar( + target, + Math.max(0, remaining), + (validation) => { + // The waiter can expire before a notification lookup settles. + // Retain its exclusion and capacity independently of the wait. + void trackAllocationQuery( + allocation.id, + () => validation, + ).catch(() => undefined); + }, + ), + ), operationTimeoutMs, ); } } catch (error) { - if (error instanceof ReconciliationLeaseLostError) throw error; + // Let the router report connection expiry. Our outer deadline can expire + // during lease or identity validation without establishing worker loss. + // A failed identity lookup is likewise inconclusive: the worker may be + // healthy behind it, so retry instead of releasing the generation. + if ( + error instanceof ReconciliationLeaseLostError || + error instanceof SidecarOperationTimeoutError || + error instanceof SidecarIdentityValidationError + ) + throw error; await replaceAfterFailure( allocation, leaseId, @@ -390,7 +472,7 @@ export function createSidecarAllocationReconciler({ return; } - await queueConnectionEvent(target, async () => { + await queueReconciliationStep(target, async () => { const active = activeAllocations.get(allocation.id); if (active?.allocation.generation !== allocation.generation) throw new ReconciliationLeaseLostError(allocation.id); @@ -400,11 +482,14 @@ export function createSidecarAllocationReconciler({ if (onReady !== undefined) { try { - await withLeaseHeartbeat( + await withReconciliationLease( allocation, leaseId, "Workflow initialization", - (context) => onReady(allocation, context), + (context) => + trackAllocationQuery(allocation.id, () => + onReady(allocation, context), + ), ); } catch (error) { if (error instanceof ReconciliationLeaseLostError) throw error; @@ -414,6 +499,7 @@ export function createSidecarAllocationReconciler({ leaseId, "sidecar_initialization_uncertain", error.message, + { onlyIfInitializationIncomplete: true }, ); return; } @@ -437,6 +523,12 @@ export function createSidecarAllocationReconciler({ } } + // A connect that lands during initialization schedules an immediate + // follow-up even on success: the new socket may be a restarted worker + // with an empty inventory (takeover suppresses the disconnect event), in + // which case the follow-up redeploys and restores it. When the worker is + // unchanged the follow-up is a no-op: deployReadyAllocation returns early + // once the workflow is active and its key is recorded. await finishReconciliation(allocation.id, (pendingConnect) => pendingConnect ? allocationStore.scheduleRetry({ @@ -469,7 +561,7 @@ export function createSidecarAllocationReconciler({ let result: EnsureSidecarResult; try { result = parseEnsureResult( - await withLeaseHeartbeat( + await withReconciliationLease( allocation, leaseId, "Sidecar ensure", @@ -536,11 +628,7 @@ export function createSidecarAllocationReconciler({ }); if (allocated !== null) { trackAllocation(allocated); - const target = { - allocationId: allocated.id, - generation: allocated.generation, - }; - if (await router.isAllocatedSidecarReady(target)) { + if (await isSidecarReady(allocated)) { await waitUntilReady(allocated, leaseId, true); return; } @@ -637,7 +725,7 @@ export function createSidecarAllocationReconciler({ let result: DestroySidecarResult; try { result = parseDestroyResult( - await withLeaseHeartbeat( + await withReconciliationLease( allocation, leaseId, "Sidecar destroy", @@ -694,6 +782,21 @@ export function createSidecarAllocationReconciler({ ) { return; } + if ( + allocation.status === "allocated" && + onInitializationRecovery !== undefined + ) { + await withReconciliationLease( + allocation, + leaseId, + "Sender deployment recovery", + (context) => + trackAllocationQuery(allocation.id, () => + onInitializationRecovery(allocation, context), + ), + operationTimeoutMs, + ); + } const provisioner = provisionerFor(allocation); if (provisioner === null) { if (allocation.status === "pending") { @@ -745,6 +848,16 @@ export function createSidecarAllocationReconciler({ ); return; case "allocated": + if (allocation.initializationLeaseId !== undefined) { + await replaceAfterFailure( + allocation, + leaseId, + "sidecar_initialization_uncertain", + "A previous initialization attempt did not record completion", + { onlyIfInitializationIncomplete: true }, + ); + return; + } await waitUntilReady(allocation, leaseId); return; case "replacing": @@ -818,6 +931,40 @@ export function createSidecarAllocationReconciler({ return pending; } + async function queueReconciliationStep( + target: AllocatedSidecarTarget, + apply: () => Promise, + ): Promise { + const active = activeAllocations.get(target.allocationId); + if (active?.allocation.generation !== target.generation) + throw new ReconciliationLeaseLostError(target.allocationId); + try { + await runSidecarOperation( + "Allocation connection events", + operationTimeoutMs, + (signal) => + queueConnectionEvent(target, async () => { + signal.throwIfAborted(); + await apply(); + }), + active.controller.signal, + ); + } catch (error) { + if (error instanceof SidecarOperationTimeoutError) { + // Stop this lease without queuing another write behind the same stalled + // event. Keep the actual operation queued until it settles, preserving + // ordering with reconnects and excluding this allocation from new claims. + const cancelled = new ReconciliationLeaseLostError( + target.allocationId, + error, + ); + active.controller.abort(cancelled); + throw cancelled; + } + throw error; + } + } + function noteDisconnect( target: AllocatedSidecarTarget, leaseInvalidated = false, @@ -876,7 +1023,14 @@ export function createSidecarAllocationReconciler({ allocationId: allocation.id, generation: allocation.generation, }; - if (await router.isAllocatedSidecarReady(target)) continue; + try { + if (await router.isAllocatedSidecarReady(target)) continue; + } catch (error) { + // Unknown readiness is not absence. Leave the allocation for the next + // repair sweep instead of scheduling a reconnect the worker may hold. + if (error instanceof SidecarIdentityValidationError) continue; + throw error; + } try { await allocationStore.scheduleReconnectIfUnscheduled({ ...target, @@ -891,35 +1045,132 @@ export function createSidecarAllocationReconciler({ } } + let admittedClaims = 0; + async function reconcileNext(): Promise { - const leaseId = createLeaseId(); - const claimStartedAt = performance.now(); - const allocation = await runSidecarOperation( - "Sidecar allocation claim", - operationTimeoutMs, - () => - allocationStore.claimNextReconcilable({ - leaseId, - leaseDurationMs, - excludedAllocationIds: [...activeAllocations.keys()], - }), - ); - if (allocation === null) return false; - // A claim started before another local claim returned can outlive its lease. - if (activeAllocations.has(allocation.id)) { - await allocationStore.parkReconciliation(allocation.id, leaseId, { - kind: "retry-after-error", - notBefore: retryAt(MAX_RETRY_BACKOFF_ATTEMPT), - }); - return true; + const retainedAllocations = new Set([ + ...connectionEvents.keys(), + ...pendingAllocationQueries.keys(), + ]); + // An active claim already owns capacity for its allocation's pending work. + for (const allocationId of activeAllocations.keys()) + retainedAllocations.delete(allocationId); + if (admittedClaims + retainedAllocations.size >= maxConcurrentClaims) + return false; + + admittedClaims += 1; + let claim: ReturnType | undefined; + try { + const leaseId = createLeaseId(); + const claimStartedAt = performance.now(); + const allocation = await runSidecarOperation( + "Sidecar allocation claim", + operationTimeoutMs, + () => { + claim = allocationStore.claimNextReconcilable({ + leaseId, + leaseDurationMs, + excludedAllocationIds: [ + ...new Set([ + ...activeAllocations.keys(), + ...connectionEvents.keys(), + ...pendingAllocationQueries.keys(), + ]), + ], + }); + return claim; + }, + ); + if (allocation === null) return false; + return await reconcileClaim(allocation, leaseId, claimStartedAt); + } finally { + const release = () => { + admittedClaims -= 1; + }; + // Keep the reservation through the claim-to-reconciliation handoff. A + // timed-out claim still owns capacity until its database query settles. + if (claim === undefined) release(); + else void claim.then(release, release); } - activeAllocations.set(allocation.id, { + } + + async function reconcileClaim( + allocation: SidecarAllocation, + leaseId: string, + claimStartedAt: number, + ): Promise { + // Local work may have started since this claim's exclusion snapshot. Let + // its lease expire without adding another database write behind that work. + if ( + connectionEvents.has(allocation.id) || + activeAllocations.has(allocation.id) || + pendingAllocationQueries.has(allocation.id) + ) + return true; + const active = { allocation, controller: new AbortController(), leaseDeadline: claimStartedAt + leaseDurationMs, pendingConnect: null, - }); + }; + activeAllocations.set(allocation.id, active); + let finished = false; + let renewing = false; + const renew = async (): Promise => { + if (finished || renewing || active.controller.signal.aborted) return; + renewing = true; + const startedAt = performance.now(); + try { + const renewed = await trackAllocationQuery(allocation.id, () => + allocationStore.extendReconciliationLease( + allocation.id, + leaseId, + leaseDurationMs, + ), + ); + if (finished || active.controller.signal.aborted) return; + if (!renewed || performance.now() >= active.leaseDeadline) { + active.controller.abort( + new ReconciliationLeaseLostError(allocation.id), + ); + } else { + // The database grants the lease during the request. Counting from its + // start avoids extending ownership by the response's transit time. + active.leaseDeadline = startedAt + leaseDurationMs; + } + } catch (error) { + if (!finished && !active.controller.signal.aborted) { + active.controller.abort( + new ReconciliationLeaseLostError(allocation.id, error), + ); + } + } finally { + renewing = false; + } + }; + // One heartbeat covers the whole claim, including database transitions + // between provider calls. Short stages must not keep postponing renewal. + const heartbeat = setInterval( + () => { + void renew(); + }, + Math.max(1, Math.floor(leaseDurationMs / 3)), + ); + let expiryTimer: ReturnType | undefined; + const checkLeaseExpiry = (): void => { + if (active.controller.signal.aborted) return; + const remaining = active.leaseDeadline - performance.now(); + if (remaining <= 0) { + active.controller.abort( + new ReconciliationLeaseLostError(allocation.id), + ); + } else { + expiryTimer = setTimeout(checkLeaseExpiry, Math.ceil(remaining)); + } + }; + checkLeaseExpiry(); try { + active.controller.signal.throwIfAborted(); await reconcile(allocation, leaseId); } catch (error) { if (error instanceof ReconciliationLeaseLostError) { @@ -927,16 +1178,10 @@ export function createSidecarAllocationReconciler({ if (cause === undefined) { logger.info`Allocation ${allocation.id} reconciliation stopped: lease ${leaseId} is no longer current`; } else { - logger.warn`Allocation ${allocation.id} reconciliation stopped because renewal of lease ${leaseId} failed: ${cause instanceof Error ? cause.message : String(cause)}`; - try { - await allocationStore.parkReconciliation(allocation.id, leaseId, { - kind: "retry-after-error", - notBefore: retryAt(MAX_RETRY_BACKOFF_ATTEMPT), - }); - } catch (parkError) { - logger.warn`Failed to park allocation ${allocation.id} after renewal failure: ${parkError instanceof Error ? parkError.message : String(parkError)}`; - } + logger.warn`Allocation ${allocation.id} reconciliation stopped because lease ${leaseId} could not be confirmed: ${cause instanceof Error ? cause.message : String(cause)}`; } + // The durable schedule survives the claim. Stop renewing and let the + // lease expire; handling a lease failure must not require another write. return true; } logger.error`Allocation ${allocation.id} reconciliation failed: ${error instanceof Error ? error.message : String(error)}`; @@ -947,6 +1192,9 @@ export function createSidecarAllocationReconciler({ }), ); } finally { + finished = true; + clearInterval(heartbeat); + clearTimeout(expiryTimer); activeAllocations.delete(allocation.id); } return true; diff --git a/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts b/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts index d1339069e2..7bb461ad05 100644 --- a/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts +++ b/vendor/intx/hub-sessions/src/workflow-dispatch-service.ts @@ -15,6 +15,10 @@ import type { AllocatedSidecarTarget, SidecarAllocationRouter, } from "./ws/sidecar-handler"; +import { + runSidecarOperation, + SidecarOperationTimeoutError, +} from "./sidecar-allocation/operation"; const logger = getLogger(["hub", "workflow-dispatch"]); @@ -73,12 +77,15 @@ export type WorkflowDispatchServiceDeps = { anchorRunId: string, ) => Promise; readonly leaseDurationMs?: number; + /** Bounds pending claims and unfinished deliveries across all drains. */ + readonly maxConcurrentDispatches?: number; readonly retryDelayMs?: (attempt: number) => number; readonly now?: () => Date; readonly createLeaseId?: () => string; }; const DEFAULT_LEASE_DURATION_MS = 30_000; +const DEFAULT_MAX_CONCURRENT_DISPATCHES = 8; function defaultRetryDelay(attempt: number): number { return Math.min(500 * 2 ** Math.min(attempt, 6), 30_000); @@ -94,6 +101,7 @@ function targetForReadyAllocation( if ( allocation.status !== "allocated" || allocation.ensureAcceptedGeneration !== allocation.generation || + allocation.initializationLeaseId !== undefined || allocation.connectDeadline !== undefined ) { return null; @@ -116,15 +124,24 @@ export function createWorkflowDispatchService({ router, resolveAnchorAddress, leaseDurationMs = DEFAULT_LEASE_DURATION_MS, + maxConcurrentDispatches = DEFAULT_MAX_CONCURRENT_DISPATCHES, retryDelayMs = defaultRetryDelay, now = () => new Date(), createLeaseId = randomLeaseId, }: WorkflowDispatchServiceDeps): WorkflowDispatchService { - if (leaseDurationMs <= 0) { - throw new Error("leaseDurationMs must be positive"); + if (!Number.isSafeInteger(leaseDurationMs) || leaseDurationMs <= 0) { + throw new Error("leaseDurationMs must be a positive integer"); + } + if ( + !Number.isSafeInteger(maxConcurrentDispatches) || + maxConcurrentDispatches <= 0 + ) { + throw new Error("maxConcurrentDispatches must be a positive integer"); } let drainPromise: Promise | null = null; + let admittedDispatches = 0; + const activeDispatches = new Set(); function retryAt(attempt: number): Date { return new Date(now().getTime() + retryDelayMs(attempt)); @@ -147,16 +164,66 @@ export function createWorkflowDispatchService({ } async function reconcileNext(): Promise { - const leaseId = createLeaseId(); - const dispatch = await dispatchStore.claimNextPending({ - leaseId, - leaseDurationMs, - }); - if (dispatch === null) return false; + if (admittedDispatches >= maxConcurrentDispatches) return false; + admittedDispatches += 1; + let delivery: Promise | undefined; + let activeDispatchId: string | undefined; + try { + const leaseId = createLeaseId(); + const claimStartedAt = performance.now(); + const dispatch = await dispatchStore.claimNextPending({ + leaseId, + leaseDurationMs, + excludedDispatchIds: [...activeDispatches], + }); + if (dispatch === null) return false; + + // A concurrent claim can outlive both its exclusion snapshot and the old + // lease. Do not start another delivery while its original I/O is pending. + if (activeDispatches.has(dispatch.id)) return true; + const remaining = leaseDurationMs - (performance.now() - claimStartedAt); + if (remaining <= 0) return true; + + activeDispatchId = dispatch.id; + activeDispatches.add(dispatch.id); + try { + await runSidecarOperation( + "Workflow dispatch", + Math.ceil(remaining), + (signal) => { + delivery = deliver(dispatch, leaseId, signal); + return delivery; + }, + ); + } catch (error) { + if (!(error instanceof SidecarOperationTimeoutError)) throw error; + // The durable lease expires independently. Do not start another database + // write here: a stuck retry write must not occupy the freed drain either. + logger.warn`Dispatch ${dispatch.id} stopped at its delivery deadline`; + } + return true; + } finally { + const release = () => { + admittedDispatches -= 1; + if (activeDispatchId !== undefined) + activeDispatches.delete(activeDispatchId); + }; + // Timeouts free the drain, but the underlying I/O retains its admission + // reservation and dispatch exclusion until it actually settles. + if (delivery === undefined) release(); + else void delivery.then(release, release); + } + } + async function deliver( + dispatch: ClaimedDispatch, + leaseId: string, + signal: AbortSignal, + ): Promise { const allocation = await allocationStore.findByAnchorRunId( dispatch.anchorRunId, ); + signal.throwIfAborted(); if (allocation === null) { await retry( dispatch, @@ -164,7 +231,7 @@ export function createWorkflowDispatchService({ "allocation_missing", `No sidecar allocation exists for workflow anchor ${dispatch.anchorRunId}`, ); - return true; + return; } const target = targetForReadyAllocation(allocation); if (target === null) { @@ -174,9 +241,10 @@ export function createWorkflowDispatchService({ "allocation_not_ready", `Sidecar allocation ${allocation.id} is not ready for delivery`, ); - return true; + return; } const agentAddress = await resolveAnchorAddress(dispatch.anchorRunId); + signal.throwIfAborted(); if (agentAddress === null) { await retry( dispatch, @@ -184,21 +252,25 @@ export function createWorkflowDispatchService({ "anchor_address_missing", `Workflow anchor ${dispatch.anchorRunId} has no routing address`, ); - return true; + return; } try { if (dispatch.kind === "signal") { - const signal = SignalDeliverFrame.assert( + const frame = SignalDeliverFrame.assert( JSON.parse(new TextDecoder().decode(dispatch.rawMessage)), ); - await router.sendSignalDeliverToAllocation(target, { - agentAddress: signal.agentAddress, - runId: signal.runId, - signalName: signal.signalName, - signalId: signal.signalId, - payload: signal.payload, - }); + await router.sendSignalDeliverToAllocation( + target, + { + agentAddress: frame.agentAddress, + runId: frame.runId, + signalName: frame.signalName, + signalId: frame.signalId, + payload: frame.payload, + }, + signal, + ); } else { // A deliverable mail dispatch always carries the sender persisted at // enqueue (the workflow_run_dispatch mail-sender check enforces it). @@ -218,12 +290,14 @@ export function createWorkflowDispatchService({ base64Encode(dispatch.rawMessage), dispatch.senderAddress, dispatch.messageId, + signal, ); } // Keep the delivery lease until the sidecar acknowledges its durable // inbox write. If that ack never arrives, lease expiry makes the same // immutable payload claimable again. } catch (error) { + signal.throwIfAborted(); await retry( dispatch, leaseId, @@ -231,7 +305,6 @@ export function createWorkflowDispatchService({ error instanceof Error ? error.message : String(error), ); } - return true; } async function reconcileUntilIdle(maxIterations = 100): Promise { diff --git a/vendor/intx/hub-sessions/src/ws/index.ts b/vendor/intx/hub-sessions/src/ws/index.ts index eb91862f45..462690a3e0 100644 --- a/vendor/intx/hub-sessions/src/ws/index.ts +++ b/vendor/intx/hub-sessions/src/ws/index.ts @@ -1,5 +1,6 @@ export { createSidecarRouter, + SidecarIdentityValidationError, type SidecarRouter, type SidecarRouterConfig, type SenderDeploySettledOutcome, @@ -7,6 +8,7 @@ export { type OAuthLoginGateOutcome, type OAuthLoginRequestOutcome, DEFAULT_OAUTH_LOGIN_TIMEOUT_MS, + type AllocatedSenderDeployAttempt, type SidecarConnection, type SidecarAuthIdentity, type SidecarAuthenticator, diff --git a/vendor/intx/hub-sessions/src/ws/pending-tracker.ts b/vendor/intx/hub-sessions/src/ws/pending-tracker.ts index 07d90e5a1f..ca7f422ab0 100644 --- a/vendor/intx/hub-sessions/src/ws/pending-tracker.ts +++ b/vendor/intx/hub-sessions/src/ws/pending-tracker.ts @@ -17,6 +17,28 @@ export type WsHandle = { close(): void; }; +/** + * Arms a one-shot timeout and returns the canceller for it. + * + * The tracker's timeouts are the only thing about it that depends on time + * passing, and a caller that wants them settled on demand rather than by + * waiting has nowhere to reach in: the global timer is not addressable from + * outside. Injecting the arming function makes the timeout observable and + * cancellable by whoever owns the tracker, so a test can fire a timeout, or + * assert one was disarmed, without a real duration elapsing. + * + * Returning the canceller rather than a handle keeps the timer's identity + * private to whichever implementation armed it. + */ +export type ScheduleTimeout = (handler: () => void, ms: number) => () => void; + +const scheduleOnGlobalTimer: ScheduleTimeout = (handler, ms) => { + const timer = setTimeout(handler, ms); + return () => { + clearTimeout(timer); + }; +}; + export type PendingEntry = { key: Key; ws: WsHandle; @@ -29,11 +51,39 @@ export type PendingEntry = { meta: Meta; resolve(value: Value): void; reject(error: string): void; - timer: ReturnType; + /** + * Disarms this entry's timeout. + * + * Runs at most once per entry, which is why it need not be idempotent and + * `ScheduleTimeout` does not ask its implementations for that: each of the + * four settle paths cancels and drops the entry from the map in one + * synchronous block, and a timeout that fires drops the entry itself, so + * nothing can reach the entry to cancel it a second time. + */ + cancelTimeout(): void; +}; + +/** + * A tracked entry plus the identity its own timeout handler recognizes it + * by. Keys are reused across round-trips, so "is the key occupied" cannot + * tell a live successor apart from the entry a given timeout belongs to; + * `token` can, because it is allocated per `register` call. It is internal + * to the tracker: `get` hands callers the `PendingEntry` view. + */ +type TrackedEntry = PendingEntry & { + readonly token: symbol; }; export class PendingTracker { - private readonly entries = new Map>(); + private readonly entries = new Map>(); + + /** + * `schedule` defaults to the global timer, which is what every production + * caller wants; it is injectable so a caller can drive the timeouts. + */ + constructor( + private readonly schedule: ScheduleTimeout = scheduleOnGlobalTimer, + ) {} /** * Register a pending round-trip and arm its timeout. The entry is stored @@ -57,7 +107,23 @@ export class PendingTracker { }, meta: Meta, ): void { - const timer = setTimeout(() => { + // Identifies the entry this call is about to register, so the handler + // below can recognize it. Allocated before `schedule`, because the + // handler closes over it and the entry itself does not exist yet. + const token = Symbol("pending-entry"); + const cancelTimeout = this.schedule(() => { + // Only the entry this call registered may be timed out. The global + // timer never fires a cancelled timeout, but `schedule` is injectable + // now and a caller's implementation is not owed that guarantee -- so + // the handler checks rather than trusting cancellation to be perfect. + // The check compares `token` rather than asking whether the key is + // occupied: keys are reused (`pendingDeploys` is keyed by agent + // address), so a stale fire can find the key held by a live successor. + // Treating that as its own entry would drop the successor without + // settling it and run this already-settled entry's per-site cleanup a + // second time. + const current = this.entries.get(key); + if (current === undefined || current.token !== token) return; this.entries.delete(key); options.reject(options.timeoutMessage); }, options.timeoutMs); @@ -67,7 +133,8 @@ export class PendingTracker { meta, resolve: options.resolve, reject: options.reject, - timer, + cancelTimeout, + token, }); } @@ -86,7 +153,7 @@ export class PendingTracker { resolve(key: Key, value: Value): boolean { const entry = this.entries.get(key); if (entry === undefined) return false; - clearTimeout(entry.timer); + entry.cancelTimeout(); this.entries.delete(key); entry.resolve(value); return true; @@ -99,7 +166,7 @@ export class PendingTracker { reject(key: Key, error: string): boolean { const entry = this.entries.get(key); if (entry === undefined) return false; - clearTimeout(entry.timer); + entry.cancelTimeout(); this.entries.delete(key); entry.reject(error); return true; @@ -115,7 +182,7 @@ export class PendingTracker { delete(key: Key): void { const entry = this.entries.get(key); if (entry === undefined) return; - clearTimeout(entry.timer); + entry.cancelTimeout(); this.entries.delete(key); } @@ -127,7 +194,7 @@ export class PendingTracker { rejectAllForWs(ws: WsHandle, error: string): void { for (const [key, entry] of this.entries) { if (entry.ws !== ws) continue; - clearTimeout(entry.timer); + entry.cancelTimeout(); this.entries.delete(key); entry.reject(error); } diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts b/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts index 08a2014cf7..e853e3838b 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-handler.test-helpers.ts @@ -50,16 +50,40 @@ export const TEST_CONFIG: HarnessConfig = { export function createMockWs(): WsHandle & { sent: string[]; closed: boolean; + /** + * Resolve once `predicate` holds over the frames sent so far, re-checking + * on each send. + * + * Several of the router's paths write to the socket from a fire-and-forget + * continuation -- the mail redelivery retry is one -- so a test that + * triggers one has no return value to await. The send is the event. + */ + awaitSent(predicate: (sent: readonly string[]) => boolean): Promise; } { + let waiters: (() => void)[] = []; return { sent: [], closed: false, send(data: string) { this.sent.push(data); + const waking = waiters; + waiters = []; + for (const wake of waking) wake(); }, close() { this.closed = true; }, + async awaitSent(predicate) { + for (;;) { + // Re-checked on every pass, so a frame sent before this call resolves + // it rather than leaving it waiting for another send. + const sent = new Promise((resolve) => { + waiters.push(resolve); + }); + if (predicate(this.sent)) return; + await sent; + } + }, }; } diff --git a/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts b/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts index b0be486cef..22396bfd3b 100644 --- a/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts +++ b/vendor/intx/hub-sessions/src/ws/sidecar-handler.ts @@ -71,8 +71,9 @@ export interface DeployFrameFailure extends Error { function deployFrameFailure( message: string, frameSent: boolean, + cause?: unknown, ): DeployFrameFailure { - return Object.assign(new Error(message), { frameSent }); + return Object.assign(new Error(message, { cause }), { frameSent }); } export function isDeployFrameFailure(err: unknown): err is DeployFrameFailure { @@ -83,6 +84,21 @@ export function isDeployFrameFailure(err: unknown): err is DeployFrameFailure { ); } +/** + * Identity validation failed or remained pending at the connection deadline. + * Readiness is unknown: the worker may be healthy behind the lookup, so + * callers must retry rather than treat this as a missed connection deadline. + */ +export class SidecarIdentityValidationError extends Error { + constructor(allocationId: string, generation: number, cause?: unknown) { + super( + `Cannot validate sidecar identity for allocation ${allocationId} generation ${String(generation)}`, + { cause }, + ); + this.name = "SidecarIdentityValidationError"; + } +} + export type SidecarConnection = { sidecarId: string; identity: SidecarAuthIdentity; @@ -218,6 +234,10 @@ export type OAuthLoginRequestOutcome = /** Default whole-login timeout: a human completes the consent page. */ export const DEFAULT_OAUTH_LOGIN_TIMEOUT_MS = 5 * 60_000; +export type AllocatedSenderDeployAttempt = AllocatedSidecarTarget & { + readonly leaseId: string; +}; + export type SidecarRouter = { handleOpen(ws: WsHandle): void; handleMessage(ws: WsHandle, data: string): void; @@ -266,14 +286,13 @@ export type SidecarRouter = { * sender parked while its public key was not yet recorded. A `recorded` * outcome wakes the parked mail and re-drives its delivery now that the key * co-delivers; a `failed` outcome drains it to `mail.outbound.undelivered`. - * Called from BOTH durable key-record sites (the non-allocated deploy-ack - * projection and the allocated anchor-key update) immediately after the write. - * The `address` MUST be byte-identical to the sender address the mail was sent - * under, or no parked entry matches. Idempotent by sender: a second settle, or - * a settle after the TTL already drained the mail, is a no-op. + * Allocated callbacks name their exact attempt; recovery may settle the + * previous attempt by generation after claiming its reconciliation lease. + * An address-only settlement belongs to a non-allocated deployment and cannot + * settle an allocated attempt. Stale or repeated settlements are no-ops. */ noteSenderDeploySettled( - address: string, + sender: string | AllocatedSidecarTarget | AllocatedSenderDeployAttempt, outcome: SenderDeploySettledOutcome, ): void; /** @@ -281,10 +300,13 @@ export type SidecarRouter = { * emit and its anchor-key update. An allocated run records its key later than * the deploy ack clears `pendingDeploys`, so this marker covers the allocated * pre-ack window that `pendingDeploys` alone under-covers. `noteSenderDeploySettled` - * clears it on both the recorded and failed outcomes, keeping the start/settle - * bracket balanced. + * clears it only when the durable outcome is known. Cancellation leaves it + * pending for recovery, so a lost publication response cannot discard mail. */ - noteSenderDeployStarted(address: string): void; + noteSenderDeployStarted( + address: string, + attempt: AllocatedSenderDeployAttempt, + ): void; /** * Returns the current connector-thread state for the named agent, or * `null` if the agent has no active connector thread (or if the @@ -397,14 +419,24 @@ export type SidecarAllocationRouter = { * terminal. Durable identity validation rejects later stale reconnects. */ retireAllocation(target: AllocatedSidecarTarget): void; - /** Resolve once the exact authenticated allocation generation is connected. */ + /** + * Resolve once the exact authenticated allocation generation is connected. + * Throws `SidecarIdentityValidationError` when readiness cannot be + * determined; only confirmed absence surfaces as a connection timeout. + * `onValidation` observes notification lookups that may outlive this wait. + */ waitForAllocatedSidecar( target: AllocatedSidecarTarget, timeoutMs: number, + onValidation?: (validation: Promise) => void, ): Promise; - /** Check exact allocated readiness without parking a reconciliation worker. */ + /** + * Check exact allocated readiness without parking a reconciliation worker. + * Throws `SidecarIdentityValidationError` when identity validation fails; + * `false` means the worker is confirmed absent or stale. + */ isAllocatedSidecarReady(target: AllocatedSidecarTarget): Promise; - /** Check whether the exact generation already hosts its workflow supervisor. */ + /** Check for an active supervisor, throwing when identity validation fails. */ isAllocatedWorkflowActive(target: AllocatedSidecarTarget): Promise; /** Probe a workflow on the exact provisioned allocation generation. */ sendProbeToAllocation( @@ -419,6 +451,7 @@ export type SidecarAllocationRouter = { config: HarnessConfig, workflow?: AgentDeployFrame["workflow"], signal?: AbortSignal, + beforeSend?: () => Promise, ): Promise<{ publicKey: string }>; sendPackToAllocation( target: AllocatedSidecarTarget, @@ -468,6 +501,7 @@ export type SidecarAllocationRouter = { rawMessage: string, authenticatedSender: string, messageId: string, + signal?: AbortSignal, ): Promise; /** Deliver an idempotent signal to the exact provisioned generation. */ sendSignalDeliverToAllocation( @@ -479,6 +513,7 @@ export type SidecarAllocationRouter = { signalId: string; payload: unknown; }, + signal?: AbortSignal, ): Promise; }; @@ -537,6 +572,26 @@ export type SidecarRouterConfig = { /** Interval between redelivery attempts of a connected-window `mail.inbound` * the sidecar has not yet acknowledged with `mail.inbound.ack`. */ mailAckRetryIntervalMs?: number; + /** + * Arms the mail-redelivery retry and the connection-liveness timers, and + * returns each one's canceller. Defaults to the global timer, which is what + * production wants. + * + * The intervals beside it say how long until something should happen; this + * says what makes it happen. With only the intervals injectable, a test had + * to shorten one and then sleep past it, which turns an assertion about + * WHETHER something happened into a bet on how much the machine got through + * -- and, for the liveness deadline, on a pause landing inside a window + * rather than past it. + * + * REQUIRED of the returned canceller: calling it more than once must be + * harmless. A pending-mail entry outlives a disconnect, so the disconnect + * path cancels its retry and a later reconnect can cancel the same one + * again. `clearTimeout` on an already-cleared timer is a no-op, which is + * what makes the default satisfy this; a substitute must arrange the same, + * typically by flipping a flag. + */ + scheduleTimeout?: (handler: () => void, ms: number) => () => void; /** Maximum redelivery attempts before the hub stops retrying an un-acked * connected-window `mail.inbound`. Bounds the retry so a sidecar that never * acks does not accumulate an unbounded timer per delivery. */ @@ -587,6 +642,12 @@ export function createSidecarRouter( disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, + scheduleTimeout = (handler: () => void, ms: number) => { + const handle = setTimeout(handler, ms); + return () => { + clearTimeout(handle); + }; + }, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config; @@ -610,6 +671,9 @@ export function createSidecarRouter( resolve(): void; reject(error: Error): void; timer: ReturnType; + validationFailure?: SidecarIdentityValidationError; + validations: Set>; + onValidation?: (validation: Promise) => void; }; const allocationWaiters = new Map>(); // agentAddress → ws handle (routing table) @@ -627,7 +691,10 @@ export function createSidecarRouter( // so pendingDeploys alone under-covers the allocated pre-ack window. session- // service brackets this marker across its deploy try/catch: set before the // deploy emit, cleared by noteSenderDeploySettled on record or failure. - const allocatedKeyRecordInFlight = new Set(); + const allocatedKeyRecordInFlight = new Map< + string, + AllocatedSenderDeployAttempt + >(); // agentAddress → queued frames for disconnected agents awaiting reconnect type DisconnectedAgent = { queue: HubFrame[]; @@ -648,7 +715,14 @@ export function createSidecarRouter( messageId: string; frame: HubFrame; attempts: number; - timer: ReturnType; + /** + * Disarms this entry's redelivery retry. Safe to call more than once: + * `scheduleTimeout` requires an idempotent canceller. The second call is + * reached when a reconnect drops a generation-local entry the disconnect + * had already cancelled -- the disconnect keeps the entry for replay, so + * its spent canceller is still on it. + */ + cancelRetry: () => void; // When this mail triggers a workflow run, the run's already-materialized // grants ride alongside it. Redelivery replays this snapshot as a // `run.grants` frame AHEAD of the mail so the redelivered trigger lands on @@ -702,7 +776,7 @@ export function createSidecarRouter( // identically to a null entry (no active thread). const connectorStates = new Map(); // ws handle → liveness timer (reset on each ping from the sidecar) - const livenessTimers = new Map>(); + const livenessTimers = new Map void>(); // Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue` // for the split and the invariant behind it). A frame that establishes or @@ -812,8 +886,8 @@ export function createSidecarRouter( function scheduleMailRetry( agentAddress: string, messageId: string, - ): ReturnType { - return setTimeout(() => { + ): () => void { + return scheduleTimeout(() => { void retryPendingMail(agentAddress, messageId).catch((err: unknown) => { logger.warn`Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`; }); @@ -841,13 +915,13 @@ export function createSidecarRouter( pendingMail.set(agentAddress, byId); } const existing = byId.get(messageId); - if (existing !== undefined) clearTimeout(existing.timer); + if (existing !== undefined) existing.cancelRetry(); byId.set(messageId, { agentAddress, messageId, frame, attempts: 0, - timer: scheduleMailRetry(agentAddress, messageId), + cancelRetry: scheduleMailRetry(agentAddress, messageId), ...(runGrants !== undefined ? { runGrants } : {}), ...(allocatedTarget !== undefined ? { allocatedTarget } : {}), }); @@ -1045,7 +1119,7 @@ export function createSidecarRouter( if (!(await replaySendPendingMail(conn, entry))) return; entry.attempts += 1; - entry.timer = scheduleMailRetry(agentAddress, messageId); + entry.cancelRetry = scheduleMailRetry(agentAddress, messageId); } function resolvePendingMail(agentAddress: string, messageId: string): void { @@ -1053,7 +1127,7 @@ export function createSidecarRouter( if (byId === undefined) return; const entry = byId.get(messageId); if (entry === undefined) return; - clearTimeout(entry.timer); + entry.cancelRetry(); deletePendingMail(byId, agentAddress, messageId); } @@ -1067,7 +1141,7 @@ export function createSidecarRouter( function retainPendingMailForAddress(agentAddress: string): void { const byId = pendingMail.get(agentAddress); if (byId === undefined) return; - for (const entry of byId.values()) clearTimeout(entry.timer); + for (const entry of byId.values()) entry.cancelRetry(); const existing = pendingMailRetention.get(agentAddress); if (existing !== undefined) clearTimeout(existing); const timer = setTimeout(() => { @@ -1115,13 +1189,13 @@ export function createSidecarRouter( // The Hub-owned dispatch row survives generation replacement and will // be requeued by the allocation-ready callback. Do not leak or replay // this generation-local retry entry onto a different worker. - clearTimeout(entry.timer); + entry.cancelRetry(); deletePendingMail(byId, agentAddress, entry.messageId); continue; } if (!(await replaySendPendingMail(conn, entry))) continue; entry.attempts = 0; - entry.timer = scheduleMailRetry(agentAddress, entry.messageId); + entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId); } if (byId.size > 0) { logger.info`Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`; @@ -1130,14 +1204,14 @@ export function createSidecarRouter( function resetLivenessTimer(ws: WsHandle): void { const existing = livenessTimers.get(ws); - if (existing !== undefined) clearTimeout(existing); + if (existing !== undefined) existing(); - const timer = setTimeout(() => { + const cancel = scheduleTimeout(() => { livenessTimers.delete(ws); logger.warn`Sidecar ping timeout, closing connection`; ws.close(); }, pingTimeoutMs); - livenessTimers.set(ws, timer); + livenessTimers.set(ws, cancel); } function handlePing(ws: WsHandle): void { @@ -1467,15 +1541,54 @@ export function createSidecarRouter( const waiters = allocationWaiters.get(allocationId); const current = allocatedConnections.get(allocationId); if (waiters === undefined || current === undefined) return; - if (!(await validateSidecarIdentity(current.identity, "readiness"))) return; + const matchingWaiters = [...waiters].filter( + (waiter) => waiter.generation === current.identity.generation, + ); + if (matchingWaiters.length === 0) return; + const validation = Promise.resolve().then(() => + validateSidecarIdentity(current.identity, "readiness"), + ); + for (const waiter of matchingWaiters) { + waiter.validations.add(validation); + waiter.onValidation?.(validation); + } + let identityCurrent: boolean; + try { + identityCurrent = await validation; + } catch (cause) { + // A failed revalidation leaves the waiters parked: a later register + // revalidates, and at expiry the wait reports the failure rather than a + // missed deadline. Registration itself was already gated, so this must + // not fail the connection that just registered. + const validationFailure = new SidecarIdentityValidationError( + allocationId, + current.identity.generation, + cause, + ); + for (const waiter of matchingWaiters) { + waiter.validationFailure = validationFailure; + } + return; + } finally { + for (const waiter of matchingWaiters) { + waiter.validations.delete(validation); + } + } + // A clean validation supersedes earlier failures: expiry must report the + // current reading, not a stale transient. + for (const waiter of matchingWaiters) { + delete waiter.validationFailure; + } + if (!identityCurrent || allocatedConnections.get(allocationId) !== current) + return; - for (const waiter of [...waiters]) { - if (waiter.generation !== current.identity.generation) continue; + for (const waiter of matchingWaiters) { + if (!waiters.delete(waiter)) continue; clearTimeout(waiter.timer); - waiters.delete(waiter); waiter.resolve(); } - if (waiters.size === 0) allocationWaiters.delete(allocationId); + if (waiters.size === 0 && allocationWaiters.get(allocationId) === waiters) + allocationWaiters.delete(allocationId); } async function handleAllocatedRegister( @@ -1757,15 +1870,42 @@ export function createSidecarRouter( logger.warn`Dropping ${String(entries.length)} deferred message(s) from ${authenticatedSender}: ${reason}`; } - function noteSenderDeployStarted(address: string): void { - allocatedKeyRecordInFlight.add(address); + function noteSenderDeployStarted( + address: string, + attempt: AllocatedSenderDeployAttempt, + ): void { + if (allocatedKeyRecordInFlight.has(address)) { + throw new Error(`Sender deployment ${address} has an unresolved attempt`); + } + allocatedKeyRecordInFlight.set(address, attempt); } function noteSenderDeploySettled( + sender: string | AllocatedSidecarTarget | AllocatedSenderDeployAttempt, + outcome: SenderDeploySettledOutcome, + ): void { + if (typeof sender !== "string") { + for (const [address, attempt] of [...allocatedKeyRecordInFlight]) { + if ( + attempt.allocationId !== sender.allocationId || + attempt.generation !== sender.generation || + ("leaseId" in sender && attempt.leaseId !== sender.leaseId) + ) { + continue; + } + allocatedKeyRecordInFlight.delete(address); + settleSenderMail(address, outcome); + } + return; + } + if (allocatedKeyRecordInFlight.has(sender)) return; + settleSenderMail(sender, outcome); + } + + function settleSenderMail( address: string, outcome: SenderDeploySettledOutcome, ): void { - allocatedKeyRecordInFlight.delete(address); if ("failed" in outcome) { drainDeferredSenderMail( address, @@ -1775,16 +1915,16 @@ export function createSidecarRouter( } for (const entry of claimAllDeferredSenderMail(address)) { // Re-drive delivery as its OWN task, off the settle's stack, so delivery - // work never runs on the deploy-ack handler's stack or reorders against - // it. The re-drive re-enters handleMailOutbound; the key is recorded (the - // durable write happens-before this settle), so it resolves the sender key - // and delivers inline. + // work never runs on the deploy-ack handler's stack. Carry the confirmed + // key: another attempt may start before this task runs, and must not + // capture this mail or change the key that authenticates it. void Promise.resolve() .then(() => handleMailOutbound( entry.rawMessage, entry.authenticatedSender, entry.recipients, + outcome.recorded, ), ) .catch((err: unknown) => { @@ -1877,6 +2017,7 @@ export function createSidecarRouter( rawMessage: string, authenticatedSender: string, recipients: string[], + recordedSenderKey?: string, ): Promise { // A mail addressed to more than one workflow deployment would birth a // run per recipient from a single inbound mail. The stable runId @@ -1911,11 +2052,20 @@ export function createSidecarRouter( // until the key lands rather than delivering it keyless, which a strict // recipient drops as an unknown sender. A parked message returns here and is // re-driven later by a settle or the TTL. - const resolution = await resolveSenderIdentitiesOrPark( - rawMessage, - authenticatedSender, - recipients, - ); + const resolution = + recordedSenderKey === undefined + ? await resolveSenderIdentitiesOrPark( + rawMessage, + authenticatedSender, + recipients, + ) + : { + deliver: true as const, + senderIdentities: senderIdentitiesFromKey( + authenticatedSender, + recordedSenderKey, + ), + }; if (!resolution.deliver) return; const senderIdentities = resolution.senderIdentities; @@ -2008,6 +2158,9 @@ export function createSidecarRouter( // cached" invariant holds; a recipient with no cached key resolves such // mail as `unknown`, which its admission policy rejects by default (a // workflow may relax `unknown` to admit). + // Finish asynchronous preparation before sending the grants and mail + // together, keeping another delivery's key out of the gap between them. + const messageId = await deriveMessageId(base64Decode(rawMessage)); // Send the run's grants ahead of the mail. A `false` here means the // deployment is unroutable. Do not route the mail that would dispatch // it; the grants-only reservation remains the canonical snapshot for a @@ -2028,7 +2181,6 @@ export function createSidecarRouter( // mail's own id (derived over the same bytes the sidecar derives), so a // redelivery replays identically and the downstream RunStarted / // stable-runId dedup makes it effectively-once. - const messageId = await deriveMessageId(base64Decode(rawMessage)); const outcome: "routed" | "unrouted" = routeMail( recipient, rawMessage, @@ -2206,9 +2358,9 @@ export function createSidecarRouter( connections.delete(ws); // Cancel the liveness timer for this connection. - const livenessTimer = livenessTimers.get(ws); - if (livenessTimer !== undefined) { - clearTimeout(livenessTimer); + const cancelLiveness = livenessTimers.get(ws); + if (cancelLiveness !== undefined) { + cancelLiveness(); livenessTimers.delete(ws); } @@ -2598,6 +2750,20 @@ export function createSidecarRouter( } allocationFences.set(allocationId, generation); + // A durable generation advance resolves unfinished initialization as failed. + // This also covers a cleanup transaction whose response was lost: the next + // reconciliation rebuilds this fence before it can start a replacement. + for (const attempt of [...allocatedKeyRecordInFlight.values()]) { + if ( + attempt.allocationId === allocationId && + attempt.generation < generation + ) { + noteSenderDeploySettled(attempt, { + failed: `Allocation ${allocationId} advanced beyond the deployment attempt`, + }); + } + } + const current = allocatedConnections.get(allocationId); if (current !== undefined && current.identity.generation !== generation) { handleClose(current.ws); @@ -2625,6 +2791,19 @@ export function createSidecarRouter( disconnectAllocation(target); allocationFences.delete(target.allocationId); + // The fence is gone, so a lingering attempt can never settle normally. + // Fail it here rather than leaving a marker that blocks the address. + for (const attempt of [...allocatedKeyRecordInFlight.values()]) { + if ( + attempt.allocationId === target.allocationId && + attempt.generation <= target.generation + ) { + noteSenderDeploySettled(attempt, { + failed: `Allocation ${target.allocationId} generation ${String(target.generation)} retired`, + }); + } + } + const waiters = allocationWaiters.get(target.allocationId); if (waiters === undefined) return; allocationWaiters.delete(target.allocationId); @@ -2656,7 +2835,17 @@ export function createSidecarRouter( `Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`, ); } - if (!(await validateSidecarIdentity(current.identity, use))) { + let identityCurrent: boolean; + try { + identityCurrent = await validateSidecarIdentity(current.identity, use); + } catch (cause) { + throw new SidecarIdentityValidationError( + target.allocationId, + target.generation, + cause, + ); + } + if (!identityCurrent) { if (allocatedConnections.get(target.allocationId) === current) { handleClose(current.ws); current.ws.close(); @@ -2710,7 +2899,11 @@ export function createSidecarRouter( try { await getProvisionedConnection(target, "readiness"); return true; - } catch { + } catch (error) { + // A failed validation is unknown, not absent: the worker may be healthy + // behind a failed lookup, so report it distinctly instead of answering + // `false` and letting the caller release a live worker. + if (error instanceof SidecarIdentityValidationError) throw error; return false; } } @@ -2722,7 +2915,8 @@ export function createSidecarRouter( const { conn } = await getAllocatedConnection(target, "readiness"); if (conn.identity.kind !== "allocated") return false; return conn.workflowAddresses.has(conn.identity.workflowRunAddress); - } catch { + } catch (error) { + if (error instanceof SidecarIdentityValidationError) throw error; return false; } } @@ -2730,14 +2924,26 @@ export function createSidecarRouter( async function waitForAllocatedSidecar( target: AllocatedSidecarTarget, timeoutMs: number, + onValidation?: (validation: Promise) => void, ): Promise { - if (await isAllocatedSidecarReady(target)) return; + // An indeterminable worker waits out the unknown while time remains: only + // confirmed absence may surface as a connection timeout. At expiry the + // wait reports the validation failure rather than a missed deadline, so + // the caller retries instead of releasing a worker that may be healthy. + let validationFailure: SidecarIdentityValidationError | undefined; + try { + if (await isAllocatedSidecarReady(target)) return; + } catch (error) { + if (!(error instanceof SidecarIdentityValidationError)) throw error; + validationFailure = error; + } if (allocationFences.get(target.allocationId) !== target.generation) { throw new Error( `Allocation ${target.allocationId} generation ${String(target.generation)} is not current`, ); } if (timeoutMs <= 0) { + if (validationFailure !== undefined) throw validationFailure; throw new Error( `Timed out waiting for allocated sidecar ${target.allocationId}`, ); @@ -2746,6 +2952,8 @@ export function createSidecarRouter( await new Promise((resolve, reject) => { const waiter: AllocationWaiter = { generation: target.generation, + validations: new Set(), + ...(onValidation !== undefined ? { onValidation } : {}), resolve, reject, timer: setTimeout(() => { @@ -2755,11 +2963,18 @@ export function createSidecarRouter( allocationWaiters.delete(target.allocationId); } reject( - new Error( - `Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`, - ), + waiter.validationFailure ?? + (waiter.validations.size > 0 + ? new SidecarIdentityValidationError( + target.allocationId, + target.generation, + ) + : new Error( + `Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`, + )), ); }, timeoutMs), + ...(validationFailure !== undefined ? { validationFailure } : {}), }; let waiters = allocationWaiters.get(target.allocationId); if (waiters === undefined) { @@ -3019,8 +3234,11 @@ export function createSidecarRouter( rawMessage: string, authenticatedSender: string, messageId: string, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const { ws, conn } = await getAllocatedConnection(target, "routing"); + signal?.throwIfAborted(); if (addressIndex.get(agentAddress) !== ws) { throw new Error( `Address ${agentAddress} is not routed on allocation ${target.allocationId}`, @@ -3043,6 +3261,7 @@ export function createSidecarRouter( lookups.resolveSenderKey !== undefined ? await lookups.resolveSenderKey(authenticatedSender) : null; + signal?.throwIfAborted(); // Co-deliver the resolved key on the run's grants barrier, omitting a null // key so it is never cached (see deliverMailToRecipient). The same list // rides the pending-mail entry so the reconnect replay carries it too. @@ -3226,28 +3445,54 @@ export function createSidecarRouter( harnessConfig: HarnessConfig, workflow?: AgentDeployFrame["workflow"], signal?: AbortSignal, + beforeSend?: () => Promise, ): Promise<{ publicKey: string }> { - signal?.throwIfAborted(); - const { ws, conn } = await getAllocatedConnection(target, "routing"); - signal?.throwIfAborted(); - if (agentAddress !== conn.identity.workflowRunAddress) { - throw new Error( - `Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`, + try { + signal?.throwIfAborted(); + const { ws, conn } = await getAllocatedConnection(target, "routing"); + signal?.throwIfAborted(); + if (agentAddress !== conn.identity.workflowRunAddress) { + throw new Error( + `Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`, + ); + } + const existing = addressIndex.get(agentAddress); + if (existing !== undefined && existing !== ws) { + throw new Error( + `Deployment ${agentAddress} is already routed to another sidecar`, + ); + } + if (hubPublicKeyHex === undefined) + throw new Error("Hub signing key is required for agent deployment"); + if (pendingDeploys.has(agentAddress)) + throw new Error( + `Deploy already in progress for agent "${agentAddress}"`, + ); + await beforeSend?.(); + signal?.throwIfAborted(); + if ( + allocatedConnections.get(target.allocationId)?.ws !== ws || + allocationFences.get(target.allocationId) !== target.generation + ) { + throw new Error( + `Allocated sidecar connection changed for allocation ${target.allocationId}`, + ); + } + // Return without awaiting: only pre-send failures belong to this catch. + return sendAgentDeployOnConnection( + ws, + conn, + agentAddress, + harnessConfig, + workflow, ); - } - const existing = addressIndex.get(agentAddress); - if (existing !== undefined && existing !== ws) { - throw new Error( - `Deployment ${agentAddress} is already routed to another sidecar`, + } catch (cause) { + throw deployFrameFailure( + cause instanceof Error ? cause.message : String(cause), + false, + cause, ); } - return sendAgentDeployOnConnection( - ws, - conn, - agentAddress, - harnessConfig, - workflow, - ); } /** @@ -3672,8 +3917,11 @@ export function createSidecarRouter( signalId: string; payload: unknown; }, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const { ws, conn } = await getAllocatedConnection(target, "routing"); + signal?.throwIfAborted(); if (addressIndex.get(opts.agentAddress) !== ws) { throw new Error( `Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`, diff --git a/vendor/intx/inference/VENDORED-FROM b/vendor/intx/inference/VENDORED-FROM index e53d0681a2..98d577621f 100644 --- a/vendor/intx/inference/VENDORED-FROM +++ b/vendor/intx/inference/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/inference) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. providers/google-genai-files.ts builds its upload body as `new Uint8Array(bytes)`: TS 6's lib.dom `BodyInit` rejects `Uint8Array` and workbench compiles this source under DOM-lib packages (upstream compiles ESNext-only under TS 5.9). CL-7443 retires CL-7190's message_response resume branch in reactor.ts's resumePendingOperation/timeoutMessageFor (and its answer_result resume mode), plus the reactor.test.ts/testing/fakes.ts regression harness CL-7190 added to cover it: `ask_user` posts its question and ends the turn instead of parking a gate, so the reactor is back to zero local delta and zero tests, matching upstream. diff --git a/vendor/intx/inference/package.json b/vendor/intx/inference/package.json index 683c817c70..28bf1ecbfa 100644 --- a/vendor/intx/inference/package.json +++ b/vendor/intx/inference/package.json @@ -1,7 +1,7 @@ { "name": "@intx/inference", "description": "Provider-agnostic inference runtime with Anthropic, OpenAI, and Google GenAI adapters", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -18,7 +18,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:" }, diff --git a/vendor/intx/mail-memory/VENDORED-FROM b/vendor/intx/mail-memory/VENDORED-FROM index b652dfc4f5..9f3049781b 100644 --- a/vendor/intx/mail-memory/VENDORED-FROM +++ b/vendor/intx/mail-memory/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mail-memory) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mail-memory/package.json b/vendor/intx/mail-memory/package.json index 3f928472d9..a8d4e17087 100644 --- a/vendor/intx/mail-memory/package.json +++ b/vendor/intx/mail-memory/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mail-memory", "description": "In-memory MessageTransport for single-process and test environments", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,8 +14,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.3.0", - "@intx/log": "0.3.0", + "@intx/crypto": "0.4.0", + "@intx/log": "0.4.0", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", diff --git a/vendor/intx/mailbox/VENDORED-FROM b/vendor/intx/mailbox/VENDORED-FROM index 9da79cdb65..f55254f0f3 100644 --- a/vendor/intx/mailbox/VENDORED-FROM +++ b/vendor/intx/mailbox/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mailbox) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mailbox/package.json b/vendor/intx/mailbox/package.json index e56df10940..27aecaa4b9 100644 --- a/vendor/intx/mailbox/package.json +++ b/vendor/intx/mailbox/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mailbox", "description": "Storage-agnostic IMAP mailbox model with search, threading, and fetch projections", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,7 +14,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx/mime/VENDORED-FROM b/vendor/intx/mime/VENDORED-FROM index 64a384a6ef..315c4c0a47 100644 --- a/vendor/intx/mime/VENDORED-FROM +++ b/vendor/intx/mime/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/mime) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/mime/package.json b/vendor/intx/mime/package.json index 0d690396da..94f172a824 100644 --- a/vendor/intx/mime/package.json +++ b/vendor/intx/mime/package.json @@ -1,7 +1,7 @@ { "name": "@intx/mime", "description": "RFC 2822 message assembly and parsing with PGP detached signatures", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -14,7 +14,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/types": "workspace:*", "arktype": "catalog:" }, diff --git a/vendor/intx/types/VENDORED-FROM b/vendor/intx/types/VENDORED-FROM index 43124c5319..77c532f935 100644 --- a/vendor/intx/types/VENDORED-FROM +++ b/vendor/intx/types/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/types) -Commit: c1b5bb73ff4815414af44e4662214e054e9e70e2 (origin/main, 2026-09-06) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-7443 retires CL-7190's local delta ("message_response" added to signals.ts's signalKinds, alongside signalKindToGateType): `ask_user` posts its question and ends the turn instead of parking a signal, so signals.ts is back to upstream's approval-only shape. CL-6396: AgentEventFrame gains an optional childRunId so a sidecar can name the occurrence (turn__) the event belongs to. diff --git a/vendor/intx/types/package.json b/vendor/intx/types/package.json index b8d740deb3..8f5a0ae819 100644 --- a/vendor/intx/types/package.json +++ b/vendor/intx/types/package.json @@ -1,7 +1,7 @@ { "name": "@intx/types", "description": "Runtime validators, API contract types, runtime interfaces, and sidecar wire frames for Interchange", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { diff --git a/vendor/intx/workflow-deploy/VENDORED-FROM b/vendor/intx/workflow-deploy/VENDORED-FROM index d8a18502bd..dfd09f1070 100644 --- a/vendor/intx/workflow-deploy/VENDORED-FROM +++ b/vendor/intx/workflow-deploy/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow-deploy) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. No source delta. diff --git a/vendor/intx/workflow-deploy/package.json b/vendor/intx/workflow-deploy/package.json index d16200a4f0..bdf98db0fe 100644 --- a/vendor/intx/workflow-deploy/package.json +++ b/vendor/intx/workflow-deploy/package.json @@ -1,7 +1,7 @@ { "name": "@intx/workflow-deploy", "description": "Deploy-time validation, capability walk, and operator-approval gating for workflows", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -19,7 +19,7 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/tool-packaging": "0.3.0", + "@intx/tool-packaging": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*" }, diff --git a/vendor/intx/workflow-host/README.md b/vendor/intx/workflow-host/README.md index 9b4c1a74fe..5ad593d565 100644 --- a/vendor/intx/workflow-host/README.md +++ b/vendor/intx/workflow-host/README.md @@ -287,7 +287,19 @@ The contract is intentionally narrow: factory returns `RunWorkflowChildBindings`: substrate `RepoStore`, principal, per-deployment repo ids, scheduler, step invoker, child spawner, grant evaluator. The factory consumes the typed struct, - never `NodeJS.ProcessEnv` directly. + never `NodeJS.ProcessEnv` directly. Each spawner declares + `hasUpstreamSignalResolver` on the runtime env it builds: `true` + for a run an answer can reach -- the deployment's own addressable + run, or a suspendable body whose container relays a decision back + down -- and `false` for a terminal `childWorkflow` child, which + carries no address and is run to its terminal rather than driven + across parks. The field is required, so an omission is a compile + error; a wrong `true` is not. Declared on a terminal-child seam it + type-checks and reinstates the hang the flag exists to remove. The + in-tree seams show both answers: `buildRuntimeEnv` declares `true` + for the deployment's own run, while in the sidecar's substrate + factory `createSidecarRunChild` declares `false` and + `createSidecarSpawnSuspendableChild` declares `true`. 3. **The helper fails loudly.** A missing or malformed spawn-time env throws via `parseSpawnTimeEnv`; a substrate-config key the host listed but the supervisor did not populate throws before the diff --git a/vendor/intx/workflow-host/VENDORED-FROM b/vendor/intx/workflow-host/VENDORED-FROM index 0b3edb2990..be45d8f62f 100644 --- a/vendor/intx/workflow-host/VENDORED-FROM +++ b/vendor/intx/workflow-host/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow-host) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. CL-6448: the suspendable-child (onTrigger body) spawn seam threads the parent child's credentials-backed authorize and MailPartReader: RunSuspendableChild's input and createInMemorySpawnSuspendableChild's opts gain the two optional fields (adapters/spawn-child.ts), and child/run-child.ts hoists `authorize` above the body resolver and passes both when building it, so a tool-bearing body agent gates its tool calls through the same per-step grant snapshot a top-level step does and reads an attachments-only inbound mail's parts instead of throwing. Upstream runs bodies toolless and has no analog. child/run-child.ts also resolves a step's grants entry through findStepGrantsEntry (head collapse: a body step whose own id is absent from a single-step deployment's snapshot resolves to the sole entry). CL-6396: the event-channel envelope, sender, and receiver carry an optional childRunId; run-child stamps spawnInput.childRunId onto body-child events so overlapping turns HMAC-bind the occurrence they belong to. Retired at this pin: the CL-6448 credentialWiring thread (upstream now passes the run's live credential-material cell into every body spawn natively as `credentialMaterial`). Retired at earlier pins: the empty-mail drop (CL-6164) and the action/loop runtime bind (CL-6325). Temporary delta: buildInboundMessageFromMail omits `subject` when the decoded header is empty (a present-but-textless Subject header), since createInboundMessage rejects an empty subject and fails the step; see VENDORED.md's kill condition. diff --git a/vendor/intx/workflow-host/package.json b/vendor/intx/workflow-host/package.json index 5ed0a72c99..e5058ae510 100644 --- a/vendor/intx/workflow-host/package.json +++ b/vendor/intx/workflow-host/package.json @@ -1,7 +1,7 @@ { "name": "@intx/workflow-host", "description": "Production-host implementations of the abstract WorkflowRuntimeEnv from @intx/workflow", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { @@ -15,14 +15,14 @@ }, "dependencies": { "@intx/agent": "workspace:*", - "@intx/crypto": "0.3.0", + "@intx/crypto": "0.4.0", "@intx/hub-sessions": "workspace:*", "@intx/inference": "workspace:*", - "@intx/log": "0.3.0", + "@intx/log": "0.4.0", "@intx/mail-memory": "workspace:*", "@intx/mailbox": "workspace:*", "@intx/mime": "workspace:*", - "@intx/storage-isogit": "0.3.0", + "@intx/storage-isogit": "0.4.0", "@intx/types": "workspace:*", "@intx/workflow": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx/workflow-host/src/child/run-child.ts b/vendor/intx/workflow-host/src/child/run-child.ts index c9ad4e6f80..92eb80358f 100644 --- a/vendor/intx/workflow-host/src/child/run-child.ts +++ b/vendor/intx/workflow-host/src/child/run-child.ts @@ -1753,6 +1753,10 @@ function buildRuntimeEnv(args: { authorize: args.authorize, invokeStep, spawnChild, + // The deployment's addressable run. Its parks are registered with the hub + // through the notify sink below, and a resolved decision is delivered back + // onto this run's own channel. + hasUpstreamSignalResolver: true, // Resolve a loop's `while`/`carry` refs against the closure's loop module. // Every ref was force-resolved at establish, so a lookup here cannot fail // for a definition that passed startup. diff --git a/vendor/intx/workflow-host/src/seams/scheduler.ts b/vendor/intx/workflow-host/src/seams/scheduler.ts index 79884b1743..1abd960ff7 100644 --- a/vendor/intx/workflow-host/src/seams/scheduler.ts +++ b/vendor/intx/workflow-host/src/seams/scheduler.ts @@ -112,13 +112,34 @@ export type SchedulerOpts = { * callbacks and to skip past-due cron entries on recovery. */ clock: () => Date; + /** + * Arms a one-shot timer and returns its canceller. Defaults to the global + * timer, which is what production wants. + * + * The clock above decides WHEN a timer should fire; this decides what + * actually fires it. Without both, a caller can compute a deterministic + * delay and still have to wait out the real interval to observe the + * firing -- which left this module's own tests waiting past a `fireAt`, + * and, in one case, waiting longer than a cancelled timer's delay to argue + * from silence that the cancel had worked. + */ + scheduleTimeout?: (handler: () => void, ms: number) => () => void; }; type QueuedTimer = { runId: string; timerId: string; fireAtMs: number; - timeout: ReturnType; + /** + * Disarms this timer. + * + * Runs at most once per queued timer, which is why it need not be + * idempotent and `scheduleTimeout` does not ask its implementations for + * that: `stop` and `cancelQueued` drop the queue entry in the same + * synchronous block as the cancel, and a fired timer drops its own entry + * before awaiting anything, so nothing can reach it to cancel it again. + */ + cancelTimeout: () => void; cron: boolean; }; @@ -162,6 +183,16 @@ export type SchedulerHandle = { export function createWorkflowHostScheduler( opts: SchedulerOpts, ): SchedulerHandle { + // Production arms the global timer; a caller supplying its own can fire a + // queued timer on demand instead of waiting out its delay. + const schedule = + opts.scheduleTimeout ?? + ((handler: () => void, ms: number) => { + const handle = setTimeout(handler, ms); + return () => { + clearTimeout(handle); + }; + }); const queues = new Map(); const liveSubscriptions: { abort: AbortController; @@ -194,7 +225,7 @@ export function createWorkflowHostScheduler( const key = queueKey(runId, timerId); if (queues.has(key)) return; // idempotent const delayMs = Math.max(0, fireAtMs - opts.clock().getTime()); - const timeout = setTimeout(() => { + const cancelTimeout = schedule(() => { void fireTimer(runId, timerId).catch((cause) => { // The scheduler's commit failed. Surface as unhandled so // operators see it; the runtime body's awaiter will hang @@ -206,7 +237,7 @@ export function createWorkflowHostScheduler( ); }); }, delayMs); - queues.set(key, { runId, timerId, fireAtMs, timeout, cron }); + queues.set(key, { runId, timerId, fireAtMs, cancelTimeout, cron }); } function startLiveSubscription(repoId: RepoId): void { @@ -327,7 +358,7 @@ export function createWorkflowHostScheduler( async stop() { if (stopped) return; stopped = true; - for (const t of queues.values()) clearTimeout(t.timeout); + for (const t of queues.values()) t.cancelTimeout(); queues.clear(); for (const sub of liveSubscriptions.splice(0)) { sub.abort.abort(); @@ -340,7 +371,7 @@ export function createWorkflowHostScheduler( const key = queueKey(runId, timerId); const entry = queues.get(key); if (entry === undefined) return; - clearTimeout(entry.timeout); + entry.cancelTimeout(); queues.delete(key); }, queuedTimers() { diff --git a/vendor/intx/workflow-host/src/supervisor/supervisor.ts b/vendor/intx/workflow-host/src/supervisor/supervisor.ts index 3dc1a28757..58ab4035c3 100644 --- a/vendor/intx/workflow-host/src/supervisor/supervisor.ts +++ b/vendor/intx/workflow-host/src/supervisor/supervisor.ts @@ -1090,6 +1090,10 @@ export function createWorkflowSupervisor( state.phase !== "running" || spawnContext === null ) { + // The backoff above announced a respawn before parking. Report the bail + // too, so a deployment that stops respawning does not leave an operator + // with a promised respawn and no record of what happened to it. + logger.info`respawn backoff elapsed but the crashed cohort is no longer the running one; skipping respawn (phase=${state.phase}, armed generation ${String(armedGeneration)}, current generation ${String(childGeneration)}): ${reason}`; return; } const priorRunning = state; diff --git a/vendor/intx/workflow/README.md b/vendor/intx/workflow/README.md index c283528dff..8ef4e684b4 100644 --- a/vendor/intx/workflow/README.md +++ b/vendor/intx/workflow/README.md @@ -33,6 +33,82 @@ DI seams for mail bus / signing key / subprocess spawner), see and the agent-deploy-trivial-workflow dichotomy, see `@intx/workflow-deploy`. +## Tools are available wherever inference runs + +Every agent step can call tools, no matter which primitive encloses +it. A step at the top level, a step inside a `loop` body, a step +inside an inline `onTrigger` section body, and a step inside a +`childWorkflow` child all get the same tool-bearing execution +environment. There is no nesting depth and no primitive at which an +agent silently runs at reduced capability. + +This is an invariant of the system, not a property of the primitives +that happen to exist today. A new body-bearing primitive inherits it: +if the primitive runs inference, its steps get tools. + +The `childWorkflow` boundary is where that word carries weight. A +step in a child gets the same tools, and a tool declared +`approval: "ask"` reaches the provider exactly as it does at the top +level. What the child cannot do is hold the resulting approval: +nothing upstream can answer a park inside it, so the runtime refuses +the park and fails the step, naming the gate. The capability is not +quietly withdrawn -- the step that depends on it stops, and the +operator's approval decision is the reason a run failed rather than +a decision that had no effect. Keep such a step in the deployment's +own run, in a `loop` body, or in an `onTrigger` section body, where +a decision can be relayed down. + +A toolless execution path is not an acceptable shortcut, and the +reason is that it is silent. The deploy-time capability walk descends +into every inline body, so an agent's `tool:` reaches the +frozen surface and the operator is required to approve it before the +deploy is accepted. A runtime that then builds that agent without its +tools does not fail: the provider receives an empty tool list, the +step completes, and nothing is logged at any level. The operator made +a security decision that had no effect, and the only evidence is the +work the agent did not do. Two layers answering the same question +differently is the defect, whichever layer is more permissive. + +The corollary for the authorization layer: a tool reaching the +provider is not authority to invoke it. The run's grants still gate +every call, and a spawned body's grants are capped to what the body's +own definition declares. Tools being present is what makes that gate +meaningful — a gate over an empty toolset decides nothing. + +## Walking a definition's steps + +`stepOrder` lists the steps of ONE definition record. A workflow's +executable surface is larger: a `loop` carries an inline body, and an +inline `onTrigger` section or `childWorkflow` carries a full nested +definition. A consumer that answers "which steps run here" off +`stepOrder` alone under-counts every nested body. + +`walkStepTree` is the one traversal every such consumer goes through. It +visits steps in pre-order (a step before the bodies it carries, and a +body's steps before the next sibling), throws on a `stepOrder` entry with +no matching step, and takes the descent as an explicit argument rather +than an implicit house rule. Two named descents cover the cases in use: + +- `EXECUTABLE_STEP_DESCENT` — loop bodies, inline onTrigger bodies, and + inline childWorkflow bodies. This is the setting that yields every step + id the deployment can execute. `executableStepIds(definition)` is the + deduplicated id list under it, and the deploy-time capability walk + descends with it so an operator approves everything a step can run. +- `LOOP_BODY_DESCENT` — loop bodies only. This is the setting that stays + inside one flat step-id namespace: a loop body resolves against the + enclosing definition's map, while an inline section or child body is + lifted to its own definition keyed under its own ref. The deploy's + per-step inference-source pin descends with it. + +`walkWorkflowSteps` and `walkNestedWorkflowSteps` are the live +`WorkflowDefinition` entry points; they read each primitive's nested +bodies through `nestedWorkflowBodies`, whose switch is exhaustive so a +newly-added primitive kind fails at compile time rather than silently +reading as a leaf. `walkStepTree` itself is generic over the step and +tree types, so the inert wire projection — whose step values are +`unknown` and are validated as the caller descends — rides the same +traversal. + ## Consuming a real agent step's structured output Structural selectors (`map.over`, `input.from`, `project`, `merge`) do @@ -91,6 +167,39 @@ A loop body may also spawn a `childWorkflow` grandchild: the child is lifted to a ref and runs as its own child run, depth-counted against the tree-wide spawn ceiling exactly like any other child. +A `childWorkflow` child cannot hold a human gate, and neither can anything +inside it. The relay that makes a loop iteration's pause durable is severed at +the `childWorkflow` boundary: a child run carries no address of its own, and +the spawning step waits for its terminal rather than driving it across parks, +so no decision can be routed back down. A body nested inside the child +inherits that answer, so this holds at any depth rather than only for a gate +written directly in the child. + +An untimed wait beneath that boundary fails the step that asked for it, naming +the gate, rather than waiting on a signal nobody can send. A gate carrying a +`timeout` is unaffected, because its own timer resolves it without anything +upstream. + +Where the spawn step sits at a workflow root, an `onFailure` handler on it +absorbs this failure like any other child failure, so a workflow that routes +around child failures swallows it; leave that step unrouted if you want to see +it. A spawn step in a `loop` body cannot carry `onFailure` at all -- routing is +honored only at a workflow root, and a loop body is not one -- so the failure +surfaces there regardless. + +To hold a human gate, keep the `awaitSignal` in a run the control plane can +address: the deployment's own run, whether directly or in a `loop` body or +`onTrigger` section body of it, whose container relays the decision down. What +no container can do is reach across the `childWorkflow` boundary. + +Of those homes the `onTrigger` section body carries a caveat the relay does not +cause. A section body's run id omits the enclosing run, so a second top-level +run of the same deployment finds the first run's body log already terminal and +short-circuits it: the body never executes, nobody is asked, and the run still +reports a clean terminal status (tracked as INTR-552). A gate a repeating +deployment depends on belongs in the deployment's own run or in a `loop` body, +whose iteration run ids re-root per run. + A loop body may contain a nested `loop`. An inner loop resolves its body ref from the same top-level bodies map (a loop iteration inherits its parent's env), its body-child run ids carry the container run id so iterations stay unique @@ -113,7 +222,9 @@ layer. Practical guidance: use a loop to repeat a self-contained unit -- which may park on an `awaitSignal`, spawn a `childWorkflow`, or run a nested `loop` -- until a -pure `while`/`carry` says stop. Model a `sleep` delay at a top-level step or an +pure `while`/`carry` says stop. Keep the gate in the body itself: an untimed +`awaitSignal` inside a `childWorkflow` the body spawns, at any depth, fails the +step rather than waiting. Model a `sleep` delay at a top-level step or an onTrigger section, not in a loop body. Keep a loop body's action idempotent where practical, since a mid-invocation crash fails the run and the effect is never re-run. diff --git a/vendor/intx/workflow/VENDORED-FROM b/vendor/intx/workflow/VENDORED-FROM index 52b447dce0..7ecaf186ff 100644 --- a/vendor/intx/workflow/VENDORED-FROM +++ b/vendor/intx/workflow/VENDORED-FROM @@ -1,4 +1,4 @@ Source: https://github.com/faremeter/interchange (packages/workflow) -Commit: 692c31068cbb636ebb19c6d179ec6fe126a2d4e6 (origin/main, 2026-09-03) +Commit: 5453d0b01da9d749aa446c4ba34fcf2aeb4549c0 (origin/main, v0.4.0) License: LGPL-2.1-only (see vendor/intx/LICENSE) Local modifications: exports map repointed from the upstream intx-src condition to direct TypeScript source resolution (types/default -> ./src/...); dist references removed. Upstream test files are not vendored (they import the unvendored @intx/inference-testing and @intx/test-harness test packages); only the tests covering a workbench delta are kept. One source delta (corbitsdev/workbench#709): run.ts's per-step timeout is an execution budget — trigger-idle input parks disarm the step timer across the wait and re-arm a fresh budget on resume, while the approval park stays armed (snapshot-bound, still bounded by `step.timeout`); pinned in runtime/run.test.ts (only tests covering a workbench delta are vendored). diff --git a/vendor/intx/workflow/package.json b/vendor/intx/workflow/package.json index dd7e4b2dab..3d8480301a 100644 --- a/vendor/intx/workflow/package.json +++ b/vendor/intx/workflow/package.json @@ -1,7 +1,7 @@ { "name": "@intx/workflow", "description": "Definition surface, state machine, and abstract runtime for multi-step agent workflows", - "version": "0.3.0", + "version": "0.4.0", "license": "LGPL-2.1-only", "type": "module", "exports": { diff --git a/vendor/intx/workflow/src/definition/primitives.ts b/vendor/intx/workflow/src/definition/primitives.ts index 0a39ce9b21..d53738a1cd 100644 --- a/vendor/intx/workflow/src/definition/primitives.ts +++ b/vendor/intx/workflow/src/definition/primitives.ts @@ -128,6 +128,27 @@ export interface GatePrimitive extends PrimitiveBase { else: string; } +/** + * Parks the run until a signal named `name` is delivered, then completes with + * that signal's payload. This is the human gate: the run holds here while a + * decision is made somewhere outside it. + * + * `timeout` (milliseconds) gives the gate a second exit. Such a gate resolves + * on its own timer with nothing delivered, routing to the step `onTimeout` + * names, or failing the step when no route is declared. Because the timer runs + * in the same process as the gate, a timed gate needs nothing from outside the + * run and is valid anywhere. + * + * An untimed gate has the one exit, so it can only end where something outside + * the run can deliver -- a run the control plane can address, or a container + * relaying a decision down. In a run tree with neither, the runtime refuses the + * gate at the park, before any suspension is recorded, rather than waiting on a + * signal that can never arrive. A `childWorkflow` child is the boundary that + * produces such a tree; see {@link ChildWorkflowPrimitive}. + * + * The same refusal covers the parks the runtime takes on a step's behalf, so + * this rule is not confined to a gate an author writes here. + */ export interface AwaitSignalPrimitive extends PrimitiveBase { kind: "awaitSignal"; name: string; @@ -149,6 +170,22 @@ export interface SleepPrimitive extends PrimitiveBase { * body and a `loop` body), so the child's grants fold into the parent's * approved surface and no separately-deployed asset is read. `input` * selects the child run's launch payload. + * + * The child is driven to its terminal, not across parks, and carries no + * address of its own, so nothing upstream can answer a park inside it. Every + * untimed park beneath this boundary -- at any depth, including inside a + * `loop` body in the child -- is refused at the park and fails the child. That + * is an untimed {@link AwaitSignalPrimitive}, and also the parks the runtime + * takes on a step's behalf without the author writing one: an agent step + * suspending on a tool declared `approval: "ask"`, and the input park a step + * with a trigger budget takes between deliveries. A child containing no + * `awaitSignal` is not thereby safe. Give such a gate a `timeout`, or hold it + * in a run the control plane can address. + * + * The refusal surfaces to the parent as an ordinary child failure. Routing it + * with `onFailure` is available only where `onFailure` is honored at all: on a + * direct entry of a workflow root. A spawn inside a `loop` body may not carry + * `onFailure`, so there the failure surfaces unrouted. */ export interface ChildWorkflowPrimitive extends PrimitiveBase { kind: "childWorkflow"; @@ -253,7 +290,11 @@ export type ActionHandler = ( * resume and must be side-effect free. The loop body may park on an * `awaitSignal` and resume, may spawn a `childWorkflow` grandchild, and * may contain a nested `loop` (bounded depth), but may not contain a - * `sleep` or `onTrigger` (all enforced at definition time). + * `sleep` or `onTrigger` (all enforced at definition time). The + * grandchild is a further boundary that definition time does not + * police: an untimed park anywhere beneath it is refused at runtime, so + * what the body itself may hold, the grandchild may not. See + * {@link ChildWorkflowPrimitive}. * * The loop step's own output -- what `steps..output` selects -- is * `{ outcome, iterations, carry, final }`: diff --git a/vendor/intx/workflow/src/runlocal/run-local.ts b/vendor/intx/workflow/src/runlocal/run-local.ts index 731ced2fdb..10f1ce505c 100644 --- a/vendor/intx/workflow/src/runlocal/run-local.ts +++ b/vendor/intx/workflow/src/runlocal/run-local.ts @@ -80,6 +80,16 @@ export interface RunLocalOptions extends RuntimeRunOptions { clock?: () => Date; /** Inject a deterministic id generator for tests. */ newId?: (prefix: string) => string; + /** + * Whether a park in this run tree can be answered from outside it. Required + * rather than defaulted, because an absent value would have to mean + * permissive and a call site that forgot it would silently inherit a park + * nothing can answer. Pass true for a run the control plane can address: a + * top-level local run, whose caller holds the handle that delivers. The + * terminal-child spawner passes false, since nothing can address a child + * run. + */ + hasUpstreamSignalResolver: boolean; } /** @@ -147,6 +157,7 @@ export function runLocal( clock, newId, drain: createNoopDrainController(rewritten), + hasUpstreamSignalResolver: options.hasUpstreamSignalResolver, }; // Wired after construction because the loop-iteration executor closes over // the env it belongs to, so each iteration's body runs under the parent's @@ -272,6 +283,14 @@ export function createDefaultActionInvoker( resolver: ((ref: string) => ActionHandler) | undefined, ): ActionInvoker { return async ({ handler, input, requires, authzContext, signal }) => { + // Refuse before anything is constructed, so a cancelled run resolves no + // handler and touches no ledger. An action is single-attempt with + // observable side effects and there is no retry to reconsider the + // decision, so starting one for a run already known to be cancelled is + // not recoverable downstream. The step invoker refuses the same way. + if (signal.aborted) { + throw abortReason(signal); + } if (!resolver) { throw new Error( `action ${handler} requires an actionResolver; pass one to runLocal({ actionResolver })`, @@ -343,7 +362,7 @@ export function createSpawnLoopIteration( }; } -function createInMemorySpawnChild( +export function createInMemorySpawnChild( bodies: ReadonlyMap, inherited: InheritedChildOptions, ): SpawnChildWorkflow { @@ -355,6 +374,18 @@ function createInMemorySpawnChild( depth, maxChildSpawnDepth, }) => { + // Four awaits separate the child run id allocation from this call, one of + // them a durable flush, so a cancel can land before the spawner runs. The + // abort bridge below subscribes to an edge and would miss one already + // past, leaving the child uncancelled and this function awaiting a + // terminal that never comes. Refusing outright also avoids writing a whole + // child log subtree for a run already known to be cancelled, and matches + // what the deployed spawn adapter does, so a local rehearsal does not + // diverge from production. + if (signal.aborted) { + throw abortReason(signal); + } + const resolved = bodies.get(definitionRef); if (resolved === undefined) { // The runtime dispatched a childWorkflow ref with no lifted definition. @@ -377,6 +408,10 @@ function createInMemorySpawnChild( runId: childRunId, depth, maxChildSpawnDepth, + // Terminal: the parent awaits this child's terminal rather than driving + // it across parks, and nothing upstream can address the child run, so a + // park inside it could never be answered. + hasUpstreamSignalResolver: false, }); const onParentAbort = (): void => { void child.cancel("supervisor-operator", "parent cancelled"); @@ -394,6 +429,16 @@ function createInMemorySpawnChild( }; } +/** + * The error an aborted signal should surface: its own reason when it carries + * one, so a cancel's cause is not replaced by a generic abort. + */ +function abortReason(signal: AbortSignal): Error { + const reason: unknown = signal.reason; + if (reason instanceof Error) return reason; + return new DOMException("aborted", "AbortError"); +} + function defaultClock(): Date { return new Date(); } diff --git a/vendor/intx/workflow/src/runtime/abort-bridge.ts b/vendor/intx/workflow/src/runtime/abort-bridge.ts new file mode 100644 index 0000000000..b1e588a0a3 --- /dev/null +++ b/vendor/intx/workflow/src/runtime/abort-bridge.ts @@ -0,0 +1,54 @@ +// Bridging an outer abort signal into a local reaction. +// +// An `AbortSignal` carries both an edge (the `abort` event) and a level +// (`.aborted`). A bridge built only from the edge misses a signal that already +// fired, and the runtime is full of places where it can have: every one of +// these bridges is built after an await, usually a durable commit or flush, so +// the window between the caller's decision to abort and the bridge existing is +// real rather than theoretical. +// +// The failure is quiet. Nothing throws; the local controller simply never +// aborts. Five bridges were written without the level check. Two left +// something waiting forever: `parkOnSignalResult` awaited a signal nobody +// would send, and the local child spawner left the child uncancelled while +// its parent awaited a terminal that never came. Two let work proceed for a +// cancelled run: `runStep` handed the invoker a signal that could never fire, +// so an agent send spent inference until the step's timeout, and +// `createDefaultActionInvoker` ran a handler whose effect nothing downstream +// can undo. `runAction` is the fifth and joins those two -- the signal it +// builds is what the action invoker's entry refusal reads, so an edge-only +// bridge there hides the abort from the handler and defeats that refusal +// with it. +// +// Consolidated here so the next bridge is a call rather than a fresh chance to +// get it wrong. Four sites stay hand-written and are not this shape: +// `createStepAbort` also consults drain and returns early on it, and +// `waitForTimer`, `createInMemorySpawnChild` and `createDefaultActionInvoker` +// throw rather than react. The spawner's listener is the one outer-abort +// bridge here still built from the edge alone; it is safe because nothing +// awaits between its entry throw and the registration, so the window a level +// check closes elsewhere never opens there. Its exclusion above names that +// entry throw, not the listener. + +/** + * Run `onAbort` when `outer` aborts, including when it already has. + * + * Returns a detach function for callers whose reaction outlives the thing it + * guards. Callers that hold the bridge for the lifetime of the surrounding + * operation can ignore it -- the listener is registered `once`. + */ +export function bridgeAbort( + outer: AbortSignal, + onAbort: () => void, +): () => void { + if (outer.aborted) { + onAbort(); + return () => { + /* nothing was attached */ + }; + } + outer.addEventListener("abort", onAbort, { once: true }); + return () => { + outer.removeEventListener("abort", onAbort); + }; +} diff --git a/vendor/intx/workflow/src/runtime/env.ts b/vendor/intx/workflow/src/runtime/env.ts index 617fc4a254..ee42982c6c 100644 --- a/vendor/intx/workflow/src/runtime/env.ts +++ b/vendor/intx/workflow/src/runtime/env.ts @@ -584,6 +584,26 @@ export interface WorkflowRuntimeEnv { * controller whose signal never fires. */ drain: DrainController; + /** + * Whether a park in this run tree has anything upstream that could resolve + * it. + * + * An untimed park waits on a signal that has to arrive from outside the + * run. That only happens when the run is reachable from the control plane: + * either it is the deployment's addressable run, or it is a suspendable + * child whose container relays a decision back down. A terminal + * `childWorkflow` child is neither -- it has no address of its own and no + * relay above it -- so a park there can never be answered. + * + * The seam that spawns a run is the only layer that knows which case it is + * in, so it states the fact here and the runtime enforces it at the park. + * + * Required rather than optional on purpose. An absent flag would have to + * mean permissive, and a host that forgot to set it would silently inherit + * the unanswerable park this exists to prevent. Requiring it makes that a + * compile error instead. + */ + hasUpstreamSignalResolver: boolean; /** * Optional suspension-notify sink. Fired once each time a step commits a * `SignalAwaited` on a reserved `signalName(correlationId)` channel -- the diff --git a/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts b/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts index 7f4c64a64d..4108fcbce9 100644 --- a/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts +++ b/vendor/intx/workflow/src/runtime/loop-iteration-handle.ts @@ -17,6 +17,13 @@ // - `onPark`/`onSignalPark`, which `createSuspendableChildHandle` owns: the // body's parks feed the handle's queue for the container to proxy up, not // the parent env's supervisor-facing `onPark`. +// One inherited field is nonetheless restated on the child env rather than +// left to the spread: +// - `hasUpstreamSignalResolver`. A loop iteration is answerable exactly when +// its container is, and `parkOnSignalResult` reads that field to refuse an +// untimed gate nothing can answer. Carrying it silently would leave the +// other half of that guard invisible from here, where a future change to +// the spread could drop it without a reader noticing. // Everything else -- including `readParkedApprovalOps`, the child's own // mid-park crash-recovery read hook -- is inherited unchanged. @@ -64,6 +71,9 @@ export function createLoopIterationHandle( ...inherited, signalChannel: args.signalChannel, drain: createNoopDrainController(args.definition), + // Restated rather than left to the spread: an iteration is answerable + // exactly when its container is, and `parkOnSignalResult` refuses on it. + hasUpstreamSignalResolver: baseEnv.hasUpstreamSignalResolver, }; return createSuspendableChildHandle(childEnv, { definition: args.definition, diff --git a/vendor/intx/workflow/src/runtime/run.ts b/vendor/intx/workflow/src/runtime/run.ts index a96f10dc2c..48ab6478ad 100644 --- a/vendor/intx/workflow/src/runtime/run.ts +++ b/vendor/intx/workflow/src/runtime/run.ts @@ -61,6 +61,7 @@ import type { WorkflowRuntimeEnv, } from "./env"; import { shouldAbortForDrain } from "./drain"; +import { bridgeAbort } from "./abort-bridge"; import { assertSpawnDepthWithinLimit, resolveMaxChildSpawnDepth, @@ -194,15 +195,9 @@ export function runtimeRun( // `StepFailed` -> `RunFailed`) rather than cancelling. See // `RuntimeRunOptions.localAbort`. if (options.localAbort !== undefined) { - if (options.localAbort.aborted) { + bridgeAbort(options.localAbort, () => { cancelController.abort(); - } else { - options.localAbort.addEventListener( - "abort", - () => cancelController.abort(), - { once: true }, - ); - } + }); } const completePromise = executeRun( definition, @@ -1488,7 +1483,11 @@ async function runStep( stepStartedEmitted = true; } - // Build per-step abort: timeout AND outer cancellation both abort. + // Build per-step abort: timeout AND outer cancellation both abort. The + // durable commit above is an await, so check the outer signal's level + // before subscribing to its edge; an abort raised during that commit + // would otherwise never reach the invoker, which would then work on + // behalf of a run that is already cancelled. // `step.timeout` is an execution budget: it bounds active agent // invocations, not parked-idle waits. Each input park below disarms // it before waiting and re-arms a fresh budget on resume; the @@ -1500,7 +1499,7 @@ async function runStep( const onOuter = () => { stepAbort.abort(); }; - abort.addEventListener("abort", onOuter, { once: true }); + bridgeAbort(abort, onOuter); let timer: ReturnType | undefined; const armStepTimer = (): void => { if (timer !== undefined) clearTimeout(timer); @@ -1890,6 +1889,12 @@ async function runStep( * runner. The per-effect ledger is a deeper exactly-once line of defense * for effects routed through `perform`; the barrier here is what makes the * action non-re-invocable at the runtime layer. + * + * Cancellation splits along the same line. The runtime owes that a handler + * is never STARTED for a run already known to be cancelled, which the + * invoker enforces by refusing a pre-aborted signal at entry. Stopping once + * started is the author's half: the handler is handed the signal, and the + * runtime cannot make a side effect already in flight transactional. */ async function runAction( definition: WorkflowDefinition, @@ -1931,11 +1936,18 @@ async function runAction( started = await commitDurable(env, runId, startedEvent); void started; + // As in `runStep`: the durable commit above is an await, so check the outer + // signal's level before subscribing to its edge, or an abort raised during + // the commit never becomes observable to the handler at all. + // + // Observable is the whole of what this bridge promises. Whether a handler + // stops once it can see the abort is the handler's own choice, and whether + // the effect is started at all belongs to the invoker one layer down. const actionAbort = new AbortController(); const onOuter = (): void => { actionAbort.abort(); }; - abort.addEventListener("abort", onOuter, { once: true }); + bridgeAbort(abort, onOuter); let timer: ReturnType | undefined; if (primitive.timeout !== undefined) { timer = setTimeout(() => { @@ -2781,11 +2793,20 @@ async function driveContainerSignalRelay( // owned channel -- the same way the body's leaf gate surfaced up to here. // Nesting composes: a loop inside a loop (or inside an onTrigger section) // proxies the park one layer at a time until it reaches the run whose channel - // has a real upstream. At the top level the sink is unset, so this is a no-op - // and the container awaits the real channel directly. Fired on the fresh drive - // only, after the durable relay await above -- the reestablish path re-drives - // `raceContainerSignalRelay` directly and each level re-establishes from its - // own durable await, so it needs no re-fire. + // has a real upstream. An unset sink means this run owns the outermost + // channel, which is true of two different runs: an addressable top-level run, + // whose caller delivers, and a terminal `childWorkflow` child, which nothing + // can address. Only the first has an upstream, so the no-op below is a + // genuine wait in one case and would be an endless one in the other. + // The second case never reaches here. `parkOnSignalResult` is the sole reader + // of `hasUpstreamSignalResolver` and the sole enforcement point: it refuses + // the body's untimed gate at the park, so the body fails instead of relaying + // a name up and this drive is never entered. That is why no guard is + // repeated here -- one invariant enforced in two places drifts apart, and the + // reader that must keep it is the one holding the durable park. + // Fired on the fresh drive only, after the durable relay await above -- the + // reestablish path re-drives `raceContainerSignalRelay` directly and each + // level re-establishes from its own durable await, so it needs no re-fire. if (env.onSignalPark !== undefined) { env.onSignalPark({ runId, name }); } @@ -2828,11 +2849,7 @@ async function raceContainerSignalRelay( const onOuterAbort = (): void => { raceAbort.abort(); }; - if (abort.aborted) { - raceAbort.abort(); - } else { - abort.addEventListener("abort", onOuterAbort, { once: true }); - } + bridgeAbort(abort, onOuterAbort); const pSignal = env.signalChannel .awaitNext(name, raceAbort.signal) .then((r) => ({ tag: "signal" as const, r })); @@ -4299,6 +4316,30 @@ async function parkOnSignalResult( state: ReturnType, abort: AbortSignal, ): Promise { + // An untimed park waits for a signal from outside this run, so it can only + // be answered where something upstream can deliver one. A terminal child has + // no address of its own and no container relaying decisions down to it, so + // the wait would never end: the child parks, the spawner waits on a terminal + // that never comes, and an approval nobody can see holds the whole tree. + // + // A body nested inside such a child inherits the same answer, so this fires + // for a gate at any depth beneath the boundary rather than only a direct + // one. The advice below must therefore not send an author to a loop or + // onTrigger body: that is where they already are. + // + // Refuse before anything durable is written, so the run fails at the step + // that asked for the impossible and no suspension is recorded for a + // correlation the control plane will never hear about. A timed gate is + // exempt -- its own timer resolves it in process, needing nothing upstream. + if (!env.hasUpstreamSignalResolver && opts.timeout === undefined) { + throw new Error( + `step ${opts.stepId} waits on ${opts.signalName} with no timeout, but ` + + `nothing outside this run can deliver it. A childWorkflow child is ` + + `run to its terminal rather than driven across parks, so no gate ` + + `beneath that boundary can be answered, at any depth. Give the gate ` + + `a timeout, or move it into a run the control plane can address`, + ); + } // Re-emit `SignalAwaited` only when the gate is not already awaiting it. // On a re-park resume the gate is already `awaiting-signal` (StepStarted // + SignalAwaited durable), so this is skipped and the tail re-parks on @@ -4478,7 +4519,17 @@ async function parkOnSignalResult( const onOuterAbort = (): void => { combinedAbort.abort(); }; - abort.addEventListener("abort", onOuterAbort, { once: true }); + // The durable flush above is an await, so the outer signal may already have + // aborted by the time this bridge is built. An abort is an edge, not a + // level: a listener attached after the fact never fires, and the park below + // would then wait on a signal nothing will send. + // + // Abort the combined controller rather than throwing, which is what + // `waitForTimer` does from the same position. `awaitNext` consults its + // pre-delivery queue before the abort signal, so a signal that arrived + // before the park is still consumed as durable progress; throwing would + // discard it. + bridgeAbort(abort, onOuterAbort); // Listen for drain transitions that land mid-await. const onDrain = (): void => { if (shouldAbortForDrain(env.drain, opts.stepId)) { diff --git a/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts b/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts index e2f7486f10..5710e6e904 100644 --- a/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts +++ b/vendor/intx/workflow/src/runtime/suspendable-child-handle.ts @@ -22,6 +22,7 @@ import type { SuspendableChildPark, WorkflowRuntimeEnv, } from "./env"; +import { bridgeAbort } from "./abort-bridge"; import { runtimeRun } from "./run"; import type { WorkflowEvent } from "../state-machine/index"; @@ -137,11 +138,7 @@ export function createSuspendableChildHandle( const onParentAbort = (): void => { localTeardown.abort(); }; - if (signal.aborted) { - onParentAbort(); - } else { - signal.addEventListener("abort", onParentAbort, { once: true }); - } + const detachParentAbort = bridgeAbort(signal, onParentAbort); // On resume, drive the run from its durable log; the body step re-parks // silently (a re-park does not re-fire onPark), and the caller relays the @@ -168,7 +165,7 @@ export function createSuspendableChildHandle( failure = cause instanceof Error ? cause : new Error(String(cause)); }) .finally(() => { - signal.removeEventListener("abort", onParentAbort); + detachParentAbort(); if (cleanup !== undefined) void cleanup(); notify(); });