From b1e6caa5d695b74c2023ec24af33b09afb344754 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:15 -0700 Subject: [PATCH 01/48] Add test for workflowDeployBody's catalog-offering shape --- scripts/e2e/harness.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 5b4d14800..37f4f93e4 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -11,6 +11,7 @@ import { createCleanupHarness, parseEnvFileDatabaseUrl, runCleanups, + workflowDeployBody, type SpawnedApp, } from "./harness.ts"; @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => { }); }); +describe("workflowDeployBody", () => { + test("carries catalog offering ids, never a raw provider/apiKey source", () => { + const body = workflowDeployBody({ + assetId: "asset-1", + commitSha: "deadbeef", + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).toEqual({ + source: { + kind: "asset", + assetId: "asset-1", + package: { format: "source", commitSha: "deadbeef" }, + }, + entry: (body as { entry: string }).entry, + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).not.toHaveProperty("sources"); + expect(body).not.toHaveProperty("defaultSource"); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( From fae77c0682222b8149cb08b6b169e1caab79d33b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:23 -0700 Subject: [PATCH 02/48] e2e: deploy workflows by catalog offering ids The native workflow deploy route now requires sourceOfferingIds/ defaultSourceOfferingId resolved against the tenant's own catalog, rather than a raw provider/baseURL/apiKey source chain. Update workflowDeployBody accordingly, and extract the zero-cost noop catalog chain smoke-webhook.test.ts already built (an anthropic- plugin provider pointed at the hub's noop-inference endpoint) into a shared seedNoopCatalogOffering harness helper. Every deploying e2e suite now seeds or resolves a real catalog offering before deploying: walking-skeleton keeps a never-real placeholder host (assertNeverRealProvider still guards it), chat resolves the offering seedCatalog already planted rather than seeding a second chain, and heartbeat/workbench-digest/smoke-webhook seed the noop chain via the shared helper. cl-6324-launch-proof's section-mode deploy now reuses the offering id the proof already resolved when it narrowed the bench catalog to its pinned model. --- scripts/e2e/chat.test.ts | 48 +++++++++-- scripts/e2e/cl-6324-launch-proof.ts | 29 ++++--- scripts/e2e/harness.ts | 121 +++++++++++++++++++++++---- scripts/e2e/heartbeat.test.ts | 19 +++-- scripts/e2e/smoke-webhook.test.ts | 89 +++----------------- scripts/e2e/walking-skeleton.test.ts | 23 +++-- scripts/e2e/workbench-digest.test.ts | 19 +++-- 7 files changed, 219 insertions(+), 129 deletions(-) diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index deea7c826..da9b255c8 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { log: () => undefined, }); + // The echo deploy resolves against the catalog offering `seedCatalog` + // just planted for the same provider/model this workflow declares, + // rather than seeding a second inference chain. + const echoModelsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/models`, + undefined, + user1.cookies, + ); + expectStatus("list catalog models", echoModelsListed, 200); + const echoModelRows = ( + echoModelsListed.data as { data: { id: string; canonicalName: string }[] } + ).data; + const echoModelId = echoModelRows.find( + (row) => row.canonicalName === "claude-sonnet-5", + )?.id; + if (echoModelId === undefined) { + throw new Error( + `no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`, + ); + } + + const echoOfferingsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings`, + undefined, + user1.cookies, + ); + expectStatus("list catalog offerings", echoOfferingsListed, 200); + const echoOfferingRows = ( + echoOfferingsListed.data as { data: { id: string; modelId: string }[] } + ).data; + const echoOfferingId = echoOfferingRows.find( + (row) => row.modelId === echoModelId, + )?.id; + if (echoOfferingId === undefined) { + throw new Error( + `no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`, + ); + } + // Seed the echo workflow as a deployed, invitable definition: the // same asset-publish → git-token → smart-HTTP push → native deploy // path `scripts/e2e/walking-skeleton.test.ts` proves end to end. @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { workflowDeployBody({ assetId: echoAssetId, commitSha: echoPushed.commitSha, - sourceId: "src-echo-e2e", - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [echoOfferingId], + defaultSourceOfferingId: echoOfferingId, }), user1.cookies, ); diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index 674315090..ef3be6297 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,7 +369,7 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - await hop("narrow the bench catalog to the pinned model", async () => { + const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { await plantGrant("model-offering:*", "read"); await plantGrant("model-offering:*", "manage"); await plantGrant("model:*", "read"); @@ -409,8 +409,13 @@ async function main(): Promise { "data", "model offerings", ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; for (const offering of offeringRows) { - if (offering.modelId === pinned.id || offering.disabled) continue; + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; const patched = await api( hub.baseUrl, "PATCH", @@ -420,10 +425,17 @@ async function main(): Promise { ); expectStatus(`disable offering ${offering.id}`, patched, 200); } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } console.log( ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); + return pinnedOffering.id; }); const assistantDefinitionId = await hop( @@ -694,7 +706,6 @@ async function main(): Promise { // own durable event log, which is the artefact the milestone's // `RunStarted` assertion was always about. const SECTION_ASSET_NAME = "cl6324-section"; - const SECTION_SOURCE_ID = "cl6324-section-source"; const SECTION_TURN_TIMEOUT_MS = 180_000; const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`; @@ -780,16 +791,8 @@ async function main(): Promise { package: { format: "source", commitSha: pushed.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SECTION_SOURCE_ID, - provider: model.provider, - baseURL: model.baseURL, - apiKey: model.apiKey, - model: model.model, - }, - ], - defaultSource: SECTION_SOURCE_ID, + sourceOfferingIds: [pinnedOfferingId], + defaultSourceOfferingId: pinnedOfferingId, }, user.cookies, ); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 099a7a210..27cd39204 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -653,16 +653,17 @@ export async function pushWorkflowSource(options: { /** * The deploy body a code-sourced asset deployment takes: the pushed * commit is the definition's pin, and the entry names the - * `interchange.workflow` module the sidecar evaluates. + * `interchange.workflow` module the sidecar evaluates. The native route + * resolves inference against the tenant's own catalog, so the caller + * supplies the ordered catalog offering ids to deploy against rather + * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` + * for the zero-cost way to obtain one. */ export function workflowDeployBody(options: { assetId: string; commitSha: string; - sourceId: string; - provider: string; - baseURL: string; - apiKey: string; - model: string; + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; }): Record { return { source: { @@ -671,15 +672,103 @@ export function workflowDeployBody(options: { package: { format: "source", commitSha: options.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: options.sourceId, - provider: options.provider, - baseURL: options.baseURL, - apiKey: options.apiKey, - model: options.model, - }, - ], - defaultSource: options.sourceId, + sourceOfferingIds: options.sourceOfferingIds, + defaultSourceOfferingId: options.defaultSourceOfferingId, }; } + +/** + * Plants the zero-cost catalog chain every deploying e2e suite needs: a + * catalog model, an `anthropic`-plugin provider/credential pointed at + * `noopBaseUrl` (never a real provider host — enforced below), and the + * catalog offering joining them. Returns the offering id a deploy's + * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. + * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + */ +export async function seedNoopCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + noopBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringFieldOf(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringFieldOf(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringFieldOf( + credential.data, + "id", + "create credential", + ); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.noopBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringFieldOf( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; +} + +function stringFieldOf(data: unknown, field: string, what: string): string { + if (typeof data === "object" && data !== null && field in data) { + const value = (data as Record)[field]; + if (typeof value === "string" && value !== "") return value; + } + throw new Error( + `${what}: missing string field "${field}": ${JSON.stringify(data)}`, + ); +} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index c19a6b6a2..9aefb900f 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -203,16 +204,22 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-heartbeat-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 1d248defa..760aca6a3 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -30,7 +30,6 @@ import { } from "../../workflows/heartbeat/src/index.ts"; import { api, - assertNeverRealProvider, connectE2eDb, createCleanupHarness, e2eDatabaseUrl, @@ -39,6 +38,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -138,77 +138,17 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real - // model. Mirrors `workbench-digest.test.ts`'s noop catalog seeding. + // model. const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - assertNeverRealProvider(noopBaseUrl, "noop catalog provider baseURL"); - await hop("noop catalog seeding", async () => { - const model = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringField(model.data, "id", "create catalog model"); - - const provider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringField(provider.data, "id", "create provider"); - - const credential = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringField( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: noopBaseUrl, - credentialId, - }, + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies2) => + api(hub.baseUrl, method, path, body, cookies2), + tenantId, cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringField( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - }); + noopBaseUrl, + }), + ); const assetName = "heartbeat"; const { assetId, commitSha } = await hop( @@ -256,16 +196,11 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); const definitionId = await hop("workflow deploy", async () => { - const sourceId = "src-smoke-webhook-e2e"; - assertNeverRealProvider(noopBaseUrl, "workflow deploy source baseURL"); const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: noopBaseUrl, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; for (;;) { diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 833612dd6..8937f9b5e 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -35,6 +35,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -199,20 +200,30 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { }, ); + // Hop: noop catalog seeding. The native deploy resolves inference + // against a real catalog offering; this suite's chosen offering + // still names an unreachable placeholder host, since deployment + // never calls inference and full run-completion is not asserted. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: "https://inference.invalid", + }), + ); + // Hop: workflow deploy via the native deploy API. Retries while // the hub still answers 502 (the sidecar's dial-in may not have // completed yet); any other failure is final. The inference // source is a placeholder — deployment does not call inference. const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-echo-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index ecf2dc56e..8917bf823 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -207,16 +208,22 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-workbench-digest-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; From b03ce16b54f9de7e1bb8f14a7efce6a50cd7f9ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:43:45 -0700 Subject: [PATCH 03/48] Fix formatting --- scripts/e2e/cl-6324-launch-proof.ts | 125 ++++++++++++++-------------- scripts/e2e/harness.ts | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index ef3be6297..80766c99d 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,74 +369,77 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { - await plantGrant("model-offering:*", "read"); - await plantGrant("model-offering:*", "manage"); - await plantGrant("model:*", "read"); + const pinnedOfferingId = await hop( + "narrow the bench catalog to the pinned model", + async () => { + await plantGrant("model-offering:*", "read"); + await plantGrant("model-offering:*", "manage"); + await plantGrant("model:*", "read"); - const models = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench catalog models", models, 200); - const modelRows = arrayField(models.data, "data", "catalog models") as { - id: string; - canonicalName: string; - }[]; - const pinned = modelRows.find( - (row) => row.canonicalName === proofModelSource.model, - ); - if (pinned === undefined) { - throw new Error( - `the bench catalog carries no model named ${proofModelSource.model}; ` + - `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + const models = await api( + hub.baseUrl, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, + undefined, + user.cookies, ); - } - - const offerings = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench model offerings", offerings, 200); - const offeringRows = arrayField( - offerings.data, - "data", - "model offerings", - ) as { id: string; modelId: string; disabled: boolean }[]; - let pinnedOffering: { id: string; modelId: string } | undefined; - for (const offering of offeringRows) { - if (offering.modelId === pinned.id) { - pinnedOffering = offering; - continue; + expectStatus("list the bench catalog models", models, 200); + const modelRows = arrayField(models.data, "data", "catalog models") as { + id: string; + canonicalName: string; + }[]; + const pinned = modelRows.find( + (row) => row.canonicalName === proofModelSource.model, + ); + if (pinned === undefined) { + throw new Error( + `the bench catalog carries no model named ${proofModelSource.model}; ` + + `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + ); } - if (offering.disabled) continue; - const patched = await api( + + const offerings = await api( hub.baseUrl, - "PATCH", - `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, - { disabled: true }, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, + undefined, user.cookies, ); - expectStatus(`disable offering ${offering.id}`, patched, 200); - } - if (pinnedOffering === undefined) { - throw new Error( - `no catalog offering resolves the pinned model ${proofModelSource.model} ` + - `(id ${pinned.id})`, + expectStatus("list the bench model offerings", offerings, 200); + const offeringRows = arrayField( + offerings.data, + "data", + "model offerings", + ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; + for (const offering of offeringRows) { + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; + const patched = await api( + hub.baseUrl, + "PATCH", + `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, + { disabled: true }, + user.cookies, + ); + expectStatus(`disable offering ${offering.id}`, patched, 200); + } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } + console.log( + ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); - } - console.log( - ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + - `(${String(offeringRows.length - 1)} other offerings disabled)`, - ); - return pinnedOffering.id; - }); + return pinnedOffering.id; + }, + ); const assistantDefinitionId = await hop( "PROOF 1 — 'assistant' is invitable tenant-wide", diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 27cd39204..cebc4e53d 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -760,7 +760,9 @@ export async function seedNoopCatalogOffering(options: { cookies, ); expectStatus("create catalog offering", offering, 201); - return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; + return { + offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), + }; } function stringFieldOf(data: unknown, field: string, what: string): string { From f981798f2aad5db7e0a8c14a0234e8e1860c4cbe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:53:52 -0700 Subject: [PATCH 04/48] Add tests for deploying default workflows by catalog offering ids seedTenant currently deploys every default workflow with a bare provider/baseURL/apiKey source, but the hub's deploy route now requires sourceOfferingIds/defaultSourceOfferingId resolved from the tenant's catalog. Update the seeding, onboarding, and hub config test suites to expect the new offering-id based deploy body and provider/model-only ModelSource shape ahead of the implementation change. --- apps/hub/test/config.test.ts | 1 - .../test/complete-credential.test.ts | 55 +++++++-- packages/onboarding/test/provision.test.ts | 101 +++++++++++++++++ .../test/routine-launchability.test.ts | 2 - packages/seeding/test/seed.test.ts | 106 +++++++++++++++--- 5 files changed, 237 insertions(+), 28 deletions(-) diff --git a/apps/hub/test/config.test.ts b/apps/hub/test/config.test.ts index 9d3435117..b4d41908e 100644 --- a/apps/hub/test/config.test.ts +++ b/apps/hub/test/config.test.ts @@ -277,7 +277,6 @@ describe("readHubConfig", () => { expect(config.seedModel).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", apiKey: "sk-ant-test", }); }); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c9ac97c2d..8a4ae6a78 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -63,6 +63,36 @@ function seedHandshake(method: string, path: string) { return { ...handshake, cookies: [] }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461). A test that +// never seeds its own catalog (it stubs `seedCatalogFn` instead) still +// needs one listable offering for the "assistant" workflow's own deploy +// to succeed — this fixed single row is that. +function fixedOfferingsResponse() { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function principalsResponse() { return { status: 200, @@ -165,8 +195,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-ant", }); }); @@ -192,8 +220,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "llama3.2", - baseURL: "http://localhost:11434/v1", - apiKey: "ollama", }); }); @@ -339,7 +365,12 @@ describe("modelSourceFor", () => { expect(result.model).toBe("gpt-oss:20b"); }); - test("ollama's baseURLOverride is normalized to the /v1 form", async () => { + // `modelSourceFor` resolves a provider/model pair only — the deploy + // itself resolves inference from the tenant's own catalog offerings + // (CL-7461), so a `baseURLOverride` here is accepted (every caller + // already resolved one for `seedCatalog`'s earlier catalog plant) but + // never affects this resolution. + test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { return resolvedCatalogResponse([ @@ -365,8 +396,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "qwen3.8:27b", - baseURL: "https://home-mac.example.ts.net/v1", - apiKey: "ollama", }); }); }); @@ -808,6 +837,12 @@ describe("completeCredentialSetup", () => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}`) { return tenantResponse(); } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) @@ -1791,6 +1826,12 @@ describe("ensureSeeded (the slow half)", () => { let deploymentCreatePosts = 0; const api: ApiCall = async (method, path, body) => { + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index af1594cbc..b1feabe08 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -87,6 +87,41 @@ function collector() { return { lines, log: (line: string) => lines.push(line) }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461); every test +// that seeds a real default workflow needs at least one listable, since +// none of these tests seed a catalog of its own. +function catalogOfferingsResponse( + method: string, + path: string, + tenantId: string, +): { status: number; data: unknown; cookies: string[] } | undefined { + if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + return undefined; + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function firstLoginSeedHub(args: { expectedParentId?: string }) { let principalsCalls = 0; const startedRuns: string[] = []; @@ -206,6 +241,33 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { ) { return { status: 200, data: [], cookies: [] }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/workflows/definitions`) @@ -638,6 +700,33 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if (method === "GET" && path === "/api/me/principals") { return membership(); } @@ -890,6 +979,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1019,6 +1110,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1133,6 +1226,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1243,6 +1338,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1485,6 +1582,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1832,6 +1931,8 @@ describe("provisionPersonalTenantIfNeeded", () => { [], ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, diff --git a/packages/seeding/test/routine-launchability.test.ts b/packages/seeding/test/routine-launchability.test.ts index 427fa0b00..746e0e202 100644 --- a/packages/seeding/test/routine-launchability.test.ts +++ b/packages/seeding/test/routine-launchability.test.ts @@ -27,8 +27,6 @@ import { const FAKE_MODEL: ModelSource = { provider: "ollama", model: "qwen-test", - baseURL: "http://localhost:11434", - apiKey: "test-key", }; type SerializedStepDefinition = { diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index b6291fc94..720250165 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -38,10 +38,38 @@ installDisposableHubDataDir(); const MODEL = { provider: "anthropic", model: "claude-sonnet-4-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-test", }; +// The tenant's one real catalog offering — every `seedTenant` test that +// deploys a real (non-noop-pinned) workflow needs at least one of these +// listable, since `ensureDeployment` now resolves its deploy source from +// the tenant's catalog rather than a bare `ModelSource`. +const OFFERING_ID = "off_1"; + +function offeringsPage( + ids: readonly string[] = [OFFERING_ID], +): { status: number; data: unknown } { + return { + status: 200, + data: { + data: ids.map((id, index) => ({ + id, + tenantId: TENANT_ID, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + })), + nextCursor: null, + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -98,6 +126,40 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/workflows/deployments` ) return { status: 200, data: [] }; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return offeringsPage(); + // The noop catalog chain `ensureNoopCatalogOffering` plants for a + // modelSource-pinned workflow (heartbeat) — every seedTenant test that + // deploys one needs this fresh chain to succeed, so it lives in the + // shared fallback rather than every such test repeating it. + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/models`) + return { status: 201, data: catalogModelRow("mdl_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_noop", "prv_noop", "noop-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_noop", "noop", "cre_noop", "anthropic"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 201, + data: catalogOfferingRow("off_noop", "mdl_noop", "cpv_noop"), + }; const handshake = pristineScheduledDefinitionHandshake(method, path); if (handshake) return handshake; return undefined; @@ -860,11 +922,9 @@ describe("seedTenant", () => { expect(assistant?.displayName).toBe("Myra"); }); - test("NOOP_MODEL_SOURCE resolves to the hub's own noop-inference endpoint", () => { - const resolved = NOOP_MODEL_SOURCE("http://localhost:3000"); - expect(resolved.baseURL).toBe( - "http://localhost:3000/api/chat/noop-inference", - ); + test("NOOP_MODEL_SOURCE names the noop provider/model pair", () => { + const resolved = NOOP_MODEL_SOURCE(); + expect(resolved.provider).toBe("anthropic"); expect(resolved.model).toBe("noop"); }); @@ -934,8 +994,8 @@ describe("seedTenant", () => { (w) => w.assetName === "heartbeat", ); if (!heartbeat) throw new Error("expected the heartbeat workflow"); - const resolved = heartbeat.modelSource?.("http://localhost:3000"); - expect(resolved).toEqual(NOOP_MODEL_SOURCE("http://localhost:3000")); + const resolved = heartbeat.modelSource?.(); + expect(resolved).toEqual(NOOP_MODEL_SOURCE()); }); test("fresh run pushes, deploys, and confirms the heartbeat workflow against the noop source", async () => { @@ -1012,12 +1072,17 @@ describe("seedTenant", () => { expect(definition.triggers[0]?.to).toBe(`heartbeat@${TENANT_DOMAIN}`); expect(definition.stepOrder).toEqual(["heartbeat"]); - // The deploy's own source, not the tenant's real MODEL, is what - // proves the noop pin took effect: it must name the noop provider - // fixture, not the ordinary anthropic/claude-sonnet-4-5 model this - // test file's `args()` helper hands every other workflow. - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).toBe("noop"); + // The deploy's own source offering, not the tenant's real catalog + // offering, is what proves the noop pin took effect: it must name + // the dedicated noop offering `ensureNoopCatalogOffering` plants, + // never the tenant's own `OFFERING_ID` this test file's `baseRoutes` + // fixture hands every real workflow. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual(["off_noop"]); + expect(deployedBody.defaultSourceOfferingId).toBe("off_noop"); const output = lines.join("\n"); expect(output).toContain("deployed workflow heartbeat as dep_3"); @@ -1135,9 +1200,14 @@ describe("seedTenant", () => { ]); expect(definition.stepOrder).toEqual(["workbench-digest"]); - // Defaults deploy against the tenant's real model (not noop). - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).not.toBe("noop"); + // Defaults deploy against the tenant's real catalog offering, never + // the noop one. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual([OFFERING_ID]); + expect(deployedBody.defaultSourceOfferingId).toBe(OFFERING_ID); const output = lines.join("\n"); expect(output).toContain("deployed workflow workbench-digest as dep_4"); From 1ce35c4fa2202730eabe41578a95978cff05c8cf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:00 -0700 Subject: [PATCH 05/48] Seeding: deploy default workflows by catalog offering ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureDeployment posted a bare {provider, baseURL, apiKey, model} source, but the hub's deploy route requires sourceOfferingIds and a defaultSourceOfferingId resolved from the tenant's catalog, so seeding a connect flow's "assistant" workflow 400'd. seedTenant now resolves the tenant's real catalog offerings (sorted by priority, lowest wins the default) for a normal workflow, and plants a dedicated noop catalog offering for a modelSource-pinned one (heartbeat), pointed at the hub's own noop-inference endpoint. ModelSource shrinks to a bare provider/model pair — the baseURL/apiKey it used to carry only mattered for the old deploy body, and callers that still need a key for seeding a catalog (system-seed's root tenant, config's env-key auto-plant) keep it on their own local type instead. --- apps/hub/src/config.ts | 8 +- apps/hub/src/system-seed.ts | 23 +- .../onboarding/src/complete-credential.ts | 51 +-- packages/seeding/src/seed.ts | 316 +++++++++++++++--- 4 files changed, 305 insertions(+), 93 deletions(-) diff --git a/apps/hub/src/config.ts b/apps/hub/src/config.ts index d9fdad334..1d3afcf79 100644 --- a/apps/hub/src/config.ts +++ b/apps/hub/src/config.ts @@ -272,7 +272,6 @@ function parsePositiveMsEnv( const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; // Matches boot-time seeding's own defaults (`system-seed.ts`) exactly, // so a zero-.env-edit local checkout that seeds its admin account @@ -282,10 +281,14 @@ const DEFAULT_PLANT_ADMIN_EMAIL = "alice@example.com"; const DEFAULT_PLANT_ADMIN_PASSWORD = "password123"; const DEFAULT_PLANT_ORG_SLUG = "workbench"; +/** The root tenant's boot-time seed model: a provider/model pair + * `seedTenant` names in every deployed definition, plus the real (or + * placeholder) key `seedCatalog` plants a launchable credential with. No + * `baseURL` — a workflow deploy resolves inference from the tenant's + * catalog offerings, never a bare source tuple (CL-7461). */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; readonly apiKey: string; }; @@ -624,7 +627,6 @@ function seedModelFrom(parsed: ParsedHubEnv): ModelSource | undefined { return { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey, }; } diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 7f9291fc5..52f0ff8e6 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -26,7 +26,6 @@ import { publishCorbitsToolsRegistry, DEFAULT_WORKFLOWS, PLACEHOLDER_CATALOG_API_KEY, - type ModelSource, } from "@corbits/seeding"; import { createHubAPI, signIn, type ApiCall } from "@corbits/hub-api-client"; import { findPersonalTenant } from "@workbench/onboarding"; @@ -43,14 +42,26 @@ const DEFAULT_POLL_INTERVAL_MS = 2_000; // and the catalog is still browsable, just not launchable. const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; -function resolvedModel(seedModel: ModelSource | undefined): ModelSource { +/** + * The root tenant's seed model: a provider/model pair for + * `seedTenant`'s deployed definitions, plus the real (or placeholder) + * API key `seedCatalog` needs to plant a launchable credential. Distinct + * from `@corbits/seeding`'s `ModelSource`, which carries no key — + * `seedTenant` never needs one; the deploy itself resolves inference + * from the tenant's catalog offerings (CL-7461). + */ +export type SeedModelConfig = { + readonly provider: string; + readonly model: string; + readonly apiKey: string; +}; + +function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey: PLACEHOLDER_CATALOG_API_KEY, } ); @@ -60,7 +71,7 @@ export type SystemSeedDeps = { baseUrl: string; orgSlug: string; admin: { email: string; password: string }; - seedModel?: ModelSource; + seedModel?: SeedModelConfig; deadlineMs?: number; pollIntervalMs?: number; }; @@ -116,7 +127,7 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { principalId: tenant.principalId, domain: tenant.tenantDomain, }, - model, + model: { provider: model.provider, model: model.model }, pushWorkflow, log: (line) => log.info`${line}`, workflows: DEFAULT_WORKFLOWS, diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index 5af80e3f7..f5f4a930a 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -60,10 +60,7 @@ import { TenantResponse, paginatedSchema, } from "@intx/types"; -import { - ollamaOpenAICompatBaseURL, - type SupportedCredentialProvider, -} from "@corbits/connections/credential-test"; +import { type SupportedCredentialProvider } from "@corbits/connections/credential-test"; import { preferCompletionCapable } from "@corbits/connections/model-capability"; import { CATALOG_SEEDS, @@ -280,13 +277,8 @@ async function resolveOllamaModelSource( api: ApiCall, cookies: string[], tenantId: string, - apiKey: string, - baseURLOverride: string | undefined, ): Promise { const catalogSeed = CATALOG_SEEDS.ollama; - const baseURL = ollamaOpenAICompatBaseURL( - baseURLOverride ?? catalogSeed.provider.baseURL, - ); const response = await api( "GET", @@ -368,39 +360,32 @@ async function resolveOllamaModelSource( return { provider: winner.plugin, model: winner.canonicalName, - baseURL, - apiKey, }; } -/** The `ModelSource` `ensureSeeded` deploys every default workflow - * against. Every provider but `ollama` has a fixed, always-available - * curated model list (`CATALOG_SEEDS`), so its curated default is a safe - * pin. `ollama` is the one provider whose actual model list is - * per-instance and can diverge from that curated name entirely - * (CL-6366) — its resolution defers to `resolveOllamaModelSource`, which - * reads back what `seedCatalog` actually found on the instance rather - * than repeating the static pin. `baseURLOverride` is the - * configurable-base-URL seam every provider but `ollama` ignores (a - * fixed origin); for `ollama` it is the root the person actually pointed - * their instance at, normalized to the OpenAI-compatible `/v1` form this - * deploys against. */ +/** The `ModelSource` `ensureSeeded` names in every deployed definition's + * inference preferences — never what the deploy itself resolves + * inference against (the tenant's catalog offerings, CL-7461). Every + * provider but `ollama` has a fixed, always-available curated model list + * (`CATALOG_SEEDS`), so its curated default is a safe pin. `ollama` is + * the one provider whose actual model list is per-instance and can + * diverge from that curated name entirely (CL-6366) — its resolution + * defers to `resolveOllamaModelSource`, which reads back what + * `seedCatalog` actually found on the instance rather than repeating the + * static pin. `apiKey`/`baseURLOverride` are accepted only because every + * caller already resolved them for `seedCatalog`'s own credential/catalog + * plant, which runs ahead of this in the same connect flow; this + * function itself needs neither. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - apiKey: string, - baseURLOverride?: string, + _apiKey: string, + _baseURLOverride?: string, ): Promise { if (provider === "ollama") { - return resolveOllamaModelSource( - api, - cookies, - tenantId, - apiKey, - baseURLOverride, - ); + return resolveOllamaModelSource(api, cookies, tenantId); } const catalogSeed = CATALOG_SEEDS[provider]; @@ -413,8 +398,6 @@ export async function modelSourceFor( return { provider: catalogSeed.provider.plugin, model: defaultModel.canonicalName, - baseURL: catalogSeed.provider.baseURL, - apiKey, }; } diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 2b34e98c3..20e350303 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -141,11 +141,6 @@ const DILIGENCE_BRIEF_TURN_TIMEOUT_MS = 2 * 60 * 1000; const RUN_START_TIMEOUT_MS = 30_000; const RUN_POLL_INTERVAL_MS = 1000; -// The deploy source the per-step agents launch against. The id is the -// routing key `defaultSource` must name; with exactly one source there -// is exactly one honest value for it. -const SEED_SOURCE_ID = "default"; - // The provider/model pair `noop-inference` (packages/chat/src/noop-inference.ts) // answers for any request, regardless of what is actually sent — the // route ignores its body and `x-api-key` entirely. Naming a distinct @@ -154,24 +149,31 @@ const SEED_SOURCE_ID = "default"; // UI and logs. const NOOP_PROVIDER = "anthropic"; const NOOP_MODEL = "noop"; +// The catalog provider/credential name a noop-pinned deployment's own +// offering is planted under — distinct from any real curated provider +// name (`CATALOG_SEEDS`'s keys), so `ensureNoopCatalogOffering` never +// collides with a tenant's real catalog and is trivially recognizable +// in the hub's own catalog UI. +const NOOP_CATALOG_PROVIDER_NAME = "noop"; +// Priority is meaningless for this offering — it is never included in +// a real workflow's `sourceOfferingIds` (`ensureNoopCatalogOffering`'s +// id is threaded through explicitly, never discovered by sorting), so +// any fixed value is honest here. +const NOOP_OFFERING_PRIORITY = 0; /** - * A `ModelSource` pointed at the hub's own `noop-inference` endpoint + * The provider/model pair naming the hub's own `noop-inference` endpoint * instead of a real provider — the same substitution * `packages/chat/src/platform-adapter.ts`'s `noopSourcesOverride` makes - * for workbench-host launches, reused here so a workflow deployed with - * this source resolves every turn instantly against a constant, - * locally served reply and never reaches a real model. `hubUrl` is the - * same base URL `seedTenant` already receives, so no new configuration - * is required to use it. + * for workbench-host launches, reused here so a workflow's rendered + * definition names this pair in its inference preferences. The actual + * deploy source is a catalog offering pinned at this same endpoint + * (`ensureNoopCatalogOffering`), not this pair directly — a workflow + * deployment resolves inference from catalog offering ids, never a bare + * provider/model/baseURL/apiKey tuple (CL-7461). */ -export function NOOP_MODEL_SOURCE(hubUrl: string): ModelSource { - return { - provider: NOOP_PROVIDER, - model: NOOP_MODEL, - baseURL: `${hubUrl}/api/chat/noop-inference`, - apiKey: "noop", - }; +export function NOOP_MODEL_SOURCE(): ModelSource { + return { provider: NOOP_PROVIDER, model: NOOP_MODEL }; } const GitTokenMintResponse = type({ id: "string", secret: "string" }); @@ -191,11 +193,18 @@ const WorkflowRunTriggerResponse = type({ messageId: "string", }); +/** + * A provider/model pair a deployed workflow's rendered definition names + * in its inference preferences (`DefaultWorkflow.buildJson`'s second + * argument). What a deployment actually resolves inference against is + * the tenant's catalog offerings, resolved separately by `ensureDeployment` + * (or, for a noop-pinned workflow, `ensureNoopCatalogOffering`) — this + * type carries no `baseURL`/`apiKey` because `seedTenant` never needs + * either (CL-7461). + */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; - readonly apiKey: string; }; /** @@ -240,9 +249,9 @@ export type DefaultWorkflow = { * Renders the definition's JSON given the tenant's mail domain and the * ordered provider/model preferences to deploy against. Takes the bare * preference list — never a full `ModelSource` — so this same - * function serves both `seedTenant`'s HTTP-deploy path (which also - * needs a `ModelSource`'s `baseURL`/`apiKey` for the deployment's - * `sources`, resolved separately) and a native in-process deploy path + * function serves both `seedTenant`'s HTTP-deploy path (whose actual + * deploy source is a set of catalog offering ids, resolved separately + * by `ensureDeployment`) and a native in-process deploy path * (`apps/hub/src/templates/block-workflows.ts`) that only ever has the * tenant's real, possibly multi-entry inference preferences on hand. */ @@ -251,15 +260,16 @@ export type DefaultWorkflow = { inferencePreferences: readonly InferencePreference[], ) => string; /** - * Overrides the deploy's inference source for this workflow only, - * given the hub's own base URL. Present on the catalog-test workflow - * `heartbeat`, which must stay free to run continuously: it names - * `NOOP_MODEL_SOURCE` instead of the tenant's real catalog model. - * Absent on every conversational workflow and on the seeded - * workbench-digest automation, which deploy against the tenant's real - * model. + * Overrides the deploy's inference source for this workflow only. + * Present on the catalog-test workflow `heartbeat`, which must stay + * free to run continuously: it names `NOOP_MODEL_SOURCE` instead of + * the tenant's real catalog model, and `seedTenant` deploys it against + * a dedicated noop catalog offering (`ensureNoopCatalogOffering`) + * rather than the tenant's own resolved offerings. Absent on every + * conversational workflow and on the seeded workbench-digest + * automation, which deploy against the tenant's real model. */ - modelSource?: (hubUrl: string) => ModelSource; + modelSource?: () => ModelSource; /** * When true, PUT the authored definition to `stopped` after deploy so * a native ScheduleTrigger does not fire every tenant at the next @@ -901,7 +911,8 @@ async function ensureDeployment( assetId: string; assetName: string; commitSha: string; - model: ModelSource; + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; }, log: (line: string) => void, ): Promise { @@ -950,16 +961,8 @@ async function ensureDeployment( package: { format: "source", commitSha: args.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SEED_SOURCE_ID, - provider: args.model.provider, - baseURL: args.model.baseURL, - apiKey: args.model.apiKey, - model: args.model.model, - }, - ], - defaultSource: SEED_SOURCE_ID, + sourceOfferingIds: args.sourceOfferingIds, + defaultSourceOfferingId: args.defaultSourceOfferingId, }, cookies, ); @@ -984,6 +987,170 @@ async function ensureDeployment( return deployment.id; } +/** + * Lists every catalog offering owned directly by a tenant, paging past + * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list + * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" + * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s + * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s + * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog + * directly on that same tenant, so the owned list already carries + * everything a plain `workbench seed` or the self-served connect flow + * ever seeded, with no HTTP surface exposing the resolved view to a + * caller outside the hub process. + */ +async function listTenantCatalogOfferings( + api: ApiCall, + cookies: string[], + tenantId: string, +): Promise<(typeof ModelOfferingResponse.infer)[]> { + const items: (typeof ModelOfferingResponse.infer)[] = []; + let cursor: string | undefined; + for (;;) { + const query = + cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings${query}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + items.push(...page.data); + if (page.nextCursor === null) return items; + if (items.length > 10_000) { + throw new HubApiError( + `catalog offerings list for tenant ${tenantId} did not terminate while paging`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + cursor = page.nextCursor; + } +} + +/** + * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow + * deploys against: every enabled offering the tenant owns, ordered by + * priority ascending, with the lowest-priority offering as the default — + * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and + * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply + * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the + * noop offering (`ensureNoopCatalogOffering`) from this list when one + * has already been planted on the same tenant this run — it must never + * win a real workflow's default just because it happens to sort first. + * + * Throws when nothing is left to deploy against: "seeded but not + * launchable" (`seedCatalog` with no credential) is a valid catalog + * state, but a workflow deploy that names zero sources is not a + * lesser success, it is the failure this function exists to catch + * before the hub's own `sourceOfferingIds` validation would. + */ +async function resolveRealSourceOfferingIds( + api: ApiCall, + cookies: string[], + tenantId: string, + excludeOfferingId: string | undefined, +): Promise<{ + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; +}> { + const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) + .filter( + (offering) => + !offering.disabled && offering.id !== excludeOfferingId, + ) + .sort((a, b) => a.priority - b.priority); + const defaultSourceOfferingId = offerings[0]?.id; + if (defaultSourceOfferingId === undefined) { + throw new HubApiError( + "this tenant has no catalog offerings to deploy against — nothing is launchable yet", + "seed the tenant's catalog with a real provider key (or a placeholder credential for a keyless dev/CI run), then re-run: workbench seed", + ); + } + return { + sourceOfferingIds: offerings.map((offering) => offering.id), + defaultSourceOfferingId, + }; +} + +/** + * Plants (or finds) the dedicated catalog offering a noop-pinned + * workflow (`DefaultWorkflow.modelSource`, e.g. `heartbeat`) deploys + * against: a catalog model named `NOOP_MODEL`, a provider pointed at the + * hub's own `noop-inference` endpoint, and a placeholder credential — + * the same `ensureCatalogModel` / `ensureProvider` / `ensureCredential` / + * `ensureCatalogProvider` / `ensureCatalogOffering` sequence + * `seedCatalog` runs for a real provider, run here for this one + * synthetic one. Idempotent, same as every other seed step: a re-run + * finds the existing rows by name and reuses them. + */ +async function ensureNoopCatalogOffering( + api: ApiCall, + cookies: string[], + tenantId: string, + hubUrl: string, + log: (line: string) => void, +): Promise { + const baseURL = `${hubUrl}/api/chat/noop-inference`; + const modelId = await ensureCatalogModel( + api, + cookies, + { tenantId, canonicalName: NOOP_MODEL }, + log, + ); + const providerId = await ensureProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + apiBaseUrl: baseURL, + }, + log, + ); + const credentialId = await ensureCredential( + api, + cookies, + { + tenantId, + providerId, + name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), + secret: PLACEHOLDER_CATALOG_API_KEY, + type: "api_key", + }, + log, + ); + const catalogProviderId = await ensureCatalogProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + baseURL, + credentialId, + }, + log, + ); + return ensureCatalogOffering( + api, + cookies, + { + tenantId, + modelId, + providerId: catalogProviderId, + priority: NOOP_OFFERING_PRIORITY, + capabilities: [], + }, + log, + ); +} + /** * After deploy, PUT a pristine authored definition to `stopped` so a native * ScheduleTrigger does not fire every tenant. A member-restored row @@ -1160,6 +1327,14 @@ export type SeedTenantArgs = { cookies: string[]; hubUrl: string; tenant: SeedTenant; + /** + * The provider/model pair a real (non-noop-pinned) workflow's rendered + * definition names in its inference preferences. The deploy itself + * resolves inference from the tenant's own catalog offerings + * (`resolveRealSourceOfferingIds`), not from this pair — a tenant with + * no catalog offerings still fails clearly at deploy time even if this + * pair happens to name a real model. + */ model: ModelSource; pushWorkflow: WorkflowPusher; log: (line: string) => void; @@ -1228,9 +1403,41 @@ export async function seedTenant(args: SeedTenantArgs): Promise { await plantDefaultSkills(api, cookies, tenant.tenantId, log); + // Resolved lazily and cached across the loop below: most seed runs + // deploy several workflows against the same tenant catalog state, so + // this fetches the tenant's real offerings (or plants the noop one) + // at most once each, however many workflows ask for them. + let noopOfferingId: string | undefined; + let realOfferings: + | { sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string } + | undefined; + let confirmed = 0; for (const workflow of workflows) { - const workflowModel = workflow.modelSource?.(hubUrl) ?? model; + const workflowModel = workflow.modelSource?.() ?? model; + + let sourceOfferingIds: readonly string[]; + let defaultSourceOfferingId: string; + if (workflow.modelSource !== undefined) { + noopOfferingId ??= await ensureNoopCatalogOffering( + api, + cookies, + tenant.tenantId, + hubUrl, + log, + ); + sourceOfferingIds = [noopOfferingId]; + defaultSourceOfferingId = noopOfferingId; + } else { + realOfferings ??= await resolveRealSourceOfferingIds( + api, + cookies, + tenant.tenantId, + noopOfferingId, + ); + ({ sourceOfferingIds, defaultSourceOfferingId } = realOfferings); + } + const assetId = await ensureWorkflowAsset( api, cookies, @@ -1265,7 +1472,8 @@ export async function seedTenant(args: SeedTenantArgs): Promise { assetId, assetName: workflow.assetName, commitSha: pushed.commitSha, - model: workflowModel, + sourceOfferingIds, + defaultSourceOfferingId, }, log, ); @@ -1666,7 +1874,7 @@ async function ensureCatalogOffering( quirks?: Record; }, log: (line: string) => void, -): Promise { +): Promise { const body: Record = { modelId: args.modelId, providerId: args.providerId, @@ -1681,9 +1889,13 @@ async function ensureCatalogOffering( cookies, ); if (created.status === 201) { - parseAs(ModelOfferingResponse, created.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + created.data, + "catalog offering response", + ); log("created catalog offering"); - return; + return offering.id; } if (created.status === 409) { let cursor: string | null = null; @@ -1715,7 +1927,7 @@ async function ensureCatalogOffering( } if (existing.priority === args.priority) { log("catalog offering already exists (skipped)"); - return; + return existing.id; } const updated = await api( @@ -1730,9 +1942,13 @@ async function ensureCatalogOffering( "check the hub logs for the underlying failure, then re-run: workbench seed", ); } - parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + updated.data, + "catalog offering response", + ); log("updated catalog offering priority"); - return; + return offering.id; } throw new HubApiError( `the hub rejected creation of the catalog offering with status ${created.status}: ${JSON.stringify(created.data)}`, From 2587cd2b5af3ecbb27bb13bcb652910bd10b29aa Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:46 -0700 Subject: [PATCH 06/48] longevity-sim: deploy workflows by catalog offering ids Another workflowDeployBody caller CI's typecheck job caught: this stack seeds one catalog offering per real inference target already, so track each one's id and pass it through to the deploy body instead of the raw provider/baseURL/apiKey shape. --- packages/longevity-sim/src/stack.ts | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/longevity-sim/src/stack.ts b/packages/longevity-sim/src/stack.ts index a7dc91776..59c4ff72a 100644 --- a/packages/longevity-sim/src/stack.ts +++ b/packages/longevity-sim/src/stack.ts @@ -437,7 +437,9 @@ export async function bootLongevityStack( // credential row still requires some string, so each target's own // `apiKey` (expected to be the same placeholder) is threaded through // unchanged rather than this package inventing its own convention. - async function seedCatalogChain(target: InferenceTarget): Promise { + async function seedCatalogChain( + target: InferenceTarget, + ): Promise<{ offeringId: string }> { const model = await api( hub.baseUrl, "POST", @@ -509,10 +511,15 @@ export async function bootLongevityStack( owner.cookies, ); expectStatus(`create catalog offering ${target.model}`, offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; } + const offeringIdByTargetLabel = new Map(); for (const target of options.realTargets) { - await seedCatalogChain(target); + const { offeringId } = await seedCatalogChain(target); + offeringIdByTargetLabel.set(target.label, offeringId); } for (const skill of options.skills ?? []) { @@ -668,6 +675,13 @@ export async function bootLongevityStack( workflowJson, }); + const offeringId = offeringIdByTargetLabel.get(target.label); + if (offeringId === undefined) { + throw new Error( + `agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`, + ); + } + const deployDeadline = Date.now() + 90_000; for (;;) { if (sidecar.exited()) { @@ -682,11 +696,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId, commitSha: pushed.commitSha, - sourceId: `src-agent-${spec.handle}`, - provider: target.provider, - baseURL: target.baseURL, - apiKey: target.apiKey, - model: target.model, + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }), owner.cookies, ); @@ -847,6 +858,12 @@ export async function bootLongevityStack( if (routineTarget === undefined) { throw new Error("unreachable: realTargets checked non-empty above"); } + const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label); + if (routineOfferingId === undefined) { + throw new Error( + `no catalog offering seeded for realTarget "${routineTarget.label}"`, + ); + } const assetName = "longevity-heartbeat"; const heartbeatAssetCreated = await api( hub.baseUrl, @@ -912,11 +929,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId: heartbeatAssetId, commitSha: heartbeatPushed.commitSha, - sourceId: "src-longevity-heartbeat", - provider: "openai-compatible", - baseURL: routineTarget.baseURL, - apiKey: routineTarget.apiKey, - model: routineTarget.model, + sourceOfferingIds: [routineOfferingId], + defaultSourceOfferingId: routineOfferingId, }), owner.cookies, ); From 2430449e41b11a77a87cc902e4e2f7c264736125 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:56:16 -0700 Subject: [PATCH 07/48] Fix formatting: prettier --- apps/hub/src/system-seed.ts | 4 +++- packages/seeding/src/seed.ts | 3 +-- packages/seeding/test/seed.test.ts | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 52f0ff8e6..1637f2e89 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -57,7 +57,9 @@ export type SeedModelConfig = { readonly apiKey: string; }; -function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { +function resolvedModel( + seedModel: SeedModelConfig | undefined, +): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 20e350303..24fc1ce6e 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -1060,8 +1060,7 @@ async function resolveRealSourceOfferingIds( }> { const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) .filter( - (offering) => - !offering.disabled && offering.id !== excludeOfferingId, + (offering) => !offering.disabled && offering.id !== excludeOfferingId, ) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 720250165..99f972804 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -46,9 +46,10 @@ const MODEL = { // the tenant's catalog rather than a bare `ModelSource`. const OFFERING_ID = "off_1"; -function offeringsPage( - ids: readonly string[] = [OFFERING_ID], -): { status: number; data: unknown } { +function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { return { status: 200, data: { From 602ad68f72898a1342ec88f5090eac80999493a4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:25 -0700 Subject: [PATCH 08/48] sandbox-sidecar: add tests for generation-0 ensure/destroy requests The allocation service's initial sidecar_allocation and workflow_probe rows start at generation 0 (@intx/db enforces generation >= 0, not > 0), so the first ensure() for a freshly allocated sidecar must be accepted at generation 0. These tests cover that acceptance, later generations, non-integer/negative rejection, and that stale-generation fencing still rejects an older generation once a newer one has been observed. --- .../sandbox-sidecar/src/provisioner.test.ts | 154 ++++++++++++++++++ .../sandbox-sidecar/src/state-store.test.ts | 10 ++ 2 files changed, 164 insertions(+) create mode 100644 packages/sandbox-sidecar/src/provisioner.test.ts diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts new file mode 100644 index 000000000..753478bf0 --- /dev/null +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { createSidecarProvisioner } from "./provisioner"; +import type { SidecarBackend } from "./backend"; + +const TOKEN = "s3cr3t-bootstrap-token"; + +let dataDir: string; +let store: AllocationStateStore; +let started: string[]; + +beforeEach(async () => { + dataDir = await mkdtemp(join(tmpdir(), "sandbox-sidecar-provisioner-")); + store = createAllocationStateStore(join(dataDir, "state.json")); + started = []; +}); + +afterEach(async () => { + await rm(dataDir, { recursive: true, force: true }); +}); + +function fakeBackend(): SidecarBackend { + let counter = 0; + return { + async startUnit() { + counter += 1; + const externalRef = `unit-${String(counter)}`; + started.push(externalRef); + return externalRef; + }, + async stopUnit() {}, + async findUnitsByAllocation() { + return []; + }, + }; +} + +function baseEnsureRequest(overrides: Partial> = {}) { + return { + allocationId: "alloc-1", + generation: 0, + tenantId: "tenant-1", + anchorRunId: "run-1", + sidecarId: "sidecar-1", + token: TOKEN, + hubWebSocketUrl: "wss://hub.example.com/ws", + ...overrides, + } as never; +} + +function makeProvisioner(backend: SidecarBackend = fakeBackend()) { + return createSidecarProvisioner({ + id: "test-provisioner", + apiVersion: 1, + bindingFingerprint: "test", + capabilities: [], + backend, + store, + }); +} + +describe("ensure request validation", () => { + test("accepts generation 0, the allocation service's initial generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); + }); + + test("accepts a later generation", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: -1 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("rejects a non-integer generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1.5 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("still rejects a stale generation once fencing has observed a later one", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toMatchObject({ + kind: "rejected", + code: "stale_generation", + }); + expect(started).toHaveLength(1); + }); +}); + +describe("destroy request validation", () => { + test("accepts generation 0", async () => { + const provisioner = makeProvisioner(); + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + + expect(result).toEqual({ kind: "destroyed" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: -1, + }); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_destroy_request", + }); + }); +}); diff --git a/packages/sandbox-sidecar/src/state-store.test.ts b/packages/sandbox-sidecar/src/state-store.test.ts index d9bdadffb..abd723afd 100644 --- a/packages/sandbox-sidecar/src/state-store.test.ts +++ b/packages/sandbox-sidecar/src/state-store.test.ts @@ -30,6 +30,16 @@ describe("observeEnsure", () => { expect(result.kind).toBe("observed"); }); + test("accepts generation 0, the allocation service's initial generation", async () => { + const store = createAllocationStateStore(statePath()); + const result = await store.observeEnsure({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + expect(result.kind).toBe("observed"); + }); + test("is idempotent when observed again at the same generation", async () => { const store = createAllocationStateStore(statePath()); await store.observeEnsure({ From e01fedad96a1991c9b05eb612462fb18bd654db4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:30 -0700 Subject: [PATCH 09/48] sandbox-sidecar: accept the allocation service's initial generation @intx/db's sidecar_allocation and workflow_probe tables both check generation >= 0 and the allocation service records the first row for an allocation at generation 0, but validateEnsureRequest() and validateDestroyRequest() in packages/sandbox-sidecar rejected anything that wasn't strictly positive, and the on-disk fencing schema in state-store.ts required generation > 0. The first ensure() for any newly allocated sidecar was rejected with 'generation must be a positive integer', failing every deploy. Accept generation >= 0 in both validators and the state-store schema; stale-generation fencing in observeEnsure()/observeDestroy() already compares against the stored generation rather than assuming positivity, so it is unaffected. --- packages/sandbox-sidecar/src/provisioner.ts | 8 ++++---- packages/sandbox-sidecar/src/state-store.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.ts b/packages/sandbox-sidecar/src/provisioner.ts index 209d9e725..9771be319 100644 --- a/packages/sandbox-sidecar/src/provisioner.ts +++ b/packages/sandbox-sidecar/src/provisioner.ts @@ -218,8 +218,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { if (request.token === "") return "token must not be empty"; if (request.hubWebSocketUrl === "") return "hubWebSocketUrl must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } @@ -227,8 +227,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { function validateDestroyRequest(request: DestroySidecarRequest): string | null { if (request.allocationId === "") return "allocationId must not be empty"; if (request.sidecarId === "") return "sidecarId must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } diff --git a/packages/sandbox-sidecar/src/state-store.ts b/packages/sandbox-sidecar/src/state-store.ts index 12e5bed2a..9e600e228 100644 --- a/packages/sandbox-sidecar/src/state-store.ts +++ b/packages/sandbox-sidecar/src/state-store.ts @@ -12,7 +12,7 @@ import { type } from "arktype"; const AllocationRecord = type({ allocationId: "string > 0", sidecarId: "string > 0", - generation: "number.integer > 0", + generation: "number.integer >= 0", desiredState: "'ensured' | 'destroyed'", externalRef: "string | null", tokenHashSha256: "string | null", From c6da1fa7ba9e2607098f1e566456e8ebc8b84b9b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:37:36 -0700 Subject: [PATCH 10/48] sandbox-sidecar: fix formatting in provisioner test --- .../sandbox-sidecar/src/provisioner.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts index 753478bf0..b858c03cc 100644 --- a/packages/sandbox-sidecar/src/provisioner.test.ts +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -3,7 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { + createAllocationStateStore, + type AllocationStateStore, +} from "./state-store"; import { createSidecarProvisioner } from "./provisioner"; import type { SidecarBackend } from "./backend"; @@ -67,7 +70,9 @@ describe("ensure request validation", () => { test("accepts generation 0, the allocation service's initial generation", async () => { const provisioner = makeProvisioner(); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); }); @@ -76,7 +81,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 0 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); }); @@ -113,7 +120,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 1 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toMatchObject({ kind: "rejected", From 824c2b69f0c53998d64e5bebcb051c27b7d3ca4c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:42 -0700 Subject: [PATCH 11/48] Add tests for resolved catalog offerings and modelSourceFor's trimmed signature Covers the new GET .../catalog/resolved-offerings route (denied grant, and both directly-owned and ancestor-inherited offerings sorted by priority) and updates seeding/onboarding test fixtures to the route's response shape ahead of the implementation change. --- .../test/resolved-offerings-routes.test.ts | 99 +++++++++++++++++++ .../test/complete-credential.test.ts | 94 ++++++------------ packages/onboarding/test/provision.test.ts | 48 +++------ packages/seeding/test/seed.test.ts | 23 +++++ 4 files changed, 166 insertions(+), 98 deletions(-) create mode 100644 packages/inference-catalog/test/resolved-offerings-routes.test.ts diff --git a/packages/inference-catalog/test/resolved-offerings-routes.test.ts b/packages/inference-catalog/test/resolved-offerings-routes.test.ts new file mode 100644 index 000000000..897e05313 --- /dev/null +++ b/packages/inference-catalog/test/resolved-offerings-routes.test.ts @@ -0,0 +1,99 @@ +// The resolved-offerings route exists so an out-of-process caller +// (`workbench seed`, CL-7461) can deploy against exactly what the hub's +// own workflow deployer would deploy against — including offerings a +// tenant only inherits from an ancestor, which the tenant-owned +// `GET .../catalog/offerings` route never lists. +import { Hono } from "hono"; +import { describe, expect, test } from "bun:test"; + +import type { TenantEnv } from "@intx/hub-api"; +import type { ResolvedOffering } from "@intx/db"; + +import { createResolvedOfferingsRoutes } from "../src/resolved-offerings-routes"; +import { offering } from "./fixtures"; + +const TENANT = { + id: "bench-1", + name: "Bench One", + slug: "bench-one", + domain: "bench-one.example", + parentId: null, + config: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), +}; + +function app( + listOfferings: (tenantId: string) => Promise, + allow = true, +) { + const host = new Hono(); + host.use("*", async (c, next) => { + c.set("tenant", TENANT); + await next(); + }); + host.route( + "/", + createResolvedOfferingsRoutes({ + listOfferings, + requireGrant: () => async (c, next) => { + if (!allow) { + return c.json({ error: { code: "forbidden", message: "no" } }, 403); + } + return next(); + }, + }), + ); + return host; +} + +describe("GET resolved-offerings", () => { + test("a denied grant is rejected before the offerings are read", async () => { + const response = await app(async () => { + throw new Error("must not be called when the grant is denied"); + }, false).request("/"); + expect(response.status).toBe(403); + }); + + test("lists both directly-owned and ancestor-inherited offerings, sorted by priority", async () => { + const owned = offering({ + id: "owned", + canonicalName: "thrifty", + providerName: "globex", + capabilities: ["plain-text"], + priority: 2, + }); + const inherited = offering({ + id: "inherited", + canonicalName: "lavish", + providerName: "acme", + capabilities: ["plain-text"], + priority: 1, + inherited: true, + }); + + const response = await app(async (tenantId) => { + expect(tenantId).toBe(TENANT.id); + return [owned, inherited]; + }).request("/"); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + offerings: { + id: string; + priority: number; + modelId: string; + providerId: string; + origin: { tenantId: string; direct: boolean }; + }[]; + }; + expect(body.offerings.map((o) => o.id)).toEqual(["inherited", "owned"]); + const inheritedEntry = body.offerings.find((o) => o.id === "inherited"); + expect(inheritedEntry?.origin).toEqual({ + tenantId: "parent-bench", + direct: false, + }); + const ownedEntry = body.offerings.find((o) => o.id === "owned"); + expect(ownedEntry?.origin).toEqual({ tenantId: TENANT.id, direct: true }); + }); +}); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 8a4ae6a78..16e81d0df 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -72,22 +72,15 @@ function fixedOfferingsResponse() { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -179,19 +172,12 @@ function ownedCatalogModelsResponse(canonicalNames: string[]) { } describe("modelSourceFor", () => { - test("every other provider ignores a baseURLOverride and never calls the hub", async () => { + test("every other provider resolves to its curated default and never calls the hub", async () => { const api: ApiCall = (async () => { throw new Error("must not be called for a fixed curated provider"); }) as ApiCall; expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "anthropic", - "sk-ant", - "https://ignored", - ), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "anthropic"), ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", @@ -216,7 +202,7 @@ describe("modelSourceFor", () => { }; expect( - await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama", "ollama"), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama"), ).toEqual({ provider: "openai-compatible", model: "llama3.2", @@ -250,7 +236,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -285,7 +270,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); @@ -323,7 +307,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -360,44 +343,9 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); - - // `modelSourceFor` resolves a provider/model pair only — the deploy - // itself resolves inference from the tenant's own catalog offerings - // (CL-7461), so a `baseURLOverride` here is accepted (every caller - // already resolved one for `seedCatalog`'s earlier catalog plant) but - // never affects this resolution. - test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { - const api: ApiCall = async (method, path) => { - if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { - return resolvedCatalogResponse([ - { - canonicalName: "qwen3.8:27b", - providerName: "ollama", - capabilities: ["plain-text"], - }, - ]); - } - throw new Error(`unexpected call: ${method} ${path}`); - }; - - expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "ollama", - "ollama", - "https://home-mac.example.ts.net", - ), - ).toEqual({ - provider: "openai-compatible", - model: "qwen3.8:27b", - }); - }); }); describe("completeCredentialSetup", () => { @@ -780,12 +728,11 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, - data: { data: [], nextCursor: null }, + data: { offerings: [] }, cookies: [], }; } @@ -839,7 +786,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } @@ -1422,8 +1369,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { return { status: 200, @@ -1446,6 +1392,24 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) { + return { + status: 200, + data: { + offerings: catalogOfferings.map((o) => ({ + id: o.id, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -1828,7 +1792,7 @@ describe("ensureSeeded (the slow half)", () => { const api: ApiCall = async (method, path, body) => { if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index b1feabe08..aff387d48 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -96,27 +96,23 @@ function catalogOfferingsResponse( path: string, tenantId: string, ): { status: number; data: unknown; cookies: string[] } | undefined { - if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + if ( + method !== "GET" || + path !== `/api/tenants/${tenantId}/catalog/resolved-offerings` + ) return undefined; return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -243,27 +239,20 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -702,27 +691,20 @@ describe("provisionPersonalTenantIfNeeded", () => { if (registry !== undefined) return registry; if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 99f972804..d1bea7bcd 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -71,6 +71,24 @@ function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { }; } +function resolvedOfferingsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { + return { + status: 200, + data: { + offerings: ids.map((id, index) => ({ + id, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -132,6 +150,11 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) return offeringsPage(); + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) + return resolvedOfferingsPage(); // The noop catalog chain `ensureNoopCatalogOffering` plants for a // modelSource-pinned workflow (heartbeat) — every seedTenant test that // deploys one needs this fresh chain to succeed, so it lives in the From ec8755106e08316e1a4a12c57f8121849be6fcb6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:49 -0700 Subject: [PATCH 12/48] Seed deploys against a tenant's resolved catalog offerings, not just owned ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveRealSourceOfferingIds listed only offerings a tenant owns directly, missing anything inherited from an ancestor tenant — the hub's own workflow deployer already resolves that wider, ancestor-inheriting view via listVisibleOfferings. Adds GET /api/tenants/:tenantId/catalog/resolved-offerings (mounted from apps/hub, implemented in @corbits/inference-catalog so the deployer and this route share one source of truth) and points seed at it instead. Also drops the two long-dead unused parameters from onboarding's modelSourceFor, and exports ensureNoopCatalogOffering from @corbits/seeding for the e2e harness to reuse. --- apps/hub/src/index.ts | 16 ++++ packages/inference-catalog/src/index.ts | 4 + .../src/resolved-offerings-routes.ts | 44 ++++++++++ .../onboarding/src/complete-credential.ts | 9 +- packages/seeding/src/index.ts | 1 + packages/seeding/src/seed.ts | 84 +++++++++---------- 6 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 packages/inference-catalog/src/resolved-offerings-routes.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 39d52b853..687bcde64 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -150,6 +150,7 @@ import { applyInferenceCatalogMigrations, createBenchModelPolicyRoutes, createPostgresBenchModelPolicyStore, + createResolvedOfferingsRoutes, createWorkflowCatalogRoutes, } from "@corbits/inference-catalog"; import { generateId } from "@intx/hub-common"; @@ -1877,6 +1878,21 @@ export async function createHub(config: HubConfig) { }), }), ); + // Resolved catalog offerings: the same ancestor-inheriting view + // `listVisibleOfferings` gives `workflowDeployer` above, exposed over + // HTTP so an out-of-process deployer (`workbench seed`) can deploy + // against exactly what the hub itself would deploy against, not just + // the offerings a tenant owns directly. + app.route( + `${TENANT_PREFIX}/catalog/resolved-offerings`, + createResolvedOfferingsRoutes({ + listOfferings: (tenantId) => listVisibleOfferings(db, tenantId), + requireGrant: createRequireGrant({ + grantStore: chatGrantStore, + conditionRegistry: chatConditionRegistry, + }), + }), + ); // Bench purpose/type: benches are Interchange tenants, so this is a // package-owned side-table keyed by tenant id, migrated at hub start // like insights and preferences. diff --git a/packages/inference-catalog/src/index.ts b/packages/inference-catalog/src/index.ts index b7e90138a..01af47a3e 100644 --- a/packages/inference-catalog/src/index.ts +++ b/packages/inference-catalog/src/index.ts @@ -63,6 +63,10 @@ export { createBenchModelPolicyRoutes, type CreateBenchModelPolicyRoutesDeps, } from "./routes"; +export { + createResolvedOfferingsRoutes, + type CreateResolvedOfferingsRoutesDeps, +} from "./resolved-offerings-routes"; export { createWorkflowCatalogRoutes, estimateUsd, diff --git a/packages/inference-catalog/src/resolved-offerings-routes.ts b/packages/inference-catalog/src/resolved-offerings-routes.ts new file mode 100644 index 000000000..ec1cfae2f --- /dev/null +++ b/packages/inference-catalog/src/resolved-offerings-routes.ts @@ -0,0 +1,44 @@ +// The tenant-owned `GET .../catalog/offerings` route only ever lists rows a +// tenant planted directly on itself; it has no way to show offerings a +// tenant inherits from an ancestor tenant. `listVisibleOfferings` +// (`@intx/db`) already computes that resolved, inheritance-aware view for +// the hub's own workflow deployer — this route exposes the same view over +// HTTP so an out-of-process caller (`workbench seed`, in particular) can +// deploy against exactly what the hub itself would deploy against. +// +// Takes a `listOfferings` port rather than a `db` handle, same as +// `workflow-catalog-routes.ts`, so the route is testable with literals. +import { Hono } from "hono"; +import type { ResolvedOffering } from "@intx/db"; +import type { RequireGrant, TenantEnv } from "@intx/hub-api"; + +export type CreateResolvedOfferingsRoutesDeps = { + readonly listOfferings: ( + tenantId: string, + ) => Promise; + readonly requireGrant: RequireGrant; +}; + +export function createResolvedOfferingsRoutes( + deps: CreateResolvedOfferingsRoutesDeps, +): Hono { + const app = new Hono(); + + app.get("/", deps.requireGrant("model-offering:*", "read"), async (c) => { + const tenant = c.get("tenant"); + const resolved = [...(await deps.listOfferings(tenant.id))].sort( + (a, b) => a.offering.priority - b.offering.priority, + ); + return c.json({ + offerings: resolved.map((entry) => ({ + id: entry.offering.id, + priority: entry.offering.priority, + modelId: entry.offering.modelId, + providerId: entry.offering.providerId, + origin: entry.origin, + })), + }); + }); + + return app; +} diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index f5f4a930a..c292bc897 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -372,17 +372,12 @@ async function resolveOllamaModelSource( * diverge from that curated name entirely (CL-6366) — its resolution * defers to `resolveOllamaModelSource`, which reads back what * `seedCatalog` actually found on the instance rather than repeating the - * static pin. `apiKey`/`baseURLOverride` are accepted only because every - * caller already resolved them for `seedCatalog`'s own credential/catalog - * plant, which runs ahead of this in the same connect flow; this - * function itself needs neither. */ + * static pin. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - _apiKey: string, - _baseURLOverride?: string, ): Promise { if (provider === "ollama") { return resolveOllamaModelSource(api, cookies, tenantId); @@ -496,8 +491,6 @@ export async function ensureSeeded( args.cookies, args.tenant.tenantId, args.provider, - args.apiKey, - args.baseURLOverride, ), pushWorkflow: args.pushWorkflow, log: args.log, diff --git a/packages/seeding/src/index.ts b/packages/seeding/src/index.ts index 78506cd4f..e67e47a58 100644 --- a/packages/seeding/src/index.ts +++ b/packages/seeding/src/index.ts @@ -26,6 +26,7 @@ export { inferenceCredentialName, PLACEHOLDER_CATALOG_API_KEY, ensureCredential, + ensureNoopCatalogOffering, ensureProvider, reconcileSeedGrants, seedCatalog, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 24fc1ce6e..21ca42794 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -987,55 +987,51 @@ async function ensureDeployment( return deployment.id; } +const ResolvedOfferingResponse = type({ + id: "string", + priority: "number", + modelId: "string", + providerId: "string", + origin: { tenantId: "string", direct: "boolean" }, +}); + +const ResolvedOfferingsResponse = type({ + offerings: ResolvedOfferingResponse.array(), +}); + /** - * Lists every catalog offering owned directly by a tenant, paging past - * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list - * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" - * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s - * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s - * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog - * directly on that same tenant, so the owned list already carries - * everything a plain `workbench seed` or the self-served connect flow - * ever seeded, with no HTTP surface exposing the resolved view to a - * caller outside the hub process. + * Lists the offerings visible to a tenant, including any inherited from + * an ancestor tenant — the same resolved view `listVisibleOfferings` + * (`@intx/db`) gives the hub's own workflow deployer + * (`apps/hub/src/index.ts`'s `workflowDeployer`), exposed at + * `GET .../catalog/resolved-offerings`. Seeding a tenant whose catalog is + * inherited rather than directly owned (e.g. a sub-tenant under a + * parent that already carries a real provider key) needs this resolved + * list, not the tenant-owned-only `GET .../catalog/offerings`. */ -async function listTenantCatalogOfferings( +async function listResolvedCatalogOfferings( api: ApiCall, cookies: string[], tenantId: string, -): Promise<(typeof ModelOfferingResponse.infer)[]> { - const items: (typeof ModelOfferingResponse.infer)[] = []; - let cursor: string | undefined; - for (;;) { - const query = - cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; - const listed = await api( - "GET", - `/api/tenants/${tenantId}/catalog/offerings${query}`, - undefined, - cookies, - ); - const page = parseAs( - paginatedSchema(ModelOfferingResponse), - listed.data, - "catalog offerings response", - ); - items.push(...page.data); - if (page.nextCursor === null) return items; - if (items.length > 10_000) { - throw new HubApiError( - `catalog offerings list for tenant ${tenantId} did not terminate while paging`, - "check the hub logs for the underlying failure, then re-run: workbench seed", - ); - } - cursor = page.nextCursor; - } +): Promise<(typeof ResolvedOfferingResponse.infer)[]> { + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/resolved-offerings`, + undefined, + cookies, + ); + return parseAs( + ResolvedOfferingsResponse, + listed.data, + "resolved catalog offerings response", + ).offerings; } /** * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow - * deploys against: every enabled offering the tenant owns, ordered by - * priority ascending, with the lowest-priority offering as the default — + * deploys against: every offering visible to the tenant (owned or + * inherited from an ancestor), ordered by priority ascending, with the + * lowest-priority offering as the default — * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the @@ -1058,10 +1054,8 @@ async function resolveRealSourceOfferingIds( sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string; }> { - const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) - .filter( - (offering) => !offering.disabled && offering.id !== excludeOfferingId, - ) + const offerings = (await listResolvedCatalogOfferings(api, cookies, tenantId)) + .filter((offering) => offering.id !== excludeOfferingId) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; if (defaultSourceOfferingId === undefined) { @@ -1087,7 +1081,7 @@ async function resolveRealSourceOfferingIds( * synthetic one. Idempotent, same as every other seed step: a re-run * finds the existing rows by name and reuses them. */ -async function ensureNoopCatalogOffering( +export async function ensureNoopCatalogOffering( api: ApiCall, cookies: string[], tenantId: string, From bcb8703e02e1a669a5d16651045a604993aeaf2e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:05 -0700 Subject: [PATCH 13/48] Add a regression test pinning the shared-sidecar auth gap (CL-7472) At the interchange pin this branch stacks on (692c3106), the hub authenticates a sidecar's register/reconnect frame only against a sidecar row that resolves to a live sidecar_allocation (or workflow_probe) row. Upstream dropped the older "shared" credential scope this repo's e2e harness used to provision a single, standalone sidecar identity for, so a provisionSidecar-only row now connects and is rejected with "invalid token" in a loop rather than ever dialing in. This test pins that negative so a future re-vendor that silently restores (or relies on) the old scope surfaces here first. --- scripts/e2e/harness.test.ts | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 37f4f93e4..1a4f3fce8 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -6,10 +6,14 @@ import { describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; +import path from "node:path"; +import { resetSchema, setupDatabase } from "../db-setup.ts"; import { createCleanupHarness, + e2eDatabaseUrl, parseEnvFileDatabaseUrl, + provisionSidecar, runCleanups, workflowDeployBody, type SpawnedApp, @@ -127,3 +131,71 @@ describe("parseEnvFileDatabaseUrl", () => { ).toBe("postgres://second/db"); }); }); + +// CL-7472 regression: at pin 692c3106, the hub authenticates a +// sidecar's register/reconnect frame only against a `sidecar` row that +// resolves to a live `sidecar_allocation` (or `workflow_probe`) row — +// upstream dropped the older "shared" credential scope this repo used +// to provision a single, standalone sidecar identity for (see +// `provisionSidecar`'s own doc comment). CI's e2e suites hit exactly +// this: a `provisionSidecar`-only row connects and is rejected with +// "invalid token" in a loop (never a working dial-in), which is why +// every suite now lets the hub's own process-provisioner spawn a +// dedicated, already-allocated sidecar per deployment instead (see +// `startHub`'s doc comment). This test pins the negative: an +// unallocated identity must keep failing to authenticate, so a future +// upstream re-vendor that silently restores (or a caller that +// silently starts relying on) the old "shared" scope surfaces here +// first rather than as a mystery disconnect loop in CI. +describe("provisionSidecar", () => { + const databaseUrl = e2eDatabaseUrl(); + const HUB_DIR = path.resolve(import.meta.dir, "..", "..", "apps", "hub"); + + test.skipIf(databaseUrl === undefined)( + "a provisioned identity with no allocation or probe never authenticates", + async () => { + const url = databaseUrl; + if (url === undefined) throw new Error("unreachable: suite is skipped"); + + await resetSchema(url); + await setupDatabase(url); + + const sidecarId = "harness-test-unallocated-sidecar"; + const token = crypto.randomUUID(); + await provisionSidecar(url, sidecarId, token); + + // Resolved through the hub's own dependency tree, exactly as + // `connectE2eDb` resolves `postgres` above — scripts/e2e is not + // itself a workspace member of these `@intx/*` packages. + const dbModule = (await import( + Bun.resolveSync("@intx/db", HUB_DIR) + )) as { createDB: (raw: unknown) => { db: unknown; close: () => Promise } }; + const hubSessionsModule = (await import( + Bun.resolveSync("@intx/hub-sessions", HUB_DIR) + )) as { + createSidecarCredentialResolver: (deps: { db: unknown }) => { + resolve: (token: string) => Promise; + }; + }; + + const parsed = new URL(url); + const db = dbModule.createDB({ + host: parsed.hostname, + port: parsed.port === "" ? 5432 : Number(parsed.port), + user: decodeURIComponent(parsed.username), + password: decodeURIComponent(parsed.password), + database: parsed.pathname.replace(/^\//, ""), + }); + try { + const resolver = hubSessionsModule.createSidecarCredentialResolver({ + db: db.db, + }); + const identity = await resolver.resolve(token); + expect(identity).toBeNull(); + } finally { + await db.close(); + } + }, + 60_000, + ); +}); From ccbe497e63bf84527e9ad01dcc840b7be2155e10 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:16 -0700 Subject: [PATCH 14/48] e2e: stop pre-provisioning a shared sidecar identity (CL-7472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every e2e suite dial-in relied on a single, harness-started sidecar process registered against a bare `provisionSidecar` identity row. At pin 692c3106, the hub only accepts a register/reconnect frame from a sidecar whose identity resolves to a live sidecar_allocation (or workflow_probe) row; a standalone identity with neither now fails the handshake and gets rejected with "invalid token" every ~3s, forever (chat mint 500s, deploys never dial in — CI run 34042907014). The hub's own process-provisioner already spawns a dedicated, correctly-allocated sidecar process per deployment on demand — the same path production already takes for a real workflow run or a minted agent chat. Every affected suite (walking skeleton, chat, heartbeat, workbench-digest, smoke-chat, smoke-webhook, greeting-delivery, channel-reconnect, local-rip, and the isolation suite) now lets that happen instead of dialing in its own sidecar: the existing 502/500 retry-until-ready loops already cover the wait, and diagnostics that used to read the one sidecar process's output now read the hub's — the process-provisioner backend spawns each sidecar with its stdio inherited from the hub, so the hub's own captured output already carries every spawned sidecar's log lines. `provisionSidecar`/`startSidecar` stay in the harness (documented as non-authenticating at this pin) for the handful of standalone proof scripts not yet migrated off a directly-controlled sidecar process. --- scripts/e2e/channel-reconnect.test.ts | 37 +++++----------- scripts/e2e/chat.test.ts | 61 ++++++++++----------------- scripts/e2e/greeting-delivery.test.ts | 36 ++++------------ scripts/e2e/harness.ts | 26 +++++++++--- scripts/e2e/heartbeat.test.ts | 29 ++----------- scripts/e2e/local-rip.test.ts | 24 ++--------- scripts/e2e/smoke-chat.test.ts | 29 ++----------- scripts/e2e/smoke-webhook.test.ts | 38 +++-------------- scripts/e2e/walking-skeleton.test.ts | 45 +++++--------------- scripts/e2e/workbench-digest.test.ts | 29 ++----------- test/isolation/setup.ts | 49 ++++++--------------- 11 files changed, 109 insertions(+), 294 deletions(-) diff --git a/scripts/e2e/channel-reconnect.test.ts b/scripts/e2e/channel-reconnect.test.ts index 59e254c19..3c3e8e637 100644 --- a/scripts/e2e/channel-reconnect.test.ts +++ b/scripts/e2e/channel-reconnect.test.ts @@ -29,12 +29,9 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -76,7 +73,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { let hubPort: number; let sessionSecret: string; let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let cookies: string[]; let tenantId: string; @@ -93,10 +89,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "workbench-reconnect-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hubPort = freePort(); sessionSecret = Buffer.from( crypto.getRandomValues(new Uint8Array(32)), @@ -110,14 +102,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-reconnect-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); const email = `workbench-reconnect-e2e-${crypto.randomUUID()}@example.invalid`; @@ -159,9 +143,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -174,7 +158,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -244,10 +228,10 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { // hub process on the same port against the same database, exactly // as a real restart looks from the sidecar's and the browser's // side: same DB rows, same session cookie, new process. The port - // must stay the same — the sidecar's own `HUB_WS_URL` is fixed at - // its boot and is never reconfigured here (only the hub restarts), - // so a different port would leave the sidecar dialing a hub that - // is no longer there. + // must stay the same — the process-provisioner-spawned sidecar's + // own `HUB_WS_URL` is fixed at its boot and is never reconfigured + // here (only the hub restarts), so a different port would leave the + // sidecar dialing a hub that is no longer there. await hub.stop(); const restarted = await startHub({ @@ -269,9 +253,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 180_000; let posted: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited after hub restart; output:\n${sidecar.output()}`, + `hub exited after hub restart; output:\n${hub.output()}`, ); } posted = await postMessage(after); @@ -279,8 +263,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `message after hub restart never accepted: status=${posted.status} ` + - `${JSON.stringify(posted.data)}\nsidecar output:\n${sidecar.output()}` + - `\nhub output:\n${hub.output()}`, + `${JSON.stringify(posted.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index da9b255c8..c50da0894 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -46,14 +46,11 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, pushWorkflowSource, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -139,7 +136,6 @@ const textPart = (text: string): Part[] => [{ kind: "text", text }]; describe.skipIf(databaseUrl === undefined)("chat e2e", () => { let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let user1: SignedUpUser; let user2: SignedUpUser; @@ -159,10 +155,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "chat-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hub = await startHub({ databaseUrl: url, port: freePort(), @@ -173,16 +165,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-chat-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); // Two independent browser-shaped accounts, each carrying its own @@ -358,15 +340,15 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { ), }); - // Retries while the hub still answers 502 (the sidecar's dial-in - // may not have completed yet), matching `createWorkbench`'s own - // retry loop below. + // Retries while the hub still answers 502 (the process + // provisioner's spawned sidecar may not have dialed in yet), + // matching `createWorkbench`'s own retry loop below. const echoDeployDeadline = Date.now() + 60_000; let echoDeployed: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before echo deploy; output:\n${sidecar.output()}`, + `hub exited before echo deploy; output:\n${hub.output()}`, ); } echoDeployed = await api( @@ -384,7 +366,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > echoDeployDeadline) { throw new Error( `echo workflow never became deployable (hub kept answering 502): ` + - `${JSON.stringify(echoDeployed.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(echoDeployed.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -393,22 +375,22 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }, 120_000); // Launching a workbench is the go/no-go signal for the whole suite: it - // launches the anchor instance in-process, which needs the - // sidecar's dial-in to have completed. `packages/chat/src/routes.ts` - // has no 502-style retry translation of its own (unlike the native - // workflow deploy route), so a launch attempted before the sidecar - // connects fails with an uncaught 500 — retried here directly until - // the sidecar is ready, exactly as the walking skeleton retries a - // 502 for the native deploy route. + // launches the anchor instance in-process, which needs its + // process-provisioner-spawned sidecar's dial-in to have completed. + // `packages/chat/src/routes.ts` has no 502-style retry translation of + // its own (unlike the native workflow deploy route), so a launch + // attempted before the sidecar connects fails with an uncaught 500 — + // retried here directly until the sidecar is ready, exactly as the + // walking skeleton retries a 502 for the native deploy route. async function createWorkbench( body: Record, ): Promise { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -421,7 +403,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -1021,10 +1003,13 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { // temporarily reverting the self-anchor fix locally: the sidecar log // then carries ten `enqueueInbox failed, withholding ack` lines, one // per rejected pack, for this same suite; with the fix applied, zero. + // Every process-provisioner-spawned sidecar inherits the hub + // process's own stdio (`process-runner.ts`'s `stdout: "inherit"`), so + // `hub.output()` already carries every sidecar's log lines. test("no chat/folded run's workflow-run pack was ever permanently rejected", () => { - const sidecarOutput = sidecar.output(); - expect(sidecarOutput).not.toContain("enqueueInbox failed"); - expect(sidecarOutput).not.toContain("withholding ack"); - expect(sidecarOutput).not.toContain("rejecting inbound mail"); + const hubOutput = hub.output(); + expect(hubOutput).not.toContain("enqueueInbox failed"); + expect(hubOutput).not.toContain("withholding ack"); + expect(hubOutput).not.toContain("rejecting inbound mail"); }); }); diff --git a/scripts/e2e/greeting-delivery.test.ts b/scripts/e2e/greeting-delivery.test.ts index 0ac6e26bb..2521206f1 100644 --- a/scripts/e2e/greeting-delivery.test.ts +++ b/scripts/e2e/greeting-delivery.test.ts @@ -41,12 +41,9 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const { tempDir, track } = createCleanupHarness(); @@ -135,10 +132,6 @@ describe.skipIf(databaseUrl === undefined)( expect(report.action).toBe("migrated"); }); - const sidecarId = "greeting-delivery-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -151,16 +144,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar: SpawnedApp = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-greeting-delivery-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -236,9 +219,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { @@ -269,9 +252,9 @@ describe.skipIf(databaseUrl === undefined)( async function deploySeededWorkflows(): Promise { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -356,9 +339,9 @@ describe.skipIf(databaseUrl === undefined)( const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before chat creation; output:\n${sidecar.output()}`, + `hub exited before chat creation; output:\n${hub.output()}`, ); } res = await api( @@ -372,7 +355,7 @@ describe.skipIf(databaseUrl === undefined)( if (Date.now() > deadline) { throw new Error( `chat never became mintable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -448,7 +431,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no agent-authored message landed in chat ${chatId} within ` + `60s of mint with zero user messages sent; messages seen: ` + - `${JSON.stringify(items)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(items)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -527,8 +510,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no answer to turn ${turn} within 180s; agent messages: ` + `${JSON.stringify(agentTexts)}\nhub output (tail):\n` + - `${hub.output().slice(-60000)}\nsidecar output (tail):\n` + - `${sidecar.output().slice(-6000)}`, + `${hub.output().slice(-60000)}`, ); } await Bun.sleep(2000); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index cebc4e53d..1e884e0f9 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -221,6 +221,20 @@ export async function connectE2eDb(databaseUrl: string): Promise { * WebSocket dial-in against the token hash on the `sidecar` table, so * the row must exist before the sidecar process starts. Mirrors * Interchange's dev provisioning (id + sha256(token), placeholder url). + * + * At pin 692c3106, the hub only accepts a register/reconnect frame from + * a sidecar identity that resolves to a live `sidecar_allocation` (or + * `workflow_probe`) row — the standalone, unallocated ("shared" + * credential scope) sidecar this used to provision no longer + * authenticates at all (CL-7472), so a caller of this function gets a + * connect-and-immediately-reject loop, not a usable dial-in. Every + * suite this repo runs against CI now lets the hub's own + * process-provisioner spawn a dedicated sidecar per allocation instead + * (see `startHub`'s doc comment and `scripts/e2e/chat.test.ts`); this + * function is kept only for the handful of scripts not yet migrated off + * a directly-controlled sidecar process (e.g. ones that kill and + * restart a specific sidecar mid-scenario) and is not a working dial-in + * path until they are. */ export async function provisionSidecar( databaseUrl: string, @@ -400,11 +414,13 @@ export async function startHub(options: { } /** - * Boot the sidecar as a real process, pointed at the hub's WebSocket - * dial-in route with the provisioned identity. Connection readiness is - * observed by the deploy call itself (the hub answers 502 until a - * sidecar is connected), so this only guards against an immediate - * crash. + * Boot a standalone sidecar as a real process, pointed at the hub's + * WebSocket dial-in route with a `provisionSidecar`-issued identity. + * See `provisionSidecar`'s own doc comment: at pin 692c3106 this + * identity never authenticates (CL-7472), so the process this starts + * connects and is immediately rejected in a loop rather than dialing + * in. Kept only for the scripts not yet migrated to the hub's + * automatic, per-allocation process-provisioner spawn. */ export function startSidecar(options: { hubPort: number; diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 9aefb900f..8c924a6af 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -95,12 +93,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-heartbeat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -114,19 +106,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-heartbeat-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Heartbeat Tester", @@ -224,10 +203,8 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -240,7 +217,7 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 2b4f4ec9c..b1919de02 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -81,9 +81,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -187,10 +185,6 @@ describe.skipIf(databaseUrl === undefined)( } }); - const sidecarId = "local-rip-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -208,16 +202,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-local-rip-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -341,9 +325,9 @@ describe.skipIf(databaseUrl === undefined)( ): Promise>> { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -398,9 +382,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { diff --git a/scripts/e2e/smoke-chat.test.ts b/scripts/e2e/smoke-chat.test.ts index 98b598332..eaf0da399 100644 --- a/scripts/e2e/smoke-chat.test.ts +++ b/scripts/e2e/smoke-chat.test.ts @@ -20,9 +20,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -58,13 +56,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { }); const hubDataDir = await tempDir("e2e-smoke-chat-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-chat-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-chat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -78,18 +69,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -123,9 +102,9 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { // through the transient 500 the same way chat.test.ts does. const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } const res = await api( @@ -141,7 +120,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { } if (Date.now() > deadline) { throw new Error( - `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -171,7 +150,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { if (Date.now() > deadline) { throw new Error( `post message kept answering 500: ${JSON.stringify(res.data)}` + - `\nsidecar output:\n${sidecar.output()}`, + `\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 760aca6a3..d834aa623 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -12,9 +12,8 @@ // events by bare id (the native events route requires a deployment-anchor // run). This test instead reads the run row the ingress delivery created // straight out of the database — a harness-side fact with no route, -// exactly as `provisionSidecar` already does for the sidecar identity -// row — and confirms the trigger itself recorded the delivery -// (`lastFiredAt`). +// via `connectE2eDb` — and confirms the trigger itself recorded the +// delivery (`lastFiredAt`). import { signPayload, @@ -36,12 +35,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -77,13 +74,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const hubDataDir = await tempDir("e2e-smoke-webhook-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-webhook-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-webhook"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -97,18 +87,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -204,10 +182,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } const res = await api( hub.baseUrl, @@ -219,7 +195,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { if (res.status === 502) { if (Date.now() > deadline) { throw new Error( - `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -346,8 +322,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // No platform route reads a non-anchored run's row by bare id // (see the file header); this reads the fact the ingress // route's own launch is required to have produced, the same - // way `provisionSidecar` reaches the database directly for a - // fact no route exposes. + // way `connectE2eDb` reaches the database directly for a fact + // no route exposes. const sql = await connectE2eDb(url); try { const rows = await sql.unsafe( diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 8937f9b5e..e876f6bfa 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -33,12 +33,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, } from "./harness.ts"; @@ -81,15 +79,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { expect(second.migrations).toBe(first.migrations); }); - // Hop: sidecar provisioning. The identity row the hub checks the - // sidecar's dial-in token against. - const sidecarId = "sidecar-e2e"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - - // Hop: hub boot. The composition root as a real process. + // Hop: hub boot. The composition root as a real process. The hub's + // own process provisioner spawns a dedicated sidecar for every + // allocation on demand (one `apps/sidecar` per deployment) — there + // is no shared, pre-provisioned sidecar identity to dial in here. const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -103,21 +96,6 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { return handle; }); - // Hop: sidecar boot. Dial-in readiness is observed at the deploy - // hop (the hub answers 502 until a sidecar is connected). - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-sidecar-data-"), - }); - track(app); - return app; - }); - // Hop: sign-up. A browser-shaped account creation through the // platform's better-auth surface; the session cookie carries the // rest of the skeleton. @@ -215,9 +193,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { ); // Hop: workflow deploy via the native deploy API. Retries while - // the hub still answers 502 (the sidecar's dial-in may not have - // completed yet); any other failure is final. The inference - // source is a placeholder — deployment does not call inference. + // the hub still answers 502 (the process provisioner's spawned + // sidecar may not have dialed in yet); any other failure is final. + // The inference source is a placeholder — deployment does not call + // inference. const deploymentId = await hop("workflow deploy", async () => { const body = workflowDeployBody({ assetId, @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 8917bf823..f3d41cd06 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -100,12 +98,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-workbench-digest"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -119,19 +111,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-digest-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Workbench Digest Tester", @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/test/isolation/setup.ts b/test/isolation/setup.ts index 3644750fa..07271b0c1 100644 --- a/test/isolation/setup.ts +++ b/test/isolation/setup.ts @@ -10,11 +10,8 @@ import path from "node:path"; import { freePort, - provisionSidecar, startHub, - startSidecar, type HubHandle, - type SpawnedApp, } from "../../scripts/e2e/harness.ts"; import { assertDatabaseConfigured } from "../../scripts/e2e/db-gate.ts"; @@ -90,29 +87,21 @@ export async function prepareDatabase(databaseUrl: string): Promise { } /** - * Boots the real hub and a real sidecar as spawned processes, exactly - * as `scripts/e2e/harness.ts` does for the walking skeleton — a - * chat-workbench create launches a real agent instance over the - * sidecar's WebSocket dial-in, so an in-process hub with no sidecar - * attached cannot exercise the "chat workbench move" block below; it - * would fail every one of those cases with "no sidecar available" - * rather than proving isolation. `AppLike.request` wraps `fetch` - * against the hub's real port, matching how a client actually reaches - * it. + * Boots the real hub as a spawned process, exactly as + * `scripts/e2e/harness.ts` does for the walking skeleton — a + * chat-workbench create launches a real agent instance over its own + * process-provisioner-spawned sidecar's WebSocket dial-in, so an + * in-process hub with no sidecar plugin wired up cannot exercise the + * "chat workbench move" block below; it would fail every one of those + * cases with "no sidecar available" rather than proving isolation. + * `AppLike.request` wraps `fetch` against the hub's real port, matching + * how a client actually reaches it. */ export async function bootIsolationHub( databaseUrl: string, ): Promise { const root = mkdtempSync(path.join(tmpdir(), "isolation-suite-")); const hubDataDir = path.join(root, "hub-data"); - const sidecarDataDir = path.join(root, "sidecar-data"); - - // Unique per run: the `sidecar` table's id is a primary key, and a - // prior run's row is never cleaned up (the isolation suite runs - // against a shared database, never a scratch one it owns outright). - const sidecarId = `sidecar-isolation-suite-${crypto.randomUUID()}`; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(databaseUrl, sidecarId, sidecarToken); const hub: HubHandle = await startHub({ databaseUrl, @@ -133,23 +122,15 @@ export async function bootIsolationHub( }, }); - const sidecar: SpawnedApp = startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }); - const app: AppLike = { request: (requestPath, init) => fetch(`${hub.baseUrl}${requestPath}`, init), }; - await waitForSidecarDialIn(app, sidecar); + await waitForSidecarDialIn(app, hub); return { app, shutdown: async () => { - await sidecar.stop(); await hub.stop(); rmSync(root, { recursive: true, force: true }); }, @@ -169,7 +150,7 @@ export async function bootIsolationHub( */ async function waitForSidecarDialIn( app: AppLike, - sidecar: SpawnedApp, + hub: HubHandle, ): Promise { const cookie = await signUpUser( app, @@ -194,10 +175,8 @@ async function waitForSidecarDialIn( const deadline = Date.now() + 30_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before dialing in; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before dialing in; output:\n${hub.output()}`); } const workbenchResponse = await app.request( `/api/tenants/${tenant.id}/chat/workbenches`, @@ -212,7 +191,7 @@ async function waitForSidecarDialIn( throw new Error( `sidecar never dialed in within 30s (workbench create kept ` + `failing with ${workbenchResponse.status}): ` + - `${await workbenchResponse.text()}\nsidecar output:\n${sidecar.output()}`, + `${await workbenchResponse.text()}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(500); From 308ab2526a5018b2e65ae4bff4eb17805ecefdc7 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:21 -0700 Subject: [PATCH 15/48] ci: upload hub/sidecar e2e logs as a build artifact (CL-7472) The e2e job's hub/sidecar output was never captured, so the CI run that surfaced the sidecar disconnect loop (34042907014) had no sidecar-side log to read past the transcript. Point E2E_LOG_DIR at a runner-temp directory and upload it on every run, pass or fail. --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a35363400..4e67a69e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,15 @@ jobs: run: bun run test:e2e env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Upload hub/sidecar logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs + path: ${{ runner.temp }}/e2e-logs + if-no-files-found: ignore isolation: runs-on: ubuntu-latest From b72964e24b7d8b384459eca23fa1b15ab45a105f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:01 -0700 Subject: [PATCH 16/48] Add tests for tenant-scoped workbench_launch address resolution (CL-7474) --- packages/chat/src/agent-binding.test.ts | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index ed89cec9d..cd59d2172 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -96,6 +96,56 @@ describe("readBindingByAddress", () => { }); }); +describe("readBindingByAddress: tenant scoping (CL-7474)", () => { + // Two tenants that each independently invited "the same" agent + // (definitionId happens to differ per tenant in practice, but the + // launch rows below are shaped exactly like two independent + // launches — different stable ids, different run ids, different + // tenants). `instanceId`/`currentRunId` are collision-resistant + // generated ids, not scoped to a tenant, so nothing but an explicit + // `expectedTenantId` check stops tenant B's caller from resolving + // tenant A's row if it ever guessed or replayed tenant A's address. + const tenantALaunch: LaunchRow = { + tenantId: "tnt_a", + instanceId: "run_a1", + currentRunId: "run_a1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const tenantBLaunch: LaunchRow = { + tenantId: "tnt_b", + instanceId: "run_b1", + currentRunId: "run_b1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const db = fakeDb([tenantALaunch, tenantBLaunch]); + + test("each tenant's DM resolves to its own distinct run id and address", async () => { + const a = await readBindingByAddress(db, "run_a1@acme.example", "tnt_a"); + const b = await readBindingByAddress(db, "run_b1@acme.example", "tnt_b"); + expect(a?.currentRunId).toBe("run_a1"); + expect(b?.currentRunId).toBe("run_b1"); + expect(a?.currentRunId).not.toBe(b?.currentRunId); + expect(a?.roomAddress).not.toBe(b?.roomAddress); + }); + + test("a bench invite in tenant B never resolves tenant A's launch", async () => { + expect( + await readBindingByAddress(db, "run_a1@acme.example", "tnt_b"), + ).toBeUndefined(); + expect( + await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), + ).toBeUndefined(); + }); + + test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { + expect( + (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, + ).toBe("tnt_a"); + }); +}); + describe("resolveRoomAddress", () => { test("leaves a non-participant address alone", async () => { expect( From 262a381344ead7f2dec2a47f17f371b3ff4070bb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:09 -0700 Subject: [PATCH 17/48] Chat: workbench_launch address/stableId lookups are tenant-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readBindingByAddress and resolveLiveByStableId took an address or stable id and resolved whichever workbench_launch row matched it, regardless of which tenant that row belonged to. Every caller that already knows its own tenant (sendMail's requireLive, and refreshAgentInstanceFromDefinition) now passes it through and gets undefined instead of another tenant's launch if the two ever disagree. The one legitimate tenant-agnostic caller — the event stream handler discovering which tenant an inbound address even belongs to before it has one to check against — keeps working by simply omitting the new optional parameter. Also fixes offeringDigest: it joined catalog offering ids with a NUL byte, which is not valid UTF-8 to Postgres the moment a tenant's catalog carries more than one visible offering ("invalid byte sequence for encoding \"UTF8\": 0x00"), permanently failing the workbench_launch insert for every such launch. This is what was failing 4 of scripts/e2e/chat.test.ts's cases under per-allocation sidecars, where seedCatalog now produces more than one offering. --- packages/chat/src/agent-binding.ts | 25 ++++++++++++++++++++++--- packages/chat/src/platform-adapter.ts | 23 +++++++++++++++++------ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index d53399366..6d3917d4f 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,13 +135,25 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, + expectedTenantId?: string, ): Promise { const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - if (byStableId !== undefined) return bindingFrom(byStableId, domain); - const byRunId = await readLaunchRow(db, "currentRunId", localPart); - return byRunId === undefined ? undefined : bindingFrom(byRunId, domain); + const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + if (row === undefined) return undefined; + // A caller that already knows which tenant it is acting for must + // never accept a binding minted by another one — `instanceId` and + // `currentRunId` are collision-resistant generated ids, not scoped to + // a tenant, so nothing else stops an address that happens to name a + // row from a different tenant from resolving here. A caller with no + // tenant of its own yet (the event-stream handler discovering which + // tenant an inbound address even belongs to) passes no + // `expectedTenantId` and gets the address-only resolution it needs. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } + return bindingFrom(row, domain); } /** @@ -183,9 +195,16 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, + expectedTenantId?: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; + // See the matching note on `readBindingByAddress`: a caller that + // knows its own tenant must never be handed another tenant's launch + // for a stable id it happens to guess or receive by mistake. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 6490af3c7..f871887da 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -258,7 +258,15 @@ export function createHubChatPlatform( }; function offeringDigest(sourceOfferingIds: readonly string[]): string { - return sourceOfferingIds.join("\0"); + // `sources_digest` is a `text` column and this digest rides straight + // into it (`platform-adapter.ts`'s `insert(workbenchLaunch)`). A + // `\0`-joined id list is not valid UTF-8 text to Postgres + // (`invalid byte sequence for encoding "UTF8": 0x00`) the moment a + // tenant's catalog carries more than one visible offering, which + // permanently failed every agent launch for such a tenant. `\n` never + // appears inside a generated offering id, so this stays a lossless, + // order-sensitive join — just one Postgres can actually store. + return sourceOfferingIds.join("\n"); } async function catalogOfferings(tenantId: string): Promise<{ @@ -913,8 +921,11 @@ export function createHubChatPlatform( * participant id — not the room's own address, once anything has been * relaunched. */ - async function requireLive(stableId: string): Promise { - const live = await resolveLiveByStableId(deps.db, stableId); + async function requireLive( + stableId: string, + expectedTenantId?: string, + ): Promise { + const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1176,7 +1187,7 @@ export function createHubChatPlatform( _workbenchId, address, ): Promise { - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddress(deps.db, address, tenantId); if (binding === undefined) return; const live = await resolveLiveAgent(deps.db, binding); const definitionId = live?.run.definitionId; @@ -1216,7 +1227,7 @@ export function createHubChatPlatform( // The stable id names the room's participant; the run it resolves // to is whichever one is alive right now, which is a different // run (and a different address) after every relaunch. - const { binding } = await requireLive(input.workbenchId); + const { binding } = await requireLive(input.workbenchId, input.tenantId); const liveAddress = binding.liveAddress; // Wake before send: a sleeping instance (the lifecycle package's @@ -1270,7 +1281,7 @@ export function createHubChatPlatform( } catch (error) { throw wrapWakeInferenceError(error); } - const delivery = await requireLive(input.workbenchId); + const delivery = await requireLive(input.workbenchId, input.tenantId); const deliveryAddress = delivery.binding.liveAddress; // Tracking here (not only at launch) brings instances that were // already resident before this hub process started — restored by From b75eebc071aaa7060109de799db16375504bc0bf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:01:01 -0700 Subject: [PATCH 18/48] Chat: run prettier on the tenant-scoping change --- packages/chat/src/agent-binding.ts | 3 ++- packages/chat/src/platform-adapter.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 6d3917d4f..18bad4513 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -140,7 +140,8 @@ export async function readBindingByAddress( const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index f871887da..602ec0b0d 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -925,7 +925,11 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); + const live = await resolveLiveByStableId( + deps.db, + stableId, + expectedTenantId, + ); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } From 0eef0e0034af3e0c5cf7870b1d0e3cba8cfb72ba Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:26:38 -0700 Subject: [PATCH 19/48] Chat: tenant-scoped binding reads require the tenant; the one unscoped read is named --- packages/chat/src/agent-binding.test.ts | 22 +++++-- packages/chat/src/agent-binding.ts | 82 +++++++++++++++++++------ packages/chat/src/chat-orchestrator.ts | 10 ++- packages/chat/src/platform-adapter.ts | 31 +++++++--- 4 files changed, 109 insertions(+), 36 deletions(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index cd59d2172..947404cf3 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from "bun:test"; import { isBeyondWake, readBindingByAddress, + readBindingByAddressAnyTenant, resolveRoomAddress, } from "./agent-binding"; @@ -71,6 +72,7 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_original@acme.example", + "ten_1", ); expect(binding?.stableId).toBe("run_original"); expect(binding?.currentRunId).toBe("run_fresh"); @@ -85,13 +87,18 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_fresh@acme.example", + "ten_1", ); expect(binding?.roomAddress).toBe("run_original@acme.example"); }); test("is undefined for an address this package never launched", async () => { expect( - await readBindingByAddress(fakeDb([relaunched]), "echo_1@acme.example"), + await readBindingByAddress( + fakeDb([relaunched]), + "echo_1@acme.example", + "ten_1", + ), ).toBeUndefined(); }); }); @@ -139,10 +146,17 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { ).toBeUndefined(); }); - test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { +}); + +describe("readBindingByAddressAnyTenant", () => { + test("the address-only resolution event-stream discovery needs, with no tenant to check against", async () => { + const anyTenantDb = fakeDb([ + { ...relaunched, instanceId: "run_a1", currentRunId: "run_a1" }, + ]); expect( - (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, - ).toBe("tnt_a"); + (await readBindingByAddressAnyTenant(anyTenantDb, "run_a1@acme.example")) + ?.tenantId, + ).toBe("ten_1"); }); }); diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 18bad4513..47b4d0d71 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,26 +135,48 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, - expectedTenantId?: string, + tenantId: string, ): Promise { - const domain = requireDomain(address); - const localPart = localPartOf(address); - const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = - byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = await readBindingRowByAddress(db, address); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and // `currentRunId` are collision-resistant generated ids, not scoped to // a tenant, so nothing else stops an address that happens to name a - // row from a different tenant from resolving here. A caller with no - // tenant of its own yet (the event-stream handler discovering which - // tenant an inbound address even belongs to) passes no - // `expectedTenantId` and gets the address-only resolution it needs. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } - return bindingFrom(row, domain); + // row from a different tenant from resolving here. `tenantId` is + // required precisely so a caller cannot silently fall back to the + // unscoped read by forgetting to pass it — see + // `readBindingByAddressAnyTenant` for the one place that read is + // actually correct. + if (row.row.tenantId !== tenantId) return undefined; + return bindingFrom(row.row, row.domain); +} + +/** + * The unscoped counterpart of `readBindingByAddress`, for the one caller + * that genuinely cannot know the tenant yet: the inbound event-stream + * handler discovering which tenant an address even belongs to before it + * can act as that tenant. Every other caller already knows its own + * tenant and must use `readBindingByAddress` instead. + */ +export async function readBindingByAddressAnyTenant( + db: DB["db"], + address: string, +): Promise { + const row = await readBindingRowByAddress(db, address); + return row === undefined ? undefined : bindingFrom(row.row, row.domain); +} + +async function readBindingRowByAddress( + db: DB["db"], + address: string, +): Promise<{ row: LaunchRow; domain: string } | undefined> { + const domain = requireDomain(address); + const localPart = localPartOf(address); + const byStableId = await readLaunchRow(db, "instanceId", localPart); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + return row === undefined ? undefined : { row, domain }; } /** @@ -167,7 +189,7 @@ export async function resolveRoomAddress( db: DB["db"], liveAddress: string, ): Promise { - const binding = await readBindingByAddress(db, liveAddress); + const binding = await readBindingByAddressAnyTenant(db, liveAddress); return binding?.roomAddress ?? liveAddress; } @@ -196,16 +218,36 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, - expectedTenantId?: string, + tenantId: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; // See the matching note on `readBindingByAddress`: a caller that // knows its own tenant must never be handed another tenant's launch - // for a stable id it happens to guess or receive by mistake. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } + // for a stable id it happens to guess or receive by mistake. Required, + // not optional — see `resolveLiveByStableIdAnyTenant` for the unscoped + // read. + if (row.tenantId !== tenantId) return undefined; + return resolveLiveRunForRow(db, row); +} + +/** + * The unscoped counterpart of `resolveLiveByStableId`, for a caller that + * has no tenant of its own to check against (e.g. it is resolving a + * stable id it does not yet know the tenant for). + */ +export async function resolveLiveByStableIdAnyTenant( + db: DB["db"], + stableId: string, +): Promise { + const row = await readLaunchRow(db, "instanceId", stableId); + return row === undefined ? undefined : resolveLiveRunForRow(db, row); +} + +async function resolveLiveRunForRow( + db: DB["db"], + row: LaunchRow, +): Promise { const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/chat-orchestrator.ts b/packages/chat/src/chat-orchestrator.ts index 808c755ba..73ce91b0c 100644 --- a/packages/chat/src/chat-orchestrator.ts +++ b/packages/chat/src/chat-orchestrator.ts @@ -57,7 +57,10 @@ import { consumerFacingInferenceText } from "./consumer-inference-text"; import type { ConnectedProviderLister } from "./inference-preferences"; import { mentionedParticipants } from "./mentions"; import { domainOf, localPartOf } from "./agent-address"; -import { readBindingByAddress, resolveLiveAgent } from "./agent-binding"; +import { + readBindingByAddressAnyTenant, + resolveLiveAgent, +} from "./agent-binding"; import { parseParticipants, type ParticipantRecord } from "./participants"; import type { Part, TextPart } from "./parts"; import type { ChatPlatform } from "./platform-port"; @@ -448,7 +451,10 @@ async function resolveMemberWorkbenches( // event's address against `workflow_run.address` directly: after a // relaunch the two differ, and only the mapping knows that the run // announcing itself under a fresh address is the same room teammate. - const binding = await readBindingByAddress(deps.db, agentAddress); + // The tenant an inbound event's address belongs to is exactly what + // this discovery is trying to find out, so it cannot pass one yet — + // see `readBindingByAddressAnyTenant`. + const binding = await readBindingByAddressAnyTenant(deps.db, agentAddress); if (binding === undefined) { // Not every agent address on the event stream belongs to a chat // workbench (an echo instance, say) — an address this package's own diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 602ec0b0d..39e52ff2c 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -33,11 +33,13 @@ import { listLaunchesBeyondWake, listLaunchesForTenant, readBindingByAddress, + readBindingByAddressAnyTenant, readPriorRuns, recordSourcesDigest, repointBinding, resolveLiveAgent, resolveLiveByStableId, + resolveLiveByStableIdAnyTenant, type AgentBinding, type LiveAgent, } from "./agent-binding"; @@ -703,7 +705,13 @@ export function createHubChatPlatform( * that treatment, just triggered by content drift instead of death. */ async function wakeByAddress(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // Every caller of `wakeByAddress` already reached `address` through + // a tenant-scoped resolution of its own (`sendMail`'s + // `requireLive(workbenchId, tenantId)`, `ensureAwake`'s caller-held + // address) — re-deriving the binding here is bookkeeping for a wake + // already authorized upstream, not a new trust boundary crossing, so + // the unscoped read is correct. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error( `No workbench_launch binding for address "${address}"; instances ` + @@ -809,7 +817,9 @@ export function createHubChatPlatform( * check that used to never run for it. */ async function reconcileDriftedRun(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // See the matching note in `wakeByAddress`: `address` is always + // reached through a tenant-scoped resolution upstream. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return false; const live = await resolveLiveAgent(deps.db, binding); if (live === undefined || live.run.address === null) return false; @@ -925,11 +935,10 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId( - deps.db, - stableId, - expectedTenantId, - ); + const live = + expectedTenantId === undefined + ? await resolveLiveByStableIdAnyTenant(deps.db, stableId) + : await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1166,7 +1175,9 @@ export function createHubChatPlatform( }, async resolveDefinitionIdByAddress(address): Promise { - const binding = await readBindingByAddress(deps.db, address); + // No tenant is threaded through this platform method's own + // signature; unscoped by necessity, not by omission. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return undefined; const live = await resolveLiveAgent(deps.db, binding); return live?.run.definitionId ?? undefined; @@ -1409,7 +1420,7 @@ export function createHubChatPlatform( let cancelled = false; let unsubscribeAgent: (() => void) | undefined; - void resolveLiveByStableId(deps.db, workbenchId) + void resolveLiveByStableIdAnyTenant(deps.db, workbenchId) .then((live) => { if (cancelled || live === undefined) return; unsubscribeAgent = deps.sidecarRouter.subscribeAgent( @@ -1437,7 +1448,7 @@ export function createHubChatPlatform( // undelivered-mail handler holds whatever the envelope named), so // the lifecycle is driven on the LIVE address it resolves to — // that is the only address the sidecar ever announces. - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error(`No workbench_launch binding for address "${address}"`); } From b6774510f8c69df16be37b566295d8f49b402ad3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:08 -0700 Subject: [PATCH 20/48] Add tests for the shared retry-until-routable delivery helper --- packages/webhook-triggers/test/launch.test.ts | 36 +++++++++ .../src/deliver-when-routable.test.ts | 74 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 packages/workflows/src/deliver-when-routable.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index f1a9bb2c1..fa7559086 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -137,6 +137,7 @@ let sendUserMessageCalls: unknown[] = []; let sendUserMessageImpl: () => Promise = async () => new Uint8Array([1]); let cryptoGetKeys: string[] = []; +let isRoutableForTest = true; function baseDeps() { return { @@ -177,6 +178,7 @@ function baseDeps() { return sendUserMessageImpl(); }, }, + isRoutable: () => isRoutableForTest, }; } @@ -191,6 +193,7 @@ function resetLaunchSpies() { visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS]; frozenProjection = INERT_PROJECTION; sendUserMessageImpl = async () => new Uint8Array([1]); + isRoutableForTest = true; } describe("launchWebhookTrigger", () => { @@ -214,6 +217,39 @@ describe("launchWebhookTrigger", () => { expect(sendUserMessageCalls).toHaveLength(1); }); + // CL-7476: a freshly provisioned run's sidecar takes several seconds to + // boot and register with the hub. The opening mail can land in that + // gap and fail "agent is unreachable" even though the run deployed + // fine — `deliverWhenRoutable` must retry once the address becomes + // routable rather than dropping the mail for good. + test("retries the opening mail once the freshly provisioned run becomes routable", async () => { + resetLaunchSpies(); + isRoutableForTest = false; + let sendAttempts = 0; + sendUserMessageImpl = async () => { + sendAttempts += 1; + if (sendAttempts === 1) { + throw new Error("agent is unreachable: ins_1@ten1.workbench.test"); + } + return new Uint8Array([1]); + }; + + const deps = baseDeps(); + const sendPromise = launchWebhookTrigger(deps, TRIGGER, { status: "ok" }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + isRoutableForTest = true; + + const result = await sendPromise; + + expect(result).toEqual({ + instanceId: INTERCHANGE_RUN_ID, + triggerAddress: INTERCHANGE_ADDRESS, + }); + expect(sendUserMessageCalls).toHaveLength(2); + expect(reportErrorCalls).toHaveLength(0); + }); + test("reports the delivery failure with the run's context", async () => { resetLaunchSpies(); const deliveryError = new Error("sidecar unreachable"); diff --git a/packages/workflows/src/deliver-when-routable.test.ts b/packages/workflows/src/deliver-when-routable.test.ts new file mode 100644 index 000000000..b283de6a1 --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { deliverWhenRoutable } from "./deliver-when-routable"; + +function unreachable(): Error { + return new Error("agent is unreachable: run_1@ten1.workbench.test"); +} + +describe("deliverWhenRoutable", () => { + test("retries once the address is routable after an unreachable send", async () => { + let sendCalls = 0; + let routablePolls = 0; + const sleeps: number[] = []; + + const result = await deliverWhenRoutable({ + send: async () => { + sendCalls++; + if (sendCalls === 1) { + throw unreachable(); + } + return "delivered"; + }, + isRoutable: () => { + routablePolls++; + return routablePolls >= 2; + }, + sleep: async (ms) => { + sleeps.push(ms); + }, + }); + + expect(result).toBe("delivered"); + expect(sendCalls).toBe(2); + expect(routablePolls).toBe(2); + expect(sleeps.length).toBe(1); + }); + + test("rethrows the original unreachable error once the deadline passes", async () => { + let now = 0; + const originalNow = Date.now; + Date.now = () => now; + try { + let sendCalls = 0; + await expect( + deliverWhenRoutable({ + send: async () => { + sendCalls++; + throw unreachable(); + }, + isRoutable: () => false, + deadlineMs: 500, + pollIntervalMs: 100, + sleep: async (ms) => { + now += ms; + }, + }), + ).rejects.toThrow("agent is unreachable"); + expect(sendCalls).toBe(1); + } finally { + Date.now = originalNow; + } + }); + + test("does not retry an error other than unreachable", async () => { + const err = new Error("boom"); + await expect( + deliverWhenRoutable({ + send: async () => { + throw err; + }, + isRoutable: () => true, + }), + ).rejects.toBe(err); + }); +}); From 5cb3164f04a8ae0406b4be5a50bb50eb379107ff Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:12 -0700 Subject: [PATCH 21/48] Deliver a webhook trigger's opening mail once its sidecar becomes routable (CL-7476) --- apps/hub/src/index.ts | 2 + packages/webhook-triggers/src/launch.ts | 32 +++++++--- .../workflows/src/deliver-when-routable.ts | 63 +++++++++++++++++++ packages/workflows/src/index.ts | 5 ++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 packages/workflows/src/deliver-when-routable.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 687bcde64..3914c6bef 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2258,6 +2258,8 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, credentialCipher, + isRoutable: (address) => + sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 0e6e03b68..6f3b3adfa 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -14,6 +14,7 @@ // reports through `@corbits/error-sink`, naming the run. import { reportError } from "@corbits/error-sink"; import { + deliverWhenRoutable, readDefinitionProjection, readFoldedBody, WORKFLOW_SOURCE_ENTRY, @@ -46,6 +47,15 @@ export type LaunchWebhookTriggerDeps = { "prepareProvisionedDeployment" >; sessionService: Pick; + /** + * Reads the hub's live sidecar routing table. A freshly provisioned + * run's sidecar takes several seconds to boot and register (CL-7476) + * — a webhook's opening mail can land in that gap and fail with + * "agent is unreachable" even though the run itself deployed fine. + * `deliverWhenRoutable` polls this until the address is routable + * before its one retry. + */ + isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; /** * Host-supplied cipher. Interchange decrypts bindings inside @@ -201,15 +211,19 @@ export async function launchWebhookTrigger( prepared.anchorRunId, ); try { - await deps.sessionService.sendUserMessage({ - agentAddress: prepared.deploymentAddress, - from: `webhook-trigger:${trigger.id}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content, - sessionId, - tenantId: trigger.tenantId, - cryptoProvider, + await deliverWhenRoutable({ + send: () => + deps.sessionService.sendUserMessage({ + agentAddress: prepared.deploymentAddress, + from: `webhook-trigger:${trigger.id}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content, + sessionId, + tenantId: trigger.tenantId, + cryptoProvider, + }), + isRoutable: () => deps.isRoutable(prepared.deploymentAddress), }); } catch (error) { reportError(error, { diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts new file mode 100644 index 000000000..af6aa21c1 --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.ts @@ -0,0 +1,63 @@ +// CL-7476: since the process provisioner started spawning one sidecar per +// allocation, a freshly launched run's first mail can arrive before that +// sidecar has finished booting and registering with the hub. The send +// fails with "agent is unreachable" even though nothing is actually wrong +// — the address just isn't routable yet. Chat's send/fan-out path, its +// invite first-turn delivery, and the webhook trigger's ingress delivery +// each hit this same race independently; this is the one shared retry +// they all go through instead of three ad hoc copies. +const DEFAULT_DEADLINE_MS = 20_000; +const DEFAULT_POLL_INTERVAL_MS = 250; + +export function isAgentUnreachableError(err: unknown): boolean { + return err instanceof Error && err.message.includes("agent is unreachable"); +} + +export type DeliverWhenRoutableOptions = { + /** Performs the actual delivery. Called once, and again after the + * address becomes routable if the first attempt fails unreachable. */ + send: () => Promise; + /** Reads the hub's live routing table for this delivery's address. */ + isRoutable: () => boolean; + /** Defaults to matching Interchange's "agent is unreachable" message. */ + isUnreachable?: (err: unknown) => boolean; + /** How long to keep polling before giving up. Defaults to a budget a + * touch past the process provisioner's own boot-and-register time. */ + deadlineMs?: number; + pollIntervalMs?: number; + sleep?: (ms: number) => Promise; +}; + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Attempts `send`; if it fails with "agent is unreachable", polls + * `isRoutable` until the address is routable (or the deadline passes) + * and sends exactly once more. Any other failure, or an unreachable + * failure that never resolves before the deadline, is rethrown as-is — + * this never swallows the original error or retries indefinitely. + */ +export async function deliverWhenRoutable( + opts: DeliverWhenRoutableOptions, +): Promise { + const isUnreachable = opts.isUnreachable ?? isAgentUnreachableError; + try { + return await opts.send(); + } catch (err) { + if (!isUnreachable(err)) { + throw err; + } + const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS); + const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const sleep = opts.sleep ?? defaultSleep; + while (!opts.isRoutable()) { + if (Date.now() >= deadline) { + throw err; + } + await sleep(pollIntervalMs); + } + return await opts.send(); + } +} diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 9fefe188a..92c132180 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,11 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + deliverWhenRoutable, + isAgentUnreachableError, + type DeliverWhenRoutableOptions, +} from "./deliver-when-routable"; export { listScheduledWorkflowDefinitions, scheduledDefinitionsFromRows, From a0c8a27ab25a0fa239060eb282e8b6e9b1ffbbbc Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:16 -0700 Subject: [PATCH 22/48] Log a skip, not an error, when root tenant seed has no operator key configured --- apps/hub/src/system-seed.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 1637f2e89..f2f9df7c0 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -35,6 +35,12 @@ const log = getLogger(["hub", "system-seed"]); const DEFAULT_DEADLINE_MS = 60_000; const DEFAULT_POLL_INTERVAL_MS = 2_000; +// Matches `packages/seeding/src/seed.ts`'s `resolveRealSourceOfferingIds` +// failure verbatim — the one deploy failure that is expected, not +// transient, when no operator seed key is configured. +const NO_CATALOG_OFFERINGS_REASON = + "this tenant has no catalog offerings to deploy against"; + // Matches `workbench seed`'s own default model source (readSeedConfig in // the now-deleted `packages/cli/src/config.ts`): real anthropic/ // claude-sonnet-5 when a hub-owned key is configured (`config.seedModel`), @@ -150,6 +156,18 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { } catch (cause) { const reason = cause instanceof Error ? cause.message : String(cause); if (Date.now() >= deadline) { + // No operator seed key means `seedCatalog` can only ever plant a + // placeholder credential — never a real, launchable offering — + // so a default workflow's deploy permanently has nothing to + // deploy against until an operator configures one. That is the + // expected shape of an unconfigured dev/CI boot, not a fault the + // next boot can retry its way out of, so it is a logged skip + // rather than an error (which `reportError` would otherwise + // paint red on every single boot). + if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; + return; + } reportError(cause, { operation: "system-seed.seedRootTenant" }); log.error`root tenant seed did not complete before its deadline (last error: ${reason}); the next boot will retry`; return; From 1fd89ddc9b40f5a2173f19eb58629baf66412f78 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:37:08 -0700 Subject: [PATCH 23/48] Run prettier --- apps/hub/src/system-seed.ts | 5 ++++- packages/chat/src/agent-binding.test.ts | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index f2f9df7c0..039ad2b4c 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -164,7 +164,10 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { // next boot can retry its way out of, so it is a logged skip // rather than an error (which `reportError` would otherwise // paint red on every single boot). - if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + if ( + deps.seedModel === undefined && + reason.includes(NO_CATALOG_OFFERINGS_REASON) + ) { log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; return; } diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf3..f11d61a60 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 3f8fdc95da60a75a6ebbdf51aba59f34c62fea92 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:38:30 -0700 Subject: [PATCH 24/48] Chat: run prettier on the tenant-scoping test file --- packages/chat/src/agent-binding.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf3..f11d61a60 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 46a4a9301e1b0504dcc1ca91a5a326a254c262a2 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:00 -0700 Subject: [PATCH 25/48] Add a DB-gated test for recording a run's agent_session (CL-7477) resolveRunSessionId (Interchange's mail-routing lookup) reads agent_session keyed by a run's principal, but nothing writes that table for a native launch since folded-runs was deleted -- a run's first turn silently fails to persist mail. Proves recordAgentSessionForRun writes a row resolveRunSessionId reads back, and updates the webhook launch fake DB to model workflow_run/agent_session the way the fix now touches them. --- packages/webhook-triggers/test/launch.test.ts | 18 +++ packages/workflows/test/agent-session.test.ts | 106 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 packages/workflows/test/agent-session.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index fa7559086..177a5090b 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,7 +88,25 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, + // `recordAgentSessionForRun` reads the just-provisioned run back by + // id — production never inserts this row itself (Interchange's + // `prepareProvisionedDeployment` mints the anchor row + // transactionally before returning it), so this models the same + // row for `INTERCHANGE_RUN_ID`. + workflowRun: { + findFirst: async () => ({ + id: INTERCHANGE_RUN_ID, + tenantId: TENANT_ROW.id, + definitionId: DEFINITION_ROW.id, + principalId: "prn_interchange", + }), + }, }, + insert: (_table: unknown) => ({ + values: (_values: unknown) => ({ + onConflictDoNothing: async (_opts: unknown) => {}, + }), + }), }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts new file mode 100644 index 000000000..a7446adda --- /dev/null +++ b/packages/workflows/test/agent-session.test.ts @@ -0,0 +1,106 @@ +// DB-gated: skipped when DATABASE_URL is unreachable, matching this +// repo's convention for tests that talk to a real Postgres (see +// `packages/access-tools/test/routes.test.ts`). +// +// CL-7477: before this fix, nothing wrote `agent_session` for a native +// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could +// never find a session for a run's principal and every launch's first +// turn failed to persist mail. This proves `recordAgentSessionForRun` +// writes a row `resolveRunSessionId` actually reads back. +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; + +import { createDB, schema, type DB } from "@intx/db"; +import { generateId } from "@intx/hub-common"; +import { resolveRunSessionId } from "@intx/hub-sessions"; +import { dbGate } from "../../../scripts/e2e/db-gate"; + +import { recordAgentSessionForRun } from "../src/launch/agent-session"; + +function dbConfigFromUrl(databaseUrl: string) { + const url = new URL(databaseUrl); + return { + host: url.hostname, + port: url.port === "" ? 5432 : Number(url.port), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + database: url.pathname.replace(/^\//, ""), + }; +} + +const databaseUrl = process.env["DATABASE_URL"]; +const describeIfDb = dbGate(databaseUrl, import.meta.path); + +describeIfDb("recordAgentSessionForRun", () => { + let db: DB; + + const tenantId = generateId("tenant"); + const principalId = generateId("principal"); + const definitionId = generateId("workflowDefinition"); + const anchorRunId = generateId("workflowRun"); + const sessionId = generateId("session"); + + beforeAll(async () => { + if (databaseUrl === undefined) return; + db = createDB(dbConfigFromUrl(databaseUrl)); + + await db.db.insert(schema.tenant).values({ + id: tenantId, + name: "Agent Session Test Tenant", + slug: `agent-session-${tenantId}`, + domain: `agent-session-${tenantId}.localhost`, + parentId: null, + config: null, + }); + + await db.db.insert(schema.principal).values({ + id: principalId, + tenantId, + kind: "workflow", + refId: anchorRunId, + status: "active", + }); + + await db.db.insert(schema.workflowDefinition).values({ + id: definitionId, + tenantId, + creatorPrincipalId: principalId, + assetId: null, + name: "agent-session-test-definition", + }); + + await db.db.insert(schema.workflowRun).values({ + id: anchorRunId, + definitionId, + anchorRunId, + tenantId, + principalId, + address: `${anchorRunId}@agent-session-${tenantId}.localhost`, + status: "running", + }); + }); + + afterAll(async () => { + if (databaseUrl === undefined) return; + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, anchorRunId)); + await db.db + .delete(schema.workflowDefinition) + .where(eq(schema.workflowDefinition.id, definitionId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, principalId)); + await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); + }); + + test("writes an agent_session resolveRunSessionId reads back", async () => { + await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + + const resolved = await resolveRunSessionId(db.db, principalId); + expect(resolved).toBe(sessionId); + }); +}); From 39ea012bd83690459520821c1e078c6de92c75fb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:15 -0700 Subject: [PATCH 26/48] Native launch records the run's agent_session so mail and spans persist (CL-7477) Before #632 deleted folded-runs, mintFoldedRun was the only writer of agent_session, the table resolveRunSessionId (Interchange's mail-routing lookup, vendor/intx/hub-sessions/src/hub-session-lookups.ts) reads by a run's principal to find its session. Every current native launch path mints its own sessionId and hands it to prepareProvisionedDeployment, but nothing ever wrote it into agent_session -- a run's first turn logged "Endpoint ... has no session for address" and silently dropped its mail. workflow_run.principal_id is still null the instant prepareProvisionedDeployment returns: a provisioned anchor is born "deployed" with no principal, and only the run's first trigger reconciles one onto it (Interchange's anchorWithPrincipal), through the exact same trigger path every sendUserMessage drives. So the write has to land after that first send, not right after provisioning. @corbits/workflows now exports recordAgentSessionForRun, a shared helper that reads a run's tenant/definition/principal fresh off workflow_run and inserts the session row (idempotent, since every send after the first is a no-op). Every call site that sends a message wires it in right after that send: webhook-triggers' launch, agent-directory's one-shot prompt, and chat's sendRunMail (the one chokepoint every chat send -- invite, wake, relaunch -- funnels through, since an invite alone provisions a run with no message and would only ever see a still-null principal). apps/hub's agent-authored-workflow deployer mints a run with no message of its own and its first trigger runs entirely through Interchange's native route with no Workbench hook to record afterward, so it's left alone. Session teardown reuses the same read: chat's relaunch marks the outgoing run's session ended before repointing the room, and the one-shot prompt's teardown ends its own session once its single turn settles. --- apps/hub/src/index.ts | 9 ++ .../agent-directory/src/one-shot-prompt.ts | 51 +++++++-- packages/chat/src/platform-adapter.ts | 10 ++ packages/chat/src/send-run-mail.ts | 21 ++++ packages/webhook-triggers/src/launch.ts | 21 ++++ packages/workflows/src/index.ts | 6 + .../workflows/src/launch/agent-session.ts | 106 ++++++++++++++++++ 7 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 packages/workflows/src/launch/agent-session.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bef0a6d6c..c3352c5a3 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2017,6 +2017,15 @@ export async function createHub(config: HubConfig) { } try { + // Not `recordAgentSessionForRun` here: this mints a deployment + // with no message sent against it — a `workflow_run` born this + // way carries no principal until its first trigger (a call this + // deployer never makes) reconciles one onto it, so the call + // would only ever no-op. That trigger runs entirely through + // Interchange's own native route, with no Workbench-owned hook + // to record the session afterward (CL-7477 tracks the chat, + // webhook, and one-shot-prompt launch paths, which all send a + // message of their own and record there instead). const prepared = await workflowAllocationService.prepareProvisionedDeployment({ tenantId, diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index bf742b2d3..f3084ea66 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -8,8 +8,10 @@ import type { AgentLifecycle } from "@corbits/agent-lifecycle"; import { connectorReplyContent, messageRunEnded } from "@corbits/agent-events"; import { reportError } from "@corbits/error-sink"; import { + endAgentSessionForRun, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -162,6 +164,10 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null until its first trigger + // reconciles one onto it — `runOneShotPrompt` records the session + // right after the opening prompt actually sends. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -260,6 +266,14 @@ export async function runOneShotPrompt( settled = true; clearTimeout(timer); unsubscribe(); + try { + await endAgentSessionForRun(deps.db, launched.runId); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.end-session", + extra: { runId: launched.runId, reason }, + }); + } try { await deps.undeploy(launched.address, reason); } catch (err) { @@ -290,18 +304,33 @@ export async function runOneShotPrompt( content: input.prompt, cryptoProvider, }); - return; + } else { + await deps.sessionService.sendUserMessage({ + agentAddress: launched.address, + from: `${input.principalId}@${tenantRow.domain}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content: input.prompt, + sessionId: launched.sessionId, + tenantId: input.tenantId, + cryptoProvider, + }); + } + // The send above just drove the run's trigger path, the only + // thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now is + // what makes this ever find one. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.recordAgentSession", + extra: { address: launched.address }, + }); } - await deps.sessionService.sendUserMessage({ - agentAddress: launched.address, - from: `${input.principalId}@${tenantRow.domain}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content: input.prompt, - sessionId: launched.sessionId, - tenantId: input.tenantId, - cryptoProvider, - }); } catch (cause) { reportError(cause, { operation: "agent-directory.one-shot.send", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 39e52ff2c..5a432c5d7 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,6 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, + endAgentSessionForPrincipal, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -342,6 +343,11 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null (an invite sits + // un-triggered until someone actually writes into it), so this + // would only ever no-op. `sendRunMail` records it instead, once the + // run's first turn has actually reconciled a principal onto it. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -470,6 +476,10 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); + if (run.principalId !== null) { + await endAgentSessionForPrincipal(deps.db, run.principalId); + } + await repointBinding( deps.db, binding, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index 909fefa09..e79a51e8f 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,5 +1,7 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. +import { recordAgentSessionForRun } from "@corbits/workflows"; +import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; @@ -100,5 +102,24 @@ export async function sendRunMail( const mailId = crypto.randomUUID(); const now = new Date(); const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); + // `sendUserMessage` just drove this run's trigger path, which is the + // only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — so the run is mail-routable + // now if it never was before. Best-effort and idempotent past the + // first successful call; a failure here must not swallow the mail + // that already sent. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: params.sessionId, + address: params.agentAddress, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendRunMail.recordAgentSession", + tenantId: params.tenantId, + agentId: params.agentAddress, + extra: { sessionId: params.sessionId }, + }); + } return recordRunMail(deps, params, mailId, now, rawMIME); } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6f3b3adfa..6fda59774 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,6 +17,7 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -237,6 +238,26 @@ export async function launchWebhookTrigger( }); } + // The delivery attempt above is what drove the run's trigger path, + // the only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now, after + // that attempt, is what makes this call ever find one. Best-effort: + // a run that never became routable stays without a session until its + // first successful send. + try { + await recordAgentSessionForRun(deps.db, { + sessionId, + anchorRunId: prepared.anchorRunId, + }); + } catch (error) { + reportError(error, { + operation: "webhookTriggers.launch.recordAgentSession", + tenantId: trigger.tenantId, + agentId: prepared.deploymentAddress, + extra: { instanceId: prepared.anchorRunId }, + }); + } + return { instanceId: prepared.anchorRunId, triggerAddress: prepared.deploymentAddress, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 92c132180..7eac44dbb 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,12 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + recordAgentSessionForRun, + endAgentSessionForPrincipal, + endAgentSessionForRun, + type RecordAgentSessionParams, +} from "./launch/agent-session"; export { deliverWhenRoutable, isAgentUnreachableError, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts new file mode 100644 index 000000000..f95c53960 --- /dev/null +++ b/packages/workflows/src/launch/agent-session.ts @@ -0,0 +1,106 @@ +// Every Workbench launch path provisions through Interchange's +// `prepareProvisionedDeployment` with a `sessionId` it mints itself, but +// nothing ever wrote that id into `agent_session` (vendor-owned, +// `vendor/intx/db/src/schema/sessions.ts`) — the table +// `resolveRunSessionId` (`vendor/intx/hub-sessions/src/hub-session-lookups.ts`) +// reads to route a run's outbound mail. Before the folded-runs package +// was deleted it was the only writer of that table; every current +// launcher must now do this itself, through this one shared helper, so +// mail and spans persist against a real session from the run's first +// turn (CL-7477). +// +// Timing matters: `workflow_run.principal_id` (the FK `agent_session` +// keys on) is still null the instant `prepareProvisionedDeployment` +// returns — a provisioned anchor is born "deployed" with no principal, +// and only the run's first trigger reconciles one onto it (Interchange's +// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`), +// through the exact same trigger path every `sendUserMessage` drives. So +// this cannot run right after provisioning (an invite sits un-triggered +// until someone actually writes into it) — call it after a send that +// just delivered the run's first turn instead. A run with no principal +// yet is a normal, expected state, not a bug: this returns `false` +// rather than throwing, and idempotent past its first successful call. +import { eq } from "drizzle-orm"; +import type { DB } from "@intx/db"; +import { agentSession, workflowRun } from "@intx/db/schema"; + +export type RecordAgentSessionParams = { + /** The id `prepareProvisionedDeployment` was called with. */ + readonly sessionId: string; +} & ({ readonly anchorRunId: string } | { readonly address: string }); + +/** + * Inserts the `agent_session` row a run needs to be mail-routable: the + * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps + * its old name but targets `workflow_definition` now), and `principalId` + * are all read fresh off the `workflow_run` row, never guessed by the + * caller. Returns `false` (and writes nothing) when the row has no + * principal yet, `true` once the session is recorded (or already was — + * a second call for the same `sessionId` is a no-op, not a conflict). + */ +export async function recordAgentSessionForRun( + db: DB["db"], + params: RecordAgentSessionParams, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: + "anchorRunId" in params + ? eq(workflowRun.id, params.anchorRunId) + : eq(workflowRun.address, params.address), + }); + if (runRow === undefined || runRow.principalId === null) { + return false; + } + + const now = new Date(); + await db + .insert(agentSession) + .values({ + id: params.sessionId, + tenantId: runRow.tenantId, + agentId: runRow.definitionId, + principalId: runRow.principalId, + status: "active", + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: agentSession.id }); + return true; +} + +/** + * Marks a run's `agent_session` ended — called only from the places that + * already know the run died: a chat relaunch (the old run's session) or + * a one-shot prompt's teardown. No sweeper; a run nobody reacts to stays + * `active` until something does. + */ +export async function endAgentSessionForPrincipal( + db: DB["db"], + principalId: string, +): Promise { + const endedAt = new Date(); + await db + .update(agentSession) + .set({ status: "ended", endedAt, updatedAt: endedAt }) + .where(eq(agentSession.principalId, principalId)); +} + +/** + * Same as `endAgentSessionForPrincipal`, keyed by the run's own id + * instead — for a caller (e.g. a one-shot prompt's teardown) that only + * ever held the run id, never minted or looked up its principal. + * No-ops for a run row already gone, or one never triggered (no + * principal, hence no session was ever recorded for it). + */ +export async function endAgentSessionForRun( + db: DB["db"], + anchorRunId: string, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: eq(workflowRun.id, anchorRunId), + }); + if (runRow === undefined || runRow.principalId === null) { + return; + } + await endAgentSessionForPrincipal(db, runRow.principalId); +} From f31e0e100206c1af55908abf698ad40e01302a6d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:01 -0700 Subject: [PATCH 27/48] Add tests for the run's event collector create/abandon wiring --- packages/webhook-triggers/test/launch.test.ts | 3 ++ packages/workflows/test/agent-session.test.ts | 53 ++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 177a5090b..01b3b6785 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -197,6 +197,9 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, + eventCollectors: { + create: () => undefined, + }, }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index a7446adda..3ae50e221 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -15,7 +15,10 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { recordAgentSessionForRun } from "../src/launch/agent-session"; +import { + endAgentSessionForRun, + recordAgentSessionForRun, +} from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -98,9 +101,55 @@ describeIfDb("recordAgentSessionForRun", () => { }); test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + await recordAgentSessionForRun( + db.db, + { sessionId, anchorRunId }, + { create: () => undefined }, + ); const resolved = await resolveRunSessionId(db.db, principalId); expect(resolved).toBe(sessionId); }); + + test("creates the run's event collector with its address/tenant/session/run id", async () => { + const createCalls: unknown[] = []; + const otherSessionId = generateId("session"); + + await recordAgentSessionForRun( + db.db, + { sessionId: otherSessionId, anchorRunId }, + { + create: (...args: unknown[]) => { + createCalls.push(args); + }, + }, + ); + + expect(createCalls).toEqual([ + [ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + tenantId, + otherSessionId, + anchorRunId, + ], + ]); + + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, otherSessionId)); + }); + + test("abandons the run's event collector when the session ends", async () => { + const abandonCalls: string[] = []; + + await endAgentSessionForRun(db.db, anchorRunId, { + abandon: (address: string) => { + abandonCalls.push(address); + }, + }); + + expect(abandonCalls).toEqual([ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + ]); + }); }); From 9260c3a8b06d1f9f227551bb88df0c6573ed7872 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:07 -0700 Subject: [PATCH 28/48] Native launch creates the run's event collector so turns are recorded (CL-7479) --- apps/hub/src/index.ts | 2 + .../agent-directory/src/one-shot-prompt.ts | 20 ++++++--- packages/chat/src/platform-adapter.ts | 7 ++- packages/chat/src/send-run-mail.ts | 18 +++++--- packages/webhook-triggers/src/launch.ts | 14 ++++-- packages/workflows/src/index.ts | 1 + .../workflows/src/launch/agent-session.ts | 45 +++++++++++++++++-- 7 files changed, 85 insertions(+), 22 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index c3352c5a3..1faa70f3a 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2272,6 +2272,7 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, + eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, @@ -3069,6 +3070,7 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, sessionService, + eventCollectors, }, runnerInput, ), diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index f3084ea66..36efb6022 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -13,6 +13,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -57,6 +58,7 @@ export type OneShotRunnerDeps = { "prepareProvisionedDeployment" >; readonly sessionService: Pick; + readonly eventCollectors: EventCollectorPort; /** * Test seam only. Production never sets these; they default to * Interchange `prepareProvisionedDeployment` and `sendUserMessage`. @@ -267,7 +269,11 @@ export async function runOneShotPrompt( clearTimeout(timer); unsubscribe(); try { - await endAgentSessionForRun(deps.db, launched.runId); + await endAgentSessionForRun( + deps.db, + launched.runId, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.end-session", @@ -321,10 +327,14 @@ export async function runOneShotPrompt( // provisioned `workflow_run` (CL-7477) — recording only now is // what makes this ever find one. try { - await recordAgentSessionForRun(deps.db, { - sessionId: launched.sessionId, - anchorRunId: launched.runId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.recordAgentSession", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 5a432c5d7..8b820493d 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,7 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, - endAgentSessionForPrincipal, + endAgentSessionForRun, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -258,6 +258,7 @@ export function createHubChatPlatform( db: deps.db, sessionService: deps.sessionService, sidecarRouter: deps.sidecarRouter, + eventCollectors: deps.eventCollectors, }; function offeringDigest(sourceOfferingIds: readonly string[]): string { @@ -476,9 +477,7 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); - if (run.principalId !== null) { - await endAgentSessionForPrincipal(deps.db, run.principalId); - } + await endAgentSessionForRun(deps.db, run.id, deps.eventCollectors); await repointBinding( deps.db, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index e79a51e8f..b570a22d2 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,6 +1,9 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. -import { recordAgentSessionForRun } from "@corbits/workflows"; +import { + recordAgentSessionForRun, + type EventCollectorPort, +} from "@corbits/workflows"; import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; @@ -11,6 +14,7 @@ export type RunMailDeps = { db: DB["db"]; sessionService: Pick; sidecarRouter: Pick; + eventCollectors: Pick; }; export type SendRunMailParams = { @@ -109,10 +113,14 @@ export async function sendRunMail( // first successful call; a failure here must not swallow the mail // that already sent. try { - await recordAgentSessionForRun(deps.db, { - sessionId: params.sessionId, - address: params.agentAddress, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: params.sessionId, + address: params.agentAddress, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "chat.sendRunMail.recordAgentSession", diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6fda59774..22e686d40 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -19,6 +19,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -58,6 +59,7 @@ export type LaunchWebhookTriggerDeps = { */ isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; + eventCollectors: Pick; /** * Host-supplied cipher. Interchange decrypts bindings inside * `prepareProvisionedDeployment`; this port still owns the field so @@ -245,10 +247,14 @@ export async function launchWebhookTrigger( // a run that never became routable stays without a session until its // first successful send. try { - await recordAgentSessionForRun(deps.db, { - sessionId, - anchorRunId: prepared.anchorRunId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId, + anchorRunId: prepared.anchorRunId, + }, + deps.eventCollectors, + ); } catch (error) { reportError(error, { operation: "webhookTriggers.launch.recordAgentSession", diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 7eac44dbb..244d0478a 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -60,6 +60,7 @@ export { endAgentSessionForPrincipal, endAgentSessionForRun, type RecordAgentSessionParams, + type EventCollectorPort, } from "./launch/agent-session"; export { deliverWhenRoutable, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index f95c53960..58ec8fd40 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -23,12 +23,27 @@ import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { agentSession, workflowRun } from "@intx/db/schema"; +import type { EventCollectorRegistry } from "@intx/hub-sessions"; export type RecordAgentSessionParams = { /** The id `prepareProvisionedDeployment` was called with. */ readonly sessionId: string; } & ({ readonly anchorRunId: string } | { readonly address: string }); +/** + * The one port every launcher threads the same wrapped + * `EventCollectorRegistry` through (`apps/hub/src/index.ts`'s + * `eventCollectors`) — never a second registry construction. `create` + * is what actually makes `inference_turn`/`turn_part` rows exist for a + * run (CL-7479: nothing else ever called it once `folded-runs` was + * deleted); `abandon` tears the collector down when a run's session + * ends. + */ +export type EventCollectorPort = Pick< + EventCollectorRegistry, + "create" | "abandon" +>; + /** * Inserts the `agent_session` row a run needs to be mail-routable: the * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps @@ -41,6 +56,7 @@ export type RecordAgentSessionParams = { export async function recordAgentSessionForRun( db: DB["db"], params: RecordAgentSessionParams, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: @@ -65,6 +81,19 @@ export async function recordAgentSessionForRun( updatedAt: now, }) .onConflictDoNothing({ target: agentSession.id }); + + const agentAddress = "address" in params ? params.address : runRow.address; + if (agentAddress !== null) { + // Idempotent across wakes: a collector already live for this + // address is replaced, matching the registry's own contract + // (`event-collector-registry.ts`'s `create`). + eventCollectors.create( + agentAddress, + runRow.tenantId, + params.sessionId, + runRow.id, + ); + } return true; } @@ -87,14 +116,19 @@ export async function endAgentSessionForPrincipal( /** * Same as `endAgentSessionForPrincipal`, keyed by the run's own id - * instead — for a caller (e.g. a one-shot prompt's teardown) that only - * ever held the run id, never minted or looked up its principal. - * No-ops for a run row already gone, or one never triggered (no - * principal, hence no session was ever recorded for it). + * instead — for a caller (e.g. a one-shot prompt's teardown, or a chat + * relaunch replacing a terminal run) that only ever held the run id, + * never minted or looked up its principal. Also abandons the outgoing + * run's event collector, through the same shared + * `EventCollectorPort` every launcher threads — one mechanism for + * ending a run's session. No-ops for a run row already gone, or one + * never triggered (no principal, hence no session was ever recorded + * for it). */ export async function endAgentSessionForRun( db: DB["db"], anchorRunId: string, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, anchorRunId), @@ -103,4 +137,7 @@ export async function endAgentSessionForRun( return; } await endAgentSessionForPrincipal(db, runRow.principalId); + if (runRow.address !== null) { + eventCollectors.abandon(runRow.address); + } } From f6369d9a0ed09bb08be2ecb7d27d8ee8d96b518e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:42:35 -0700 Subject: [PATCH 29/48] Add tests for lazy, idempotent run-session ensure (CL-7480) --- packages/workflows/test/agent-session.test.ts | 142 +++++++++++------- 1 file changed, 86 insertions(+), 56 deletions(-) diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index 3ae50e221..de783406f 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,11 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477: before this fix, nothing wrote `agent_session` for a native -// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could -// never find a session for a run's principal and every launch's first -// turn failed to persist mail. This proves `recordAgentSessionForRun` -// writes a row `resolveRunSessionId` actually reads back. +// CL-7477/CL-7480: nothing writes `agent_session` for a native launch +// until something calls `ensureRunSession` — this proves it writes a +// row `resolveRunSessionId` (vendor's mail-routing lookup) actually +// reads back, only once the run is anchored with a principal, and stays +// idempotent (one insert, one collector) across repeated calls. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -15,10 +15,7 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { - endAgentSessionForRun, - recordAgentSessionForRun, -} from "../src/launch/agent-session"; +import { ensureRunSession } from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -34,14 +31,16 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("recordAgentSessionForRun", () => { +describeIfDb("ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); const principalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchorRunId = generateId("workflowRun"); + const anchoredRunId = generateId("workflowRun"); + const unanchoredRunId = generateId("workflowRun"); const sessionId = generateId("session"); + const domain = `agent-session-${tenantId}.localhost`; beforeAll(async () => { if (databaseUrl === undefined) return; @@ -51,7 +50,7 @@ describeIfDb("recordAgentSessionForRun", () => { id: tenantId, name: "Agent Session Test Tenant", slug: `agent-session-${tenantId}`, - domain: `agent-session-${tenantId}.localhost`, + domain, parentId: null, config: null, }); @@ -60,7 +59,7 @@ describeIfDb("recordAgentSessionForRun", () => { id: principalId, tenantId, kind: "workflow", - refId: anchorRunId, + refId: anchoredRunId, status: "active", }); @@ -72,13 +71,38 @@ describeIfDb("recordAgentSessionForRun", () => { name: "agent-session-test-definition", }); + // Anchored: has a principal, so `ensureRunSession` can insert its + // `agent_session` row. await db.db.insert(schema.workflowRun).values({ - id: anchorRunId, + id: anchoredRunId, definitionId, - anchorRunId, + anchorRunId: anchoredRunId, tenantId, principalId, - address: `${anchorRunId}@agent-session-${tenantId}.localhost`, + address: `${anchoredRunId}@${domain}`, + status: "running", + }); + await db.db.insert(schema.workflowRunLaunchSpec).values({ + anchorRunId: anchoredRunId, + sessionId, + deploymentDomain: domain, + sourceAuthorityPrincipalId: principalId, + frozenApprovalBundle: {}, + sourceOfferingIds: [], + defaultSourceOfferingId: "off_test", + deployContent: { systemPrompt: "" }, + }); + + // Unanchored: `prepareProvisionedDeployment` minted the row, but no + // trigger has reconciled a principal onto it yet — the normal state + // for a freshly provisioned invite. + await db.db.insert(schema.workflowRun).values({ + id: unanchoredRunId, + definitionId, + anchorRunId: unanchoredRunId, + tenantId, + principalId: null, + address: `${unanchoredRunId}@${domain}`, status: "running", }); }); @@ -88,9 +112,15 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db .delete(schema.agentSession) .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRunLaunchSpec) + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, anchoredRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchorRunId)); + .where(eq(schema.workflowRun.id, anchoredRunId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, unanchoredRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); @@ -100,56 +130,56 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun( - db.db, - { sessionId, anchorRunId }, - { create: () => undefined }, - ); - - const resolved = await resolveRunSessionId(db.db, principalId); - expect(resolved).toBe(sessionId); - }); - - test("creates the run's event collector with its address/tenant/session/run id", async () => { + test("an unanchored run returns null and writes nothing", async () => { const createCalls: unknown[] = []; - const otherSessionId = generateId("session"); - - await recordAgentSessionForRun( - db.db, - { sessionId: otherSessionId, anchorRunId }, - { + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: (...args: unknown[]) => { createCalls.push(args); }, + has: () => false, }, - ); - - expect(createCalls).toEqual([ - [ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - tenantId, - otherSessionId, - anchorRunId, - ], - ]); + runId: unanchoredRunId, + }); - await db.db - .delete(schema.agentSession) - .where(eq(schema.agentSession.id, otherSessionId)); + expect(resolved).toBeNull(); + expect(createCalls).toEqual([]); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.agentId, definitionId), + }); + expect(sessionRow).toBeUndefined(); }); - test("abandons the run's event collector when the session ends", async () => { - const abandonCalls: string[] = []; - - await endAgentSessionForRun(db.db, anchorRunId, { - abandon: (address: string) => { - abandonCalls.push(address); + test("an anchored run inserts its session and creates its collector once across two calls", async () => { + const createCalls: unknown[] = []; + let hasCollector = false; + const eventCollectors = { + create: (...args: unknown[]) => { + createCalls.push(args); + hasCollector = true; }, + has: () => hasCollector, + }; + + const first = await ensureRunSession({ + db: db.db, + eventCollectors, + runId: anchoredRunId, + }); + const second = await ensureRunSession({ + db: db.db, + eventCollectors, + runId: anchoredRunId, }); - expect(abandonCalls).toEqual([ - `${anchorRunId}@agent-session-${tenantId}.localhost`, + expect(first).toBe(sessionId); + expect(second).toBe(sessionId); + expect(createCalls).toEqual([ + [`${anchoredRunId}@${domain}`, tenantId, sessionId, anchoredRunId], ]); + + const resolved = await resolveRunSessionId(db.db, principalId); + expect(resolved).toBe(sessionId); }); }); From d6eda397f142b9fb94403173a1cee8d69f6fa3e8 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:42:41 -0700 Subject: [PATCH 30/48] Hub ensures a run's session and collector on first use (CL-7480) Replaces the post-send recordAgentSessionForRun (CL-7477/CL-7479) with one idempotent ensureRunSession, called lazily at the hub seams that actually see a run become mail-routable: the mailbox persist wrapper and the event-collector dispatch path, plus chat's pre-send mail dispatch. Pre-creating the run's principal to dodge the timing isn't an option -- Interchange's grant materialization treats a conflict on that insert as grants already committed and throws. --- apps/hub/src/index.ts | 60 +++++++++++- apps/hub/src/mailbox-persist.ts | 45 +++++++++ .../agent-directory/src/one-shot-prompt.ts | 31 ++---- packages/chat/src/platform-adapter.ts | 28 +++++- packages/chat/src/send-run-mail.ts | 34 +------ packages/webhook-triggers/src/launch.ts | 31 +----- packages/webhook-triggers/test/launch.test.ts | 21 ---- packages/workflows/src/index.ts | 3 +- .../workflows/src/launch/agent-session.ts | 97 ++++++++++--------- 9 files changed, 198 insertions(+), 152 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 1faa70f3a..bd04767ec 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -168,6 +168,7 @@ import { import { createHubMailboxAuthorizeSender, createHubMailboxResolveRefs, + createHubPersistMailWithSessionEnsure, } from "./mailbox-persist"; import { createCommandRegistry, @@ -196,6 +197,7 @@ import { createScheduledWorkflowRoutes, renderWorkflowSourceTree, WORKFLOW_SOURCE_ENTRY, + ensureRunSession, } from "@corbits/workflows"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -219,6 +221,7 @@ import { createWorkflowAllocationService, createWorkflowDispatchService, DEFAULT_ASSET_REF, + resolveRoutableAddress, WorkflowProvisioningError, type AgentRepoStore, type EventCollectorRegistry, @@ -713,6 +716,15 @@ export async function createHub(config: HubConfig) { args: Parameters[0], ) => Promise; } = {}; + // Forward reference: `eventCollectors` (the wrapped + // `EventCollectorRegistry`) isn't constructed until later in this + // composition, but `createHubPersistMailWithSessionEnsure` below needs + // it. Set once `eventCollectors` exists; every persistMail call before + // that point (there are none — the server isn't serving requests yet) + // would just no-op. + const eventCollectorsRef: { + current?: Pick; + } = {}; // CL-6499 (native multi-step routines): materializes a mail-triggered // run's authorization grants from its deploy-approved snapshot, so // ANY plain mail delivered to a workflow deployment's address — not @@ -746,7 +758,11 @@ export async function createHub(config: HubConfig) { // workbench ref is present before the post-commit bus event fires -- // no out-of-band UPDATE, no polling read. persistMail: createMailboxPersist(mailboxDb, { - upstream: baseLookups.persistMail, + upstream: createHubPersistMailWithSessionEnsure( + db, + eventCollectorsRef, + baseLookups.persistMail, + ), authorizeSender: createHubMailboxAuthorizeSender(db), bus: mailboxBus, resolveRefs: createHubMailboxResolveRefs(chatStore, roomMessages), @@ -885,6 +901,17 @@ export async function createHub(config: HubConfig) { baseEventCollectors.create(agentAddress, tenantId, sessionId, runId); }, dispatch(agentAddress, event) { + // CL-7480: a run's turn events can start arriving before anything + // else ever recorded its session — the first inbound trigger that + // just reconciled its principal races this same dispatch. No live + // collector for the address is exactly that case: ensure the + // session (and, inside it, the collector) before delegating, + // rather than silently dropping the event for a collector that + // never gets created. + if (!baseEventCollectors.has(agentAddress)) { + void ensureCollectorThenDispatch(agentAddress, event); + return; + } turnLatency.onEvent(agentAddress, event); baseEventCollectors.dispatch(agentAddress, event); // Mirrors the registry's own `isTerminal` check (event-collector-registry.ts) @@ -901,6 +928,36 @@ export async function createHub(config: HubConfig) { baseEventCollectors.abandon(agentAddress); }, }; + eventCollectorsRef.current = eventCollectors; + // Named so `dispatch`'s no-collector branch above reads clearly; not + // inlined there because it needs to be `async` and `dispatch` itself + // must stay synchronous (the `EventCollectorRegistry` contract). + async function ensureCollectorThenDispatch( + agentAddress: string, + event: Parameters[1], + ): Promise { + try { + const run = await resolveRoutableAddress(db, agentAddress); + if (run !== undefined) { + await ensureRunSession({ + db, + eventCollectors: baseEventCollectors, + runId: run.id, + }); + } + } catch (err) { + reportError(err, { + operation: "hub.eventCollectors.ensureRunSession", + extra: { agentAddress }, + }); + } + turnLatency.onEvent(agentAddress, event); + baseEventCollectors.dispatch(agentAddress, event); + const isTerminal = + event.type === "reactor.done" || + (event.type === "reactor.error" && event.data.fatal); + if (isTerminal) turnLatency.onSessionEnd(agentAddress); + } createHubSessionOrchestrator({ events: sidecarRouter.events, router: sidecarRouter, @@ -2272,7 +2329,6 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, - eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index 827dad17e..f8aa0e2f0 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -16,6 +16,7 @@ import { tenant as tenantTable } from "@intx/db/schema"; import type { AuthorizeMailboxSender, CreateMailboxPersistOpts, + MailboxPersistArgs, MailboxRef, } from "@corbits/mailbox"; import { resolveRoutableAddress } from "@intx/hub-sessions"; @@ -24,6 +25,8 @@ import { type ChatStore, type RoomMessageStore, } from "@corbits/chat"; +import { ensureRunSession, type EventCollectorPort } from "@corbits/workflows"; +import { reportError } from "@corbits/error-sink"; /** * `@corbits/mailbox` does not export `ResolveMailboxRefs` itself (only the @@ -57,6 +60,48 @@ type ResolveMailboxRefs = NonNullable< * fact from two directions: one extra indexed lookup by address per frame, * not per recipient. */ +/** + * Wraps the hub's own `persistMail` (vendor's `baseLookups.persistMail`) + * so a run's `agent_session` and event collector exist before that write + * runs — CL-7480: the first inbound mail on a freshly triggered run is + * often the earliest point the run is mail-routable at all, since + * `workflow_run.principal_id` only reconciles onto the trigger that just + * fired. Resolves the frame's own sender address to its run + * (`resolveRoutableAddress`, the same resolver `persistMail` itself + * uses) rather than assuming the caller already knows it; a sender that + * is not a live run resolves to `undefined` and this is a no-op past + * that point. Best-effort: `ensureRunSession` failing must not swallow + * the mail upstream is about to persist regardless, so this only + * degrades to "no session yet" rather than ever throwing past itself. + * + * `eventCollectorsRef` is a forward reference (the same pattern this + * file's other composition-order refs use, e.g. `apps/hub/src/index.ts`'s + * `grantAllowanceGateRef`): the wrapped `eventCollectors` registry isn't + * constructed until after `lookups` is, so this only reads `.current` + * inside the returned closure, never at wiring time. + */ +export function createHubPersistMailWithSessionEnsure( + db: DB["db"], + eventCollectorsRef: { current?: Pick }, + upstream: (args: MailboxPersistArgs) => Promise, +): (args: MailboxPersistArgs) => Promise { + return async (args) => { + try { + const eventCollectors = eventCollectorsRef.current; + const sender = await resolveRoutableAddress(db, args.senderAddress); + if (sender !== undefined && eventCollectors !== undefined) { + await ensureRunSession({ db, eventCollectors, runId: sender.id }); + } + } catch (err) { + reportError(err, { + operation: "hub.mailboxPersist.ensureRunSession", + extra: { senderAddress: args.senderAddress }, + }); + } + return upstream(args); + }; +} + export function createHubMailboxAuthorizeSender( db: DB["db"], ): AuthorizeMailboxSender { diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index 36efb6022..1851527d4 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -11,7 +11,6 @@ import { endAgentSessionForRun, readDefinitionProjection, readFoldedBody, - recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, type EventCollectorPort, } from "@corbits/workflows"; @@ -166,10 +165,10 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // Not `ensureRunSession` here: a freshly provisioned run's // `workflow_run.principal_id` is still null until its first trigger - // reconciles one onto it — `runOneShotPrompt` records the session - // right after the opening prompt actually sends. + // reconciles one onto it, and the hub's own mail/dispatch seams + // ensure the session lazily once that happens (CL-7480). return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -322,25 +321,11 @@ export async function runOneShotPrompt( cryptoProvider, }); } - // The send above just drove the run's trigger path, the only - // thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — recording only now is - // what makes this ever find one. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId: launched.sessionId, - anchorRunId: launched.runId, - }, - deps.eventCollectors, - ); - } catch (err) { - reportError(err, { - operation: "agent-directory.one-shot.recordAgentSession", - extra: { address: launched.address }, - }); - } + // A run's session and event collector are ensured lazily now, at + // the hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped + // `eventCollectors` dispatch in `apps/hub/src/index.ts`) — + // CL-7480. Nothing here needs to record it. } catch (cause) { reportError(cause, { operation: "agent-directory.one-shot.send", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 8b820493d..a546564e7 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -17,6 +17,7 @@ import { authoredDefinitionCandidates, DefinitionProjectionMissingError, endAgentSessionForRun, + ensureRunSession, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -258,7 +259,6 @@ export function createHubChatPlatform( db: deps.db, sessionService: deps.sessionService, sidecarRouter: deps.sidecarRouter, - eventCollectors: deps.eventCollectors, }; function offeringDigest(sourceOfferingIds: readonly string[]): string { @@ -344,11 +344,12 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // Not `ensureRunSession` here: a freshly provisioned run's // `workflow_run.principal_id` is still null (an invite sits // un-triggered until someone actually writes into it), so this - // would only ever no-op. `sendRunMail` records it instead, once the - // run's first turn has actually reconciled a principal onto it. + // would only ever no-op. The hub's own mail/dispatch seams ensure + // the session lazily instead, once the run's first turn has + // actually reconciled a principal onto it (CL-7480). return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -1313,6 +1314,25 @@ export function createHubChatPlatform( // idle sweep the moment they see traffic. lifecycle?.track(deliveryAddress); + // CL-7480: a run just woken/reconciled above can be routable + // before anything ever recorded its session — waking it is not + // itself a mail or dispatch seam, so nothing else along this path + // would ensure one. Best-effort: an ensure failure here still + // falls through to `resolveRunSessionIdOrThrow`'s own error. + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: delivery.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: deliveryAddress, + extra: { runId: delivery.run.id }, + }); + } const sessionId = await resolveRunSessionIdOrThrow(deps.db, delivery.run); const domain = domainOf(deliveryAddress); // `fromWorkbenchId` names the room a dispatch speaks for. A room diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index b570a22d2..139dfd98a 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,10 +1,5 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. -import { - recordAgentSessionForRun, - type EventCollectorPort, -} from "@corbits/workflows"; -import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; @@ -14,7 +9,6 @@ export type RunMailDeps = { db: DB["db"]; sessionService: Pick; sidecarRouter: Pick; - eventCollectors: Pick; }; export type SendRunMailParams = { @@ -106,28 +100,10 @@ export async function sendRunMail( const mailId = crypto.randomUUID(); const now = new Date(); const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); - // `sendUserMessage` just drove this run's trigger path, which is the - // only thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — so the run is mail-routable - // now if it never was before. Best-effort and idempotent past the - // first successful call; a failure here must not swallow the mail - // that already sent. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId: params.sessionId, - address: params.agentAddress, - }, - deps.eventCollectors, - ); - } catch (err) { - reportError(err, { - operation: "chat.sendRunMail.recordAgentSession", - tenantId: params.tenantId, - agentId: params.agentAddress, - extra: { sessionId: params.sessionId }, - }); - } + // A run's session and event collector are ensured lazily now, at the + // hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` + // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs + // to record it. return recordRunMail(deps, params, mailId, now, rawMIME); } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 22e686d40..4a0331aca 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,9 +17,7 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, - recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, - type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -59,7 +57,6 @@ export type LaunchWebhookTriggerDeps = { */ isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; - eventCollectors: Pick; /** * Host-supplied cipher. Interchange decrypts bindings inside * `prepareProvisionedDeployment`; this port still owns the field so @@ -240,29 +237,11 @@ export async function launchWebhookTrigger( }); } - // The delivery attempt above is what drove the run's trigger path, - // the only thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — recording only now, after - // that attempt, is what makes this call ever find one. Best-effort: - // a run that never became routable stays without a session until its - // first successful send. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId, - anchorRunId: prepared.anchorRunId, - }, - deps.eventCollectors, - ); - } catch (error) { - reportError(error, { - operation: "webhookTriggers.launch.recordAgentSession", - tenantId: trigger.tenantId, - agentId: prepared.deploymentAddress, - extra: { instanceId: prepared.anchorRunId }, - }); - } + // A run's session and event collector are ensured lazily now, at the + // hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` + // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs + // to record it. return { instanceId: prepared.anchorRunId, diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 01b3b6785..fa7559086 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,25 +88,7 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, - // `recordAgentSessionForRun` reads the just-provisioned run back by - // id — production never inserts this row itself (Interchange's - // `prepareProvisionedDeployment` mints the anchor row - // transactionally before returning it), so this models the same - // row for `INTERCHANGE_RUN_ID`. - workflowRun: { - findFirst: async () => ({ - id: INTERCHANGE_RUN_ID, - tenantId: TENANT_ROW.id, - definitionId: DEFINITION_ROW.id, - principalId: "prn_interchange", - }), - }, }, - insert: (_table: unknown) => ({ - values: (_values: unknown) => ({ - onConflictDoNothing: async (_opts: unknown) => {}, - }), - }), }; } @@ -197,9 +179,6 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, - eventCollectors: { - create: () => undefined, - }, }; } diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 244d0478a..74828b7b2 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -56,10 +56,9 @@ export { } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; export { - recordAgentSessionForRun, + ensureRunSession, endAgentSessionForPrincipal, endAgentSessionForRun, - type RecordAgentSessionParams, type EventCollectorPort, } from "./launch/agent-session"; export { diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index 58ec8fd40..b6ba1100a 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -13,23 +13,29 @@ // keys on) is still null the instant `prepareProvisionedDeployment` // returns — a provisioned anchor is born "deployed" with no principal, // and only the run's first trigger reconciles one onto it (Interchange's -// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`), -// through the exact same trigger path every `sendUserMessage` drives. So -// this cannot run right after provisioning (an invite sits un-triggered -// until someone actually writes into it) — call it after a send that -// just delivered the run's first turn instead. A run with no principal -// yet is a normal, expected state, not a bug: this returns `false` -// rather than throwing, and idempotent past its first successful call. +// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`). +// Pre-creating the run's principal to dodge this is not an option either: +// Interchange's own grant materialization +// (`vendor/intx/hub-api/src/run-grant-materialization.ts`) inserts the +// principal row `onConflictDoNothing` and treats a conflict as "grants +// already committed", throwing rather than re-materializing — so nothing +// upstream of that first trigger may write the principal first. +// +// So `ensureRunSession` (CL-7480) is lazy and idempotent rather than +// called once right after a send: every seam that might be the run's +// first mail-routable moment (a hub mailbox persist, a session +// orchestrator dispatch with no live collector yet) calls it before +// doing its own work. A run with no principal yet is a normal, expected +// state, not a bug — this returns `null` rather than throwing. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { agentSession, workflowRun } from "@intx/db/schema"; +import { + agentSession, + workflowRun, + workflowRunLaunchSpec, +} from "@intx/db/schema"; import type { EventCollectorRegistry } from "@intx/hub-sessions"; -export type RecordAgentSessionParams = { - /** The id `prepareProvisionedDeployment` was called with. */ - readonly sessionId: string; -} & ({ readonly anchorRunId: string } | { readonly address: string }); - /** * The one port every launcher threads the same wrapped * `EventCollectorRegistry` through (`apps/hub/src/index.ts`'s @@ -37,42 +43,52 @@ export type RecordAgentSessionParams = { * is what actually makes `inference_turn`/`turn_part` rows exist for a * run (CL-7479: nothing else ever called it once `folded-runs` was * deleted); `abandon` tears the collector down when a run's session - * ends. + * ends; `has` lets a caller check whether a collector already exists + * before creating a second one. */ export type EventCollectorPort = Pick< EventCollectorRegistry, - "create" | "abandon" + "create" | "abandon" | "has" >; /** - * Inserts the `agent_session` row a run needs to be mail-routable: the - * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps - * its old name but targets `workflow_definition` now), and `principalId` - * are all read fresh off the `workflow_run` row, never guessed by the - * caller. Returns `false` (and writes nothing) when the row has no - * principal yet, `true` once the session is recorded (or already was — - * a second call for the same `sessionId` is a no-op, not a conflict). + * Idempotently makes a run's session and event collector exist, the + * instant that becomes possible: the run's `tenantId`, `definitionId` + * (the `agent_id` FK), and `principalId` are all read fresh off the + * `workflow_run` row, never guessed by the caller. Returns `null` (and + * writes nothing) when the row has no principal yet — an un-anchored + * run is not a bug, just not ensurable yet. Returns the run's + * `sessionId` once the session exists (freshly inserted or already + * there — a second call for the same run is a no-op, not a conflict), + * with the run's event collector created too if none was live for its + * address. */ -export async function recordAgentSessionForRun( - db: DB["db"], - params: RecordAgentSessionParams, - eventCollectors: Pick, -): Promise { +export async function ensureRunSession(params: { + readonly db: DB["db"]; + readonly eventCollectors: Pick; + readonly runId: string; +}): Promise { + const { db, eventCollectors, runId } = params; const runRow = await db.query.workflowRun.findFirst({ - where: - "anchorRunId" in params - ? eq(workflowRun.id, params.anchorRunId) - : eq(workflowRun.address, params.address), + where: eq(workflowRun.id, runId), }); if (runRow === undefined || runRow.principalId === null) { - return false; + return null; + } + + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, runId), + }); + if (launchSpecRow === undefined) { + return null; } + const sessionId = launchSpecRow.sessionId; const now = new Date(); await db .insert(agentSession) .values({ - id: params.sessionId, + id: sessionId, tenantId: runRow.tenantId, agentId: runRow.definitionId, principalId: runRow.principalId, @@ -82,19 +98,10 @@ export async function recordAgentSessionForRun( }) .onConflictDoNothing({ target: agentSession.id }); - const agentAddress = "address" in params ? params.address : runRow.address; - if (agentAddress !== null) { - // Idempotent across wakes: a collector already live for this - // address is replaced, matching the registry's own contract - // (`event-collector-registry.ts`'s `create`). - eventCollectors.create( - agentAddress, - runRow.tenantId, - params.sessionId, - runRow.id, - ); + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { + eventCollectors.create(runRow.address, runRow.tenantId, sessionId, runRow.id); } - return true; + return sessionId; } /** From 63b2d2e9949557d7c48d1463cc1baf54841c4c19 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:53:48 -0700 Subject: [PATCH 31/48] Fix prettier formatting in agent-session.ts (CL-7480) --- packages/workflows/src/launch/agent-session.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index b6ba1100a..0e2cbff99 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -99,7 +99,12 @@ export async function ensureRunSession(params: { .onConflictDoNothing({ target: agentSession.id }); if (runRow.address !== null && !eventCollectors.has(runRow.address)) { - eventCollectors.create(runRow.address, runRow.tenantId, sessionId, runRow.id); + eventCollectors.create( + runRow.address, + runRow.tenantId, + sessionId, + runRow.id, + ); } return sessionId; } From af790b2ece08c3bcd216c4385c7511e4cb12a850 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:27:06 -0700 Subject: [PATCH 32/48] Add tests for eager session recording and launch-spec session lookup --- packages/chat/test/platform-adapter.test.ts | 134 ++++++++++++- packages/webhook-triggers/test/launch.test.ts | 27 +++ packages/workflows/test/agent-session.test.ts | 178 +++++++++++------- 3 files changed, 269 insertions(+), 70 deletions(-) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 29281d330..9f3b5656a 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -34,6 +34,7 @@ import { workflowDefinition, workflowDefinitionVersion, workflowRun, + workflowRunLaunchSpec, } from "@intx/db/schema"; import { SessionLaunchError } from "@intx/hub-sessions"; import type { EventCollectorRegistry, SidecarRouter } from "@intx/hub-sessions"; @@ -343,6 +344,19 @@ function createFakeDb(opts: { * production code asks for, in any order. */ wireProjectionsByDefinitionId?: Record | undefined; + /** + * The `workflow_run_launch_spec` row `resolveRunSessionIdOrThrow` + * (CL-7481) reads by run id — the session id is fixed at provision + * time, unlike the principal it used to be looked up by. Explicit + * only where a test cares about an un-anchored run's session + * resolving before its principal ever exists; every other test here + * predates CL-7481 and seeds an `agentSession` row directly, so the + * fallback below derives one from that instead of touching every + * call site. + */ + workflowRunLaunchSpecRow?: + | { anchorRunId: string; sessionId: string } + | undefined; }) { const inserted: { table: unknown; values: unknown }[] = []; const updated: { table: unknown; values: unknown }[] = []; @@ -427,13 +441,64 @@ function createFakeDb(opts: { status: "pending", }; } - return inserted.findLast((row) => row.table === workflowRun) - ?.values as typeof opts.workflowRunRow | undefined; + const insertedRow = inserted.findLast( + (row) => row.table === workflowRun, + )?.values as typeof opts.workflowRunRow | undefined; + if (insertedRow !== undefined) return insertedRow; + // A run this fixture never configured: the freshly minted + // `anchorRunId` a fake `prepareProvisionedDeployment` just + // returned, which never wrote a `workflowRun` row into this + // fake db (Interchange's own provisioning is entirely mocked + // out here). `recordAgentSessionAtProvision` (CL-7481) reads + // the run row it just "provisioned" immediately after — this + // synthesizes the generic shape production code would see, + // so that read succeeds without every test wiring one up. + if (id === undefined) return undefined; + const domain = + opts.tenantRow?.domain ?? + opts.workflowRunRow?.address?.split("@")[1] ?? + "ten1.workbench.test"; + return { + id, + tenantId: opts.tenantRow?.id ?? "ten_1", + definitionId: opts.definitionId, + address: `${id}@${domain}`, + principalId: null, + }; }, }, sessionMail: { findFirst: async () => opts.sessionMailRow, }, + agentSession: { + findFirst: async (query?: { where?: unknown }) => { + const [id] = + query?.where !== undefined ? boundStringValues(query.where) : []; + return inserted.findLast( + (row) => + row.table === agentSession && + (id === undefined || + (row.values as { id?: string }).id === id), + )?.values as { id: string; principalId: string } | undefined; + }, + }, + workflowRunLaunchSpec: { + findFirst: async (query?: { where?: unknown }) => { + const [id] = + query?.where !== undefined ? boundStringValues(query.where) : []; + if (opts.workflowRunLaunchSpecRow !== undefined) { + return id === undefined || + id === opts.workflowRunLaunchSpecRow.anchorRunId + ? opts.workflowRunLaunchSpecRow + : undefined; + } + const seededSession = inserted.find( + (row) => row.table === agentSession, + )?.values as { id: string } | undefined; + if (seededSession === undefined) return undefined; + return { anchorRunId: id, sessionId: seededSession.id }; + }, + }, workflowDefinition: { findFirst: async () => opts.workflowDefinitionRow ?? @@ -1020,6 +1085,71 @@ describe("createHubChatPlatform", () => { ); }); + test("sendMail on a freshly provisioned, un-anchored run resolves the launch-spec session id (CL-7481)", async () => { + resolveDefinitionSourcesResult = { + ok: true, + materials: [], + sources: [ + { + id: "off_1", + provider: "anthropic", + baseURL: "https://inference.invalid", + credentialId: "cred_placeholder", + model: "claude-sonnet-5", + }, + ], + defaultSource: "off_1", + }; + + // No principal on the run yet — an invite sits un-triggered until + // its first message. Before CL-7481, `resolveRunSessionIdOrThrow` + // looked the session up by this (null) principal and found nothing; + // now it reads the launch spec `recordAgentSessionAtProvision` + // wrote at provision time, keyed by the run's own id instead. + const db = createFakeDb({ + assetRow: { + tenantId: "ten_1", + creatorPrincipalId: "prin_creator", + name: "workbench-1", + displayName: null, + }, + definitionId: "wfd_workbench1", + workflowRunRow: { + id: "ins_workbench1", + address: "ins_workbench1@ten1.workbench.test", + principalId: null, + }, + workflowRunLaunchSpecRow: { + anchorRunId: "ins_workbench1", + sessionId: "ses_unanchored", + }, + }); + + const sessionService = createFakeSessionService(); + const sidecarRouter = createFakeSidecarRouter(); + + const platform = createPlatform({ + toolGrantsForPins: () => [], + db: db as never, + sessionService, + sidecarRouter, + eventCollectors: createFakeEventCollectors(), + }); + + const sent = await platform.sendMail({ + tenantId: "ten_1", + workbenchId: "ins_workbench1", + principalId: "prin_sender", + content: { content: "hello workbench" }, + }); + + expect(sent.id).toBeTruthy(); + const call = sessionService.sendUserMessageCalls[0] as { + sessionId: string; + }; + expect(call.sessionId).toBe("ses_unanchored"); + }); + test("sendMail signs with the injected crypto cache keyed by workbenchId", async () => { resolveDefinitionSourcesResult = { ok: true, diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index fa7559086..dad471615 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,7 +88,32 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, + workflowRun: { + findFirst: async () => ({ + id: INTERCHANGE_RUN_ID, + tenantId: TENANT_ROW.id, + definitionId: DEFINITION_ROW.id, + address: INTERCHANGE_ADDRESS, + principalId: null, + }), + }, + }, + insert: () => ({ + values: () => ({ + onConflictDoNothing: async () => undefined, + }), + }), + }; +} + +let eventCollectorCreateCalls: unknown[] = []; + +function createFakeEventCollectors() { + return { + create: (...args: unknown[]) => { + eventCollectorCreateCalls.push(args); }, + has: () => false, }; } @@ -179,6 +204,7 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, + eventCollectors: createFakeEventCollectors(), }; } @@ -190,6 +216,7 @@ function resetLaunchSpies() { sendUserMessageCalls = []; cryptoGetKeys = []; reportErrorCalls = []; + eventCollectorCreateCalls = []; visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS]; frozenProjection = INERT_PROJECTION; sendUserMessageImpl = async () => new Uint8Array([1]); diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index de783406f..e26b19aa4 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,16 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477/CL-7480: nothing writes `agent_session` for a native launch -// until something calls `ensureRunSession` — this proves it writes a -// row `resolveRunSessionId` (vendor's mail-routing lookup) actually -// reads back, only once the run is anchored with a principal, and stays -// idempotent (one insert, one collector) across repeated calls. +// CL-7481: `recordAgentSessionAtProvision` is the eager write every +// native launcher makes right after `prepareProvisionedDeployment` +// returns — this proves it lands the `agent_session` row under the +// *deploying* principal (the run's own principal does not exist yet) +// and creates the run's event collector exactly once, idempotently. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this proves the session +// moves onto that principal without ever touching the session id, and +// that a run with no launch spec row is a bug this throws on rather +// than papering over. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -15,7 +20,10 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { ensureRunSession } from "../src/launch/agent-session"; +import { + ensureRunSession, + recordAgentSessionAtProvision, +} from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -31,14 +39,15 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("ensureRunSession", () => { +describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); - const principalId = generateId("principal"); + const deployingPrincipalId = generateId("principal"); + const runPrincipalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchoredRunId = generateId("workflowRun"); - const unanchoredRunId = generateId("workflowRun"); + const provisionedRunId = generateId("workflowRun"); + const noLaunchSpecRunId = generateId("workflowRun"); const sessionId = generateId("session"); const domain = `agent-session-${tenantId}.localhost`; @@ -55,55 +64,65 @@ describeIfDb("ensureRunSession", () => { config: null, }); - await db.db.insert(schema.principal).values({ - id: principalId, - tenantId, - kind: "workflow", - refId: anchoredRunId, - status: "active", - }); + await db.db.insert(schema.principal).values([ + { + id: deployingPrincipalId, + tenantId, + kind: "human", + refId: "usr_test", + status: "active", + }, + { + id: runPrincipalId, + tenantId, + kind: "workflow", + refId: provisionedRunId, + status: "active", + }, + ]); await db.db.insert(schema.workflowDefinition).values({ id: definitionId, tenantId, - creatorPrincipalId: principalId, + creatorPrincipalId: deployingPrincipalId, assetId: null, name: "agent-session-test-definition", }); - // Anchored: has a principal, so `ensureRunSession` can insert its - // `agent_session` row. + // Born the instant `prepareProvisionedDeployment` returns: a run row + // exists, its own principal is still null, and its launch spec names + // the session `recordAgentSessionAtProvision` will record. await db.db.insert(schema.workflowRun).values({ - id: anchoredRunId, + id: provisionedRunId, definitionId, - anchorRunId: anchoredRunId, + anchorRunId: provisionedRunId, tenantId, - principalId, - address: `${anchoredRunId}@${domain}`, - status: "running", + principalId: null, + address: `${provisionedRunId}@${domain}`, + status: "deployed", }); await db.db.insert(schema.workflowRunLaunchSpec).values({ - anchorRunId: anchoredRunId, + anchorRunId: provisionedRunId, sessionId, deploymentDomain: domain, - sourceAuthorityPrincipalId: principalId, + sourceAuthorityPrincipalId: deployingPrincipalId, frozenApprovalBundle: {}, sourceOfferingIds: [], defaultSourceOfferingId: "off_test", deployContent: { systemPrompt: "" }, }); - // Unanchored: `prepareProvisionedDeployment` minted the row, but no - // trigger has reconciled a principal onto it yet — the normal state - // for a freshly provisioned invite. + // A run with no launch spec row at all: a launcher that skipped the + // shared provisioning path — a bug `ensureRunSession` must surface, + // not silently no-op past. await db.db.insert(schema.workflowRun).values({ - id: unanchoredRunId, + id: noLaunchSpecRunId, definitionId, - anchorRunId: unanchoredRunId, + anchorRunId: noLaunchSpecRunId, tenantId, principalId: null, - address: `${unanchoredRunId}@${domain}`, - status: "running", + address: `${noLaunchSpecRunId}@${domain}`, + status: "deployed", }); }); @@ -114,44 +133,26 @@ describeIfDb("ensureRunSession", () => { .where(eq(schema.agentSession.id, sessionId)); await db.db .delete(schema.workflowRunLaunchSpec) - .where(eq(schema.workflowRunLaunchSpec.anchorRunId, anchoredRunId)); + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchoredRunId)); + .where(eq(schema.workflowRun.id, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, unanchoredRunId)); + .where(eq(schema.workflowRun.id, noLaunchSpecRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); await db.db .delete(schema.principal) - .where(eq(schema.principal.id, principalId)); + .where(eq(schema.principal.id, runPrincipalId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, deployingPrincipalId)); await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("an unanchored run returns null and writes nothing", async () => { - const createCalls: unknown[] = []; - const resolved = await ensureRunSession({ - db: db.db, - eventCollectors: { - create: (...args: unknown[]) => { - createCalls.push(args); - }, - has: () => false, - }, - runId: unanchoredRunId, - }); - - expect(resolved).toBeNull(); - expect(createCalls).toEqual([]); - const sessionRow = await db.db.query.agentSession.findFirst({ - where: eq(schema.agentSession.agentId, definitionId), - }); - expect(sessionRow).toBeUndefined(); - }); - - test("an anchored run inserts its session and creates its collector once across two calls", async () => { + test("records the session under the deploying principal and creates the collector once", async () => { const createCalls: unknown[] = []; let hasCollector = false; const eventCollectors = { @@ -162,24 +163,65 @@ describeIfDb("ensureRunSession", () => { has: () => hasCollector, }; - const first = await ensureRunSession({ + await recordAgentSessionAtProvision({ db: db.db, eventCollectors, - runId: anchoredRunId, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, }); - const second = await ensureRunSession({ + // A second call for the same run is a no-op, not a conflict, and + // must not create a second collector. + await recordAgentSessionAtProvision({ db: db.db, eventCollectors, - runId: anchoredRunId, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, }); - expect(first).toBe(sessionId); - expect(second).toBe(sessionId); expect(createCalls).toEqual([ - [`${anchoredRunId}@${domain}`, tenantId, sessionId, anchoredRunId], + [`${provisionedRunId}@${domain}`, tenantId, sessionId, provisionedRunId], ]); - const resolved = await resolveRunSessionId(db.db, principalId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(deployingPrincipalId); + expect(sessionRow?.agentId).toBe(definitionId); + }); + + test("re-keys the session onto the run principal once Interchange anchors it", async () => { + await db.db + .update(schema.workflowRun) + .set({ principalId: runPrincipalId, status: "running" }) + .where(eq(schema.workflowRun.id, provisionedRunId)); + + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => true }, + runId: provisionedRunId, + }); + expect(resolved).toBe(sessionId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(runPrincipalId); + + // Interchange's own mail-routing lookup now finds the session by the + // run's own principal, exactly as it would for a folded run. + const viaVendorLookup = await resolveRunSessionId(db.db, runPrincipalId); + expect(viaVendorLookup).toBe(sessionId); + }); + + test("throws for a run with no launch spec row", async () => { + await expect( + ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => false }, + runId: noLaunchSpecRunId, + }), + ).rejects.toThrow(/no workflow_run_launch_spec/); }); }); From ae8d33b66ba331f0221e010bb72f96c890a9eac5 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:27:10 -0700 Subject: [PATCH 33/48] Record a run's session at provision; re-key it when Interchange anchors the run --- apps/hub/src/index.ts | 28 ++-- apps/hub/src/mailbox-persist.ts | 25 +++- .../agent-directory/src/one-shot-prompt.ts | 12 +- packages/chat/src/platform-adapter.ts | 14 +- packages/chat/src/run-session.ts | 25 ++-- packages/webhook-triggers/src/launch.ts | 23 ++- packages/workflows/src/index.ts | 1 + .../workflows/src/launch/agent-session.ts | 133 ++++++++++++++---- 8 files changed, 194 insertions(+), 67 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bd04767ec..fbf44a094 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -198,6 +198,7 @@ import { renderWorkflowSourceTree, WORKFLOW_SOURCE_ENTRY, ensureRunSession, + recordAgentSessionAtProvision, } from "@corbits/workflows"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -2074,15 +2075,7 @@ export async function createHub(config: HubConfig) { } try { - // Not `recordAgentSessionForRun` here: this mints a deployment - // with no message sent against it — a `workflow_run` born this - // way carries no principal until its first trigger (a call this - // deployer never makes) reconciles one onto it, so the call - // would only ever no-op. That trigger runs entirely through - // Interchange's own native route, with no Workbench-owned hook - // to record the session afterward (CL-7477 tracks the chat, - // webhook, and one-shot-prompt launch paths, which all send a - // message of their own and record there instead). + const sessionId = generateId("session"); const prepared = await workflowAllocationService.prepareProvisionedDeployment({ tenantId, @@ -2095,12 +2088,26 @@ export async function createHub(config: HubConfig) { }, entry, definitionAssetId: assetId, - sessionId: generateId("session"), + sessionId, sourceAuthorityPrincipalId: principalId, sourceOfferingIds, defaultSourceOfferingId, deployContent: { systemPrompt: "" }, }); + // The eager record every native launcher makes right after + // `prepareProvisionedDeployment` returns (CL-7481): this + // deployment mints no opening message of its own, but the + // trigger that eventually reconciles a principal onto it runs + // entirely through Interchange's own native route, with no + // Workbench-owned hook downstream to record the session + // afterward — so this is the only chance to record it at all. + await recordAgentSessionAtProvision({ + db, + eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: principalId, + }); return { deploymentId: prepared.anchorRunId, definitionAssetId: assetId, @@ -2326,6 +2333,7 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, credentialCipher, + eventCollectors, isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index f8aa0e2f0..26aa08748 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -27,6 +27,9 @@ import { } from "@corbits/chat"; import { ensureRunSession, type EventCollectorPort } from "@corbits/workflows"; import { reportError } from "@corbits/error-sink"; +import { getLogger } from "@intx/log"; + +const logger = getLogger(["hub", "mailbox-persist"]); /** * `@corbits/mailbox` does not export `ResolveMailboxRefs` itself (only the @@ -80,15 +83,15 @@ type ResolveMailboxRefs = NonNullable< * constructed until after `lookups` is, so this only reads `.current` * inside the returned closure, never at wiring time. */ -export function createHubPersistMailWithSessionEnsure( +export function createHubPersistMailWithSessionEnsure( db: DB["db"], eventCollectorsRef: { current?: Pick }, upstream: (args: MailboxPersistArgs) => Promise, ): (args: MailboxPersistArgs) => Promise { return async (args) => { + const sender = await resolveRoutableAddress(db, args.senderAddress); try { const eventCollectors = eventCollectorsRef.current; - const sender = await resolveRoutableAddress(db, args.senderAddress); if (sender !== undefined && eventCollectors !== undefined) { await ensureRunSession({ db, eventCollectors, runId: sender.id }); } @@ -98,6 +101,24 @@ export function createHubPersistMailWithSessionEnsure( extra: { senderAddress: args.senderAddress }, }); } + // The vendored `session_mail` ledger (`upstream`, `baseLookups.persistMail`) + // keys on the sender run's principal, which does not exist until the + // run's first trigger anchors one — a run's own first outbound mail can + // reach this wrapper before that happens. `@corbits/mailbox`'s own write + // (this function's caller, `createMailboxPersist`) is keyed on the run + // address instead and runs regardless, so it is already the durable + // record for this frame; skip the vendored delegate rather than let it + // throw "no session for address" past the caller. + if (sender !== undefined && sender.sessionId === null) { + logger.debug( + "skipping vendored persistMail for run {runId}: no session yet", + { runId: sender.id, senderAddress: args.senderAddress }, + ); + // No `session_mail` rows were written for this frame: an empty + // result, not a thrown error, so a caller iterating this like + // `handleMailPersist` does simply emits no `mail.persisted` events. + return [] as unknown as R; + } return upstream(args); }; } diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index 1851527d4..b03351e45 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -11,6 +11,7 @@ import { endAgentSessionForRun, readDefinitionProjection, readFoldedBody, + recordAgentSessionAtProvision, WORKFLOW_SOURCE_ENTRY, type EventCollectorPort, } from "@corbits/workflows"; @@ -165,10 +166,13 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `ensureRunSession` here: a freshly provisioned run's - // `workflow_run.principal_id` is still null until its first trigger - // reconciles one onto it, and the hub's own mail/dispatch seams - // ensure the session lazily once that happens (CL-7480). + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: input.principalId, + }); return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index a546564e7..b7c5c5147 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -19,6 +19,7 @@ import { endAgentSessionForRun, ensureRunSession, readFoldedBody, + recordAgentSessionAtProvision, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; @@ -344,12 +345,13 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `ensureRunSession` here: a freshly provisioned run's - // `workflow_run.principal_id` is still null (an invite sits - // un-triggered until someone actually writes into it), so this - // would only ever no-op. The hub's own mail/dispatch seams ensure - // the session lazily instead, once the run's first turn has - // actually reconciled a principal onto it (CL-7480). + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: input.sourceAuthorityPrincipalId, + }); return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, diff --git a/packages/chat/src/run-session.ts b/packages/chat/src/run-session.ts index 79ffea89c..54a366a33 100644 --- a/packages/chat/src/run-session.ts +++ b/packages/chat/src/run-session.ts @@ -1,19 +1,26 @@ -// Resolves a provisioned run's session id via Interchange's -// shared-principal bridge rather than any name derived from the run id. +// Resolves a provisioned run's session id off its own launch spec +// (`workflow_run_launch_spec.session_id`, written once at provision by +// every native launcher's `recordAgentSessionAtProvision` — CL-7481) +// rather than Interchange's principal-keyed `resolveRunSessionId`. A +// run's session id is fixed at provision time and never changes; its +// principal is not assigned until the run's first trigger anchors one, +// so a principal-keyed lookup returns nothing for exactly the run states +// (freshly provisioned, un-anchored) chat most needs to resolve. +import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { resolveRunSessionId } from "@intx/hub-sessions"; +import { workflowRunLaunchSpec } from "@intx/db/schema"; export async function resolveRunSessionIdOrThrow( db: DB["db"], - run: { principalId: string | null }, + run: { id: string }, ): Promise { - const sessionId = await resolveRunSessionId(db, run.principalId, { - includeEnded: true, + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, run.id), }); - if (sessionId === null) { + if (launchSpecRow === undefined) { throw new Error( - "no agent_session found for this run's principal; provision may not have completed", + `no workflow_run_launch_spec for run "${run.id}"; provision may not have completed`, ); } - return sessionId; + return launchSpecRow.sessionId; } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 4a0331aca..d215dfb1a 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,7 +17,9 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, + recordAgentSessionAtProvision, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -47,6 +49,13 @@ export type LaunchWebhookTriggerDeps = { "prepareProvisionedDeployment" >; sessionService: Pick; + /** + * The same wrapped `EventCollectorRegistry` every native launcher + * threads through (`apps/hub/src/index.ts`'s `eventCollectors`) — this + * launcher records the run's `agent_session` and event collector the + * instant `prepareProvisionedDeployment` returns (CL-7481). + */ + eventCollectors: Pick; /** * Reads the hub's live sidecar routing table. A freshly provisioned * run's sidecar takes several seconds to boot and register (CL-7476) @@ -196,6 +205,14 @@ export async function launchWebhookTrigger( : {}), }); + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: trigger.createdBy, + }); + await deps.persistLaunch({ tenantId: trigger.tenantId, instanceId: prepared.anchorRunId, @@ -237,12 +254,6 @@ export async function launchWebhookTrigger( }); } - // A run's session and event collector are ensured lazily now, at the - // hub seams that actually see the run become mail-routable - // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` - // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs - // to record it. - return { instanceId: prepared.anchorRunId, triggerAddress: prepared.deploymentAddress, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 74828b7b2..eefa55a85 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -57,6 +57,7 @@ export { export { scheduleCronFromProjection } from "./schedule/from-projection"; export { ensureRunSession, + recordAgentSessionAtProvision, endAgentSessionForPrincipal, endAgentSessionForRun, type EventCollectorPort, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index 0e2cbff99..f73c2a0d0 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -21,12 +21,21 @@ // already committed", throwing rather than re-materializing — so nothing // upstream of that first trigger may write the principal first. // -// So `ensureRunSession` (CL-7480) is lazy and idempotent rather than -// called once right after a send: every seam that might be the run's -// first mail-routable moment (a hub mailbox persist, a session -// orchestrator dispatch with no live collector yet) calls it before -// doing its own work. A run with no principal yet is a normal, expected -// state, not a bug — this returns `null` rather than throwing. +// CL-7480 made `ensureRunSession` lazy: called from every seam that +// might be a run's first mail-routable moment, no-opping until a +// principal existed to key the row on. That left a real gap — a run's +// very first outbound message can itself be that seam, both for chat's +// `sendMail` (which looks the session up by run principal) and for the +// vendored `persistMail` a heartbeat run's own first outbound mail hits +// — and both landed on an un-anchored run with nothing recorded yet. +// CL-7481 fixes this at the root: `recordAgentSessionAtProvision` writes +// the row immediately after `prepareProvisionedDeployment` returns, +// keyed on the deploying principal (`sourceAuthorityPrincipalId`) since +// the run principal does not exist yet — every other field the launcher +// already knows or can read straight off the fresh run row. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this moves `principal_id` +// onto that principal without ever touching the session id. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { @@ -52,37 +61,37 @@ export type EventCollectorPort = Pick< >; /** - * Idempotently makes a run's session and event collector exist, the - * instant that becomes possible: the run's `tenantId`, `definitionId` - * (the `agent_id` FK), and `principalId` are all read fresh off the - * `workflow_run` row, never guessed by the caller. Returns `null` (and - * writes nothing) when the row has no principal yet — an un-anchored - * run is not a bug, just not ensurable yet. Returns the run's - * `sessionId` once the session exists (freshly inserted or already - * there — a second call for the same run is a no-op, not a conflict), - * with the run's event collector created too if none was live for its - * address. + * The one shared write every native launcher calls right after its own + * `prepareProvisionedDeployment` returns (CL-7481) — a run row exists + * the instant that call resolves, so the session it will use is known + * then too: the launch-spec `sessionId` it just minted, `tenantId` and + * `definitionId` read fresh off the new `workflow_run` row (Interchange + * resolves the asset id the caller passed to a real `workflow_definition` + * row internally; callers never see that id themselves), and the + * deploying principal (`sourceAuthorityPrincipalId`) standing in for the + * run's own principal, which does not exist yet. `onConflictDoNothing` + * makes a second call for the same run id a no-op. Every launcher threads + * the same wrapped `EventCollectorRegistry` through here so the run's + * `inference_turn`/`turn_part` rows have somewhere to land from its very + * first turn. */ -export async function ensureRunSession(params: { +export async function recordAgentSessionAtProvision(params: { readonly db: DB["db"]; readonly eventCollectors: Pick; readonly runId: string; -}): Promise { - const { db, eventCollectors, runId } = params; + readonly sessionId: string; + readonly sourceAuthorityPrincipalId: string; +}): Promise { + const { db, eventCollectors, runId, sessionId, sourceAuthorityPrincipalId } = + params; const runRow = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, runId), }); - if (runRow === undefined || runRow.principalId === null) { - return null; - } - - const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ - where: eq(workflowRunLaunchSpec.anchorRunId, runId), - }); - if (launchSpecRow === undefined) { - return null; + if (runRow === undefined) { + throw new Error( + `recordAgentSessionAtProvision: no workflow_run "${runId}" — prepareProvisionedDeployment must have already returned`, + ); } - const sessionId = launchSpecRow.sessionId; const now = new Date(); await db @@ -91,13 +100,77 @@ export async function ensureRunSession(params: { id: sessionId, tenantId: runRow.tenantId, agentId: runRow.definitionId, - principalId: runRow.principalId, + principalId: sourceAuthorityPrincipalId, status: "active", createdAt: now, updatedAt: now, }) .onConflictDoNothing({ target: agentSession.id }); + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { + eventCollectors.create( + runRow.address, + runRow.tenantId, + sessionId, + runRow.id, + ); + } +} + +/** + * Re-keys a run's `agent_session` onto the run's own principal once + * Interchange anchors one (`anchorWithPrincipal`, the run's first + * trigger) — before that, the session row is recorded under the + * deploying principal by `recordAgentSessionAtProvision`, since the run + * principal does not exist yet. The session id never changes; only + * `principal_id` moves, and only once (a session already re-keyed to the + * current run principal is left alone). The launch-spec row is the + * source of truth for which session belongs to this run — its absence + * means a launcher provisioned this run without going through the + * shared path, which is a bug, not a state to paper over. Also creates + * the run's event collector if none is live for its address (a fresh + * process has no in-memory collectors regardless of what is on disk). + */ +export async function ensureRunSession(params: { + readonly db: DB["db"]; + readonly eventCollectors: Pick; + readonly runId: string; +}): Promise { + const { db, eventCollectors, runId } = params; + const runRow = await db.query.workflowRun.findFirst({ + where: eq(workflowRun.id, runId), + }); + if (runRow === undefined) { + return null; + } + + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, runId), + }); + if (launchSpecRow === undefined) { + throw new Error( + `ensureRunSession: no workflow_run_launch_spec for run "${runId}" — every launcher records one at provision time`, + ); + } + const sessionId = launchSpecRow.sessionId; + + if (runRow.principalId !== null) { + const sessionRow = await db.query.agentSession.findFirst({ + where: eq(agentSession.id, sessionId), + }); + if (sessionRow === undefined) { + throw new Error( + `ensureRunSession: no agent_session "${sessionId}" for run "${runId}" — recordAgentSessionAtProvision should have created it at launch`, + ); + } + if (sessionRow.principalId !== runRow.principalId) { + await db + .update(agentSession) + .set({ principalId: runRow.principalId, updatedAt: new Date() }) + .where(eq(agentSession.id, sessionId)); + } + } + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { eventCollectors.create( runRow.address, From 2b5dd7a608ad2a654cff7a558d1d4b79950384fe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:39:42 -0700 Subject: [PATCH 34/48] Fix CI: prettier formatting, principal enum value, and workflowRunLaunchSpec fakes --- apps/hub/src/mailbox-persist.ts | 4 +++- packages/chat/test/platform-adapter.test.ts | 7 ++----- packages/chat/test/relaunch-close.test.ts | 16 ++++++++++++++++ packages/workflows/test/agent-session.test.ts | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index 26aa08748..6d37382a6 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -83,7 +83,9 @@ type ResolveMailboxRefs = NonNullable< * constructed until after `lookups` is, so this only reads `.current` * inside the returned closure, never at wiring time. */ -export function createHubPersistMailWithSessionEnsure( +export function createHubPersistMailWithSessionEnsure< + R extends readonly unknown[], +>( db: DB["db"], eventCollectorsRef: { current?: Pick }, upstream: (args: MailboxPersistArgs) => Promise, diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 9f3b5656a..120538c92 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -34,7 +34,6 @@ import { workflowDefinition, workflowDefinitionVersion, workflowRun, - workflowRunLaunchSpec, } from "@intx/db/schema"; import { SessionLaunchError } from "@intx/hub-sessions"; import type { EventCollectorRegistry, SidecarRouter } from "@intx/hub-sessions"; @@ -355,8 +354,7 @@ function createFakeDb(opts: { * call site. */ workflowRunLaunchSpecRow?: - | { anchorRunId: string; sessionId: string } - | undefined; + { anchorRunId: string; sessionId: string } | undefined; }) { const inserted: { table: unknown; values: unknown }[] = []; const updated: { table: unknown; values: unknown }[] = []; @@ -477,8 +475,7 @@ function createFakeDb(opts: { return inserted.findLast( (row) => row.table === agentSession && - (id === undefined || - (row.values as { id?: string }).id === id), + (id === undefined || (row.values as { id?: string }).id === id), )?.values as { id: string; principalId: string } | undefined; }, }, diff --git a/packages/chat/test/relaunch-close.test.ts b/packages/chat/test/relaunch-close.test.ts index 4e85d0626..35f2c0627 100644 --- a/packages/chat/test/relaunch-close.test.ts +++ b/packages/chat/test/relaunch-close.test.ts @@ -78,6 +78,22 @@ function createFakeDb(opts: { ); }, }, + // CL-7481: `resolveRunSessionIdOrThrow` reads the launch spec by + // run id instead of walking through the run's principal — the + // session id it returns still matches this fixture's + // `ses_` convention, since every run here is already + // "anchored" with a fixed principal from the moment it's defined. + workflowRunLaunchSpec: { + findFirst: async ({ where }: { where: unknown }) => { + const [anchorRunId] = comparedValues(where); + const run = opts.runs.find((row) => row.id === anchorRunId); + if (run === undefined) return undefined; + return { + anchorRunId, + sessionId: `ses_${run.principalId ?? ""}`, + }; + }, + }, }, select: () => ({ from: (table: unknown) => ({ diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index e26b19aa4..6fe8dcfb6 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -68,7 +68,7 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { { id: deployingPrincipalId, tenantId, - kind: "human", + kind: "user", refId: "usr_test", status: "active", }, From c57dd9d75327e738894292a0bb006f5c7d607c17 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:26 -0700 Subject: [PATCH 35/48] Fix merge fallout: restore CL-7481 agent-session exports and drop a duplicate key Merging cl-7107's repin/re-vendor audit into this branch left packages/workflows/src/index.ts re-exporting the pre-CL-7481 agent-session API (recordAgentSessionForRun) that no longer exists, its matching test file testing that same stale API, and a duplicated eventCollectors key in apps/hub/src/index.ts's webhook-trigger launch wiring. Restores the current exports (recordAgentSessionAtProvision, ensureRunSession) and their test, and drops the duplicate object key. --- apps/hub/src/index.ts | 1 - packages/workflows/src/index.ts | 4 +- packages/workflows/test/agent-session.test.ts | 204 ++++++++++++------ 3 files changed, 140 insertions(+), 69 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index b62dbcb1f..fbf44a094 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2337,7 +2337,6 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, - eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 244d0478a..64bfbdff9 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -56,10 +56,10 @@ export { } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; export { - recordAgentSessionForRun, + recordAgentSessionAtProvision, + ensureRunSession, endAgentSessionForPrincipal, endAgentSessionForRun, - type RecordAgentSessionParams, type EventCollectorPort, } from "./launch/agent-session"; export { diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index 3ae50e221..6fe8dcfb6 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,16 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477: before this fix, nothing wrote `agent_session` for a native -// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could -// never find a session for a run's principal and every launch's first -// turn failed to persist mail. This proves `recordAgentSessionForRun` -// writes a row `resolveRunSessionId` actually reads back. +// CL-7481: `recordAgentSessionAtProvision` is the eager write every +// native launcher makes right after `prepareProvisionedDeployment` +// returns — this proves it lands the `agent_session` row under the +// *deploying* principal (the run's own principal does not exist yet) +// and creates the run's event collector exactly once, idempotently. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this proves the session +// moves onto that principal without ever touching the session id, and +// that a run with no launch spec row is a bug this throws on rather +// than papering over. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -16,8 +21,8 @@ import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; import { - endAgentSessionForRun, - recordAgentSessionForRun, + ensureRunSession, + recordAgentSessionAtProvision, } from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { @@ -34,14 +39,17 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("recordAgentSessionForRun", () => { +describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); - const principalId = generateId("principal"); + const deployingPrincipalId = generateId("principal"); + const runPrincipalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchorRunId = generateId("workflowRun"); + const provisionedRunId = generateId("workflowRun"); + const noLaunchSpecRunId = generateId("workflowRun"); const sessionId = generateId("session"); + const domain = `agent-session-${tenantId}.localhost`; beforeAll(async () => { if (databaseUrl === undefined) return; @@ -51,35 +59,70 @@ describeIfDb("recordAgentSessionForRun", () => { id: tenantId, name: "Agent Session Test Tenant", slug: `agent-session-${tenantId}`, - domain: `agent-session-${tenantId}.localhost`, + domain, parentId: null, config: null, }); - await db.db.insert(schema.principal).values({ - id: principalId, - tenantId, - kind: "workflow", - refId: anchorRunId, - status: "active", - }); + await db.db.insert(schema.principal).values([ + { + id: deployingPrincipalId, + tenantId, + kind: "user", + refId: "usr_test", + status: "active", + }, + { + id: runPrincipalId, + tenantId, + kind: "workflow", + refId: provisionedRunId, + status: "active", + }, + ]); await db.db.insert(schema.workflowDefinition).values({ id: definitionId, tenantId, - creatorPrincipalId: principalId, + creatorPrincipalId: deployingPrincipalId, assetId: null, name: "agent-session-test-definition", }); + // Born the instant `prepareProvisionedDeployment` returns: a run row + // exists, its own principal is still null, and its launch spec names + // the session `recordAgentSessionAtProvision` will record. + await db.db.insert(schema.workflowRun).values({ + id: provisionedRunId, + definitionId, + anchorRunId: provisionedRunId, + tenantId, + principalId: null, + address: `${provisionedRunId}@${domain}`, + status: "deployed", + }); + await db.db.insert(schema.workflowRunLaunchSpec).values({ + anchorRunId: provisionedRunId, + sessionId, + deploymentDomain: domain, + sourceAuthorityPrincipalId: deployingPrincipalId, + frozenApprovalBundle: {}, + sourceOfferingIds: [], + defaultSourceOfferingId: "off_test", + deployContent: { systemPrompt: "" }, + }); + + // A run with no launch spec row at all: a launcher that skipped the + // shared provisioning path — a bug `ensureRunSession` must surface, + // not silently no-op past. await db.db.insert(schema.workflowRun).values({ - id: anchorRunId, + id: noLaunchSpecRunId, definitionId, - anchorRunId, + anchorRunId: noLaunchSpecRunId, tenantId, - principalId, - address: `${anchorRunId}@agent-session-${tenantId}.localhost`, - status: "running", + principalId: null, + address: `${noLaunchSpecRunId}@${domain}`, + status: "deployed", }); }); @@ -88,68 +131,97 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db .delete(schema.agentSession) .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRunLaunchSpec) + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, provisionedRunId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchorRunId)); + .where(eq(schema.workflowRun.id, noLaunchSpecRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); await db.db .delete(schema.principal) - .where(eq(schema.principal.id, principalId)); + .where(eq(schema.principal.id, runPrincipalId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, deployingPrincipalId)); await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun( - db.db, - { sessionId, anchorRunId }, - { create: () => undefined }, - ); - - const resolved = await resolveRunSessionId(db.db, principalId); - expect(resolved).toBe(sessionId); - }); - - test("creates the run's event collector with its address/tenant/session/run id", async () => { + test("records the session under the deploying principal and creates the collector once", async () => { const createCalls: unknown[] = []; - const otherSessionId = generateId("session"); - - await recordAgentSessionForRun( - db.db, - { sessionId: otherSessionId, anchorRunId }, - { - create: (...args: unknown[]) => { - createCalls.push(args); - }, + let hasCollector = false; + const eventCollectors = { + create: (...args: unknown[]) => { + createCalls.push(args); + hasCollector = true; }, - ); + has: () => hasCollector, + }; + + await recordAgentSessionAtProvision({ + db: db.db, + eventCollectors, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, + }); + // A second call for the same run is a no-op, not a conflict, and + // must not create a second collector. + await recordAgentSessionAtProvision({ + db: db.db, + eventCollectors, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, + }); expect(createCalls).toEqual([ - [ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - tenantId, - otherSessionId, - anchorRunId, - ], + [`${provisionedRunId}@${domain}`, tenantId, sessionId, provisionedRunId], ]); - await db.db - .delete(schema.agentSession) - .where(eq(schema.agentSession.id, otherSessionId)); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(deployingPrincipalId); + expect(sessionRow?.agentId).toBe(definitionId); }); - test("abandons the run's event collector when the session ends", async () => { - const abandonCalls: string[] = []; + test("re-keys the session onto the run principal once Interchange anchors it", async () => { + await db.db + .update(schema.workflowRun) + .set({ principalId: runPrincipalId, status: "running" }) + .where(eq(schema.workflowRun.id, provisionedRunId)); + + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => true }, + runId: provisionedRunId, + }); - await endAgentSessionForRun(db.db, anchorRunId, { - abandon: (address: string) => { - abandonCalls.push(address); - }, + expect(resolved).toBe(sessionId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), }); + expect(sessionRow?.principalId).toBe(runPrincipalId); - expect(abandonCalls).toEqual([ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - ]); + // Interchange's own mail-routing lookup now finds the session by the + // run's own principal, exactly as it would for a folded run. + const viaVendorLookup = await resolveRunSessionId(db.db, runPrincipalId); + expect(viaVendorLookup).toBe(sessionId); + }); + + test("throws for a run with no launch spec row", async () => { + await expect( + ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => false }, + runId: noLaunchSpecRunId, + }), + ).rejects.toThrow(/no workflow_run_launch_spec/); }); }); From 42f051164eff60ddaed7e32b58090d48740be9f0 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:31 -0700 Subject: [PATCH 36/48] Add tests for wake retry re-resolving the live address (CL-7486) sendRunMailWithReclaimRetry retried every attempt against the address it started with. Proves the reproduction of the production failure: a participant already one relaunch generation in that dies again mid-retry must have its next retry chase the run that replaced it, not keep hammering the address that just died. --- packages/chat/test/platform-adapter.test.ts | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 120538c92..89ad655d5 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -2781,6 +2781,139 @@ describe("createHubChatPlatform", () => { }); }); + // CL-7486: `sendRunMailWithReclaimRetry` used to retry every attempt + // against the address it started with, even after a wake in between + // attempts relaunched the run. A run already one relaunch generation + // in (`instanceId` stable, `currentRunId` already repointed once) that + // dies again while this loop is retrying reproduces the production + // failure exactly: a second relaunch repoints `currentRunId` again, + // and the address this loop still holds is now neither the stable + // `instanceId` nor the current `currentRunId` — invisible to + // `readBindingByAddressAnyTenant`, so the next wake for it throws "No + // workbench_launch binding" instead of finding the run that replaced + // it. + describe("sendRunMailWithReclaimRetry re-resolves the live address", () => { + test("a relaunch between retries is chased to its new address, not retried against the one that died", async () => { + resolveDefinitionSourcesResult = { + ok: true, + materials: [], + sources: [ + { + id: "off_1", + provider: "anthropic", + baseURL: "https://inference.invalid", + credentialId: "cred_placeholder", + model: "claude-sonnet-5", + }, + ], + defaultSource: "off_1", + }; + + const staleAddress = "run_gen1@ten1.workbench.test"; + const db = createFakeDb({ + assetRow: { + tenantId: "ten_1", + creatorPrincipalId: "prin_creator", + name: "ins_room1", + displayName: null, + }, + definitionId: "wfd_room1", + workflowRunRow: { + id: "run_gen1", + address: staleAddress, + principalId: "prin_room1", + definitionId: "wfd_room1", + // Already dead by the time the reclaim retry's own wake reads + // it — the run died again while this loop was mid-retry, not + // at the top of `sendMail` (which would have relaunched it + // before ever reaching this retry loop at all). + status: "failed", + }, + workflowDefinitionRow: { + id: "wfd_room1", + tenantId: "ten_1", + status: "deployed", + origin: "authored", + assetId: "asst_room1", + }, + workbenchLaunchRow: { + tenantId: "ten_1", + // `instanceId` is the room's own stable id, already distinct + // from `currentRunId` — this participant has been relaunched + // once before this test even starts. + instanceId: "ins_room1", + currentRunId: "run_gen1", + foldedBody: { + systemPrompt: "host prompt", + model: "claude-sonnet-5", + toolPackagePins: [], + grantRequirements: [], + credentialBindings: [], + }, + }, + }); + db.inserted.push({ + table: agentSession, + values: { id: "ses_run1", principalId: "prin_room1" }, + }); + + const sessionService = createFakeSessionService(); + sessionService.sendUserMessage = async (params: unknown) => { + sessionService.sendUserMessageCalls.push(params); + const { agentAddress } = params as { agentAddress: string }; + // The dead run never becomes reachable again — only a send to + // whatever address the relaunch actually produced can succeed. + if (agentAddress === staleAddress) { + throw new Error("agent is unreachable"); + } + return new TextEncoder().encode("raw-mime-bytes"); + }; + + // Routable so `sendMail`'s own opening wake-before-send gate skips + // waking it — the relaunch this test cares about must happen + // inside the reclaim-retry loop below, not before it. + const sidecarRouter = createFakeSidecarRouter({ + routableAddresses: [staleAddress], + }); + + const platform = createPlatform({ + toolGrantsForPins: () => [], + db: db as never, + sessionService, + sidecarRouter, + eventCollectors: createFakeEventCollectors(), + reclaimRetryDelaysMs: [1], + mailDeliveryTimeoutMs: 5_000, + }); + + const sent = await platform.sendMail({ + tenantId: "ten_1", + workbenchId: "ins_room1", + principalId: "prin_sender", + content: { content: "hello" }, + }); + + expect(sent.id).toBeTruthy(); + expect(sessionService.sendUserMessageCalls).toHaveLength(2); + const [first, second] = sessionService.sendUserMessageCalls as { + agentAddress: string; + }[]; + expect(first?.agentAddress).toBe(staleAddress); + // The retry that follows the relaunch targets the run that + // replaced `run_gen1`, never the dead address the loop started + // with. + expect(second?.agentAddress).not.toBe(staleAddress); + + const repointed = db.updated.at(-1)?.values as { + currentRunId: string; + priorRunIds: string[]; + }; + const secondAddress = second?.agentAddress ?? ""; + expect(repointed.currentRunId).toBe(secondAddress.split("@")[0] ?? ""); + expect(repointed.priorRunIds).toEqual(["run_gen1"]); + }); + }); + // Proves the actual lever an edited system prompt reaches a running // instance through: `wakeByAddress` (exercised via `sendMail`'s // wake-on-send path above) replays `workbench_launch.foldedBody` From c5cbcebe417a51018a868b831334a6171a617c33 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:41 -0700 Subject: [PATCH 37/48] Chat: wake retry chases the run a relaunch replaced it with (CL-7486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sendRunMailWithReclaimRetry retried every delivery attempt, and woke, against the address it captured at the top of the loop. A wake in between retries can relaunch the run (its dead-run branch repoints the room's stable participant at a fresh address without telling this loop), and a participant already one relaunch generation in that dies again mid-retry leaves this loop holding an address that is no longer either the stable participant id or the current run id — invisible to readBindingByAddressAnyTenant. The next wake for it then threw "No workbench_launch binding" instead of finding the run that replaced it, which is what chat e2e's mention-fanout and invite-drives-a-turn cases were hitting. Re-resolves the live address (and its session) off the stable participant id after each wake, so the retry loop always chases whichever run is actually current. Also hardens publishCorbitsToolsRegistry's create-then-list-on-conflict ensure: two overlapping seed runs racing the same tenant's package registry asset can still throw a raw asset_tenant_kind_name violation if the substrate's own 409 mapping is bypassed; the ensure now also recognizes that violation by name and resolves the winner's row instead of throwing. --- packages/chat/src/platform-adapter.ts | 63 ++++++++++++++++--- packages/tool-registry-publish/src/publish.ts | 14 ++++- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index b7c5c5147..111317134 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -989,10 +989,24 @@ export function createHubChatPlatform( * more time regardless. Exhausting every delay means the condition * is not transient and the caller gets a clean `AgentUnreachableError` * rather than an unhandled 500. + * + * CL-7486: a wake this loop runs between retries can itself relaunch + * the run — `wakeByAddress`'s dead-run branch repoints the room's + * stable participant at a fresh address without telling this loop. + * Retrying `sendRunMail` against the address it started with would + * then keep hammering a run that will never become routable again, + * and the wake ahead of the NEXT retry would fail outright (no + * `workbench_launch` row still names that address at all) instead of + * the "unreachable" this loop knows how to ride out. `reresolve` + * re-reads the live address (and its session) off the stable + * participant id before every wake/retry, so this loop always chases + * whichever run is actually current. */ async function sendRunMailWithReclaimRetry( params: SendRunMailParams, + reresolve: () => Promise<{ agentAddress: string; sessionId: string }>, ): Promise>> { + let current = params; let loggedRetryStart = false; for (let attempt = 0; ; attempt++) { try { @@ -1001,29 +1015,44 @@ export function createHubChatPlatform( // to accept regardless — the signal parameter is unused here, // same abandon-on-timeout behavior as before. return await withTimeout( - () => sendRunMail(runMailDeps, params), + () => sendRunMail(runMailDeps, current), MAIL_DELIVERY_TIMEOUT_MS, - `mail to ${params.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, + `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, ); } catch (err) { const delay = RECLAIM_RETRY_DELAYS_MS[attempt]; if (!isAgentUnreachable(err) || delay === undefined) { if (loggedRetryStart) { - wakeLogger.info`mail to ${params.agentAddress} exhausted every reclaim retry; giving up`; + wakeLogger.info`mail to ${current.agentAddress} exhausted every reclaim retry; giving up`; } if (isAgentUnreachable(err)) { - throw new AgentUnreachableError(params.agentAddress, { + throw new AgentUnreachableError(current.agentAddress, { cause: err, }); } throw err; } if (!loggedRetryStart) { - wakeLogger.info`mail to ${params.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; + wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; loggedRetryStart = true; } await sleep(delay); - await wakeByAddressBounded(params.agentAddress); + // Wake whatever this loop still believes is live — `wakeByAddress` + // itself tolerates an address one relaunch behind the room's + // current one (its lookup matches either the stable participant + // id or the live run id) — and only THEN re-resolve, since the + // wake just run is exactly the kind of relaunch that can leave + // `current.agentAddress` pointing at a run that no longer exists + // at all. Re-resolving before the wake instead would still hand + // the next retry whatever address was current a moment ago, + // stale again the instant this wake relaunches it. + await wakeByAddressBounded(current.agentAddress); + const fresh = await reresolve(); + current = { + ...current, + agentAddress: fresh.agentAddress, + sessionId: fresh.sessionId, + }; } } } @@ -1399,7 +1428,27 @@ export function createHubChatPlatform( ? { references: input.content.references } : {}), }; - const sent = await sendRunMailWithReclaimRetry(withThreading); + const sent = await sendRunMailWithReclaimRetry(withThreading, async () => { + const relive = await requireLive(input.workbenchId, input.tenantId); + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: relive.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: relive.binding.liveAddress, + extra: { runId: relive.run.id }, + }); + } + return { + agentAddress: relive.binding.liveAddress, + sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), + }; + }); lifecycle?.recordActivity(deliveryAddress); diff --git a/packages/tool-registry-publish/src/publish.ts b/packages/tool-registry-publish/src/publish.ts index 301bdd6c6..d6eebbd36 100644 --- a/packages/tool-registry-publish/src/publish.ts +++ b/packages/tool-registry-publish/src/publish.ts @@ -110,7 +110,19 @@ async function ensureRegistryAsset( if (created.status === 201) { return parseSchema(AssetResponse, created.data, "create asset response").id; } - if (created.status !== 409) { + // CL-7486: two overlapping seed runs for the same tenant (the root + // tenant's own boot-time seed racing a second caller seeding that + // same tenant) can both pass the `listRegistryAsset` check above + // before either has created the row, then both post-create — the + // loser hits the `asset_tenant_kind_name` unique constraint. The + // substrate answers that as `409` in the common case; asserted here + // by name too, in case a raw constraint violation surfaces as a + // generic 500 instead, so the loser still resolves the winner's row + // rather than throwing on a race this ensure is meant to tolerate. + const isNameCollision = + created.status === 409 || + JSON.stringify(created.data).includes("asset_tenant_kind_name"); + if (!isNameCollision) { throw new Error( `publishCorbitsToolsRegistry: failed to create the ${CORBITS_TOOLS_REGISTRY} asset: ${String(created.status)}`, ); From ebc173bdd5aaa12cad9068df676586788a827314 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:16:53 -0700 Subject: [PATCH 38/48] Fix webhook-triggers launch test fixture: eventCollectors needs has() --- packages/webhook-triggers/test/launch.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 01b3b6785..e671da009 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -199,6 +199,7 @@ function baseDeps() { isRoutable: () => isRoutableForTest, eventCollectors: { create: () => undefined, + has: () => false, }, }; } From 097b172570f4e9fa08e0ce4bde57aecc4f62c05f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:49:03 -0700 Subject: [PATCH 39/48] Add test coverage for routable-wait relaunch chasing in sendRunMailWithReclaimRetry Drives the CL-7486 reclaim-retry test off routing-table transitions instead of fixed backoff timing, and updates the CL-7214 coalescing test's fake sidecar router to model a redeploy actually registering. --- packages/chat/test/platform-adapter.test.ts | 32 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 89ad655d5..c9583847f 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -2726,6 +2726,7 @@ describe("createHubChatPlatform", () => { // in flight when the concurrent `ensureAwake` call joins it. let deployCallCount = 0; const gatedDeployReleases: (() => void)[] = []; + const sidecarRouter = createFakeSidecarRouter({ routableAddresses: [] }); const allocationService = createFakeWorkflowAllocationService(); const originalPrepare = allocationService.prepareProvisionedDeployment.bind(allocationService); @@ -2738,6 +2739,14 @@ describe("createHubChatPlatform", () => { await new Promise((resolve) => { gatedDeployReleases.push(resolve); }); + // Models the sidecar registering once the reclaim retry's own + // redeploy actually completes — the routable-wait + // `sendRunMailWithReclaimRetry` now does (CL-7488) needs this + // to see the retried send through. The cold wake ahead of the + // first send attempt deliberately stays unrouted so that + // attempt still finds `wakeByAddress`'s own routability check + // false and takes the reclaim-retry path this test exercises. + sidecarRouter.routableAddresses.push(result.deploymentAddress); } return result; }; @@ -2746,10 +2755,10 @@ describe("createHubChatPlatform", () => { toolGrantsForPins: () => [], db: db as never, sessionService, - sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), + sidecarRouter, eventCollectors: createFakeEventCollectors(), lifecycle: { idleSleepMs: 60_000 }, - reclaimRetryDelaysMs: [1], + routableWaitDeadlineMs: 5_000, mailDeliveryTimeoutMs: 5_000, workflowAllocationService: allocationService, }); @@ -2876,13 +2885,30 @@ describe("createHubChatPlatform", () => { routableAddresses: [staleAddress], }); + // Models the sidecar registering the relaunch's fresh address the + // moment its deploy actually completes — `sendRunMailWithReclaimRetry` + // now waits on this routing-table transition (CL-7488) instead of a + // fixed backoff, so the retried send needs it to flip before it can + // see the run through. + const allocationService = createFakeWorkflowAllocationService(); + const originalPrepare = + allocationService.prepareProvisionedDeployment.bind(allocationService); + allocationService.prepareProvisionedDeployment = async ( + params: Parameters[0], + ) => { + const result = await originalPrepare(params); + sidecarRouter.routableAddresses.push(result.deploymentAddress); + return result; + }; + const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, sessionService, sidecarRouter, eventCollectors: createFakeEventCollectors(), - reclaimRetryDelaysMs: [1], + workflowAllocationService: allocationService, + routableWaitDeadlineMs: 5_000, mailDeliveryTimeoutMs: 5_000, }); From 153b81822ea425800a6f44f64c67db515076ce7e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:49:10 -0700 Subject: [PATCH 40/48] Chat: wait for the routing table instead of a fixed backoff before giving up on first-turn delivery sendRunMailWithReclaimRetry used to exhaust a ~7.75s backoff and report the invited agent unreachable while its sidecar was still booting and registering with the hub (CL-7476) -- fast enough to make the fan-out relaunch a perfectly healthy run. It now shares deliverWhenRoutable with the webhook trigger's ingress delivery: wake once on the first unreachable failure (still chasing a mid-wait relaunch by re-resolving the live address inside the retry's send closure), then wait on the hub's routing table up to the same boot-and-register budget (DEFAULT_ROUTABLE_DEADLINE_MS, now exported from @corbits/workflows) before retrying exactly once more. --- packages/chat/src/platform-adapter.ts | 191 +++++++++--------- .../workflows/src/deliver-when-routable.ts | 8 +- packages/workflows/src/index.ts | 1 + 3 files changed, 97 insertions(+), 103 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 111317134..16b2da3f6 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,8 +16,10 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, + deliverWhenRoutable, endAgentSessionForRun, ensureRunSession, + isAgentUnreachableError, readFoldedBody, recordAgentSessionAtProvision, resolveNewestProjectedDefinition, @@ -139,13 +141,6 @@ export type CreateHubChatPlatformDeps = { * `ensureAwake` to redeploy a non-routable target before sending. */ lifecycle?: { idleSleepMs: number; sweepIntervalMs?: number }; - /** - * `sendRunMailWithReclaimRetry`'s backoff between retries of a - * mail send that failed with "agent is unreachable" (see - * RECLAIM_RETRY_DELAYS_MS below). Injectable so tests exercise the - * backoff in milliseconds instead of the production ~8s budget. - */ - reclaimRetryDelaysMs?: readonly number[]; /** * Per-attempt wall-clock bound on `sendRunMail` inside * `sendRunMailWithReclaimRetry` (CL-6644). Defaults to @@ -153,6 +148,16 @@ export type CreateHubChatPlatformDeps = { * in milliseconds instead of the production ~30s budget. */ mailDeliveryTimeoutMs?: number; + /** + * How long `sendRunMailWithReclaimRetry` waits for a freshly + * woken/relaunched run's address to show up in the hub's routing + * table before giving up (see `deliverWhenRoutable` in + * `@corbits/workflows`). Defaults to `DEFAULT_ROUTABLE_DEADLINE_MS` + * — the same boot-and-register budget the webhook trigger's ingress + * delivery waits on (CL-7476). Injectable so tests exercise the wait + * in milliseconds instead of the production ~20s budget. + */ + routableWaitDeadlineMs?: number; /** * The invite-launch model fallback (see `./inference-preferences.ts`'s * `createWorkbenchHostInferencePreferencesResolver`): a @@ -409,35 +414,17 @@ export function createHubChatPlatform( const lifecycle = deps.lifecycle !== undefined ? buildLifecycle(deps.lifecycle) : undefined; - // Bounded backoff for a wake racing the sidecar's own post-restart - // reclaim, and separately for a mail delivery racing the same - // window: 250ms, 500ms, 1s, 2s, 4s — a ~7.75s budget, long enough - // for a normal reconnect challenge to settle without leaving a - // sender stuck for much longer than that. - const RECLAIM_RETRY_DELAYS_MS = deps.reclaimRetryDelaysMs ?? [ - 250, 500, 1000, 2000, 4000, - ]; - - // CL-6644: the reclaim-retry loop's own delays only run between - // attempts that already failed LOUD with "agent is unreachable" -- - // they bound nothing about an attempt that instead stalls forever - // (a sidecar ack that never comes, a wedged promise anywhere in - // `sendRunMail`'s call chain) without ever throwing. Each - // attempt gets the same kind of wall-clock bound `wakeByAddressBounded` - // already puts on the wake itself, so that hang becomes a rejection + // CL-6644: a per-attempt bound on `sendRunMail` inside + // `sendRunMailWithReclaimRetry` -- a send that stalls forever (a + // sidecar ack that never comes, a wedged promise anywhere in + // `sendRunMail`'s call chain) without ever throwing gets the same + // kind of wall-clock bound `wakeByAddressBounded` already puts on + // the wake itself, so that hang becomes a rejection // `dispatchTurnBatch`'s catch can report and notify on, instead of a // promise nothing ever settles. const MAIL_DELIVERY_TIMEOUT_MS = deps.mailDeliveryTimeoutMs ?? DEFAULT_WAKE_TIMEOUT_MS; - function sleep(ms: number): Promise { - return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); - } - - function isAgentUnreachable(err: unknown): boolean { - return err instanceof Error && err.message.includes("agent is unreachable"); - } - function isRoutable(address: string): boolean { return deps.sidecarRouter.getRoutableAddresses().includes(address); } @@ -977,83 +964,89 @@ export function createHubChatPlatform( } /** - * `sendRunMail` delivers synchronously against the sidecar's - * current routable set — the same in-memory index `isRoutable` reads - * — so a send that lands in the same post-restart reclaim window - * `wakeByAddress` above tolerates can still fail with "agent is - * unreachable" even right after a successful (or no-op) wake. Each - * retry forces a fresh wake first: if the earlier reclaim tore the - * agent down, this becomes the genuine redeploy that recovers it; if - * the reclaim is still in flight, the wake itself waits it out (or - * redeploys once its own budget is exhausted) and the delay gives it - * more time regardless. Exhausting every delay means the condition - * is not transient and the caller gets a clean `AgentUnreachableError` - * rather than an unhandled 500. + * `sendRunMail` delivers synchronously against the sidecar's current + * routable set — the same in-memory index `isRoutable` reads — so a + * send lands in "agent is unreachable" for two different reasons + * that both need the same treatment: a run whose sidecar just + * finished a provisioned deploy and hasn't registered yet (CL-7476 — + * `deliverWhenRoutable` from `@corbits/workflows` is the exact same + * wait the webhook trigger's ingress delivery already uses for this), + * and a run that this loop's own wake tore down and is mid-reclaim. + * `deliverWhenRoutable` sends once, and — only on an unreachable + * failure — polls the routing table for `current.agentAddress` up to + * its shared deadline (`DEFAULT_ROUTABLE_DEADLINE_MS`, the same + * boot-and-register budget the webhook path waits on) before sending + * exactly once more. * - * CL-7486: a wake this loop runs between retries can itself relaunch - * the run — `wakeByAddress`'s dead-run branch repoints the room's - * stable participant at a fresh address without telling this loop. - * Retrying `sendRunMail` against the address it started with would - * then keep hammering a run that will never become routable again, - * and the wake ahead of the NEXT retry would fail outright (no - * `workbench_launch` row still names that address at all) instead of - * the "unreachable" this loop knows how to ride out. `reresolve` - * re-reads the live address (and its session) off the stable - * participant id before every wake/retry, so this loop always chases - * whichever run is actually current. + * CL-7486: a wake this loop runs on the first failure can itself + * relaunch the run — `wakeByAddress`'s dead-run branch repoints the + * room's stable participant at a fresh address without telling this + * loop. Retrying `sendRunMail` against the address it started with + * would then keep hammering a run that will never become routable + * again. `reresolve` re-reads the live address (and its session) off + * the stable participant id every time `send` runs — including the + * one `deliverWhenRoutable` runs again after the routing-table wait — + * so this loop always chases whichever run is actually current, even + * one relaunched again while it was waiting. */ async function sendRunMailWithReclaimRetry( params: SendRunMailParams, reresolve: () => Promise<{ agentAddress: string; sessionId: string }>, ): Promise>> { let current = params; - let loggedRetryStart = false; - for (let attempt = 0; ; attempt++) { - try { - // CL-7193: `sendRunMail` does a DB write plus a sidecar - // delivery that shouldn't be half-cancelled, and has no signal - // to accept regardless — the signal parameter is unused here, - // same abandon-on-timeout behavior as before. - return await withTimeout( - () => sendRunMail(runMailDeps, current), - MAIL_DELIVERY_TIMEOUT_MS, - `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, - ); - } catch (err) { - const delay = RECLAIM_RETRY_DELAYS_MS[attempt]; - if (!isAgentUnreachable(err) || delay === undefined) { - if (loggedRetryStart) { - wakeLogger.info`mail to ${current.agentAddress} exhausted every reclaim retry; giving up`; - } - if (isAgentUnreachable(err)) { - throw new AgentUnreachableError(current.agentAddress, { - cause: err, - }); + let wakedOnFailure = false; + async function sendCurrent() { + // CL-7193: `sendRunMail` does a DB write plus a sidecar delivery + // that shouldn't be half-cancelled, and has no signal to accept + // regardless — the signal parameter is unused here, same + // abandon-on-timeout behavior as before. + return withTimeout( + () => sendRunMail(runMailDeps, current), + MAIL_DELIVERY_TIMEOUT_MS, + `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, + ); + } + try { + return await deliverWhenRoutable({ + send: async () => { + try { + return await sendCurrent(); + } catch (err) { + if (!isAgentUnreachableError(err) || wakedOnFailure) throw err; + // Wake exactly once, the moment the first attempt reports + // unreachable: if the run had already died, this is the + // genuine redeploy that recovers it; if a deploy is merely + // still booting, `wakeByAddress`/`lifecycle.ensureAwake` + // no-op past the routability check and this becomes a + // cheap reconcile. Re-resolving only after the wake means a + // relaunch it just performed is exactly what the fresh read + // picks up, rather than the address that was current a + // moment before. + wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; waiting for it to become routable`; + wakedOnFailure = true; + await wakeByAddressBounded(current.agentAddress); + const fresh = await reresolve(); + current = { + ...current, + agentAddress: fresh.agentAddress, + sessionId: fresh.sessionId, + }; + throw err; } - throw err; - } - if (!loggedRetryStart) { - wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; - loggedRetryStart = true; - } - await sleep(delay); - // Wake whatever this loop still believes is live — `wakeByAddress` - // itself tolerates an address one relaunch behind the room's - // current one (its lookup matches either the stable participant - // id or the live run id) — and only THEN re-resolve, since the - // wake just run is exactly the kind of relaunch that can leave - // `current.agentAddress` pointing at a run that no longer exists - // at all. Re-resolving before the wake instead would still hand - // the next retry whatever address was current a moment ago, - // stale again the instant this wake relaunches it. - await wakeByAddressBounded(current.agentAddress); - const fresh = await reresolve(); - current = { - ...current, - agentAddress: fresh.agentAddress, - sessionId: fresh.sessionId, - }; + }, + isRoutable: () => isRoutable(current.agentAddress), + ...(deps.routableWaitDeadlineMs !== undefined + ? { deadlineMs: deps.routableWaitDeadlineMs } + : {}), + }); + } catch (err) { + if (isAgentUnreachableError(err)) { + wakeLogger.info`mail to ${current.agentAddress} exhausted the routable-wait budget; giving up`; + throw new AgentUnreachableError(current.agentAddress, { + cause: err, + }); } + throw err; } } diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts index a48e99403..666cba20b 100644 --- a/packages/workflows/src/deliver-when-routable.ts +++ b/packages/workflows/src/deliver-when-routable.ts @@ -3,9 +3,9 @@ // sidecar has finished booting and registering with the hub, so the send // fails with "agent is unreachable" even though nothing is actually wrong // — the address just isn't routable yet. The webhook trigger's ingress -// delivery goes through this helper; chat keeps its own existing -// wake-and-retry loop (`sendRunMailWithReclaimRetry`) instead. -const DEFAULT_DEADLINE_MS = 20_000; +// delivery goes through this helper; chat's `sendRunMailWithReclaimRetry` +// (CL-7488) shares this same budget rather than declaring its own. +export const DEFAULT_ROUTABLE_DEADLINE_MS = 20_000; const DEFAULT_POLL_INTERVAL_MS = 250; export function isAgentUnreachableError(err: unknown): boolean { @@ -48,7 +48,7 @@ export async function deliverWhenRoutable( if (!isUnreachable(err)) { throw err; } - const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS); + const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; const sleep = opts.sleep ?? defaultSleep; while (!opts.isRoutable()) { diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 64bfbdff9..ba84a329b 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -65,6 +65,7 @@ export { export { deliverWhenRoutable, isAgentUnreachableError, + DEFAULT_ROUTABLE_DEADLINE_MS, type DeliverWhenRoutableOptions, } from "./deliver-when-routable"; export { From 320deb222d83895a6db13c032a16fd2c2f2b9426 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:53:02 -0700 Subject: [PATCH 41/48] Fix formatting --- packages/chat/src/platform-adapter.ts | 45 ++++++++++--------- .../workflows/src/deliver-when-routable.ts | 3 +- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 16b2da3f6..745ac214f 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -1421,27 +1421,30 @@ export function createHubChatPlatform( ? { references: input.content.references } : {}), }; - const sent = await sendRunMailWithReclaimRetry(withThreading, async () => { - const relive = await requireLive(input.workbenchId, input.tenantId); - try { - await ensureRunSession({ - db: deps.db, - eventCollectors: deps.eventCollectors, - runId: relive.run.id, - }); - } catch (err) { - reportError(err, { - operation: "chat.sendMail.ensureRunSession", - tenantId: input.tenantId, - agentId: relive.binding.liveAddress, - extra: { runId: relive.run.id }, - }); - } - return { - agentAddress: relive.binding.liveAddress, - sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), - }; - }); + const sent = await sendRunMailWithReclaimRetry( + withThreading, + async () => { + const relive = await requireLive(input.workbenchId, input.tenantId); + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: relive.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: relive.binding.liveAddress, + extra: { runId: relive.run.id }, + }); + } + return { + agentAddress: relive.binding.liveAddress, + sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), + }; + }, + ); lifecycle?.recordActivity(deliveryAddress); diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts index 666cba20b..cee443ed9 100644 --- a/packages/workflows/src/deliver-when-routable.ts +++ b/packages/workflows/src/deliver-when-routable.ts @@ -48,7 +48,8 @@ export async function deliverWhenRoutable( if (!isUnreachable(err)) { throw err; } - const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); + const deadline = + Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; const sleep = opts.sleep ?? defaultSleep; while (!opts.isRoutable()) { From 3cfb6dec3e23461fd8296ff8909d54c2da45c11c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 15:09:27 -0700 Subject: [PATCH 42/48] Add test coverage for ensureRunSession creating a session for an API-deployed run --- packages/workflows/test/agent-session.test.ts | 93 ++++++++++++++++++- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index 6fe8dcfb6..7837ec3cb 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -7,10 +7,13 @@ // returns — this proves it lands the `agent_session` row under the // *deploying* principal (the run's own principal does not exist yet) // and creates the run's event collector exactly once, idempotently. -// `ensureRunSession` is now purely a re-key: once the run's first -// trigger anchors a real principal onto it, this proves the session -// moves onto that principal without ever touching the session id, and -// that a run with no launch spec row is a bug this throws on rather +// `ensureRunSession` is a true upsert: once the run's first trigger +// anchors a real principal onto it, this proves the session moves onto +// that principal without ever touching the session id when a row +// already exists, that it creates the row from scratch for a run +// deployed straight through Interchange's own deployments route (no +// Workbench launcher ever ran `recordAgentSessionAtProvision` for it), +// and that a run with no launch spec row is a bug this throws on rather // than papering over. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -48,7 +51,10 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { const definitionId = generateId("workflowDefinition"); const provisionedRunId = generateId("workflowRun"); const noLaunchSpecRunId = generateId("workflowRun"); + const apiDeployedRunId = generateId("workflowRun"); + const apiDeployedPrincipalId = generateId("principal"); const sessionId = generateId("session"); + const apiDeployedSessionId = generateId("session"); const domain = `agent-session-${tenantId}.localhost`; beforeAll(async () => { @@ -79,6 +85,13 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { refId: provisionedRunId, status: "active", }, + { + id: apiDeployedPrincipalId, + tenantId, + kind: "workflow", + refId: apiDeployedRunId, + status: "active", + }, ]); await db.db.insert(schema.workflowDefinition).values({ @@ -124,6 +137,32 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { address: `${noLaunchSpecRunId}@${domain}`, status: "deployed", }); + + // Deployed through Interchange's own deployments route rather than a + // Workbench launcher: Interchange itself writes the launch spec (it + // mints the session id), but nothing ever calls + // `recordAgentSessionAtProvision` for this path, so no `agent_session` + // row exists yet. Already anchored with its own principal by the time + // `ensureRunSession` is reached, same as at persist/dispatch. + await db.db.insert(schema.workflowRun).values({ + id: apiDeployedRunId, + definitionId, + anchorRunId: apiDeployedRunId, + tenantId, + principalId: apiDeployedPrincipalId, + address: `${apiDeployedRunId}@${domain}`, + status: "running", + }); + await db.db.insert(schema.workflowRunLaunchSpec).values({ + anchorRunId: apiDeployedRunId, + sessionId: apiDeployedSessionId, + deploymentDomain: domain, + sourceAuthorityPrincipalId: deployingPrincipalId, + frozenApprovalBundle: {}, + sourceOfferingIds: [], + defaultSourceOfferingId: "off_test", + deployContent: { systemPrompt: "" }, + }); }); afterAll(async () => { @@ -140,12 +179,24 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { await db.db .delete(schema.workflowRun) .where(eq(schema.workflowRun.id, noLaunchSpecRunId)); + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, apiDeployedSessionId)); + await db.db + .delete(schema.workflowRunLaunchSpec) + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, apiDeployedRunId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, apiDeployedRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); await db.db .delete(schema.principal) .where(eq(schema.principal.id, runPrincipalId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, apiDeployedPrincipalId)); await db.db .delete(schema.principal) .where(eq(schema.principal.id, deployingPrincipalId)); @@ -215,6 +266,40 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { expect(viaVendorLookup).toBe(sessionId); }); + test("creates the session and collector on first ensure for an API-deployed run", async () => { + const createCalls: unknown[] = []; + let hasCollector = false; + const eventCollectors = { + create: (...args: unknown[]) => { + createCalls.push(args); + hasCollector = true; + }, + has: () => hasCollector, + }; + + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors, + runId: apiDeployedRunId, + }); + + expect(resolved).toBe(apiDeployedSessionId); + expect(createCalls).toEqual([ + [ + `${apiDeployedRunId}@${domain}`, + tenantId, + apiDeployedSessionId, + apiDeployedRunId, + ], + ]); + + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, apiDeployedSessionId), + }); + expect(sessionRow?.principalId).toBe(apiDeployedPrincipalId); + expect(sessionRow?.agentId).toBe(definitionId); + }); + test("throws for a run with no launch spec row", async () => { await expect( ensureRunSession({ From 6547f36f5c26339a9eddc7ac140544f17f8cabdf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 15:09:33 -0700 Subject: [PATCH 43/48] ensureRunSession: create the session row instead of throwing when it's missing Runs deployed straight through Interchange's own deployments route never pass through a Workbench launcher, so recordAgentSessionAtProvision never runs for them and no agent_session row exists for the run. By the time persist/dispatch call ensureRunSession the run is anchored with a principal, so the seam has everything needed to create the row itself instead of treating the missing row as a bug. --- .../workflows/src/launch/agent-session.ts | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index f73c2a0d0..e4c972ef4 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -33,9 +33,17 @@ // keyed on the deploying principal (`sourceAuthorityPrincipalId`) since // the run principal does not exist yet — every other field the launcher // already knows or can read straight off the fresh run row. -// `ensureRunSession` is now purely a re-key: once the run's first -// trigger anchors a real principal onto it, this moves `principal_id` -// onto that principal without ever touching the session id. +// `ensureRunSession` re-keys onto the run's own principal once one is +// anchored, moving `principal_id` without ever touching the session id. +// +// That eager write only ever runs for a Workbench launcher, though — a +// run deployed straight through Interchange's own +// `POST /api/tenants/:id/workflows/deployments` route (the e2e suites, +// or any other API client) never passes through one, so no +// `agent_session` row exists for it until something creates one. +// CL-7489 makes `ensureRunSession` a true upsert: by the time +// persist/dispatch call it the run is anchored, so a missing row is +// created rather than treated as a bug. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { @@ -118,18 +126,24 @@ export async function recordAgentSessionAtProvision(params: { } /** - * Re-keys a run's `agent_session` onto the run's own principal once - * Interchange anchors one (`anchorWithPrincipal`, the run's first - * trigger) — before that, the session row is recorded under the - * deploying principal by `recordAgentSessionAtProvision`, since the run - * principal does not exist yet. The session id never changes; only - * `principal_id` moves, and only once (a session already re-keyed to the - * current run principal is left alone). The launch-spec row is the - * source of truth for which session belongs to this run — its absence - * means a launcher provisioned this run without going through the - * shared path, which is a bug, not a state to paper over. Also creates - * the run's event collector if none is live for its address (a fresh - * process has no in-memory collectors regardless of what is on disk). + * True upsert of a run's `agent_session`, keyed by the run's own + * principal once Interchange anchors one (`anchorWithPrincipal`, the + * run's first trigger). Before that, `recordAgentSessionAtProvision` + * records the row under the deploying principal for a Workbench-launched + * run — but a run deployed straight through Interchange's own + * `POST /api/tenants/:id/workflows/deployments` route (the e2e suites, + * or any other API client) never passes through a Workbench launcher, so + * that eager write never runs for it, and no `agent_session` row exists + * yet when this seam is first reached. By the time persist/dispatch call + * this, though, the run is anchored (`runRow.principalId` is set), so + * everything needed to create the row is on hand: this inserts it rather + * than treating the missing row as a bug. The launch-spec row is still + * the source of truth for which session id belongs to this run — its + * absence means a launcher provisioned this run without going through + * the shared path, which is a bug, not a state to paper over. Also + * creates the run's event collector if none is live for its address (a + * fresh process has no in-memory collectors regardless of what is on + * disk). */ export async function ensureRunSession(params: { readonly db: DB["db"]; @@ -159,11 +173,20 @@ export async function ensureRunSession(params: { where: eq(agentSession.id, sessionId), }); if (sessionRow === undefined) { - throw new Error( - `ensureRunSession: no agent_session "${sessionId}" for run "${runId}" — recordAgentSessionAtProvision should have created it at launch`, - ); - } - if (sessionRow.principalId !== runRow.principalId) { + const now = new Date(); + await db + .insert(agentSession) + .values({ + id: sessionId, + tenantId: runRow.tenantId, + agentId: runRow.definitionId, + principalId: runRow.principalId, + status: "active", + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: agentSession.id }); + } else if (sessionRow.principalId !== runRow.principalId) { await db .update(agentSession) .set({ principalId: runRow.principalId, updatedAt: new Date() }) From b9785a9cf1810d4322117f15dcc7460d7613cf66 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 15:34:22 -0700 Subject: [PATCH 44/48] Add tests for waiting on a booting run instead of relaunching it isBeyondWake's completed-run test assumed status alone marks a run terminal; a deployment settled "completed" before its first trigger is still booting, not dead, so the predicate now needs a run that took a turn to treat "completed" as terminal. Update platform-adapter's lifecycle/ensureAwake/sendMail coverage to match: a live run that is merely not yet routable is waited on rather than redeployed, so those tests now exercise a genuinely terminal run instead of an offline one. --- packages/chat/src/agent-binding.test.ts | 19 +- packages/chat/test/platform-adapter.test.ts | 215 ++++++-------------- 2 files changed, 85 insertions(+), 149 deletions(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index f11d61a60..e9967fb04 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -186,12 +186,29 @@ describe("isBeyondWake", () => { ).toBe(false); }); - test("a completed run is beyond waking — wake is a fresh provision", async () => { + test("a completed run that took a turn is beyond waking — wake is a fresh provision", async () => { expect( await isBeyondWake(fakeDb([]), { id: "run_done", status: "completed", + principalId: "prin_1", }), ).toBe(true); }); + + // CL-7490: a deployment `markTerminal`'d before its first trigger ever + // landed settles "completed" too (see `workflow-run-store.ts`'s + // `markTerminal`), and the store cannot otherwise tell that apart from a + // run still mid-boot. Treating it as beyond waking here is exactly the + // bug that made a booting run's status read as terminal and relaunched + // it out from under its own in-flight sidecar registration. + test("a completed run that never took a turn is not beyond waking — the store cannot tell it apart from still booting", async () => { + expect( + await isBeyondWake(fakeDb([]), { + id: "run_never_triggered", + status: "completed", + principalId: null, + }), + ).toBe(false); + }); }); diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index c9583847f..40a5e2ab5 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -870,10 +870,15 @@ describe("createHubChatPlatform", () => { displayName: null, }, definitionId: "wfd_workbench1", + // CL-7490: `wakeByAddress` only ever redeploys a genuinely dead + // run now — a merely not-yet-routable one is booting and is + // waited on, never redeployed — so this run must be terminal for + // the deploy failure below to actually be reached. workflowRunRow: { id: "ins_workbench1", address: "ins_workbench1@ten1.workbench.test", principalId: "prin_run1", + status: "failed", }, workbenchLaunchRow: { tenantId: "ten_1", @@ -947,10 +952,13 @@ describe("createHubChatPlatform", () => { displayName: null, }, definitionId: "wfd_workbench1", + // CL-7490: same reasoning as above — the run must be terminal for + // `wakeByAddress` to attempt a redeploy at all. workflowRunRow: { id: "ins_workbench1", address: "ins_workbench1@ten1.workbench.test", principalId: "prin_run1", + status: "failed", }, workbenchLaunchRow: { tenantId: "ten_1", @@ -2028,7 +2036,12 @@ describe("createHubChatPlatform", () => { // when `deps.lifecycle` is configured, and that `sendMail` actually // redeploys a non-routable target before sending. describe("lifecycle wiring", () => { - test("sendMail wakes a non-routable workbench by redeploying before sending, then sends", async () => { + // CL-7490: a provisioned anchor stays "deployed" until its first + // trigger, so a run mid-boot looks exactly like this — live, not yet + // routable. `wakeByAddress` must not relaunch it; the run is waited + // on (`deliverWhenRoutable`, via `sendRunMailWithReclaimRetry`) until + // the sidecar finishes registering, then delivered once. + test("sendMail waits on a deployed, not-yet-routable workbench and delivers once it becomes routable, without relaunching it", async () => { resolveDefinitionSourcesResult = { ok: true, materials: [], @@ -2044,6 +2057,7 @@ describe("createHubChatPlatform", () => { defaultSource: "off_1", }; + const address = "ins_workbench1@ten1.workbench.test"; const db = createFakeDb({ assetRow: { tenantId: "ten_1", @@ -2054,9 +2068,10 @@ describe("createHubChatPlatform", () => { definitionId: "wfd_workbench1", workflowRunRow: { id: "ins_workbench1", - address: "ins_workbench1@ten1.workbench.test", - principalId: "prin_run1", + address, + principalId: null, definitionId: "wfd_workbench1", + status: "deployed", }, workflowDefinitionRow: { id: "wfd_workbench1", @@ -2083,9 +2098,22 @@ describe("createHubChatPlatform", () => { }); const sessionService = createFakeSessionService(); - // Not in the sidecar's routable set: this workbench is asleep (or - // never came back after a restart) when the send arrives. + let sendAttempts = 0; + sessionService.sendUserMessage = async (params: unknown) => { + sessionService.sendUserMessageCalls.push(params); + sendAttempts += 1; + // The first attempt lands while the sidecar is still booting. + if (sendAttempts === 1) { + throw new Error("agent is unreachable"); + } + return new TextEncoder().encode("raw-mime-bytes"); + }; + // Not routable yet — the sidecar is still finishing its boot; + // it registers a moment later, on its own, with no redeploy. const sidecarRouter = createFakeSidecarRouter({ routableAddresses: [] }); + setTimeout(() => { + sidecarRouter.routableAddresses.push(address); + }, 10); const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ @@ -2105,8 +2133,10 @@ describe("createHubChatPlatform", () => { }); expect(sent.id).toBeTruthy(); - expect(lastAllocationService.prepareCalls).toHaveLength(1); - expect(sessionService.sendUserMessageCalls).toHaveLength(1); + // No relaunch: the booting run kept its own run id and address — + // `wakeByAddress` never called `prepareProvisionedDeployment`. + expect(lastAllocationService.prepareCalls).toHaveLength(0); + expect(sessionService.sendUserMessageCalls).toHaveLength(2); }); // CL-6267: the sidecar's own park/wake handler now owns respawning @@ -2507,7 +2537,13 @@ describe("createHubChatPlatform", () => { expect(lastAllocationService.prepareCalls).toHaveLength(0); }); - test("redeploys a non-routable address when lifecycle is configured", async () => { + // CL-7490: `ensureAwake` (via `wakeByAddress`) must not redeploy a + // merely not-yet-routable run — only a genuinely terminal one. A + // `deployed`/`running` run that has not yet registered is left alone + // here (no unit test needed for that: `wakeByAddress` becomes a + // pure no-op, nothing to assert beyond "did not call prepare", which + // the failed-run tests below cover by contrast). + test("relaunches a failed, non-routable address exactly once when lifecycle is configured", async () => { resolveDefinitionSourcesResult = { ok: true, materials: [], @@ -2535,6 +2571,7 @@ describe("createHubChatPlatform", () => { id: "ins_workbench1", address, principalId: "prin_run1", + status: "failed", }, workbenchLaunchRow: { tenantId: "ten_1", @@ -2568,7 +2605,7 @@ describe("createHubChatPlatform", () => { expect(lastAllocationService.prepareCalls).toHaveLength(1); }); - test("redeploys a non-routable address when lifecycle is not configured", async () => { + test("relaunches a failed, non-routable address exactly once when lifecycle is not configured", async () => { resolveDefinitionSourcesResult = { ok: true, materials: [], @@ -2596,6 +2633,7 @@ describe("createHubChatPlatform", () => { id: "ins_workbench1", address, principalId: "prin_run1", + status: "failed", }, workbenchLaunchRow: { tenantId: "ten_1", @@ -2652,143 +2690,16 @@ describe("createHubChatPlatform", () => { }); }); - // CL-7214: `sendRunMailWithReclaimRetry`'s reclaim-retry loop used - // to call `wakeByAddress` directly, bypassing `lifecycle.ensureAwake`'s - // per-address coalescing entirely. Proves that a reclaim-retry wake and - // an independent, concurrent `ensureAwake` call for the same address - // now coalesce onto the same in-flight wake — never dispatching a - // second, concurrent `wakeByAddress` that would race the first on the - // same `session_asset` primary key and git ref. - describe("wakeByAddressBounded reclaim-retry coalescing", () => { - test("a reclaim-retry wake and a concurrent ensureAwake call for the same address never redeploy it twice", async () => { - resolveDefinitionSourcesResult = { - ok: true, - materials: [], - sources: [ - { - id: "off_1", - provider: "anthropic", - baseURL: "https://inference.invalid", - credentialId: "cred_placeholder", - model: "claude-sonnet-5", - }, - ], - defaultSource: "off_1", - }; - const address = "ins_workbench1@ten1.workbench.test"; - const db = createFakeDb({ - assetRow: { - tenantId: "ten_1", - creatorPrincipalId: "prin_creator", - name: "workbench-1", - displayName: null, - }, - definitionId: "wfd_workbench1", - workflowRunRow: { - id: "ins_workbench1", - address, - principalId: "prin_run1", - }, - workbenchLaunchRow: { - tenantId: "ten_1", - instanceId: "ins_workbench1", - foldedBody: { - systemPrompt: "host prompt", - model: "claude-sonnet-5", - toolPackagePins: [], - grantRequirements: [], - credentialBindings: [], - }, - }, - }); - db.inserted.push({ - table: agentSession, - values: { id: "ses_run1", principalId: "prin_run1" }, - }); - - const sessionService = createFakeSessionService(); - let sendAttempts = 0; - sessionService.sendUserMessage = async (params: unknown) => { - sessionService.sendUserMessageCalls.push(params); - sendAttempts += 1; - // The first delivery attempt fails as "agent is unreachable", - // forcing `sendFoldedMailWithReclaimRetry` down its reclaim - // path — the call site that used to bypass coalescing. - if (sendAttempts === 1) { - throw new Error("agent is unreachable"); - } - return new TextEncoder().encode("raw-mime-bytes"); - }; - - // The first provision (the cold wake before the first send attempt) - // resolves immediately; every prepare after it is held open until - // the test releases it, so the reclaim-retry's redeploy is still - // in flight when the concurrent `ensureAwake` call joins it. - let deployCallCount = 0; - const gatedDeployReleases: (() => void)[] = []; - const sidecarRouter = createFakeSidecarRouter({ routableAddresses: [] }); - const allocationService = createFakeWorkflowAllocationService(); - const originalPrepare = - allocationService.prepareProvisionedDeployment.bind(allocationService); - allocationService.prepareProvisionedDeployment = async ( - params: Parameters[0], - ) => { - const result = await originalPrepare(params); - deployCallCount += 1; - if (deployCallCount >= 2) { - await new Promise((resolve) => { - gatedDeployReleases.push(resolve); - }); - // Models the sidecar registering once the reclaim retry's own - // redeploy actually completes — the routable-wait - // `sendRunMailWithReclaimRetry` now does (CL-7488) needs this - // to see the retried send through. The cold wake ahead of the - // first send attempt deliberately stays unrouted so that - // attempt still finds `wakeByAddress`'s own routability check - // false and takes the reclaim-retry path this test exercises. - sidecarRouter.routableAddresses.push(result.deploymentAddress); - } - return result; - }; - - const platform = createPlatform({ - toolGrantsForPins: () => [], - db: db as never, - sessionService, - sidecarRouter, - eventCollectors: createFakeEventCollectors(), - lifecycle: { idleSleepMs: 60_000 }, - routableWaitDeadlineMs: 5_000, - mailDeliveryTimeoutMs: 5_000, - workflowAllocationService: allocationService, - }); - - const sendMailPromise = platform.sendMail({ - tenantId: "ten_1", - workbenchId: "ins_workbench1", - principalId: "prin_sender", - content: { content: "hello workbench" }, - }); - - // Let the cold wake (prepare #1) resolve, the first send attempt - // fail, and the reclaim retry's own wake (prepare #2) start and - // gate. - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(deployCallCount).toBe(2); - - // A second, independent caller asks for the same address while - // prepare #2 is still in flight — the concurrent-wake race CL-7214 - // closes. - const ensureAwakePromise = platform.ensureAwake(address); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(deployCallCount).toBe(2); - - gatedDeployReleases.forEach((release) => release()); - await Promise.all([sendMailPromise, ensureAwakePromise]); - - expect(deployCallCount).toBe(2); - }); - }); + // CL-7214 proved `sendRunMailWithReclaimRetry`'s reclaim-retry wake and + // an independent, concurrent `ensureAwake` call coalesced onto the same + // in-flight redeploy rather than racing two `prepareProvisionedDeployment` + // calls on the same `session_asset` primary key and git ref. That + // redeploy is no longer wakeByAddress's behavior for a live, merely + // not-yet-routable run (CL-7490) — the only thing left to redeploy is a + // genuinely terminal run, and the coalescing that guards a concurrent + // redeploy of it is `@corbits/agent-lifecycle`'s `pendingWakes` map, + // proven directly in that package's own test suite (see the note above + // `describe("lifecycle wiring", ...)`), not re-proven here. // CL-7486: `sendRunMailWithReclaimRetry` used to retry every attempt // against the address it started with, even after a wake in between @@ -2954,7 +2865,7 @@ describe("createHubChatPlatform", () => { model: "claude-sonnet-5", }); - function buildRefreshableDb() { + function buildRefreshableDb(workflowRunStatus?: string) { return createFakeDb({ assetRow: { tenantId: "ten_1", @@ -2968,6 +2879,9 @@ describe("createHubChatPlatform", () => { address: "agent1@ten1.workbench.test", principalId: "prin_agent1", definitionId: "wfd_agent1", + ...(workflowRunStatus !== undefined + ? { status: workflowRunStatus } + : {}), }, workflowDefinitionRow: { id: "wfd_agent1", @@ -3032,7 +2946,12 @@ describe("createHubChatPlatform", () => { defaultSource: "off_1", }; - const db = buildRefreshableDb(); + // CL-7490: `wakeByAddress` no longer redeploys a merely + // not-yet-routable run — only a genuinely dead one, so this run + // must actually be terminal (not just offline) for `sendMail`'s + // wake-on-send path to be the one that carries the refreshed + // system prompt into a fresh deploy. + const db = buildRefreshableDb("failed"); db.inserted.push({ table: agentSession, values: { id: "ses_agent1", principalId: "prin_agent1" }, From a1370dcf507fb632057d56201b02c47100ede501 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 15:34:30 -0700 Subject: [PATCH 45/48] Chat: a booting run is waited on, not relaunched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provisioned anchor stays "deployed" until its first trigger, so a run mid-boot is indistinguishable from an offline one by routability alone. wakeByAddress's fallback branch treated the two the same and redeployed either — minting a fresh run and address out from under the sidecar's own in-flight registration, which supersedes its allocation and restarts the boot. Repeated on every wake attempt, this produced an endless reconnect loop with no run ever finishing its boot. wakeByAddress now only relaunches a run whose status is genuinely terminal; a live run that is not yet routable is left alone; the caller's own routable wait (deliverWhenRoutable, via sendRunMailWithReclaimRetry) is what actually resolves it once the sidecar finishes registering. isBeyondWake also no longer treats every "completed" status as terminal outright: a deployment settled complete before its first trigger ever landed is still indistinguishable from one mid-boot, so it is only terminal once the store can tell it took a turn (a principal was ever anchored to it). --- packages/chat/src/agent-binding.ts | 39 ++++++++++++++++++------- packages/chat/src/platform-adapter.ts | 41 ++++++++------------------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 47b4d0d71..dd61aa6a2 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -314,27 +314,46 @@ export async function readPriorRuns( } /** - * The statuses a `workflow_run` can hold that mean "this run will never - * accept mail again". Wake of a reaped or lost instance is a fresh - * `prepareProvisionedDeployment`, not a reuse of this row. + * The statuses a `workflow_run` can hold that are terminal regardless of + * whether the run ever took a turn. `"completed"` is deliberately absent + * here — a deployment `markTerminal`'d before its first trigger settles + * "completed" too (see `workflow-run-store.ts`'s `markTerminal`), and that + * is a run that is still booting, not one that will "never accept mail + * again". `isBeyondWake` below folds `"completed"` back in only once it + * can tell the two apart. */ -const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ +const UNCONDITIONALLY_TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ "failed", "cancelled", "canceled", - "completed", ]); /** - * Whether this run is dead: terminal in `workflow_run.status`. A run - * this returns true for cannot be woken — it can only be relaunched as - * a fresh provisioned deployment. + * Whether this run is dead: terminal in `workflow_run.status`, in a way + * that actually means "this run will never accept mail again" rather than + * "this deployment never got as far as its first trigger". A run this + * returns true for cannot be woken — it can only be relaunched as a fresh + * provisioned deployment. A run this returns false for — `"deployed"` or + * `"running"`, or `"completed"` with no principal ever anchored to it — is + * still live or still booting: waking it means waiting for it to become + * routable (`deliverWhenRoutable`), never relaunching it (CL-7490). + * + * `principalId` is the signal for "ever took a turn": `anchorWithPrincipal` + * sets it in the same update that flips `"deployed"` -> `"running"` on a + * deployment's first trigger, and a child run is born with one already. A + * `"completed"` run with no `principalId` can only be a deployment torn + * down before that first trigger ever landed — the store cannot tell that + * apart from a mid-boot deployment misreporting itself terminal, so it is + * treated as still booting rather than relaunched out from under a caller + * that is only waiting on it. */ export async function isBeyondWake( _db: DB["db"], - run: { id: string; status: string }, + run: { id: string; status: string; principalId?: string | null }, ): Promise { - return TERMINAL_RUN_STATUSES.has(run.status); + if (UNCONDITIONALLY_TERMINAL_RUN_STATUSES.has(run.status)) return true; + if (run.status !== "completed") return false; + return run.principalId !== null && run.principalId !== undefined; } /** diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 745ac214f..5e8f97181 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -733,35 +733,18 @@ export function createHubChatPlatform( return; } - if (live.run.definitionId === null) { - throw new Error( - `Cannot wake "${binding.stableId}": its run ${live.run.id} names no definition`, - ); - } - const definitionAssetId = await resolveDefinitionAssetId( - live.run.definitionId, - ); - if (definitionAssetId === undefined) { - throw new Error( - `Cannot wake "${binding.stableId}": definition ${live.run.definitionId} has no workflow asset`, - ); - } - const prepared = await provisionOnAsset({ - tenantId: binding.tenantId, - deploymentDomain: domainOf(binding.roomAddress), - sourceAuthorityPrincipalId: - await sourceAuthorityForAsset(definitionAssetId), - definitionAssetId, - foldedBody: binding.foldedBody, - }); - await repointBinding( - deps.db, - binding, - prepared.runId, - prepared.sourcesDigest, - ); - lifecycle?.untrack(binding.liveAddress); - lifecycle?.track(prepared.address); + // CL-7490: not terminal, and not (yet) routable. A provisioned + // anchor stays "deployed" until its first trigger — this is + // exactly what a run mid-boot looks like from here, and relaunching + // it out from under its own in-flight sidecar registration is what + // supersedes the allocation and restarts the boot forever (the + // "Deployed agent" line never appears; a fresh sidecar/allocation + // shows up every few seconds instead). There is nothing this + // function can safely do for a live-but-not-yet-routable run: the + // caller's own routable wait (`deliverWhenRoutable`, wired through + // `sendRunMailWithReclaimRetry`) is what actually resolves this, by + // polling the routing table until the boot finishes or its budget + // runs out. A no-op here is the correct "nothing more to do". } catch (error) { throw wrapWakeInferenceError(error); } From b913397e898d072c55cdddcecf8438d6d602e7f5 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 16:15:46 -0700 Subject: [PATCH 46/48] Add tests for chat's run-trigger delivery (CL-7490) Retargets platform-adapter's fan-out assertions from a raw sessionService.sendUserMessage call to the new RunTriggerClient seam: the fake now records anchorRunId-keyed trigger calls and mints a messageId the same shape the real route does, so the reclaim-retry and threading assertions exercise the seam production code will actually use. --- packages/chat/test/platform-adapter.test.ts | 294 ++++++++++---------- packages/chat/test/relaunch-close.test.ts | 2 +- 2 files changed, 145 insertions(+), 151 deletions(-) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 40a5e2ab5..73d418937 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -15,7 +15,7 @@ // `@intx/hub-api`'s own contract, not this package's, and is not // re-proven here. // -// `sessionService`/`repoStore`/`sidecarRouter` are fakes recording +// `runTrigger`/`repoStore`/`sidecarRouter` are fakes recording // their calls, and `db` is a minimal chainable stand-in for the // drizzle query builder (no database involved) so the mapping is // exercised without a real Postgres. @@ -30,6 +30,7 @@ import type { DefinitionSourceResolution } from "@intx/hub-api"; import { agentSession, asset, + principal as principalTable, sessionMail, workflowDefinition, workflowDefinitionVersion, @@ -40,6 +41,10 @@ import type { EventCollectorRegistry, SidecarRouter } from "@intx/hub-sessions"; import { workbenchLaunch } from "../src/schema"; import type { CreateHubChatPlatformDeps } from "../src/platform-adapter"; import { MODEL_UNAVAILABLE_CONSUMER_MESSAGE } from "../src/model-unavailable"; +import type { + RunTriggerClient, + TriggerWorkflowRunMailInput, +} from "../src/run-trigger-client"; import { createCryptoProviderCache, type CryptoProviderCache, @@ -535,6 +540,23 @@ function createFakeDb(opts: { }; } if (table === asset) return selectChain([opts.assetRow]); + // The run-trigger client's auth resolution + // (`resolveTriggerAuthUserId` in `../src/platform-adapter.ts`) + // looks up the better-auth user behind a principal id. No + // test here cares which synthetic user a trigger call + // authenticates as, only that it succeeds, so this answers + // every lookup with the same fixed "user" principal. + if (table === principalTable) { + return selectChain([{ refId: "user_test", kind: "user" }]); + } + if (table === workflowDefinition) { + return selectChain([ + { + assetId: + opts.workflowDefinitionRow?.assetId ?? "ast_definition1", + }, + ]); + } if (table === workflowDefinitionVersion) { // `loadFrozenWireProjection` filters on // `and(eq(definitionId, id), eq(version, "1"))`; the first @@ -670,58 +692,35 @@ function createFakeEventCollectors( }; } -type AdoptedDeployCall = { - anchorRunId: string; - agentAddress: string; +type FakeRunTrigger = RunTriggerClient & { + triggerCalls: TriggerWorkflowRunMailInput[]; }; -type FakeSessionService = { - adoptedDeployCalls: unknown[]; - sendUserMessageCalls: unknown[]; - sendUserMessage: (params: unknown) => Promise; - stageWorkflowStep: () => Promise; - deployInstanceAtHead: () => Promise; - deployAdoptedWorkflowFromSource: (params: AdoptedDeployCall) => Promise<{ - anchorRunId: string; - deploymentAddress: string; - publicKey: string; - }>; - deployWorkflowDefinition: () => Promise; - endSession: () => Promise; -}; - -function createFakeSessionService(): FakeSessionService { - const adoptedDeployCalls: unknown[] = []; - const sendUserMessageCalls: unknown[] = []; - return { - adoptedDeployCalls, - sendUserMessageCalls, - async stageWorkflowStep() {}, - async deployInstanceAtHead() { - throw new Error( - "deployInstanceAtHead must not be called: invite, wake, and relaunch provision via Interchange prepareProvisionedDeployment", - ); - }, - async deployAdoptedWorkflowFromSource(params: AdoptedDeployCall) { - adoptedDeployCalls.push(params); +/** + * Fakes `@corbits/chat`'s `RunTriggerClient` (CL-7490): the seam + * `sendRunMail` now delivers every provisioned run's mail through, + * replacing the old raw `runTrigger.triggerMail`. Each call + * mints a fresh `messageId` (``) the same shape + * `vendor/intx/hub-api/src/workflow-run-trigger.ts`'s real route mints, + * so `mailIdFromBracketMessageId` and reply-threading assertions behave + * exactly as they would against the real route. + */ +function createFakeRunTrigger(): FakeRunTrigger { + const triggerCalls: TriggerWorkflowRunMailInput[] = []; + let counter = 0; + const fake: FakeRunTrigger = { + triggerCalls, + async triggerMail(input: TriggerWorkflowRunMailInput) { + triggerCalls.push(input); + counter += 1; return { - anchorRunId: params.anchorRunId, - deploymentAddress: params.agentAddress, - publicKey: "test-public-key", + runId: input.anchorRunId, + address: `${input.anchorRunId}@ten1.workbench.test`, + messageId: ``, }; }, - async deployWorkflowDefinition() { - throw new Error( - "deployWorkflowDefinition must not be called: launchWorkbench " + - "provisions via Interchange prepareProvisionedDeployment", - ); - }, - async sendUserMessage(params: unknown) { - sendUserMessageCalls.push(params); - return new TextEncoder().encode("raw-mime-bytes"); - }, - async endSession() {}, }; + return fake; } function createFakeSidecarRouter( @@ -782,7 +781,7 @@ function createFakeSidecarRouter( }, // Every launch and wake produces the run's `run.grants` frame before its // first mail. Always routable: the frame is sent after the deploy the - // fake `sessionService` just acked, and that deploy is what makes the + // fake `runTrigger` just acked, and that deploy is what makes the // address resident — `routableAddresses` models residency BEFORE the // wake (what `getRoutableAddresses` answers), not after it. sendRunGrants(address: string, runId: string, stepGrants: unknown) { @@ -896,7 +895,7 @@ describe("createHubChatPlatform", () => { table: agentSession, values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const deployError = new Error("sidecar unreachable"); const allocationService = createFakeWorkflowAllocationService(); allocationService.prepareProvisionedDeployment = async () => { @@ -908,7 +907,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors, workflowAllocationService: allocationService, @@ -976,7 +975,7 @@ describe("createHubChatPlatform", () => { table: agentSession, values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const allocationService = createFakeWorkflowAllocationService(); allocationService.prepareProvisionedDeployment = async () => { throw new SessionLaunchError("start", new Error("ack timeout"), true); @@ -987,7 +986,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors, workflowAllocationService: allocationService, @@ -1010,7 +1009,7 @@ describe("createHubChatPlatform", () => { // its replies are real, so it still fails loud without a catalog // source; proven alongside `launchInvite`'s other tests below. - test("sendMail resolves the workbench's run's session via the shared principal and delivers via sessionService", async () => { + test("sendMail resolves the workbench's run's session via the shared principal and delivers via runTrigger", async () => { resolveDefinitionSourcesResult = { ok: true, materials: [], @@ -1047,13 +1046,13 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors: createFakeEventCollectors(), }); @@ -1066,15 +1065,10 @@ describe("createHubChatPlatform", () => { }); expect(sent.id).toBeTruthy(); - expect(sessionService.sendUserMessageCalls).toHaveLength(1); - const call = sessionService.sendUserMessageCalls[0] as { - agentAddress: string; - content: string; - sessionId: string; - }; - expect(call.agentAddress).toBe("ins_workbench1@ten1.workbench.test"); - expect(call.content).toBe("hello workbench"); - expect(call.sessionId).toBe("ses_run1"); + expect(runTrigger.triggerCalls).toHaveLength(1); + const call = runTrigger.triggerCalls[0]; + expect(call?.anchorRunId).toBe("ins_workbench1"); + expect(call?.content).toBe("hello workbench"); const mailInsert = db.inserted.find((row) => row.table === sessionMail); expect(mailInsert?.values).toMatchObject({ @@ -1130,13 +1124,13 @@ describe("createHubChatPlatform", () => { }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors: createFakeEventCollectors(), }); @@ -1149,10 +1143,8 @@ describe("createHubChatPlatform", () => { }); expect(sent.id).toBeTruthy(); - const call = sessionService.sendUserMessageCalls[0] as { - sessionId: string; - }; - expect(call.sessionId).toBe("ses_unanchored"); + const mailInsert = db.inserted.find((row) => row.table === sessionMail); + expect(mailInsert?.values).toMatchObject({ sessionId: "ses_unanchored" }); }); test("sendMail signs with the injected crypto cache keyed by workbenchId", async () => { @@ -1190,10 +1182,14 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const getKeys: string[] = []; const injectedProvider = { getPublicKey: () => new Uint8Array([7, 2, 8, 4]), + // Exercised for real now (CL-7490): `sendRunMail` signs its own + // local `session_mail` copy directly rather than through a fake + // `sessionService.sendUserMessage`. + sign: async () => new Uint8Array(64), }; const cryptoProviders: CryptoProviderCache = { get: async (key) => { @@ -1205,7 +1201,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), cryptoProviders, @@ -1218,11 +1214,11 @@ describe("createHubChatPlatform", () => { content: { content: "hello workbench" }, }); + // The crypto cache is only consulted for the local `session_mail` + // signing copy now (CL-7490) — delivery itself goes through the + // run-trigger client, which signs its own copy independently. expect(getKeys).toEqual(["ins_workbench1"]); - const call = sessionService.sendUserMessageCalls[0] as { - cryptoProvider: unknown; - }; - expect(call.cryptoProvider).toBe(injectedProvider); + expect(runTrigger.triggerCalls).toHaveLength(1); }); test("sendMail rejects within the mail-delivery deadline instead of hanging forever when delivery never settles (CL-6644)", async () => { @@ -1260,20 +1256,20 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); // Models the observed CL-6644 symptom: the post-deploy delivery - // step (`sessionService.sendUserMessage`, reached through + // step (`runTrigger.triggerMail`, reached through // `sendFoldedMail`) never resolves and never rejects -- a wedged // ack, not a thrown "agent is unreachable" the reclaim-retry loop // already knows how to handle. Before the fix, `sendMail`'s // returned promise stayed pending forever with nothing logged. - sessionService.sendUserMessage = () => new Promise(() => {}); + runTrigger.triggerMail = () => new Promise(() => {}); const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors: createFakeEventCollectors(), mailDeliveryTimeoutMs: 20, @@ -1326,14 +1322,14 @@ describe("createHubChatPlatform", () => { wfd_echo: inertProjection({ id: "wfd_echo" }), }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const sidecarRouter = createFakeSidecarRouter({ routableAddresses: [] }); const eventCollectors = createFakeEventCollectors(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors, }); @@ -1403,7 +1399,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1471,7 +1467,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1495,7 +1491,7 @@ describe("createHubChatPlatform", () => { createHubChatPlatform({ toolGrantsForPins: () => [], db: {} as never, - sessionService: {} as never, + runTrigger: {} as never, sidecarRouter: {} as never, eventCollectors: createFakeEventCollectors(), credentialCipher: undefined as never, @@ -1508,7 +1504,7 @@ describe("createHubChatPlatform", () => { createHubChatPlatform({ toolGrantsForPins: () => [], db: {} as never, - sessionService: {} as never, + runTrigger: {} as never, sidecarRouter: {} as never, eventCollectors: createFakeEventCollectors(), credentialCipher: {} as never, @@ -1526,7 +1522,7 @@ describe("createHubChatPlatform", () => { const platform = createHubChatPlatform({ toolGrantsForPins: () => [], db: {} as never, - sessionService: {} as never, + runTrigger: {} as never, sidecarRouter: {} as never, eventCollectors: createFakeEventCollectors(), credentialCipher: { @@ -1564,7 +1560,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -1616,7 +1612,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1688,7 +1684,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1756,7 +1752,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1785,7 +1781,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -1829,7 +1825,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1878,7 +1874,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1920,7 +1916,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -1971,7 +1967,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -1997,13 +1993,13 @@ describe("createHubChatPlatform", () => { principalId: "prin_run1", }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const sidecarRouter = createFakeSidecarRouter(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors: createFakeEventCollectors(), }); @@ -2097,16 +2093,17 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); let sendAttempts = 0; - sessionService.sendUserMessage = async (params: unknown) => { - sessionService.sendUserMessageCalls.push(params); + const baseTriggerMail = runTrigger.triggerMail; + runTrigger.triggerMail = async (input) => { sendAttempts += 1; // The first attempt lands while the sidecar is still booting. if (sendAttempts === 1) { + runTrigger.triggerCalls.push(input); throw new Error("agent is unreachable"); } - return new TextEncoder().encode("raw-mime-bytes"); + return baseTriggerMail(input); }; // Not routable yet — the sidecar is still finishing its boot; // it registers a moment later, on its own, with no redeploy. @@ -2119,7 +2116,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors, lifecycle: { idleSleepMs: 60_000 }, @@ -2136,7 +2133,7 @@ describe("createHubChatPlatform", () => { // No relaunch: the booting run kept its own run id and address — // `wakeByAddress` never called `prepareProvisionedDeployment`. expect(lastAllocationService.prepareCalls).toHaveLength(0); - expect(sessionService.sendUserMessageCalls).toHaveLength(2); + expect(runTrigger.triggerCalls).toHaveLength(2); }); // CL-6267: the sidecar's own park/wake handler now owns respawning @@ -2199,7 +2196,7 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const sidecarRouter = createFakeSidecarRouter({ routableAddresses: ["ins_workbench1@ten1.workbench.test"], }); @@ -2208,7 +2205,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors, lifecycle: { idleSleepMs: 60_000 }, @@ -2224,7 +2221,7 @@ describe("createHubChatPlatform", () => { expect(sent.id).toBeTruthy(); expect(lastAllocationService.prepareCalls).toHaveLength(0); expect(sidecarRouter.sendAgentUndeployCalls).toHaveLength(0); - expect(sessionService.sendUserMessageCalls).toHaveLength(1); + expect(runTrigger.triggerCalls).toHaveLength(1); }); test("the idle sweep never undeploys an address the event collector reports as busy", async () => { @@ -2273,7 +2270,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter, eventCollectors, lifecycle: { idleSleepMs: 5, sweepIntervalMs: 5 }, @@ -2337,7 +2334,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter, eventCollectors, lifecycle: { idleSleepMs: 5, sweepIntervalMs: 5 }, @@ -2403,7 +2400,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter, eventCollectors, lifecycle: { idleSleepMs: 5, sweepIntervalMs: 5 }, @@ -2446,7 +2443,7 @@ describe("createHubChatPlatform", () => { createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -2479,7 +2476,7 @@ describe("createHubChatPlatform", () => { createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), lifecycle: { idleSleepMs: 60_000 }, @@ -2521,11 +2518,11 @@ describe("createHubChatPlatform", () => { }, }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [address], }), @@ -2590,11 +2587,11 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), lifecycle: { idleSleepMs: 60_000 }, @@ -2652,11 +2649,11 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_run1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -2679,7 +2676,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), }); @@ -2777,16 +2774,17 @@ describe("createHubChatPlatform", () => { values: { id: "ses_run1", principalId: "prin_room1" }, }); - const sessionService = createFakeSessionService(); - sessionService.sendUserMessage = async (params: unknown) => { - sessionService.sendUserMessageCalls.push(params); - const { agentAddress } = params as { agentAddress: string }; + const staleRunId = "run_gen1"; + const runTrigger = createFakeRunTrigger(); + const baseTriggerMail = runTrigger.triggerMail; + runTrigger.triggerMail = async (input) => { // The dead run never becomes reachable again — only a send to - // whatever address the relaunch actually produced can succeed. - if (agentAddress === staleAddress) { + // whatever run the relaunch actually produced can succeed. + if (input.anchorRunId === staleRunId) { + runTrigger.triggerCalls.push(input); throw new Error("agent is unreachable"); } - return new TextEncoder().encode("raw-mime-bytes"); + return baseTriggerMail(input); }; // Routable so `sendMail`'s own opening wake-before-send gate skips @@ -2815,7 +2813,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter, eventCollectors: createFakeEventCollectors(), workflowAllocationService: allocationService, @@ -2831,22 +2829,18 @@ describe("createHubChatPlatform", () => { }); expect(sent.id).toBeTruthy(); - expect(sessionService.sendUserMessageCalls).toHaveLength(2); - const [first, second] = sessionService.sendUserMessageCalls as { - agentAddress: string; - }[]; - expect(first?.agentAddress).toBe(staleAddress); + expect(runTrigger.triggerCalls).toHaveLength(2); + const [first, second] = runTrigger.triggerCalls; + expect(first?.anchorRunId).toBe(staleRunId); // The retry that follows the relaunch targets the run that - // replaced `run_gen1`, never the dead address the loop started - // with. - expect(second?.agentAddress).not.toBe(staleAddress); + // replaced `run_gen1`, never the dead run the loop started with. + expect(second?.anchorRunId).not.toBe(staleRunId); const repointed = db.updated.at(-1)?.values as { currentRunId: string; priorRunIds: string[]; }; - const secondAddress = second?.agentAddress ?? ""; - expect(repointed.currentRunId).toBe(secondAddress.split("@")[0] ?? ""); + expect(repointed.currentRunId).toBe(second?.anchorRunId ?? ""); expect(repointed.priorRunIds).toEqual(["run_gen1"]); }); }); @@ -2910,7 +2904,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -2956,11 +2950,11 @@ describe("createHubChatPlatform", () => { table: agentSession, values: { id: "ses_agent1", principalId: "prin_agent1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), eventCollectors: createFakeEventCollectors(), lifecycle: { idleSleepMs: 60_000 }, @@ -3057,7 +3051,7 @@ describe("createHubChatPlatform", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter(), eventCollectors: createFakeEventCollectors(), }); @@ -3182,7 +3176,7 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter, eventCollectors: createFakeEventCollectors(), }); @@ -3256,11 +3250,11 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { table: agentSession, values: { id: "ses_stale", principalId: "prin_room1" }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, sidecarRouter: createFakeSidecarRouter({ routableAddresses: ["run_stale@ten1.workbench.test"], }), @@ -3347,7 +3341,7 @@ describe("createHubChatPlatform stale-definition reconciliation", () => { const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: ["run_standalone@ten1.workbench.test"], }), @@ -3407,12 +3401,12 @@ describe("createHubChatPlatform relaunch sweep", () => { noopInference: opts.noopInference ?? true, }, }); - const sessionService = createFakeSessionService(); + const runTrigger = createFakeRunTrigger(); const notices: unknown[] = []; const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService, + runTrigger, // Routable, and dead anyway: that combination is exactly what the // wake path cannot fix — boot restore re-announced the address, so // nothing looks broken until the next message is dropped. @@ -3422,7 +3416,7 @@ describe("createHubChatPlatform relaunch sweep", () => { eventCollectors: createFakeEventCollectors(), relaunchNotice: { current: (notice) => notices.push(notice) }, }); - return { db, platform, sessionService, notices }; + return { db, platform, runTrigger, notices }; } test("relaunches a routable-but-dead participant and tells the room", async () => { @@ -3624,7 +3618,7 @@ describe("createHubChatPlatform inference-source rotation reconciliation", () => const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: ["run_live@ten1.workbench.test"], }), @@ -3853,7 +3847,7 @@ describe("createHubChatPlatform pinned-tool-package connect reconciliation", () const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, - sessionService: createFakeSessionService(), + runTrigger: createFakeRunTrigger(), sidecarRouter: createFakeSidecarRouter({ routableAddresses: ["run_live@ten1.workbench.test"], }), diff --git a/packages/chat/test/relaunch-close.test.ts b/packages/chat/test/relaunch-close.test.ts index 35f2c0627..b4a81f59f 100644 --- a/packages/chat/test/relaunch-close.test.ts +++ b/packages/chat/test/relaunch-close.test.ts @@ -128,7 +128,7 @@ function createPlatform(db: never) { return createHubChatPlatform({ db, toolGrantsForPins: () => [], - sessionService: {} as never, + runTrigger: {} as never, repoStore: { resolveRef: async () => "sha_test" }, sidecarRouter: { getRoutableAddresses: () => [] } as never, eventCollectors: {} as never, From 651ea2ed2c83d6e8b975f165ca0ea508d55337ad Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 16:15:59 -0700 Subject: [PATCH 47/48] Chat delivers room mail through Interchange's run-trigger route (CL-7490) A provisioned run is deployed to its sidecar lazily by its first native trigger (vendor/intx/hub-api's createWorkflowRunTrigger: anchor -> grants/principal -> durable dispatch's agent.deploy -> deliver mail). Chat sent via sessionService.sendUserMessage, which delivers straight over the sidecar's ordinary channel and never reaches that durable-dispatch branch, so an invited run never triggered natively never got its first agent.deploy and stayed permanently unreachable. createWorkflowRunTrigger itself is not exported from @intx/hub-api (it backs the route, not a published seam), so chat now calls the route itself: a new RunTriggerClient (run-trigger-client.ts) posts to POST /api/tenants/:tenantId/workflows/runs/:runId/mail in-process against the hub's own composed Hono app. The route authenticates through @intx/hub-api's ordinary session contract, which chat cannot present a browser cookie for (most sends have no inbound HTTP request in scope at all), so the client signs a short-lived internal token naming the principal to authenticate as (run-trigger-internal-auth.ts); apps/hub's own getSession implementation -- a seam @intx/hub-api documents as pluggable -- verifies that token before falling through to the real cookie-session path. Neither side touches vendor/intx. send-run-mail.ts now delivers through the trigger client and uses its returned messageId for session_mail correlation, still assembling a local signed MIME copy (with @intx/mime/@intx/crypto directly) purely for this workbench's own history and attachment blob reads -- the trigger route returns no raw bytes, and nothing here delivers that local copy anywhere a second time. --- apps/hub/src/index.ts | 53 +++++- packages/chat/src/index.ts | 14 ++ packages/chat/src/platform-adapter.ts | 88 ++++++++- packages/chat/src/run-trigger-client.ts | 169 ++++++++++++++++++ .../chat/src/run-trigger-internal-auth.ts | 73 ++++++++ packages/chat/src/send-run-mail.ts | 128 +++++++++---- 6 files changed, 490 insertions(+), 35 deletions(-) create mode 100644 packages/chat/src/run-trigger-client.ts create mode 100644 packages/chat/src/run-trigger-internal-auth.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index fbf44a094..b864db59b 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -23,6 +23,7 @@ import { model, modelPricing, tenant as tenantTable, + user as userTable, workflowDefinition, } from "@intx/db/schema"; import { and, eq, inArray } from "drizzle-orm"; @@ -92,6 +93,7 @@ import { createHubChatPlatform, createNoopInferenceRoutes, createRelaunchNoticePoster, + createRunTriggerClient, createWorkflowParticipantRoutes, isWorkbenchHostDefinitionName, listConnectedProviders, @@ -105,6 +107,7 @@ import { workbenchLaunchPersistExtra, createCryptoProviderCache, tagCredentialCipher, + verifyInternalRunTriggerToken, } from "@corbits/chat"; import { createDrizzleMailboxWriter, @@ -1129,7 +1132,47 @@ export async function createHub(config: HubConfig) { workflowAllocationService, workflowDispatchService, credentialCipher, + // `@intx/hub-api`'s `GetSession` is a deliberately pluggable seam + // (its own doc comment: "so a third-party identity provider can be + // plugged in"), and this composition root owns it. `@corbits/chat`'s + // run-trigger client (CL-7490) has no browser cookie to present for + // most of its calls — mention fan-out, a relaunch resend, the + // mailbox-fanout persist seam all run with no inbound HTTP request + // in scope — so it signs a short-lived internal token instead (see + // `@corbits/chat`'s `run-trigger-internal-auth.ts`) naming the + // better-auth user it authenticates as. This checks that token + // FIRST and only falls through to the real cookie-session path when + // the header is absent; a PRESENT-but-invalid token fails closed + // (401 via a null session) rather than silently retrying as a + // session, the same posture `workflow-run-deploy-auth` documents for + // its own bearer mirror in `vendor/intx/hub-api`. getSession: async (headers) => { + const internalRunTriggerToken = headers.get( + "x-corbits-internal-run-trigger", + ); + if (internalRunTriggerToken !== null) { + const userId = verifyInternalRunTriggerToken( + config.sessionSecret, + internalRunTriggerToken, + ); + if (userId === null) return null; + const userRow = await db.query.user.findFirst({ + where: eq(userTable.id, userId), + }); + if (userRow === undefined) return null; + const now = new Date(); + return { + user: userRow, + session: { + id: `internal_run_trigger_${userRow.id}`, + createdAt: now, + updatedAt: now, + userId: userRow.id, + expiresAt: new Date(now.getTime() + 30_000), + token: internalRunTriggerToken, + }, + }; + } const result = await auth.api.getSession({ headers }); return result ? { user: result.user, session: result.session } : null; }, @@ -1430,9 +1473,17 @@ export async function createHub(config: HubConfig) { // across consumers. TTL-bounded by `createCryptoProviderCache` itself // (CL-7223). const cryptoProviders = createCryptoProviderCache(); + // Delivers chat's outbound run mail through Interchange's own + // workflow-run mail-trigger route (CL-7490), in-process against this + // same `app` — see `@corbits/chat`'s `run-trigger-client.ts` and the + // `getSession` internal-token check above for how it authenticates. + const runTrigger = createRunTriggerClient({ + app, + internalAuthSecret: config.sessionSecret, + }); const chatPlatform = createHubChatPlatform({ db, - sessionService, + runTrigger, repoStore: agentRepoStore.repoStore, sidecarRouter, eventCollectors, diff --git a/packages/chat/src/index.ts b/packages/chat/src/index.ts index cc3770f38..6f4aa1636 100644 --- a/packages/chat/src/index.ts +++ b/packages/chat/src/index.ts @@ -322,6 +322,20 @@ export type { CreateHubChatPlatformDeps, HubChatPlatform, } from "./platform-adapter"; +export { + createRunTriggerClient, + RunTriggerError, +} from "./run-trigger-client"; +export type { + RunTriggerClient, + RunTriggerClientDeps, + TriggerWorkflowRunMailInput, + TriggeredWorkflowRunMail, +} from "./run-trigger-client"; +export { + signInternalRunTriggerToken, + verifyInternalRunTriggerToken, +} from "./run-trigger-internal-auth"; export { createCryptoProviderCache } from "./crypto-cache"; export type { CryptoProviderCache } from "./crypto-cache"; export { tagCredentialCipher } from "./credential-cipher-tag"; diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 5e8f97181..5809b292c 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -31,6 +31,7 @@ import type { CryptoProviderCache } from "./crypto-cache"; import { InferenceResolutionError } from "./model-unavailable"; import type { ToolGrantsForPins } from "./pin-ports"; import { resolveRunSessionIdOrThrow } from "./run-session"; +import type { RunTriggerClient } from "./run-trigger-client"; import { sendRunMail, type SendRunMailParams } from "./send-run-mail"; import { findStandingLaunchByDefinition, @@ -53,6 +54,7 @@ import type { DB } from "@intx/db"; import { listVisibleOfferings } from "@intx/db"; import { asset as assetTable, + principal as principalTable, sessionMail, tenant as tenantTable, workflowDefinition, @@ -68,7 +70,6 @@ import { DEFAULT_ASSET_REF, type EventCollectorRegistry, type RepoStore, - type SessionService, type SidecarRouter, type WorkflowAllocationService, } from "@intx/hub-sessions"; @@ -94,7 +95,15 @@ function domainOf(address: string): string { export type CreateHubChatPlatformDeps = { db: DB["db"]; - sessionService: Pick; + /** + * Delivers outbound run mail through Interchange's own workflow-run + * mail-trigger route (CL-7490) — see `./run-trigger-client.ts` for why + * a raw `sessionService.sendUserMessage` can no longer stand in for + * this: it never reaches the trigger route's durable-dispatch branch, + * so a provisioned run invited but never yet triggered natively never + * gets its first `agent.deploy`. + */ + runTrigger: RunTriggerClient; repoStore: Pick; /** * Interchange's provisioned-deployment front. Invite, wake, and @@ -263,10 +272,68 @@ export function createHubChatPlatform( void tagCredentialCipher(deps.credentialCipher); const runMailDeps = { db: deps.db, - sessionService: deps.sessionService, + runTrigger: deps.runTrigger, sidecarRouter: deps.sidecarRouter, }; + /** + * Resolves the principal a run-trigger's DB row names, then the + * better-auth `user.id` behind it — the identity + * `./run-trigger-client.ts` authenticates the trigger call as. Only a + * `kind: "user"` principal has a real login identity to authenticate + * as; anything else is a wiring bug this throws loud on rather than + * silently authenticating as nobody. + */ + async function refIdForUserPrincipal(principalId: string): Promise { + const rows = await deps.db + .select({ refId: principalTable.refId, kind: principalTable.kind }) + .from(principalTable) + .where(eq(principalTable.id, principalId)) + .limit(1); + const row = rows[0]; + if (row === undefined || row.kind !== "user") { + throw new Error( + `principal "${principalId}" is not a user principal chat can authenticate the run trigger as`, + ); + } + return row.refId; + } + + /** + * The identity a `sendMail` call authenticates its run-trigger + * delivery as (see `./run-trigger-client.ts`'s `authAsUserId`): + * the sending human when `sendMail` named one directly, and + * otherwise the run's own workflow definition asset's + * `creatorPrincipalId` — the same authority + * `sourceAuthorityForAsset` already resolves for provisioning. + */ + async function resolveTriggerAuthUserId(input: { + readonly principalId: string | undefined; + readonly definitionId: string | null; + }): Promise { + if (input.principalId !== undefined) { + return refIdForUserPrincipal(input.principalId); + } + if (input.definitionId === null) { + throw new Error( + "run has no definitionId to resolve a run-trigger authority from", + ); + } + const rows = await deps.db + .select({ assetId: workflowDefinition.assetId }) + .from(workflowDefinition) + .where(eq(workflowDefinition.id, input.definitionId)) + .limit(1); + const assetId = rows[0]?.assetId; + if (assetId === null || assetId === undefined) { + throw new Error( + `workflow definition "${input.definitionId}" has no asset`, + ); + } + const creatorPrincipalId = await sourceAuthorityForAsset(assetId); + return refIdForUserPrincipal(creatorPrincipalId); + } + function offeringDigest(sourceOfferingIds: readonly string[]): string { // `sources_digest` is a `text` column and this digest rides straight // into it (`platform-adapter.ts`'s `insert(workbenchLaunch)`). A @@ -974,7 +1041,11 @@ export function createHubChatPlatform( */ async function sendRunMailWithReclaimRetry( params: SendRunMailParams, - reresolve: () => Promise<{ agentAddress: string; sessionId: string }>, + reresolve: () => Promise<{ + agentAddress: string; + anchorRunId: string; + sessionId: string; + }>, ): Promise>> { let current = params; let wakedOnFailure = false; @@ -1012,6 +1083,7 @@ export function createHubChatPlatform( current = { ...current, agentAddress: fresh.agentAddress, + anchorRunId: fresh.anchorRunId, sessionId: fresh.sessionId, }; throw err; @@ -1370,14 +1442,21 @@ export function createHubChatPlatform( }), ); + const authAsUserId = await resolveTriggerAuthUserId({ + principalId: input.principalId, + definitionId: delivery.run.definitionId, + }); + const sendMailBase = { tenantId: input.tenantId, sessionId, agentAddress: deliveryAddress, + anchorRunId: delivery.run.id, from, domain, content: input.content.content, cryptoProvider, + authAsUserId, }; const withAttachments = attachments !== undefined @@ -1424,6 +1503,7 @@ export function createHubChatPlatform( } return { agentAddress: relive.binding.liveAddress, + anchorRunId: relive.run.id, sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), }; }, diff --git a/packages/chat/src/run-trigger-client.ts b/packages/chat/src/run-trigger-client.ts new file mode 100644 index 000000000..135cda2dc --- /dev/null +++ b/packages/chat/src/run-trigger-client.ts @@ -0,0 +1,169 @@ +// A thin client over Interchange's own workflow-run mail-trigger route +// (`POST /api/tenants/:tenantId/workflows/runs/:runId/mail`, mounted by +// `vendor/intx/hub-api`'s `createRunRoutes`). CL-7490: a provisioned +// run's first `agent.deploy` only fires from inside that route's +// durable-dispatch branch (`createWorkflowRunTrigger` in +// `vendor/intx/hub-api/src/workflow-run-trigger.ts`) — a raw +// `sessionService.sendUserMessage` never reaches it, so an invited run +// that has never been triggered natively stays undeployed and +// unreachable forever. `createWorkflowRunTrigger` itself is not +// exported from `@intx/hub-api` (it is the route's own private +// dependency, not a published seam), so this calls the mounted HTTP +// route instead of reimplementing what it does. +// +// The call is in-process against the hub's own composed Hono `app` — +// no network hop, no separate origin to configure — authenticated the +// way any other request to that route is: through `@intx/hub-api`'s +// session contract. Chat has no browser cookie to present (most sends +// happen with no inbound HTTP request in scope at all — mention +// fan-out, a relaunch resend, the mailbox-fanout persist seam), so it +// presents a short-lived signed token instead (see +// `./run-trigger-internal-auth.ts`) that `apps/hub`'s own `getSession` +// implementation verifies before falling through to the real +// cookie-session path. Neither side touches `vendor/intx`. +import { type } from "arktype"; + +import type { MailContent } from "./codec"; +import { signInternalRunTriggerToken } from "./run-trigger-internal-auth"; + +// Mirrors `vendor/intx/hub-api/src/workflow-run-trigger.ts`'s own +// `WorkflowRunTriggerResponse` — not imported directly since +// `@intx/hub-api` does not re-export it from its package entry point. +// Parsed here rather than trusted as `as` cast: this is a trust +// boundary (an HTTP response), even though the call never leaves the +// process. +const TriggerResponse = type({ + runId: "string", + address: "string", + messageId: "string", +}); + +const TriggerErrorBody = type({ + error: { + code: "string", + message: "string", + }, +}); + +export class RunTriggerError extends Error { + constructor( + readonly status: number, + readonly code: string, + message: string, + ) { + super(`workflow-run trigger failed (${status} ${code}): ${message}`); + this.name = "RunTriggerError"; + } +} + +export type RunTriggerClientDeps = { + /** + * The hub's own composed Hono app (`@intx/hub-api`'s `App`, built by + * `createApp`). Calling `.request()` on it dispatches straight + * through the route's own middleware chain in-process — the same + * path a real inbound request to the hub takes. + */ + app: { + request: ( + input: string, + init?: RequestInit, + ) => Response | Promise; + }; + /** + * The shared secret `./run-trigger-internal-auth.ts` signs the + * per-call token with. Must be the same value `apps/hub` wires into + * its `getSession` verifier — the host's composition root, never a + * package default, owns that pairing. + */ + internalAuthSecret: string; + /** + * Mints the token asserting which better-auth `user.id` this trigger + * call authenticates as. Separated from `internalAuthSecret` only so + * a test can supply a fixed clock; production always uses + * `signInternalRunTriggerToken` from `./run-trigger-internal-auth.ts`. + */ + signToken?: (secret: string, userId: string) => string; +}; + +export type TriggerWorkflowRunMailInput = { + readonly tenantId: string; + /** The deployment's anchor run id (`:runId` in the route path). */ + readonly anchorRunId: string; + readonly content: string; + readonly attachments?: MailContent["attachments"]; + /** + * The better-auth `user.id` this call authenticates as. The route + * derives the delivered mail's `From` header from whatever principal + * this resolves to in the target tenant, so this should be the real + * sending human when one exists (`sendMail`'s own `principalId`), + * and the workflow definition asset's `creatorPrincipalId` (resolved + * to its owning user) otherwise — see the callers in + * `./platform-adapter.ts`. + */ + readonly authAsUserId: string; +}; + +export type TriggeredWorkflowRunMail = { + readonly runId: string; + readonly address: string; + readonly messageId: string; +}; + +export type RunTriggerClient = { + triggerMail( + input: TriggerWorkflowRunMailInput, + ): Promise; +}; + +async function readJson(response: Response): Promise { + try { + return await response.json(); + } catch { + return null; + } +} + +export function createRunTriggerClient( + deps: RunTriggerClientDeps, +): RunTriggerClient { + return { + async triggerMail(input): Promise { + const signToken = deps.signToken ?? signInternalRunTriggerToken; + const token = signToken(deps.internalAuthSecret, input.authAsUserId); + const body: { content: string; attachments?: MailContent["attachments"] } = + input.attachments !== undefined + ? { content: input.content, attachments: input.attachments } + : { content: input.content }; + + const response = await deps.app.request( + `/api/tenants/${input.tenantId}/workflows/runs/${input.anchorRunId}/mail`, + { + method: "POST", + headers: { + "content-type": "application/json", + "x-corbits-internal-run-trigger": token, + }, + body: JSON.stringify(body), + }, + ); + + const json = await readJson(response); + if (response.status !== 202) { + const parsedError = TriggerErrorBody(json); + const { code, message } = + parsedError instanceof type.errors + ? { code: "unknown", message: response.statusText } + : parsedError.error; + throw new RunTriggerError(response.status, code, message); + } + + const parsed = TriggerResponse(json); + if (parsed instanceof type.errors) { + throw new Error( + `malformed workflow-run trigger response: ${parsed.summary}`, + ); + } + return parsed; + }, + }; +} diff --git a/packages/chat/src/run-trigger-internal-auth.ts b/packages/chat/src/run-trigger-internal-auth.ts new file mode 100644 index 000000000..b48242b52 --- /dev/null +++ b/packages/chat/src/run-trigger-internal-auth.ts @@ -0,0 +1,73 @@ +// A server-side identity credential for chat's own calls into the hub's +// `/api/tenants/:tenantId/workflows/runs/:runId/mail` route (see +// `./run-trigger-client.ts`). That route authenticates a caller through +// `@intx/hub-api`'s ordinary session-cookie path — the right answer for a +// human's browser, but chat's own dispatch (mention fan-out, relaunch +// resends, the mailbox-fanout persist seam) runs with no browser request +// in scope at all and cannot present one. +// +// `@intx/hub-api`'s `GetSession` contract is deliberately pluggable (see +// its own doc comment: "so a third-party identity provider can be +// plugged in"), and `apps/hub` is the one composing it (`getSession` is +// supplied there, not fixed by the vendored package). This module is the +// shared signer/verifier for a short-lived, HMAC-signed token naming the +// principal to authenticate as: chat signs one per trigger call +// (`./run-trigger-client.ts`), and `apps/hub`'s `getSession` verifies it +// before falling through to the real cookie-session path. Neither side +// touches `vendor/intx` — this is host-owned wiring through an +// already-open extension seam, not a fork of the session contract. +// +// The signed identity is a better-auth `user.id` (what `resolveTenant` +// looks up a tenant `principal` by by `refId`), never a raw +// `chat.workbench_launch`/`workflow_run` id — those name a run or a +// room, not an authenticated user. +import { createHmac, timingSafeEqual } from "node:crypto"; + +const TOKEN_TTL_MS = 30_000; + +function sign(secret: string, payload: string): string { + return createHmac("sha256", secret).update(payload).digest("base64url"); +} + +/** + * Mint a token asserting `userId` is the caller, valid for + * `TOKEN_TTL_MS` from `now`. Bound to a real wall-clock deadline rather + * than being a durable credential — this authenticates one trigger call, + * not a session a caller could hold onto. + */ +export function signInternalRunTriggerToken( + secret: string, + userId: string, + now: number = Date.now(), +): string { + const expiresAt = now + TOKEN_TTL_MS; + const payload = `${userId}.${expiresAt}`; + return `${payload}.${sign(secret, payload)}`; +} + +/** + * Verify a token minted by `signInternalRunTriggerToken`, returning the + * asserted `userId` when the signature checks out and the token has not + * expired, `null` otherwise. Constant-time signature comparison so a + * caller cannot learn anything about the expected signature from timing. + */ +export function verifyInternalRunTriggerToken( + secret: string, + token: string, + now: number = Date.now(), +): string | null { + const parts = token.split("."); + if (parts.length !== 3) return null; + const [userId, expiresAtRaw, signature] = parts; + if (userId === undefined || userId === "") return null; + if (expiresAtRaw === undefined || signature === undefined) return null; + const expiresAt = Number(expiresAtRaw); + if (!Number.isFinite(expiresAt) || expiresAt < now) return null; + + const expected = sign(secret, `${userId}.${expiresAtRaw}`); + const presented = Buffer.from(signature); + const wanted = Buffer.from(expected); + if (presented.length !== wanted.length) return null; + if (!timingSafeEqual(presented, wanted)) return null; + return userId; +} diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index 139dfd98a..86412bd9d 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,13 +1,36 @@ -// Signs one inbound message via Interchange `sendUserMessage` and -// records it on session_mail so the run's own live subscribers see it. +// Delivers one inbound message to a provisioned run through Interchange's +// own workflow-run mail-trigger route (`./run-trigger-client.ts`), then +// records a local, chat-signed copy on `session_mail` so this workbench's +// own subscribers (the `mail.delivered` event and `fetchBlob`'s outbound +// attachment reads) see it exactly as before (CL-7490). +// +// CL-7490: `sessionService.sendUserMessage` used to both sign AND +// deliver. It never reached the trigger route's durable-dispatch branch, +// so a provisioned run invited but never yet triggered natively never +// got its first `agent.deploy` and stayed permanently unreachable. The +// trigger route now owns delivery (and, on its first call, the deploy); +// it returns no raw MIME bytes, so this still assembles and signs its +// own local copy — with `@intx/mime`/`@intx/crypto`, the same primitives +// the route itself uses — purely for that local record. It is never +// delivered a second time: the sidecar send lives entirely on the +// trigger route's side of this call. +import { + assembleMessage, + assembleSignedContent, + createDetachedSignatureFromProvider, + type MessageHeaders, +} from "@intx/mime"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; -import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; +import type { SidecarRouter } from "@intx/hub-sessions"; import type { CryptoProvider, MessageAttachment } from "@intx/types/runtime"; +import { mailIdFromBracketMessageId } from "./turn-mail-correlation"; +import type { RunTriggerClient } from "./run-trigger-client"; + export type RunMailDeps = { db: DB["db"]; - sessionService: Pick; + runTrigger: RunTriggerClient; sidecarRouter: Pick; }; @@ -15,6 +38,9 @@ export type SendRunMailParams = { tenantId: string; sessionId: string; agentAddress: string; + /** The deployment's anchor run id — `:runId` in the trigger route's + * path, distinct from `agentAddress` (which names the deployment). */ + anchorRunId: string; from: string; domain: string; content: string; @@ -24,6 +50,12 @@ export type SendRunMailParams = { inReplyTo?: string; references?: readonly string[]; cryptoProvider: CryptoProvider; + /** + * The better-auth `user.id` this delivery authenticates the trigger + * call as — see `./run-trigger-client.ts`'s own doc comment on + * `authAsUserId` for how a caller resolves this. + */ + authAsUserId: string; }; export type SentRunMail = { @@ -31,36 +63,65 @@ export type SentRunMail = { readonly createdAt: string; }; -async function deliverRunMailMIME( - deps: Pick, +function toWireAttachments( + attachments: MessageAttachment[] | undefined, +): { mimeType: string; data: string; name?: string }[] | undefined { + if (attachments === undefined || attachments.length === 0) return undefined; + return attachments.map((attachment) => ({ + mimeType: attachment.contentType, + data: Buffer.from(attachment.data).toString("base64"), + ...(attachment.name !== undefined ? { name: attachment.name } : {}), + })); +} + +/** + * Assemble and sign a local MIME copy of the message this delivery sent, + * for `session_mail` — never delivered anywhere itself. Threading + * headers (`inReplyTo`/`references`) are preserved here even though the + * trigger route's own `SendMessage` body carries neither (CL-7490: "a + * trigger occurrence is threading-less at the mail boundary" — + * `vendor/intx/hub-api/src/workflow-run-trigger.ts`), so the run itself + * never sees this message as a reply; only this workbench's own local + * history does. + */ +async function signLocalRunMailCopy( params: SendRunMailParams, - mailId: string, + messageId: string, now: Date, ): Promise { - const userMessageParams = { - agentAddress: params.agentAddress, + const headers: MessageHeaders = { from: params.from, - messageId: params.messageId ?? `<${mailId}@${params.domain}>`, + to: [params.agentAddress], + cc: undefined, date: now, - content: params.content, - sessionId: params.sessionId, - tenantId: params.tenantId, - cryptoProvider: params.cryptoProvider, + messageId, + subject: undefined, + inReplyTo: params.inReplyTo ?? params.replyTo, + references: + params.references !== undefined ? [...params.references] : undefined, + mimeVersion: "1.0", + interchangeType: "conversation.message", + interchangeCorrelationId: undefined, + interchangeTenantId: params.tenantId, + interchangeAgentId: undefined, + interchangeSessionId: params.sessionId, + interchangeOfferingId: undefined, + interchangeSchemaVersion: undefined, + traceparent: undefined, + tracestate: undefined, }; - const withAttachments = - params.attachments !== undefined - ? { ...userMessageParams, attachments: params.attachments } - : userMessageParams; - const inReplyTo = params.inReplyTo ?? params.replyTo; - const withReplyTo = - inReplyTo !== undefined - ? { ...withAttachments, inReplyTo } - : withAttachments; - const withReferences = - params.references !== undefined && params.references.length > 0 - ? { ...withReplyTo, references: [...params.references] } - : withReplyTo; - return deps.sessionService.sendUserMessage(withReferences); + const signedContent = assembleSignedContent({ + kind: "conversation", + text: params.content, + ...(params.attachments !== undefined + ? { attachments: params.attachments } + : {}), + }); + const signature = await createDetachedSignatureFromProvider( + signedContent, + params.cryptoProvider, + ); + return assembleMessage(headers, signedContent, signature); } async function recordRunMail( @@ -97,9 +158,16 @@ export async function sendRunMail( deps: RunMailDeps, params: SendRunMailParams, ): Promise { - const mailId = crypto.randomUUID(); const now = new Date(); - const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); + const triggered = await deps.runTrigger.triggerMail({ + tenantId: params.tenantId, + anchorRunId: params.anchorRunId, + content: params.content, + attachments: toWireAttachments(params.attachments), + authAsUserId: params.authAsUserId, + }); + const mailId = mailIdFromBracketMessageId(triggered.messageId); + const rawMIME = await signLocalRunMailCopy(params, triggered.messageId, now); // A run's session and event collector are ensured lazily now, at the // hub seams that actually see the run become mail-routable // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` From 6122e092511891a532e28f0afa8e71e36176fd19 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 16:26:00 -0700 Subject: [PATCH 48/48] Fix lint formatting and report-error finding in run-trigger-client Prettier formatting on index.ts/run-trigger-client.ts, and the report-error check's one real finding: readJson's parse-failure catch now carries a report-error-ignore note explaining that a malformed response body is still caught (loudly) by the caller's own arktype parse of the response. --- packages/chat/src/index.ts | 5 +---- packages/chat/src/run-trigger-client.ts | 10 +++++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/chat/src/index.ts b/packages/chat/src/index.ts index 6f4aa1636..ac5f1935a 100644 --- a/packages/chat/src/index.ts +++ b/packages/chat/src/index.ts @@ -322,10 +322,7 @@ export type { CreateHubChatPlatformDeps, HubChatPlatform, } from "./platform-adapter"; -export { - createRunTriggerClient, - RunTriggerError, -} from "./run-trigger-client"; +export { createRunTriggerClient, RunTriggerError } from "./run-trigger-client"; export type { RunTriggerClient, RunTriggerClientDeps, diff --git a/packages/chat/src/run-trigger-client.ts b/packages/chat/src/run-trigger-client.ts index 135cda2dc..7d22d2951 100644 --- a/packages/chat/src/run-trigger-client.ts +++ b/packages/chat/src/run-trigger-client.ts @@ -119,6 +119,11 @@ async function readJson(response: Response): Promise { try { return await response.json(); } catch { + // report-error-ignore: a response body that fails to parse as JSON + // is handled by the caller's own arktype parse — `TriggerErrorBody` + // or `TriggerResponse` rejecting `null` — which throws a real, + // reported error naming the malformed response; this local catch + // only stops a parse failure itself from masking that path. return null; } } @@ -130,7 +135,10 @@ export function createRunTriggerClient( async triggerMail(input): Promise { const signToken = deps.signToken ?? signInternalRunTriggerToken; const token = signToken(deps.internalAuthSecret, input.authAsUserId); - const body: { content: string; attachments?: MailContent["attachments"] } = + const body: { + content: string; + attachments?: MailContent["attachments"]; + } = input.attachments !== undefined ? { content: input.content, attachments: input.attachments } : { content: input.content };