From b1e6caa5d695b74c2023ec24af33b09afb344754 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:15 -0700 Subject: [PATCH 01/41] Add test for workflowDeployBody's catalog-offering shape --- scripts/e2e/harness.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 5b4d148005..37f4f93e4e 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -11,6 +11,7 @@ import { createCleanupHarness, parseEnvFileDatabaseUrl, runCleanups, + workflowDeployBody, type SpawnedApp, } from "./harness.ts"; @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => { }); }); +describe("workflowDeployBody", () => { + test("carries catalog offering ids, never a raw provider/apiKey source", () => { + const body = workflowDeployBody({ + assetId: "asset-1", + commitSha: "deadbeef", + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).toEqual({ + source: { + kind: "asset", + assetId: "asset-1", + package: { format: "source", commitSha: "deadbeef" }, + }, + entry: (body as { entry: string }).entry, + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).not.toHaveProperty("sources"); + expect(body).not.toHaveProperty("defaultSource"); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( From fae77c0682222b8149cb08b6b169e1caab79d33b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:23 -0700 Subject: [PATCH 02/41] e2e: deploy workflows by catalog offering ids The native workflow deploy route now requires sourceOfferingIds/ defaultSourceOfferingId resolved against the tenant's own catalog, rather than a raw provider/baseURL/apiKey source chain. Update workflowDeployBody accordingly, and extract the zero-cost noop catalog chain smoke-webhook.test.ts already built (an anthropic- plugin provider pointed at the hub's noop-inference endpoint) into a shared seedNoopCatalogOffering harness helper. Every deploying e2e suite now seeds or resolves a real catalog offering before deploying: walking-skeleton keeps a never-real placeholder host (assertNeverRealProvider still guards it), chat resolves the offering seedCatalog already planted rather than seeding a second chain, and heartbeat/workbench-digest/smoke-webhook seed the noop chain via the shared helper. cl-6324-launch-proof's section-mode deploy now reuses the offering id the proof already resolved when it narrowed the bench catalog to its pinned model. --- scripts/e2e/chat.test.ts | 48 +++++++++-- scripts/e2e/cl-6324-launch-proof.ts | 29 ++++--- scripts/e2e/harness.ts | 121 +++++++++++++++++++++++---- scripts/e2e/heartbeat.test.ts | 19 +++-- scripts/e2e/smoke-webhook.test.ts | 89 +++----------------- scripts/e2e/walking-skeleton.test.ts | 23 +++-- scripts/e2e/workbench-digest.test.ts | 19 +++-- 7 files changed, 219 insertions(+), 129 deletions(-) diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index deea7c8268..da9b255c89 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { log: () => undefined, }); + // The echo deploy resolves against the catalog offering `seedCatalog` + // just planted for the same provider/model this workflow declares, + // rather than seeding a second inference chain. + const echoModelsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/models`, + undefined, + user1.cookies, + ); + expectStatus("list catalog models", echoModelsListed, 200); + const echoModelRows = ( + echoModelsListed.data as { data: { id: string; canonicalName: string }[] } + ).data; + const echoModelId = echoModelRows.find( + (row) => row.canonicalName === "claude-sonnet-5", + )?.id; + if (echoModelId === undefined) { + throw new Error( + `no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`, + ); + } + + const echoOfferingsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings`, + undefined, + user1.cookies, + ); + expectStatus("list catalog offerings", echoOfferingsListed, 200); + const echoOfferingRows = ( + echoOfferingsListed.data as { data: { id: string; modelId: string }[] } + ).data; + const echoOfferingId = echoOfferingRows.find( + (row) => row.modelId === echoModelId, + )?.id; + if (echoOfferingId === undefined) { + throw new Error( + `no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`, + ); + } + // Seed the echo workflow as a deployed, invitable definition: the // same asset-publish → git-token → smart-HTTP push → native deploy // path `scripts/e2e/walking-skeleton.test.ts` proves end to end. @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { workflowDeployBody({ assetId: echoAssetId, commitSha: echoPushed.commitSha, - sourceId: "src-echo-e2e", - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [echoOfferingId], + defaultSourceOfferingId: echoOfferingId, }), user1.cookies, ); diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index 6743150908..ef3be6297c 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,7 +369,7 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - await hop("narrow the bench catalog to the pinned model", async () => { + const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { await plantGrant("model-offering:*", "read"); await plantGrant("model-offering:*", "manage"); await plantGrant("model:*", "read"); @@ -409,8 +409,13 @@ async function main(): Promise { "data", "model offerings", ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; for (const offering of offeringRows) { - if (offering.modelId === pinned.id || offering.disabled) continue; + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; const patched = await api( hub.baseUrl, "PATCH", @@ -420,10 +425,17 @@ async function main(): Promise { ); expectStatus(`disable offering ${offering.id}`, patched, 200); } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } console.log( ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); + return pinnedOffering.id; }); const assistantDefinitionId = await hop( @@ -694,7 +706,6 @@ async function main(): Promise { // own durable event log, which is the artefact the milestone's // `RunStarted` assertion was always about. const SECTION_ASSET_NAME = "cl6324-section"; - const SECTION_SOURCE_ID = "cl6324-section-source"; const SECTION_TURN_TIMEOUT_MS = 180_000; const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`; @@ -780,16 +791,8 @@ async function main(): Promise { package: { format: "source", commitSha: pushed.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SECTION_SOURCE_ID, - provider: model.provider, - baseURL: model.baseURL, - apiKey: model.apiKey, - model: model.model, - }, - ], - defaultSource: SECTION_SOURCE_ID, + sourceOfferingIds: [pinnedOfferingId], + defaultSourceOfferingId: pinnedOfferingId, }, user.cookies, ); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 099a7a2109..27cd392047 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -653,16 +653,17 @@ export async function pushWorkflowSource(options: { /** * The deploy body a code-sourced asset deployment takes: the pushed * commit is the definition's pin, and the entry names the - * `interchange.workflow` module the sidecar evaluates. + * `interchange.workflow` module the sidecar evaluates. The native route + * resolves inference against the tenant's own catalog, so the caller + * supplies the ordered catalog offering ids to deploy against rather + * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` + * for the zero-cost way to obtain one. */ export function workflowDeployBody(options: { assetId: string; commitSha: string; - sourceId: string; - provider: string; - baseURL: string; - apiKey: string; - model: string; + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; }): Record { return { source: { @@ -671,15 +672,103 @@ export function workflowDeployBody(options: { package: { format: "source", commitSha: options.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: options.sourceId, - provider: options.provider, - baseURL: options.baseURL, - apiKey: options.apiKey, - model: options.model, - }, - ], - defaultSource: options.sourceId, + sourceOfferingIds: options.sourceOfferingIds, + defaultSourceOfferingId: options.defaultSourceOfferingId, }; } + +/** + * Plants the zero-cost catalog chain every deploying e2e suite needs: a + * catalog model, an `anthropic`-plugin provider/credential pointed at + * `noopBaseUrl` (never a real provider host — enforced below), and the + * catalog offering joining them. Returns the offering id a deploy's + * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. + * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + */ +export async function seedNoopCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + noopBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringFieldOf(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringFieldOf(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringFieldOf( + credential.data, + "id", + "create credential", + ); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.noopBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringFieldOf( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; +} + +function stringFieldOf(data: unknown, field: string, what: string): string { + if (typeof data === "object" && data !== null && field in data) { + const value = (data as Record)[field]; + if (typeof value === "string" && value !== "") return value; + } + throw new Error( + `${what}: missing string field "${field}": ${JSON.stringify(data)}`, + ); +} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index c19a6b6a2c..9aefb900fb 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -203,16 +204,22 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-heartbeat-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 1d248defaf..760aca6a34 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -30,7 +30,6 @@ import { } from "../../workflows/heartbeat/src/index.ts"; import { api, - assertNeverRealProvider, connectE2eDb, createCleanupHarness, e2eDatabaseUrl, @@ -39,6 +38,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -138,77 +138,17 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real - // model. Mirrors `workbench-digest.test.ts`'s noop catalog seeding. + // model. const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - assertNeverRealProvider(noopBaseUrl, "noop catalog provider baseURL"); - await hop("noop catalog seeding", async () => { - const model = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringField(model.data, "id", "create catalog model"); - - const provider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringField(provider.data, "id", "create provider"); - - const credential = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringField( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: noopBaseUrl, - credentialId, - }, + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies2) => + api(hub.baseUrl, method, path, body, cookies2), + tenantId, cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringField( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - }); + noopBaseUrl, + }), + ); const assetName = "heartbeat"; const { assetId, commitSha } = await hop( @@ -256,16 +196,11 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); const definitionId = await hop("workflow deploy", async () => { - const sourceId = "src-smoke-webhook-e2e"; - assertNeverRealProvider(noopBaseUrl, "workflow deploy source baseURL"); const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: noopBaseUrl, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; for (;;) { diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 833612dd67..8937f9b5e5 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -35,6 +35,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -199,20 +200,30 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { }, ); + // Hop: noop catalog seeding. The native deploy resolves inference + // against a real catalog offering; this suite's chosen offering + // still names an unreachable placeholder host, since deployment + // never calls inference and full run-completion is not asserted. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: "https://inference.invalid", + }), + ); + // Hop: workflow deploy via the native deploy API. Retries while // the hub still answers 502 (the sidecar's dial-in may not have // completed yet); any other failure is final. The inference // source is a placeholder — deployment does not call inference. const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-echo-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index ecf2dc56eb..8917bf8236 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -207,16 +208,22 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-workbench-digest-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; From b03ce16b54f9de7e1bb8f14a7efce6a50cd7f9ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:43:45 -0700 Subject: [PATCH 03/41] Fix formatting --- scripts/e2e/cl-6324-launch-proof.ts | 125 ++++++++++++++-------------- scripts/e2e/harness.ts | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index ef3be6297c..80766c99db 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,74 +369,77 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { - await plantGrant("model-offering:*", "read"); - await plantGrant("model-offering:*", "manage"); - await plantGrant("model:*", "read"); + const pinnedOfferingId = await hop( + "narrow the bench catalog to the pinned model", + async () => { + await plantGrant("model-offering:*", "read"); + await plantGrant("model-offering:*", "manage"); + await plantGrant("model:*", "read"); - const models = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench catalog models", models, 200); - const modelRows = arrayField(models.data, "data", "catalog models") as { - id: string; - canonicalName: string; - }[]; - const pinned = modelRows.find( - (row) => row.canonicalName === proofModelSource.model, - ); - if (pinned === undefined) { - throw new Error( - `the bench catalog carries no model named ${proofModelSource.model}; ` + - `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + const models = await api( + hub.baseUrl, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, + undefined, + user.cookies, ); - } - - const offerings = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench model offerings", offerings, 200); - const offeringRows = arrayField( - offerings.data, - "data", - "model offerings", - ) as { id: string; modelId: string; disabled: boolean }[]; - let pinnedOffering: { id: string; modelId: string } | undefined; - for (const offering of offeringRows) { - if (offering.modelId === pinned.id) { - pinnedOffering = offering; - continue; + expectStatus("list the bench catalog models", models, 200); + const modelRows = arrayField(models.data, "data", "catalog models") as { + id: string; + canonicalName: string; + }[]; + const pinned = modelRows.find( + (row) => row.canonicalName === proofModelSource.model, + ); + if (pinned === undefined) { + throw new Error( + `the bench catalog carries no model named ${proofModelSource.model}; ` + + `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + ); } - if (offering.disabled) continue; - const patched = await api( + + const offerings = await api( hub.baseUrl, - "PATCH", - `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, - { disabled: true }, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, + undefined, user.cookies, ); - expectStatus(`disable offering ${offering.id}`, patched, 200); - } - if (pinnedOffering === undefined) { - throw new Error( - `no catalog offering resolves the pinned model ${proofModelSource.model} ` + - `(id ${pinned.id})`, + expectStatus("list the bench model offerings", offerings, 200); + const offeringRows = arrayField( + offerings.data, + "data", + "model offerings", + ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; + for (const offering of offeringRows) { + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; + const patched = await api( + hub.baseUrl, + "PATCH", + `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, + { disabled: true }, + user.cookies, + ); + expectStatus(`disable offering ${offering.id}`, patched, 200); + } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } + console.log( + ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); - } - console.log( - ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + - `(${String(offeringRows.length - 1)} other offerings disabled)`, - ); - return pinnedOffering.id; - }); + return pinnedOffering.id; + }, + ); const assistantDefinitionId = await hop( "PROOF 1 — 'assistant' is invitable tenant-wide", diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 27cd392047..cebc4e53db 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -760,7 +760,9 @@ export async function seedNoopCatalogOffering(options: { cookies, ); expectStatus("create catalog offering", offering, 201); - return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; + return { + offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), + }; } function stringFieldOf(data: unknown, field: string, what: string): string { From f981798f2aad5db7e0a8c14a0234e8e1860c4cbe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:53:52 -0700 Subject: [PATCH 04/41] Add tests for deploying default workflows by catalog offering ids seedTenant currently deploys every default workflow with a bare provider/baseURL/apiKey source, but the hub's deploy route now requires sourceOfferingIds/defaultSourceOfferingId resolved from the tenant's catalog. Update the seeding, onboarding, and hub config test suites to expect the new offering-id based deploy body and provider/model-only ModelSource shape ahead of the implementation change. --- apps/hub/test/config.test.ts | 1 - .../test/complete-credential.test.ts | 55 +++++++-- packages/onboarding/test/provision.test.ts | 101 +++++++++++++++++ .../test/routine-launchability.test.ts | 2 - packages/seeding/test/seed.test.ts | 106 +++++++++++++++--- 5 files changed, 237 insertions(+), 28 deletions(-) diff --git a/apps/hub/test/config.test.ts b/apps/hub/test/config.test.ts index 9d34351174..b4d41908e4 100644 --- a/apps/hub/test/config.test.ts +++ b/apps/hub/test/config.test.ts @@ -277,7 +277,6 @@ describe("readHubConfig", () => { expect(config.seedModel).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", apiKey: "sk-ant-test", }); }); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c9ac97c2db..8a4ae6a789 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -63,6 +63,36 @@ function seedHandshake(method: string, path: string) { return { ...handshake, cookies: [] }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461). A test that +// never seeds its own catalog (it stubs `seedCatalogFn` instead) still +// needs one listable offering for the "assistant" workflow's own deploy +// to succeed — this fixed single row is that. +function fixedOfferingsResponse() { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function principalsResponse() { return { status: 200, @@ -165,8 +195,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-ant", }); }); @@ -192,8 +220,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "llama3.2", - baseURL: "http://localhost:11434/v1", - apiKey: "ollama", }); }); @@ -339,7 +365,12 @@ describe("modelSourceFor", () => { expect(result.model).toBe("gpt-oss:20b"); }); - test("ollama's baseURLOverride is normalized to the /v1 form", async () => { + // `modelSourceFor` resolves a provider/model pair only — the deploy + // itself resolves inference from the tenant's own catalog offerings + // (CL-7461), so a `baseURLOverride` here is accepted (every caller + // already resolved one for `seedCatalog`'s earlier catalog plant) but + // never affects this resolution. + test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { return resolvedCatalogResponse([ @@ -365,8 +396,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "qwen3.8:27b", - baseURL: "https://home-mac.example.ts.net/v1", - apiKey: "ollama", }); }); }); @@ -808,6 +837,12 @@ describe("completeCredentialSetup", () => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}`) { return tenantResponse(); } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) @@ -1791,6 +1826,12 @@ describe("ensureSeeded (the slow half)", () => { let deploymentCreatePosts = 0; const api: ApiCall = async (method, path, body) => { + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index af1594cbc6..b1feabe081 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -87,6 +87,41 @@ function collector() { return { lines, log: (line: string) => lines.push(line) }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461); every test +// that seeds a real default workflow needs at least one listable, since +// none of these tests seed a catalog of its own. +function catalogOfferingsResponse( + method: string, + path: string, + tenantId: string, +): { status: number; data: unknown; cookies: string[] } | undefined { + if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + return undefined; + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function firstLoginSeedHub(args: { expectedParentId?: string }) { let principalsCalls = 0; const startedRuns: string[] = []; @@ -206,6 +241,33 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { ) { return { status: 200, data: [], cookies: [] }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/workflows/definitions`) @@ -638,6 +700,33 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if (method === "GET" && path === "/api/me/principals") { return membership(); } @@ -890,6 +979,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1019,6 +1110,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1133,6 +1226,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1243,6 +1338,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1485,6 +1582,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1832,6 +1931,8 @@ describe("provisionPersonalTenantIfNeeded", () => { [], ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, diff --git a/packages/seeding/test/routine-launchability.test.ts b/packages/seeding/test/routine-launchability.test.ts index 427fa0b001..746e0e2021 100644 --- a/packages/seeding/test/routine-launchability.test.ts +++ b/packages/seeding/test/routine-launchability.test.ts @@ -27,8 +27,6 @@ import { const FAKE_MODEL: ModelSource = { provider: "ollama", model: "qwen-test", - baseURL: "http://localhost:11434", - apiKey: "test-key", }; type SerializedStepDefinition = { diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index b6291fc941..720250165c 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -38,10 +38,38 @@ installDisposableHubDataDir(); const MODEL = { provider: "anthropic", model: "claude-sonnet-4-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-test", }; +// The tenant's one real catalog offering — every `seedTenant` test that +// deploys a real (non-noop-pinned) workflow needs at least one of these +// listable, since `ensureDeployment` now resolves its deploy source from +// the tenant's catalog rather than a bare `ModelSource`. +const OFFERING_ID = "off_1"; + +function offeringsPage( + ids: readonly string[] = [OFFERING_ID], +): { status: number; data: unknown } { + return { + status: 200, + data: { + data: ids.map((id, index) => ({ + id, + tenantId: TENANT_ID, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + })), + nextCursor: null, + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -98,6 +126,40 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/workflows/deployments` ) return { status: 200, data: [] }; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return offeringsPage(); + // The noop catalog chain `ensureNoopCatalogOffering` plants for a + // modelSource-pinned workflow (heartbeat) — every seedTenant test that + // deploys one needs this fresh chain to succeed, so it lives in the + // shared fallback rather than every such test repeating it. + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/models`) + return { status: 201, data: catalogModelRow("mdl_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_noop", "prv_noop", "noop-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_noop", "noop", "cre_noop", "anthropic"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 201, + data: catalogOfferingRow("off_noop", "mdl_noop", "cpv_noop"), + }; const handshake = pristineScheduledDefinitionHandshake(method, path); if (handshake) return handshake; return undefined; @@ -860,11 +922,9 @@ describe("seedTenant", () => { expect(assistant?.displayName).toBe("Myra"); }); - test("NOOP_MODEL_SOURCE resolves to the hub's own noop-inference endpoint", () => { - const resolved = NOOP_MODEL_SOURCE("http://localhost:3000"); - expect(resolved.baseURL).toBe( - "http://localhost:3000/api/chat/noop-inference", - ); + test("NOOP_MODEL_SOURCE names the noop provider/model pair", () => { + const resolved = NOOP_MODEL_SOURCE(); + expect(resolved.provider).toBe("anthropic"); expect(resolved.model).toBe("noop"); }); @@ -934,8 +994,8 @@ describe("seedTenant", () => { (w) => w.assetName === "heartbeat", ); if (!heartbeat) throw new Error("expected the heartbeat workflow"); - const resolved = heartbeat.modelSource?.("http://localhost:3000"); - expect(resolved).toEqual(NOOP_MODEL_SOURCE("http://localhost:3000")); + const resolved = heartbeat.modelSource?.(); + expect(resolved).toEqual(NOOP_MODEL_SOURCE()); }); test("fresh run pushes, deploys, and confirms the heartbeat workflow against the noop source", async () => { @@ -1012,12 +1072,17 @@ describe("seedTenant", () => { expect(definition.triggers[0]?.to).toBe(`heartbeat@${TENANT_DOMAIN}`); expect(definition.stepOrder).toEqual(["heartbeat"]); - // The deploy's own source, not the tenant's real MODEL, is what - // proves the noop pin took effect: it must name the noop provider - // fixture, not the ordinary anthropic/claude-sonnet-4-5 model this - // test file's `args()` helper hands every other workflow. - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).toBe("noop"); + // The deploy's own source offering, not the tenant's real catalog + // offering, is what proves the noop pin took effect: it must name + // the dedicated noop offering `ensureNoopCatalogOffering` plants, + // never the tenant's own `OFFERING_ID` this test file's `baseRoutes` + // fixture hands every real workflow. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual(["off_noop"]); + expect(deployedBody.defaultSourceOfferingId).toBe("off_noop"); const output = lines.join("\n"); expect(output).toContain("deployed workflow heartbeat as dep_3"); @@ -1135,9 +1200,14 @@ describe("seedTenant", () => { ]); expect(definition.stepOrder).toEqual(["workbench-digest"]); - // Defaults deploy against the tenant's real model (not noop). - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).not.toBe("noop"); + // Defaults deploy against the tenant's real catalog offering, never + // the noop one. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual([OFFERING_ID]); + expect(deployedBody.defaultSourceOfferingId).toBe(OFFERING_ID); const output = lines.join("\n"); expect(output).toContain("deployed workflow workbench-digest as dep_4"); From 1ce35c4fa2202730eabe41578a95978cff05c8cf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:00 -0700 Subject: [PATCH 05/41] Seeding: deploy default workflows by catalog offering ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureDeployment posted a bare {provider, baseURL, apiKey, model} source, but the hub's deploy route requires sourceOfferingIds and a defaultSourceOfferingId resolved from the tenant's catalog, so seeding a connect flow's "assistant" workflow 400'd. seedTenant now resolves the tenant's real catalog offerings (sorted by priority, lowest wins the default) for a normal workflow, and plants a dedicated noop catalog offering for a modelSource-pinned one (heartbeat), pointed at the hub's own noop-inference endpoint. ModelSource shrinks to a bare provider/model pair — the baseURL/apiKey it used to carry only mattered for the old deploy body, and callers that still need a key for seeding a catalog (system-seed's root tenant, config's env-key auto-plant) keep it on their own local type instead. --- apps/hub/src/config.ts | 8 +- apps/hub/src/system-seed.ts | 23 +- .../onboarding/src/complete-credential.ts | 51 +-- packages/seeding/src/seed.ts | 316 +++++++++++++++--- 4 files changed, 305 insertions(+), 93 deletions(-) diff --git a/apps/hub/src/config.ts b/apps/hub/src/config.ts index d9fdad334a..1d3afcf793 100644 --- a/apps/hub/src/config.ts +++ b/apps/hub/src/config.ts @@ -272,7 +272,6 @@ function parsePositiveMsEnv( const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; // Matches boot-time seeding's own defaults (`system-seed.ts`) exactly, // so a zero-.env-edit local checkout that seeds its admin account @@ -282,10 +281,14 @@ const DEFAULT_PLANT_ADMIN_EMAIL = "alice@example.com"; const DEFAULT_PLANT_ADMIN_PASSWORD = "password123"; const DEFAULT_PLANT_ORG_SLUG = "workbench"; +/** The root tenant's boot-time seed model: a provider/model pair + * `seedTenant` names in every deployed definition, plus the real (or + * placeholder) key `seedCatalog` plants a launchable credential with. No + * `baseURL` — a workflow deploy resolves inference from the tenant's + * catalog offerings, never a bare source tuple (CL-7461). */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; readonly apiKey: string; }; @@ -624,7 +627,6 @@ function seedModelFrom(parsed: ParsedHubEnv): ModelSource | undefined { return { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey, }; } diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 7f9291fc54..52f0ff8e66 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -26,7 +26,6 @@ import { publishCorbitsToolsRegistry, DEFAULT_WORKFLOWS, PLACEHOLDER_CATALOG_API_KEY, - type ModelSource, } from "@corbits/seeding"; import { createHubAPI, signIn, type ApiCall } from "@corbits/hub-api-client"; import { findPersonalTenant } from "@workbench/onboarding"; @@ -43,14 +42,26 @@ const DEFAULT_POLL_INTERVAL_MS = 2_000; // and the catalog is still browsable, just not launchable. const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; -function resolvedModel(seedModel: ModelSource | undefined): ModelSource { +/** + * The root tenant's seed model: a provider/model pair for + * `seedTenant`'s deployed definitions, plus the real (or placeholder) + * API key `seedCatalog` needs to plant a launchable credential. Distinct + * from `@corbits/seeding`'s `ModelSource`, which carries no key — + * `seedTenant` never needs one; the deploy itself resolves inference + * from the tenant's catalog offerings (CL-7461). + */ +export type SeedModelConfig = { + readonly provider: string; + readonly model: string; + readonly apiKey: string; +}; + +function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey: PLACEHOLDER_CATALOG_API_KEY, } ); @@ -60,7 +71,7 @@ export type SystemSeedDeps = { baseUrl: string; orgSlug: string; admin: { email: string; password: string }; - seedModel?: ModelSource; + seedModel?: SeedModelConfig; deadlineMs?: number; pollIntervalMs?: number; }; @@ -116,7 +127,7 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { principalId: tenant.principalId, domain: tenant.tenantDomain, }, - model, + model: { provider: model.provider, model: model.model }, pushWorkflow, log: (line) => log.info`${line}`, workflows: DEFAULT_WORKFLOWS, diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index 5af80e3f77..f5f4a930aa 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -60,10 +60,7 @@ import { TenantResponse, paginatedSchema, } from "@intx/types"; -import { - ollamaOpenAICompatBaseURL, - type SupportedCredentialProvider, -} from "@corbits/connections/credential-test"; +import { type SupportedCredentialProvider } from "@corbits/connections/credential-test"; import { preferCompletionCapable } from "@corbits/connections/model-capability"; import { CATALOG_SEEDS, @@ -280,13 +277,8 @@ async function resolveOllamaModelSource( api: ApiCall, cookies: string[], tenantId: string, - apiKey: string, - baseURLOverride: string | undefined, ): Promise { const catalogSeed = CATALOG_SEEDS.ollama; - const baseURL = ollamaOpenAICompatBaseURL( - baseURLOverride ?? catalogSeed.provider.baseURL, - ); const response = await api( "GET", @@ -368,39 +360,32 @@ async function resolveOllamaModelSource( return { provider: winner.plugin, model: winner.canonicalName, - baseURL, - apiKey, }; } -/** The `ModelSource` `ensureSeeded` deploys every default workflow - * against. Every provider but `ollama` has a fixed, always-available - * curated model list (`CATALOG_SEEDS`), so its curated default is a safe - * pin. `ollama` is the one provider whose actual model list is - * per-instance and can diverge from that curated name entirely - * (CL-6366) — its resolution defers to `resolveOllamaModelSource`, which - * reads back what `seedCatalog` actually found on the instance rather - * than repeating the static pin. `baseURLOverride` is the - * configurable-base-URL seam every provider but `ollama` ignores (a - * fixed origin); for `ollama` it is the root the person actually pointed - * their instance at, normalized to the OpenAI-compatible `/v1` form this - * deploys against. */ +/** The `ModelSource` `ensureSeeded` names in every deployed definition's + * inference preferences — never what the deploy itself resolves + * inference against (the tenant's catalog offerings, CL-7461). Every + * provider but `ollama` has a fixed, always-available curated model list + * (`CATALOG_SEEDS`), so its curated default is a safe pin. `ollama` is + * the one provider whose actual model list is per-instance and can + * diverge from that curated name entirely (CL-6366) — its resolution + * defers to `resolveOllamaModelSource`, which reads back what + * `seedCatalog` actually found on the instance rather than repeating the + * static pin. `apiKey`/`baseURLOverride` are accepted only because every + * caller already resolved them for `seedCatalog`'s own credential/catalog + * plant, which runs ahead of this in the same connect flow; this + * function itself needs neither. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - apiKey: string, - baseURLOverride?: string, + _apiKey: string, + _baseURLOverride?: string, ): Promise { if (provider === "ollama") { - return resolveOllamaModelSource( - api, - cookies, - tenantId, - apiKey, - baseURLOverride, - ); + return resolveOllamaModelSource(api, cookies, tenantId); } const catalogSeed = CATALOG_SEEDS[provider]; @@ -413,8 +398,6 @@ export async function modelSourceFor( return { provider: catalogSeed.provider.plugin, model: defaultModel.canonicalName, - baseURL: catalogSeed.provider.baseURL, - apiKey, }; } diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 2b34e98c3d..20e350303e 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -141,11 +141,6 @@ const DILIGENCE_BRIEF_TURN_TIMEOUT_MS = 2 * 60 * 1000; const RUN_START_TIMEOUT_MS = 30_000; const RUN_POLL_INTERVAL_MS = 1000; -// The deploy source the per-step agents launch against. The id is the -// routing key `defaultSource` must name; with exactly one source there -// is exactly one honest value for it. -const SEED_SOURCE_ID = "default"; - // The provider/model pair `noop-inference` (packages/chat/src/noop-inference.ts) // answers for any request, regardless of what is actually sent — the // route ignores its body and `x-api-key` entirely. Naming a distinct @@ -154,24 +149,31 @@ const SEED_SOURCE_ID = "default"; // UI and logs. const NOOP_PROVIDER = "anthropic"; const NOOP_MODEL = "noop"; +// The catalog provider/credential name a noop-pinned deployment's own +// offering is planted under — distinct from any real curated provider +// name (`CATALOG_SEEDS`'s keys), so `ensureNoopCatalogOffering` never +// collides with a tenant's real catalog and is trivially recognizable +// in the hub's own catalog UI. +const NOOP_CATALOG_PROVIDER_NAME = "noop"; +// Priority is meaningless for this offering — it is never included in +// a real workflow's `sourceOfferingIds` (`ensureNoopCatalogOffering`'s +// id is threaded through explicitly, never discovered by sorting), so +// any fixed value is honest here. +const NOOP_OFFERING_PRIORITY = 0; /** - * A `ModelSource` pointed at the hub's own `noop-inference` endpoint + * The provider/model pair naming the hub's own `noop-inference` endpoint * instead of a real provider — the same substitution * `packages/chat/src/platform-adapter.ts`'s `noopSourcesOverride` makes - * for workbench-host launches, reused here so a workflow deployed with - * this source resolves every turn instantly against a constant, - * locally served reply and never reaches a real model. `hubUrl` is the - * same base URL `seedTenant` already receives, so no new configuration - * is required to use it. + * for workbench-host launches, reused here so a workflow's rendered + * definition names this pair in its inference preferences. The actual + * deploy source is a catalog offering pinned at this same endpoint + * (`ensureNoopCatalogOffering`), not this pair directly — a workflow + * deployment resolves inference from catalog offering ids, never a bare + * provider/model/baseURL/apiKey tuple (CL-7461). */ -export function NOOP_MODEL_SOURCE(hubUrl: string): ModelSource { - return { - provider: NOOP_PROVIDER, - model: NOOP_MODEL, - baseURL: `${hubUrl}/api/chat/noop-inference`, - apiKey: "noop", - }; +export function NOOP_MODEL_SOURCE(): ModelSource { + return { provider: NOOP_PROVIDER, model: NOOP_MODEL }; } const GitTokenMintResponse = type({ id: "string", secret: "string" }); @@ -191,11 +193,18 @@ const WorkflowRunTriggerResponse = type({ messageId: "string", }); +/** + * A provider/model pair a deployed workflow's rendered definition names + * in its inference preferences (`DefaultWorkflow.buildJson`'s second + * argument). What a deployment actually resolves inference against is + * the tenant's catalog offerings, resolved separately by `ensureDeployment` + * (or, for a noop-pinned workflow, `ensureNoopCatalogOffering`) — this + * type carries no `baseURL`/`apiKey` because `seedTenant` never needs + * either (CL-7461). + */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; - readonly apiKey: string; }; /** @@ -240,9 +249,9 @@ export type DefaultWorkflow = { * Renders the definition's JSON given the tenant's mail domain and the * ordered provider/model preferences to deploy against. Takes the bare * preference list — never a full `ModelSource` — so this same - * function serves both `seedTenant`'s HTTP-deploy path (which also - * needs a `ModelSource`'s `baseURL`/`apiKey` for the deployment's - * `sources`, resolved separately) and a native in-process deploy path + * function serves both `seedTenant`'s HTTP-deploy path (whose actual + * deploy source is a set of catalog offering ids, resolved separately + * by `ensureDeployment`) and a native in-process deploy path * (`apps/hub/src/templates/block-workflows.ts`) that only ever has the * tenant's real, possibly multi-entry inference preferences on hand. */ @@ -251,15 +260,16 @@ export type DefaultWorkflow = { inferencePreferences: readonly InferencePreference[], ) => string; /** - * Overrides the deploy's inference source for this workflow only, - * given the hub's own base URL. Present on the catalog-test workflow - * `heartbeat`, which must stay free to run continuously: it names - * `NOOP_MODEL_SOURCE` instead of the tenant's real catalog model. - * Absent on every conversational workflow and on the seeded - * workbench-digest automation, which deploy against the tenant's real - * model. + * Overrides the deploy's inference source for this workflow only. + * Present on the catalog-test workflow `heartbeat`, which must stay + * free to run continuously: it names `NOOP_MODEL_SOURCE` instead of + * the tenant's real catalog model, and `seedTenant` deploys it against + * a dedicated noop catalog offering (`ensureNoopCatalogOffering`) + * rather than the tenant's own resolved offerings. Absent on every + * conversational workflow and on the seeded workbench-digest + * automation, which deploy against the tenant's real model. */ - modelSource?: (hubUrl: string) => ModelSource; + modelSource?: () => ModelSource; /** * When true, PUT the authored definition to `stopped` after deploy so * a native ScheduleTrigger does not fire every tenant at the next @@ -901,7 +911,8 @@ async function ensureDeployment( assetId: string; assetName: string; commitSha: string; - model: ModelSource; + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; }, log: (line: string) => void, ): Promise { @@ -950,16 +961,8 @@ async function ensureDeployment( package: { format: "source", commitSha: args.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SEED_SOURCE_ID, - provider: args.model.provider, - baseURL: args.model.baseURL, - apiKey: args.model.apiKey, - model: args.model.model, - }, - ], - defaultSource: SEED_SOURCE_ID, + sourceOfferingIds: args.sourceOfferingIds, + defaultSourceOfferingId: args.defaultSourceOfferingId, }, cookies, ); @@ -984,6 +987,170 @@ async function ensureDeployment( return deployment.id; } +/** + * Lists every catalog offering owned directly by a tenant, paging past + * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list + * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" + * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s + * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s + * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog + * directly on that same tenant, so the owned list already carries + * everything a plain `workbench seed` or the self-served connect flow + * ever seeded, with no HTTP surface exposing the resolved view to a + * caller outside the hub process. + */ +async function listTenantCatalogOfferings( + api: ApiCall, + cookies: string[], + tenantId: string, +): Promise<(typeof ModelOfferingResponse.infer)[]> { + const items: (typeof ModelOfferingResponse.infer)[] = []; + let cursor: string | undefined; + for (;;) { + const query = + cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings${query}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + items.push(...page.data); + if (page.nextCursor === null) return items; + if (items.length > 10_000) { + throw new HubApiError( + `catalog offerings list for tenant ${tenantId} did not terminate while paging`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + cursor = page.nextCursor; + } +} + +/** + * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow + * deploys against: every enabled offering the tenant owns, ordered by + * priority ascending, with the lowest-priority offering as the default — + * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and + * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply + * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the + * noop offering (`ensureNoopCatalogOffering`) from this list when one + * has already been planted on the same tenant this run — it must never + * win a real workflow's default just because it happens to sort first. + * + * Throws when nothing is left to deploy against: "seeded but not + * launchable" (`seedCatalog` with no credential) is a valid catalog + * state, but a workflow deploy that names zero sources is not a + * lesser success, it is the failure this function exists to catch + * before the hub's own `sourceOfferingIds` validation would. + */ +async function resolveRealSourceOfferingIds( + api: ApiCall, + cookies: string[], + tenantId: string, + excludeOfferingId: string | undefined, +): Promise<{ + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; +}> { + const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) + .filter( + (offering) => + !offering.disabled && offering.id !== excludeOfferingId, + ) + .sort((a, b) => a.priority - b.priority); + const defaultSourceOfferingId = offerings[0]?.id; + if (defaultSourceOfferingId === undefined) { + throw new HubApiError( + "this tenant has no catalog offerings to deploy against — nothing is launchable yet", + "seed the tenant's catalog with a real provider key (or a placeholder credential for a keyless dev/CI run), then re-run: workbench seed", + ); + } + return { + sourceOfferingIds: offerings.map((offering) => offering.id), + defaultSourceOfferingId, + }; +} + +/** + * Plants (or finds) the dedicated catalog offering a noop-pinned + * workflow (`DefaultWorkflow.modelSource`, e.g. `heartbeat`) deploys + * against: a catalog model named `NOOP_MODEL`, a provider pointed at the + * hub's own `noop-inference` endpoint, and a placeholder credential — + * the same `ensureCatalogModel` / `ensureProvider` / `ensureCredential` / + * `ensureCatalogProvider` / `ensureCatalogOffering` sequence + * `seedCatalog` runs for a real provider, run here for this one + * synthetic one. Idempotent, same as every other seed step: a re-run + * finds the existing rows by name and reuses them. + */ +async function ensureNoopCatalogOffering( + api: ApiCall, + cookies: string[], + tenantId: string, + hubUrl: string, + log: (line: string) => void, +): Promise { + const baseURL = `${hubUrl}/api/chat/noop-inference`; + const modelId = await ensureCatalogModel( + api, + cookies, + { tenantId, canonicalName: NOOP_MODEL }, + log, + ); + const providerId = await ensureProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + apiBaseUrl: baseURL, + }, + log, + ); + const credentialId = await ensureCredential( + api, + cookies, + { + tenantId, + providerId, + name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), + secret: PLACEHOLDER_CATALOG_API_KEY, + type: "api_key", + }, + log, + ); + const catalogProviderId = await ensureCatalogProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + baseURL, + credentialId, + }, + log, + ); + return ensureCatalogOffering( + api, + cookies, + { + tenantId, + modelId, + providerId: catalogProviderId, + priority: NOOP_OFFERING_PRIORITY, + capabilities: [], + }, + log, + ); +} + /** * After deploy, PUT a pristine authored definition to `stopped` so a native * ScheduleTrigger does not fire every tenant. A member-restored row @@ -1160,6 +1327,14 @@ export type SeedTenantArgs = { cookies: string[]; hubUrl: string; tenant: SeedTenant; + /** + * The provider/model pair a real (non-noop-pinned) workflow's rendered + * definition names in its inference preferences. The deploy itself + * resolves inference from the tenant's own catalog offerings + * (`resolveRealSourceOfferingIds`), not from this pair — a tenant with + * no catalog offerings still fails clearly at deploy time even if this + * pair happens to name a real model. + */ model: ModelSource; pushWorkflow: WorkflowPusher; log: (line: string) => void; @@ -1228,9 +1403,41 @@ export async function seedTenant(args: SeedTenantArgs): Promise { await plantDefaultSkills(api, cookies, tenant.tenantId, log); + // Resolved lazily and cached across the loop below: most seed runs + // deploy several workflows against the same tenant catalog state, so + // this fetches the tenant's real offerings (or plants the noop one) + // at most once each, however many workflows ask for them. + let noopOfferingId: string | undefined; + let realOfferings: + | { sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string } + | undefined; + let confirmed = 0; for (const workflow of workflows) { - const workflowModel = workflow.modelSource?.(hubUrl) ?? model; + const workflowModel = workflow.modelSource?.() ?? model; + + let sourceOfferingIds: readonly string[]; + let defaultSourceOfferingId: string; + if (workflow.modelSource !== undefined) { + noopOfferingId ??= await ensureNoopCatalogOffering( + api, + cookies, + tenant.tenantId, + hubUrl, + log, + ); + sourceOfferingIds = [noopOfferingId]; + defaultSourceOfferingId = noopOfferingId; + } else { + realOfferings ??= await resolveRealSourceOfferingIds( + api, + cookies, + tenant.tenantId, + noopOfferingId, + ); + ({ sourceOfferingIds, defaultSourceOfferingId } = realOfferings); + } + const assetId = await ensureWorkflowAsset( api, cookies, @@ -1265,7 +1472,8 @@ export async function seedTenant(args: SeedTenantArgs): Promise { assetId, assetName: workflow.assetName, commitSha: pushed.commitSha, - model: workflowModel, + sourceOfferingIds, + defaultSourceOfferingId, }, log, ); @@ -1666,7 +1874,7 @@ async function ensureCatalogOffering( quirks?: Record; }, log: (line: string) => void, -): Promise { +): Promise { const body: Record = { modelId: args.modelId, providerId: args.providerId, @@ -1681,9 +1889,13 @@ async function ensureCatalogOffering( cookies, ); if (created.status === 201) { - parseAs(ModelOfferingResponse, created.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + created.data, + "catalog offering response", + ); log("created catalog offering"); - return; + return offering.id; } if (created.status === 409) { let cursor: string | null = null; @@ -1715,7 +1927,7 @@ async function ensureCatalogOffering( } if (existing.priority === args.priority) { log("catalog offering already exists (skipped)"); - return; + return existing.id; } const updated = await api( @@ -1730,9 +1942,13 @@ async function ensureCatalogOffering( "check the hub logs for the underlying failure, then re-run: workbench seed", ); } - parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + updated.data, + "catalog offering response", + ); log("updated catalog offering priority"); - return; + return offering.id; } throw new HubApiError( `the hub rejected creation of the catalog offering with status ${created.status}: ${JSON.stringify(created.data)}`, From 2587cd2b5af3ecbb27bb13bcb652910bd10b29aa Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:46 -0700 Subject: [PATCH 06/41] longevity-sim: deploy workflows by catalog offering ids Another workflowDeployBody caller CI's typecheck job caught: this stack seeds one catalog offering per real inference target already, so track each one's id and pass it through to the deploy body instead of the raw provider/baseURL/apiKey shape. --- packages/longevity-sim/src/stack.ts | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/longevity-sim/src/stack.ts b/packages/longevity-sim/src/stack.ts index a7dc917761..59c4ff72aa 100644 --- a/packages/longevity-sim/src/stack.ts +++ b/packages/longevity-sim/src/stack.ts @@ -437,7 +437,9 @@ export async function bootLongevityStack( // credential row still requires some string, so each target's own // `apiKey` (expected to be the same placeholder) is threaded through // unchanged rather than this package inventing its own convention. - async function seedCatalogChain(target: InferenceTarget): Promise { + async function seedCatalogChain( + target: InferenceTarget, + ): Promise<{ offeringId: string }> { const model = await api( hub.baseUrl, "POST", @@ -509,10 +511,15 @@ export async function bootLongevityStack( owner.cookies, ); expectStatus(`create catalog offering ${target.model}`, offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; } + const offeringIdByTargetLabel = new Map(); for (const target of options.realTargets) { - await seedCatalogChain(target); + const { offeringId } = await seedCatalogChain(target); + offeringIdByTargetLabel.set(target.label, offeringId); } for (const skill of options.skills ?? []) { @@ -668,6 +675,13 @@ export async function bootLongevityStack( workflowJson, }); + const offeringId = offeringIdByTargetLabel.get(target.label); + if (offeringId === undefined) { + throw new Error( + `agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`, + ); + } + const deployDeadline = Date.now() + 90_000; for (;;) { if (sidecar.exited()) { @@ -682,11 +696,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId, commitSha: pushed.commitSha, - sourceId: `src-agent-${spec.handle}`, - provider: target.provider, - baseURL: target.baseURL, - apiKey: target.apiKey, - model: target.model, + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }), owner.cookies, ); @@ -847,6 +858,12 @@ export async function bootLongevityStack( if (routineTarget === undefined) { throw new Error("unreachable: realTargets checked non-empty above"); } + const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label); + if (routineOfferingId === undefined) { + throw new Error( + `no catalog offering seeded for realTarget "${routineTarget.label}"`, + ); + } const assetName = "longevity-heartbeat"; const heartbeatAssetCreated = await api( hub.baseUrl, @@ -912,11 +929,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId: heartbeatAssetId, commitSha: heartbeatPushed.commitSha, - sourceId: "src-longevity-heartbeat", - provider: "openai-compatible", - baseURL: routineTarget.baseURL, - apiKey: routineTarget.apiKey, - model: routineTarget.model, + sourceOfferingIds: [routineOfferingId], + defaultSourceOfferingId: routineOfferingId, }), owner.cookies, ); From 2430449e41b11a77a87cc902e4e2f7c264736125 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:56:16 -0700 Subject: [PATCH 07/41] Fix formatting: prettier --- apps/hub/src/system-seed.ts | 4 +++- packages/seeding/src/seed.ts | 3 +-- packages/seeding/test/seed.test.ts | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 52f0ff8e66..1637f2e89d 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -57,7 +57,9 @@ export type SeedModelConfig = { readonly apiKey: string; }; -function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { +function resolvedModel( + seedModel: SeedModelConfig | undefined, +): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 20e350303e..24fc1ce6eb 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -1060,8 +1060,7 @@ async function resolveRealSourceOfferingIds( }> { const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) .filter( - (offering) => - !offering.disabled && offering.id !== excludeOfferingId, + (offering) => !offering.disabled && offering.id !== excludeOfferingId, ) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 720250165c..99f9728045 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -46,9 +46,10 @@ const MODEL = { // the tenant's catalog rather than a bare `ModelSource`. const OFFERING_ID = "off_1"; -function offeringsPage( - ids: readonly string[] = [OFFERING_ID], -): { status: number; data: unknown } { +function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { return { status: 200, data: { From 602ad68f72898a1342ec88f5090eac80999493a4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:25 -0700 Subject: [PATCH 08/41] sandbox-sidecar: add tests for generation-0 ensure/destroy requests The allocation service's initial sidecar_allocation and workflow_probe rows start at generation 0 (@intx/db enforces generation >= 0, not > 0), so the first ensure() for a freshly allocated sidecar must be accepted at generation 0. These tests cover that acceptance, later generations, non-integer/negative rejection, and that stale-generation fencing still rejects an older generation once a newer one has been observed. --- .../sandbox-sidecar/src/provisioner.test.ts | 154 ++++++++++++++++++ .../sandbox-sidecar/src/state-store.test.ts | 10 ++ 2 files changed, 164 insertions(+) create mode 100644 packages/sandbox-sidecar/src/provisioner.test.ts diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts new file mode 100644 index 0000000000..753478bf01 --- /dev/null +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { createSidecarProvisioner } from "./provisioner"; +import type { SidecarBackend } from "./backend"; + +const TOKEN = "s3cr3t-bootstrap-token"; + +let dataDir: string; +let store: AllocationStateStore; +let started: string[]; + +beforeEach(async () => { + dataDir = await mkdtemp(join(tmpdir(), "sandbox-sidecar-provisioner-")); + store = createAllocationStateStore(join(dataDir, "state.json")); + started = []; +}); + +afterEach(async () => { + await rm(dataDir, { recursive: true, force: true }); +}); + +function fakeBackend(): SidecarBackend { + let counter = 0; + return { + async startUnit() { + counter += 1; + const externalRef = `unit-${String(counter)}`; + started.push(externalRef); + return externalRef; + }, + async stopUnit() {}, + async findUnitsByAllocation() { + return []; + }, + }; +} + +function baseEnsureRequest(overrides: Partial> = {}) { + return { + allocationId: "alloc-1", + generation: 0, + tenantId: "tenant-1", + anchorRunId: "run-1", + sidecarId: "sidecar-1", + token: TOKEN, + hubWebSocketUrl: "wss://hub.example.com/ws", + ...overrides, + } as never; +} + +function makeProvisioner(backend: SidecarBackend = fakeBackend()) { + return createSidecarProvisioner({ + id: "test-provisioner", + apiVersion: 1, + bindingFingerprint: "test", + capabilities: [], + backend, + store, + }); +} + +describe("ensure request validation", () => { + test("accepts generation 0, the allocation service's initial generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); + }); + + test("accepts a later generation", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: -1 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("rejects a non-integer generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1.5 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("still rejects a stale generation once fencing has observed a later one", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toMatchObject({ + kind: "rejected", + code: "stale_generation", + }); + expect(started).toHaveLength(1); + }); +}); + +describe("destroy request validation", () => { + test("accepts generation 0", async () => { + const provisioner = makeProvisioner(); + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + + expect(result).toEqual({ kind: "destroyed" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: -1, + }); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_destroy_request", + }); + }); +}); diff --git a/packages/sandbox-sidecar/src/state-store.test.ts b/packages/sandbox-sidecar/src/state-store.test.ts index d9bdadffb0..abd723afd8 100644 --- a/packages/sandbox-sidecar/src/state-store.test.ts +++ b/packages/sandbox-sidecar/src/state-store.test.ts @@ -30,6 +30,16 @@ describe("observeEnsure", () => { expect(result.kind).toBe("observed"); }); + test("accepts generation 0, the allocation service's initial generation", async () => { + const store = createAllocationStateStore(statePath()); + const result = await store.observeEnsure({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + expect(result.kind).toBe("observed"); + }); + test("is idempotent when observed again at the same generation", async () => { const store = createAllocationStateStore(statePath()); await store.observeEnsure({ From e01fedad96a1991c9b05eb612462fb18bd654db4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:30 -0700 Subject: [PATCH 09/41] sandbox-sidecar: accept the allocation service's initial generation @intx/db's sidecar_allocation and workflow_probe tables both check generation >= 0 and the allocation service records the first row for an allocation at generation 0, but validateEnsureRequest() and validateDestroyRequest() in packages/sandbox-sidecar rejected anything that wasn't strictly positive, and the on-disk fencing schema in state-store.ts required generation > 0. The first ensure() for any newly allocated sidecar was rejected with 'generation must be a positive integer', failing every deploy. Accept generation >= 0 in both validators and the state-store schema; stale-generation fencing in observeEnsure()/observeDestroy() already compares against the stored generation rather than assuming positivity, so it is unaffected. --- packages/sandbox-sidecar/src/provisioner.ts | 8 ++++---- packages/sandbox-sidecar/src/state-store.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.ts b/packages/sandbox-sidecar/src/provisioner.ts index 209d9e725a..9771be3198 100644 --- a/packages/sandbox-sidecar/src/provisioner.ts +++ b/packages/sandbox-sidecar/src/provisioner.ts @@ -218,8 +218,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { if (request.token === "") return "token must not be empty"; if (request.hubWebSocketUrl === "") return "hubWebSocketUrl must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } @@ -227,8 +227,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { function validateDestroyRequest(request: DestroySidecarRequest): string | null { if (request.allocationId === "") return "allocationId must not be empty"; if (request.sidecarId === "") return "sidecarId must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } diff --git a/packages/sandbox-sidecar/src/state-store.ts b/packages/sandbox-sidecar/src/state-store.ts index 12e5bed2a9..9e600e2286 100644 --- a/packages/sandbox-sidecar/src/state-store.ts +++ b/packages/sandbox-sidecar/src/state-store.ts @@ -12,7 +12,7 @@ import { type } from "arktype"; const AllocationRecord = type({ allocationId: "string > 0", sidecarId: "string > 0", - generation: "number.integer > 0", + generation: "number.integer >= 0", desiredState: "'ensured' | 'destroyed'", externalRef: "string | null", tokenHashSha256: "string | null", From c6da1fa7ba9e2607098f1e566456e8ebc8b84b9b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:37:36 -0700 Subject: [PATCH 10/41] sandbox-sidecar: fix formatting in provisioner test --- .../sandbox-sidecar/src/provisioner.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts index 753478bf01..b858c03cc6 100644 --- a/packages/sandbox-sidecar/src/provisioner.test.ts +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -3,7 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { + createAllocationStateStore, + type AllocationStateStore, +} from "./state-store"; import { createSidecarProvisioner } from "./provisioner"; import type { SidecarBackend } from "./backend"; @@ -67,7 +70,9 @@ describe("ensure request validation", () => { test("accepts generation 0, the allocation service's initial generation", async () => { const provisioner = makeProvisioner(); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); }); @@ -76,7 +81,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 0 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); }); @@ -113,7 +120,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 1 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toMatchObject({ kind: "rejected", From 824c2b69f0c53998d64e5bebcb051c27b7d3ca4c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:42 -0700 Subject: [PATCH 11/41] Add tests for resolved catalog offerings and modelSourceFor's trimmed signature Covers the new GET .../catalog/resolved-offerings route (denied grant, and both directly-owned and ancestor-inherited offerings sorted by priority) and updates seeding/onboarding test fixtures to the route's response shape ahead of the implementation change. --- .../test/resolved-offerings-routes.test.ts | 99 +++++++++++++++++++ .../test/complete-credential.test.ts | 94 ++++++------------ packages/onboarding/test/provision.test.ts | 48 +++------ packages/seeding/test/seed.test.ts | 23 +++++ 4 files changed, 166 insertions(+), 98 deletions(-) create mode 100644 packages/inference-catalog/test/resolved-offerings-routes.test.ts diff --git a/packages/inference-catalog/test/resolved-offerings-routes.test.ts b/packages/inference-catalog/test/resolved-offerings-routes.test.ts new file mode 100644 index 0000000000..897e053130 --- /dev/null +++ b/packages/inference-catalog/test/resolved-offerings-routes.test.ts @@ -0,0 +1,99 @@ +// The resolved-offerings route exists so an out-of-process caller +// (`workbench seed`, CL-7461) can deploy against exactly what the hub's +// own workflow deployer would deploy against — including offerings a +// tenant only inherits from an ancestor, which the tenant-owned +// `GET .../catalog/offerings` route never lists. +import { Hono } from "hono"; +import { describe, expect, test } from "bun:test"; + +import type { TenantEnv } from "@intx/hub-api"; +import type { ResolvedOffering } from "@intx/db"; + +import { createResolvedOfferingsRoutes } from "../src/resolved-offerings-routes"; +import { offering } from "./fixtures"; + +const TENANT = { + id: "bench-1", + name: "Bench One", + slug: "bench-one", + domain: "bench-one.example", + parentId: null, + config: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), +}; + +function app( + listOfferings: (tenantId: string) => Promise, + allow = true, +) { + const host = new Hono(); + host.use("*", async (c, next) => { + c.set("tenant", TENANT); + await next(); + }); + host.route( + "/", + createResolvedOfferingsRoutes({ + listOfferings, + requireGrant: () => async (c, next) => { + if (!allow) { + return c.json({ error: { code: "forbidden", message: "no" } }, 403); + } + return next(); + }, + }), + ); + return host; +} + +describe("GET resolved-offerings", () => { + test("a denied grant is rejected before the offerings are read", async () => { + const response = await app(async () => { + throw new Error("must not be called when the grant is denied"); + }, false).request("/"); + expect(response.status).toBe(403); + }); + + test("lists both directly-owned and ancestor-inherited offerings, sorted by priority", async () => { + const owned = offering({ + id: "owned", + canonicalName: "thrifty", + providerName: "globex", + capabilities: ["plain-text"], + priority: 2, + }); + const inherited = offering({ + id: "inherited", + canonicalName: "lavish", + providerName: "acme", + capabilities: ["plain-text"], + priority: 1, + inherited: true, + }); + + const response = await app(async (tenantId) => { + expect(tenantId).toBe(TENANT.id); + return [owned, inherited]; + }).request("/"); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + offerings: { + id: string; + priority: number; + modelId: string; + providerId: string; + origin: { tenantId: string; direct: boolean }; + }[]; + }; + expect(body.offerings.map((o) => o.id)).toEqual(["inherited", "owned"]); + const inheritedEntry = body.offerings.find((o) => o.id === "inherited"); + expect(inheritedEntry?.origin).toEqual({ + tenantId: "parent-bench", + direct: false, + }); + const ownedEntry = body.offerings.find((o) => o.id === "owned"); + expect(ownedEntry?.origin).toEqual({ tenantId: TENANT.id, direct: true }); + }); +}); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 8a4ae6a789..16e81d0dfd 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -72,22 +72,15 @@ function fixedOfferingsResponse() { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -179,19 +172,12 @@ function ownedCatalogModelsResponse(canonicalNames: string[]) { } describe("modelSourceFor", () => { - test("every other provider ignores a baseURLOverride and never calls the hub", async () => { + test("every other provider resolves to its curated default and never calls the hub", async () => { const api: ApiCall = (async () => { throw new Error("must not be called for a fixed curated provider"); }) as ApiCall; expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "anthropic", - "sk-ant", - "https://ignored", - ), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "anthropic"), ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", @@ -216,7 +202,7 @@ describe("modelSourceFor", () => { }; expect( - await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama", "ollama"), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama"), ).toEqual({ provider: "openai-compatible", model: "llama3.2", @@ -250,7 +236,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -285,7 +270,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); @@ -323,7 +307,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -360,44 +343,9 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); - - // `modelSourceFor` resolves a provider/model pair only — the deploy - // itself resolves inference from the tenant's own catalog offerings - // (CL-7461), so a `baseURLOverride` here is accepted (every caller - // already resolved one for `seedCatalog`'s earlier catalog plant) but - // never affects this resolution. - test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { - const api: ApiCall = async (method, path) => { - if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { - return resolvedCatalogResponse([ - { - canonicalName: "qwen3.8:27b", - providerName: "ollama", - capabilities: ["plain-text"], - }, - ]); - } - throw new Error(`unexpected call: ${method} ${path}`); - }; - - expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "ollama", - "ollama", - "https://home-mac.example.ts.net", - ), - ).toEqual({ - provider: "openai-compatible", - model: "qwen3.8:27b", - }); - }); }); describe("completeCredentialSetup", () => { @@ -780,12 +728,11 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, - data: { data: [], nextCursor: null }, + data: { offerings: [] }, cookies: [], }; } @@ -839,7 +786,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } @@ -1422,8 +1369,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { return { status: 200, @@ -1446,6 +1392,24 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) { + return { + status: 200, + data: { + offerings: catalogOfferings.map((o) => ({ + id: o.id, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -1828,7 +1792,7 @@ describe("ensureSeeded (the slow half)", () => { const api: ApiCall = async (method, path, body) => { if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index b1feabe081..aff387d489 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -96,27 +96,23 @@ function catalogOfferingsResponse( path: string, tenantId: string, ): { status: number; data: unknown; cookies: string[] } | undefined { - if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + if ( + method !== "GET" || + path !== `/api/tenants/${tenantId}/catalog/resolved-offerings` + ) return undefined; return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -243,27 +239,20 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -702,27 +691,20 @@ describe("provisionPersonalTenantIfNeeded", () => { if (registry !== undefined) return registry; if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 99f9728045..d1bea7bcdb 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -71,6 +71,24 @@ function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { }; } +function resolvedOfferingsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { + return { + status: 200, + data: { + offerings: ids.map((id, index) => ({ + id, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -132,6 +150,11 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) return offeringsPage(); + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) + return resolvedOfferingsPage(); // The noop catalog chain `ensureNoopCatalogOffering` plants for a // modelSource-pinned workflow (heartbeat) — every seedTenant test that // deploys one needs this fresh chain to succeed, so it lives in the From ec8755106e08316e1a4a12c57f8121849be6fcb6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:49 -0700 Subject: [PATCH 12/41] Seed deploys against a tenant's resolved catalog offerings, not just owned ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveRealSourceOfferingIds listed only offerings a tenant owns directly, missing anything inherited from an ancestor tenant — the hub's own workflow deployer already resolves that wider, ancestor-inheriting view via listVisibleOfferings. Adds GET /api/tenants/:tenantId/catalog/resolved-offerings (mounted from apps/hub, implemented in @corbits/inference-catalog so the deployer and this route share one source of truth) and points seed at it instead. Also drops the two long-dead unused parameters from onboarding's modelSourceFor, and exports ensureNoopCatalogOffering from @corbits/seeding for the e2e harness to reuse. --- apps/hub/src/index.ts | 16 ++++ packages/inference-catalog/src/index.ts | 4 + .../src/resolved-offerings-routes.ts | 44 ++++++++++ .../onboarding/src/complete-credential.ts | 9 +- packages/seeding/src/index.ts | 1 + packages/seeding/src/seed.ts | 84 +++++++++---------- 6 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 packages/inference-catalog/src/resolved-offerings-routes.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 39d52b853d..687bcde64e 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -150,6 +150,7 @@ import { applyInferenceCatalogMigrations, createBenchModelPolicyRoutes, createPostgresBenchModelPolicyStore, + createResolvedOfferingsRoutes, createWorkflowCatalogRoutes, } from "@corbits/inference-catalog"; import { generateId } from "@intx/hub-common"; @@ -1877,6 +1878,21 @@ export async function createHub(config: HubConfig) { }), }), ); + // Resolved catalog offerings: the same ancestor-inheriting view + // `listVisibleOfferings` gives `workflowDeployer` above, exposed over + // HTTP so an out-of-process deployer (`workbench seed`) can deploy + // against exactly what the hub itself would deploy against, not just + // the offerings a tenant owns directly. + app.route( + `${TENANT_PREFIX}/catalog/resolved-offerings`, + createResolvedOfferingsRoutes({ + listOfferings: (tenantId) => listVisibleOfferings(db, tenantId), + requireGrant: createRequireGrant({ + grantStore: chatGrantStore, + conditionRegistry: chatConditionRegistry, + }), + }), + ); // Bench purpose/type: benches are Interchange tenants, so this is a // package-owned side-table keyed by tenant id, migrated at hub start // like insights and preferences. diff --git a/packages/inference-catalog/src/index.ts b/packages/inference-catalog/src/index.ts index b7e90138a8..01af47a3e3 100644 --- a/packages/inference-catalog/src/index.ts +++ b/packages/inference-catalog/src/index.ts @@ -63,6 +63,10 @@ export { createBenchModelPolicyRoutes, type CreateBenchModelPolicyRoutesDeps, } from "./routes"; +export { + createResolvedOfferingsRoutes, + type CreateResolvedOfferingsRoutesDeps, +} from "./resolved-offerings-routes"; export { createWorkflowCatalogRoutes, estimateUsd, diff --git a/packages/inference-catalog/src/resolved-offerings-routes.ts b/packages/inference-catalog/src/resolved-offerings-routes.ts new file mode 100644 index 0000000000..ec1cfae2f7 --- /dev/null +++ b/packages/inference-catalog/src/resolved-offerings-routes.ts @@ -0,0 +1,44 @@ +// The tenant-owned `GET .../catalog/offerings` route only ever lists rows a +// tenant planted directly on itself; it has no way to show offerings a +// tenant inherits from an ancestor tenant. `listVisibleOfferings` +// (`@intx/db`) already computes that resolved, inheritance-aware view for +// the hub's own workflow deployer — this route exposes the same view over +// HTTP so an out-of-process caller (`workbench seed`, in particular) can +// deploy against exactly what the hub itself would deploy against. +// +// Takes a `listOfferings` port rather than a `db` handle, same as +// `workflow-catalog-routes.ts`, so the route is testable with literals. +import { Hono } from "hono"; +import type { ResolvedOffering } from "@intx/db"; +import type { RequireGrant, TenantEnv } from "@intx/hub-api"; + +export type CreateResolvedOfferingsRoutesDeps = { + readonly listOfferings: ( + tenantId: string, + ) => Promise; + readonly requireGrant: RequireGrant; +}; + +export function createResolvedOfferingsRoutes( + deps: CreateResolvedOfferingsRoutesDeps, +): Hono { + const app = new Hono(); + + app.get("/", deps.requireGrant("model-offering:*", "read"), async (c) => { + const tenant = c.get("tenant"); + const resolved = [...(await deps.listOfferings(tenant.id))].sort( + (a, b) => a.offering.priority - b.offering.priority, + ); + return c.json({ + offerings: resolved.map((entry) => ({ + id: entry.offering.id, + priority: entry.offering.priority, + modelId: entry.offering.modelId, + providerId: entry.offering.providerId, + origin: entry.origin, + })), + }); + }); + + return app; +} diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index f5f4a930aa..c292bc897f 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -372,17 +372,12 @@ async function resolveOllamaModelSource( * diverge from that curated name entirely (CL-6366) — its resolution * defers to `resolveOllamaModelSource`, which reads back what * `seedCatalog` actually found on the instance rather than repeating the - * static pin. `apiKey`/`baseURLOverride` are accepted only because every - * caller already resolved them for `seedCatalog`'s own credential/catalog - * plant, which runs ahead of this in the same connect flow; this - * function itself needs neither. */ + * static pin. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - _apiKey: string, - _baseURLOverride?: string, ): Promise { if (provider === "ollama") { return resolveOllamaModelSource(api, cookies, tenantId); @@ -496,8 +491,6 @@ export async function ensureSeeded( args.cookies, args.tenant.tenantId, args.provider, - args.apiKey, - args.baseURLOverride, ), pushWorkflow: args.pushWorkflow, log: args.log, diff --git a/packages/seeding/src/index.ts b/packages/seeding/src/index.ts index 78506cd4fb..e67e47a58d 100644 --- a/packages/seeding/src/index.ts +++ b/packages/seeding/src/index.ts @@ -26,6 +26,7 @@ export { inferenceCredentialName, PLACEHOLDER_CATALOG_API_KEY, ensureCredential, + ensureNoopCatalogOffering, ensureProvider, reconcileSeedGrants, seedCatalog, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 24fc1ce6eb..21ca427949 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -987,55 +987,51 @@ async function ensureDeployment( return deployment.id; } +const ResolvedOfferingResponse = type({ + id: "string", + priority: "number", + modelId: "string", + providerId: "string", + origin: { tenantId: "string", direct: "boolean" }, +}); + +const ResolvedOfferingsResponse = type({ + offerings: ResolvedOfferingResponse.array(), +}); + /** - * Lists every catalog offering owned directly by a tenant, paging past - * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list - * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" - * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s - * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s - * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog - * directly on that same tenant, so the owned list already carries - * everything a plain `workbench seed` or the self-served connect flow - * ever seeded, with no HTTP surface exposing the resolved view to a - * caller outside the hub process. + * Lists the offerings visible to a tenant, including any inherited from + * an ancestor tenant — the same resolved view `listVisibleOfferings` + * (`@intx/db`) gives the hub's own workflow deployer + * (`apps/hub/src/index.ts`'s `workflowDeployer`), exposed at + * `GET .../catalog/resolved-offerings`. Seeding a tenant whose catalog is + * inherited rather than directly owned (e.g. a sub-tenant under a + * parent that already carries a real provider key) needs this resolved + * list, not the tenant-owned-only `GET .../catalog/offerings`. */ -async function listTenantCatalogOfferings( +async function listResolvedCatalogOfferings( api: ApiCall, cookies: string[], tenantId: string, -): Promise<(typeof ModelOfferingResponse.infer)[]> { - const items: (typeof ModelOfferingResponse.infer)[] = []; - let cursor: string | undefined; - for (;;) { - const query = - cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; - const listed = await api( - "GET", - `/api/tenants/${tenantId}/catalog/offerings${query}`, - undefined, - cookies, - ); - const page = parseAs( - paginatedSchema(ModelOfferingResponse), - listed.data, - "catalog offerings response", - ); - items.push(...page.data); - if (page.nextCursor === null) return items; - if (items.length > 10_000) { - throw new HubApiError( - `catalog offerings list for tenant ${tenantId} did not terminate while paging`, - "check the hub logs for the underlying failure, then re-run: workbench seed", - ); - } - cursor = page.nextCursor; - } +): Promise<(typeof ResolvedOfferingResponse.infer)[]> { + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/resolved-offerings`, + undefined, + cookies, + ); + return parseAs( + ResolvedOfferingsResponse, + listed.data, + "resolved catalog offerings response", + ).offerings; } /** * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow - * deploys against: every enabled offering the tenant owns, ordered by - * priority ascending, with the lowest-priority offering as the default — + * deploys against: every offering visible to the tenant (owned or + * inherited from an ancestor), ordered by priority ascending, with the + * lowest-priority offering as the default — * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the @@ -1058,10 +1054,8 @@ async function resolveRealSourceOfferingIds( sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string; }> { - const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) - .filter( - (offering) => !offering.disabled && offering.id !== excludeOfferingId, - ) + const offerings = (await listResolvedCatalogOfferings(api, cookies, tenantId)) + .filter((offering) => offering.id !== excludeOfferingId) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; if (defaultSourceOfferingId === undefined) { @@ -1087,7 +1081,7 @@ async function resolveRealSourceOfferingIds( * synthetic one. Idempotent, same as every other seed step: a re-run * finds the existing rows by name and reuses them. */ -async function ensureNoopCatalogOffering( +export async function ensureNoopCatalogOffering( api: ApiCall, cookies: string[], tenantId: string, From bcb8703e02e1a669a5d16651045a604993aeaf2e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:05 -0700 Subject: [PATCH 13/41] Add a regression test pinning the shared-sidecar auth gap (CL-7472) At the interchange pin this branch stacks on (692c3106), the hub authenticates a sidecar's register/reconnect frame only against a sidecar row that resolves to a live sidecar_allocation (or workflow_probe) row. Upstream dropped the older "shared" credential scope this repo's e2e harness used to provision a single, standalone sidecar identity for, so a provisionSidecar-only row now connects and is rejected with "invalid token" in a loop rather than ever dialing in. This test pins that negative so a future re-vendor that silently restores (or relies on) the old scope surfaces here first. --- scripts/e2e/harness.test.ts | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 37f4f93e4e..1a4f3fce85 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -6,10 +6,14 @@ import { describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; +import path from "node:path"; +import { resetSchema, setupDatabase } from "../db-setup.ts"; import { createCleanupHarness, + e2eDatabaseUrl, parseEnvFileDatabaseUrl, + provisionSidecar, runCleanups, workflowDeployBody, type SpawnedApp, @@ -127,3 +131,71 @@ describe("parseEnvFileDatabaseUrl", () => { ).toBe("postgres://second/db"); }); }); + +// CL-7472 regression: at pin 692c3106, the hub authenticates a +// sidecar's register/reconnect frame only against a `sidecar` row that +// resolves to a live `sidecar_allocation` (or `workflow_probe`) row — +// upstream dropped the older "shared" credential scope this repo used +// to provision a single, standalone sidecar identity for (see +// `provisionSidecar`'s own doc comment). CI's e2e suites hit exactly +// this: a `provisionSidecar`-only row connects and is rejected with +// "invalid token" in a loop (never a working dial-in), which is why +// every suite now lets the hub's own process-provisioner spawn a +// dedicated, already-allocated sidecar per deployment instead (see +// `startHub`'s doc comment). This test pins the negative: an +// unallocated identity must keep failing to authenticate, so a future +// upstream re-vendor that silently restores (or a caller that +// silently starts relying on) the old "shared" scope surfaces here +// first rather than as a mystery disconnect loop in CI. +describe("provisionSidecar", () => { + const databaseUrl = e2eDatabaseUrl(); + const HUB_DIR = path.resolve(import.meta.dir, "..", "..", "apps", "hub"); + + test.skipIf(databaseUrl === undefined)( + "a provisioned identity with no allocation or probe never authenticates", + async () => { + const url = databaseUrl; + if (url === undefined) throw new Error("unreachable: suite is skipped"); + + await resetSchema(url); + await setupDatabase(url); + + const sidecarId = "harness-test-unallocated-sidecar"; + const token = crypto.randomUUID(); + await provisionSidecar(url, sidecarId, token); + + // Resolved through the hub's own dependency tree, exactly as + // `connectE2eDb` resolves `postgres` above — scripts/e2e is not + // itself a workspace member of these `@intx/*` packages. + const dbModule = (await import( + Bun.resolveSync("@intx/db", HUB_DIR) + )) as { createDB: (raw: unknown) => { db: unknown; close: () => Promise } }; + const hubSessionsModule = (await import( + Bun.resolveSync("@intx/hub-sessions", HUB_DIR) + )) as { + createSidecarCredentialResolver: (deps: { db: unknown }) => { + resolve: (token: string) => Promise; + }; + }; + + const parsed = new URL(url); + const db = dbModule.createDB({ + host: parsed.hostname, + port: parsed.port === "" ? 5432 : Number(parsed.port), + user: decodeURIComponent(parsed.username), + password: decodeURIComponent(parsed.password), + database: parsed.pathname.replace(/^\//, ""), + }); + try { + const resolver = hubSessionsModule.createSidecarCredentialResolver({ + db: db.db, + }); + const identity = await resolver.resolve(token); + expect(identity).toBeNull(); + } finally { + await db.close(); + } + }, + 60_000, + ); +}); From ccbe497e63bf84527e9ad01dcc840b7be2155e10 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:16 -0700 Subject: [PATCH 14/41] e2e: stop pre-provisioning a shared sidecar identity (CL-7472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every e2e suite dial-in relied on a single, harness-started sidecar process registered against a bare `provisionSidecar` identity row. At pin 692c3106, the hub only accepts a register/reconnect frame from a sidecar whose identity resolves to a live sidecar_allocation (or workflow_probe) row; a standalone identity with neither now fails the handshake and gets rejected with "invalid token" every ~3s, forever (chat mint 500s, deploys never dial in — CI run 34042907014). The hub's own process-provisioner already spawns a dedicated, correctly-allocated sidecar process per deployment on demand — the same path production already takes for a real workflow run or a minted agent chat. Every affected suite (walking skeleton, chat, heartbeat, workbench-digest, smoke-chat, smoke-webhook, greeting-delivery, channel-reconnect, local-rip, and the isolation suite) now lets that happen instead of dialing in its own sidecar: the existing 502/500 retry-until-ready loops already cover the wait, and diagnostics that used to read the one sidecar process's output now read the hub's — the process-provisioner backend spawns each sidecar with its stdio inherited from the hub, so the hub's own captured output already carries every spawned sidecar's log lines. `provisionSidecar`/`startSidecar` stay in the harness (documented as non-authenticating at this pin) for the handful of standalone proof scripts not yet migrated off a directly-controlled sidecar process. --- scripts/e2e/channel-reconnect.test.ts | 37 +++++----------- scripts/e2e/chat.test.ts | 61 ++++++++++----------------- scripts/e2e/greeting-delivery.test.ts | 36 ++++------------ scripts/e2e/harness.ts | 26 +++++++++--- scripts/e2e/heartbeat.test.ts | 29 ++----------- scripts/e2e/local-rip.test.ts | 24 ++--------- scripts/e2e/smoke-chat.test.ts | 29 ++----------- scripts/e2e/smoke-webhook.test.ts | 38 +++-------------- scripts/e2e/walking-skeleton.test.ts | 45 +++++--------------- scripts/e2e/workbench-digest.test.ts | 29 ++----------- test/isolation/setup.ts | 49 ++++++--------------- 11 files changed, 109 insertions(+), 294 deletions(-) diff --git a/scripts/e2e/channel-reconnect.test.ts b/scripts/e2e/channel-reconnect.test.ts index 59e254c197..3c3e8e6377 100644 --- a/scripts/e2e/channel-reconnect.test.ts +++ b/scripts/e2e/channel-reconnect.test.ts @@ -29,12 +29,9 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -76,7 +73,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { let hubPort: number; let sessionSecret: string; let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let cookies: string[]; let tenantId: string; @@ -93,10 +89,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "workbench-reconnect-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hubPort = freePort(); sessionSecret = Buffer.from( crypto.getRandomValues(new Uint8Array(32)), @@ -110,14 +102,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-reconnect-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); const email = `workbench-reconnect-e2e-${crypto.randomUUID()}@example.invalid`; @@ -159,9 +143,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -174,7 +158,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -244,10 +228,10 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { // hub process on the same port against the same database, exactly // as a real restart looks from the sidecar's and the browser's // side: same DB rows, same session cookie, new process. The port - // must stay the same — the sidecar's own `HUB_WS_URL` is fixed at - // its boot and is never reconfigured here (only the hub restarts), - // so a different port would leave the sidecar dialing a hub that - // is no longer there. + // must stay the same — the process-provisioner-spawned sidecar's + // own `HUB_WS_URL` is fixed at its boot and is never reconfigured + // here (only the hub restarts), so a different port would leave the + // sidecar dialing a hub that is no longer there. await hub.stop(); const restarted = await startHub({ @@ -269,9 +253,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 180_000; let posted: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited after hub restart; output:\n${sidecar.output()}`, + `hub exited after hub restart; output:\n${hub.output()}`, ); } posted = await postMessage(after); @@ -279,8 +263,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `message after hub restart never accepted: status=${posted.status} ` + - `${JSON.stringify(posted.data)}\nsidecar output:\n${sidecar.output()}` + - `\nhub output:\n${hub.output()}`, + `${JSON.stringify(posted.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index da9b255c89..c50da08946 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -46,14 +46,11 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, pushWorkflowSource, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -139,7 +136,6 @@ const textPart = (text: string): Part[] => [{ kind: "text", text }]; describe.skipIf(databaseUrl === undefined)("chat e2e", () => { let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let user1: SignedUpUser; let user2: SignedUpUser; @@ -159,10 +155,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "chat-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hub = await startHub({ databaseUrl: url, port: freePort(), @@ -173,16 +165,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-chat-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); // Two independent browser-shaped accounts, each carrying its own @@ -358,15 +340,15 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { ), }); - // Retries while the hub still answers 502 (the sidecar's dial-in - // may not have completed yet), matching `createWorkbench`'s own - // retry loop below. + // Retries while the hub still answers 502 (the process + // provisioner's spawned sidecar may not have dialed in yet), + // matching `createWorkbench`'s own retry loop below. const echoDeployDeadline = Date.now() + 60_000; let echoDeployed: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before echo deploy; output:\n${sidecar.output()}`, + `hub exited before echo deploy; output:\n${hub.output()}`, ); } echoDeployed = await api( @@ -384,7 +366,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > echoDeployDeadline) { throw new Error( `echo workflow never became deployable (hub kept answering 502): ` + - `${JSON.stringify(echoDeployed.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(echoDeployed.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -393,22 +375,22 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }, 120_000); // Launching a workbench is the go/no-go signal for the whole suite: it - // launches the anchor instance in-process, which needs the - // sidecar's dial-in to have completed. `packages/chat/src/routes.ts` - // has no 502-style retry translation of its own (unlike the native - // workflow deploy route), so a launch attempted before the sidecar - // connects fails with an uncaught 500 — retried here directly until - // the sidecar is ready, exactly as the walking skeleton retries a - // 502 for the native deploy route. + // launches the anchor instance in-process, which needs its + // process-provisioner-spawned sidecar's dial-in to have completed. + // `packages/chat/src/routes.ts` has no 502-style retry translation of + // its own (unlike the native workflow deploy route), so a launch + // attempted before the sidecar connects fails with an uncaught 500 — + // retried here directly until the sidecar is ready, exactly as the + // walking skeleton retries a 502 for the native deploy route. async function createWorkbench( body: Record, ): Promise { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -421,7 +403,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -1021,10 +1003,13 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { // temporarily reverting the self-anchor fix locally: the sidecar log // then carries ten `enqueueInbox failed, withholding ack` lines, one // per rejected pack, for this same suite; with the fix applied, zero. + // Every process-provisioner-spawned sidecar inherits the hub + // process's own stdio (`process-runner.ts`'s `stdout: "inherit"`), so + // `hub.output()` already carries every sidecar's log lines. test("no chat/folded run's workflow-run pack was ever permanently rejected", () => { - const sidecarOutput = sidecar.output(); - expect(sidecarOutput).not.toContain("enqueueInbox failed"); - expect(sidecarOutput).not.toContain("withholding ack"); - expect(sidecarOutput).not.toContain("rejecting inbound mail"); + const hubOutput = hub.output(); + expect(hubOutput).not.toContain("enqueueInbox failed"); + expect(hubOutput).not.toContain("withholding ack"); + expect(hubOutput).not.toContain("rejecting inbound mail"); }); }); diff --git a/scripts/e2e/greeting-delivery.test.ts b/scripts/e2e/greeting-delivery.test.ts index 0ac6e26bb2..2521206f1a 100644 --- a/scripts/e2e/greeting-delivery.test.ts +++ b/scripts/e2e/greeting-delivery.test.ts @@ -41,12 +41,9 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const { tempDir, track } = createCleanupHarness(); @@ -135,10 +132,6 @@ describe.skipIf(databaseUrl === undefined)( expect(report.action).toBe("migrated"); }); - const sidecarId = "greeting-delivery-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -151,16 +144,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar: SpawnedApp = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-greeting-delivery-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -236,9 +219,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { @@ -269,9 +252,9 @@ describe.skipIf(databaseUrl === undefined)( async function deploySeededWorkflows(): Promise { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -356,9 +339,9 @@ describe.skipIf(databaseUrl === undefined)( const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before chat creation; output:\n${sidecar.output()}`, + `hub exited before chat creation; output:\n${hub.output()}`, ); } res = await api( @@ -372,7 +355,7 @@ describe.skipIf(databaseUrl === undefined)( if (Date.now() > deadline) { throw new Error( `chat never became mintable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -448,7 +431,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no agent-authored message landed in chat ${chatId} within ` + `60s of mint with zero user messages sent; messages seen: ` + - `${JSON.stringify(items)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(items)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -527,8 +510,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no answer to turn ${turn} within 180s; agent messages: ` + `${JSON.stringify(agentTexts)}\nhub output (tail):\n` + - `${hub.output().slice(-60000)}\nsidecar output (tail):\n` + - `${sidecar.output().slice(-6000)}`, + `${hub.output().slice(-60000)}`, ); } await Bun.sleep(2000); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index cebc4e53db..1e884e0f9e 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -221,6 +221,20 @@ export async function connectE2eDb(databaseUrl: string): Promise { * WebSocket dial-in against the token hash on the `sidecar` table, so * the row must exist before the sidecar process starts. Mirrors * Interchange's dev provisioning (id + sha256(token), placeholder url). + * + * At pin 692c3106, the hub only accepts a register/reconnect frame from + * a sidecar identity that resolves to a live `sidecar_allocation` (or + * `workflow_probe`) row — the standalone, unallocated ("shared" + * credential scope) sidecar this used to provision no longer + * authenticates at all (CL-7472), so a caller of this function gets a + * connect-and-immediately-reject loop, not a usable dial-in. Every + * suite this repo runs against CI now lets the hub's own + * process-provisioner spawn a dedicated sidecar per allocation instead + * (see `startHub`'s doc comment and `scripts/e2e/chat.test.ts`); this + * function is kept only for the handful of scripts not yet migrated off + * a directly-controlled sidecar process (e.g. ones that kill and + * restart a specific sidecar mid-scenario) and is not a working dial-in + * path until they are. */ export async function provisionSidecar( databaseUrl: string, @@ -400,11 +414,13 @@ export async function startHub(options: { } /** - * Boot the sidecar as a real process, pointed at the hub's WebSocket - * dial-in route with the provisioned identity. Connection readiness is - * observed by the deploy call itself (the hub answers 502 until a - * sidecar is connected), so this only guards against an immediate - * crash. + * Boot a standalone sidecar as a real process, pointed at the hub's + * WebSocket dial-in route with a `provisionSidecar`-issued identity. + * See `provisionSidecar`'s own doc comment: at pin 692c3106 this + * identity never authenticates (CL-7472), so the process this starts + * connects and is immediately rejected in a loop rather than dialing + * in. Kept only for the scripts not yet migrated to the hub's + * automatic, per-allocation process-provisioner spawn. */ export function startSidecar(options: { hubPort: number; diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 9aefb900fb..8c924a6af9 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -95,12 +93,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-heartbeat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -114,19 +106,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-heartbeat-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Heartbeat Tester", @@ -224,10 +203,8 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -240,7 +217,7 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 2b4f4ec9c5..b1919de025 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -81,9 +81,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -187,10 +185,6 @@ describe.skipIf(databaseUrl === undefined)( } }); - const sidecarId = "local-rip-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -208,16 +202,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-local-rip-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -341,9 +325,9 @@ describe.skipIf(databaseUrl === undefined)( ): Promise>> { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -398,9 +382,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { diff --git a/scripts/e2e/smoke-chat.test.ts b/scripts/e2e/smoke-chat.test.ts index 98b5983321..eaf0da399c 100644 --- a/scripts/e2e/smoke-chat.test.ts +++ b/scripts/e2e/smoke-chat.test.ts @@ -20,9 +20,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -58,13 +56,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { }); const hubDataDir = await tempDir("e2e-smoke-chat-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-chat-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-chat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -78,18 +69,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -123,9 +102,9 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { // through the transient 500 the same way chat.test.ts does. const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } const res = await api( @@ -141,7 +120,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { } if (Date.now() > deadline) { throw new Error( - `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -171,7 +150,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { if (Date.now() > deadline) { throw new Error( `post message kept answering 500: ${JSON.stringify(res.data)}` + - `\nsidecar output:\n${sidecar.output()}`, + `\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 760aca6a34..d834aa623e 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -12,9 +12,8 @@ // events by bare id (the native events route requires a deployment-anchor // run). This test instead reads the run row the ingress delivery created // straight out of the database — a harness-side fact with no route, -// exactly as `provisionSidecar` already does for the sidecar identity -// row — and confirms the trigger itself recorded the delivery -// (`lastFiredAt`). +// via `connectE2eDb` — and confirms the trigger itself recorded the +// delivery (`lastFiredAt`). import { signPayload, @@ -36,12 +35,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -77,13 +74,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const hubDataDir = await tempDir("e2e-smoke-webhook-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-webhook-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-webhook"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -97,18 +87,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -204,10 +182,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } const res = await api( hub.baseUrl, @@ -219,7 +195,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { if (res.status === 502) { if (Date.now() > deadline) { throw new Error( - `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -346,8 +322,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // No platform route reads a non-anchored run's row by bare id // (see the file header); this reads the fact the ingress // route's own launch is required to have produced, the same - // way `provisionSidecar` reaches the database directly for a - // fact no route exposes. + // way `connectE2eDb` reaches the database directly for a fact + // no route exposes. const sql = await connectE2eDb(url); try { const rows = await sql.unsafe( diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 8937f9b5e5..e876f6bfaf 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -33,12 +33,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, } from "./harness.ts"; @@ -81,15 +79,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { expect(second.migrations).toBe(first.migrations); }); - // Hop: sidecar provisioning. The identity row the hub checks the - // sidecar's dial-in token against. - const sidecarId = "sidecar-e2e"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - - // Hop: hub boot. The composition root as a real process. + // Hop: hub boot. The composition root as a real process. The hub's + // own process provisioner spawns a dedicated sidecar for every + // allocation on demand (one `apps/sidecar` per deployment) — there + // is no shared, pre-provisioned sidecar identity to dial in here. const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -103,21 +96,6 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { return handle; }); - // Hop: sidecar boot. Dial-in readiness is observed at the deploy - // hop (the hub answers 502 until a sidecar is connected). - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-sidecar-data-"), - }); - track(app); - return app; - }); - // Hop: sign-up. A browser-shaped account creation through the // platform's better-auth surface; the session cookie carries the // rest of the skeleton. @@ -215,9 +193,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { ); // Hop: workflow deploy via the native deploy API. Retries while - // the hub still answers 502 (the sidecar's dial-in may not have - // completed yet); any other failure is final. The inference - // source is a placeholder — deployment does not call inference. + // the hub still answers 502 (the process provisioner's spawned + // sidecar may not have dialed in yet); any other failure is final. + // The inference source is a placeholder — deployment does not call + // inference. const deploymentId = await hop("workflow deploy", async () => { const body = workflowDeployBody({ assetId, @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 8917bf8236..f3d41cd06a 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -100,12 +98,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-workbench-digest"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -119,19 +111,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-digest-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Workbench Digest Tester", @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/test/isolation/setup.ts b/test/isolation/setup.ts index 3644750fa6..07271b0c16 100644 --- a/test/isolation/setup.ts +++ b/test/isolation/setup.ts @@ -10,11 +10,8 @@ import path from "node:path"; import { freePort, - provisionSidecar, startHub, - startSidecar, type HubHandle, - type SpawnedApp, } from "../../scripts/e2e/harness.ts"; import { assertDatabaseConfigured } from "../../scripts/e2e/db-gate.ts"; @@ -90,29 +87,21 @@ export async function prepareDatabase(databaseUrl: string): Promise { } /** - * Boots the real hub and a real sidecar as spawned processes, exactly - * as `scripts/e2e/harness.ts` does for the walking skeleton — a - * chat-workbench create launches a real agent instance over the - * sidecar's WebSocket dial-in, so an in-process hub with no sidecar - * attached cannot exercise the "chat workbench move" block below; it - * would fail every one of those cases with "no sidecar available" - * rather than proving isolation. `AppLike.request` wraps `fetch` - * against the hub's real port, matching how a client actually reaches - * it. + * Boots the real hub as a spawned process, exactly as + * `scripts/e2e/harness.ts` does for the walking skeleton — a + * chat-workbench create launches a real agent instance over its own + * process-provisioner-spawned sidecar's WebSocket dial-in, so an + * in-process hub with no sidecar plugin wired up cannot exercise the + * "chat workbench move" block below; it would fail every one of those + * cases with "no sidecar available" rather than proving isolation. + * `AppLike.request` wraps `fetch` against the hub's real port, matching + * how a client actually reaches it. */ export async function bootIsolationHub( databaseUrl: string, ): Promise { const root = mkdtempSync(path.join(tmpdir(), "isolation-suite-")); const hubDataDir = path.join(root, "hub-data"); - const sidecarDataDir = path.join(root, "sidecar-data"); - - // Unique per run: the `sidecar` table's id is a primary key, and a - // prior run's row is never cleaned up (the isolation suite runs - // against a shared database, never a scratch one it owns outright). - const sidecarId = `sidecar-isolation-suite-${crypto.randomUUID()}`; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(databaseUrl, sidecarId, sidecarToken); const hub: HubHandle = await startHub({ databaseUrl, @@ -133,23 +122,15 @@ export async function bootIsolationHub( }, }); - const sidecar: SpawnedApp = startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }); - const app: AppLike = { request: (requestPath, init) => fetch(`${hub.baseUrl}${requestPath}`, init), }; - await waitForSidecarDialIn(app, sidecar); + await waitForSidecarDialIn(app, hub); return { app, shutdown: async () => { - await sidecar.stop(); await hub.stop(); rmSync(root, { recursive: true, force: true }); }, @@ -169,7 +150,7 @@ export async function bootIsolationHub( */ async function waitForSidecarDialIn( app: AppLike, - sidecar: SpawnedApp, + hub: HubHandle, ): Promise { const cookie = await signUpUser( app, @@ -194,10 +175,8 @@ async function waitForSidecarDialIn( const deadline = Date.now() + 30_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before dialing in; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before dialing in; output:\n${hub.output()}`); } const workbenchResponse = await app.request( `/api/tenants/${tenant.id}/chat/workbenches`, @@ -212,7 +191,7 @@ async function waitForSidecarDialIn( throw new Error( `sidecar never dialed in within 30s (workbench create kept ` + `failing with ${workbenchResponse.status}): ` + - `${await workbenchResponse.text()}\nsidecar output:\n${sidecar.output()}`, + `${await workbenchResponse.text()}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(500); From 308ab2526a5018b2e65ae4bff4eb17805ecefdc7 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:21 -0700 Subject: [PATCH 15/41] ci: upload hub/sidecar e2e logs as a build artifact (CL-7472) The e2e job's hub/sidecar output was never captured, so the CI run that surfaced the sidecar disconnect loop (34042907014) had no sidecar-side log to read past the transcript. Point E2E_LOG_DIR at a runner-temp directory and upload it on every run, pass or fail. --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a353634006..4e67a69e02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,15 @@ jobs: run: bun run test:e2e env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Upload hub/sidecar logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs + path: ${{ runner.temp }}/e2e-logs + if-no-files-found: ignore isolation: runs-on: ubuntu-latest From b72964e24b7d8b384459eca23fa1b15ab45a105f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:01 -0700 Subject: [PATCH 16/41] Add tests for tenant-scoped workbench_launch address resolution (CL-7474) --- packages/chat/src/agent-binding.test.ts | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index ed89cec9db..cd59d2172e 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -96,6 +96,56 @@ describe("readBindingByAddress", () => { }); }); +describe("readBindingByAddress: tenant scoping (CL-7474)", () => { + // Two tenants that each independently invited "the same" agent + // (definitionId happens to differ per tenant in practice, but the + // launch rows below are shaped exactly like two independent + // launches — different stable ids, different run ids, different + // tenants). `instanceId`/`currentRunId` are collision-resistant + // generated ids, not scoped to a tenant, so nothing but an explicit + // `expectedTenantId` check stops tenant B's caller from resolving + // tenant A's row if it ever guessed or replayed tenant A's address. + const tenantALaunch: LaunchRow = { + tenantId: "tnt_a", + instanceId: "run_a1", + currentRunId: "run_a1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const tenantBLaunch: LaunchRow = { + tenantId: "tnt_b", + instanceId: "run_b1", + currentRunId: "run_b1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const db = fakeDb([tenantALaunch, tenantBLaunch]); + + test("each tenant's DM resolves to its own distinct run id and address", async () => { + const a = await readBindingByAddress(db, "run_a1@acme.example", "tnt_a"); + const b = await readBindingByAddress(db, "run_b1@acme.example", "tnt_b"); + expect(a?.currentRunId).toBe("run_a1"); + expect(b?.currentRunId).toBe("run_b1"); + expect(a?.currentRunId).not.toBe(b?.currentRunId); + expect(a?.roomAddress).not.toBe(b?.roomAddress); + }); + + test("a bench invite in tenant B never resolves tenant A's launch", async () => { + expect( + await readBindingByAddress(db, "run_a1@acme.example", "tnt_b"), + ).toBeUndefined(); + expect( + await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), + ).toBeUndefined(); + }); + + test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { + expect( + (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, + ).toBe("tnt_a"); + }); +}); + describe("resolveRoomAddress", () => { test("leaves a non-participant address alone", async () => { expect( From 262a381344ead7f2dec2a47f17f371b3ff4070bb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:09 -0700 Subject: [PATCH 17/41] Chat: workbench_launch address/stableId lookups are tenant-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readBindingByAddress and resolveLiveByStableId took an address or stable id and resolved whichever workbench_launch row matched it, regardless of which tenant that row belonged to. Every caller that already knows its own tenant (sendMail's requireLive, and refreshAgentInstanceFromDefinition) now passes it through and gets undefined instead of another tenant's launch if the two ever disagree. The one legitimate tenant-agnostic caller — the event stream handler discovering which tenant an inbound address even belongs to before it has one to check against — keeps working by simply omitting the new optional parameter. Also fixes offeringDigest: it joined catalog offering ids with a NUL byte, which is not valid UTF-8 to Postgres the moment a tenant's catalog carries more than one visible offering ("invalid byte sequence for encoding \"UTF8\": 0x00"), permanently failing the workbench_launch insert for every such launch. This is what was failing 4 of scripts/e2e/chat.test.ts's cases under per-allocation sidecars, where seedCatalog now produces more than one offering. --- packages/chat/src/agent-binding.ts | 25 ++++++++++++++++++++++--- packages/chat/src/platform-adapter.ts | 23 +++++++++++++++++------ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index d53399366c..6d3917d4f0 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,13 +135,25 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, + expectedTenantId?: string, ): Promise { const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - if (byStableId !== undefined) return bindingFrom(byStableId, domain); - const byRunId = await readLaunchRow(db, "currentRunId", localPart); - return byRunId === undefined ? undefined : bindingFrom(byRunId, domain); + const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + if (row === undefined) return undefined; + // A caller that already knows which tenant it is acting for must + // never accept a binding minted by another one — `instanceId` and + // `currentRunId` are collision-resistant generated ids, not scoped to + // a tenant, so nothing else stops an address that happens to name a + // row from a different tenant from resolving here. A caller with no + // tenant of its own yet (the event-stream handler discovering which + // tenant an inbound address even belongs to) passes no + // `expectedTenantId` and gets the address-only resolution it needs. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } + return bindingFrom(row, domain); } /** @@ -183,9 +195,16 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, + expectedTenantId?: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; + // See the matching note on `readBindingByAddress`: a caller that + // knows its own tenant must never be handed another tenant's launch + // for a stable id it happens to guess or receive by mistake. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 6490af3c7e..f871887dae 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -258,7 +258,15 @@ export function createHubChatPlatform( }; function offeringDigest(sourceOfferingIds: readonly string[]): string { - return sourceOfferingIds.join("\0"); + // `sources_digest` is a `text` column and this digest rides straight + // into it (`platform-adapter.ts`'s `insert(workbenchLaunch)`). A + // `\0`-joined id list is not valid UTF-8 text to Postgres + // (`invalid byte sequence for encoding "UTF8": 0x00`) the moment a + // tenant's catalog carries more than one visible offering, which + // permanently failed every agent launch for such a tenant. `\n` never + // appears inside a generated offering id, so this stays a lossless, + // order-sensitive join — just one Postgres can actually store. + return sourceOfferingIds.join("\n"); } async function catalogOfferings(tenantId: string): Promise<{ @@ -913,8 +921,11 @@ export function createHubChatPlatform( * participant id — not the room's own address, once anything has been * relaunched. */ - async function requireLive(stableId: string): Promise { - const live = await resolveLiveByStableId(deps.db, stableId); + async function requireLive( + stableId: string, + expectedTenantId?: string, + ): Promise { + const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1176,7 +1187,7 @@ export function createHubChatPlatform( _workbenchId, address, ): Promise { - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddress(deps.db, address, tenantId); if (binding === undefined) return; const live = await resolveLiveAgent(deps.db, binding); const definitionId = live?.run.definitionId; @@ -1216,7 +1227,7 @@ export function createHubChatPlatform( // The stable id names the room's participant; the run it resolves // to is whichever one is alive right now, which is a different // run (and a different address) after every relaunch. - const { binding } = await requireLive(input.workbenchId); + const { binding } = await requireLive(input.workbenchId, input.tenantId); const liveAddress = binding.liveAddress; // Wake before send: a sleeping instance (the lifecycle package's @@ -1270,7 +1281,7 @@ export function createHubChatPlatform( } catch (error) { throw wrapWakeInferenceError(error); } - const delivery = await requireLive(input.workbenchId); + const delivery = await requireLive(input.workbenchId, input.tenantId); const deliveryAddress = delivery.binding.liveAddress; // Tracking here (not only at launch) brings instances that were // already resident before this hub process started — restored by From b75eebc071aaa7060109de799db16375504bc0bf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:01:01 -0700 Subject: [PATCH 18/41] Chat: run prettier on the tenant-scoping change --- packages/chat/src/agent-binding.ts | 3 ++- packages/chat/src/platform-adapter.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 6d3917d4f0..18bad4513c 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -140,7 +140,8 @@ export async function readBindingByAddress( const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index f871887dae..602ec0b0d1 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -925,7 +925,11 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); + const live = await resolveLiveByStableId( + deps.db, + stableId, + expectedTenantId, + ); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } From 0eef0e0034af3e0c5cf7870b1d0e3cba8cfb72ba Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:26:38 -0700 Subject: [PATCH 19/41] Chat: tenant-scoped binding reads require the tenant; the one unscoped read is named --- packages/chat/src/agent-binding.test.ts | 22 +++++-- packages/chat/src/agent-binding.ts | 82 +++++++++++++++++++------ packages/chat/src/chat-orchestrator.ts | 10 ++- packages/chat/src/platform-adapter.ts | 31 +++++++--- 4 files changed, 109 insertions(+), 36 deletions(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index cd59d2172e..947404cf34 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from "bun:test"; import { isBeyondWake, readBindingByAddress, + readBindingByAddressAnyTenant, resolveRoomAddress, } from "./agent-binding"; @@ -71,6 +72,7 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_original@acme.example", + "ten_1", ); expect(binding?.stableId).toBe("run_original"); expect(binding?.currentRunId).toBe("run_fresh"); @@ -85,13 +87,18 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_fresh@acme.example", + "ten_1", ); expect(binding?.roomAddress).toBe("run_original@acme.example"); }); test("is undefined for an address this package never launched", async () => { expect( - await readBindingByAddress(fakeDb([relaunched]), "echo_1@acme.example"), + await readBindingByAddress( + fakeDb([relaunched]), + "echo_1@acme.example", + "ten_1", + ), ).toBeUndefined(); }); }); @@ -139,10 +146,17 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { ).toBeUndefined(); }); - test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { +}); + +describe("readBindingByAddressAnyTenant", () => { + test("the address-only resolution event-stream discovery needs, with no tenant to check against", async () => { + const anyTenantDb = fakeDb([ + { ...relaunched, instanceId: "run_a1", currentRunId: "run_a1" }, + ]); expect( - (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, - ).toBe("tnt_a"); + (await readBindingByAddressAnyTenant(anyTenantDb, "run_a1@acme.example")) + ?.tenantId, + ).toBe("ten_1"); }); }); diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 18bad4513c..47b4d0d714 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,26 +135,48 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, - expectedTenantId?: string, + tenantId: string, ): Promise { - const domain = requireDomain(address); - const localPart = localPartOf(address); - const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = - byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = await readBindingRowByAddress(db, address); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and // `currentRunId` are collision-resistant generated ids, not scoped to // a tenant, so nothing else stops an address that happens to name a - // row from a different tenant from resolving here. A caller with no - // tenant of its own yet (the event-stream handler discovering which - // tenant an inbound address even belongs to) passes no - // `expectedTenantId` and gets the address-only resolution it needs. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } - return bindingFrom(row, domain); + // row from a different tenant from resolving here. `tenantId` is + // required precisely so a caller cannot silently fall back to the + // unscoped read by forgetting to pass it — see + // `readBindingByAddressAnyTenant` for the one place that read is + // actually correct. + if (row.row.tenantId !== tenantId) return undefined; + return bindingFrom(row.row, row.domain); +} + +/** + * The unscoped counterpart of `readBindingByAddress`, for the one caller + * that genuinely cannot know the tenant yet: the inbound event-stream + * handler discovering which tenant an address even belongs to before it + * can act as that tenant. Every other caller already knows its own + * tenant and must use `readBindingByAddress` instead. + */ +export async function readBindingByAddressAnyTenant( + db: DB["db"], + address: string, +): Promise { + const row = await readBindingRowByAddress(db, address); + return row === undefined ? undefined : bindingFrom(row.row, row.domain); +} + +async function readBindingRowByAddress( + db: DB["db"], + address: string, +): Promise<{ row: LaunchRow; domain: string } | undefined> { + const domain = requireDomain(address); + const localPart = localPartOf(address); + const byStableId = await readLaunchRow(db, "instanceId", localPart); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + return row === undefined ? undefined : { row, domain }; } /** @@ -167,7 +189,7 @@ export async function resolveRoomAddress( db: DB["db"], liveAddress: string, ): Promise { - const binding = await readBindingByAddress(db, liveAddress); + const binding = await readBindingByAddressAnyTenant(db, liveAddress); return binding?.roomAddress ?? liveAddress; } @@ -196,16 +218,36 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, - expectedTenantId?: string, + tenantId: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; // See the matching note on `readBindingByAddress`: a caller that // knows its own tenant must never be handed another tenant's launch - // for a stable id it happens to guess or receive by mistake. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } + // for a stable id it happens to guess or receive by mistake. Required, + // not optional — see `resolveLiveByStableIdAnyTenant` for the unscoped + // read. + if (row.tenantId !== tenantId) return undefined; + return resolveLiveRunForRow(db, row); +} + +/** + * The unscoped counterpart of `resolveLiveByStableId`, for a caller that + * has no tenant of its own to check against (e.g. it is resolving a + * stable id it does not yet know the tenant for). + */ +export async function resolveLiveByStableIdAnyTenant( + db: DB["db"], + stableId: string, +): Promise { + const row = await readLaunchRow(db, "instanceId", stableId); + return row === undefined ? undefined : resolveLiveRunForRow(db, row); +} + +async function resolveLiveRunForRow( + db: DB["db"], + row: LaunchRow, +): Promise { const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/chat-orchestrator.ts b/packages/chat/src/chat-orchestrator.ts index 808c755ba2..73ce91b0cb 100644 --- a/packages/chat/src/chat-orchestrator.ts +++ b/packages/chat/src/chat-orchestrator.ts @@ -57,7 +57,10 @@ import { consumerFacingInferenceText } from "./consumer-inference-text"; import type { ConnectedProviderLister } from "./inference-preferences"; import { mentionedParticipants } from "./mentions"; import { domainOf, localPartOf } from "./agent-address"; -import { readBindingByAddress, resolveLiveAgent } from "./agent-binding"; +import { + readBindingByAddressAnyTenant, + resolveLiveAgent, +} from "./agent-binding"; import { parseParticipants, type ParticipantRecord } from "./participants"; import type { Part, TextPart } from "./parts"; import type { ChatPlatform } from "./platform-port"; @@ -448,7 +451,10 @@ async function resolveMemberWorkbenches( // event's address against `workflow_run.address` directly: after a // relaunch the two differ, and only the mapping knows that the run // announcing itself under a fresh address is the same room teammate. - const binding = await readBindingByAddress(deps.db, agentAddress); + // The tenant an inbound event's address belongs to is exactly what + // this discovery is trying to find out, so it cannot pass one yet — + // see `readBindingByAddressAnyTenant`. + const binding = await readBindingByAddressAnyTenant(deps.db, agentAddress); if (binding === undefined) { // Not every agent address on the event stream belongs to a chat // workbench (an echo instance, say) — an address this package's own diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 602ec0b0d1..39e52ff2c8 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -33,11 +33,13 @@ import { listLaunchesBeyondWake, listLaunchesForTenant, readBindingByAddress, + readBindingByAddressAnyTenant, readPriorRuns, recordSourcesDigest, repointBinding, resolveLiveAgent, resolveLiveByStableId, + resolveLiveByStableIdAnyTenant, type AgentBinding, type LiveAgent, } from "./agent-binding"; @@ -703,7 +705,13 @@ export function createHubChatPlatform( * that treatment, just triggered by content drift instead of death. */ async function wakeByAddress(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // Every caller of `wakeByAddress` already reached `address` through + // a tenant-scoped resolution of its own (`sendMail`'s + // `requireLive(workbenchId, tenantId)`, `ensureAwake`'s caller-held + // address) — re-deriving the binding here is bookkeeping for a wake + // already authorized upstream, not a new trust boundary crossing, so + // the unscoped read is correct. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error( `No workbench_launch binding for address "${address}"; instances ` + @@ -809,7 +817,9 @@ export function createHubChatPlatform( * check that used to never run for it. */ async function reconcileDriftedRun(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // See the matching note in `wakeByAddress`: `address` is always + // reached through a tenant-scoped resolution upstream. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return false; const live = await resolveLiveAgent(deps.db, binding); if (live === undefined || live.run.address === null) return false; @@ -925,11 +935,10 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId( - deps.db, - stableId, - expectedTenantId, - ); + const live = + expectedTenantId === undefined + ? await resolveLiveByStableIdAnyTenant(deps.db, stableId) + : await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1166,7 +1175,9 @@ export function createHubChatPlatform( }, async resolveDefinitionIdByAddress(address): Promise { - const binding = await readBindingByAddress(deps.db, address); + // No tenant is threaded through this platform method's own + // signature; unscoped by necessity, not by omission. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return undefined; const live = await resolveLiveAgent(deps.db, binding); return live?.run.definitionId ?? undefined; @@ -1409,7 +1420,7 @@ export function createHubChatPlatform( let cancelled = false; let unsubscribeAgent: (() => void) | undefined; - void resolveLiveByStableId(deps.db, workbenchId) + void resolveLiveByStableIdAnyTenant(deps.db, workbenchId) .then((live) => { if (cancelled || live === undefined) return; unsubscribeAgent = deps.sidecarRouter.subscribeAgent( @@ -1437,7 +1448,7 @@ export function createHubChatPlatform( // undelivered-mail handler holds whatever the envelope named), so // the lifecycle is driven on the LIVE address it resolves to — // that is the only address the sidecar ever announces. - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error(`No workbench_launch binding for address "${address}"`); } From b6774510f8c69df16be37b566295d8f49b402ad3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:08 -0700 Subject: [PATCH 20/41] Add tests for the shared retry-until-routable delivery helper --- packages/webhook-triggers/test/launch.test.ts | 36 +++++++++ .../src/deliver-when-routable.test.ts | 74 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 packages/workflows/src/deliver-when-routable.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index f1a9bb2c1f..fa75590866 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -137,6 +137,7 @@ let sendUserMessageCalls: unknown[] = []; let sendUserMessageImpl: () => Promise = async () => new Uint8Array([1]); let cryptoGetKeys: string[] = []; +let isRoutableForTest = true; function baseDeps() { return { @@ -177,6 +178,7 @@ function baseDeps() { return sendUserMessageImpl(); }, }, + isRoutable: () => isRoutableForTest, }; } @@ -191,6 +193,7 @@ function resetLaunchSpies() { visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS]; frozenProjection = INERT_PROJECTION; sendUserMessageImpl = async () => new Uint8Array([1]); + isRoutableForTest = true; } describe("launchWebhookTrigger", () => { @@ -214,6 +217,39 @@ describe("launchWebhookTrigger", () => { expect(sendUserMessageCalls).toHaveLength(1); }); + // CL-7476: a freshly provisioned run's sidecar takes several seconds to + // boot and register with the hub. The opening mail can land in that + // gap and fail "agent is unreachable" even though the run deployed + // fine — `deliverWhenRoutable` must retry once the address becomes + // routable rather than dropping the mail for good. + test("retries the opening mail once the freshly provisioned run becomes routable", async () => { + resetLaunchSpies(); + isRoutableForTest = false; + let sendAttempts = 0; + sendUserMessageImpl = async () => { + sendAttempts += 1; + if (sendAttempts === 1) { + throw new Error("agent is unreachable: ins_1@ten1.workbench.test"); + } + return new Uint8Array([1]); + }; + + const deps = baseDeps(); + const sendPromise = launchWebhookTrigger(deps, TRIGGER, { status: "ok" }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + isRoutableForTest = true; + + const result = await sendPromise; + + expect(result).toEqual({ + instanceId: INTERCHANGE_RUN_ID, + triggerAddress: INTERCHANGE_ADDRESS, + }); + expect(sendUserMessageCalls).toHaveLength(2); + expect(reportErrorCalls).toHaveLength(0); + }); + test("reports the delivery failure with the run's context", async () => { resetLaunchSpies(); const deliveryError = new Error("sidecar unreachable"); diff --git a/packages/workflows/src/deliver-when-routable.test.ts b/packages/workflows/src/deliver-when-routable.test.ts new file mode 100644 index 0000000000..b283de6a1a --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { deliverWhenRoutable } from "./deliver-when-routable"; + +function unreachable(): Error { + return new Error("agent is unreachable: run_1@ten1.workbench.test"); +} + +describe("deliverWhenRoutable", () => { + test("retries once the address is routable after an unreachable send", async () => { + let sendCalls = 0; + let routablePolls = 0; + const sleeps: number[] = []; + + const result = await deliverWhenRoutable({ + send: async () => { + sendCalls++; + if (sendCalls === 1) { + throw unreachable(); + } + return "delivered"; + }, + isRoutable: () => { + routablePolls++; + return routablePolls >= 2; + }, + sleep: async (ms) => { + sleeps.push(ms); + }, + }); + + expect(result).toBe("delivered"); + expect(sendCalls).toBe(2); + expect(routablePolls).toBe(2); + expect(sleeps.length).toBe(1); + }); + + test("rethrows the original unreachable error once the deadline passes", async () => { + let now = 0; + const originalNow = Date.now; + Date.now = () => now; + try { + let sendCalls = 0; + await expect( + deliverWhenRoutable({ + send: async () => { + sendCalls++; + throw unreachable(); + }, + isRoutable: () => false, + deadlineMs: 500, + pollIntervalMs: 100, + sleep: async (ms) => { + now += ms; + }, + }), + ).rejects.toThrow("agent is unreachable"); + expect(sendCalls).toBe(1); + } finally { + Date.now = originalNow; + } + }); + + test("does not retry an error other than unreachable", async () => { + const err = new Error("boom"); + await expect( + deliverWhenRoutable({ + send: async () => { + throw err; + }, + isRoutable: () => true, + }), + ).rejects.toBe(err); + }); +}); From 5cb3164f04a8ae0406b4be5a50bb50eb379107ff Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:12 -0700 Subject: [PATCH 21/41] Deliver a webhook trigger's opening mail once its sidecar becomes routable (CL-7476) --- apps/hub/src/index.ts | 2 + packages/webhook-triggers/src/launch.ts | 32 +++++++--- .../workflows/src/deliver-when-routable.ts | 63 +++++++++++++++++++ packages/workflows/src/index.ts | 5 ++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 packages/workflows/src/deliver-when-routable.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 687bcde64e..3914c6bef5 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2258,6 +2258,8 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, credentialCipher, + isRoutable: (address) => + sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 0e6e03b68d..6f3b3adfaa 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -14,6 +14,7 @@ // reports through `@corbits/error-sink`, naming the run. import { reportError } from "@corbits/error-sink"; import { + deliverWhenRoutable, readDefinitionProjection, readFoldedBody, WORKFLOW_SOURCE_ENTRY, @@ -46,6 +47,15 @@ export type LaunchWebhookTriggerDeps = { "prepareProvisionedDeployment" >; sessionService: Pick; + /** + * Reads the hub's live sidecar routing table. A freshly provisioned + * run's sidecar takes several seconds to boot and register (CL-7476) + * — a webhook's opening mail can land in that gap and fail with + * "agent is unreachable" even though the run itself deployed fine. + * `deliverWhenRoutable` polls this until the address is routable + * before its one retry. + */ + isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; /** * Host-supplied cipher. Interchange decrypts bindings inside @@ -201,15 +211,19 @@ export async function launchWebhookTrigger( prepared.anchorRunId, ); try { - await deps.sessionService.sendUserMessage({ - agentAddress: prepared.deploymentAddress, - from: `webhook-trigger:${trigger.id}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content, - sessionId, - tenantId: trigger.tenantId, - cryptoProvider, + await deliverWhenRoutable({ + send: () => + deps.sessionService.sendUserMessage({ + agentAddress: prepared.deploymentAddress, + from: `webhook-trigger:${trigger.id}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content, + sessionId, + tenantId: trigger.tenantId, + cryptoProvider, + }), + isRoutable: () => deps.isRoutable(prepared.deploymentAddress), }); } catch (error) { reportError(error, { diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts new file mode 100644 index 0000000000..af6aa21c15 --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.ts @@ -0,0 +1,63 @@ +// CL-7476: since the process provisioner started spawning one sidecar per +// allocation, a freshly launched run's first mail can arrive before that +// sidecar has finished booting and registering with the hub. The send +// fails with "agent is unreachable" even though nothing is actually wrong +// — the address just isn't routable yet. Chat's send/fan-out path, its +// invite first-turn delivery, and the webhook trigger's ingress delivery +// each hit this same race independently; this is the one shared retry +// they all go through instead of three ad hoc copies. +const DEFAULT_DEADLINE_MS = 20_000; +const DEFAULT_POLL_INTERVAL_MS = 250; + +export function isAgentUnreachableError(err: unknown): boolean { + return err instanceof Error && err.message.includes("agent is unreachable"); +} + +export type DeliverWhenRoutableOptions = { + /** Performs the actual delivery. Called once, and again after the + * address becomes routable if the first attempt fails unreachable. */ + send: () => Promise; + /** Reads the hub's live routing table for this delivery's address. */ + isRoutable: () => boolean; + /** Defaults to matching Interchange's "agent is unreachable" message. */ + isUnreachable?: (err: unknown) => boolean; + /** How long to keep polling before giving up. Defaults to a budget a + * touch past the process provisioner's own boot-and-register time. */ + deadlineMs?: number; + pollIntervalMs?: number; + sleep?: (ms: number) => Promise; +}; + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Attempts `send`; if it fails with "agent is unreachable", polls + * `isRoutable` until the address is routable (or the deadline passes) + * and sends exactly once more. Any other failure, or an unreachable + * failure that never resolves before the deadline, is rethrown as-is — + * this never swallows the original error or retries indefinitely. + */ +export async function deliverWhenRoutable( + opts: DeliverWhenRoutableOptions, +): Promise { + const isUnreachable = opts.isUnreachable ?? isAgentUnreachableError; + try { + return await opts.send(); + } catch (err) { + if (!isUnreachable(err)) { + throw err; + } + const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS); + const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const sleep = opts.sleep ?? defaultSleep; + while (!opts.isRoutable()) { + if (Date.now() >= deadline) { + throw err; + } + await sleep(pollIntervalMs); + } + return await opts.send(); + } +} diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 9fefe188af..92c132180b 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,11 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + deliverWhenRoutable, + isAgentUnreachableError, + type DeliverWhenRoutableOptions, +} from "./deliver-when-routable"; export { listScheduledWorkflowDefinitions, scheduledDefinitionsFromRows, From a0c8a27ab25a0fa239060eb282e8b6e9b1ffbbbc Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:16 -0700 Subject: [PATCH 22/41] Log a skip, not an error, when root tenant seed has no operator key configured --- apps/hub/src/system-seed.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 1637f2e89d..f2f9df7c0c 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -35,6 +35,12 @@ const log = getLogger(["hub", "system-seed"]); const DEFAULT_DEADLINE_MS = 60_000; const DEFAULT_POLL_INTERVAL_MS = 2_000; +// Matches `packages/seeding/src/seed.ts`'s `resolveRealSourceOfferingIds` +// failure verbatim — the one deploy failure that is expected, not +// transient, when no operator seed key is configured. +const NO_CATALOG_OFFERINGS_REASON = + "this tenant has no catalog offerings to deploy against"; + // Matches `workbench seed`'s own default model source (readSeedConfig in // the now-deleted `packages/cli/src/config.ts`): real anthropic/ // claude-sonnet-5 when a hub-owned key is configured (`config.seedModel`), @@ -150,6 +156,18 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { } catch (cause) { const reason = cause instanceof Error ? cause.message : String(cause); if (Date.now() >= deadline) { + // No operator seed key means `seedCatalog` can only ever plant a + // placeholder credential — never a real, launchable offering — + // so a default workflow's deploy permanently has nothing to + // deploy against until an operator configures one. That is the + // expected shape of an unconfigured dev/CI boot, not a fault the + // next boot can retry its way out of, so it is a logged skip + // rather than an error (which `reportError` would otherwise + // paint red on every single boot). + if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; + return; + } reportError(cause, { operation: "system-seed.seedRootTenant" }); log.error`root tenant seed did not complete before its deadline (last error: ${reason}); the next boot will retry`; return; From 1fd89ddc9b40f5a2173f19eb58629baf66412f78 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:37:08 -0700 Subject: [PATCH 23/41] Run prettier --- apps/hub/src/system-seed.ts | 5 ++++- packages/chat/src/agent-binding.test.ts | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index f2f9df7c0c..039ad2b4c5 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -164,7 +164,10 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { // next boot can retry its way out of, so it is a logged skip // rather than an error (which `reportError` would otherwise // paint red on every single boot). - if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + if ( + deps.seedModel === undefined && + reason.includes(NO_CATALOG_OFFERINGS_REASON) + ) { log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; return; } diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf34..f11d61a60f 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 3f8fdc95da60a75a6ebbdf51aba59f34c62fea92 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:38:30 -0700 Subject: [PATCH 24/41] Chat: run prettier on the tenant-scoping test file --- packages/chat/src/agent-binding.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf34..f11d61a60f 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 46a4a9301e1b0504dcc1ca91a5a326a254c262a2 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:00 -0700 Subject: [PATCH 25/41] Add a DB-gated test for recording a run's agent_session (CL-7477) resolveRunSessionId (Interchange's mail-routing lookup) reads agent_session keyed by a run's principal, but nothing writes that table for a native launch since folded-runs was deleted -- a run's first turn silently fails to persist mail. Proves recordAgentSessionForRun writes a row resolveRunSessionId reads back, and updates the webhook launch fake DB to model workflow_run/agent_session the way the fix now touches them. --- packages/webhook-triggers/test/launch.test.ts | 18 +++ packages/workflows/test/agent-session.test.ts | 106 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 packages/workflows/test/agent-session.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index fa75590866..177a5090bc 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,7 +88,25 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, + // `recordAgentSessionForRun` reads the just-provisioned run back by + // id — production never inserts this row itself (Interchange's + // `prepareProvisionedDeployment` mints the anchor row + // transactionally before returning it), so this models the same + // row for `INTERCHANGE_RUN_ID`. + workflowRun: { + findFirst: async () => ({ + id: INTERCHANGE_RUN_ID, + tenantId: TENANT_ROW.id, + definitionId: DEFINITION_ROW.id, + principalId: "prn_interchange", + }), + }, }, + insert: (_table: unknown) => ({ + values: (_values: unknown) => ({ + onConflictDoNothing: async (_opts: unknown) => {}, + }), + }), }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts new file mode 100644 index 0000000000..a7446adda4 --- /dev/null +++ b/packages/workflows/test/agent-session.test.ts @@ -0,0 +1,106 @@ +// DB-gated: skipped when DATABASE_URL is unreachable, matching this +// repo's convention for tests that talk to a real Postgres (see +// `packages/access-tools/test/routes.test.ts`). +// +// CL-7477: before this fix, nothing wrote `agent_session` for a native +// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could +// never find a session for a run's principal and every launch's first +// turn failed to persist mail. This proves `recordAgentSessionForRun` +// writes a row `resolveRunSessionId` actually reads back. +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; + +import { createDB, schema, type DB } from "@intx/db"; +import { generateId } from "@intx/hub-common"; +import { resolveRunSessionId } from "@intx/hub-sessions"; +import { dbGate } from "../../../scripts/e2e/db-gate"; + +import { recordAgentSessionForRun } from "../src/launch/agent-session"; + +function dbConfigFromUrl(databaseUrl: string) { + const url = new URL(databaseUrl); + return { + host: url.hostname, + port: url.port === "" ? 5432 : Number(url.port), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + database: url.pathname.replace(/^\//, ""), + }; +} + +const databaseUrl = process.env["DATABASE_URL"]; +const describeIfDb = dbGate(databaseUrl, import.meta.path); + +describeIfDb("recordAgentSessionForRun", () => { + let db: DB; + + const tenantId = generateId("tenant"); + const principalId = generateId("principal"); + const definitionId = generateId("workflowDefinition"); + const anchorRunId = generateId("workflowRun"); + const sessionId = generateId("session"); + + beforeAll(async () => { + if (databaseUrl === undefined) return; + db = createDB(dbConfigFromUrl(databaseUrl)); + + await db.db.insert(schema.tenant).values({ + id: tenantId, + name: "Agent Session Test Tenant", + slug: `agent-session-${tenantId}`, + domain: `agent-session-${tenantId}.localhost`, + parentId: null, + config: null, + }); + + await db.db.insert(schema.principal).values({ + id: principalId, + tenantId, + kind: "workflow", + refId: anchorRunId, + status: "active", + }); + + await db.db.insert(schema.workflowDefinition).values({ + id: definitionId, + tenantId, + creatorPrincipalId: principalId, + assetId: null, + name: "agent-session-test-definition", + }); + + await db.db.insert(schema.workflowRun).values({ + id: anchorRunId, + definitionId, + anchorRunId, + tenantId, + principalId, + address: `${anchorRunId}@agent-session-${tenantId}.localhost`, + status: "running", + }); + }); + + afterAll(async () => { + if (databaseUrl === undefined) return; + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, anchorRunId)); + await db.db + .delete(schema.workflowDefinition) + .where(eq(schema.workflowDefinition.id, definitionId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, principalId)); + await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); + }); + + test("writes an agent_session resolveRunSessionId reads back", async () => { + await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + + const resolved = await resolveRunSessionId(db.db, principalId); + expect(resolved).toBe(sessionId); + }); +}); From 39ea012bd83690459520821c1e078c6de92c75fb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:15 -0700 Subject: [PATCH 26/41] Native launch records the run's agent_session so mail and spans persist (CL-7477) Before #632 deleted folded-runs, mintFoldedRun was the only writer of agent_session, the table resolveRunSessionId (Interchange's mail-routing lookup, vendor/intx/hub-sessions/src/hub-session-lookups.ts) reads by a run's principal to find its session. Every current native launch path mints its own sessionId and hands it to prepareProvisionedDeployment, but nothing ever wrote it into agent_session -- a run's first turn logged "Endpoint ... has no session for address" and silently dropped its mail. workflow_run.principal_id is still null the instant prepareProvisionedDeployment returns: a provisioned anchor is born "deployed" with no principal, and only the run's first trigger reconciles one onto it (Interchange's anchorWithPrincipal), through the exact same trigger path every sendUserMessage drives. So the write has to land after that first send, not right after provisioning. @corbits/workflows now exports recordAgentSessionForRun, a shared helper that reads a run's tenant/definition/principal fresh off workflow_run and inserts the session row (idempotent, since every send after the first is a no-op). Every call site that sends a message wires it in right after that send: webhook-triggers' launch, agent-directory's one-shot prompt, and chat's sendRunMail (the one chokepoint every chat send -- invite, wake, relaunch -- funnels through, since an invite alone provisions a run with no message and would only ever see a still-null principal). apps/hub's agent-authored-workflow deployer mints a run with no message of its own and its first trigger runs entirely through Interchange's native route with no Workbench hook to record afterward, so it's left alone. Session teardown reuses the same read: chat's relaunch marks the outgoing run's session ended before repointing the room, and the one-shot prompt's teardown ends its own session once its single turn settles. --- apps/hub/src/index.ts | 9 ++ .../agent-directory/src/one-shot-prompt.ts | 51 +++++++-- packages/chat/src/platform-adapter.ts | 10 ++ packages/chat/src/send-run-mail.ts | 21 ++++ packages/webhook-triggers/src/launch.ts | 21 ++++ packages/workflows/src/index.ts | 6 + .../workflows/src/launch/agent-session.ts | 106 ++++++++++++++++++ 7 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 packages/workflows/src/launch/agent-session.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bef0a6d6c2..c3352c5a35 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2017,6 +2017,15 @@ export async function createHub(config: HubConfig) { } try { + // Not `recordAgentSessionForRun` here: this mints a deployment + // with no message sent against it — a `workflow_run` born this + // way carries no principal until its first trigger (a call this + // deployer never makes) reconciles one onto it, so the call + // would only ever no-op. That trigger runs entirely through + // Interchange's own native route, with no Workbench-owned hook + // to record the session afterward (CL-7477 tracks the chat, + // webhook, and one-shot-prompt launch paths, which all send a + // message of their own and record there instead). const prepared = await workflowAllocationService.prepareProvisionedDeployment({ tenantId, diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index bf742b2d33..f3084ea665 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -8,8 +8,10 @@ import type { AgentLifecycle } from "@corbits/agent-lifecycle"; import { connectorReplyContent, messageRunEnded } from "@corbits/agent-events"; import { reportError } from "@corbits/error-sink"; import { + endAgentSessionForRun, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -162,6 +164,10 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null until its first trigger + // reconciles one onto it — `runOneShotPrompt` records the session + // right after the opening prompt actually sends. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -260,6 +266,14 @@ export async function runOneShotPrompt( settled = true; clearTimeout(timer); unsubscribe(); + try { + await endAgentSessionForRun(deps.db, launched.runId); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.end-session", + extra: { runId: launched.runId, reason }, + }); + } try { await deps.undeploy(launched.address, reason); } catch (err) { @@ -290,18 +304,33 @@ export async function runOneShotPrompt( content: input.prompt, cryptoProvider, }); - return; + } else { + await deps.sessionService.sendUserMessage({ + agentAddress: launched.address, + from: `${input.principalId}@${tenantRow.domain}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content: input.prompt, + sessionId: launched.sessionId, + tenantId: input.tenantId, + cryptoProvider, + }); + } + // The send above just drove the run's trigger path, the only + // thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now is + // what makes this ever find one. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.recordAgentSession", + extra: { address: launched.address }, + }); } - await deps.sessionService.sendUserMessage({ - agentAddress: launched.address, - from: `${input.principalId}@${tenantRow.domain}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content: input.prompt, - sessionId: launched.sessionId, - tenantId: input.tenantId, - cryptoProvider, - }); } catch (cause) { reportError(cause, { operation: "agent-directory.one-shot.send", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 39e52ff2c8..5a432c5d7e 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,6 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, + endAgentSessionForPrincipal, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -342,6 +343,11 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null (an invite sits + // un-triggered until someone actually writes into it), so this + // would only ever no-op. `sendRunMail` records it instead, once the + // run's first turn has actually reconciled a principal onto it. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -470,6 +476,10 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); + if (run.principalId !== null) { + await endAgentSessionForPrincipal(deps.db, run.principalId); + } + await repointBinding( deps.db, binding, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index 909fefa093..e79a51e8fb 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,5 +1,7 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. +import { recordAgentSessionForRun } from "@corbits/workflows"; +import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; @@ -100,5 +102,24 @@ export async function sendRunMail( const mailId = crypto.randomUUID(); const now = new Date(); const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); + // `sendUserMessage` just drove this run's trigger path, which is the + // only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — so the run is mail-routable + // now if it never was before. Best-effort and idempotent past the + // first successful call; a failure here must not swallow the mail + // that already sent. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: params.sessionId, + address: params.agentAddress, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendRunMail.recordAgentSession", + tenantId: params.tenantId, + agentId: params.agentAddress, + extra: { sessionId: params.sessionId }, + }); + } return recordRunMail(deps, params, mailId, now, rawMIME); } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6f3b3adfaa..6fda597742 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,6 +17,7 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -237,6 +238,26 @@ export async function launchWebhookTrigger( }); } + // The delivery attempt above is what drove the run's trigger path, + // the only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now, after + // that attempt, is what makes this call ever find one. Best-effort: + // a run that never became routable stays without a session until its + // first successful send. + try { + await recordAgentSessionForRun(deps.db, { + sessionId, + anchorRunId: prepared.anchorRunId, + }); + } catch (error) { + reportError(error, { + operation: "webhookTriggers.launch.recordAgentSession", + tenantId: trigger.tenantId, + agentId: prepared.deploymentAddress, + extra: { instanceId: prepared.anchorRunId }, + }); + } + return { instanceId: prepared.anchorRunId, triggerAddress: prepared.deploymentAddress, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 92c132180b..7eac44dbb2 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,12 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + recordAgentSessionForRun, + endAgentSessionForPrincipal, + endAgentSessionForRun, + type RecordAgentSessionParams, +} from "./launch/agent-session"; export { deliverWhenRoutable, isAgentUnreachableError, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts new file mode 100644 index 0000000000..f95c539608 --- /dev/null +++ b/packages/workflows/src/launch/agent-session.ts @@ -0,0 +1,106 @@ +// Every Workbench launch path provisions through Interchange's +// `prepareProvisionedDeployment` with a `sessionId` it mints itself, but +// nothing ever wrote that id into `agent_session` (vendor-owned, +// `vendor/intx/db/src/schema/sessions.ts`) — the table +// `resolveRunSessionId` (`vendor/intx/hub-sessions/src/hub-session-lookups.ts`) +// reads to route a run's outbound mail. Before the folded-runs package +// was deleted it was the only writer of that table; every current +// launcher must now do this itself, through this one shared helper, so +// mail and spans persist against a real session from the run's first +// turn (CL-7477). +// +// Timing matters: `workflow_run.principal_id` (the FK `agent_session` +// keys on) is still null the instant `prepareProvisionedDeployment` +// returns — a provisioned anchor is born "deployed" with no principal, +// and only the run's first trigger reconciles one onto it (Interchange's +// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`), +// through the exact same trigger path every `sendUserMessage` drives. So +// this cannot run right after provisioning (an invite sits un-triggered +// until someone actually writes into it) — call it after a send that +// just delivered the run's first turn instead. A run with no principal +// yet is a normal, expected state, not a bug: this returns `false` +// rather than throwing, and idempotent past its first successful call. +import { eq } from "drizzle-orm"; +import type { DB } from "@intx/db"; +import { agentSession, workflowRun } from "@intx/db/schema"; + +export type RecordAgentSessionParams = { + /** The id `prepareProvisionedDeployment` was called with. */ + readonly sessionId: string; +} & ({ readonly anchorRunId: string } | { readonly address: string }); + +/** + * Inserts the `agent_session` row a run needs to be mail-routable: the + * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps + * its old name but targets `workflow_definition` now), and `principalId` + * are all read fresh off the `workflow_run` row, never guessed by the + * caller. Returns `false` (and writes nothing) when the row has no + * principal yet, `true` once the session is recorded (or already was — + * a second call for the same `sessionId` is a no-op, not a conflict). + */ +export async function recordAgentSessionForRun( + db: DB["db"], + params: RecordAgentSessionParams, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: + "anchorRunId" in params + ? eq(workflowRun.id, params.anchorRunId) + : eq(workflowRun.address, params.address), + }); + if (runRow === undefined || runRow.principalId === null) { + return false; + } + + const now = new Date(); + await db + .insert(agentSession) + .values({ + id: params.sessionId, + tenantId: runRow.tenantId, + agentId: runRow.definitionId, + principalId: runRow.principalId, + status: "active", + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: agentSession.id }); + return true; +} + +/** + * Marks a run's `agent_session` ended — called only from the places that + * already know the run died: a chat relaunch (the old run's session) or + * a one-shot prompt's teardown. No sweeper; a run nobody reacts to stays + * `active` until something does. + */ +export async function endAgentSessionForPrincipal( + db: DB["db"], + principalId: string, +): Promise { + const endedAt = new Date(); + await db + .update(agentSession) + .set({ status: "ended", endedAt, updatedAt: endedAt }) + .where(eq(agentSession.principalId, principalId)); +} + +/** + * Same as `endAgentSessionForPrincipal`, keyed by the run's own id + * instead — for a caller (e.g. a one-shot prompt's teardown) that only + * ever held the run id, never minted or looked up its principal. + * No-ops for a run row already gone, or one never triggered (no + * principal, hence no session was ever recorded for it). + */ +export async function endAgentSessionForRun( + db: DB["db"], + anchorRunId: string, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: eq(workflowRun.id, anchorRunId), + }); + if (runRow === undefined || runRow.principalId === null) { + return; + } + await endAgentSessionForPrincipal(db, runRow.principalId); +} From f31e0e100206c1af55908abf698ad40e01302a6d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:01 -0700 Subject: [PATCH 27/41] Add tests for the run's event collector create/abandon wiring --- packages/webhook-triggers/test/launch.test.ts | 3 ++ packages/workflows/test/agent-session.test.ts | 53 ++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 177a5090bc..01b3b6785c 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -197,6 +197,9 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, + eventCollectors: { + create: () => undefined, + }, }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index a7446adda4..3ae50e221e 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -15,7 +15,10 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { recordAgentSessionForRun } from "../src/launch/agent-session"; +import { + endAgentSessionForRun, + recordAgentSessionForRun, +} from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -98,9 +101,55 @@ describeIfDb("recordAgentSessionForRun", () => { }); test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + await recordAgentSessionForRun( + db.db, + { sessionId, anchorRunId }, + { create: () => undefined }, + ); const resolved = await resolveRunSessionId(db.db, principalId); expect(resolved).toBe(sessionId); }); + + test("creates the run's event collector with its address/tenant/session/run id", async () => { + const createCalls: unknown[] = []; + const otherSessionId = generateId("session"); + + await recordAgentSessionForRun( + db.db, + { sessionId: otherSessionId, anchorRunId }, + { + create: (...args: unknown[]) => { + createCalls.push(args); + }, + }, + ); + + expect(createCalls).toEqual([ + [ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + tenantId, + otherSessionId, + anchorRunId, + ], + ]); + + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, otherSessionId)); + }); + + test("abandons the run's event collector when the session ends", async () => { + const abandonCalls: string[] = []; + + await endAgentSessionForRun(db.db, anchorRunId, { + abandon: (address: string) => { + abandonCalls.push(address); + }, + }); + + expect(abandonCalls).toEqual([ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + ]); + }); }); From 9260c3a8b06d1f9f227551bb88df0c6573ed7872 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:07 -0700 Subject: [PATCH 28/41] Native launch creates the run's event collector so turns are recorded (CL-7479) --- apps/hub/src/index.ts | 2 + .../agent-directory/src/one-shot-prompt.ts | 20 ++++++--- packages/chat/src/platform-adapter.ts | 7 ++- packages/chat/src/send-run-mail.ts | 18 +++++--- packages/webhook-triggers/src/launch.ts | 14 ++++-- packages/workflows/src/index.ts | 1 + .../workflows/src/launch/agent-session.ts | 45 +++++++++++++++++-- 7 files changed, 85 insertions(+), 22 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index c3352c5a35..1faa70f3a7 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2272,6 +2272,7 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, + eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, @@ -3069,6 +3070,7 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, sessionService, + eventCollectors, }, runnerInput, ), diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index f3084ea665..36efb6022e 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -13,6 +13,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -57,6 +58,7 @@ export type OneShotRunnerDeps = { "prepareProvisionedDeployment" >; readonly sessionService: Pick; + readonly eventCollectors: EventCollectorPort; /** * Test seam only. Production never sets these; they default to * Interchange `prepareProvisionedDeployment` and `sendUserMessage`. @@ -267,7 +269,11 @@ export async function runOneShotPrompt( clearTimeout(timer); unsubscribe(); try { - await endAgentSessionForRun(deps.db, launched.runId); + await endAgentSessionForRun( + deps.db, + launched.runId, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.end-session", @@ -321,10 +327,14 @@ export async function runOneShotPrompt( // provisioned `workflow_run` (CL-7477) — recording only now is // what makes this ever find one. try { - await recordAgentSessionForRun(deps.db, { - sessionId: launched.sessionId, - anchorRunId: launched.runId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.recordAgentSession", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 5a432c5d7e..8b820493d7 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,7 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, - endAgentSessionForPrincipal, + endAgentSessionForRun, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -258,6 +258,7 @@ export function createHubChatPlatform( db: deps.db, sessionService: deps.sessionService, sidecarRouter: deps.sidecarRouter, + eventCollectors: deps.eventCollectors, }; function offeringDigest(sourceOfferingIds: readonly string[]): string { @@ -476,9 +477,7 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); - if (run.principalId !== null) { - await endAgentSessionForPrincipal(deps.db, run.principalId); - } + await endAgentSessionForRun(deps.db, run.id, deps.eventCollectors); await repointBinding( deps.db, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index e79a51e8fb..b570a22d20 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,6 +1,9 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. -import { recordAgentSessionForRun } from "@corbits/workflows"; +import { + recordAgentSessionForRun, + type EventCollectorPort, +} from "@corbits/workflows"; import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; @@ -11,6 +14,7 @@ export type RunMailDeps = { db: DB["db"]; sessionService: Pick; sidecarRouter: Pick; + eventCollectors: Pick; }; export type SendRunMailParams = { @@ -109,10 +113,14 @@ export async function sendRunMail( // first successful call; a failure here must not swallow the mail // that already sent. try { - await recordAgentSessionForRun(deps.db, { - sessionId: params.sessionId, - address: params.agentAddress, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: params.sessionId, + address: params.agentAddress, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "chat.sendRunMail.recordAgentSession", diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6fda597742..22e686d40e 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -19,6 +19,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -58,6 +59,7 @@ export type LaunchWebhookTriggerDeps = { */ isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; + eventCollectors: Pick; /** * Host-supplied cipher. Interchange decrypts bindings inside * `prepareProvisionedDeployment`; this port still owns the field so @@ -245,10 +247,14 @@ export async function launchWebhookTrigger( // a run that never became routable stays without a session until its // first successful send. try { - await recordAgentSessionForRun(deps.db, { - sessionId, - anchorRunId: prepared.anchorRunId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId, + anchorRunId: prepared.anchorRunId, + }, + deps.eventCollectors, + ); } catch (error) { reportError(error, { operation: "webhookTriggers.launch.recordAgentSession", diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 7eac44dbb2..244d0478ab 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -60,6 +60,7 @@ export { endAgentSessionForPrincipal, endAgentSessionForRun, type RecordAgentSessionParams, + type EventCollectorPort, } from "./launch/agent-session"; export { deliverWhenRoutable, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index f95c539608..58ec8fd409 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -23,12 +23,27 @@ import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { agentSession, workflowRun } from "@intx/db/schema"; +import type { EventCollectorRegistry } from "@intx/hub-sessions"; export type RecordAgentSessionParams = { /** The id `prepareProvisionedDeployment` was called with. */ readonly sessionId: string; } & ({ readonly anchorRunId: string } | { readonly address: string }); +/** + * The one port every launcher threads the same wrapped + * `EventCollectorRegistry` through (`apps/hub/src/index.ts`'s + * `eventCollectors`) — never a second registry construction. `create` + * is what actually makes `inference_turn`/`turn_part` rows exist for a + * run (CL-7479: nothing else ever called it once `folded-runs` was + * deleted); `abandon` tears the collector down when a run's session + * ends. + */ +export type EventCollectorPort = Pick< + EventCollectorRegistry, + "create" | "abandon" +>; + /** * Inserts the `agent_session` row a run needs to be mail-routable: the * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps @@ -41,6 +56,7 @@ export type RecordAgentSessionParams = { export async function recordAgentSessionForRun( db: DB["db"], params: RecordAgentSessionParams, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: @@ -65,6 +81,19 @@ export async function recordAgentSessionForRun( updatedAt: now, }) .onConflictDoNothing({ target: agentSession.id }); + + const agentAddress = "address" in params ? params.address : runRow.address; + if (agentAddress !== null) { + // Idempotent across wakes: a collector already live for this + // address is replaced, matching the registry's own contract + // (`event-collector-registry.ts`'s `create`). + eventCollectors.create( + agentAddress, + runRow.tenantId, + params.sessionId, + runRow.id, + ); + } return true; } @@ -87,14 +116,19 @@ export async function endAgentSessionForPrincipal( /** * Same as `endAgentSessionForPrincipal`, keyed by the run's own id - * instead — for a caller (e.g. a one-shot prompt's teardown) that only - * ever held the run id, never minted or looked up its principal. - * No-ops for a run row already gone, or one never triggered (no - * principal, hence no session was ever recorded for it). + * instead — for a caller (e.g. a one-shot prompt's teardown, or a chat + * relaunch replacing a terminal run) that only ever held the run id, + * never minted or looked up its principal. Also abandons the outgoing + * run's event collector, through the same shared + * `EventCollectorPort` every launcher threads — one mechanism for + * ending a run's session. No-ops for a run row already gone, or one + * never triggered (no principal, hence no session was ever recorded + * for it). */ export async function endAgentSessionForRun( db: DB["db"], anchorRunId: string, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, anchorRunId), @@ -103,4 +137,7 @@ export async function endAgentSessionForRun( return; } await endAgentSessionForPrincipal(db, runRow.principalId); + if (runRow.address !== null) { + eventCollectors.abandon(runRow.address); + } } From f6369d9a0ed09bb08be2ecb7d27d8ee8d96b518e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:42:35 -0700 Subject: [PATCH 29/41] Add tests for lazy, idempotent run-session ensure (CL-7480) --- packages/workflows/test/agent-session.test.ts | 142 +++++++++++------- 1 file changed, 86 insertions(+), 56 deletions(-) diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index 3ae50e221e..de783406f3 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,11 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477: before this fix, nothing wrote `agent_session` for a native -// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could -// never find a session for a run's principal and every launch's first -// turn failed to persist mail. This proves `recordAgentSessionForRun` -// writes a row `resolveRunSessionId` actually reads back. +// CL-7477/CL-7480: nothing writes `agent_session` for a native launch +// until something calls `ensureRunSession` — this proves it writes a +// row `resolveRunSessionId` (vendor's mail-routing lookup) actually +// reads back, only once the run is anchored with a principal, and stays +// idempotent (one insert, one collector) across repeated calls. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -15,10 +15,7 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { - endAgentSessionForRun, - recordAgentSessionForRun, -} from "../src/launch/agent-session"; +import { ensureRunSession } from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -34,14 +31,16 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("recordAgentSessionForRun", () => { +describeIfDb("ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); const principalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchorRunId = generateId("workflowRun"); + const anchoredRunId = generateId("workflowRun"); + const unanchoredRunId = generateId("workflowRun"); const sessionId = generateId("session"); + const domain = `agent-session-${tenantId}.localhost`; beforeAll(async () => { if (databaseUrl === undefined) return; @@ -51,7 +50,7 @@ describeIfDb("recordAgentSessionForRun", () => { id: tenantId, name: "Agent Session Test Tenant", slug: `agent-session-${tenantId}`, - domain: `agent-session-${tenantId}.localhost`, + domain, parentId: null, config: null, }); @@ -60,7 +59,7 @@ describeIfDb("recordAgentSessionForRun", () => { id: principalId, tenantId, kind: "workflow", - refId: anchorRunId, + refId: anchoredRunId, status: "active", }); @@ -72,13 +71,38 @@ describeIfDb("recordAgentSessionForRun", () => { name: "agent-session-test-definition", }); + // Anchored: has a principal, so `ensureRunSession` can insert its + // `agent_session` row. await db.db.insert(schema.workflowRun).values({ - id: anchorRunId, + id: anchoredRunId, definitionId, - anchorRunId, + anchorRunId: anchoredRunId, tenantId, principalId, - address: `${anchorRunId}@agent-session-${tenantId}.localhost`, + address: `${anchoredRunId}@${domain}`, + status: "running", + }); + await db.db.insert(schema.workflowRunLaunchSpec).values({ + anchorRunId: anchoredRunId, + sessionId, + deploymentDomain: domain, + sourceAuthorityPrincipalId: principalId, + frozenApprovalBundle: {}, + sourceOfferingIds: [], + defaultSourceOfferingId: "off_test", + deployContent: { systemPrompt: "" }, + }); + + // Unanchored: `prepareProvisionedDeployment` minted the row, but no + // trigger has reconciled a principal onto it yet — the normal state + // for a freshly provisioned invite. + await db.db.insert(schema.workflowRun).values({ + id: unanchoredRunId, + definitionId, + anchorRunId: unanchoredRunId, + tenantId, + principalId: null, + address: `${unanchoredRunId}@${domain}`, status: "running", }); }); @@ -88,9 +112,15 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db .delete(schema.agentSession) .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRunLaunchSpec) + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, anchoredRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchorRunId)); + .where(eq(schema.workflowRun.id, anchoredRunId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, unanchoredRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); @@ -100,56 +130,56 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun( - db.db, - { sessionId, anchorRunId }, - { create: () => undefined }, - ); - - const resolved = await resolveRunSessionId(db.db, principalId); - expect(resolved).toBe(sessionId); - }); - - test("creates the run's event collector with its address/tenant/session/run id", async () => { + test("an unanchored run returns null and writes nothing", async () => { const createCalls: unknown[] = []; - const otherSessionId = generateId("session"); - - await recordAgentSessionForRun( - db.db, - { sessionId: otherSessionId, anchorRunId }, - { + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: (...args: unknown[]) => { createCalls.push(args); }, + has: () => false, }, - ); - - expect(createCalls).toEqual([ - [ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - tenantId, - otherSessionId, - anchorRunId, - ], - ]); + runId: unanchoredRunId, + }); - await db.db - .delete(schema.agentSession) - .where(eq(schema.agentSession.id, otherSessionId)); + expect(resolved).toBeNull(); + expect(createCalls).toEqual([]); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.agentId, definitionId), + }); + expect(sessionRow).toBeUndefined(); }); - test("abandons the run's event collector when the session ends", async () => { - const abandonCalls: string[] = []; - - await endAgentSessionForRun(db.db, anchorRunId, { - abandon: (address: string) => { - abandonCalls.push(address); + test("an anchored run inserts its session and creates its collector once across two calls", async () => { + const createCalls: unknown[] = []; + let hasCollector = false; + const eventCollectors = { + create: (...args: unknown[]) => { + createCalls.push(args); + hasCollector = true; }, + has: () => hasCollector, + }; + + const first = await ensureRunSession({ + db: db.db, + eventCollectors, + runId: anchoredRunId, + }); + const second = await ensureRunSession({ + db: db.db, + eventCollectors, + runId: anchoredRunId, }); - expect(abandonCalls).toEqual([ - `${anchorRunId}@agent-session-${tenantId}.localhost`, + expect(first).toBe(sessionId); + expect(second).toBe(sessionId); + expect(createCalls).toEqual([ + [`${anchoredRunId}@${domain}`, tenantId, sessionId, anchoredRunId], ]); + + const resolved = await resolveRunSessionId(db.db, principalId); + expect(resolved).toBe(sessionId); }); }); From d6eda397f142b9fb94403173a1cee8d69f6fa3e8 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:42:41 -0700 Subject: [PATCH 30/41] Hub ensures a run's session and collector on first use (CL-7480) Replaces the post-send recordAgentSessionForRun (CL-7477/CL-7479) with one idempotent ensureRunSession, called lazily at the hub seams that actually see a run become mail-routable: the mailbox persist wrapper and the event-collector dispatch path, plus chat's pre-send mail dispatch. Pre-creating the run's principal to dodge the timing isn't an option -- Interchange's grant materialization treats a conflict on that insert as grants already committed and throws. --- apps/hub/src/index.ts | 60 +++++++++++- apps/hub/src/mailbox-persist.ts | 45 +++++++++ .../agent-directory/src/one-shot-prompt.ts | 31 ++---- packages/chat/src/platform-adapter.ts | 28 +++++- packages/chat/src/send-run-mail.ts | 34 +------ packages/webhook-triggers/src/launch.ts | 31 +----- packages/webhook-triggers/test/launch.test.ts | 21 ---- packages/workflows/src/index.ts | 3 +- .../workflows/src/launch/agent-session.ts | 97 ++++++++++--------- 9 files changed, 198 insertions(+), 152 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 1faa70f3a7..bd04767ec3 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -168,6 +168,7 @@ import { import { createHubMailboxAuthorizeSender, createHubMailboxResolveRefs, + createHubPersistMailWithSessionEnsure, } from "./mailbox-persist"; import { createCommandRegistry, @@ -196,6 +197,7 @@ import { createScheduledWorkflowRoutes, renderWorkflowSourceTree, WORKFLOW_SOURCE_ENTRY, + ensureRunSession, } from "@corbits/workflows"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -219,6 +221,7 @@ import { createWorkflowAllocationService, createWorkflowDispatchService, DEFAULT_ASSET_REF, + resolveRoutableAddress, WorkflowProvisioningError, type AgentRepoStore, type EventCollectorRegistry, @@ -713,6 +716,15 @@ export async function createHub(config: HubConfig) { args: Parameters[0], ) => Promise; } = {}; + // Forward reference: `eventCollectors` (the wrapped + // `EventCollectorRegistry`) isn't constructed until later in this + // composition, but `createHubPersistMailWithSessionEnsure` below needs + // it. Set once `eventCollectors` exists; every persistMail call before + // that point (there are none — the server isn't serving requests yet) + // would just no-op. + const eventCollectorsRef: { + current?: Pick; + } = {}; // CL-6499 (native multi-step routines): materializes a mail-triggered // run's authorization grants from its deploy-approved snapshot, so // ANY plain mail delivered to a workflow deployment's address — not @@ -746,7 +758,11 @@ export async function createHub(config: HubConfig) { // workbench ref is present before the post-commit bus event fires -- // no out-of-band UPDATE, no polling read. persistMail: createMailboxPersist(mailboxDb, { - upstream: baseLookups.persistMail, + upstream: createHubPersistMailWithSessionEnsure( + db, + eventCollectorsRef, + baseLookups.persistMail, + ), authorizeSender: createHubMailboxAuthorizeSender(db), bus: mailboxBus, resolveRefs: createHubMailboxResolveRefs(chatStore, roomMessages), @@ -885,6 +901,17 @@ export async function createHub(config: HubConfig) { baseEventCollectors.create(agentAddress, tenantId, sessionId, runId); }, dispatch(agentAddress, event) { + // CL-7480: a run's turn events can start arriving before anything + // else ever recorded its session — the first inbound trigger that + // just reconciled its principal races this same dispatch. No live + // collector for the address is exactly that case: ensure the + // session (and, inside it, the collector) before delegating, + // rather than silently dropping the event for a collector that + // never gets created. + if (!baseEventCollectors.has(agentAddress)) { + void ensureCollectorThenDispatch(agentAddress, event); + return; + } turnLatency.onEvent(agentAddress, event); baseEventCollectors.dispatch(agentAddress, event); // Mirrors the registry's own `isTerminal` check (event-collector-registry.ts) @@ -901,6 +928,36 @@ export async function createHub(config: HubConfig) { baseEventCollectors.abandon(agentAddress); }, }; + eventCollectorsRef.current = eventCollectors; + // Named so `dispatch`'s no-collector branch above reads clearly; not + // inlined there because it needs to be `async` and `dispatch` itself + // must stay synchronous (the `EventCollectorRegistry` contract). + async function ensureCollectorThenDispatch( + agentAddress: string, + event: Parameters[1], + ): Promise { + try { + const run = await resolveRoutableAddress(db, agentAddress); + if (run !== undefined) { + await ensureRunSession({ + db, + eventCollectors: baseEventCollectors, + runId: run.id, + }); + } + } catch (err) { + reportError(err, { + operation: "hub.eventCollectors.ensureRunSession", + extra: { agentAddress }, + }); + } + turnLatency.onEvent(agentAddress, event); + baseEventCollectors.dispatch(agentAddress, event); + const isTerminal = + event.type === "reactor.done" || + (event.type === "reactor.error" && event.data.fatal); + if (isTerminal) turnLatency.onSessionEnd(agentAddress); + } createHubSessionOrchestrator({ events: sidecarRouter.events, router: sidecarRouter, @@ -2272,7 +2329,6 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, - eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index 827dad17e1..f8aa0e2f01 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -16,6 +16,7 @@ import { tenant as tenantTable } from "@intx/db/schema"; import type { AuthorizeMailboxSender, CreateMailboxPersistOpts, + MailboxPersistArgs, MailboxRef, } from "@corbits/mailbox"; import { resolveRoutableAddress } from "@intx/hub-sessions"; @@ -24,6 +25,8 @@ import { type ChatStore, type RoomMessageStore, } from "@corbits/chat"; +import { ensureRunSession, type EventCollectorPort } from "@corbits/workflows"; +import { reportError } from "@corbits/error-sink"; /** * `@corbits/mailbox` does not export `ResolveMailboxRefs` itself (only the @@ -57,6 +60,48 @@ type ResolveMailboxRefs = NonNullable< * fact from two directions: one extra indexed lookup by address per frame, * not per recipient. */ +/** + * Wraps the hub's own `persistMail` (vendor's `baseLookups.persistMail`) + * so a run's `agent_session` and event collector exist before that write + * runs — CL-7480: the first inbound mail on a freshly triggered run is + * often the earliest point the run is mail-routable at all, since + * `workflow_run.principal_id` only reconciles onto the trigger that just + * fired. Resolves the frame's own sender address to its run + * (`resolveRoutableAddress`, the same resolver `persistMail` itself + * uses) rather than assuming the caller already knows it; a sender that + * is not a live run resolves to `undefined` and this is a no-op past + * that point. Best-effort: `ensureRunSession` failing must not swallow + * the mail upstream is about to persist regardless, so this only + * degrades to "no session yet" rather than ever throwing past itself. + * + * `eventCollectorsRef` is a forward reference (the same pattern this + * file's other composition-order refs use, e.g. `apps/hub/src/index.ts`'s + * `grantAllowanceGateRef`): the wrapped `eventCollectors` registry isn't + * constructed until after `lookups` is, so this only reads `.current` + * inside the returned closure, never at wiring time. + */ +export function createHubPersistMailWithSessionEnsure( + db: DB["db"], + eventCollectorsRef: { current?: Pick }, + upstream: (args: MailboxPersistArgs) => Promise, +): (args: MailboxPersistArgs) => Promise { + return async (args) => { + try { + const eventCollectors = eventCollectorsRef.current; + const sender = await resolveRoutableAddress(db, args.senderAddress); + if (sender !== undefined && eventCollectors !== undefined) { + await ensureRunSession({ db, eventCollectors, runId: sender.id }); + } + } catch (err) { + reportError(err, { + operation: "hub.mailboxPersist.ensureRunSession", + extra: { senderAddress: args.senderAddress }, + }); + } + return upstream(args); + }; +} + export function createHubMailboxAuthorizeSender( db: DB["db"], ): AuthorizeMailboxSender { diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index 36efb6022e..1851527d4f 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -11,7 +11,6 @@ import { endAgentSessionForRun, readDefinitionProjection, readFoldedBody, - recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, type EventCollectorPort, } from "@corbits/workflows"; @@ -166,10 +165,10 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // Not `ensureRunSession` here: a freshly provisioned run's // `workflow_run.principal_id` is still null until its first trigger - // reconciles one onto it — `runOneShotPrompt` records the session - // right after the opening prompt actually sends. + // reconciles one onto it, and the hub's own mail/dispatch seams + // ensure the session lazily once that happens (CL-7480). return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -322,25 +321,11 @@ export async function runOneShotPrompt( cryptoProvider, }); } - // The send above just drove the run's trigger path, the only - // thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — recording only now is - // what makes this ever find one. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId: launched.sessionId, - anchorRunId: launched.runId, - }, - deps.eventCollectors, - ); - } catch (err) { - reportError(err, { - operation: "agent-directory.one-shot.recordAgentSession", - extra: { address: launched.address }, - }); - } + // A run's session and event collector are ensured lazily now, at + // the hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped + // `eventCollectors` dispatch in `apps/hub/src/index.ts`) — + // CL-7480. Nothing here needs to record it. } catch (cause) { reportError(cause, { operation: "agent-directory.one-shot.send", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 8b820493d7..a546564e75 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -17,6 +17,7 @@ import { authoredDefinitionCandidates, DefinitionProjectionMissingError, endAgentSessionForRun, + ensureRunSession, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -258,7 +259,6 @@ export function createHubChatPlatform( db: deps.db, sessionService: deps.sessionService, sidecarRouter: deps.sidecarRouter, - eventCollectors: deps.eventCollectors, }; function offeringDigest(sourceOfferingIds: readonly string[]): string { @@ -344,11 +344,12 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // Not `ensureRunSession` here: a freshly provisioned run's // `workflow_run.principal_id` is still null (an invite sits // un-triggered until someone actually writes into it), so this - // would only ever no-op. `sendRunMail` records it instead, once the - // run's first turn has actually reconciled a principal onto it. + // would only ever no-op. The hub's own mail/dispatch seams ensure + // the session lazily instead, once the run's first turn has + // actually reconciled a principal onto it (CL-7480). return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -1313,6 +1314,25 @@ export function createHubChatPlatform( // idle sweep the moment they see traffic. lifecycle?.track(deliveryAddress); + // CL-7480: a run just woken/reconciled above can be routable + // before anything ever recorded its session — waking it is not + // itself a mail or dispatch seam, so nothing else along this path + // would ensure one. Best-effort: an ensure failure here still + // falls through to `resolveRunSessionIdOrThrow`'s own error. + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: delivery.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: deliveryAddress, + extra: { runId: delivery.run.id }, + }); + } const sessionId = await resolveRunSessionIdOrThrow(deps.db, delivery.run); const domain = domainOf(deliveryAddress); // `fromWorkbenchId` names the room a dispatch speaks for. A room diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index b570a22d20..139dfd98a4 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,10 +1,5 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. -import { - recordAgentSessionForRun, - type EventCollectorPort, -} from "@corbits/workflows"; -import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; @@ -14,7 +9,6 @@ export type RunMailDeps = { db: DB["db"]; sessionService: Pick; sidecarRouter: Pick; - eventCollectors: Pick; }; export type SendRunMailParams = { @@ -106,28 +100,10 @@ export async function sendRunMail( const mailId = crypto.randomUUID(); const now = new Date(); const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); - // `sendUserMessage` just drove this run's trigger path, which is the - // only thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — so the run is mail-routable - // now if it never was before. Best-effort and idempotent past the - // first successful call; a failure here must not swallow the mail - // that already sent. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId: params.sessionId, - address: params.agentAddress, - }, - deps.eventCollectors, - ); - } catch (err) { - reportError(err, { - operation: "chat.sendRunMail.recordAgentSession", - tenantId: params.tenantId, - agentId: params.agentAddress, - extra: { sessionId: params.sessionId }, - }); - } + // A run's session and event collector are ensured lazily now, at the + // hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` + // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs + // to record it. return recordRunMail(deps, params, mailId, now, rawMIME); } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 22e686d40e..4a0331aca1 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,9 +17,7 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, - recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, - type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -59,7 +57,6 @@ export type LaunchWebhookTriggerDeps = { */ isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; - eventCollectors: Pick; /** * Host-supplied cipher. Interchange decrypts bindings inside * `prepareProvisionedDeployment`; this port still owns the field so @@ -240,29 +237,11 @@ export async function launchWebhookTrigger( }); } - // The delivery attempt above is what drove the run's trigger path, - // the only thing that ever reconciles a principal onto a freshly - // provisioned `workflow_run` (CL-7477) — recording only now, after - // that attempt, is what makes this call ever find one. Best-effort: - // a run that never became routable stays without a session until its - // first successful send. - try { - await recordAgentSessionForRun( - deps.db, - { - sessionId, - anchorRunId: prepared.anchorRunId, - }, - deps.eventCollectors, - ); - } catch (error) { - reportError(error, { - operation: "webhookTriggers.launch.recordAgentSession", - tenantId: trigger.tenantId, - agentId: prepared.deploymentAddress, - extra: { instanceId: prepared.anchorRunId }, - }); - } + // A run's session and event collector are ensured lazily now, at the + // hub seams that actually see the run become mail-routable + // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` + // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs + // to record it. return { instanceId: prepared.anchorRunId, diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 01b3b6785c..fa75590866 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,25 +88,7 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, - // `recordAgentSessionForRun` reads the just-provisioned run back by - // id — production never inserts this row itself (Interchange's - // `prepareProvisionedDeployment` mints the anchor row - // transactionally before returning it), so this models the same - // row for `INTERCHANGE_RUN_ID`. - workflowRun: { - findFirst: async () => ({ - id: INTERCHANGE_RUN_ID, - tenantId: TENANT_ROW.id, - definitionId: DEFINITION_ROW.id, - principalId: "prn_interchange", - }), - }, }, - insert: (_table: unknown) => ({ - values: (_values: unknown) => ({ - onConflictDoNothing: async (_opts: unknown) => {}, - }), - }), }; } @@ -197,9 +179,6 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, - eventCollectors: { - create: () => undefined, - }, }; } diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 244d0478ab..74828b7b2a 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -56,10 +56,9 @@ export { } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; export { - recordAgentSessionForRun, + ensureRunSession, endAgentSessionForPrincipal, endAgentSessionForRun, - type RecordAgentSessionParams, type EventCollectorPort, } from "./launch/agent-session"; export { diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index 58ec8fd409..b6ba1100a1 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -13,23 +13,29 @@ // keys on) is still null the instant `prepareProvisionedDeployment` // returns — a provisioned anchor is born "deployed" with no principal, // and only the run's first trigger reconciles one onto it (Interchange's -// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`), -// through the exact same trigger path every `sendUserMessage` drives. So -// this cannot run right after provisioning (an invite sits un-triggered -// until someone actually writes into it) — call it after a send that -// just delivered the run's first turn instead. A run with no principal -// yet is a normal, expected state, not a bug: this returns `false` -// rather than throwing, and idempotent past its first successful call. +// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`). +// Pre-creating the run's principal to dodge this is not an option either: +// Interchange's own grant materialization +// (`vendor/intx/hub-api/src/run-grant-materialization.ts`) inserts the +// principal row `onConflictDoNothing` and treats a conflict as "grants +// already committed", throwing rather than re-materializing — so nothing +// upstream of that first trigger may write the principal first. +// +// So `ensureRunSession` (CL-7480) is lazy and idempotent rather than +// called once right after a send: every seam that might be the run's +// first mail-routable moment (a hub mailbox persist, a session +// orchestrator dispatch with no live collector yet) calls it before +// doing its own work. A run with no principal yet is a normal, expected +// state, not a bug — this returns `null` rather than throwing. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { agentSession, workflowRun } from "@intx/db/schema"; +import { + agentSession, + workflowRun, + workflowRunLaunchSpec, +} from "@intx/db/schema"; import type { EventCollectorRegistry } from "@intx/hub-sessions"; -export type RecordAgentSessionParams = { - /** The id `prepareProvisionedDeployment` was called with. */ - readonly sessionId: string; -} & ({ readonly anchorRunId: string } | { readonly address: string }); - /** * The one port every launcher threads the same wrapped * `EventCollectorRegistry` through (`apps/hub/src/index.ts`'s @@ -37,42 +43,52 @@ export type RecordAgentSessionParams = { * is what actually makes `inference_turn`/`turn_part` rows exist for a * run (CL-7479: nothing else ever called it once `folded-runs` was * deleted); `abandon` tears the collector down when a run's session - * ends. + * ends; `has` lets a caller check whether a collector already exists + * before creating a second one. */ export type EventCollectorPort = Pick< EventCollectorRegistry, - "create" | "abandon" + "create" | "abandon" | "has" >; /** - * Inserts the `agent_session` row a run needs to be mail-routable: the - * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps - * its old name but targets `workflow_definition` now), and `principalId` - * are all read fresh off the `workflow_run` row, never guessed by the - * caller. Returns `false` (and writes nothing) when the row has no - * principal yet, `true` once the session is recorded (or already was — - * a second call for the same `sessionId` is a no-op, not a conflict). + * Idempotently makes a run's session and event collector exist, the + * instant that becomes possible: the run's `tenantId`, `definitionId` + * (the `agent_id` FK), and `principalId` are all read fresh off the + * `workflow_run` row, never guessed by the caller. Returns `null` (and + * writes nothing) when the row has no principal yet — an un-anchored + * run is not a bug, just not ensurable yet. Returns the run's + * `sessionId` once the session exists (freshly inserted or already + * there — a second call for the same run is a no-op, not a conflict), + * with the run's event collector created too if none was live for its + * address. */ -export async function recordAgentSessionForRun( - db: DB["db"], - params: RecordAgentSessionParams, - eventCollectors: Pick, -): Promise { +export async function ensureRunSession(params: { + readonly db: DB["db"]; + readonly eventCollectors: Pick; + readonly runId: string; +}): Promise { + const { db, eventCollectors, runId } = params; const runRow = await db.query.workflowRun.findFirst({ - where: - "anchorRunId" in params - ? eq(workflowRun.id, params.anchorRunId) - : eq(workflowRun.address, params.address), + where: eq(workflowRun.id, runId), }); if (runRow === undefined || runRow.principalId === null) { - return false; + return null; + } + + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, runId), + }); + if (launchSpecRow === undefined) { + return null; } + const sessionId = launchSpecRow.sessionId; const now = new Date(); await db .insert(agentSession) .values({ - id: params.sessionId, + id: sessionId, tenantId: runRow.tenantId, agentId: runRow.definitionId, principalId: runRow.principalId, @@ -82,19 +98,10 @@ export async function recordAgentSessionForRun( }) .onConflictDoNothing({ target: agentSession.id }); - const agentAddress = "address" in params ? params.address : runRow.address; - if (agentAddress !== null) { - // Idempotent across wakes: a collector already live for this - // address is replaced, matching the registry's own contract - // (`event-collector-registry.ts`'s `create`). - eventCollectors.create( - agentAddress, - runRow.tenantId, - params.sessionId, - runRow.id, - ); + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { + eventCollectors.create(runRow.address, runRow.tenantId, sessionId, runRow.id); } - return true; + return sessionId; } /** From 63b2d2e9949557d7c48d1463cc1baf54841c4c19 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:53:48 -0700 Subject: [PATCH 31/41] Fix prettier formatting in agent-session.ts (CL-7480) --- packages/workflows/src/launch/agent-session.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index b6ba1100a1..0e2cbff99f 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -99,7 +99,12 @@ export async function ensureRunSession(params: { .onConflictDoNothing({ target: agentSession.id }); if (runRow.address !== null && !eventCollectors.has(runRow.address)) { - eventCollectors.create(runRow.address, runRow.tenantId, sessionId, runRow.id); + eventCollectors.create( + runRow.address, + runRow.tenantId, + sessionId, + runRow.id, + ); } return sessionId; } From af790b2ece08c3bcd216c4385c7511e4cb12a850 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:27:06 -0700 Subject: [PATCH 32/41] Add tests for eager session recording and launch-spec session lookup --- packages/chat/test/platform-adapter.test.ts | 134 ++++++++++++- packages/webhook-triggers/test/launch.test.ts | 27 +++ packages/workflows/test/agent-session.test.ts | 178 +++++++++++------- 3 files changed, 269 insertions(+), 70 deletions(-) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 29281d3308..9f3b5656aa 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -34,6 +34,7 @@ import { workflowDefinition, workflowDefinitionVersion, workflowRun, + workflowRunLaunchSpec, } from "@intx/db/schema"; import { SessionLaunchError } from "@intx/hub-sessions"; import type { EventCollectorRegistry, SidecarRouter } from "@intx/hub-sessions"; @@ -343,6 +344,19 @@ function createFakeDb(opts: { * production code asks for, in any order. */ wireProjectionsByDefinitionId?: Record | undefined; + /** + * The `workflow_run_launch_spec` row `resolveRunSessionIdOrThrow` + * (CL-7481) reads by run id — the session id is fixed at provision + * time, unlike the principal it used to be looked up by. Explicit + * only where a test cares about an un-anchored run's session + * resolving before its principal ever exists; every other test here + * predates CL-7481 and seeds an `agentSession` row directly, so the + * fallback below derives one from that instead of touching every + * call site. + */ + workflowRunLaunchSpecRow?: + | { anchorRunId: string; sessionId: string } + | undefined; }) { const inserted: { table: unknown; values: unknown }[] = []; const updated: { table: unknown; values: unknown }[] = []; @@ -427,13 +441,64 @@ function createFakeDb(opts: { status: "pending", }; } - return inserted.findLast((row) => row.table === workflowRun) - ?.values as typeof opts.workflowRunRow | undefined; + const insertedRow = inserted.findLast( + (row) => row.table === workflowRun, + )?.values as typeof opts.workflowRunRow | undefined; + if (insertedRow !== undefined) return insertedRow; + // A run this fixture never configured: the freshly minted + // `anchorRunId` a fake `prepareProvisionedDeployment` just + // returned, which never wrote a `workflowRun` row into this + // fake db (Interchange's own provisioning is entirely mocked + // out here). `recordAgentSessionAtProvision` (CL-7481) reads + // the run row it just "provisioned" immediately after — this + // synthesizes the generic shape production code would see, + // so that read succeeds without every test wiring one up. + if (id === undefined) return undefined; + const domain = + opts.tenantRow?.domain ?? + opts.workflowRunRow?.address?.split("@")[1] ?? + "ten1.workbench.test"; + return { + id, + tenantId: opts.tenantRow?.id ?? "ten_1", + definitionId: opts.definitionId, + address: `${id}@${domain}`, + principalId: null, + }; }, }, sessionMail: { findFirst: async () => opts.sessionMailRow, }, + agentSession: { + findFirst: async (query?: { where?: unknown }) => { + const [id] = + query?.where !== undefined ? boundStringValues(query.where) : []; + return inserted.findLast( + (row) => + row.table === agentSession && + (id === undefined || + (row.values as { id?: string }).id === id), + )?.values as { id: string; principalId: string } | undefined; + }, + }, + workflowRunLaunchSpec: { + findFirst: async (query?: { where?: unknown }) => { + const [id] = + query?.where !== undefined ? boundStringValues(query.where) : []; + if (opts.workflowRunLaunchSpecRow !== undefined) { + return id === undefined || + id === opts.workflowRunLaunchSpecRow.anchorRunId + ? opts.workflowRunLaunchSpecRow + : undefined; + } + const seededSession = inserted.find( + (row) => row.table === agentSession, + )?.values as { id: string } | undefined; + if (seededSession === undefined) return undefined; + return { anchorRunId: id, sessionId: seededSession.id }; + }, + }, workflowDefinition: { findFirst: async () => opts.workflowDefinitionRow ?? @@ -1020,6 +1085,71 @@ describe("createHubChatPlatform", () => { ); }); + test("sendMail on a freshly provisioned, un-anchored run resolves the launch-spec session id (CL-7481)", async () => { + resolveDefinitionSourcesResult = { + ok: true, + materials: [], + sources: [ + { + id: "off_1", + provider: "anthropic", + baseURL: "https://inference.invalid", + credentialId: "cred_placeholder", + model: "claude-sonnet-5", + }, + ], + defaultSource: "off_1", + }; + + // No principal on the run yet — an invite sits un-triggered until + // its first message. Before CL-7481, `resolveRunSessionIdOrThrow` + // looked the session up by this (null) principal and found nothing; + // now it reads the launch spec `recordAgentSessionAtProvision` + // wrote at provision time, keyed by the run's own id instead. + const db = createFakeDb({ + assetRow: { + tenantId: "ten_1", + creatorPrincipalId: "prin_creator", + name: "workbench-1", + displayName: null, + }, + definitionId: "wfd_workbench1", + workflowRunRow: { + id: "ins_workbench1", + address: "ins_workbench1@ten1.workbench.test", + principalId: null, + }, + workflowRunLaunchSpecRow: { + anchorRunId: "ins_workbench1", + sessionId: "ses_unanchored", + }, + }); + + const sessionService = createFakeSessionService(); + const sidecarRouter = createFakeSidecarRouter(); + + const platform = createPlatform({ + toolGrantsForPins: () => [], + db: db as never, + sessionService, + sidecarRouter, + eventCollectors: createFakeEventCollectors(), + }); + + const sent = await platform.sendMail({ + tenantId: "ten_1", + workbenchId: "ins_workbench1", + principalId: "prin_sender", + content: { content: "hello workbench" }, + }); + + expect(sent.id).toBeTruthy(); + const call = sessionService.sendUserMessageCalls[0] as { + sessionId: string; + }; + expect(call.sessionId).toBe("ses_unanchored"); + }); + test("sendMail signs with the injected crypto cache keyed by workbenchId", async () => { resolveDefinitionSourcesResult = { ok: true, diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index fa75590866..dad4716158 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,7 +88,32 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, + workflowRun: { + findFirst: async () => ({ + id: INTERCHANGE_RUN_ID, + tenantId: TENANT_ROW.id, + definitionId: DEFINITION_ROW.id, + address: INTERCHANGE_ADDRESS, + principalId: null, + }), + }, + }, + insert: () => ({ + values: () => ({ + onConflictDoNothing: async () => undefined, + }), + }), + }; +} + +let eventCollectorCreateCalls: unknown[] = []; + +function createFakeEventCollectors() { + return { + create: (...args: unknown[]) => { + eventCollectorCreateCalls.push(args); }, + has: () => false, }; } @@ -179,6 +204,7 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, + eventCollectors: createFakeEventCollectors(), }; } @@ -190,6 +216,7 @@ function resetLaunchSpies() { sendUserMessageCalls = []; cryptoGetKeys = []; reportErrorCalls = []; + eventCollectorCreateCalls = []; visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS]; frozenProjection = INERT_PROJECTION; sendUserMessageImpl = async () => new Uint8Array([1]); diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index de783406f3..e26b19aa44 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,16 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477/CL-7480: nothing writes `agent_session` for a native launch -// until something calls `ensureRunSession` — this proves it writes a -// row `resolveRunSessionId` (vendor's mail-routing lookup) actually -// reads back, only once the run is anchored with a principal, and stays -// idempotent (one insert, one collector) across repeated calls. +// CL-7481: `recordAgentSessionAtProvision` is the eager write every +// native launcher makes right after `prepareProvisionedDeployment` +// returns — this proves it lands the `agent_session` row under the +// *deploying* principal (the run's own principal does not exist yet) +// and creates the run's event collector exactly once, idempotently. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this proves the session +// moves onto that principal without ever touching the session id, and +// that a run with no launch spec row is a bug this throws on rather +// than papering over. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -15,7 +20,10 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { ensureRunSession } from "../src/launch/agent-session"; +import { + ensureRunSession, + recordAgentSessionAtProvision, +} from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -31,14 +39,15 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("ensureRunSession", () => { +describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); - const principalId = generateId("principal"); + const deployingPrincipalId = generateId("principal"); + const runPrincipalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchoredRunId = generateId("workflowRun"); - const unanchoredRunId = generateId("workflowRun"); + const provisionedRunId = generateId("workflowRun"); + const noLaunchSpecRunId = generateId("workflowRun"); const sessionId = generateId("session"); const domain = `agent-session-${tenantId}.localhost`; @@ -55,55 +64,65 @@ describeIfDb("ensureRunSession", () => { config: null, }); - await db.db.insert(schema.principal).values({ - id: principalId, - tenantId, - kind: "workflow", - refId: anchoredRunId, - status: "active", - }); + await db.db.insert(schema.principal).values([ + { + id: deployingPrincipalId, + tenantId, + kind: "human", + refId: "usr_test", + status: "active", + }, + { + id: runPrincipalId, + tenantId, + kind: "workflow", + refId: provisionedRunId, + status: "active", + }, + ]); await db.db.insert(schema.workflowDefinition).values({ id: definitionId, tenantId, - creatorPrincipalId: principalId, + creatorPrincipalId: deployingPrincipalId, assetId: null, name: "agent-session-test-definition", }); - // Anchored: has a principal, so `ensureRunSession` can insert its - // `agent_session` row. + // Born the instant `prepareProvisionedDeployment` returns: a run row + // exists, its own principal is still null, and its launch spec names + // the session `recordAgentSessionAtProvision` will record. await db.db.insert(schema.workflowRun).values({ - id: anchoredRunId, + id: provisionedRunId, definitionId, - anchorRunId: anchoredRunId, + anchorRunId: provisionedRunId, tenantId, - principalId, - address: `${anchoredRunId}@${domain}`, - status: "running", + principalId: null, + address: `${provisionedRunId}@${domain}`, + status: "deployed", }); await db.db.insert(schema.workflowRunLaunchSpec).values({ - anchorRunId: anchoredRunId, + anchorRunId: provisionedRunId, sessionId, deploymentDomain: domain, - sourceAuthorityPrincipalId: principalId, + sourceAuthorityPrincipalId: deployingPrincipalId, frozenApprovalBundle: {}, sourceOfferingIds: [], defaultSourceOfferingId: "off_test", deployContent: { systemPrompt: "" }, }); - // Unanchored: `prepareProvisionedDeployment` minted the row, but no - // trigger has reconciled a principal onto it yet — the normal state - // for a freshly provisioned invite. + // A run with no launch spec row at all: a launcher that skipped the + // shared provisioning path — a bug `ensureRunSession` must surface, + // not silently no-op past. await db.db.insert(schema.workflowRun).values({ - id: unanchoredRunId, + id: noLaunchSpecRunId, definitionId, - anchorRunId: unanchoredRunId, + anchorRunId: noLaunchSpecRunId, tenantId, principalId: null, - address: `${unanchoredRunId}@${domain}`, - status: "running", + address: `${noLaunchSpecRunId}@${domain}`, + status: "deployed", }); }); @@ -114,44 +133,26 @@ describeIfDb("ensureRunSession", () => { .where(eq(schema.agentSession.id, sessionId)); await db.db .delete(schema.workflowRunLaunchSpec) - .where(eq(schema.workflowRunLaunchSpec.anchorRunId, anchoredRunId)); + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchoredRunId)); + .where(eq(schema.workflowRun.id, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, unanchoredRunId)); + .where(eq(schema.workflowRun.id, noLaunchSpecRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); await db.db .delete(schema.principal) - .where(eq(schema.principal.id, principalId)); + .where(eq(schema.principal.id, runPrincipalId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, deployingPrincipalId)); await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("an unanchored run returns null and writes nothing", async () => { - const createCalls: unknown[] = []; - const resolved = await ensureRunSession({ - db: db.db, - eventCollectors: { - create: (...args: unknown[]) => { - createCalls.push(args); - }, - has: () => false, - }, - runId: unanchoredRunId, - }); - - expect(resolved).toBeNull(); - expect(createCalls).toEqual([]); - const sessionRow = await db.db.query.agentSession.findFirst({ - where: eq(schema.agentSession.agentId, definitionId), - }); - expect(sessionRow).toBeUndefined(); - }); - - test("an anchored run inserts its session and creates its collector once across two calls", async () => { + test("records the session under the deploying principal and creates the collector once", async () => { const createCalls: unknown[] = []; let hasCollector = false; const eventCollectors = { @@ -162,24 +163,65 @@ describeIfDb("ensureRunSession", () => { has: () => hasCollector, }; - const first = await ensureRunSession({ + await recordAgentSessionAtProvision({ db: db.db, eventCollectors, - runId: anchoredRunId, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, }); - const second = await ensureRunSession({ + // A second call for the same run is a no-op, not a conflict, and + // must not create a second collector. + await recordAgentSessionAtProvision({ db: db.db, eventCollectors, - runId: anchoredRunId, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, }); - expect(first).toBe(sessionId); - expect(second).toBe(sessionId); expect(createCalls).toEqual([ - [`${anchoredRunId}@${domain}`, tenantId, sessionId, anchoredRunId], + [`${provisionedRunId}@${domain}`, tenantId, sessionId, provisionedRunId], ]); - const resolved = await resolveRunSessionId(db.db, principalId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(deployingPrincipalId); + expect(sessionRow?.agentId).toBe(definitionId); + }); + + test("re-keys the session onto the run principal once Interchange anchors it", async () => { + await db.db + .update(schema.workflowRun) + .set({ principalId: runPrincipalId, status: "running" }) + .where(eq(schema.workflowRun.id, provisionedRunId)); + + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => true }, + runId: provisionedRunId, + }); + expect(resolved).toBe(sessionId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(runPrincipalId); + + // Interchange's own mail-routing lookup now finds the session by the + // run's own principal, exactly as it would for a folded run. + const viaVendorLookup = await resolveRunSessionId(db.db, runPrincipalId); + expect(viaVendorLookup).toBe(sessionId); + }); + + test("throws for a run with no launch spec row", async () => { + await expect( + ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => false }, + runId: noLaunchSpecRunId, + }), + ).rejects.toThrow(/no workflow_run_launch_spec/); }); }); From ae8d33b66ba331f0221e010bb72f96c890a9eac5 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:27:10 -0700 Subject: [PATCH 33/41] Record a run's session at provision; re-key it when Interchange anchors the run --- apps/hub/src/index.ts | 28 ++-- apps/hub/src/mailbox-persist.ts | 25 +++- .../agent-directory/src/one-shot-prompt.ts | 12 +- packages/chat/src/platform-adapter.ts | 14 +- packages/chat/src/run-session.ts | 25 ++-- packages/webhook-triggers/src/launch.ts | 23 ++- packages/workflows/src/index.ts | 1 + .../workflows/src/launch/agent-session.ts | 133 ++++++++++++++---- 8 files changed, 194 insertions(+), 67 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bd04767ec3..fbf44a094d 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -198,6 +198,7 @@ import { renderWorkflowSourceTree, WORKFLOW_SOURCE_ENTRY, ensureRunSession, + recordAgentSessionAtProvision, } from "@corbits/workflows"; import { createSidecarProvisioner as createE2BSidecarProvisioner, @@ -2074,15 +2075,7 @@ export async function createHub(config: HubConfig) { } try { - // Not `recordAgentSessionForRun` here: this mints a deployment - // with no message sent against it — a `workflow_run` born this - // way carries no principal until its first trigger (a call this - // deployer never makes) reconciles one onto it, so the call - // would only ever no-op. That trigger runs entirely through - // Interchange's own native route, with no Workbench-owned hook - // to record the session afterward (CL-7477 tracks the chat, - // webhook, and one-shot-prompt launch paths, which all send a - // message of their own and record there instead). + const sessionId = generateId("session"); const prepared = await workflowAllocationService.prepareProvisionedDeployment({ tenantId, @@ -2095,12 +2088,26 @@ export async function createHub(config: HubConfig) { }, entry, definitionAssetId: assetId, - sessionId: generateId("session"), + sessionId, sourceAuthorityPrincipalId: principalId, sourceOfferingIds, defaultSourceOfferingId, deployContent: { systemPrompt: "" }, }); + // The eager record every native launcher makes right after + // `prepareProvisionedDeployment` returns (CL-7481): this + // deployment mints no opening message of its own, but the + // trigger that eventually reconciles a principal onto it runs + // entirely through Interchange's own native route, with no + // Workbench-owned hook downstream to record the session + // afterward — so this is the only chance to record it at all. + await recordAgentSessionAtProvision({ + db, + eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: principalId, + }); return { deploymentId: prepared.anchorRunId, definitionAssetId: assetId, @@ -2326,6 +2333,7 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, credentialCipher, + eventCollectors, isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index f8aa0e2f01..26aa08748b 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -27,6 +27,9 @@ import { } from "@corbits/chat"; import { ensureRunSession, type EventCollectorPort } from "@corbits/workflows"; import { reportError } from "@corbits/error-sink"; +import { getLogger } from "@intx/log"; + +const logger = getLogger(["hub", "mailbox-persist"]); /** * `@corbits/mailbox` does not export `ResolveMailboxRefs` itself (only the @@ -80,15 +83,15 @@ type ResolveMailboxRefs = NonNullable< * constructed until after `lookups` is, so this only reads `.current` * inside the returned closure, never at wiring time. */ -export function createHubPersistMailWithSessionEnsure( +export function createHubPersistMailWithSessionEnsure( db: DB["db"], eventCollectorsRef: { current?: Pick }, upstream: (args: MailboxPersistArgs) => Promise, ): (args: MailboxPersistArgs) => Promise { return async (args) => { + const sender = await resolveRoutableAddress(db, args.senderAddress); try { const eventCollectors = eventCollectorsRef.current; - const sender = await resolveRoutableAddress(db, args.senderAddress); if (sender !== undefined && eventCollectors !== undefined) { await ensureRunSession({ db, eventCollectors, runId: sender.id }); } @@ -98,6 +101,24 @@ export function createHubPersistMailWithSessionEnsure( extra: { senderAddress: args.senderAddress }, }); } + // The vendored `session_mail` ledger (`upstream`, `baseLookups.persistMail`) + // keys on the sender run's principal, which does not exist until the + // run's first trigger anchors one — a run's own first outbound mail can + // reach this wrapper before that happens. `@corbits/mailbox`'s own write + // (this function's caller, `createMailboxPersist`) is keyed on the run + // address instead and runs regardless, so it is already the durable + // record for this frame; skip the vendored delegate rather than let it + // throw "no session for address" past the caller. + if (sender !== undefined && sender.sessionId === null) { + logger.debug( + "skipping vendored persistMail for run {runId}: no session yet", + { runId: sender.id, senderAddress: args.senderAddress }, + ); + // No `session_mail` rows were written for this frame: an empty + // result, not a thrown error, so a caller iterating this like + // `handleMailPersist` does simply emits no `mail.persisted` events. + return [] as unknown as R; + } return upstream(args); }; } diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index 1851527d4f..b03351e45a 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -11,6 +11,7 @@ import { endAgentSessionForRun, readDefinitionProjection, readFoldedBody, + recordAgentSessionAtProvision, WORKFLOW_SOURCE_ENTRY, type EventCollectorPort, } from "@corbits/workflows"; @@ -165,10 +166,13 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `ensureRunSession` here: a freshly provisioned run's - // `workflow_run.principal_id` is still null until its first trigger - // reconciles one onto it, and the hub's own mail/dispatch seams - // ensure the session lazily once that happens (CL-7480). + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: input.principalId, + }); return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index a546564e75..b7c5c51477 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -19,6 +19,7 @@ import { endAgentSessionForRun, ensureRunSession, readFoldedBody, + recordAgentSessionAtProvision, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; @@ -344,12 +345,13 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); - // Not `ensureRunSession` here: a freshly provisioned run's - // `workflow_run.principal_id` is still null (an invite sits - // un-triggered until someone actually writes into it), so this - // would only ever no-op. The hub's own mail/dispatch seams ensure - // the session lazily instead, once the run's first turn has - // actually reconciled a principal onto it (CL-7480). + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: input.sourceAuthorityPrincipalId, + }); return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, diff --git a/packages/chat/src/run-session.ts b/packages/chat/src/run-session.ts index 79ffea89c3..54a366a33c 100644 --- a/packages/chat/src/run-session.ts +++ b/packages/chat/src/run-session.ts @@ -1,19 +1,26 @@ -// Resolves a provisioned run's session id via Interchange's -// shared-principal bridge rather than any name derived from the run id. +// Resolves a provisioned run's session id off its own launch spec +// (`workflow_run_launch_spec.session_id`, written once at provision by +// every native launcher's `recordAgentSessionAtProvision` — CL-7481) +// rather than Interchange's principal-keyed `resolveRunSessionId`. A +// run's session id is fixed at provision time and never changes; its +// principal is not assigned until the run's first trigger anchors one, +// so a principal-keyed lookup returns nothing for exactly the run states +// (freshly provisioned, un-anchored) chat most needs to resolve. +import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; -import { resolveRunSessionId } from "@intx/hub-sessions"; +import { workflowRunLaunchSpec } from "@intx/db/schema"; export async function resolveRunSessionIdOrThrow( db: DB["db"], - run: { principalId: string | null }, + run: { id: string }, ): Promise { - const sessionId = await resolveRunSessionId(db, run.principalId, { - includeEnded: true, + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, run.id), }); - if (sessionId === null) { + if (launchSpecRow === undefined) { throw new Error( - "no agent_session found for this run's principal; provision may not have completed", + `no workflow_run_launch_spec for run "${run.id}"; provision may not have completed`, ); } - return sessionId; + return launchSpecRow.sessionId; } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 4a0331aca1..d215dfb1a2 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,7 +17,9 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, + recordAgentSessionAtProvision, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -47,6 +49,13 @@ export type LaunchWebhookTriggerDeps = { "prepareProvisionedDeployment" >; sessionService: Pick; + /** + * The same wrapped `EventCollectorRegistry` every native launcher + * threads through (`apps/hub/src/index.ts`'s `eventCollectors`) — this + * launcher records the run's `agent_session` and event collector the + * instant `prepareProvisionedDeployment` returns (CL-7481). + */ + eventCollectors: Pick; /** * Reads the hub's live sidecar routing table. A freshly provisioned * run's sidecar takes several seconds to boot and register (CL-7476) @@ -196,6 +205,14 @@ export async function launchWebhookTrigger( : {}), }); + await recordAgentSessionAtProvision({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: prepared.anchorRunId, + sessionId, + sourceAuthorityPrincipalId: trigger.createdBy, + }); + await deps.persistLaunch({ tenantId: trigger.tenantId, instanceId: prepared.anchorRunId, @@ -237,12 +254,6 @@ export async function launchWebhookTrigger( }); } - // A run's session and event collector are ensured lazily now, at the - // hub seams that actually see the run become mail-routable - // (`apps/hub/src/mailbox-persist.ts`, the wrapped `eventCollectors` - // dispatch in `apps/hub/src/index.ts`) — CL-7480. Nothing here needs - // to record it. - return { instanceId: prepared.anchorRunId, triggerAddress: prepared.deploymentAddress, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 74828b7b2a..eefa55a850 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -57,6 +57,7 @@ export { export { scheduleCronFromProjection } from "./schedule/from-projection"; export { ensureRunSession, + recordAgentSessionAtProvision, endAgentSessionForPrincipal, endAgentSessionForRun, type EventCollectorPort, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index 0e2cbff99f..f73c2a0d03 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -21,12 +21,21 @@ // already committed", throwing rather than re-materializing — so nothing // upstream of that first trigger may write the principal first. // -// So `ensureRunSession` (CL-7480) is lazy and idempotent rather than -// called once right after a send: every seam that might be the run's -// first mail-routable moment (a hub mailbox persist, a session -// orchestrator dispatch with no live collector yet) calls it before -// doing its own work. A run with no principal yet is a normal, expected -// state, not a bug — this returns `null` rather than throwing. +// CL-7480 made `ensureRunSession` lazy: called from every seam that +// might be a run's first mail-routable moment, no-opping until a +// principal existed to key the row on. That left a real gap — a run's +// very first outbound message can itself be that seam, both for chat's +// `sendMail` (which looks the session up by run principal) and for the +// vendored `persistMail` a heartbeat run's own first outbound mail hits +// — and both landed on an un-anchored run with nothing recorded yet. +// CL-7481 fixes this at the root: `recordAgentSessionAtProvision` writes +// the row immediately after `prepareProvisionedDeployment` returns, +// keyed on the deploying principal (`sourceAuthorityPrincipalId`) since +// the run principal does not exist yet — every other field the launcher +// already knows or can read straight off the fresh run row. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this moves `principal_id` +// onto that principal without ever touching the session id. import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { @@ -52,37 +61,37 @@ export type EventCollectorPort = Pick< >; /** - * Idempotently makes a run's session and event collector exist, the - * instant that becomes possible: the run's `tenantId`, `definitionId` - * (the `agent_id` FK), and `principalId` are all read fresh off the - * `workflow_run` row, never guessed by the caller. Returns `null` (and - * writes nothing) when the row has no principal yet — an un-anchored - * run is not a bug, just not ensurable yet. Returns the run's - * `sessionId` once the session exists (freshly inserted or already - * there — a second call for the same run is a no-op, not a conflict), - * with the run's event collector created too if none was live for its - * address. + * The one shared write every native launcher calls right after its own + * `prepareProvisionedDeployment` returns (CL-7481) — a run row exists + * the instant that call resolves, so the session it will use is known + * then too: the launch-spec `sessionId` it just minted, `tenantId` and + * `definitionId` read fresh off the new `workflow_run` row (Interchange + * resolves the asset id the caller passed to a real `workflow_definition` + * row internally; callers never see that id themselves), and the + * deploying principal (`sourceAuthorityPrincipalId`) standing in for the + * run's own principal, which does not exist yet. `onConflictDoNothing` + * makes a second call for the same run id a no-op. Every launcher threads + * the same wrapped `EventCollectorRegistry` through here so the run's + * `inference_turn`/`turn_part` rows have somewhere to land from its very + * first turn. */ -export async function ensureRunSession(params: { +export async function recordAgentSessionAtProvision(params: { readonly db: DB["db"]; readonly eventCollectors: Pick; readonly runId: string; -}): Promise { - const { db, eventCollectors, runId } = params; + readonly sessionId: string; + readonly sourceAuthorityPrincipalId: string; +}): Promise { + const { db, eventCollectors, runId, sessionId, sourceAuthorityPrincipalId } = + params; const runRow = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, runId), }); - if (runRow === undefined || runRow.principalId === null) { - return null; - } - - const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ - where: eq(workflowRunLaunchSpec.anchorRunId, runId), - }); - if (launchSpecRow === undefined) { - return null; + if (runRow === undefined) { + throw new Error( + `recordAgentSessionAtProvision: no workflow_run "${runId}" — prepareProvisionedDeployment must have already returned`, + ); } - const sessionId = launchSpecRow.sessionId; const now = new Date(); await db @@ -91,13 +100,77 @@ export async function ensureRunSession(params: { id: sessionId, tenantId: runRow.tenantId, agentId: runRow.definitionId, - principalId: runRow.principalId, + principalId: sourceAuthorityPrincipalId, status: "active", createdAt: now, updatedAt: now, }) .onConflictDoNothing({ target: agentSession.id }); + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { + eventCollectors.create( + runRow.address, + runRow.tenantId, + sessionId, + runRow.id, + ); + } +} + +/** + * Re-keys a run's `agent_session` onto the run's own principal once + * Interchange anchors one (`anchorWithPrincipal`, the run's first + * trigger) — before that, the session row is recorded under the + * deploying principal by `recordAgentSessionAtProvision`, since the run + * principal does not exist yet. The session id never changes; only + * `principal_id` moves, and only once (a session already re-keyed to the + * current run principal is left alone). The launch-spec row is the + * source of truth for which session belongs to this run — its absence + * means a launcher provisioned this run without going through the + * shared path, which is a bug, not a state to paper over. Also creates + * the run's event collector if none is live for its address (a fresh + * process has no in-memory collectors regardless of what is on disk). + */ +export async function ensureRunSession(params: { + readonly db: DB["db"]; + readonly eventCollectors: Pick; + readonly runId: string; +}): Promise { + const { db, eventCollectors, runId } = params; + const runRow = await db.query.workflowRun.findFirst({ + where: eq(workflowRun.id, runId), + }); + if (runRow === undefined) { + return null; + } + + const launchSpecRow = await db.query.workflowRunLaunchSpec.findFirst({ + where: eq(workflowRunLaunchSpec.anchorRunId, runId), + }); + if (launchSpecRow === undefined) { + throw new Error( + `ensureRunSession: no workflow_run_launch_spec for run "${runId}" — every launcher records one at provision time`, + ); + } + const sessionId = launchSpecRow.sessionId; + + if (runRow.principalId !== null) { + const sessionRow = await db.query.agentSession.findFirst({ + where: eq(agentSession.id, sessionId), + }); + if (sessionRow === undefined) { + throw new Error( + `ensureRunSession: no agent_session "${sessionId}" for run "${runId}" — recordAgentSessionAtProvision should have created it at launch`, + ); + } + if (sessionRow.principalId !== runRow.principalId) { + await db + .update(agentSession) + .set({ principalId: runRow.principalId, updatedAt: new Date() }) + .where(eq(agentSession.id, sessionId)); + } + } + if (runRow.address !== null && !eventCollectors.has(runRow.address)) { eventCollectors.create( runRow.address, From 2b5dd7a608ad2a654cff7a558d1d4b79950384fe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:39:42 -0700 Subject: [PATCH 34/41] Fix CI: prettier formatting, principal enum value, and workflowRunLaunchSpec fakes --- apps/hub/src/mailbox-persist.ts | 4 +++- packages/chat/test/platform-adapter.test.ts | 7 ++----- packages/chat/test/relaunch-close.test.ts | 16 ++++++++++++++++ packages/workflows/test/agent-session.test.ts | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/apps/hub/src/mailbox-persist.ts b/apps/hub/src/mailbox-persist.ts index 26aa08748b..6d37382a6f 100644 --- a/apps/hub/src/mailbox-persist.ts +++ b/apps/hub/src/mailbox-persist.ts @@ -83,7 +83,9 @@ type ResolveMailboxRefs = NonNullable< * constructed until after `lookups` is, so this only reads `.current` * inside the returned closure, never at wiring time. */ -export function createHubPersistMailWithSessionEnsure( +export function createHubPersistMailWithSessionEnsure< + R extends readonly unknown[], +>( db: DB["db"], eventCollectorsRef: { current?: Pick }, upstream: (args: MailboxPersistArgs) => Promise, diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 9f3b5656aa..120538c926 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -34,7 +34,6 @@ import { workflowDefinition, workflowDefinitionVersion, workflowRun, - workflowRunLaunchSpec, } from "@intx/db/schema"; import { SessionLaunchError } from "@intx/hub-sessions"; import type { EventCollectorRegistry, SidecarRouter } from "@intx/hub-sessions"; @@ -355,8 +354,7 @@ function createFakeDb(opts: { * call site. */ workflowRunLaunchSpecRow?: - | { anchorRunId: string; sessionId: string } - | undefined; + { anchorRunId: string; sessionId: string } | undefined; }) { const inserted: { table: unknown; values: unknown }[] = []; const updated: { table: unknown; values: unknown }[] = []; @@ -477,8 +475,7 @@ function createFakeDb(opts: { return inserted.findLast( (row) => row.table === agentSession && - (id === undefined || - (row.values as { id?: string }).id === id), + (id === undefined || (row.values as { id?: string }).id === id), )?.values as { id: string; principalId: string } | undefined; }, }, diff --git a/packages/chat/test/relaunch-close.test.ts b/packages/chat/test/relaunch-close.test.ts index 4e85d0626b..35f2c0627d 100644 --- a/packages/chat/test/relaunch-close.test.ts +++ b/packages/chat/test/relaunch-close.test.ts @@ -78,6 +78,22 @@ function createFakeDb(opts: { ); }, }, + // CL-7481: `resolveRunSessionIdOrThrow` reads the launch spec by + // run id instead of walking through the run's principal — the + // session id it returns still matches this fixture's + // `ses_` convention, since every run here is already + // "anchored" with a fixed principal from the moment it's defined. + workflowRunLaunchSpec: { + findFirst: async ({ where }: { where: unknown }) => { + const [anchorRunId] = comparedValues(where); + const run = opts.runs.find((row) => row.id === anchorRunId); + if (run === undefined) return undefined; + return { + anchorRunId, + sessionId: `ses_${run.principalId ?? ""}`, + }; + }, + }, }, select: () => ({ from: (table: unknown) => ({ diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index e26b19aa44..6fe8dcfb64 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -68,7 +68,7 @@ describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { { id: deployingPrincipalId, tenantId, - kind: "human", + kind: "user", refId: "usr_test", status: "active", }, From c57dd9d75327e738894292a0bb006f5c7d607c17 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:26 -0700 Subject: [PATCH 35/41] Fix merge fallout: restore CL-7481 agent-session exports and drop a duplicate key Merging cl-7107's repin/re-vendor audit into this branch left packages/workflows/src/index.ts re-exporting the pre-CL-7481 agent-session API (recordAgentSessionForRun) that no longer exists, its matching test file testing that same stale API, and a duplicated eventCollectors key in apps/hub/src/index.ts's webhook-trigger launch wiring. Restores the current exports (recordAgentSessionAtProvision, ensureRunSession) and their test, and drops the duplicate object key. --- apps/hub/src/index.ts | 1 - packages/workflows/src/index.ts | 4 +- packages/workflows/test/agent-session.test.ts | 204 ++++++++++++------ 3 files changed, 140 insertions(+), 69 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index b62dbcb1ff..fbf44a094d 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2337,7 +2337,6 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, - eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 244d0478ab..64bfbdff98 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -56,10 +56,10 @@ export { } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; export { - recordAgentSessionForRun, + recordAgentSessionAtProvision, + ensureRunSession, endAgentSessionForPrincipal, endAgentSessionForRun, - type RecordAgentSessionParams, type EventCollectorPort, } from "./launch/agent-session"; export { diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index 3ae50e221e..6fe8dcfb64 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -2,11 +2,16 @@ // repo's convention for tests that talk to a real Postgres (see // `packages/access-tools/test/routes.test.ts`). // -// CL-7477: before this fix, nothing wrote `agent_session` for a native -// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could -// never find a session for a run's principal and every launch's first -// turn failed to persist mail. This proves `recordAgentSessionForRun` -// writes a row `resolveRunSessionId` actually reads back. +// CL-7481: `recordAgentSessionAtProvision` is the eager write every +// native launcher makes right after `prepareProvisionedDeployment` +// returns — this proves it lands the `agent_session` row under the +// *deploying* principal (the run's own principal does not exist yet) +// and creates the run's event collector exactly once, idempotently. +// `ensureRunSession` is now purely a re-key: once the run's first +// trigger anchors a real principal onto it, this proves the session +// moves onto that principal without ever touching the session id, and +// that a run with no launch spec row is a bug this throws on rather +// than papering over. import { afterAll, beforeAll, expect, test } from "bun:test"; import { eq } from "drizzle-orm"; @@ -16,8 +21,8 @@ import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; import { - endAgentSessionForRun, - recordAgentSessionForRun, + ensureRunSession, + recordAgentSessionAtProvision, } from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { @@ -34,14 +39,17 @@ function dbConfigFromUrl(databaseUrl: string) { const databaseUrl = process.env["DATABASE_URL"]; const describeIfDb = dbGate(databaseUrl, import.meta.path); -describeIfDb("recordAgentSessionForRun", () => { +describeIfDb("recordAgentSessionAtProvision / ensureRunSession", () => { let db: DB; const tenantId = generateId("tenant"); - const principalId = generateId("principal"); + const deployingPrincipalId = generateId("principal"); + const runPrincipalId = generateId("principal"); const definitionId = generateId("workflowDefinition"); - const anchorRunId = generateId("workflowRun"); + const provisionedRunId = generateId("workflowRun"); + const noLaunchSpecRunId = generateId("workflowRun"); const sessionId = generateId("session"); + const domain = `agent-session-${tenantId}.localhost`; beforeAll(async () => { if (databaseUrl === undefined) return; @@ -51,35 +59,70 @@ describeIfDb("recordAgentSessionForRun", () => { id: tenantId, name: "Agent Session Test Tenant", slug: `agent-session-${tenantId}`, - domain: `agent-session-${tenantId}.localhost`, + domain, parentId: null, config: null, }); - await db.db.insert(schema.principal).values({ - id: principalId, - tenantId, - kind: "workflow", - refId: anchorRunId, - status: "active", - }); + await db.db.insert(schema.principal).values([ + { + id: deployingPrincipalId, + tenantId, + kind: "user", + refId: "usr_test", + status: "active", + }, + { + id: runPrincipalId, + tenantId, + kind: "workflow", + refId: provisionedRunId, + status: "active", + }, + ]); await db.db.insert(schema.workflowDefinition).values({ id: definitionId, tenantId, - creatorPrincipalId: principalId, + creatorPrincipalId: deployingPrincipalId, assetId: null, name: "agent-session-test-definition", }); + // Born the instant `prepareProvisionedDeployment` returns: a run row + // exists, its own principal is still null, and its launch spec names + // the session `recordAgentSessionAtProvision` will record. + await db.db.insert(schema.workflowRun).values({ + id: provisionedRunId, + definitionId, + anchorRunId: provisionedRunId, + tenantId, + principalId: null, + address: `${provisionedRunId}@${domain}`, + status: "deployed", + }); + await db.db.insert(schema.workflowRunLaunchSpec).values({ + anchorRunId: provisionedRunId, + sessionId, + deploymentDomain: domain, + sourceAuthorityPrincipalId: deployingPrincipalId, + frozenApprovalBundle: {}, + sourceOfferingIds: [], + defaultSourceOfferingId: "off_test", + deployContent: { systemPrompt: "" }, + }); + + // A run with no launch spec row at all: a launcher that skipped the + // shared provisioning path — a bug `ensureRunSession` must surface, + // not silently no-op past. await db.db.insert(schema.workflowRun).values({ - id: anchorRunId, + id: noLaunchSpecRunId, definitionId, - anchorRunId, + anchorRunId: noLaunchSpecRunId, tenantId, - principalId, - address: `${anchorRunId}@agent-session-${tenantId}.localhost`, - status: "running", + principalId: null, + address: `${noLaunchSpecRunId}@${domain}`, + status: "deployed", }); }); @@ -88,68 +131,97 @@ describeIfDb("recordAgentSessionForRun", () => { await db.db .delete(schema.agentSession) .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRunLaunchSpec) + .where(eq(schema.workflowRunLaunchSpec.anchorRunId, provisionedRunId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, provisionedRunId)); await db.db .delete(schema.workflowRun) - .where(eq(schema.workflowRun.id, anchorRunId)); + .where(eq(schema.workflowRun.id, noLaunchSpecRunId)); await db.db .delete(schema.workflowDefinition) .where(eq(schema.workflowDefinition.id, definitionId)); await db.db .delete(schema.principal) - .where(eq(schema.principal.id, principalId)); + .where(eq(schema.principal.id, runPrincipalId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, deployingPrincipalId)); await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); }); - test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun( - db.db, - { sessionId, anchorRunId }, - { create: () => undefined }, - ); - - const resolved = await resolveRunSessionId(db.db, principalId); - expect(resolved).toBe(sessionId); - }); - - test("creates the run's event collector with its address/tenant/session/run id", async () => { + test("records the session under the deploying principal and creates the collector once", async () => { const createCalls: unknown[] = []; - const otherSessionId = generateId("session"); - - await recordAgentSessionForRun( - db.db, - { sessionId: otherSessionId, anchorRunId }, - { - create: (...args: unknown[]) => { - createCalls.push(args); - }, + let hasCollector = false; + const eventCollectors = { + create: (...args: unknown[]) => { + createCalls.push(args); + hasCollector = true; }, - ); + has: () => hasCollector, + }; + + await recordAgentSessionAtProvision({ + db: db.db, + eventCollectors, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, + }); + // A second call for the same run is a no-op, not a conflict, and + // must not create a second collector. + await recordAgentSessionAtProvision({ + db: db.db, + eventCollectors, + runId: provisionedRunId, + sessionId, + sourceAuthorityPrincipalId: deployingPrincipalId, + }); expect(createCalls).toEqual([ - [ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - tenantId, - otherSessionId, - anchorRunId, - ], + [`${provisionedRunId}@${domain}`, tenantId, sessionId, provisionedRunId], ]); - await db.db - .delete(schema.agentSession) - .where(eq(schema.agentSession.id, otherSessionId)); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), + }); + expect(sessionRow?.principalId).toBe(deployingPrincipalId); + expect(sessionRow?.agentId).toBe(definitionId); }); - test("abandons the run's event collector when the session ends", async () => { - const abandonCalls: string[] = []; + test("re-keys the session onto the run principal once Interchange anchors it", async () => { + await db.db + .update(schema.workflowRun) + .set({ principalId: runPrincipalId, status: "running" }) + .where(eq(schema.workflowRun.id, provisionedRunId)); + + const resolved = await ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => true }, + runId: provisionedRunId, + }); - await endAgentSessionForRun(db.db, anchorRunId, { - abandon: (address: string) => { - abandonCalls.push(address); - }, + expect(resolved).toBe(sessionId); + const sessionRow = await db.db.query.agentSession.findFirst({ + where: eq(schema.agentSession.id, sessionId), }); + expect(sessionRow?.principalId).toBe(runPrincipalId); - expect(abandonCalls).toEqual([ - `${anchorRunId}@agent-session-${tenantId}.localhost`, - ]); + // Interchange's own mail-routing lookup now finds the session by the + // run's own principal, exactly as it would for a folded run. + const viaVendorLookup = await resolveRunSessionId(db.db, runPrincipalId); + expect(viaVendorLookup).toBe(sessionId); + }); + + test("throws for a run with no launch spec row", async () => { + await expect( + ensureRunSession({ + db: db.db, + eventCollectors: { create: () => {}, has: () => false }, + runId: noLaunchSpecRunId, + }), + ).rejects.toThrow(/no workflow_run_launch_spec/); }); }); From 42f051164eff60ddaed7e32b58090d48740be9f0 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:31 -0700 Subject: [PATCH 36/41] Add tests for wake retry re-resolving the live address (CL-7486) sendRunMailWithReclaimRetry retried every attempt against the address it started with. Proves the reproduction of the production failure: a participant already one relaunch generation in that dies again mid-retry must have its next retry chase the run that replaced it, not keep hammering the address that just died. --- packages/chat/test/platform-adapter.test.ts | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 120538c926..89ad655d58 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -2781,6 +2781,139 @@ describe("createHubChatPlatform", () => { }); }); + // CL-7486: `sendRunMailWithReclaimRetry` used to retry every attempt + // against the address it started with, even after a wake in between + // attempts relaunched the run. A run already one relaunch generation + // in (`instanceId` stable, `currentRunId` already repointed once) that + // dies again while this loop is retrying reproduces the production + // failure exactly: a second relaunch repoints `currentRunId` again, + // and the address this loop still holds is now neither the stable + // `instanceId` nor the current `currentRunId` — invisible to + // `readBindingByAddressAnyTenant`, so the next wake for it throws "No + // workbench_launch binding" instead of finding the run that replaced + // it. + describe("sendRunMailWithReclaimRetry re-resolves the live address", () => { + test("a relaunch between retries is chased to its new address, not retried against the one that died", async () => { + resolveDefinitionSourcesResult = { + ok: true, + materials: [], + sources: [ + { + id: "off_1", + provider: "anthropic", + baseURL: "https://inference.invalid", + credentialId: "cred_placeholder", + model: "claude-sonnet-5", + }, + ], + defaultSource: "off_1", + }; + + const staleAddress = "run_gen1@ten1.workbench.test"; + const db = createFakeDb({ + assetRow: { + tenantId: "ten_1", + creatorPrincipalId: "prin_creator", + name: "ins_room1", + displayName: null, + }, + definitionId: "wfd_room1", + workflowRunRow: { + id: "run_gen1", + address: staleAddress, + principalId: "prin_room1", + definitionId: "wfd_room1", + // Already dead by the time the reclaim retry's own wake reads + // it — the run died again while this loop was mid-retry, not + // at the top of `sendMail` (which would have relaunched it + // before ever reaching this retry loop at all). + status: "failed", + }, + workflowDefinitionRow: { + id: "wfd_room1", + tenantId: "ten_1", + status: "deployed", + origin: "authored", + assetId: "asst_room1", + }, + workbenchLaunchRow: { + tenantId: "ten_1", + // `instanceId` is the room's own stable id, already distinct + // from `currentRunId` — this participant has been relaunched + // once before this test even starts. + instanceId: "ins_room1", + currentRunId: "run_gen1", + foldedBody: { + systemPrompt: "host prompt", + model: "claude-sonnet-5", + toolPackagePins: [], + grantRequirements: [], + credentialBindings: [], + }, + }, + }); + db.inserted.push({ + table: agentSession, + values: { id: "ses_run1", principalId: "prin_room1" }, + }); + + const sessionService = createFakeSessionService(); + sessionService.sendUserMessage = async (params: unknown) => { + sessionService.sendUserMessageCalls.push(params); + const { agentAddress } = params as { agentAddress: string }; + // The dead run never becomes reachable again — only a send to + // whatever address the relaunch actually produced can succeed. + if (agentAddress === staleAddress) { + throw new Error("agent is unreachable"); + } + return new TextEncoder().encode("raw-mime-bytes"); + }; + + // Routable so `sendMail`'s own opening wake-before-send gate skips + // waking it — the relaunch this test cares about must happen + // inside the reclaim-retry loop below, not before it. + const sidecarRouter = createFakeSidecarRouter({ + routableAddresses: [staleAddress], + }); + + const platform = createPlatform({ + toolGrantsForPins: () => [], + db: db as never, + sessionService, + sidecarRouter, + eventCollectors: createFakeEventCollectors(), + reclaimRetryDelaysMs: [1], + mailDeliveryTimeoutMs: 5_000, + }); + + const sent = await platform.sendMail({ + tenantId: "ten_1", + workbenchId: "ins_room1", + principalId: "prin_sender", + content: { content: "hello" }, + }); + + expect(sent.id).toBeTruthy(); + expect(sessionService.sendUserMessageCalls).toHaveLength(2); + const [first, second] = sessionService.sendUserMessageCalls as { + agentAddress: string; + }[]; + expect(first?.agentAddress).toBe(staleAddress); + // The retry that follows the relaunch targets the run that + // replaced `run_gen1`, never the dead address the loop started + // with. + expect(second?.agentAddress).not.toBe(staleAddress); + + const repointed = db.updated.at(-1)?.values as { + currentRunId: string; + priorRunIds: string[]; + }; + const secondAddress = second?.agentAddress ?? ""; + expect(repointed.currentRunId).toBe(secondAddress.split("@")[0] ?? ""); + expect(repointed.priorRunIds).toEqual(["run_gen1"]); + }); + }); + // Proves the actual lever an edited system prompt reaches a running // instance through: `wakeByAddress` (exercised via `sendMail`'s // wake-on-send path above) replays `workbench_launch.foldedBody` From c5cbcebe417a51018a868b831334a6171a617c33 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:15:41 -0700 Subject: [PATCH 37/41] Chat: wake retry chases the run a relaunch replaced it with (CL-7486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sendRunMailWithReclaimRetry retried every delivery attempt, and woke, against the address it captured at the top of the loop. A wake in between retries can relaunch the run (its dead-run branch repoints the room's stable participant at a fresh address without telling this loop), and a participant already one relaunch generation in that dies again mid-retry leaves this loop holding an address that is no longer either the stable participant id or the current run id — invisible to readBindingByAddressAnyTenant. The next wake for it then threw "No workbench_launch binding" instead of finding the run that replaced it, which is what chat e2e's mention-fanout and invite-drives-a-turn cases were hitting. Re-resolves the live address (and its session) off the stable participant id after each wake, so the retry loop always chases whichever run is actually current. Also hardens publishCorbitsToolsRegistry's create-then-list-on-conflict ensure: two overlapping seed runs racing the same tenant's package registry asset can still throw a raw asset_tenant_kind_name violation if the substrate's own 409 mapping is bypassed; the ensure now also recognizes that violation by name and resolves the winner's row instead of throwing. --- packages/chat/src/platform-adapter.ts | 63 ++++++++++++++++--- packages/tool-registry-publish/src/publish.ts | 14 ++++- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index b7c5c51477..1113171348 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -989,10 +989,24 @@ export function createHubChatPlatform( * more time regardless. Exhausting every delay means the condition * is not transient and the caller gets a clean `AgentUnreachableError` * rather than an unhandled 500. + * + * CL-7486: a wake this loop runs between retries can itself relaunch + * the run — `wakeByAddress`'s dead-run branch repoints the room's + * stable participant at a fresh address without telling this loop. + * Retrying `sendRunMail` against the address it started with would + * then keep hammering a run that will never become routable again, + * and the wake ahead of the NEXT retry would fail outright (no + * `workbench_launch` row still names that address at all) instead of + * the "unreachable" this loop knows how to ride out. `reresolve` + * re-reads the live address (and its session) off the stable + * participant id before every wake/retry, so this loop always chases + * whichever run is actually current. */ async function sendRunMailWithReclaimRetry( params: SendRunMailParams, + reresolve: () => Promise<{ agentAddress: string; sessionId: string }>, ): Promise>> { + let current = params; let loggedRetryStart = false; for (let attempt = 0; ; attempt++) { try { @@ -1001,29 +1015,44 @@ export function createHubChatPlatform( // to accept regardless — the signal parameter is unused here, // same abandon-on-timeout behavior as before. return await withTimeout( - () => sendRunMail(runMailDeps, params), + () => sendRunMail(runMailDeps, current), MAIL_DELIVERY_TIMEOUT_MS, - `mail to ${params.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, + `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, ); } catch (err) { const delay = RECLAIM_RETRY_DELAYS_MS[attempt]; if (!isAgentUnreachable(err) || delay === undefined) { if (loggedRetryStart) { - wakeLogger.info`mail to ${params.agentAddress} exhausted every reclaim retry; giving up`; + wakeLogger.info`mail to ${current.agentAddress} exhausted every reclaim retry; giving up`; } if (isAgentUnreachable(err)) { - throw new AgentUnreachableError(params.agentAddress, { + throw new AgentUnreachableError(current.agentAddress, { cause: err, }); } throw err; } if (!loggedRetryStart) { - wakeLogger.info`mail to ${params.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; + wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; loggedRetryStart = true; } await sleep(delay); - await wakeByAddressBounded(params.agentAddress); + // Wake whatever this loop still believes is live — `wakeByAddress` + // itself tolerates an address one relaunch behind the room's + // current one (its lookup matches either the stable participant + // id or the live run id) — and only THEN re-resolve, since the + // wake just run is exactly the kind of relaunch that can leave + // `current.agentAddress` pointing at a run that no longer exists + // at all. Re-resolving before the wake instead would still hand + // the next retry whatever address was current a moment ago, + // stale again the instant this wake relaunches it. + await wakeByAddressBounded(current.agentAddress); + const fresh = await reresolve(); + current = { + ...current, + agentAddress: fresh.agentAddress, + sessionId: fresh.sessionId, + }; } } } @@ -1399,7 +1428,27 @@ export function createHubChatPlatform( ? { references: input.content.references } : {}), }; - const sent = await sendRunMailWithReclaimRetry(withThreading); + const sent = await sendRunMailWithReclaimRetry(withThreading, async () => { + const relive = await requireLive(input.workbenchId, input.tenantId); + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: relive.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: relive.binding.liveAddress, + extra: { runId: relive.run.id }, + }); + } + return { + agentAddress: relive.binding.liveAddress, + sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), + }; + }); lifecycle?.recordActivity(deliveryAddress); diff --git a/packages/tool-registry-publish/src/publish.ts b/packages/tool-registry-publish/src/publish.ts index 301bdd6c64..d6eebbd36d 100644 --- a/packages/tool-registry-publish/src/publish.ts +++ b/packages/tool-registry-publish/src/publish.ts @@ -110,7 +110,19 @@ async function ensureRegistryAsset( if (created.status === 201) { return parseSchema(AssetResponse, created.data, "create asset response").id; } - if (created.status !== 409) { + // CL-7486: two overlapping seed runs for the same tenant (the root + // tenant's own boot-time seed racing a second caller seeding that + // same tenant) can both pass the `listRegistryAsset` check above + // before either has created the row, then both post-create — the + // loser hits the `asset_tenant_kind_name` unique constraint. The + // substrate answers that as `409` in the common case; asserted here + // by name too, in case a raw constraint violation surfaces as a + // generic 500 instead, so the loser still resolves the winner's row + // rather than throwing on a race this ensure is meant to tolerate. + const isNameCollision = + created.status === 409 || + JSON.stringify(created.data).includes("asset_tenant_kind_name"); + if (!isNameCollision) { throw new Error( `publishCorbitsToolsRegistry: failed to create the ${CORBITS_TOOLS_REGISTRY} asset: ${String(created.status)}`, ); From ebc173bdd5aaa12cad9068df676586788a827314 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:16:53 -0700 Subject: [PATCH 38/41] Fix webhook-triggers launch test fixture: eventCollectors needs has() --- packages/webhook-triggers/test/launch.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 01b3b6785c..e671da0093 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -199,6 +199,7 @@ function baseDeps() { isRoutable: () => isRoutableForTest, eventCollectors: { create: () => undefined, + has: () => false, }, }; } From 097b172570f4e9fa08e0ce4bde57aecc4f62c05f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:49:03 -0700 Subject: [PATCH 39/41] Add test coverage for routable-wait relaunch chasing in sendRunMailWithReclaimRetry Drives the CL-7486 reclaim-retry test off routing-table transitions instead of fixed backoff timing, and updates the CL-7214 coalescing test's fake sidecar router to model a redeploy actually registering. --- packages/chat/test/platform-adapter.test.ts | 32 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/chat/test/platform-adapter.test.ts b/packages/chat/test/platform-adapter.test.ts index 89ad655d58..c9583847ff 100644 --- a/packages/chat/test/platform-adapter.test.ts +++ b/packages/chat/test/platform-adapter.test.ts @@ -2726,6 +2726,7 @@ describe("createHubChatPlatform", () => { // in flight when the concurrent `ensureAwake` call joins it. let deployCallCount = 0; const gatedDeployReleases: (() => void)[] = []; + const sidecarRouter = createFakeSidecarRouter({ routableAddresses: [] }); const allocationService = createFakeWorkflowAllocationService(); const originalPrepare = allocationService.prepareProvisionedDeployment.bind(allocationService); @@ -2738,6 +2739,14 @@ describe("createHubChatPlatform", () => { await new Promise((resolve) => { gatedDeployReleases.push(resolve); }); + // Models the sidecar registering once the reclaim retry's own + // redeploy actually completes — the routable-wait + // `sendRunMailWithReclaimRetry` now does (CL-7488) needs this + // to see the retried send through. The cold wake ahead of the + // first send attempt deliberately stays unrouted so that + // attempt still finds `wakeByAddress`'s own routability check + // false and takes the reclaim-retry path this test exercises. + sidecarRouter.routableAddresses.push(result.deploymentAddress); } return result; }; @@ -2746,10 +2755,10 @@ describe("createHubChatPlatform", () => { toolGrantsForPins: () => [], db: db as never, sessionService, - sidecarRouter: createFakeSidecarRouter({ routableAddresses: [] }), + sidecarRouter, eventCollectors: createFakeEventCollectors(), lifecycle: { idleSleepMs: 60_000 }, - reclaimRetryDelaysMs: [1], + routableWaitDeadlineMs: 5_000, mailDeliveryTimeoutMs: 5_000, workflowAllocationService: allocationService, }); @@ -2876,13 +2885,30 @@ describe("createHubChatPlatform", () => { routableAddresses: [staleAddress], }); + // Models the sidecar registering the relaunch's fresh address the + // moment its deploy actually completes — `sendRunMailWithReclaimRetry` + // now waits on this routing-table transition (CL-7488) instead of a + // fixed backoff, so the retried send needs it to flip before it can + // see the run through. + const allocationService = createFakeWorkflowAllocationService(); + const originalPrepare = + allocationService.prepareProvisionedDeployment.bind(allocationService); + allocationService.prepareProvisionedDeployment = async ( + params: Parameters[0], + ) => { + const result = await originalPrepare(params); + sidecarRouter.routableAddresses.push(result.deploymentAddress); + return result; + }; + const platform = createPlatform({ toolGrantsForPins: () => [], db: db as never, sessionService, sidecarRouter, eventCollectors: createFakeEventCollectors(), - reclaimRetryDelaysMs: [1], + workflowAllocationService: allocationService, + routableWaitDeadlineMs: 5_000, mailDeliveryTimeoutMs: 5_000, }); From 153b81822ea425800a6f44f64c67db515076ce7e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:49:10 -0700 Subject: [PATCH 40/41] Chat: wait for the routing table instead of a fixed backoff before giving up on first-turn delivery sendRunMailWithReclaimRetry used to exhaust a ~7.75s backoff and report the invited agent unreachable while its sidecar was still booting and registering with the hub (CL-7476) -- fast enough to make the fan-out relaunch a perfectly healthy run. It now shares deliverWhenRoutable with the webhook trigger's ingress delivery: wake once on the first unreachable failure (still chasing a mid-wait relaunch by re-resolving the live address inside the retry's send closure), then wait on the hub's routing table up to the same boot-and-register budget (DEFAULT_ROUTABLE_DEADLINE_MS, now exported from @corbits/workflows) before retrying exactly once more. --- packages/chat/src/platform-adapter.ts | 191 +++++++++--------- .../workflows/src/deliver-when-routable.ts | 8 +- packages/workflows/src/index.ts | 1 + 3 files changed, 97 insertions(+), 103 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 1113171348..16b2da3f6b 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,8 +16,10 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, + deliverWhenRoutable, endAgentSessionForRun, ensureRunSession, + isAgentUnreachableError, readFoldedBody, recordAgentSessionAtProvision, resolveNewestProjectedDefinition, @@ -139,13 +141,6 @@ export type CreateHubChatPlatformDeps = { * `ensureAwake` to redeploy a non-routable target before sending. */ lifecycle?: { idleSleepMs: number; sweepIntervalMs?: number }; - /** - * `sendRunMailWithReclaimRetry`'s backoff between retries of a - * mail send that failed with "agent is unreachable" (see - * RECLAIM_RETRY_DELAYS_MS below). Injectable so tests exercise the - * backoff in milliseconds instead of the production ~8s budget. - */ - reclaimRetryDelaysMs?: readonly number[]; /** * Per-attempt wall-clock bound on `sendRunMail` inside * `sendRunMailWithReclaimRetry` (CL-6644). Defaults to @@ -153,6 +148,16 @@ export type CreateHubChatPlatformDeps = { * in milliseconds instead of the production ~30s budget. */ mailDeliveryTimeoutMs?: number; + /** + * How long `sendRunMailWithReclaimRetry` waits for a freshly + * woken/relaunched run's address to show up in the hub's routing + * table before giving up (see `deliverWhenRoutable` in + * `@corbits/workflows`). Defaults to `DEFAULT_ROUTABLE_DEADLINE_MS` + * — the same boot-and-register budget the webhook trigger's ingress + * delivery waits on (CL-7476). Injectable so tests exercise the wait + * in milliseconds instead of the production ~20s budget. + */ + routableWaitDeadlineMs?: number; /** * The invite-launch model fallback (see `./inference-preferences.ts`'s * `createWorkbenchHostInferencePreferencesResolver`): a @@ -409,35 +414,17 @@ export function createHubChatPlatform( const lifecycle = deps.lifecycle !== undefined ? buildLifecycle(deps.lifecycle) : undefined; - // Bounded backoff for a wake racing the sidecar's own post-restart - // reclaim, and separately for a mail delivery racing the same - // window: 250ms, 500ms, 1s, 2s, 4s — a ~7.75s budget, long enough - // for a normal reconnect challenge to settle without leaving a - // sender stuck for much longer than that. - const RECLAIM_RETRY_DELAYS_MS = deps.reclaimRetryDelaysMs ?? [ - 250, 500, 1000, 2000, 4000, - ]; - - // CL-6644: the reclaim-retry loop's own delays only run between - // attempts that already failed LOUD with "agent is unreachable" -- - // they bound nothing about an attempt that instead stalls forever - // (a sidecar ack that never comes, a wedged promise anywhere in - // `sendRunMail`'s call chain) without ever throwing. Each - // attempt gets the same kind of wall-clock bound `wakeByAddressBounded` - // already puts on the wake itself, so that hang becomes a rejection + // CL-6644: a per-attempt bound on `sendRunMail` inside + // `sendRunMailWithReclaimRetry` -- a send that stalls forever (a + // sidecar ack that never comes, a wedged promise anywhere in + // `sendRunMail`'s call chain) without ever throwing gets the same + // kind of wall-clock bound `wakeByAddressBounded` already puts on + // the wake itself, so that hang becomes a rejection // `dispatchTurnBatch`'s catch can report and notify on, instead of a // promise nothing ever settles. const MAIL_DELIVERY_TIMEOUT_MS = deps.mailDeliveryTimeoutMs ?? DEFAULT_WAKE_TIMEOUT_MS; - function sleep(ms: number): Promise { - return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); - } - - function isAgentUnreachable(err: unknown): boolean { - return err instanceof Error && err.message.includes("agent is unreachable"); - } - function isRoutable(address: string): boolean { return deps.sidecarRouter.getRoutableAddresses().includes(address); } @@ -977,83 +964,89 @@ export function createHubChatPlatform( } /** - * `sendRunMail` delivers synchronously against the sidecar's - * current routable set — the same in-memory index `isRoutable` reads - * — so a send that lands in the same post-restart reclaim window - * `wakeByAddress` above tolerates can still fail with "agent is - * unreachable" even right after a successful (or no-op) wake. Each - * retry forces a fresh wake first: if the earlier reclaim tore the - * agent down, this becomes the genuine redeploy that recovers it; if - * the reclaim is still in flight, the wake itself waits it out (or - * redeploys once its own budget is exhausted) and the delay gives it - * more time regardless. Exhausting every delay means the condition - * is not transient and the caller gets a clean `AgentUnreachableError` - * rather than an unhandled 500. + * `sendRunMail` delivers synchronously against the sidecar's current + * routable set — the same in-memory index `isRoutable` reads — so a + * send lands in "agent is unreachable" for two different reasons + * that both need the same treatment: a run whose sidecar just + * finished a provisioned deploy and hasn't registered yet (CL-7476 — + * `deliverWhenRoutable` from `@corbits/workflows` is the exact same + * wait the webhook trigger's ingress delivery already uses for this), + * and a run that this loop's own wake tore down and is mid-reclaim. + * `deliverWhenRoutable` sends once, and — only on an unreachable + * failure — polls the routing table for `current.agentAddress` up to + * its shared deadline (`DEFAULT_ROUTABLE_DEADLINE_MS`, the same + * boot-and-register budget the webhook path waits on) before sending + * exactly once more. * - * CL-7486: a wake this loop runs between retries can itself relaunch - * the run — `wakeByAddress`'s dead-run branch repoints the room's - * stable participant at a fresh address without telling this loop. - * Retrying `sendRunMail` against the address it started with would - * then keep hammering a run that will never become routable again, - * and the wake ahead of the NEXT retry would fail outright (no - * `workbench_launch` row still names that address at all) instead of - * the "unreachable" this loop knows how to ride out. `reresolve` - * re-reads the live address (and its session) off the stable - * participant id before every wake/retry, so this loop always chases - * whichever run is actually current. + * CL-7486: a wake this loop runs on the first failure can itself + * relaunch the run — `wakeByAddress`'s dead-run branch repoints the + * room's stable participant at a fresh address without telling this + * loop. Retrying `sendRunMail` against the address it started with + * would then keep hammering a run that will never become routable + * again. `reresolve` re-reads the live address (and its session) off + * the stable participant id every time `send` runs — including the + * one `deliverWhenRoutable` runs again after the routing-table wait — + * so this loop always chases whichever run is actually current, even + * one relaunched again while it was waiting. */ async function sendRunMailWithReclaimRetry( params: SendRunMailParams, reresolve: () => Promise<{ agentAddress: string; sessionId: string }>, ): Promise>> { let current = params; - let loggedRetryStart = false; - for (let attempt = 0; ; attempt++) { - try { - // CL-7193: `sendRunMail` does a DB write plus a sidecar - // delivery that shouldn't be half-cancelled, and has no signal - // to accept regardless — the signal parameter is unused here, - // same abandon-on-timeout behavior as before. - return await withTimeout( - () => sendRunMail(runMailDeps, current), - MAIL_DELIVERY_TIMEOUT_MS, - `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, - ); - } catch (err) { - const delay = RECLAIM_RETRY_DELAYS_MS[attempt]; - if (!isAgentUnreachable(err) || delay === undefined) { - if (loggedRetryStart) { - wakeLogger.info`mail to ${current.agentAddress} exhausted every reclaim retry; giving up`; - } - if (isAgentUnreachable(err)) { - throw new AgentUnreachableError(current.agentAddress, { - cause: err, - }); + let wakedOnFailure = false; + async function sendCurrent() { + // CL-7193: `sendRunMail` does a DB write plus a sidecar delivery + // that shouldn't be half-cancelled, and has no signal to accept + // regardless — the signal parameter is unused here, same + // abandon-on-timeout behavior as before. + return withTimeout( + () => sendRunMail(runMailDeps, current), + MAIL_DELIVERY_TIMEOUT_MS, + `mail to ${current.agentAddress} did not settle within ${String(MAIL_DELIVERY_TIMEOUT_MS)}ms`, + ); + } + try { + return await deliverWhenRoutable({ + send: async () => { + try { + return await sendCurrent(); + } catch (err) { + if (!isAgentUnreachableError(err) || wakedOnFailure) throw err; + // Wake exactly once, the moment the first attempt reports + // unreachable: if the run had already died, this is the + // genuine redeploy that recovers it; if a deploy is merely + // still booting, `wakeByAddress`/`lifecycle.ensureAwake` + // no-op past the routability check and this becomes a + // cheap reconcile. Re-resolving only after the wake means a + // relaunch it just performed is exactly what the fresh read + // picks up, rather than the address that was current a + // moment before. + wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; waiting for it to become routable`; + wakedOnFailure = true; + await wakeByAddressBounded(current.agentAddress); + const fresh = await reresolve(); + current = { + ...current, + agentAddress: fresh.agentAddress, + sessionId: fresh.sessionId, + }; + throw err; } - throw err; - } - if (!loggedRetryStart) { - wakeLogger.info`mail to ${current.agentAddress} hit "agent is unreachable"; retrying with backoff while the post-restart reclaim settles`; - loggedRetryStart = true; - } - await sleep(delay); - // Wake whatever this loop still believes is live — `wakeByAddress` - // itself tolerates an address one relaunch behind the room's - // current one (its lookup matches either the stable participant - // id or the live run id) — and only THEN re-resolve, since the - // wake just run is exactly the kind of relaunch that can leave - // `current.agentAddress` pointing at a run that no longer exists - // at all. Re-resolving before the wake instead would still hand - // the next retry whatever address was current a moment ago, - // stale again the instant this wake relaunches it. - await wakeByAddressBounded(current.agentAddress); - const fresh = await reresolve(); - current = { - ...current, - agentAddress: fresh.agentAddress, - sessionId: fresh.sessionId, - }; + }, + isRoutable: () => isRoutable(current.agentAddress), + ...(deps.routableWaitDeadlineMs !== undefined + ? { deadlineMs: deps.routableWaitDeadlineMs } + : {}), + }); + } catch (err) { + if (isAgentUnreachableError(err)) { + wakeLogger.info`mail to ${current.agentAddress} exhausted the routable-wait budget; giving up`; + throw new AgentUnreachableError(current.agentAddress, { + cause: err, + }); } + throw err; } } diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts index a48e994038..666cba20b5 100644 --- a/packages/workflows/src/deliver-when-routable.ts +++ b/packages/workflows/src/deliver-when-routable.ts @@ -3,9 +3,9 @@ // sidecar has finished booting and registering with the hub, so the send // fails with "agent is unreachable" even though nothing is actually wrong // — the address just isn't routable yet. The webhook trigger's ingress -// delivery goes through this helper; chat keeps its own existing -// wake-and-retry loop (`sendRunMailWithReclaimRetry`) instead. -const DEFAULT_DEADLINE_MS = 20_000; +// delivery goes through this helper; chat's `sendRunMailWithReclaimRetry` +// (CL-7488) shares this same budget rather than declaring its own. +export const DEFAULT_ROUTABLE_DEADLINE_MS = 20_000; const DEFAULT_POLL_INTERVAL_MS = 250; export function isAgentUnreachableError(err: unknown): boolean { @@ -48,7 +48,7 @@ export async function deliverWhenRoutable( if (!isUnreachable(err)) { throw err; } - const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS); + const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; const sleep = opts.sleep ?? defaultSleep; while (!opts.isRoutable()) { diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 64bfbdff98..ba84a329b9 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -65,6 +65,7 @@ export { export { deliverWhenRoutable, isAgentUnreachableError, + DEFAULT_ROUTABLE_DEADLINE_MS, type DeliverWhenRoutableOptions, } from "./deliver-when-routable"; export { From 320deb222d83895a6db13c032a16fd2c2f2b9426 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 14:53:02 -0700 Subject: [PATCH 41/41] Fix formatting --- packages/chat/src/platform-adapter.ts | 45 ++++++++++--------- .../workflows/src/deliver-when-routable.ts | 3 +- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 16b2da3f6b..745ac214f3 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -1421,27 +1421,30 @@ export function createHubChatPlatform( ? { references: input.content.references } : {}), }; - const sent = await sendRunMailWithReclaimRetry(withThreading, async () => { - const relive = await requireLive(input.workbenchId, input.tenantId); - try { - await ensureRunSession({ - db: deps.db, - eventCollectors: deps.eventCollectors, - runId: relive.run.id, - }); - } catch (err) { - reportError(err, { - operation: "chat.sendMail.ensureRunSession", - tenantId: input.tenantId, - agentId: relive.binding.liveAddress, - extra: { runId: relive.run.id }, - }); - } - return { - agentAddress: relive.binding.liveAddress, - sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), - }; - }); + const sent = await sendRunMailWithReclaimRetry( + withThreading, + async () => { + const relive = await requireLive(input.workbenchId, input.tenantId); + try { + await ensureRunSession({ + db: deps.db, + eventCollectors: deps.eventCollectors, + runId: relive.run.id, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendMail.ensureRunSession", + tenantId: input.tenantId, + agentId: relive.binding.liveAddress, + extra: { runId: relive.run.id }, + }); + } + return { + agentAddress: relive.binding.liveAddress, + sessionId: await resolveRunSessionIdOrThrow(deps.db, relive.run), + }; + }, + ); lifecycle?.recordActivity(deliveryAddress); diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts index 666cba20b5..cee443ed92 100644 --- a/packages/workflows/src/deliver-when-routable.ts +++ b/packages/workflows/src/deliver-when-routable.ts @@ -48,7 +48,8 @@ export async function deliverWhenRoutable( if (!isUnreachable(err)) { throw err; } - const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); + const deadline = + Date.now() + (opts.deadlineMs ?? DEFAULT_ROUTABLE_DEADLINE_MS); const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; const sleep = opts.sleep ?? defaultSleep; while (!opts.isRoutable()) {