From b1e6caa5d695b74c2023ec24af33b09afb344754 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:15 -0700 Subject: [PATCH 01/28] Add test for workflowDeployBody's catalog-offering shape --- scripts/e2e/harness.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 5b4d14800..37f4f93e4 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -11,6 +11,7 @@ import { createCleanupHarness, parseEnvFileDatabaseUrl, runCleanups, + workflowDeployBody, type SpawnedApp, } from "./harness.ts"; @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => { }); }); +describe("workflowDeployBody", () => { + test("carries catalog offering ids, never a raw provider/apiKey source", () => { + const body = workflowDeployBody({ + assetId: "asset-1", + commitSha: "deadbeef", + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).toEqual({ + source: { + kind: "asset", + assetId: "asset-1", + package: { format: "source", commitSha: "deadbeef" }, + }, + entry: (body as { entry: string }).entry, + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).not.toHaveProperty("sources"); + expect(body).not.toHaveProperty("defaultSource"); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( From fae77c0682222b8149cb08b6b169e1caab79d33b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:23 -0700 Subject: [PATCH 02/28] e2e: deploy workflows by catalog offering ids The native workflow deploy route now requires sourceOfferingIds/ defaultSourceOfferingId resolved against the tenant's own catalog, rather than a raw provider/baseURL/apiKey source chain. Update workflowDeployBody accordingly, and extract the zero-cost noop catalog chain smoke-webhook.test.ts already built (an anthropic- plugin provider pointed at the hub's noop-inference endpoint) into a shared seedNoopCatalogOffering harness helper. Every deploying e2e suite now seeds or resolves a real catalog offering before deploying: walking-skeleton keeps a never-real placeholder host (assertNeverRealProvider still guards it), chat resolves the offering seedCatalog already planted rather than seeding a second chain, and heartbeat/workbench-digest/smoke-webhook seed the noop chain via the shared helper. cl-6324-launch-proof's section-mode deploy now reuses the offering id the proof already resolved when it narrowed the bench catalog to its pinned model. --- scripts/e2e/chat.test.ts | 48 +++++++++-- scripts/e2e/cl-6324-launch-proof.ts | 29 ++++--- scripts/e2e/harness.ts | 121 +++++++++++++++++++++++---- scripts/e2e/heartbeat.test.ts | 19 +++-- scripts/e2e/smoke-webhook.test.ts | 89 +++----------------- scripts/e2e/walking-skeleton.test.ts | 23 +++-- scripts/e2e/workbench-digest.test.ts | 19 +++-- 7 files changed, 219 insertions(+), 129 deletions(-) diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index deea7c826..da9b255c8 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { log: () => undefined, }); + // The echo deploy resolves against the catalog offering `seedCatalog` + // just planted for the same provider/model this workflow declares, + // rather than seeding a second inference chain. + const echoModelsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/models`, + undefined, + user1.cookies, + ); + expectStatus("list catalog models", echoModelsListed, 200); + const echoModelRows = ( + echoModelsListed.data as { data: { id: string; canonicalName: string }[] } + ).data; + const echoModelId = echoModelRows.find( + (row) => row.canonicalName === "claude-sonnet-5", + )?.id; + if (echoModelId === undefined) { + throw new Error( + `no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`, + ); + } + + const echoOfferingsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings`, + undefined, + user1.cookies, + ); + expectStatus("list catalog offerings", echoOfferingsListed, 200); + const echoOfferingRows = ( + echoOfferingsListed.data as { data: { id: string; modelId: string }[] } + ).data; + const echoOfferingId = echoOfferingRows.find( + (row) => row.modelId === echoModelId, + )?.id; + if (echoOfferingId === undefined) { + throw new Error( + `no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`, + ); + } + // Seed the echo workflow as a deployed, invitable definition: the // same asset-publish → git-token → smart-HTTP push → native deploy // path `scripts/e2e/walking-skeleton.test.ts` proves end to end. @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { workflowDeployBody({ assetId: echoAssetId, commitSha: echoPushed.commitSha, - sourceId: "src-echo-e2e", - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [echoOfferingId], + defaultSourceOfferingId: echoOfferingId, }), user1.cookies, ); diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index 674315090..ef3be6297 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,7 +369,7 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - await hop("narrow the bench catalog to the pinned model", async () => { + const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { await plantGrant("model-offering:*", "read"); await plantGrant("model-offering:*", "manage"); await plantGrant("model:*", "read"); @@ -409,8 +409,13 @@ async function main(): Promise { "data", "model offerings", ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; for (const offering of offeringRows) { - if (offering.modelId === pinned.id || offering.disabled) continue; + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; const patched = await api( hub.baseUrl, "PATCH", @@ -420,10 +425,17 @@ async function main(): Promise { ); expectStatus(`disable offering ${offering.id}`, patched, 200); } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } console.log( ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); + return pinnedOffering.id; }); const assistantDefinitionId = await hop( @@ -694,7 +706,6 @@ async function main(): Promise { // own durable event log, which is the artefact the milestone's // `RunStarted` assertion was always about. const SECTION_ASSET_NAME = "cl6324-section"; - const SECTION_SOURCE_ID = "cl6324-section-source"; const SECTION_TURN_TIMEOUT_MS = 180_000; const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`; @@ -780,16 +791,8 @@ async function main(): Promise { package: { format: "source", commitSha: pushed.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SECTION_SOURCE_ID, - provider: model.provider, - baseURL: model.baseURL, - apiKey: model.apiKey, - model: model.model, - }, - ], - defaultSource: SECTION_SOURCE_ID, + sourceOfferingIds: [pinnedOfferingId], + defaultSourceOfferingId: pinnedOfferingId, }, user.cookies, ); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 099a7a210..27cd39204 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -653,16 +653,17 @@ export async function pushWorkflowSource(options: { /** * The deploy body a code-sourced asset deployment takes: the pushed * commit is the definition's pin, and the entry names the - * `interchange.workflow` module the sidecar evaluates. + * `interchange.workflow` module the sidecar evaluates. The native route + * resolves inference against the tenant's own catalog, so the caller + * supplies the ordered catalog offering ids to deploy against rather + * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` + * for the zero-cost way to obtain one. */ export function workflowDeployBody(options: { assetId: string; commitSha: string; - sourceId: string; - provider: string; - baseURL: string; - apiKey: string; - model: string; + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; }): Record { return { source: { @@ -671,15 +672,103 @@ export function workflowDeployBody(options: { package: { format: "source", commitSha: options.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: options.sourceId, - provider: options.provider, - baseURL: options.baseURL, - apiKey: options.apiKey, - model: options.model, - }, - ], - defaultSource: options.sourceId, + sourceOfferingIds: options.sourceOfferingIds, + defaultSourceOfferingId: options.defaultSourceOfferingId, }; } + +/** + * Plants the zero-cost catalog chain every deploying e2e suite needs: a + * catalog model, an `anthropic`-plugin provider/credential pointed at + * `noopBaseUrl` (never a real provider host — enforced below), and the + * catalog offering joining them. Returns the offering id a deploy's + * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. + * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + */ +export async function seedNoopCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + noopBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringFieldOf(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringFieldOf(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringFieldOf( + credential.data, + "id", + "create credential", + ); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.noopBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringFieldOf( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; +} + +function stringFieldOf(data: unknown, field: string, what: string): string { + if (typeof data === "object" && data !== null && field in data) { + const value = (data as Record)[field]; + if (typeof value === "string" && value !== "") return value; + } + throw new Error( + `${what}: missing string field "${field}": ${JSON.stringify(data)}`, + ); +} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index c19a6b6a2..9aefb900f 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -203,16 +204,22 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-heartbeat-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 1d248defa..760aca6a3 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -30,7 +30,6 @@ import { } from "../../workflows/heartbeat/src/index.ts"; import { api, - assertNeverRealProvider, connectE2eDb, createCleanupHarness, e2eDatabaseUrl, @@ -39,6 +38,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -138,77 +138,17 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real - // model. Mirrors `workbench-digest.test.ts`'s noop catalog seeding. + // model. const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - assertNeverRealProvider(noopBaseUrl, "noop catalog provider baseURL"); - await hop("noop catalog seeding", async () => { - const model = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringField(model.data, "id", "create catalog model"); - - const provider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringField(provider.data, "id", "create provider"); - - const credential = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringField( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: noopBaseUrl, - credentialId, - }, + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies2) => + api(hub.baseUrl, method, path, body, cookies2), + tenantId, cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringField( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - }); + noopBaseUrl, + }), + ); const assetName = "heartbeat"; const { assetId, commitSha } = await hop( @@ -256,16 +196,11 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); const definitionId = await hop("workflow deploy", async () => { - const sourceId = "src-smoke-webhook-e2e"; - assertNeverRealProvider(noopBaseUrl, "workflow deploy source baseURL"); const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: noopBaseUrl, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; for (;;) { diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 833612dd6..8937f9b5e 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -35,6 +35,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -199,20 +200,30 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { }, ); + // Hop: noop catalog seeding. The native deploy resolves inference + // against a real catalog offering; this suite's chosen offering + // still names an unreachable placeholder host, since deployment + // never calls inference and full run-completion is not asserted. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: "https://inference.invalid", + }), + ); + // Hop: workflow deploy via the native deploy API. Retries while // the hub still answers 502 (the sidecar's dial-in may not have // completed yet); any other failure is final. The inference // source is a placeholder — deployment does not call inference. const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-echo-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index ecf2dc56e..8917bf823 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -207,16 +208,22 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-workbench-digest-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; From b03ce16b54f9de7e1bb8f14a7efce6a50cd7f9ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:43:45 -0700 Subject: [PATCH 03/28] Fix formatting --- scripts/e2e/cl-6324-launch-proof.ts | 125 ++++++++++++++-------------- scripts/e2e/harness.ts | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index ef3be6297..80766c99d 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,74 +369,77 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { - await plantGrant("model-offering:*", "read"); - await plantGrant("model-offering:*", "manage"); - await plantGrant("model:*", "read"); + const pinnedOfferingId = await hop( + "narrow the bench catalog to the pinned model", + async () => { + await plantGrant("model-offering:*", "read"); + await plantGrant("model-offering:*", "manage"); + await plantGrant("model:*", "read"); - const models = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench catalog models", models, 200); - const modelRows = arrayField(models.data, "data", "catalog models") as { - id: string; - canonicalName: string; - }[]; - const pinned = modelRows.find( - (row) => row.canonicalName === proofModelSource.model, - ); - if (pinned === undefined) { - throw new Error( - `the bench catalog carries no model named ${proofModelSource.model}; ` + - `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + const models = await api( + hub.baseUrl, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, + undefined, + user.cookies, ); - } - - const offerings = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench model offerings", offerings, 200); - const offeringRows = arrayField( - offerings.data, - "data", - "model offerings", - ) as { id: string; modelId: string; disabled: boolean }[]; - let pinnedOffering: { id: string; modelId: string } | undefined; - for (const offering of offeringRows) { - if (offering.modelId === pinned.id) { - pinnedOffering = offering; - continue; + expectStatus("list the bench catalog models", models, 200); + const modelRows = arrayField(models.data, "data", "catalog models") as { + id: string; + canonicalName: string; + }[]; + const pinned = modelRows.find( + (row) => row.canonicalName === proofModelSource.model, + ); + if (pinned === undefined) { + throw new Error( + `the bench catalog carries no model named ${proofModelSource.model}; ` + + `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + ); } - if (offering.disabled) continue; - const patched = await api( + + const offerings = await api( hub.baseUrl, - "PATCH", - `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, - { disabled: true }, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, + undefined, user.cookies, ); - expectStatus(`disable offering ${offering.id}`, patched, 200); - } - if (pinnedOffering === undefined) { - throw new Error( - `no catalog offering resolves the pinned model ${proofModelSource.model} ` + - `(id ${pinned.id})`, + expectStatus("list the bench model offerings", offerings, 200); + const offeringRows = arrayField( + offerings.data, + "data", + "model offerings", + ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; + for (const offering of offeringRows) { + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; + const patched = await api( + hub.baseUrl, + "PATCH", + `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, + { disabled: true }, + user.cookies, + ); + expectStatus(`disable offering ${offering.id}`, patched, 200); + } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } + console.log( + ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); - } - console.log( - ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + - `(${String(offeringRows.length - 1)} other offerings disabled)`, - ); - return pinnedOffering.id; - }); + return pinnedOffering.id; + }, + ); const assistantDefinitionId = await hop( "PROOF 1 — 'assistant' is invitable tenant-wide", diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 27cd39204..cebc4e53d 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -760,7 +760,9 @@ export async function seedNoopCatalogOffering(options: { cookies, ); expectStatus("create catalog offering", offering, 201); - return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; + return { + offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), + }; } function stringFieldOf(data: unknown, field: string, what: string): string { From f981798f2aad5db7e0a8c14a0234e8e1860c4cbe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:53:52 -0700 Subject: [PATCH 04/28] Add tests for deploying default workflows by catalog offering ids seedTenant currently deploys every default workflow with a bare provider/baseURL/apiKey source, but the hub's deploy route now requires sourceOfferingIds/defaultSourceOfferingId resolved from the tenant's catalog. Update the seeding, onboarding, and hub config test suites to expect the new offering-id based deploy body and provider/model-only ModelSource shape ahead of the implementation change. --- apps/hub/test/config.test.ts | 1 - .../test/complete-credential.test.ts | 55 +++++++-- packages/onboarding/test/provision.test.ts | 101 +++++++++++++++++ .../test/routine-launchability.test.ts | 2 - packages/seeding/test/seed.test.ts | 106 +++++++++++++++--- 5 files changed, 237 insertions(+), 28 deletions(-) diff --git a/apps/hub/test/config.test.ts b/apps/hub/test/config.test.ts index 9d3435117..b4d41908e 100644 --- a/apps/hub/test/config.test.ts +++ b/apps/hub/test/config.test.ts @@ -277,7 +277,6 @@ describe("readHubConfig", () => { expect(config.seedModel).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", apiKey: "sk-ant-test", }); }); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c9ac97c2d..8a4ae6a78 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -63,6 +63,36 @@ function seedHandshake(method: string, path: string) { return { ...handshake, cookies: [] }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461). A test that +// never seeds its own catalog (it stubs `seedCatalogFn` instead) still +// needs one listable offering for the "assistant" workflow's own deploy +// to succeed — this fixed single row is that. +function fixedOfferingsResponse() { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function principalsResponse() { return { status: 200, @@ -165,8 +195,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-ant", }); }); @@ -192,8 +220,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "llama3.2", - baseURL: "http://localhost:11434/v1", - apiKey: "ollama", }); }); @@ -339,7 +365,12 @@ describe("modelSourceFor", () => { expect(result.model).toBe("gpt-oss:20b"); }); - test("ollama's baseURLOverride is normalized to the /v1 form", async () => { + // `modelSourceFor` resolves a provider/model pair only — the deploy + // itself resolves inference from the tenant's own catalog offerings + // (CL-7461), so a `baseURLOverride` here is accepted (every caller + // already resolved one for `seedCatalog`'s earlier catalog plant) but + // never affects this resolution. + test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { return resolvedCatalogResponse([ @@ -365,8 +396,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "qwen3.8:27b", - baseURL: "https://home-mac.example.ts.net/v1", - apiKey: "ollama", }); }); }); @@ -808,6 +837,12 @@ describe("completeCredentialSetup", () => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}`) { return tenantResponse(); } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) @@ -1791,6 +1826,12 @@ describe("ensureSeeded (the slow half)", () => { let deploymentCreatePosts = 0; const api: ApiCall = async (method, path, body) => { + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index af1594cbc..b1feabe08 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -87,6 +87,41 @@ function collector() { return { lines, log: (line: string) => lines.push(line) }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461); every test +// that seeds a real default workflow needs at least one listable, since +// none of these tests seed a catalog of its own. +function catalogOfferingsResponse( + method: string, + path: string, + tenantId: string, +): { status: number; data: unknown; cookies: string[] } | undefined { + if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + return undefined; + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function firstLoginSeedHub(args: { expectedParentId?: string }) { let principalsCalls = 0; const startedRuns: string[] = []; @@ -206,6 +241,33 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { ) { return { status: 200, data: [], cookies: [] }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/workflows/definitions`) @@ -638,6 +700,33 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if (method === "GET" && path === "/api/me/principals") { return membership(); } @@ -890,6 +979,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1019,6 +1110,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1133,6 +1226,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1243,6 +1338,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1485,6 +1582,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1832,6 +1931,8 @@ describe("provisionPersonalTenantIfNeeded", () => { [], ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, diff --git a/packages/seeding/test/routine-launchability.test.ts b/packages/seeding/test/routine-launchability.test.ts index 427fa0b00..746e0e202 100644 --- a/packages/seeding/test/routine-launchability.test.ts +++ b/packages/seeding/test/routine-launchability.test.ts @@ -27,8 +27,6 @@ import { const FAKE_MODEL: ModelSource = { provider: "ollama", model: "qwen-test", - baseURL: "http://localhost:11434", - apiKey: "test-key", }; type SerializedStepDefinition = { diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index b6291fc94..720250165 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -38,10 +38,38 @@ installDisposableHubDataDir(); const MODEL = { provider: "anthropic", model: "claude-sonnet-4-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-test", }; +// The tenant's one real catalog offering — every `seedTenant` test that +// deploys a real (non-noop-pinned) workflow needs at least one of these +// listable, since `ensureDeployment` now resolves its deploy source from +// the tenant's catalog rather than a bare `ModelSource`. +const OFFERING_ID = "off_1"; + +function offeringsPage( + ids: readonly string[] = [OFFERING_ID], +): { status: number; data: unknown } { + return { + status: 200, + data: { + data: ids.map((id, index) => ({ + id, + tenantId: TENANT_ID, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + })), + nextCursor: null, + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -98,6 +126,40 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/workflows/deployments` ) return { status: 200, data: [] }; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return offeringsPage(); + // The noop catalog chain `ensureNoopCatalogOffering` plants for a + // modelSource-pinned workflow (heartbeat) — every seedTenant test that + // deploys one needs this fresh chain to succeed, so it lives in the + // shared fallback rather than every such test repeating it. + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/models`) + return { status: 201, data: catalogModelRow("mdl_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_noop", "prv_noop", "noop-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_noop", "noop", "cre_noop", "anthropic"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 201, + data: catalogOfferingRow("off_noop", "mdl_noop", "cpv_noop"), + }; const handshake = pristineScheduledDefinitionHandshake(method, path); if (handshake) return handshake; return undefined; @@ -860,11 +922,9 @@ describe("seedTenant", () => { expect(assistant?.displayName).toBe("Myra"); }); - test("NOOP_MODEL_SOURCE resolves to the hub's own noop-inference endpoint", () => { - const resolved = NOOP_MODEL_SOURCE("http://localhost:3000"); - expect(resolved.baseURL).toBe( - "http://localhost:3000/api/chat/noop-inference", - ); + test("NOOP_MODEL_SOURCE names the noop provider/model pair", () => { + const resolved = NOOP_MODEL_SOURCE(); + expect(resolved.provider).toBe("anthropic"); expect(resolved.model).toBe("noop"); }); @@ -934,8 +994,8 @@ describe("seedTenant", () => { (w) => w.assetName === "heartbeat", ); if (!heartbeat) throw new Error("expected the heartbeat workflow"); - const resolved = heartbeat.modelSource?.("http://localhost:3000"); - expect(resolved).toEqual(NOOP_MODEL_SOURCE("http://localhost:3000")); + const resolved = heartbeat.modelSource?.(); + expect(resolved).toEqual(NOOP_MODEL_SOURCE()); }); test("fresh run pushes, deploys, and confirms the heartbeat workflow against the noop source", async () => { @@ -1012,12 +1072,17 @@ describe("seedTenant", () => { expect(definition.triggers[0]?.to).toBe(`heartbeat@${TENANT_DOMAIN}`); expect(definition.stepOrder).toEqual(["heartbeat"]); - // The deploy's own source, not the tenant's real MODEL, is what - // proves the noop pin took effect: it must name the noop provider - // fixture, not the ordinary anthropic/claude-sonnet-4-5 model this - // test file's `args()` helper hands every other workflow. - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).toBe("noop"); + // The deploy's own source offering, not the tenant's real catalog + // offering, is what proves the noop pin took effect: it must name + // the dedicated noop offering `ensureNoopCatalogOffering` plants, + // never the tenant's own `OFFERING_ID` this test file's `baseRoutes` + // fixture hands every real workflow. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual(["off_noop"]); + expect(deployedBody.defaultSourceOfferingId).toBe("off_noop"); const output = lines.join("\n"); expect(output).toContain("deployed workflow heartbeat as dep_3"); @@ -1135,9 +1200,14 @@ describe("seedTenant", () => { ]); expect(definition.stepOrder).toEqual(["workbench-digest"]); - // Defaults deploy against the tenant's real model (not noop). - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).not.toBe("noop"); + // Defaults deploy against the tenant's real catalog offering, never + // the noop one. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual([OFFERING_ID]); + expect(deployedBody.defaultSourceOfferingId).toBe(OFFERING_ID); const output = lines.join("\n"); expect(output).toContain("deployed workflow workbench-digest as dep_4"); From 1ce35c4fa2202730eabe41578a95978cff05c8cf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:00 -0700 Subject: [PATCH 05/28] Seeding: deploy default workflows by catalog offering ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureDeployment posted a bare {provider, baseURL, apiKey, model} source, but the hub's deploy route requires sourceOfferingIds and a defaultSourceOfferingId resolved from the tenant's catalog, so seeding a connect flow's "assistant" workflow 400'd. seedTenant now resolves the tenant's real catalog offerings (sorted by priority, lowest wins the default) for a normal workflow, and plants a dedicated noop catalog offering for a modelSource-pinned one (heartbeat), pointed at the hub's own noop-inference endpoint. ModelSource shrinks to a bare provider/model pair — the baseURL/apiKey it used to carry only mattered for the old deploy body, and callers that still need a key for seeding a catalog (system-seed's root tenant, config's env-key auto-plant) keep it on their own local type instead. --- apps/hub/src/config.ts | 8 +- apps/hub/src/system-seed.ts | 23 +- .../onboarding/src/complete-credential.ts | 51 +-- packages/seeding/src/seed.ts | 316 +++++++++++++++--- 4 files changed, 305 insertions(+), 93 deletions(-) diff --git a/apps/hub/src/config.ts b/apps/hub/src/config.ts index d9fdad334..1d3afcf79 100644 --- a/apps/hub/src/config.ts +++ b/apps/hub/src/config.ts @@ -272,7 +272,6 @@ function parsePositiveMsEnv( const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; // Matches boot-time seeding's own defaults (`system-seed.ts`) exactly, // so a zero-.env-edit local checkout that seeds its admin account @@ -282,10 +281,14 @@ const DEFAULT_PLANT_ADMIN_EMAIL = "alice@example.com"; const DEFAULT_PLANT_ADMIN_PASSWORD = "password123"; const DEFAULT_PLANT_ORG_SLUG = "workbench"; +/** The root tenant's boot-time seed model: a provider/model pair + * `seedTenant` names in every deployed definition, plus the real (or + * placeholder) key `seedCatalog` plants a launchable credential with. No + * `baseURL` — a workflow deploy resolves inference from the tenant's + * catalog offerings, never a bare source tuple (CL-7461). */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; readonly apiKey: string; }; @@ -624,7 +627,6 @@ function seedModelFrom(parsed: ParsedHubEnv): ModelSource | undefined { return { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey, }; } diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 7f9291fc5..52f0ff8e6 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -26,7 +26,6 @@ import { publishCorbitsToolsRegistry, DEFAULT_WORKFLOWS, PLACEHOLDER_CATALOG_API_KEY, - type ModelSource, } from "@corbits/seeding"; import { createHubAPI, signIn, type ApiCall } from "@corbits/hub-api-client"; import { findPersonalTenant } from "@workbench/onboarding"; @@ -43,14 +42,26 @@ const DEFAULT_POLL_INTERVAL_MS = 2_000; // and the catalog is still browsable, just not launchable. const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; -function resolvedModel(seedModel: ModelSource | undefined): ModelSource { +/** + * The root tenant's seed model: a provider/model pair for + * `seedTenant`'s deployed definitions, plus the real (or placeholder) + * API key `seedCatalog` needs to plant a launchable credential. Distinct + * from `@corbits/seeding`'s `ModelSource`, which carries no key — + * `seedTenant` never needs one; the deploy itself resolves inference + * from the tenant's catalog offerings (CL-7461). + */ +export type SeedModelConfig = { + readonly provider: string; + readonly model: string; + readonly apiKey: string; +}; + +function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey: PLACEHOLDER_CATALOG_API_KEY, } ); @@ -60,7 +71,7 @@ export type SystemSeedDeps = { baseUrl: string; orgSlug: string; admin: { email: string; password: string }; - seedModel?: ModelSource; + seedModel?: SeedModelConfig; deadlineMs?: number; pollIntervalMs?: number; }; @@ -116,7 +127,7 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { principalId: tenant.principalId, domain: tenant.tenantDomain, }, - model, + model: { provider: model.provider, model: model.model }, pushWorkflow, log: (line) => log.info`${line}`, workflows: DEFAULT_WORKFLOWS, diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index 5af80e3f7..f5f4a930a 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -60,10 +60,7 @@ import { TenantResponse, paginatedSchema, } from "@intx/types"; -import { - ollamaOpenAICompatBaseURL, - type SupportedCredentialProvider, -} from "@corbits/connections/credential-test"; +import { type SupportedCredentialProvider } from "@corbits/connections/credential-test"; import { preferCompletionCapable } from "@corbits/connections/model-capability"; import { CATALOG_SEEDS, @@ -280,13 +277,8 @@ async function resolveOllamaModelSource( api: ApiCall, cookies: string[], tenantId: string, - apiKey: string, - baseURLOverride: string | undefined, ): Promise { const catalogSeed = CATALOG_SEEDS.ollama; - const baseURL = ollamaOpenAICompatBaseURL( - baseURLOverride ?? catalogSeed.provider.baseURL, - ); const response = await api( "GET", @@ -368,39 +360,32 @@ async function resolveOllamaModelSource( return { provider: winner.plugin, model: winner.canonicalName, - baseURL, - apiKey, }; } -/** The `ModelSource` `ensureSeeded` deploys every default workflow - * against. Every provider but `ollama` has a fixed, always-available - * curated model list (`CATALOG_SEEDS`), so its curated default is a safe - * pin. `ollama` is the one provider whose actual model list is - * per-instance and can diverge from that curated name entirely - * (CL-6366) — its resolution defers to `resolveOllamaModelSource`, which - * reads back what `seedCatalog` actually found on the instance rather - * than repeating the static pin. `baseURLOverride` is the - * configurable-base-URL seam every provider but `ollama` ignores (a - * fixed origin); for `ollama` it is the root the person actually pointed - * their instance at, normalized to the OpenAI-compatible `/v1` form this - * deploys against. */ +/** The `ModelSource` `ensureSeeded` names in every deployed definition's + * inference preferences — never what the deploy itself resolves + * inference against (the tenant's catalog offerings, CL-7461). Every + * provider but `ollama` has a fixed, always-available curated model list + * (`CATALOG_SEEDS`), so its curated default is a safe pin. `ollama` is + * the one provider whose actual model list is per-instance and can + * diverge from that curated name entirely (CL-6366) — its resolution + * defers to `resolveOllamaModelSource`, which reads back what + * `seedCatalog` actually found on the instance rather than repeating the + * static pin. `apiKey`/`baseURLOverride` are accepted only because every + * caller already resolved them for `seedCatalog`'s own credential/catalog + * plant, which runs ahead of this in the same connect flow; this + * function itself needs neither. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - apiKey: string, - baseURLOverride?: string, + _apiKey: string, + _baseURLOverride?: string, ): Promise { if (provider === "ollama") { - return resolveOllamaModelSource( - api, - cookies, - tenantId, - apiKey, - baseURLOverride, - ); + return resolveOllamaModelSource(api, cookies, tenantId); } const catalogSeed = CATALOG_SEEDS[provider]; @@ -413,8 +398,6 @@ export async function modelSourceFor( return { provider: catalogSeed.provider.plugin, model: defaultModel.canonicalName, - baseURL: catalogSeed.provider.baseURL, - apiKey, }; } diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 2b34e98c3..20e350303 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -141,11 +141,6 @@ const DILIGENCE_BRIEF_TURN_TIMEOUT_MS = 2 * 60 * 1000; const RUN_START_TIMEOUT_MS = 30_000; const RUN_POLL_INTERVAL_MS = 1000; -// The deploy source the per-step agents launch against. The id is the -// routing key `defaultSource` must name; with exactly one source there -// is exactly one honest value for it. -const SEED_SOURCE_ID = "default"; - // The provider/model pair `noop-inference` (packages/chat/src/noop-inference.ts) // answers for any request, regardless of what is actually sent — the // route ignores its body and `x-api-key` entirely. Naming a distinct @@ -154,24 +149,31 @@ const SEED_SOURCE_ID = "default"; // UI and logs. const NOOP_PROVIDER = "anthropic"; const NOOP_MODEL = "noop"; +// The catalog provider/credential name a noop-pinned deployment's own +// offering is planted under — distinct from any real curated provider +// name (`CATALOG_SEEDS`'s keys), so `ensureNoopCatalogOffering` never +// collides with a tenant's real catalog and is trivially recognizable +// in the hub's own catalog UI. +const NOOP_CATALOG_PROVIDER_NAME = "noop"; +// Priority is meaningless for this offering — it is never included in +// a real workflow's `sourceOfferingIds` (`ensureNoopCatalogOffering`'s +// id is threaded through explicitly, never discovered by sorting), so +// any fixed value is honest here. +const NOOP_OFFERING_PRIORITY = 0; /** - * A `ModelSource` pointed at the hub's own `noop-inference` endpoint + * The provider/model pair naming the hub's own `noop-inference` endpoint * instead of a real provider — the same substitution * `packages/chat/src/platform-adapter.ts`'s `noopSourcesOverride` makes - * for workbench-host launches, reused here so a workflow deployed with - * this source resolves every turn instantly against a constant, - * locally served reply and never reaches a real model. `hubUrl` is the - * same base URL `seedTenant` already receives, so no new configuration - * is required to use it. + * for workbench-host launches, reused here so a workflow's rendered + * definition names this pair in its inference preferences. The actual + * deploy source is a catalog offering pinned at this same endpoint + * (`ensureNoopCatalogOffering`), not this pair directly — a workflow + * deployment resolves inference from catalog offering ids, never a bare + * provider/model/baseURL/apiKey tuple (CL-7461). */ -export function NOOP_MODEL_SOURCE(hubUrl: string): ModelSource { - return { - provider: NOOP_PROVIDER, - model: NOOP_MODEL, - baseURL: `${hubUrl}/api/chat/noop-inference`, - apiKey: "noop", - }; +export function NOOP_MODEL_SOURCE(): ModelSource { + return { provider: NOOP_PROVIDER, model: NOOP_MODEL }; } const GitTokenMintResponse = type({ id: "string", secret: "string" }); @@ -191,11 +193,18 @@ const WorkflowRunTriggerResponse = type({ messageId: "string", }); +/** + * A provider/model pair a deployed workflow's rendered definition names + * in its inference preferences (`DefaultWorkflow.buildJson`'s second + * argument). What a deployment actually resolves inference against is + * the tenant's catalog offerings, resolved separately by `ensureDeployment` + * (or, for a noop-pinned workflow, `ensureNoopCatalogOffering`) — this + * type carries no `baseURL`/`apiKey` because `seedTenant` never needs + * either (CL-7461). + */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; - readonly apiKey: string; }; /** @@ -240,9 +249,9 @@ export type DefaultWorkflow = { * Renders the definition's JSON given the tenant's mail domain and the * ordered provider/model preferences to deploy against. Takes the bare * preference list — never a full `ModelSource` — so this same - * function serves both `seedTenant`'s HTTP-deploy path (which also - * needs a `ModelSource`'s `baseURL`/`apiKey` for the deployment's - * `sources`, resolved separately) and a native in-process deploy path + * function serves both `seedTenant`'s HTTP-deploy path (whose actual + * deploy source is a set of catalog offering ids, resolved separately + * by `ensureDeployment`) and a native in-process deploy path * (`apps/hub/src/templates/block-workflows.ts`) that only ever has the * tenant's real, possibly multi-entry inference preferences on hand. */ @@ -251,15 +260,16 @@ export type DefaultWorkflow = { inferencePreferences: readonly InferencePreference[], ) => string; /** - * Overrides the deploy's inference source for this workflow only, - * given the hub's own base URL. Present on the catalog-test workflow - * `heartbeat`, which must stay free to run continuously: it names - * `NOOP_MODEL_SOURCE` instead of the tenant's real catalog model. - * Absent on every conversational workflow and on the seeded - * workbench-digest automation, which deploy against the tenant's real - * model. + * Overrides the deploy's inference source for this workflow only. + * Present on the catalog-test workflow `heartbeat`, which must stay + * free to run continuously: it names `NOOP_MODEL_SOURCE` instead of + * the tenant's real catalog model, and `seedTenant` deploys it against + * a dedicated noop catalog offering (`ensureNoopCatalogOffering`) + * rather than the tenant's own resolved offerings. Absent on every + * conversational workflow and on the seeded workbench-digest + * automation, which deploy against the tenant's real model. */ - modelSource?: (hubUrl: string) => ModelSource; + modelSource?: () => ModelSource; /** * When true, PUT the authored definition to `stopped` after deploy so * a native ScheduleTrigger does not fire every tenant at the next @@ -901,7 +911,8 @@ async function ensureDeployment( assetId: string; assetName: string; commitSha: string; - model: ModelSource; + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; }, log: (line: string) => void, ): Promise { @@ -950,16 +961,8 @@ async function ensureDeployment( package: { format: "source", commitSha: args.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SEED_SOURCE_ID, - provider: args.model.provider, - baseURL: args.model.baseURL, - apiKey: args.model.apiKey, - model: args.model.model, - }, - ], - defaultSource: SEED_SOURCE_ID, + sourceOfferingIds: args.sourceOfferingIds, + defaultSourceOfferingId: args.defaultSourceOfferingId, }, cookies, ); @@ -984,6 +987,170 @@ async function ensureDeployment( return deployment.id; } +/** + * Lists every catalog offering owned directly by a tenant, paging past + * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list + * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" + * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s + * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s + * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog + * directly on that same tenant, so the owned list already carries + * everything a plain `workbench seed` or the self-served connect flow + * ever seeded, with no HTTP surface exposing the resolved view to a + * caller outside the hub process. + */ +async function listTenantCatalogOfferings( + api: ApiCall, + cookies: string[], + tenantId: string, +): Promise<(typeof ModelOfferingResponse.infer)[]> { + const items: (typeof ModelOfferingResponse.infer)[] = []; + let cursor: string | undefined; + for (;;) { + const query = + cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings${query}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + items.push(...page.data); + if (page.nextCursor === null) return items; + if (items.length > 10_000) { + throw new HubApiError( + `catalog offerings list for tenant ${tenantId} did not terminate while paging`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + cursor = page.nextCursor; + } +} + +/** + * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow + * deploys against: every enabled offering the tenant owns, ordered by + * priority ascending, with the lowest-priority offering as the default — + * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and + * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply + * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the + * noop offering (`ensureNoopCatalogOffering`) from this list when one + * has already been planted on the same tenant this run — it must never + * win a real workflow's default just because it happens to sort first. + * + * Throws when nothing is left to deploy against: "seeded but not + * launchable" (`seedCatalog` with no credential) is a valid catalog + * state, but a workflow deploy that names zero sources is not a + * lesser success, it is the failure this function exists to catch + * before the hub's own `sourceOfferingIds` validation would. + */ +async function resolveRealSourceOfferingIds( + api: ApiCall, + cookies: string[], + tenantId: string, + excludeOfferingId: string | undefined, +): Promise<{ + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; +}> { + const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) + .filter( + (offering) => + !offering.disabled && offering.id !== excludeOfferingId, + ) + .sort((a, b) => a.priority - b.priority); + const defaultSourceOfferingId = offerings[0]?.id; + if (defaultSourceOfferingId === undefined) { + throw new HubApiError( + "this tenant has no catalog offerings to deploy against — nothing is launchable yet", + "seed the tenant's catalog with a real provider key (or a placeholder credential for a keyless dev/CI run), then re-run: workbench seed", + ); + } + return { + sourceOfferingIds: offerings.map((offering) => offering.id), + defaultSourceOfferingId, + }; +} + +/** + * Plants (or finds) the dedicated catalog offering a noop-pinned + * workflow (`DefaultWorkflow.modelSource`, e.g. `heartbeat`) deploys + * against: a catalog model named `NOOP_MODEL`, a provider pointed at the + * hub's own `noop-inference` endpoint, and a placeholder credential — + * the same `ensureCatalogModel` / `ensureProvider` / `ensureCredential` / + * `ensureCatalogProvider` / `ensureCatalogOffering` sequence + * `seedCatalog` runs for a real provider, run here for this one + * synthetic one. Idempotent, same as every other seed step: a re-run + * finds the existing rows by name and reuses them. + */ +async function ensureNoopCatalogOffering( + api: ApiCall, + cookies: string[], + tenantId: string, + hubUrl: string, + log: (line: string) => void, +): Promise { + const baseURL = `${hubUrl}/api/chat/noop-inference`; + const modelId = await ensureCatalogModel( + api, + cookies, + { tenantId, canonicalName: NOOP_MODEL }, + log, + ); + const providerId = await ensureProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + apiBaseUrl: baseURL, + }, + log, + ); + const credentialId = await ensureCredential( + api, + cookies, + { + tenantId, + providerId, + name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), + secret: PLACEHOLDER_CATALOG_API_KEY, + type: "api_key", + }, + log, + ); + const catalogProviderId = await ensureCatalogProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + baseURL, + credentialId, + }, + log, + ); + return ensureCatalogOffering( + api, + cookies, + { + tenantId, + modelId, + providerId: catalogProviderId, + priority: NOOP_OFFERING_PRIORITY, + capabilities: [], + }, + log, + ); +} + /** * After deploy, PUT a pristine authored definition to `stopped` so a native * ScheduleTrigger does not fire every tenant. A member-restored row @@ -1160,6 +1327,14 @@ export type SeedTenantArgs = { cookies: string[]; hubUrl: string; tenant: SeedTenant; + /** + * The provider/model pair a real (non-noop-pinned) workflow's rendered + * definition names in its inference preferences. The deploy itself + * resolves inference from the tenant's own catalog offerings + * (`resolveRealSourceOfferingIds`), not from this pair — a tenant with + * no catalog offerings still fails clearly at deploy time even if this + * pair happens to name a real model. + */ model: ModelSource; pushWorkflow: WorkflowPusher; log: (line: string) => void; @@ -1228,9 +1403,41 @@ export async function seedTenant(args: SeedTenantArgs): Promise { await plantDefaultSkills(api, cookies, tenant.tenantId, log); + // Resolved lazily and cached across the loop below: most seed runs + // deploy several workflows against the same tenant catalog state, so + // this fetches the tenant's real offerings (or plants the noop one) + // at most once each, however many workflows ask for them. + let noopOfferingId: string | undefined; + let realOfferings: + | { sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string } + | undefined; + let confirmed = 0; for (const workflow of workflows) { - const workflowModel = workflow.modelSource?.(hubUrl) ?? model; + const workflowModel = workflow.modelSource?.() ?? model; + + let sourceOfferingIds: readonly string[]; + let defaultSourceOfferingId: string; + if (workflow.modelSource !== undefined) { + noopOfferingId ??= await ensureNoopCatalogOffering( + api, + cookies, + tenant.tenantId, + hubUrl, + log, + ); + sourceOfferingIds = [noopOfferingId]; + defaultSourceOfferingId = noopOfferingId; + } else { + realOfferings ??= await resolveRealSourceOfferingIds( + api, + cookies, + tenant.tenantId, + noopOfferingId, + ); + ({ sourceOfferingIds, defaultSourceOfferingId } = realOfferings); + } + const assetId = await ensureWorkflowAsset( api, cookies, @@ -1265,7 +1472,8 @@ export async function seedTenant(args: SeedTenantArgs): Promise { assetId, assetName: workflow.assetName, commitSha: pushed.commitSha, - model: workflowModel, + sourceOfferingIds, + defaultSourceOfferingId, }, log, ); @@ -1666,7 +1874,7 @@ async function ensureCatalogOffering( quirks?: Record; }, log: (line: string) => void, -): Promise { +): Promise { const body: Record = { modelId: args.modelId, providerId: args.providerId, @@ -1681,9 +1889,13 @@ async function ensureCatalogOffering( cookies, ); if (created.status === 201) { - parseAs(ModelOfferingResponse, created.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + created.data, + "catalog offering response", + ); log("created catalog offering"); - return; + return offering.id; } if (created.status === 409) { let cursor: string | null = null; @@ -1715,7 +1927,7 @@ async function ensureCatalogOffering( } if (existing.priority === args.priority) { log("catalog offering already exists (skipped)"); - return; + return existing.id; } const updated = await api( @@ -1730,9 +1942,13 @@ async function ensureCatalogOffering( "check the hub logs for the underlying failure, then re-run: workbench seed", ); } - parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + updated.data, + "catalog offering response", + ); log("updated catalog offering priority"); - return; + return offering.id; } throw new HubApiError( `the hub rejected creation of the catalog offering with status ${created.status}: ${JSON.stringify(created.data)}`, From 2587cd2b5af3ecbb27bb13bcb652910bd10b29aa Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:46 -0700 Subject: [PATCH 06/28] longevity-sim: deploy workflows by catalog offering ids Another workflowDeployBody caller CI's typecheck job caught: this stack seeds one catalog offering per real inference target already, so track each one's id and pass it through to the deploy body instead of the raw provider/baseURL/apiKey shape. --- packages/longevity-sim/src/stack.ts | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/longevity-sim/src/stack.ts b/packages/longevity-sim/src/stack.ts index a7dc91776..59c4ff72a 100644 --- a/packages/longevity-sim/src/stack.ts +++ b/packages/longevity-sim/src/stack.ts @@ -437,7 +437,9 @@ export async function bootLongevityStack( // credential row still requires some string, so each target's own // `apiKey` (expected to be the same placeholder) is threaded through // unchanged rather than this package inventing its own convention. - async function seedCatalogChain(target: InferenceTarget): Promise { + async function seedCatalogChain( + target: InferenceTarget, + ): Promise<{ offeringId: string }> { const model = await api( hub.baseUrl, "POST", @@ -509,10 +511,15 @@ export async function bootLongevityStack( owner.cookies, ); expectStatus(`create catalog offering ${target.model}`, offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; } + const offeringIdByTargetLabel = new Map(); for (const target of options.realTargets) { - await seedCatalogChain(target); + const { offeringId } = await seedCatalogChain(target); + offeringIdByTargetLabel.set(target.label, offeringId); } for (const skill of options.skills ?? []) { @@ -668,6 +675,13 @@ export async function bootLongevityStack( workflowJson, }); + const offeringId = offeringIdByTargetLabel.get(target.label); + if (offeringId === undefined) { + throw new Error( + `agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`, + ); + } + const deployDeadline = Date.now() + 90_000; for (;;) { if (sidecar.exited()) { @@ -682,11 +696,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId, commitSha: pushed.commitSha, - sourceId: `src-agent-${spec.handle}`, - provider: target.provider, - baseURL: target.baseURL, - apiKey: target.apiKey, - model: target.model, + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }), owner.cookies, ); @@ -847,6 +858,12 @@ export async function bootLongevityStack( if (routineTarget === undefined) { throw new Error("unreachable: realTargets checked non-empty above"); } + const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label); + if (routineOfferingId === undefined) { + throw new Error( + `no catalog offering seeded for realTarget "${routineTarget.label}"`, + ); + } const assetName = "longevity-heartbeat"; const heartbeatAssetCreated = await api( hub.baseUrl, @@ -912,11 +929,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId: heartbeatAssetId, commitSha: heartbeatPushed.commitSha, - sourceId: "src-longevity-heartbeat", - provider: "openai-compatible", - baseURL: routineTarget.baseURL, - apiKey: routineTarget.apiKey, - model: routineTarget.model, + sourceOfferingIds: [routineOfferingId], + defaultSourceOfferingId: routineOfferingId, }), owner.cookies, ); From 2430449e41b11a77a87cc902e4e2f7c264736125 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:56:16 -0700 Subject: [PATCH 07/28] Fix formatting: prettier --- apps/hub/src/system-seed.ts | 4 +++- packages/seeding/src/seed.ts | 3 +-- packages/seeding/test/seed.test.ts | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 52f0ff8e6..1637f2e89 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -57,7 +57,9 @@ export type SeedModelConfig = { readonly apiKey: string; }; -function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { +function resolvedModel( + seedModel: SeedModelConfig | undefined, +): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 20e350303..24fc1ce6e 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -1060,8 +1060,7 @@ async function resolveRealSourceOfferingIds( }> { const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) .filter( - (offering) => - !offering.disabled && offering.id !== excludeOfferingId, + (offering) => !offering.disabled && offering.id !== excludeOfferingId, ) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 720250165..99f972804 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -46,9 +46,10 @@ const MODEL = { // the tenant's catalog rather than a bare `ModelSource`. const OFFERING_ID = "off_1"; -function offeringsPage( - ids: readonly string[] = [OFFERING_ID], -): { status: number; data: unknown } { +function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { return { status: 200, data: { From 602ad68f72898a1342ec88f5090eac80999493a4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:25 -0700 Subject: [PATCH 08/28] sandbox-sidecar: add tests for generation-0 ensure/destroy requests The allocation service's initial sidecar_allocation and workflow_probe rows start at generation 0 (@intx/db enforces generation >= 0, not > 0), so the first ensure() for a freshly allocated sidecar must be accepted at generation 0. These tests cover that acceptance, later generations, non-integer/negative rejection, and that stale-generation fencing still rejects an older generation once a newer one has been observed. --- .../sandbox-sidecar/src/provisioner.test.ts | 154 ++++++++++++++++++ .../sandbox-sidecar/src/state-store.test.ts | 10 ++ 2 files changed, 164 insertions(+) create mode 100644 packages/sandbox-sidecar/src/provisioner.test.ts diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts new file mode 100644 index 000000000..753478bf0 --- /dev/null +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { createSidecarProvisioner } from "./provisioner"; +import type { SidecarBackend } from "./backend"; + +const TOKEN = "s3cr3t-bootstrap-token"; + +let dataDir: string; +let store: AllocationStateStore; +let started: string[]; + +beforeEach(async () => { + dataDir = await mkdtemp(join(tmpdir(), "sandbox-sidecar-provisioner-")); + store = createAllocationStateStore(join(dataDir, "state.json")); + started = []; +}); + +afterEach(async () => { + await rm(dataDir, { recursive: true, force: true }); +}); + +function fakeBackend(): SidecarBackend { + let counter = 0; + return { + async startUnit() { + counter += 1; + const externalRef = `unit-${String(counter)}`; + started.push(externalRef); + return externalRef; + }, + async stopUnit() {}, + async findUnitsByAllocation() { + return []; + }, + }; +} + +function baseEnsureRequest(overrides: Partial> = {}) { + return { + allocationId: "alloc-1", + generation: 0, + tenantId: "tenant-1", + anchorRunId: "run-1", + sidecarId: "sidecar-1", + token: TOKEN, + hubWebSocketUrl: "wss://hub.example.com/ws", + ...overrides, + } as never; +} + +function makeProvisioner(backend: SidecarBackend = fakeBackend()) { + return createSidecarProvisioner({ + id: "test-provisioner", + apiVersion: 1, + bindingFingerprint: "test", + capabilities: [], + backend, + store, + }); +} + +describe("ensure request validation", () => { + test("accepts generation 0, the allocation service's initial generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); + }); + + test("accepts a later generation", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: -1 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("rejects a non-integer generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1.5 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("still rejects a stale generation once fencing has observed a later one", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toMatchObject({ + kind: "rejected", + code: "stale_generation", + }); + expect(started).toHaveLength(1); + }); +}); + +describe("destroy request validation", () => { + test("accepts generation 0", async () => { + const provisioner = makeProvisioner(); + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + + expect(result).toEqual({ kind: "destroyed" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: -1, + }); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_destroy_request", + }); + }); +}); diff --git a/packages/sandbox-sidecar/src/state-store.test.ts b/packages/sandbox-sidecar/src/state-store.test.ts index d9bdadffb..abd723afd 100644 --- a/packages/sandbox-sidecar/src/state-store.test.ts +++ b/packages/sandbox-sidecar/src/state-store.test.ts @@ -30,6 +30,16 @@ describe("observeEnsure", () => { expect(result.kind).toBe("observed"); }); + test("accepts generation 0, the allocation service's initial generation", async () => { + const store = createAllocationStateStore(statePath()); + const result = await store.observeEnsure({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + expect(result.kind).toBe("observed"); + }); + test("is idempotent when observed again at the same generation", async () => { const store = createAllocationStateStore(statePath()); await store.observeEnsure({ From e01fedad96a1991c9b05eb612462fb18bd654db4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:30 -0700 Subject: [PATCH 09/28] sandbox-sidecar: accept the allocation service's initial generation @intx/db's sidecar_allocation and workflow_probe tables both check generation >= 0 and the allocation service records the first row for an allocation at generation 0, but validateEnsureRequest() and validateDestroyRequest() in packages/sandbox-sidecar rejected anything that wasn't strictly positive, and the on-disk fencing schema in state-store.ts required generation > 0. The first ensure() for any newly allocated sidecar was rejected with 'generation must be a positive integer', failing every deploy. Accept generation >= 0 in both validators and the state-store schema; stale-generation fencing in observeEnsure()/observeDestroy() already compares against the stored generation rather than assuming positivity, so it is unaffected. --- packages/sandbox-sidecar/src/provisioner.ts | 8 ++++---- packages/sandbox-sidecar/src/state-store.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.ts b/packages/sandbox-sidecar/src/provisioner.ts index 209d9e725..9771be319 100644 --- a/packages/sandbox-sidecar/src/provisioner.ts +++ b/packages/sandbox-sidecar/src/provisioner.ts @@ -218,8 +218,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { if (request.token === "") return "token must not be empty"; if (request.hubWebSocketUrl === "") return "hubWebSocketUrl must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } @@ -227,8 +227,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { function validateDestroyRequest(request: DestroySidecarRequest): string | null { if (request.allocationId === "") return "allocationId must not be empty"; if (request.sidecarId === "") return "sidecarId must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } diff --git a/packages/sandbox-sidecar/src/state-store.ts b/packages/sandbox-sidecar/src/state-store.ts index 12e5bed2a..9e600e228 100644 --- a/packages/sandbox-sidecar/src/state-store.ts +++ b/packages/sandbox-sidecar/src/state-store.ts @@ -12,7 +12,7 @@ import { type } from "arktype"; const AllocationRecord = type({ allocationId: "string > 0", sidecarId: "string > 0", - generation: "number.integer > 0", + generation: "number.integer >= 0", desiredState: "'ensured' | 'destroyed'", externalRef: "string | null", tokenHashSha256: "string | null", From c6da1fa7ba9e2607098f1e566456e8ebc8b84b9b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:37:36 -0700 Subject: [PATCH 10/28] sandbox-sidecar: fix formatting in provisioner test --- .../sandbox-sidecar/src/provisioner.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts index 753478bf0..b858c03cc 100644 --- a/packages/sandbox-sidecar/src/provisioner.test.ts +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -3,7 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { + createAllocationStateStore, + type AllocationStateStore, +} from "./state-store"; import { createSidecarProvisioner } from "./provisioner"; import type { SidecarBackend } from "./backend"; @@ -67,7 +70,9 @@ describe("ensure request validation", () => { test("accepts generation 0, the allocation service's initial generation", async () => { const provisioner = makeProvisioner(); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); }); @@ -76,7 +81,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 0 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); }); @@ -113,7 +120,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 1 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toMatchObject({ kind: "rejected", From 824c2b69f0c53998d64e5bebcb051c27b7d3ca4c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:42 -0700 Subject: [PATCH 11/28] Add tests for resolved catalog offerings and modelSourceFor's trimmed signature Covers the new GET .../catalog/resolved-offerings route (denied grant, and both directly-owned and ancestor-inherited offerings sorted by priority) and updates seeding/onboarding test fixtures to the route's response shape ahead of the implementation change. --- .../test/resolved-offerings-routes.test.ts | 99 +++++++++++++++++++ .../test/complete-credential.test.ts | 94 ++++++------------ packages/onboarding/test/provision.test.ts | 48 +++------ packages/seeding/test/seed.test.ts | 23 +++++ 4 files changed, 166 insertions(+), 98 deletions(-) create mode 100644 packages/inference-catalog/test/resolved-offerings-routes.test.ts diff --git a/packages/inference-catalog/test/resolved-offerings-routes.test.ts b/packages/inference-catalog/test/resolved-offerings-routes.test.ts new file mode 100644 index 000000000..897e05313 --- /dev/null +++ b/packages/inference-catalog/test/resolved-offerings-routes.test.ts @@ -0,0 +1,99 @@ +// The resolved-offerings route exists so an out-of-process caller +// (`workbench seed`, CL-7461) can deploy against exactly what the hub's +// own workflow deployer would deploy against — including offerings a +// tenant only inherits from an ancestor, which the tenant-owned +// `GET .../catalog/offerings` route never lists. +import { Hono } from "hono"; +import { describe, expect, test } from "bun:test"; + +import type { TenantEnv } from "@intx/hub-api"; +import type { ResolvedOffering } from "@intx/db"; + +import { createResolvedOfferingsRoutes } from "../src/resolved-offerings-routes"; +import { offering } from "./fixtures"; + +const TENANT = { + id: "bench-1", + name: "Bench One", + slug: "bench-one", + domain: "bench-one.example", + parentId: null, + config: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), +}; + +function app( + listOfferings: (tenantId: string) => Promise, + allow = true, +) { + const host = new Hono(); + host.use("*", async (c, next) => { + c.set("tenant", TENANT); + await next(); + }); + host.route( + "/", + createResolvedOfferingsRoutes({ + listOfferings, + requireGrant: () => async (c, next) => { + if (!allow) { + return c.json({ error: { code: "forbidden", message: "no" } }, 403); + } + return next(); + }, + }), + ); + return host; +} + +describe("GET resolved-offerings", () => { + test("a denied grant is rejected before the offerings are read", async () => { + const response = await app(async () => { + throw new Error("must not be called when the grant is denied"); + }, false).request("/"); + expect(response.status).toBe(403); + }); + + test("lists both directly-owned and ancestor-inherited offerings, sorted by priority", async () => { + const owned = offering({ + id: "owned", + canonicalName: "thrifty", + providerName: "globex", + capabilities: ["plain-text"], + priority: 2, + }); + const inherited = offering({ + id: "inherited", + canonicalName: "lavish", + providerName: "acme", + capabilities: ["plain-text"], + priority: 1, + inherited: true, + }); + + const response = await app(async (tenantId) => { + expect(tenantId).toBe(TENANT.id); + return [owned, inherited]; + }).request("/"); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + offerings: { + id: string; + priority: number; + modelId: string; + providerId: string; + origin: { tenantId: string; direct: boolean }; + }[]; + }; + expect(body.offerings.map((o) => o.id)).toEqual(["inherited", "owned"]); + const inheritedEntry = body.offerings.find((o) => o.id === "inherited"); + expect(inheritedEntry?.origin).toEqual({ + tenantId: "parent-bench", + direct: false, + }); + const ownedEntry = body.offerings.find((o) => o.id === "owned"); + expect(ownedEntry?.origin).toEqual({ tenantId: TENANT.id, direct: true }); + }); +}); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 8a4ae6a78..16e81d0df 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -72,22 +72,15 @@ function fixedOfferingsResponse() { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -179,19 +172,12 @@ function ownedCatalogModelsResponse(canonicalNames: string[]) { } describe("modelSourceFor", () => { - test("every other provider ignores a baseURLOverride and never calls the hub", async () => { + test("every other provider resolves to its curated default and never calls the hub", async () => { const api: ApiCall = (async () => { throw new Error("must not be called for a fixed curated provider"); }) as ApiCall; expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "anthropic", - "sk-ant", - "https://ignored", - ), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "anthropic"), ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", @@ -216,7 +202,7 @@ describe("modelSourceFor", () => { }; expect( - await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama", "ollama"), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama"), ).toEqual({ provider: "openai-compatible", model: "llama3.2", @@ -250,7 +236,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -285,7 +270,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); @@ -323,7 +307,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -360,44 +343,9 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); - - // `modelSourceFor` resolves a provider/model pair only — the deploy - // itself resolves inference from the tenant's own catalog offerings - // (CL-7461), so a `baseURLOverride` here is accepted (every caller - // already resolved one for `seedCatalog`'s earlier catalog plant) but - // never affects this resolution. - test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { - const api: ApiCall = async (method, path) => { - if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { - return resolvedCatalogResponse([ - { - canonicalName: "qwen3.8:27b", - providerName: "ollama", - capabilities: ["plain-text"], - }, - ]); - } - throw new Error(`unexpected call: ${method} ${path}`); - }; - - expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "ollama", - "ollama", - "https://home-mac.example.ts.net", - ), - ).toEqual({ - provider: "openai-compatible", - model: "qwen3.8:27b", - }); - }); }); describe("completeCredentialSetup", () => { @@ -780,12 +728,11 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, - data: { data: [], nextCursor: null }, + data: { offerings: [] }, cookies: [], }; } @@ -839,7 +786,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } @@ -1422,8 +1369,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { return { status: 200, @@ -1446,6 +1392,24 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) { + return { + status: 200, + data: { + offerings: catalogOfferings.map((o) => ({ + id: o.id, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -1828,7 +1792,7 @@ describe("ensureSeeded (the slow half)", () => { const api: ApiCall = async (method, path, body) => { if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index b1feabe08..aff387d48 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -96,27 +96,23 @@ function catalogOfferingsResponse( path: string, tenantId: string, ): { status: number; data: unknown; cookies: string[] } | undefined { - if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + if ( + method !== "GET" || + path !== `/api/tenants/${tenantId}/catalog/resolved-offerings` + ) return undefined; return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -243,27 +239,20 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -702,27 +691,20 @@ describe("provisionPersonalTenantIfNeeded", () => { if (registry !== undefined) return registry; if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 99f972804..d1bea7bcd 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -71,6 +71,24 @@ function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { }; } +function resolvedOfferingsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { + return { + status: 200, + data: { + offerings: ids.map((id, index) => ({ + id, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -132,6 +150,11 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) return offeringsPage(); + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) + return resolvedOfferingsPage(); // The noop catalog chain `ensureNoopCatalogOffering` plants for a // modelSource-pinned workflow (heartbeat) — every seedTenant test that // deploys one needs this fresh chain to succeed, so it lives in the From ec8755106e08316e1a4a12c57f8121849be6fcb6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:49 -0700 Subject: [PATCH 12/28] Seed deploys against a tenant's resolved catalog offerings, not just owned ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveRealSourceOfferingIds listed only offerings a tenant owns directly, missing anything inherited from an ancestor tenant — the hub's own workflow deployer already resolves that wider, ancestor-inheriting view via listVisibleOfferings. Adds GET /api/tenants/:tenantId/catalog/resolved-offerings (mounted from apps/hub, implemented in @corbits/inference-catalog so the deployer and this route share one source of truth) and points seed at it instead. Also drops the two long-dead unused parameters from onboarding's modelSourceFor, and exports ensureNoopCatalogOffering from @corbits/seeding for the e2e harness to reuse. --- apps/hub/src/index.ts | 16 ++++ packages/inference-catalog/src/index.ts | 4 + .../src/resolved-offerings-routes.ts | 44 ++++++++++ .../onboarding/src/complete-credential.ts | 9 +- packages/seeding/src/index.ts | 1 + packages/seeding/src/seed.ts | 84 +++++++++---------- 6 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 packages/inference-catalog/src/resolved-offerings-routes.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 39d52b853..687bcde64 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -150,6 +150,7 @@ import { applyInferenceCatalogMigrations, createBenchModelPolicyRoutes, createPostgresBenchModelPolicyStore, + createResolvedOfferingsRoutes, createWorkflowCatalogRoutes, } from "@corbits/inference-catalog"; import { generateId } from "@intx/hub-common"; @@ -1877,6 +1878,21 @@ export async function createHub(config: HubConfig) { }), }), ); + // Resolved catalog offerings: the same ancestor-inheriting view + // `listVisibleOfferings` gives `workflowDeployer` above, exposed over + // HTTP so an out-of-process deployer (`workbench seed`) can deploy + // against exactly what the hub itself would deploy against, not just + // the offerings a tenant owns directly. + app.route( + `${TENANT_PREFIX}/catalog/resolved-offerings`, + createResolvedOfferingsRoutes({ + listOfferings: (tenantId) => listVisibleOfferings(db, tenantId), + requireGrant: createRequireGrant({ + grantStore: chatGrantStore, + conditionRegistry: chatConditionRegistry, + }), + }), + ); // Bench purpose/type: benches are Interchange tenants, so this is a // package-owned side-table keyed by tenant id, migrated at hub start // like insights and preferences. diff --git a/packages/inference-catalog/src/index.ts b/packages/inference-catalog/src/index.ts index b7e90138a..01af47a3e 100644 --- a/packages/inference-catalog/src/index.ts +++ b/packages/inference-catalog/src/index.ts @@ -63,6 +63,10 @@ export { createBenchModelPolicyRoutes, type CreateBenchModelPolicyRoutesDeps, } from "./routes"; +export { + createResolvedOfferingsRoutes, + type CreateResolvedOfferingsRoutesDeps, +} from "./resolved-offerings-routes"; export { createWorkflowCatalogRoutes, estimateUsd, diff --git a/packages/inference-catalog/src/resolved-offerings-routes.ts b/packages/inference-catalog/src/resolved-offerings-routes.ts new file mode 100644 index 000000000..ec1cfae2f --- /dev/null +++ b/packages/inference-catalog/src/resolved-offerings-routes.ts @@ -0,0 +1,44 @@ +// The tenant-owned `GET .../catalog/offerings` route only ever lists rows a +// tenant planted directly on itself; it has no way to show offerings a +// tenant inherits from an ancestor tenant. `listVisibleOfferings` +// (`@intx/db`) already computes that resolved, inheritance-aware view for +// the hub's own workflow deployer — this route exposes the same view over +// HTTP so an out-of-process caller (`workbench seed`, in particular) can +// deploy against exactly what the hub itself would deploy against. +// +// Takes a `listOfferings` port rather than a `db` handle, same as +// `workflow-catalog-routes.ts`, so the route is testable with literals. +import { Hono } from "hono"; +import type { ResolvedOffering } from "@intx/db"; +import type { RequireGrant, TenantEnv } from "@intx/hub-api"; + +export type CreateResolvedOfferingsRoutesDeps = { + readonly listOfferings: ( + tenantId: string, + ) => Promise; + readonly requireGrant: RequireGrant; +}; + +export function createResolvedOfferingsRoutes( + deps: CreateResolvedOfferingsRoutesDeps, +): Hono { + const app = new Hono(); + + app.get("/", deps.requireGrant("model-offering:*", "read"), async (c) => { + const tenant = c.get("tenant"); + const resolved = [...(await deps.listOfferings(tenant.id))].sort( + (a, b) => a.offering.priority - b.offering.priority, + ); + return c.json({ + offerings: resolved.map((entry) => ({ + id: entry.offering.id, + priority: entry.offering.priority, + modelId: entry.offering.modelId, + providerId: entry.offering.providerId, + origin: entry.origin, + })), + }); + }); + + return app; +} diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index f5f4a930a..c292bc897 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -372,17 +372,12 @@ async function resolveOllamaModelSource( * diverge from that curated name entirely (CL-6366) — its resolution * defers to `resolveOllamaModelSource`, which reads back what * `seedCatalog` actually found on the instance rather than repeating the - * static pin. `apiKey`/`baseURLOverride` are accepted only because every - * caller already resolved them for `seedCatalog`'s own credential/catalog - * plant, which runs ahead of this in the same connect flow; this - * function itself needs neither. */ + * static pin. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - _apiKey: string, - _baseURLOverride?: string, ): Promise { if (provider === "ollama") { return resolveOllamaModelSource(api, cookies, tenantId); @@ -496,8 +491,6 @@ export async function ensureSeeded( args.cookies, args.tenant.tenantId, args.provider, - args.apiKey, - args.baseURLOverride, ), pushWorkflow: args.pushWorkflow, log: args.log, diff --git a/packages/seeding/src/index.ts b/packages/seeding/src/index.ts index 78506cd4f..e67e47a58 100644 --- a/packages/seeding/src/index.ts +++ b/packages/seeding/src/index.ts @@ -26,6 +26,7 @@ export { inferenceCredentialName, PLACEHOLDER_CATALOG_API_KEY, ensureCredential, + ensureNoopCatalogOffering, ensureProvider, reconcileSeedGrants, seedCatalog, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 24fc1ce6e..21ca42794 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -987,55 +987,51 @@ async function ensureDeployment( return deployment.id; } +const ResolvedOfferingResponse = type({ + id: "string", + priority: "number", + modelId: "string", + providerId: "string", + origin: { tenantId: "string", direct: "boolean" }, +}); + +const ResolvedOfferingsResponse = type({ + offerings: ResolvedOfferingResponse.array(), +}); + /** - * Lists every catalog offering owned directly by a tenant, paging past - * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list - * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" - * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s - * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s - * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog - * directly on that same tenant, so the owned list already carries - * everything a plain `workbench seed` or the self-served connect flow - * ever seeded, with no HTTP surface exposing the resolved view to a - * caller outside the hub process. + * Lists the offerings visible to a tenant, including any inherited from + * an ancestor tenant — the same resolved view `listVisibleOfferings` + * (`@intx/db`) gives the hub's own workflow deployer + * (`apps/hub/src/index.ts`'s `workflowDeployer`), exposed at + * `GET .../catalog/resolved-offerings`. Seeding a tenant whose catalog is + * inherited rather than directly owned (e.g. a sub-tenant under a + * parent that already carries a real provider key) needs this resolved + * list, not the tenant-owned-only `GET .../catalog/offerings`. */ -async function listTenantCatalogOfferings( +async function listResolvedCatalogOfferings( api: ApiCall, cookies: string[], tenantId: string, -): Promise<(typeof ModelOfferingResponse.infer)[]> { - const items: (typeof ModelOfferingResponse.infer)[] = []; - let cursor: string | undefined; - for (;;) { - const query = - cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; - const listed = await api( - "GET", - `/api/tenants/${tenantId}/catalog/offerings${query}`, - undefined, - cookies, - ); - const page = parseAs( - paginatedSchema(ModelOfferingResponse), - listed.data, - "catalog offerings response", - ); - items.push(...page.data); - if (page.nextCursor === null) return items; - if (items.length > 10_000) { - throw new HubApiError( - `catalog offerings list for tenant ${tenantId} did not terminate while paging`, - "check the hub logs for the underlying failure, then re-run: workbench seed", - ); - } - cursor = page.nextCursor; - } +): Promise<(typeof ResolvedOfferingResponse.infer)[]> { + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/resolved-offerings`, + undefined, + cookies, + ); + return parseAs( + ResolvedOfferingsResponse, + listed.data, + "resolved catalog offerings response", + ).offerings; } /** * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow - * deploys against: every enabled offering the tenant owns, ordered by - * priority ascending, with the lowest-priority offering as the default — + * deploys against: every offering visible to the tenant (owned or + * inherited from an ancestor), ordered by priority ascending, with the + * lowest-priority offering as the default — * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the @@ -1058,10 +1054,8 @@ async function resolveRealSourceOfferingIds( sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string; }> { - const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) - .filter( - (offering) => !offering.disabled && offering.id !== excludeOfferingId, - ) + const offerings = (await listResolvedCatalogOfferings(api, cookies, tenantId)) + .filter((offering) => offering.id !== excludeOfferingId) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; if (defaultSourceOfferingId === undefined) { @@ -1087,7 +1081,7 @@ async function resolveRealSourceOfferingIds( * synthetic one. Idempotent, same as every other seed step: a re-run * finds the existing rows by name and reuses them. */ -async function ensureNoopCatalogOffering( +export async function ensureNoopCatalogOffering( api: ApiCall, cookies: string[], tenantId: string, From bcb8703e02e1a669a5d16651045a604993aeaf2e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:05 -0700 Subject: [PATCH 13/28] Add a regression test pinning the shared-sidecar auth gap (CL-7472) At the interchange pin this branch stacks on (692c3106), the hub authenticates a sidecar's register/reconnect frame only against a sidecar row that resolves to a live sidecar_allocation (or workflow_probe) row. Upstream dropped the older "shared" credential scope this repo's e2e harness used to provision a single, standalone sidecar identity for, so a provisionSidecar-only row now connects and is rejected with "invalid token" in a loop rather than ever dialing in. This test pins that negative so a future re-vendor that silently restores (or relies on) the old scope surfaces here first. --- scripts/e2e/harness.test.ts | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 37f4f93e4..1a4f3fce8 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -6,10 +6,14 @@ import { describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; +import path from "node:path"; +import { resetSchema, setupDatabase } from "../db-setup.ts"; import { createCleanupHarness, + e2eDatabaseUrl, parseEnvFileDatabaseUrl, + provisionSidecar, runCleanups, workflowDeployBody, type SpawnedApp, @@ -127,3 +131,71 @@ describe("parseEnvFileDatabaseUrl", () => { ).toBe("postgres://second/db"); }); }); + +// CL-7472 regression: at pin 692c3106, the hub authenticates a +// sidecar's register/reconnect frame only against a `sidecar` row that +// resolves to a live `sidecar_allocation` (or `workflow_probe`) row — +// upstream dropped the older "shared" credential scope this repo used +// to provision a single, standalone sidecar identity for (see +// `provisionSidecar`'s own doc comment). CI's e2e suites hit exactly +// this: a `provisionSidecar`-only row connects and is rejected with +// "invalid token" in a loop (never a working dial-in), which is why +// every suite now lets the hub's own process-provisioner spawn a +// dedicated, already-allocated sidecar per deployment instead (see +// `startHub`'s doc comment). This test pins the negative: an +// unallocated identity must keep failing to authenticate, so a future +// upstream re-vendor that silently restores (or a caller that +// silently starts relying on) the old "shared" scope surfaces here +// first rather than as a mystery disconnect loop in CI. +describe("provisionSidecar", () => { + const databaseUrl = e2eDatabaseUrl(); + const HUB_DIR = path.resolve(import.meta.dir, "..", "..", "apps", "hub"); + + test.skipIf(databaseUrl === undefined)( + "a provisioned identity with no allocation or probe never authenticates", + async () => { + const url = databaseUrl; + if (url === undefined) throw new Error("unreachable: suite is skipped"); + + await resetSchema(url); + await setupDatabase(url); + + const sidecarId = "harness-test-unallocated-sidecar"; + const token = crypto.randomUUID(); + await provisionSidecar(url, sidecarId, token); + + // Resolved through the hub's own dependency tree, exactly as + // `connectE2eDb` resolves `postgres` above — scripts/e2e is not + // itself a workspace member of these `@intx/*` packages. + const dbModule = (await import( + Bun.resolveSync("@intx/db", HUB_DIR) + )) as { createDB: (raw: unknown) => { db: unknown; close: () => Promise } }; + const hubSessionsModule = (await import( + Bun.resolveSync("@intx/hub-sessions", HUB_DIR) + )) as { + createSidecarCredentialResolver: (deps: { db: unknown }) => { + resolve: (token: string) => Promise; + }; + }; + + const parsed = new URL(url); + const db = dbModule.createDB({ + host: parsed.hostname, + port: parsed.port === "" ? 5432 : Number(parsed.port), + user: decodeURIComponent(parsed.username), + password: decodeURIComponent(parsed.password), + database: parsed.pathname.replace(/^\//, ""), + }); + try { + const resolver = hubSessionsModule.createSidecarCredentialResolver({ + db: db.db, + }); + const identity = await resolver.resolve(token); + expect(identity).toBeNull(); + } finally { + await db.close(); + } + }, + 60_000, + ); +}); From ccbe497e63bf84527e9ad01dcc840b7be2155e10 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:16 -0700 Subject: [PATCH 14/28] e2e: stop pre-provisioning a shared sidecar identity (CL-7472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every e2e suite dial-in relied on a single, harness-started sidecar process registered against a bare `provisionSidecar` identity row. At pin 692c3106, the hub only accepts a register/reconnect frame from a sidecar whose identity resolves to a live sidecar_allocation (or workflow_probe) row; a standalone identity with neither now fails the handshake and gets rejected with "invalid token" every ~3s, forever (chat mint 500s, deploys never dial in — CI run 34042907014). The hub's own process-provisioner already spawns a dedicated, correctly-allocated sidecar process per deployment on demand — the same path production already takes for a real workflow run or a minted agent chat. Every affected suite (walking skeleton, chat, heartbeat, workbench-digest, smoke-chat, smoke-webhook, greeting-delivery, channel-reconnect, local-rip, and the isolation suite) now lets that happen instead of dialing in its own sidecar: the existing 502/500 retry-until-ready loops already cover the wait, and diagnostics that used to read the one sidecar process's output now read the hub's — the process-provisioner backend spawns each sidecar with its stdio inherited from the hub, so the hub's own captured output already carries every spawned sidecar's log lines. `provisionSidecar`/`startSidecar` stay in the harness (documented as non-authenticating at this pin) for the handful of standalone proof scripts not yet migrated off a directly-controlled sidecar process. --- scripts/e2e/channel-reconnect.test.ts | 37 +++++----------- scripts/e2e/chat.test.ts | 61 ++++++++++----------------- scripts/e2e/greeting-delivery.test.ts | 36 ++++------------ scripts/e2e/harness.ts | 26 +++++++++--- scripts/e2e/heartbeat.test.ts | 29 ++----------- scripts/e2e/local-rip.test.ts | 24 ++--------- scripts/e2e/smoke-chat.test.ts | 29 ++----------- scripts/e2e/smoke-webhook.test.ts | 38 +++-------------- scripts/e2e/walking-skeleton.test.ts | 45 +++++--------------- scripts/e2e/workbench-digest.test.ts | 29 ++----------- test/isolation/setup.ts | 49 ++++++--------------- 11 files changed, 109 insertions(+), 294 deletions(-) diff --git a/scripts/e2e/channel-reconnect.test.ts b/scripts/e2e/channel-reconnect.test.ts index 59e254c19..3c3e8e637 100644 --- a/scripts/e2e/channel-reconnect.test.ts +++ b/scripts/e2e/channel-reconnect.test.ts @@ -29,12 +29,9 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -76,7 +73,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { let hubPort: number; let sessionSecret: string; let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let cookies: string[]; let tenantId: string; @@ -93,10 +89,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "workbench-reconnect-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hubPort = freePort(); sessionSecret = Buffer.from( crypto.getRandomValues(new Uint8Array(32)), @@ -110,14 +102,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-reconnect-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); const email = `workbench-reconnect-e2e-${crypto.randomUUID()}@example.invalid`; @@ -159,9 +143,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -174,7 +158,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -244,10 +228,10 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { // hub process on the same port against the same database, exactly // as a real restart looks from the sidecar's and the browser's // side: same DB rows, same session cookie, new process. The port - // must stay the same — the sidecar's own `HUB_WS_URL` is fixed at - // its boot and is never reconfigured here (only the hub restarts), - // so a different port would leave the sidecar dialing a hub that - // is no longer there. + // must stay the same — the process-provisioner-spawned sidecar's + // own `HUB_WS_URL` is fixed at its boot and is never reconfigured + // here (only the hub restarts), so a different port would leave the + // sidecar dialing a hub that is no longer there. await hub.stop(); const restarted = await startHub({ @@ -269,9 +253,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 180_000; let posted: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited after hub restart; output:\n${sidecar.output()}`, + `hub exited after hub restart; output:\n${hub.output()}`, ); } posted = await postMessage(after); @@ -279,8 +263,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `message after hub restart never accepted: status=${posted.status} ` + - `${JSON.stringify(posted.data)}\nsidecar output:\n${sidecar.output()}` + - `\nhub output:\n${hub.output()}`, + `${JSON.stringify(posted.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index da9b255c8..c50da0894 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -46,14 +46,11 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, pushWorkflowSource, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -139,7 +136,6 @@ const textPart = (text: string): Part[] => [{ kind: "text", text }]; describe.skipIf(databaseUrl === undefined)("chat e2e", () => { let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let user1: SignedUpUser; let user2: SignedUpUser; @@ -159,10 +155,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "chat-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hub = await startHub({ databaseUrl: url, port: freePort(), @@ -173,16 +165,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-chat-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); // Two independent browser-shaped accounts, each carrying its own @@ -358,15 +340,15 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { ), }); - // Retries while the hub still answers 502 (the sidecar's dial-in - // may not have completed yet), matching `createWorkbench`'s own - // retry loop below. + // Retries while the hub still answers 502 (the process + // provisioner's spawned sidecar may not have dialed in yet), + // matching `createWorkbench`'s own retry loop below. const echoDeployDeadline = Date.now() + 60_000; let echoDeployed: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before echo deploy; output:\n${sidecar.output()}`, + `hub exited before echo deploy; output:\n${hub.output()}`, ); } echoDeployed = await api( @@ -384,7 +366,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > echoDeployDeadline) { throw new Error( `echo workflow never became deployable (hub kept answering 502): ` + - `${JSON.stringify(echoDeployed.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(echoDeployed.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -393,22 +375,22 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }, 120_000); // Launching a workbench is the go/no-go signal for the whole suite: it - // launches the anchor instance in-process, which needs the - // sidecar's dial-in to have completed. `packages/chat/src/routes.ts` - // has no 502-style retry translation of its own (unlike the native - // workflow deploy route), so a launch attempted before the sidecar - // connects fails with an uncaught 500 — retried here directly until - // the sidecar is ready, exactly as the walking skeleton retries a - // 502 for the native deploy route. + // launches the anchor instance in-process, which needs its + // process-provisioner-spawned sidecar's dial-in to have completed. + // `packages/chat/src/routes.ts` has no 502-style retry translation of + // its own (unlike the native workflow deploy route), so a launch + // attempted before the sidecar connects fails with an uncaught 500 — + // retried here directly until the sidecar is ready, exactly as the + // walking skeleton retries a 502 for the native deploy route. async function createWorkbench( body: Record, ): Promise { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -421,7 +403,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -1021,10 +1003,13 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { // temporarily reverting the self-anchor fix locally: the sidecar log // then carries ten `enqueueInbox failed, withholding ack` lines, one // per rejected pack, for this same suite; with the fix applied, zero. + // Every process-provisioner-spawned sidecar inherits the hub + // process's own stdio (`process-runner.ts`'s `stdout: "inherit"`), so + // `hub.output()` already carries every sidecar's log lines. test("no chat/folded run's workflow-run pack was ever permanently rejected", () => { - const sidecarOutput = sidecar.output(); - expect(sidecarOutput).not.toContain("enqueueInbox failed"); - expect(sidecarOutput).not.toContain("withholding ack"); - expect(sidecarOutput).not.toContain("rejecting inbound mail"); + const hubOutput = hub.output(); + expect(hubOutput).not.toContain("enqueueInbox failed"); + expect(hubOutput).not.toContain("withholding ack"); + expect(hubOutput).not.toContain("rejecting inbound mail"); }); }); diff --git a/scripts/e2e/greeting-delivery.test.ts b/scripts/e2e/greeting-delivery.test.ts index 0ac6e26bb..2521206f1 100644 --- a/scripts/e2e/greeting-delivery.test.ts +++ b/scripts/e2e/greeting-delivery.test.ts @@ -41,12 +41,9 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const { tempDir, track } = createCleanupHarness(); @@ -135,10 +132,6 @@ describe.skipIf(databaseUrl === undefined)( expect(report.action).toBe("migrated"); }); - const sidecarId = "greeting-delivery-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -151,16 +144,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar: SpawnedApp = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-greeting-delivery-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -236,9 +219,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { @@ -269,9 +252,9 @@ describe.skipIf(databaseUrl === undefined)( async function deploySeededWorkflows(): Promise { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -356,9 +339,9 @@ describe.skipIf(databaseUrl === undefined)( const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before chat creation; output:\n${sidecar.output()}`, + `hub exited before chat creation; output:\n${hub.output()}`, ); } res = await api( @@ -372,7 +355,7 @@ describe.skipIf(databaseUrl === undefined)( if (Date.now() > deadline) { throw new Error( `chat never became mintable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -448,7 +431,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no agent-authored message landed in chat ${chatId} within ` + `60s of mint with zero user messages sent; messages seen: ` + - `${JSON.stringify(items)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(items)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -527,8 +510,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no answer to turn ${turn} within 180s; agent messages: ` + `${JSON.stringify(agentTexts)}\nhub output (tail):\n` + - `${hub.output().slice(-60000)}\nsidecar output (tail):\n` + - `${sidecar.output().slice(-6000)}`, + `${hub.output().slice(-60000)}`, ); } await Bun.sleep(2000); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index cebc4e53d..1e884e0f9 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -221,6 +221,20 @@ export async function connectE2eDb(databaseUrl: string): Promise { * WebSocket dial-in against the token hash on the `sidecar` table, so * the row must exist before the sidecar process starts. Mirrors * Interchange's dev provisioning (id + sha256(token), placeholder url). + * + * At pin 692c3106, the hub only accepts a register/reconnect frame from + * a sidecar identity that resolves to a live `sidecar_allocation` (or + * `workflow_probe`) row — the standalone, unallocated ("shared" + * credential scope) sidecar this used to provision no longer + * authenticates at all (CL-7472), so a caller of this function gets a + * connect-and-immediately-reject loop, not a usable dial-in. Every + * suite this repo runs against CI now lets the hub's own + * process-provisioner spawn a dedicated sidecar per allocation instead + * (see `startHub`'s doc comment and `scripts/e2e/chat.test.ts`); this + * function is kept only for the handful of scripts not yet migrated off + * a directly-controlled sidecar process (e.g. ones that kill and + * restart a specific sidecar mid-scenario) and is not a working dial-in + * path until they are. */ export async function provisionSidecar( databaseUrl: string, @@ -400,11 +414,13 @@ export async function startHub(options: { } /** - * Boot the sidecar as a real process, pointed at the hub's WebSocket - * dial-in route with the provisioned identity. Connection readiness is - * observed by the deploy call itself (the hub answers 502 until a - * sidecar is connected), so this only guards against an immediate - * crash. + * Boot a standalone sidecar as a real process, pointed at the hub's + * WebSocket dial-in route with a `provisionSidecar`-issued identity. + * See `provisionSidecar`'s own doc comment: at pin 692c3106 this + * identity never authenticates (CL-7472), so the process this starts + * connects and is immediately rejected in a loop rather than dialing + * in. Kept only for the scripts not yet migrated to the hub's + * automatic, per-allocation process-provisioner spawn. */ export function startSidecar(options: { hubPort: number; diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 9aefb900f..8c924a6af 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -95,12 +93,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-heartbeat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -114,19 +106,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-heartbeat-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Heartbeat Tester", @@ -224,10 +203,8 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -240,7 +217,7 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 2b4f4ec9c..b1919de02 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -81,9 +81,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -187,10 +185,6 @@ describe.skipIf(databaseUrl === undefined)( } }); - const sidecarId = "local-rip-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -208,16 +202,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-local-rip-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -341,9 +325,9 @@ describe.skipIf(databaseUrl === undefined)( ): Promise>> { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -398,9 +382,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { diff --git a/scripts/e2e/smoke-chat.test.ts b/scripts/e2e/smoke-chat.test.ts index 98b598332..eaf0da399 100644 --- a/scripts/e2e/smoke-chat.test.ts +++ b/scripts/e2e/smoke-chat.test.ts @@ -20,9 +20,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -58,13 +56,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { }); const hubDataDir = await tempDir("e2e-smoke-chat-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-chat-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-chat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -78,18 +69,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -123,9 +102,9 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { // through the transient 500 the same way chat.test.ts does. const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } const res = await api( @@ -141,7 +120,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { } if (Date.now() > deadline) { throw new Error( - `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -171,7 +150,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { if (Date.now() > deadline) { throw new Error( `post message kept answering 500: ${JSON.stringify(res.data)}` + - `\nsidecar output:\n${sidecar.output()}`, + `\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 760aca6a3..d834aa623 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -12,9 +12,8 @@ // events by bare id (the native events route requires a deployment-anchor // run). This test instead reads the run row the ingress delivery created // straight out of the database — a harness-side fact with no route, -// exactly as `provisionSidecar` already does for the sidecar identity -// row — and confirms the trigger itself recorded the delivery -// (`lastFiredAt`). +// via `connectE2eDb` — and confirms the trigger itself recorded the +// delivery (`lastFiredAt`). import { signPayload, @@ -36,12 +35,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -77,13 +74,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const hubDataDir = await tempDir("e2e-smoke-webhook-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-webhook-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-webhook"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -97,18 +87,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -204,10 +182,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } const res = await api( hub.baseUrl, @@ -219,7 +195,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { if (res.status === 502) { if (Date.now() > deadline) { throw new Error( - `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -346,8 +322,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // No platform route reads a non-anchored run's row by bare id // (see the file header); this reads the fact the ingress // route's own launch is required to have produced, the same - // way `provisionSidecar` reaches the database directly for a - // fact no route exposes. + // way `connectE2eDb` reaches the database directly for a fact + // no route exposes. const sql = await connectE2eDb(url); try { const rows = await sql.unsafe( diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 8937f9b5e..e876f6bfa 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -33,12 +33,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, } from "./harness.ts"; @@ -81,15 +79,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { expect(second.migrations).toBe(first.migrations); }); - // Hop: sidecar provisioning. The identity row the hub checks the - // sidecar's dial-in token against. - const sidecarId = "sidecar-e2e"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - - // Hop: hub boot. The composition root as a real process. + // Hop: hub boot. The composition root as a real process. The hub's + // own process provisioner spawns a dedicated sidecar for every + // allocation on demand (one `apps/sidecar` per deployment) — there + // is no shared, pre-provisioned sidecar identity to dial in here. const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -103,21 +96,6 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { return handle; }); - // Hop: sidecar boot. Dial-in readiness is observed at the deploy - // hop (the hub answers 502 until a sidecar is connected). - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-sidecar-data-"), - }); - track(app); - return app; - }); - // Hop: sign-up. A browser-shaped account creation through the // platform's better-auth surface; the session cookie carries the // rest of the skeleton. @@ -215,9 +193,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { ); // Hop: workflow deploy via the native deploy API. Retries while - // the hub still answers 502 (the sidecar's dial-in may not have - // completed yet); any other failure is final. The inference - // source is a placeholder — deployment does not call inference. + // the hub still answers 502 (the process provisioner's spawned + // sidecar may not have dialed in yet); any other failure is final. + // The inference source is a placeholder — deployment does not call + // inference. const deploymentId = await hop("workflow deploy", async () => { const body = workflowDeployBody({ assetId, @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 8917bf823..f3d41cd06 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -100,12 +98,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-workbench-digest"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -119,19 +111,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-digest-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Workbench Digest Tester", @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/test/isolation/setup.ts b/test/isolation/setup.ts index 3644750fa..07271b0c1 100644 --- a/test/isolation/setup.ts +++ b/test/isolation/setup.ts @@ -10,11 +10,8 @@ import path from "node:path"; import { freePort, - provisionSidecar, startHub, - startSidecar, type HubHandle, - type SpawnedApp, } from "../../scripts/e2e/harness.ts"; import { assertDatabaseConfigured } from "../../scripts/e2e/db-gate.ts"; @@ -90,29 +87,21 @@ export async function prepareDatabase(databaseUrl: string): Promise { } /** - * Boots the real hub and a real sidecar as spawned processes, exactly - * as `scripts/e2e/harness.ts` does for the walking skeleton — a - * chat-workbench create launches a real agent instance over the - * sidecar's WebSocket dial-in, so an in-process hub with no sidecar - * attached cannot exercise the "chat workbench move" block below; it - * would fail every one of those cases with "no sidecar available" - * rather than proving isolation. `AppLike.request` wraps `fetch` - * against the hub's real port, matching how a client actually reaches - * it. + * Boots the real hub as a spawned process, exactly as + * `scripts/e2e/harness.ts` does for the walking skeleton — a + * chat-workbench create launches a real agent instance over its own + * process-provisioner-spawned sidecar's WebSocket dial-in, so an + * in-process hub with no sidecar plugin wired up cannot exercise the + * "chat workbench move" block below; it would fail every one of those + * cases with "no sidecar available" rather than proving isolation. + * `AppLike.request` wraps `fetch` against the hub's real port, matching + * how a client actually reaches it. */ export async function bootIsolationHub( databaseUrl: string, ): Promise { const root = mkdtempSync(path.join(tmpdir(), "isolation-suite-")); const hubDataDir = path.join(root, "hub-data"); - const sidecarDataDir = path.join(root, "sidecar-data"); - - // Unique per run: the `sidecar` table's id is a primary key, and a - // prior run's row is never cleaned up (the isolation suite runs - // against a shared database, never a scratch one it owns outright). - const sidecarId = `sidecar-isolation-suite-${crypto.randomUUID()}`; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(databaseUrl, sidecarId, sidecarToken); const hub: HubHandle = await startHub({ databaseUrl, @@ -133,23 +122,15 @@ export async function bootIsolationHub( }, }); - const sidecar: SpawnedApp = startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }); - const app: AppLike = { request: (requestPath, init) => fetch(`${hub.baseUrl}${requestPath}`, init), }; - await waitForSidecarDialIn(app, sidecar); + await waitForSidecarDialIn(app, hub); return { app, shutdown: async () => { - await sidecar.stop(); await hub.stop(); rmSync(root, { recursive: true, force: true }); }, @@ -169,7 +150,7 @@ export async function bootIsolationHub( */ async function waitForSidecarDialIn( app: AppLike, - sidecar: SpawnedApp, + hub: HubHandle, ): Promise { const cookie = await signUpUser( app, @@ -194,10 +175,8 @@ async function waitForSidecarDialIn( const deadline = Date.now() + 30_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before dialing in; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before dialing in; output:\n${hub.output()}`); } const workbenchResponse = await app.request( `/api/tenants/${tenant.id}/chat/workbenches`, @@ -212,7 +191,7 @@ async function waitForSidecarDialIn( throw new Error( `sidecar never dialed in within 30s (workbench create kept ` + `failing with ${workbenchResponse.status}): ` + - `${await workbenchResponse.text()}\nsidecar output:\n${sidecar.output()}`, + `${await workbenchResponse.text()}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(500); From 308ab2526a5018b2e65ae4bff4eb17805ecefdc7 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:21 -0700 Subject: [PATCH 15/28] ci: upload hub/sidecar e2e logs as a build artifact (CL-7472) The e2e job's hub/sidecar output was never captured, so the CI run that surfaced the sidecar disconnect loop (34042907014) had no sidecar-side log to read past the transcript. Point E2E_LOG_DIR at a runner-temp directory and upload it on every run, pass or fail. --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a35363400..4e67a69e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,15 @@ jobs: run: bun run test:e2e env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Upload hub/sidecar logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs + path: ${{ runner.temp }}/e2e-logs + if-no-files-found: ignore isolation: runs-on: ubuntu-latest From b72964e24b7d8b384459eca23fa1b15ab45a105f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:01 -0700 Subject: [PATCH 16/28] Add tests for tenant-scoped workbench_launch address resolution (CL-7474) --- packages/chat/src/agent-binding.test.ts | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index ed89cec9d..cd59d2172 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -96,6 +96,56 @@ describe("readBindingByAddress", () => { }); }); +describe("readBindingByAddress: tenant scoping (CL-7474)", () => { + // Two tenants that each independently invited "the same" agent + // (definitionId happens to differ per tenant in practice, but the + // launch rows below are shaped exactly like two independent + // launches — different stable ids, different run ids, different + // tenants). `instanceId`/`currentRunId` are collision-resistant + // generated ids, not scoped to a tenant, so nothing but an explicit + // `expectedTenantId` check stops tenant B's caller from resolving + // tenant A's row if it ever guessed or replayed tenant A's address. + const tenantALaunch: LaunchRow = { + tenantId: "tnt_a", + instanceId: "run_a1", + currentRunId: "run_a1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const tenantBLaunch: LaunchRow = { + tenantId: "tnt_b", + instanceId: "run_b1", + currentRunId: "run_b1", + priorRunIds: [], + foldedBody: FOLDED_BODY, + }; + const db = fakeDb([tenantALaunch, tenantBLaunch]); + + test("each tenant's DM resolves to its own distinct run id and address", async () => { + const a = await readBindingByAddress(db, "run_a1@acme.example", "tnt_a"); + const b = await readBindingByAddress(db, "run_b1@acme.example", "tnt_b"); + expect(a?.currentRunId).toBe("run_a1"); + expect(b?.currentRunId).toBe("run_b1"); + expect(a?.currentRunId).not.toBe(b?.currentRunId); + expect(a?.roomAddress).not.toBe(b?.roomAddress); + }); + + test("a bench invite in tenant B never resolves tenant A's launch", async () => { + expect( + await readBindingByAddress(db, "run_a1@acme.example", "tnt_b"), + ).toBeUndefined(); + expect( + await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), + ).toBeUndefined(); + }); + + test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { + expect( + (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, + ).toBe("tnt_a"); + }); +}); + describe("resolveRoomAddress", () => { test("leaves a non-participant address alone", async () => { expect( From 262a381344ead7f2dec2a47f17f371b3ff4070bb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:50:09 -0700 Subject: [PATCH 17/28] Chat: workbench_launch address/stableId lookups are tenant-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readBindingByAddress and resolveLiveByStableId took an address or stable id and resolved whichever workbench_launch row matched it, regardless of which tenant that row belonged to. Every caller that already knows its own tenant (sendMail's requireLive, and refreshAgentInstanceFromDefinition) now passes it through and gets undefined instead of another tenant's launch if the two ever disagree. The one legitimate tenant-agnostic caller — the event stream handler discovering which tenant an inbound address even belongs to before it has one to check against — keeps working by simply omitting the new optional parameter. Also fixes offeringDigest: it joined catalog offering ids with a NUL byte, which is not valid UTF-8 to Postgres the moment a tenant's catalog carries more than one visible offering ("invalid byte sequence for encoding \"UTF8\": 0x00"), permanently failing the workbench_launch insert for every such launch. This is what was failing 4 of scripts/e2e/chat.test.ts's cases under per-allocation sidecars, where seedCatalog now produces more than one offering. --- packages/chat/src/agent-binding.ts | 25 ++++++++++++++++++++++--- packages/chat/src/platform-adapter.ts | 23 +++++++++++++++++------ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index d53399366..6d3917d4f 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,13 +135,25 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, + expectedTenantId?: string, ): Promise { const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - if (byStableId !== undefined) return bindingFrom(byStableId, domain); - const byRunId = await readLaunchRow(db, "currentRunId", localPart); - return byRunId === undefined ? undefined : bindingFrom(byRunId, domain); + const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + if (row === undefined) return undefined; + // A caller that already knows which tenant it is acting for must + // never accept a binding minted by another one — `instanceId` and + // `currentRunId` are collision-resistant generated ids, not scoped to + // a tenant, so nothing else stops an address that happens to name a + // row from a different tenant from resolving here. A caller with no + // tenant of its own yet (the event-stream handler discovering which + // tenant an inbound address even belongs to) passes no + // `expectedTenantId` and gets the address-only resolution it needs. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } + return bindingFrom(row, domain); } /** @@ -183,9 +195,16 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, + expectedTenantId?: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; + // See the matching note on `readBindingByAddress`: a caller that + // knows its own tenant must never be handed another tenant's launch + // for a stable id it happens to guess or receive by mistake. + if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { + return undefined; + } const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 6490af3c7..f871887da 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -258,7 +258,15 @@ export function createHubChatPlatform( }; function offeringDigest(sourceOfferingIds: readonly string[]): string { - return sourceOfferingIds.join("\0"); + // `sources_digest` is a `text` column and this digest rides straight + // into it (`platform-adapter.ts`'s `insert(workbenchLaunch)`). A + // `\0`-joined id list is not valid UTF-8 text to Postgres + // (`invalid byte sequence for encoding "UTF8": 0x00`) the moment a + // tenant's catalog carries more than one visible offering, which + // permanently failed every agent launch for such a tenant. `\n` never + // appears inside a generated offering id, so this stays a lossless, + // order-sensitive join — just one Postgres can actually store. + return sourceOfferingIds.join("\n"); } async function catalogOfferings(tenantId: string): Promise<{ @@ -913,8 +921,11 @@ export function createHubChatPlatform( * participant id — not the room's own address, once anything has been * relaunched. */ - async function requireLive(stableId: string): Promise { - const live = await resolveLiveByStableId(deps.db, stableId); + async function requireLive( + stableId: string, + expectedTenantId?: string, + ): Promise { + const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1176,7 +1187,7 @@ export function createHubChatPlatform( _workbenchId, address, ): Promise { - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddress(deps.db, address, tenantId); if (binding === undefined) return; const live = await resolveLiveAgent(deps.db, binding); const definitionId = live?.run.definitionId; @@ -1216,7 +1227,7 @@ export function createHubChatPlatform( // The stable id names the room's participant; the run it resolves // to is whichever one is alive right now, which is a different // run (and a different address) after every relaunch. - const { binding } = await requireLive(input.workbenchId); + const { binding } = await requireLive(input.workbenchId, input.tenantId); const liveAddress = binding.liveAddress; // Wake before send: a sleeping instance (the lifecycle package's @@ -1270,7 +1281,7 @@ export function createHubChatPlatform( } catch (error) { throw wrapWakeInferenceError(error); } - const delivery = await requireLive(input.workbenchId); + const delivery = await requireLive(input.workbenchId, input.tenantId); const deliveryAddress = delivery.binding.liveAddress; // Tracking here (not only at launch) brings instances that were // already resident before this hub process started — restored by From b75eebc071aaa7060109de799db16375504bc0bf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:01:01 -0700 Subject: [PATCH 18/28] Chat: run prettier on the tenant-scoping change --- packages/chat/src/agent-binding.ts | 3 ++- packages/chat/src/platform-adapter.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 6d3917d4f..18bad4513 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -140,7 +140,8 @@ export async function readBindingByAddress( const domain = requireDomain(address); const localPart = localPartOf(address); const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index f871887da..602ec0b0d 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -925,7 +925,11 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId(deps.db, stableId, expectedTenantId); + const live = await resolveLiveByStableId( + deps.db, + stableId, + expectedTenantId, + ); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } From 0eef0e0034af3e0c5cf7870b1d0e3cba8cfb72ba Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:26:38 -0700 Subject: [PATCH 19/28] Chat: tenant-scoped binding reads require the tenant; the one unscoped read is named --- packages/chat/src/agent-binding.test.ts | 22 +++++-- packages/chat/src/agent-binding.ts | 82 +++++++++++++++++++------ packages/chat/src/chat-orchestrator.ts | 10 ++- packages/chat/src/platform-adapter.ts | 31 +++++++--- 4 files changed, 109 insertions(+), 36 deletions(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index cd59d2172..947404cf3 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -5,6 +5,7 @@ import { describe, expect, test } from "bun:test"; import { isBeyondWake, readBindingByAddress, + readBindingByAddressAnyTenant, resolveRoomAddress, } from "./agent-binding"; @@ -71,6 +72,7 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_original@acme.example", + "ten_1", ); expect(binding?.stableId).toBe("run_original"); expect(binding?.currentRunId).toBe("run_fresh"); @@ -85,13 +87,18 @@ describe("readBindingByAddress", () => { const binding = await readBindingByAddress( fakeDb([relaunched]), "run_fresh@acme.example", + "ten_1", ); expect(binding?.roomAddress).toBe("run_original@acme.example"); }); test("is undefined for an address this package never launched", async () => { expect( - await readBindingByAddress(fakeDb([relaunched]), "echo_1@acme.example"), + await readBindingByAddress( + fakeDb([relaunched]), + "echo_1@acme.example", + "ten_1", + ), ).toBeUndefined(); }); }); @@ -139,10 +146,17 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { ).toBeUndefined(); }); - test("omitting expectedTenantId keeps the pre-existing address-only resolution (event-stream discovery)", async () => { +}); + +describe("readBindingByAddressAnyTenant", () => { + test("the address-only resolution event-stream discovery needs, with no tenant to check against", async () => { + const anyTenantDb = fakeDb([ + { ...relaunched, instanceId: "run_a1", currentRunId: "run_a1" }, + ]); expect( - (await readBindingByAddress(db, "run_a1@acme.example"))?.tenantId, - ).toBe("tnt_a"); + (await readBindingByAddressAnyTenant(anyTenantDb, "run_a1@acme.example")) + ?.tenantId, + ).toBe("ten_1"); }); }); diff --git a/packages/chat/src/agent-binding.ts b/packages/chat/src/agent-binding.ts index 18bad4513..47b4d0d71 100644 --- a/packages/chat/src/agent-binding.ts +++ b/packages/chat/src/agent-binding.ts @@ -135,26 +135,48 @@ async function readLaunchRow( export async function readBindingByAddress( db: DB["db"], address: string, - expectedTenantId?: string, + tenantId: string, ): Promise { - const domain = requireDomain(address); - const localPart = localPartOf(address); - const byStableId = await readLaunchRow(db, "instanceId", localPart); - const row = - byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + const row = await readBindingRowByAddress(db, address); if (row === undefined) return undefined; // A caller that already knows which tenant it is acting for must // never accept a binding minted by another one — `instanceId` and // `currentRunId` are collision-resistant generated ids, not scoped to // a tenant, so nothing else stops an address that happens to name a - // row from a different tenant from resolving here. A caller with no - // tenant of its own yet (the event-stream handler discovering which - // tenant an inbound address even belongs to) passes no - // `expectedTenantId` and gets the address-only resolution it needs. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } - return bindingFrom(row, domain); + // row from a different tenant from resolving here. `tenantId` is + // required precisely so a caller cannot silently fall back to the + // unscoped read by forgetting to pass it — see + // `readBindingByAddressAnyTenant` for the one place that read is + // actually correct. + if (row.row.tenantId !== tenantId) return undefined; + return bindingFrom(row.row, row.domain); +} + +/** + * The unscoped counterpart of `readBindingByAddress`, for the one caller + * that genuinely cannot know the tenant yet: the inbound event-stream + * handler discovering which tenant an address even belongs to before it + * can act as that tenant. Every other caller already knows its own + * tenant and must use `readBindingByAddress` instead. + */ +export async function readBindingByAddressAnyTenant( + db: DB["db"], + address: string, +): Promise { + const row = await readBindingRowByAddress(db, address); + return row === undefined ? undefined : bindingFrom(row.row, row.domain); +} + +async function readBindingRowByAddress( + db: DB["db"], + address: string, +): Promise<{ row: LaunchRow; domain: string } | undefined> { + const domain = requireDomain(address); + const localPart = localPartOf(address); + const byStableId = await readLaunchRow(db, "instanceId", localPart); + const row = + byStableId ?? (await readLaunchRow(db, "currentRunId", localPart)); + return row === undefined ? undefined : { row, domain }; } /** @@ -167,7 +189,7 @@ export async function resolveRoomAddress( db: DB["db"], liveAddress: string, ): Promise { - const binding = await readBindingByAddress(db, liveAddress); + const binding = await readBindingByAddressAnyTenant(db, liveAddress); return binding?.roomAddress ?? liveAddress; } @@ -196,16 +218,36 @@ export async function resolveLiveAgent( export async function resolveLiveByStableId( db: DB["db"], stableId: string, - expectedTenantId?: string, + tenantId: string, ): Promise { const row = await readLaunchRow(db, "instanceId", stableId); if (row === undefined) return undefined; // See the matching note on `readBindingByAddress`: a caller that // knows its own tenant must never be handed another tenant's launch - // for a stable id it happens to guess or receive by mistake. - if (expectedTenantId !== undefined && row.tenantId !== expectedTenantId) { - return undefined; - } + // for a stable id it happens to guess or receive by mistake. Required, + // not optional — see `resolveLiveByStableIdAnyTenant` for the unscoped + // read. + if (row.tenantId !== tenantId) return undefined; + return resolveLiveRunForRow(db, row); +} + +/** + * The unscoped counterpart of `resolveLiveByStableId`, for a caller that + * has no tenant of its own to check against (e.g. it is resolving a + * stable id it does not yet know the tenant for). + */ +export async function resolveLiveByStableIdAnyTenant( + db: DB["db"], + stableId: string, +): Promise { + const row = await readLaunchRow(db, "instanceId", stableId); + return row === undefined ? undefined : resolveLiveRunForRow(db, row); +} + +async function resolveLiveRunForRow( + db: DB["db"], + row: LaunchRow, +): Promise { const run = await readRun(db, row.currentRunId); if (run === undefined || run.address === null) return undefined; return { binding: bindingFrom(row, requireDomain(run.address)), run }; diff --git a/packages/chat/src/chat-orchestrator.ts b/packages/chat/src/chat-orchestrator.ts index 808c755ba..73ce91b0c 100644 --- a/packages/chat/src/chat-orchestrator.ts +++ b/packages/chat/src/chat-orchestrator.ts @@ -57,7 +57,10 @@ import { consumerFacingInferenceText } from "./consumer-inference-text"; import type { ConnectedProviderLister } from "./inference-preferences"; import { mentionedParticipants } from "./mentions"; import { domainOf, localPartOf } from "./agent-address"; -import { readBindingByAddress, resolveLiveAgent } from "./agent-binding"; +import { + readBindingByAddressAnyTenant, + resolveLiveAgent, +} from "./agent-binding"; import { parseParticipants, type ParticipantRecord } from "./participants"; import type { Part, TextPart } from "./parts"; import type { ChatPlatform } from "./platform-port"; @@ -448,7 +451,10 @@ async function resolveMemberWorkbenches( // event's address against `workflow_run.address` directly: after a // relaunch the two differ, and only the mapping knows that the run // announcing itself under a fresh address is the same room teammate. - const binding = await readBindingByAddress(deps.db, agentAddress); + // The tenant an inbound event's address belongs to is exactly what + // this discovery is trying to find out, so it cannot pass one yet — + // see `readBindingByAddressAnyTenant`. + const binding = await readBindingByAddressAnyTenant(deps.db, agentAddress); if (binding === undefined) { // Not every agent address on the event stream belongs to a chat // workbench (an echo instance, say) — an address this package's own diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 602ec0b0d..39e52ff2c 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -33,11 +33,13 @@ import { listLaunchesBeyondWake, listLaunchesForTenant, readBindingByAddress, + readBindingByAddressAnyTenant, readPriorRuns, recordSourcesDigest, repointBinding, resolveLiveAgent, resolveLiveByStableId, + resolveLiveByStableIdAnyTenant, type AgentBinding, type LiveAgent, } from "./agent-binding"; @@ -703,7 +705,13 @@ export function createHubChatPlatform( * that treatment, just triggered by content drift instead of death. */ async function wakeByAddress(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // Every caller of `wakeByAddress` already reached `address` through + // a tenant-scoped resolution of its own (`sendMail`'s + // `requireLive(workbenchId, tenantId)`, `ensureAwake`'s caller-held + // address) — re-deriving the binding here is bookkeeping for a wake + // already authorized upstream, not a new trust boundary crossing, so + // the unscoped read is correct. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error( `No workbench_launch binding for address "${address}"; instances ` + @@ -809,7 +817,9 @@ export function createHubChatPlatform( * check that used to never run for it. */ async function reconcileDriftedRun(address: string): Promise { - const binding = await readBindingByAddress(deps.db, address); + // See the matching note in `wakeByAddress`: `address` is always + // reached through a tenant-scoped resolution upstream. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return false; const live = await resolveLiveAgent(deps.db, binding); if (live === undefined || live.run.address === null) return false; @@ -925,11 +935,10 @@ export function createHubChatPlatform( stableId: string, expectedTenantId?: string, ): Promise { - const live = await resolveLiveByStableId( - deps.db, - stableId, - expectedTenantId, - ); + const live = + expectedTenantId === undefined + ? await resolveLiveByStableIdAnyTenant(deps.db, stableId) + : await resolveLiveByStableId(deps.db, stableId, expectedTenantId); if (live === undefined) { throw new Error(`No live workbench run for "${stableId}"`); } @@ -1166,7 +1175,9 @@ export function createHubChatPlatform( }, async resolveDefinitionIdByAddress(address): Promise { - const binding = await readBindingByAddress(deps.db, address); + // No tenant is threaded through this platform method's own + // signature; unscoped by necessity, not by omission. + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) return undefined; const live = await resolveLiveAgent(deps.db, binding); return live?.run.definitionId ?? undefined; @@ -1409,7 +1420,7 @@ export function createHubChatPlatform( let cancelled = false; let unsubscribeAgent: (() => void) | undefined; - void resolveLiveByStableId(deps.db, workbenchId) + void resolveLiveByStableIdAnyTenant(deps.db, workbenchId) .then((live) => { if (cancelled || live === undefined) return; unsubscribeAgent = deps.sidecarRouter.subscribeAgent( @@ -1437,7 +1448,7 @@ export function createHubChatPlatform( // undelivered-mail handler holds whatever the envelope named), so // the lifecycle is driven on the LIVE address it resolves to — // that is the only address the sidecar ever announces. - const binding = await readBindingByAddress(deps.db, address); + const binding = await readBindingByAddressAnyTenant(deps.db, address); if (binding === undefined) { throw new Error(`No workbench_launch binding for address "${address}"`); } From b6774510f8c69df16be37b566295d8f49b402ad3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:08 -0700 Subject: [PATCH 20/28] Add tests for the shared retry-until-routable delivery helper --- packages/webhook-triggers/test/launch.test.ts | 36 +++++++++ .../src/deliver-when-routable.test.ts | 74 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 packages/workflows/src/deliver-when-routable.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index f1a9bb2c1..fa7559086 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -137,6 +137,7 @@ let sendUserMessageCalls: unknown[] = []; let sendUserMessageImpl: () => Promise = async () => new Uint8Array([1]); let cryptoGetKeys: string[] = []; +let isRoutableForTest = true; function baseDeps() { return { @@ -177,6 +178,7 @@ function baseDeps() { return sendUserMessageImpl(); }, }, + isRoutable: () => isRoutableForTest, }; } @@ -191,6 +193,7 @@ function resetLaunchSpies() { visibleOfferings = [...DEFAULT_VISIBLE_OFFERINGS]; frozenProjection = INERT_PROJECTION; sendUserMessageImpl = async () => new Uint8Array([1]); + isRoutableForTest = true; } describe("launchWebhookTrigger", () => { @@ -214,6 +217,39 @@ describe("launchWebhookTrigger", () => { expect(sendUserMessageCalls).toHaveLength(1); }); + // CL-7476: a freshly provisioned run's sidecar takes several seconds to + // boot and register with the hub. The opening mail can land in that + // gap and fail "agent is unreachable" even though the run deployed + // fine — `deliverWhenRoutable` must retry once the address becomes + // routable rather than dropping the mail for good. + test("retries the opening mail once the freshly provisioned run becomes routable", async () => { + resetLaunchSpies(); + isRoutableForTest = false; + let sendAttempts = 0; + sendUserMessageImpl = async () => { + sendAttempts += 1; + if (sendAttempts === 1) { + throw new Error("agent is unreachable: ins_1@ten1.workbench.test"); + } + return new Uint8Array([1]); + }; + + const deps = baseDeps(); + const sendPromise = launchWebhookTrigger(deps, TRIGGER, { status: "ok" }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + isRoutableForTest = true; + + const result = await sendPromise; + + expect(result).toEqual({ + instanceId: INTERCHANGE_RUN_ID, + triggerAddress: INTERCHANGE_ADDRESS, + }); + expect(sendUserMessageCalls).toHaveLength(2); + expect(reportErrorCalls).toHaveLength(0); + }); + test("reports the delivery failure with the run's context", async () => { resetLaunchSpies(); const deliveryError = new Error("sidecar unreachable"); diff --git a/packages/workflows/src/deliver-when-routable.test.ts b/packages/workflows/src/deliver-when-routable.test.ts new file mode 100644 index 000000000..b283de6a1 --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { deliverWhenRoutable } from "./deliver-when-routable"; + +function unreachable(): Error { + return new Error("agent is unreachable: run_1@ten1.workbench.test"); +} + +describe("deliverWhenRoutable", () => { + test("retries once the address is routable after an unreachable send", async () => { + let sendCalls = 0; + let routablePolls = 0; + const sleeps: number[] = []; + + const result = await deliverWhenRoutable({ + send: async () => { + sendCalls++; + if (sendCalls === 1) { + throw unreachable(); + } + return "delivered"; + }, + isRoutable: () => { + routablePolls++; + return routablePolls >= 2; + }, + sleep: async (ms) => { + sleeps.push(ms); + }, + }); + + expect(result).toBe("delivered"); + expect(sendCalls).toBe(2); + expect(routablePolls).toBe(2); + expect(sleeps.length).toBe(1); + }); + + test("rethrows the original unreachable error once the deadline passes", async () => { + let now = 0; + const originalNow = Date.now; + Date.now = () => now; + try { + let sendCalls = 0; + await expect( + deliverWhenRoutable({ + send: async () => { + sendCalls++; + throw unreachable(); + }, + isRoutable: () => false, + deadlineMs: 500, + pollIntervalMs: 100, + sleep: async (ms) => { + now += ms; + }, + }), + ).rejects.toThrow("agent is unreachable"); + expect(sendCalls).toBe(1); + } finally { + Date.now = originalNow; + } + }); + + test("does not retry an error other than unreachable", async () => { + const err = new Error("boom"); + await expect( + deliverWhenRoutable({ + send: async () => { + throw err; + }, + isRoutable: () => true, + }), + ).rejects.toBe(err); + }); +}); From 5cb3164f04a8ae0406b4be5a50bb50eb379107ff Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:12 -0700 Subject: [PATCH 21/28] Deliver a webhook trigger's opening mail once its sidecar becomes routable (CL-7476) --- apps/hub/src/index.ts | 2 + packages/webhook-triggers/src/launch.ts | 32 +++++++--- .../workflows/src/deliver-when-routable.ts | 63 +++++++++++++++++++ packages/workflows/src/index.ts | 5 ++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 packages/workflows/src/deliver-when-routable.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 687bcde64..3914c6bef 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2258,6 +2258,8 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, credentialCipher, + isRoutable: (address) => + sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 0e6e03b68..6f3b3adfa 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -14,6 +14,7 @@ // reports through `@corbits/error-sink`, naming the run. import { reportError } from "@corbits/error-sink"; import { + deliverWhenRoutable, readDefinitionProjection, readFoldedBody, WORKFLOW_SOURCE_ENTRY, @@ -46,6 +47,15 @@ export type LaunchWebhookTriggerDeps = { "prepareProvisionedDeployment" >; sessionService: Pick; + /** + * Reads the hub's live sidecar routing table. A freshly provisioned + * run's sidecar takes several seconds to boot and register (CL-7476) + * — a webhook's opening mail can land in that gap and fail with + * "agent is unreachable" even though the run itself deployed fine. + * `deliverWhenRoutable` polls this until the address is routable + * before its one retry. + */ + isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; /** * Host-supplied cipher. Interchange decrypts bindings inside @@ -201,15 +211,19 @@ export async function launchWebhookTrigger( prepared.anchorRunId, ); try { - await deps.sessionService.sendUserMessage({ - agentAddress: prepared.deploymentAddress, - from: `webhook-trigger:${trigger.id}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content, - sessionId, - tenantId: trigger.tenantId, - cryptoProvider, + await deliverWhenRoutable({ + send: () => + deps.sessionService.sendUserMessage({ + agentAddress: prepared.deploymentAddress, + from: `webhook-trigger:${trigger.id}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content, + sessionId, + tenantId: trigger.tenantId, + cryptoProvider, + }), + isRoutable: () => deps.isRoutable(prepared.deploymentAddress), }); } catch (error) { reportError(error, { diff --git a/packages/workflows/src/deliver-when-routable.ts b/packages/workflows/src/deliver-when-routable.ts new file mode 100644 index 000000000..af6aa21c1 --- /dev/null +++ b/packages/workflows/src/deliver-when-routable.ts @@ -0,0 +1,63 @@ +// CL-7476: since the process provisioner started spawning one sidecar per +// allocation, a freshly launched run's first mail can arrive before that +// sidecar has finished booting and registering with the hub. The send +// fails with "agent is unreachable" even though nothing is actually wrong +// — the address just isn't routable yet. Chat's send/fan-out path, its +// invite first-turn delivery, and the webhook trigger's ingress delivery +// each hit this same race independently; this is the one shared retry +// they all go through instead of three ad hoc copies. +const DEFAULT_DEADLINE_MS = 20_000; +const DEFAULT_POLL_INTERVAL_MS = 250; + +export function isAgentUnreachableError(err: unknown): boolean { + return err instanceof Error && err.message.includes("agent is unreachable"); +} + +export type DeliverWhenRoutableOptions = { + /** Performs the actual delivery. Called once, and again after the + * address becomes routable if the first attempt fails unreachable. */ + send: () => Promise; + /** Reads the hub's live routing table for this delivery's address. */ + isRoutable: () => boolean; + /** Defaults to matching Interchange's "agent is unreachable" message. */ + isUnreachable?: (err: unknown) => boolean; + /** How long to keep polling before giving up. Defaults to a budget a + * touch past the process provisioner's own boot-and-register time. */ + deadlineMs?: number; + pollIntervalMs?: number; + sleep?: (ms: number) => Promise; +}; + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Attempts `send`; if it fails with "agent is unreachable", polls + * `isRoutable` until the address is routable (or the deadline passes) + * and sends exactly once more. Any other failure, or an unreachable + * failure that never resolves before the deadline, is rethrown as-is — + * this never swallows the original error or retries indefinitely. + */ +export async function deliverWhenRoutable( + opts: DeliverWhenRoutableOptions, +): Promise { + const isUnreachable = opts.isUnreachable ?? isAgentUnreachableError; + try { + return await opts.send(); + } catch (err) { + if (!isUnreachable(err)) { + throw err; + } + const deadline = Date.now() + (opts.deadlineMs ?? DEFAULT_DEADLINE_MS); + const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const sleep = opts.sleep ?? defaultSleep; + while (!opts.isRoutable()) { + if (Date.now() >= deadline) { + throw err; + } + await sleep(pollIntervalMs); + } + return await opts.send(); + } +} diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 9fefe188a..92c132180 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,11 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + deliverWhenRoutable, + isAgentUnreachableError, + type DeliverWhenRoutableOptions, +} from "./deliver-when-routable"; export { listScheduledWorkflowDefinitions, scheduledDefinitionsFromRows, From a0c8a27ab25a0fa239060eb282e8b6e9b1ffbbbc Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:33:16 -0700 Subject: [PATCH 22/28] Log a skip, not an error, when root tenant seed has no operator key configured --- apps/hub/src/system-seed.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 1637f2e89..f2f9df7c0 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -35,6 +35,12 @@ const log = getLogger(["hub", "system-seed"]); const DEFAULT_DEADLINE_MS = 60_000; const DEFAULT_POLL_INTERVAL_MS = 2_000; +// Matches `packages/seeding/src/seed.ts`'s `resolveRealSourceOfferingIds` +// failure verbatim — the one deploy failure that is expected, not +// transient, when no operator seed key is configured. +const NO_CATALOG_OFFERINGS_REASON = + "this tenant has no catalog offerings to deploy against"; + // Matches `workbench seed`'s own default model source (readSeedConfig in // the now-deleted `packages/cli/src/config.ts`): real anthropic/ // claude-sonnet-5 when a hub-owned key is configured (`config.seedModel`), @@ -150,6 +156,18 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { } catch (cause) { const reason = cause instanceof Error ? cause.message : String(cause); if (Date.now() >= deadline) { + // No operator seed key means `seedCatalog` can only ever plant a + // placeholder credential — never a real, launchable offering — + // so a default workflow's deploy permanently has nothing to + // deploy against until an operator configures one. That is the + // expected shape of an unconfigured dev/CI boot, not a fault the + // next boot can retry its way out of, so it is a logged skip + // rather than an error (which `reportError` would otherwise + // paint red on every single boot). + if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; + return; + } reportError(cause, { operation: "system-seed.seedRootTenant" }); log.error`root tenant seed did not complete before its deadline (last error: ${reason}); the next boot will retry`; return; From 1fd89ddc9b40f5a2173f19eb58629baf66412f78 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:37:08 -0700 Subject: [PATCH 23/28] Run prettier --- apps/hub/src/system-seed.ts | 5 ++++- packages/chat/src/agent-binding.test.ts | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index f2f9df7c0..039ad2b4c 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -164,7 +164,10 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { // next boot can retry its way out of, so it is a logged skip // rather than an error (which `reportError` would otherwise // paint red on every single boot). - if (deps.seedModel === undefined && reason.includes(NO_CATALOG_OFFERINGS_REASON)) { + if ( + deps.seedModel === undefined && + reason.includes(NO_CATALOG_OFFERINGS_REASON) + ) { log.info`root tenant seed skipped: no operator seed key configured, so the tenant has no catalog offerings to deploy against yet`; return; } diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf3..f11d61a60 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 3f8fdc95da60a75a6ebbdf51aba59f34c62fea92 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 10:38:30 -0700 Subject: [PATCH 24/28] Chat: run prettier on the tenant-scoping test file --- packages/chat/src/agent-binding.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/chat/src/agent-binding.test.ts b/packages/chat/src/agent-binding.test.ts index 947404cf3..f11d61a60 100644 --- a/packages/chat/src/agent-binding.test.ts +++ b/packages/chat/src/agent-binding.test.ts @@ -145,7 +145,6 @@ describe("readBindingByAddress: tenant scoping (CL-7474)", () => { await readBindingByAddress(db, "run_b1@acme.example", "tnt_a"), ).toBeUndefined(); }); - }); describe("readBindingByAddressAnyTenant", () => { From 46a4a9301e1b0504dcc1ca91a5a326a254c262a2 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:00 -0700 Subject: [PATCH 25/28] Add a DB-gated test for recording a run's agent_session (CL-7477) resolveRunSessionId (Interchange's mail-routing lookup) reads agent_session keyed by a run's principal, but nothing writes that table for a native launch since folded-runs was deleted -- a run's first turn silently fails to persist mail. Proves recordAgentSessionForRun writes a row resolveRunSessionId reads back, and updates the webhook launch fake DB to model workflow_run/agent_session the way the fix now touches them. --- packages/webhook-triggers/test/launch.test.ts | 18 +++ packages/workflows/test/agent-session.test.ts | 106 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 packages/workflows/test/agent-session.test.ts diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index fa7559086..177a5090b 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -88,7 +88,25 @@ function createFakeDb() { query: { workflowDefinition: { findFirst: async () => DEFINITION_ROW }, tenant: { findFirst: async () => TENANT_ROW }, + // `recordAgentSessionForRun` reads the just-provisioned run back by + // id — production never inserts this row itself (Interchange's + // `prepareProvisionedDeployment` mints the anchor row + // transactionally before returning it), so this models the same + // row for `INTERCHANGE_RUN_ID`. + workflowRun: { + findFirst: async () => ({ + id: INTERCHANGE_RUN_ID, + tenantId: TENANT_ROW.id, + definitionId: DEFINITION_ROW.id, + principalId: "prn_interchange", + }), + }, }, + insert: (_table: unknown) => ({ + values: (_values: unknown) => ({ + onConflictDoNothing: async (_opts: unknown) => {}, + }), + }), }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts new file mode 100644 index 000000000..a7446adda --- /dev/null +++ b/packages/workflows/test/agent-session.test.ts @@ -0,0 +1,106 @@ +// DB-gated: skipped when DATABASE_URL is unreachable, matching this +// repo's convention for tests that talk to a real Postgres (see +// `packages/access-tools/test/routes.test.ts`). +// +// CL-7477: before this fix, nothing wrote `agent_session` for a native +// launch, so `resolveRunSessionId` (vendor's mail-routing lookup) could +// never find a session for a run's principal and every launch's first +// turn failed to persist mail. This proves `recordAgentSessionForRun` +// writes a row `resolveRunSessionId` actually reads back. +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; + +import { createDB, schema, type DB } from "@intx/db"; +import { generateId } from "@intx/hub-common"; +import { resolveRunSessionId } from "@intx/hub-sessions"; +import { dbGate } from "../../../scripts/e2e/db-gate"; + +import { recordAgentSessionForRun } from "../src/launch/agent-session"; + +function dbConfigFromUrl(databaseUrl: string) { + const url = new URL(databaseUrl); + return { + host: url.hostname, + port: url.port === "" ? 5432 : Number(url.port), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + database: url.pathname.replace(/^\//, ""), + }; +} + +const databaseUrl = process.env["DATABASE_URL"]; +const describeIfDb = dbGate(databaseUrl, import.meta.path); + +describeIfDb("recordAgentSessionForRun", () => { + let db: DB; + + const tenantId = generateId("tenant"); + const principalId = generateId("principal"); + const definitionId = generateId("workflowDefinition"); + const anchorRunId = generateId("workflowRun"); + const sessionId = generateId("session"); + + beforeAll(async () => { + if (databaseUrl === undefined) return; + db = createDB(dbConfigFromUrl(databaseUrl)); + + await db.db.insert(schema.tenant).values({ + id: tenantId, + name: "Agent Session Test Tenant", + slug: `agent-session-${tenantId}`, + domain: `agent-session-${tenantId}.localhost`, + parentId: null, + config: null, + }); + + await db.db.insert(schema.principal).values({ + id: principalId, + tenantId, + kind: "workflow", + refId: anchorRunId, + status: "active", + }); + + await db.db.insert(schema.workflowDefinition).values({ + id: definitionId, + tenantId, + creatorPrincipalId: principalId, + assetId: null, + name: "agent-session-test-definition", + }); + + await db.db.insert(schema.workflowRun).values({ + id: anchorRunId, + definitionId, + anchorRunId, + tenantId, + principalId, + address: `${anchorRunId}@agent-session-${tenantId}.localhost`, + status: "running", + }); + }); + + afterAll(async () => { + if (databaseUrl === undefined) return; + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, sessionId)); + await db.db + .delete(schema.workflowRun) + .where(eq(schema.workflowRun.id, anchorRunId)); + await db.db + .delete(schema.workflowDefinition) + .where(eq(schema.workflowDefinition.id, definitionId)); + await db.db + .delete(schema.principal) + .where(eq(schema.principal.id, principalId)); + await db.db.delete(schema.tenant).where(eq(schema.tenant.id, tenantId)); + }); + + test("writes an agent_session resolveRunSessionId reads back", async () => { + await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + + const resolved = await resolveRunSessionId(db.db, principalId); + expect(resolved).toBe(sessionId); + }); +}); From 39ea012bd83690459520821c1e078c6de92c75fb Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 11:16:15 -0700 Subject: [PATCH 26/28] Native launch records the run's agent_session so mail and spans persist (CL-7477) Before #632 deleted folded-runs, mintFoldedRun was the only writer of agent_session, the table resolveRunSessionId (Interchange's mail-routing lookup, vendor/intx/hub-sessions/src/hub-session-lookups.ts) reads by a run's principal to find its session. Every current native launch path mints its own sessionId and hands it to prepareProvisionedDeployment, but nothing ever wrote it into agent_session -- a run's first turn logged "Endpoint ... has no session for address" and silently dropped its mail. workflow_run.principal_id is still null the instant prepareProvisionedDeployment returns: a provisioned anchor is born "deployed" with no principal, and only the run's first trigger reconciles one onto it (Interchange's anchorWithPrincipal), through the exact same trigger path every sendUserMessage drives. So the write has to land after that first send, not right after provisioning. @corbits/workflows now exports recordAgentSessionForRun, a shared helper that reads a run's tenant/definition/principal fresh off workflow_run and inserts the session row (idempotent, since every send after the first is a no-op). Every call site that sends a message wires it in right after that send: webhook-triggers' launch, agent-directory's one-shot prompt, and chat's sendRunMail (the one chokepoint every chat send -- invite, wake, relaunch -- funnels through, since an invite alone provisions a run with no message and would only ever see a still-null principal). apps/hub's agent-authored-workflow deployer mints a run with no message of its own and its first trigger runs entirely through Interchange's native route with no Workbench hook to record afterward, so it's left alone. Session teardown reuses the same read: chat's relaunch marks the outgoing run's session ended before repointing the room, and the one-shot prompt's teardown ends its own session once its single turn settles. --- apps/hub/src/index.ts | 9 ++ .../agent-directory/src/one-shot-prompt.ts | 51 +++++++-- packages/chat/src/platform-adapter.ts | 10 ++ packages/chat/src/send-run-mail.ts | 21 ++++ packages/webhook-triggers/src/launch.ts | 21 ++++ packages/workflows/src/index.ts | 6 + .../workflows/src/launch/agent-session.ts | 106 ++++++++++++++++++ 7 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 packages/workflows/src/launch/agent-session.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index bef0a6d6c..c3352c5a3 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2017,6 +2017,15 @@ export async function createHub(config: HubConfig) { } try { + // Not `recordAgentSessionForRun` here: this mints a deployment + // with no message sent against it — a `workflow_run` born this + // way carries no principal until its first trigger (a call this + // deployer never makes) reconciles one onto it, so the call + // would only ever no-op. That trigger runs entirely through + // Interchange's own native route, with no Workbench-owned hook + // to record the session afterward (CL-7477 tracks the chat, + // webhook, and one-shot-prompt launch paths, which all send a + // message of their own and record there instead). const prepared = await workflowAllocationService.prepareProvisionedDeployment({ tenantId, diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index bf742b2d3..f3084ea66 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -8,8 +8,10 @@ import type { AgentLifecycle } from "@corbits/agent-lifecycle"; import { connectorReplyContent, messageRunEnded } from "@corbits/agent-events"; import { reportError } from "@corbits/error-sink"; import { + endAgentSessionForRun, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -162,6 +164,10 @@ async function provisionOnAsset( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null until its first trigger + // reconciles one onto it — `runOneShotPrompt` records the session + // right after the opening prompt actually sends. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -260,6 +266,14 @@ export async function runOneShotPrompt( settled = true; clearTimeout(timer); unsubscribe(); + try { + await endAgentSessionForRun(deps.db, launched.runId); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.end-session", + extra: { runId: launched.runId, reason }, + }); + } try { await deps.undeploy(launched.address, reason); } catch (err) { @@ -290,18 +304,33 @@ export async function runOneShotPrompt( content: input.prompt, cryptoProvider, }); - return; + } else { + await deps.sessionService.sendUserMessage({ + agentAddress: launched.address, + from: `${input.principalId}@${tenantRow.domain}`, + messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, + date: new Date(), + content: input.prompt, + sessionId: launched.sessionId, + tenantId: input.tenantId, + cryptoProvider, + }); + } + // The send above just drove the run's trigger path, the only + // thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now is + // what makes this ever find one. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }); + } catch (err) { + reportError(err, { + operation: "agent-directory.one-shot.recordAgentSession", + extra: { address: launched.address }, + }); } - await deps.sessionService.sendUserMessage({ - agentAddress: launched.address, - from: `${input.principalId}@${tenantRow.domain}`, - messageId: `<${crypto.randomUUID()}@${tenantRow.domain}>`, - date: new Date(), - content: input.prompt, - sessionId: launched.sessionId, - tenantId: input.tenantId, - cryptoProvider, - }); } catch (cause) { reportError(cause, { operation: "agent-directory.one-shot.send", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 39e52ff2c..5a432c5d7 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,6 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, + endAgentSessionForPrincipal, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -342,6 +343,11 @@ export function createHubChatPlatform( ? { toolPackagePins: input.foldedBody.toolPackagePins } : {}), }); + // Not `recordAgentSessionForRun` here: a freshly provisioned run's + // `workflow_run.principal_id` is still null (an invite sits + // un-triggered until someone actually writes into it), so this + // would only ever no-op. `sendRunMail` records it instead, once the + // run's first turn has actually reconciled a principal onto it. return { runId: prepared.anchorRunId, address: prepared.deploymentAddress, @@ -470,6 +476,10 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); + if (run.principalId !== null) { + await endAgentSessionForPrincipal(deps.db, run.principalId); + } + await repointBinding( deps.db, binding, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index 909fefa09..e79a51e8f 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,5 +1,7 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. +import { recordAgentSessionForRun } from "@corbits/workflows"; +import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; import type { SessionService, SidecarRouter } from "@intx/hub-sessions"; @@ -100,5 +102,24 @@ export async function sendRunMail( const mailId = crypto.randomUUID(); const now = new Date(); const rawMIME = await deliverRunMailMIME(deps, params, mailId, now); + // `sendUserMessage` just drove this run's trigger path, which is the + // only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — so the run is mail-routable + // now if it never was before. Best-effort and idempotent past the + // first successful call; a failure here must not swallow the mail + // that already sent. + try { + await recordAgentSessionForRun(deps.db, { + sessionId: params.sessionId, + address: params.agentAddress, + }); + } catch (err) { + reportError(err, { + operation: "chat.sendRunMail.recordAgentSession", + tenantId: params.tenantId, + agentId: params.agentAddress, + extra: { sessionId: params.sessionId }, + }); + } return recordRunMail(deps, params, mailId, now, rawMIME); } diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6f3b3adfa..6fda59774 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -17,6 +17,7 @@ import { deliverWhenRoutable, readDefinitionProjection, readFoldedBody, + recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; @@ -237,6 +238,26 @@ export async function launchWebhookTrigger( }); } + // The delivery attempt above is what drove the run's trigger path, + // the only thing that ever reconciles a principal onto a freshly + // provisioned `workflow_run` (CL-7477) — recording only now, after + // that attempt, is what makes this call ever find one. Best-effort: + // a run that never became routable stays without a session until its + // first successful send. + try { + await recordAgentSessionForRun(deps.db, { + sessionId, + anchorRunId: prepared.anchorRunId, + }); + } catch (error) { + reportError(error, { + operation: "webhookTriggers.launch.recordAgentSession", + tenantId: trigger.tenantId, + agentId: prepared.deploymentAddress, + extra: { instanceId: prepared.anchorRunId }, + }); + } + return { instanceId: prepared.anchorRunId, triggerAddress: prepared.deploymentAddress, diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 92c132180..7eac44dbb 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -55,6 +55,12 @@ export { type ZonedParts, } from "./schedule/cron"; export { scheduleCronFromProjection } from "./schedule/from-projection"; +export { + recordAgentSessionForRun, + endAgentSessionForPrincipal, + endAgentSessionForRun, + type RecordAgentSessionParams, +} from "./launch/agent-session"; export { deliverWhenRoutable, isAgentUnreachableError, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts new file mode 100644 index 000000000..f95c53960 --- /dev/null +++ b/packages/workflows/src/launch/agent-session.ts @@ -0,0 +1,106 @@ +// Every Workbench launch path provisions through Interchange's +// `prepareProvisionedDeployment` with a `sessionId` it mints itself, but +// nothing ever wrote that id into `agent_session` (vendor-owned, +// `vendor/intx/db/src/schema/sessions.ts`) — the table +// `resolveRunSessionId` (`vendor/intx/hub-sessions/src/hub-session-lookups.ts`) +// reads to route a run's outbound mail. Before the folded-runs package +// was deleted it was the only writer of that table; every current +// launcher must now do this itself, through this one shared helper, so +// mail and spans persist against a real session from the run's first +// turn (CL-7477). +// +// Timing matters: `workflow_run.principal_id` (the FK `agent_session` +// keys on) is still null the instant `prepareProvisionedDeployment` +// returns — a provisioned anchor is born "deployed" with no principal, +// and only the run's first trigger reconciles one onto it (Interchange's +// `anchorWithPrincipal`, `vendor/intx/db/src/workflow-run-store.ts`), +// through the exact same trigger path every `sendUserMessage` drives. So +// this cannot run right after provisioning (an invite sits un-triggered +// until someone actually writes into it) — call it after a send that +// just delivered the run's first turn instead. A run with no principal +// yet is a normal, expected state, not a bug: this returns `false` +// rather than throwing, and idempotent past its first successful call. +import { eq } from "drizzle-orm"; +import type { DB } from "@intx/db"; +import { agentSession, workflowRun } from "@intx/db/schema"; + +export type RecordAgentSessionParams = { + /** The id `prepareProvisionedDeployment` was called with. */ + readonly sessionId: string; +} & ({ readonly anchorRunId: string } | { readonly address: string }); + +/** + * Inserts the `agent_session` row a run needs to be mail-routable: the + * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps + * its old name but targets `workflow_definition` now), and `principalId` + * are all read fresh off the `workflow_run` row, never guessed by the + * caller. Returns `false` (and writes nothing) when the row has no + * principal yet, `true` once the session is recorded (or already was — + * a second call for the same `sessionId` is a no-op, not a conflict). + */ +export async function recordAgentSessionForRun( + db: DB["db"], + params: RecordAgentSessionParams, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: + "anchorRunId" in params + ? eq(workflowRun.id, params.anchorRunId) + : eq(workflowRun.address, params.address), + }); + if (runRow === undefined || runRow.principalId === null) { + return false; + } + + const now = new Date(); + await db + .insert(agentSession) + .values({ + id: params.sessionId, + tenantId: runRow.tenantId, + agentId: runRow.definitionId, + principalId: runRow.principalId, + status: "active", + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: agentSession.id }); + return true; +} + +/** + * Marks a run's `agent_session` ended — called only from the places that + * already know the run died: a chat relaunch (the old run's session) or + * a one-shot prompt's teardown. No sweeper; a run nobody reacts to stays + * `active` until something does. + */ +export async function endAgentSessionForPrincipal( + db: DB["db"], + principalId: string, +): Promise { + const endedAt = new Date(); + await db + .update(agentSession) + .set({ status: "ended", endedAt, updatedAt: endedAt }) + .where(eq(agentSession.principalId, principalId)); +} + +/** + * Same as `endAgentSessionForPrincipal`, keyed by the run's own id + * instead — for a caller (e.g. a one-shot prompt's teardown) that only + * ever held the run id, never minted or looked up its principal. + * No-ops for a run row already gone, or one never triggered (no + * principal, hence no session was ever recorded for it). + */ +export async function endAgentSessionForRun( + db: DB["db"], + anchorRunId: string, +): Promise { + const runRow = await db.query.workflowRun.findFirst({ + where: eq(workflowRun.id, anchorRunId), + }); + if (runRow === undefined || runRow.principalId === null) { + return; + } + await endAgentSessionForPrincipal(db, runRow.principalId); +} From f31e0e100206c1af55908abf698ad40e01302a6d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:01 -0700 Subject: [PATCH 27/28] Add tests for the run's event collector create/abandon wiring --- packages/webhook-triggers/test/launch.test.ts | 3 ++ packages/workflows/test/agent-session.test.ts | 53 ++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/webhook-triggers/test/launch.test.ts b/packages/webhook-triggers/test/launch.test.ts index 177a5090b..01b3b6785 100644 --- a/packages/webhook-triggers/test/launch.test.ts +++ b/packages/webhook-triggers/test/launch.test.ts @@ -197,6 +197,9 @@ function baseDeps() { }, }, isRoutable: () => isRoutableForTest, + eventCollectors: { + create: () => undefined, + }, }; } diff --git a/packages/workflows/test/agent-session.test.ts b/packages/workflows/test/agent-session.test.ts index a7446adda..3ae50e221 100644 --- a/packages/workflows/test/agent-session.test.ts +++ b/packages/workflows/test/agent-session.test.ts @@ -15,7 +15,10 @@ import { generateId } from "@intx/hub-common"; import { resolveRunSessionId } from "@intx/hub-sessions"; import { dbGate } from "../../../scripts/e2e/db-gate"; -import { recordAgentSessionForRun } from "../src/launch/agent-session"; +import { + endAgentSessionForRun, + recordAgentSessionForRun, +} from "../src/launch/agent-session"; function dbConfigFromUrl(databaseUrl: string) { const url = new URL(databaseUrl); @@ -98,9 +101,55 @@ describeIfDb("recordAgentSessionForRun", () => { }); test("writes an agent_session resolveRunSessionId reads back", async () => { - await recordAgentSessionForRun(db.db, { sessionId, anchorRunId }); + await recordAgentSessionForRun( + db.db, + { sessionId, anchorRunId }, + { create: () => undefined }, + ); const resolved = await resolveRunSessionId(db.db, principalId); expect(resolved).toBe(sessionId); }); + + test("creates the run's event collector with its address/tenant/session/run id", async () => { + const createCalls: unknown[] = []; + const otherSessionId = generateId("session"); + + await recordAgentSessionForRun( + db.db, + { sessionId: otherSessionId, anchorRunId }, + { + create: (...args: unknown[]) => { + createCalls.push(args); + }, + }, + ); + + expect(createCalls).toEqual([ + [ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + tenantId, + otherSessionId, + anchorRunId, + ], + ]); + + await db.db + .delete(schema.agentSession) + .where(eq(schema.agentSession.id, otherSessionId)); + }); + + test("abandons the run's event collector when the session ends", async () => { + const abandonCalls: string[] = []; + + await endAgentSessionForRun(db.db, anchorRunId, { + abandon: (address: string) => { + abandonCalls.push(address); + }, + }); + + expect(abandonCalls).toEqual([ + `${anchorRunId}@agent-session-${tenantId}.localhost`, + ]); + }); }); From 9260c3a8b06d1f9f227551bb88df0c6573ed7872 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 12:02:07 -0700 Subject: [PATCH 28/28] Native launch creates the run's event collector so turns are recorded (CL-7479) --- apps/hub/src/index.ts | 2 + .../agent-directory/src/one-shot-prompt.ts | 20 ++++++--- packages/chat/src/platform-adapter.ts | 7 ++- packages/chat/src/send-run-mail.ts | 18 +++++--- packages/webhook-triggers/src/launch.ts | 14 ++++-- packages/workflows/src/index.ts | 1 + .../workflows/src/launch/agent-session.ts | 45 +++++++++++++++++-- 7 files changed, 85 insertions(+), 22 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index c3352c5a3..1faa70f3a 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -2272,6 +2272,7 @@ export async function createHub(config: HubConfig) { isRoutable: (address) => sidecarRouter.getRoutableAddresses().includes(address), cryptoProviderCache: cryptoProviders, + eventCollectors, persistLaunch: async (input) => { await workbenchLaunchPersistExtra(input)(db); }, @@ -3069,6 +3070,7 @@ export async function createHub(config: HubConfig) { repoStore: agentRepoStore.repoStore, workflowAllocationService, sessionService, + eventCollectors, }, runnerInput, ), diff --git a/packages/agent-directory/src/one-shot-prompt.ts b/packages/agent-directory/src/one-shot-prompt.ts index f3084ea66..36efb6022 100644 --- a/packages/agent-directory/src/one-shot-prompt.ts +++ b/packages/agent-directory/src/one-shot-prompt.ts @@ -13,6 +13,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -57,6 +58,7 @@ export type OneShotRunnerDeps = { "prepareProvisionedDeployment" >; readonly sessionService: Pick; + readonly eventCollectors: EventCollectorPort; /** * Test seam only. Production never sets these; they default to * Interchange `prepareProvisionedDeployment` and `sendUserMessage`. @@ -267,7 +269,11 @@ export async function runOneShotPrompt( clearTimeout(timer); unsubscribe(); try { - await endAgentSessionForRun(deps.db, launched.runId); + await endAgentSessionForRun( + deps.db, + launched.runId, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.end-session", @@ -321,10 +327,14 @@ export async function runOneShotPrompt( // provisioned `workflow_run` (CL-7477) — recording only now is // what makes this ever find one. try { - await recordAgentSessionForRun(deps.db, { - sessionId: launched.sessionId, - anchorRunId: launched.runId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: launched.sessionId, + anchorRunId: launched.runId, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "agent-directory.one-shot.recordAgentSession", diff --git a/packages/chat/src/platform-adapter.ts b/packages/chat/src/platform-adapter.ts index 5a432c5d7..8b820493d 100644 --- a/packages/chat/src/platform-adapter.ts +++ b/packages/chat/src/platform-adapter.ts @@ -16,7 +16,7 @@ import { and, desc, eq } from "drizzle-orm"; import { authoredDefinitionCandidates, DefinitionProjectionMissingError, - endAgentSessionForPrincipal, + endAgentSessionForRun, readFoldedBody, resolveNewestProjectedDefinition, WORKFLOW_SOURCE_ENTRY, @@ -258,6 +258,7 @@ export function createHubChatPlatform( db: deps.db, sessionService: deps.sessionService, sidecarRouter: deps.sidecarRouter, + eventCollectors: deps.eventCollectors, }; function offeringDigest(sourceOfferingIds: readonly string[]): string { @@ -476,9 +477,7 @@ export function createHubChatPlatform( foldedBody: binding.foldedBody, }); - if (run.principalId !== null) { - await endAgentSessionForPrincipal(deps.db, run.principalId); - } + await endAgentSessionForRun(deps.db, run.id, deps.eventCollectors); await repointBinding( deps.db, diff --git a/packages/chat/src/send-run-mail.ts b/packages/chat/src/send-run-mail.ts index e79a51e8f..b570a22d2 100644 --- a/packages/chat/src/send-run-mail.ts +++ b/packages/chat/src/send-run-mail.ts @@ -1,6 +1,9 @@ // Signs one inbound message via Interchange `sendUserMessage` and // records it on session_mail so the run's own live subscribers see it. -import { recordAgentSessionForRun } from "@corbits/workflows"; +import { + recordAgentSessionForRun, + type EventCollectorPort, +} from "@corbits/workflows"; import { reportError } from "@corbits/error-sink"; import { sessionMail } from "@intx/db/schema"; import type { DB } from "@intx/db"; @@ -11,6 +14,7 @@ export type RunMailDeps = { db: DB["db"]; sessionService: Pick; sidecarRouter: Pick; + eventCollectors: Pick; }; export type SendRunMailParams = { @@ -109,10 +113,14 @@ export async function sendRunMail( // first successful call; a failure here must not swallow the mail // that already sent. try { - await recordAgentSessionForRun(deps.db, { - sessionId: params.sessionId, - address: params.agentAddress, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId: params.sessionId, + address: params.agentAddress, + }, + deps.eventCollectors, + ); } catch (err) { reportError(err, { operation: "chat.sendRunMail.recordAgentSession", diff --git a/packages/webhook-triggers/src/launch.ts b/packages/webhook-triggers/src/launch.ts index 6fda59774..22e686d40 100644 --- a/packages/webhook-triggers/src/launch.ts +++ b/packages/webhook-triggers/src/launch.ts @@ -19,6 +19,7 @@ import { readFoldedBody, recordAgentSessionForRun, WORKFLOW_SOURCE_ENTRY, + type EventCollectorPort, } from "@corbits/workflows"; import { listVisibleOfferings, type DB } from "@intx/db"; import { tenant as tenantTable, workflowDefinition } from "@intx/db/schema"; @@ -58,6 +59,7 @@ export type LaunchWebhookTriggerDeps = { */ isRoutable: (address: string) => boolean; cryptoProviderCache: CryptoProviderCache; + eventCollectors: Pick; /** * Host-supplied cipher. Interchange decrypts bindings inside * `prepareProvisionedDeployment`; this port still owns the field so @@ -245,10 +247,14 @@ export async function launchWebhookTrigger( // a run that never became routable stays without a session until its // first successful send. try { - await recordAgentSessionForRun(deps.db, { - sessionId, - anchorRunId: prepared.anchorRunId, - }); + await recordAgentSessionForRun( + deps.db, + { + sessionId, + anchorRunId: prepared.anchorRunId, + }, + deps.eventCollectors, + ); } catch (error) { reportError(error, { operation: "webhookTriggers.launch.recordAgentSession", diff --git a/packages/workflows/src/index.ts b/packages/workflows/src/index.ts index 7eac44dbb..244d0478a 100644 --- a/packages/workflows/src/index.ts +++ b/packages/workflows/src/index.ts @@ -60,6 +60,7 @@ export { endAgentSessionForPrincipal, endAgentSessionForRun, type RecordAgentSessionParams, + type EventCollectorPort, } from "./launch/agent-session"; export { deliverWhenRoutable, diff --git a/packages/workflows/src/launch/agent-session.ts b/packages/workflows/src/launch/agent-session.ts index f95c53960..58ec8fd40 100644 --- a/packages/workflows/src/launch/agent-session.ts +++ b/packages/workflows/src/launch/agent-session.ts @@ -23,12 +23,27 @@ import { eq } from "drizzle-orm"; import type { DB } from "@intx/db"; import { agentSession, workflowRun } from "@intx/db/schema"; +import type { EventCollectorRegistry } from "@intx/hub-sessions"; export type RecordAgentSessionParams = { /** The id `prepareProvisionedDeployment` was called with. */ readonly sessionId: string; } & ({ readonly anchorRunId: string } | { readonly address: string }); +/** + * The one port every launcher threads the same wrapped + * `EventCollectorRegistry` through (`apps/hub/src/index.ts`'s + * `eventCollectors`) — never a second registry construction. `create` + * is what actually makes `inference_turn`/`turn_part` rows exist for a + * run (CL-7479: nothing else ever called it once `folded-runs` was + * deleted); `abandon` tears the collector down when a run's session + * ends. + */ +export type EventCollectorPort = Pick< + EventCollectorRegistry, + "create" | "abandon" +>; + /** * Inserts the `agent_session` row a run needs to be mail-routable: the * run's `tenantId`, `definitionId` (the `agent_id` FK — the column keeps @@ -41,6 +56,7 @@ export type RecordAgentSessionParams = { export async function recordAgentSessionForRun( db: DB["db"], params: RecordAgentSessionParams, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: @@ -65,6 +81,19 @@ export async function recordAgentSessionForRun( updatedAt: now, }) .onConflictDoNothing({ target: agentSession.id }); + + const agentAddress = "address" in params ? params.address : runRow.address; + if (agentAddress !== null) { + // Idempotent across wakes: a collector already live for this + // address is replaced, matching the registry's own contract + // (`event-collector-registry.ts`'s `create`). + eventCollectors.create( + agentAddress, + runRow.tenantId, + params.sessionId, + runRow.id, + ); + } return true; } @@ -87,14 +116,19 @@ export async function endAgentSessionForPrincipal( /** * Same as `endAgentSessionForPrincipal`, keyed by the run's own id - * instead — for a caller (e.g. a one-shot prompt's teardown) that only - * ever held the run id, never minted or looked up its principal. - * No-ops for a run row already gone, or one never triggered (no - * principal, hence no session was ever recorded for it). + * instead — for a caller (e.g. a one-shot prompt's teardown, or a chat + * relaunch replacing a terminal run) that only ever held the run id, + * never minted or looked up its principal. Also abandons the outgoing + * run's event collector, through the same shared + * `EventCollectorPort` every launcher threads — one mechanism for + * ending a run's session. No-ops for a run row already gone, or one + * never triggered (no principal, hence no session was ever recorded + * for it). */ export async function endAgentSessionForRun( db: DB["db"], anchorRunId: string, + eventCollectors: Pick, ): Promise { const runRow = await db.query.workflowRun.findFirst({ where: eq(workflowRun.id, anchorRunId), @@ -103,4 +137,7 @@ export async function endAgentSessionForRun( return; } await endAgentSessionForPrincipal(db, runRow.principalId); + if (runRow.address !== null) { + eventCollectors.abandon(runRow.address); + } }