From b1e6caa5d695b74c2023ec24af33b09afb344754 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:15 -0700 Subject: [PATCH 01/20] Add test for workflowDeployBody's catalog-offering shape --- scripts/e2e/harness.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 5b4d14800..37f4f93e4 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -11,6 +11,7 @@ import { createCleanupHarness, parseEnvFileDatabaseUrl, runCleanups, + workflowDeployBody, type SpawnedApp, } from "./harness.ts"; @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => { }); }); +describe("workflowDeployBody", () => { + test("carries catalog offering ids, never a raw provider/apiKey source", () => { + const body = workflowDeployBody({ + assetId: "asset-1", + commitSha: "deadbeef", + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).toEqual({ + source: { + kind: "asset", + assetId: "asset-1", + package: { format: "source", commitSha: "deadbeef" }, + }, + entry: (body as { entry: string }).entry, + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).not.toHaveProperty("sources"); + expect(body).not.toHaveProperty("defaultSource"); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( From fae77c0682222b8149cb08b6b169e1caab79d33b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:23 -0700 Subject: [PATCH 02/20] e2e: deploy workflows by catalog offering ids The native workflow deploy route now requires sourceOfferingIds/ defaultSourceOfferingId resolved against the tenant's own catalog, rather than a raw provider/baseURL/apiKey source chain. Update workflowDeployBody accordingly, and extract the zero-cost noop catalog chain smoke-webhook.test.ts already built (an anthropic- plugin provider pointed at the hub's noop-inference endpoint) into a shared seedNoopCatalogOffering harness helper. Every deploying e2e suite now seeds or resolves a real catalog offering before deploying: walking-skeleton keeps a never-real placeholder host (assertNeverRealProvider still guards it), chat resolves the offering seedCatalog already planted rather than seeding a second chain, and heartbeat/workbench-digest/smoke-webhook seed the noop chain via the shared helper. cl-6324-launch-proof's section-mode deploy now reuses the offering id the proof already resolved when it narrowed the bench catalog to its pinned model. --- scripts/e2e/chat.test.ts | 48 +++++++++-- scripts/e2e/cl-6324-launch-proof.ts | 29 ++++--- scripts/e2e/harness.ts | 121 +++++++++++++++++++++++---- scripts/e2e/heartbeat.test.ts | 19 +++-- scripts/e2e/smoke-webhook.test.ts | 89 +++----------------- scripts/e2e/walking-skeleton.test.ts | 23 +++-- scripts/e2e/workbench-digest.test.ts | 19 +++-- 7 files changed, 219 insertions(+), 129 deletions(-) diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index deea7c826..da9b255c8 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { log: () => undefined, }); + // The echo deploy resolves against the catalog offering `seedCatalog` + // just planted for the same provider/model this workflow declares, + // rather than seeding a second inference chain. + const echoModelsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/models`, + undefined, + user1.cookies, + ); + expectStatus("list catalog models", echoModelsListed, 200); + const echoModelRows = ( + echoModelsListed.data as { data: { id: string; canonicalName: string }[] } + ).data; + const echoModelId = echoModelRows.find( + (row) => row.canonicalName === "claude-sonnet-5", + )?.id; + if (echoModelId === undefined) { + throw new Error( + `no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`, + ); + } + + const echoOfferingsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings`, + undefined, + user1.cookies, + ); + expectStatus("list catalog offerings", echoOfferingsListed, 200); + const echoOfferingRows = ( + echoOfferingsListed.data as { data: { id: string; modelId: string }[] } + ).data; + const echoOfferingId = echoOfferingRows.find( + (row) => row.modelId === echoModelId, + )?.id; + if (echoOfferingId === undefined) { + throw new Error( + `no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`, + ); + } + // Seed the echo workflow as a deployed, invitable definition: the // same asset-publish → git-token → smart-HTTP push → native deploy // path `scripts/e2e/walking-skeleton.test.ts` proves end to end. @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { workflowDeployBody({ assetId: echoAssetId, commitSha: echoPushed.commitSha, - sourceId: "src-echo-e2e", - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [echoOfferingId], + defaultSourceOfferingId: echoOfferingId, }), user1.cookies, ); diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index 674315090..ef3be6297 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,7 +369,7 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - await hop("narrow the bench catalog to the pinned model", async () => { + const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { await plantGrant("model-offering:*", "read"); await plantGrant("model-offering:*", "manage"); await plantGrant("model:*", "read"); @@ -409,8 +409,13 @@ async function main(): Promise { "data", "model offerings", ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; for (const offering of offeringRows) { - if (offering.modelId === pinned.id || offering.disabled) continue; + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; const patched = await api( hub.baseUrl, "PATCH", @@ -420,10 +425,17 @@ async function main(): Promise { ); expectStatus(`disable offering ${offering.id}`, patched, 200); } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } console.log( ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); + return pinnedOffering.id; }); const assistantDefinitionId = await hop( @@ -694,7 +706,6 @@ async function main(): Promise { // own durable event log, which is the artefact the milestone's // `RunStarted` assertion was always about. const SECTION_ASSET_NAME = "cl6324-section"; - const SECTION_SOURCE_ID = "cl6324-section-source"; const SECTION_TURN_TIMEOUT_MS = 180_000; const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`; @@ -780,16 +791,8 @@ async function main(): Promise { package: { format: "source", commitSha: pushed.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SECTION_SOURCE_ID, - provider: model.provider, - baseURL: model.baseURL, - apiKey: model.apiKey, - model: model.model, - }, - ], - defaultSource: SECTION_SOURCE_ID, + sourceOfferingIds: [pinnedOfferingId], + defaultSourceOfferingId: pinnedOfferingId, }, user.cookies, ); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 099a7a210..27cd39204 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -653,16 +653,17 @@ export async function pushWorkflowSource(options: { /** * The deploy body a code-sourced asset deployment takes: the pushed * commit is the definition's pin, and the entry names the - * `interchange.workflow` module the sidecar evaluates. + * `interchange.workflow` module the sidecar evaluates. The native route + * resolves inference against the tenant's own catalog, so the caller + * supplies the ordered catalog offering ids to deploy against rather + * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` + * for the zero-cost way to obtain one. */ export function workflowDeployBody(options: { assetId: string; commitSha: string; - sourceId: string; - provider: string; - baseURL: string; - apiKey: string; - model: string; + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; }): Record { return { source: { @@ -671,15 +672,103 @@ export function workflowDeployBody(options: { package: { format: "source", commitSha: options.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: options.sourceId, - provider: options.provider, - baseURL: options.baseURL, - apiKey: options.apiKey, - model: options.model, - }, - ], - defaultSource: options.sourceId, + sourceOfferingIds: options.sourceOfferingIds, + defaultSourceOfferingId: options.defaultSourceOfferingId, }; } + +/** + * Plants the zero-cost catalog chain every deploying e2e suite needs: a + * catalog model, an `anthropic`-plugin provider/credential pointed at + * `noopBaseUrl` (never a real provider host — enforced below), and the + * catalog offering joining them. Returns the offering id a deploy's + * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. + * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + */ +export async function seedNoopCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + noopBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringFieldOf(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringFieldOf(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringFieldOf( + credential.data, + "id", + "create credential", + ); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.noopBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringFieldOf( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; +} + +function stringFieldOf(data: unknown, field: string, what: string): string { + if (typeof data === "object" && data !== null && field in data) { + const value = (data as Record)[field]; + if (typeof value === "string" && value !== "") return value; + } + throw new Error( + `${what}: missing string field "${field}": ${JSON.stringify(data)}`, + ); +} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index c19a6b6a2..9aefb900f 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -203,16 +204,22 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-heartbeat-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 1d248defa..760aca6a3 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -30,7 +30,6 @@ import { } from "../../workflows/heartbeat/src/index.ts"; import { api, - assertNeverRealProvider, connectE2eDb, createCleanupHarness, e2eDatabaseUrl, @@ -39,6 +38,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -138,77 +138,17 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real - // model. Mirrors `workbench-digest.test.ts`'s noop catalog seeding. + // model. const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - assertNeverRealProvider(noopBaseUrl, "noop catalog provider baseURL"); - await hop("noop catalog seeding", async () => { - const model = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringField(model.data, "id", "create catalog model"); - - const provider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringField(provider.data, "id", "create provider"); - - const credential = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringField( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: noopBaseUrl, - credentialId, - }, + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies2) => + api(hub.baseUrl, method, path, body, cookies2), + tenantId, cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringField( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - }); + noopBaseUrl, + }), + ); const assetName = "heartbeat"; const { assetId, commitSha } = await hop( @@ -256,16 +196,11 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); const definitionId = await hop("workflow deploy", async () => { - const sourceId = "src-smoke-webhook-e2e"; - assertNeverRealProvider(noopBaseUrl, "workflow deploy source baseURL"); const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: noopBaseUrl, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; for (;;) { diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 833612dd6..8937f9b5e 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -35,6 +35,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -199,20 +200,30 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { }, ); + // Hop: noop catalog seeding. The native deploy resolves inference + // against a real catalog offering; this suite's chosen offering + // still names an unreachable placeholder host, since deployment + // never calls inference and full run-completion is not asserted. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: "https://inference.invalid", + }), + ); + // Hop: workflow deploy via the native deploy API. Retries while // the hub still answers 502 (the sidecar's dial-in may not have // completed yet); any other failure is final. The inference // source is a placeholder — deployment does not call inference. const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-echo-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index ecf2dc56e..8917bf823 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -207,16 +208,22 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-workbench-digest-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; From b03ce16b54f9de7e1bb8f14a7efce6a50cd7f9ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:43:45 -0700 Subject: [PATCH 03/20] Fix formatting --- scripts/e2e/cl-6324-launch-proof.ts | 125 ++++++++++++++-------------- scripts/e2e/harness.ts | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index ef3be6297..80766c99d 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,74 +369,77 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { - await plantGrant("model-offering:*", "read"); - await plantGrant("model-offering:*", "manage"); - await plantGrant("model:*", "read"); + const pinnedOfferingId = await hop( + "narrow the bench catalog to the pinned model", + async () => { + await plantGrant("model-offering:*", "read"); + await plantGrant("model-offering:*", "manage"); + await plantGrant("model:*", "read"); - const models = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench catalog models", models, 200); - const modelRows = arrayField(models.data, "data", "catalog models") as { - id: string; - canonicalName: string; - }[]; - const pinned = modelRows.find( - (row) => row.canonicalName === proofModelSource.model, - ); - if (pinned === undefined) { - throw new Error( - `the bench catalog carries no model named ${proofModelSource.model}; ` + - `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + const models = await api( + hub.baseUrl, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, + undefined, + user.cookies, ); - } - - const offerings = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench model offerings", offerings, 200); - const offeringRows = arrayField( - offerings.data, - "data", - "model offerings", - ) as { id: string; modelId: string; disabled: boolean }[]; - let pinnedOffering: { id: string; modelId: string } | undefined; - for (const offering of offeringRows) { - if (offering.modelId === pinned.id) { - pinnedOffering = offering; - continue; + expectStatus("list the bench catalog models", models, 200); + const modelRows = arrayField(models.data, "data", "catalog models") as { + id: string; + canonicalName: string; + }[]; + const pinned = modelRows.find( + (row) => row.canonicalName === proofModelSource.model, + ); + if (pinned === undefined) { + throw new Error( + `the bench catalog carries no model named ${proofModelSource.model}; ` + + `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + ); } - if (offering.disabled) continue; - const patched = await api( + + const offerings = await api( hub.baseUrl, - "PATCH", - `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, - { disabled: true }, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, + undefined, user.cookies, ); - expectStatus(`disable offering ${offering.id}`, patched, 200); - } - if (pinnedOffering === undefined) { - throw new Error( - `no catalog offering resolves the pinned model ${proofModelSource.model} ` + - `(id ${pinned.id})`, + expectStatus("list the bench model offerings", offerings, 200); + const offeringRows = arrayField( + offerings.data, + "data", + "model offerings", + ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; + for (const offering of offeringRows) { + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; + const patched = await api( + hub.baseUrl, + "PATCH", + `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, + { disabled: true }, + user.cookies, + ); + expectStatus(`disable offering ${offering.id}`, patched, 200); + } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } + console.log( + ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); - } - console.log( - ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + - `(${String(offeringRows.length - 1)} other offerings disabled)`, - ); - return pinnedOffering.id; - }); + return pinnedOffering.id; + }, + ); const assistantDefinitionId = await hop( "PROOF 1 — 'assistant' is invitable tenant-wide", diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 27cd39204..cebc4e53d 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -760,7 +760,9 @@ export async function seedNoopCatalogOffering(options: { cookies, ); expectStatus("create catalog offering", offering, 201); - return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; + return { + offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), + }; } function stringFieldOf(data: unknown, field: string, what: string): string { From f981798f2aad5db7e0a8c14a0234e8e1860c4cbe Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:53:52 -0700 Subject: [PATCH 04/20] Add tests for deploying default workflows by catalog offering ids seedTenant currently deploys every default workflow with a bare provider/baseURL/apiKey source, but the hub's deploy route now requires sourceOfferingIds/defaultSourceOfferingId resolved from the tenant's catalog. Update the seeding, onboarding, and hub config test suites to expect the new offering-id based deploy body and provider/model-only ModelSource shape ahead of the implementation change. --- apps/hub/test/config.test.ts | 1 - .../test/complete-credential.test.ts | 55 +++++++-- packages/onboarding/test/provision.test.ts | 101 +++++++++++++++++ .../test/routine-launchability.test.ts | 2 - packages/seeding/test/seed.test.ts | 106 +++++++++++++++--- 5 files changed, 237 insertions(+), 28 deletions(-) diff --git a/apps/hub/test/config.test.ts b/apps/hub/test/config.test.ts index 9d3435117..b4d41908e 100644 --- a/apps/hub/test/config.test.ts +++ b/apps/hub/test/config.test.ts @@ -277,7 +277,6 @@ describe("readHubConfig", () => { expect(config.seedModel).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", apiKey: "sk-ant-test", }); }); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c9ac97c2d..8a4ae6a78 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -63,6 +63,36 @@ function seedHandshake(method: string, path: string) { return { ...handshake, cookies: [] }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461). A test that +// never seeds its own catalog (it stubs `seedCatalogFn` instead) still +// needs one listable offering for the "assistant" workflow's own deploy +// to succeed — this fixed single row is that. +function fixedOfferingsResponse() { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function principalsResponse() { return { status: 200, @@ -165,8 +195,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-ant", }); }); @@ -192,8 +220,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "llama3.2", - baseURL: "http://localhost:11434/v1", - apiKey: "ollama", }); }); @@ -339,7 +365,12 @@ describe("modelSourceFor", () => { expect(result.model).toBe("gpt-oss:20b"); }); - test("ollama's baseURLOverride is normalized to the /v1 form", async () => { + // `modelSourceFor` resolves a provider/model pair only — the deploy + // itself resolves inference from the tenant's own catalog offerings + // (CL-7461), so a `baseURLOverride` here is accepted (every caller + // already resolved one for `seedCatalog`'s earlier catalog plant) but + // never affects this resolution. + test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { return resolvedCatalogResponse([ @@ -365,8 +396,6 @@ describe("modelSourceFor", () => { ).toEqual({ provider: "openai-compatible", model: "qwen3.8:27b", - baseURL: "https://home-mac.example.ts.net/v1", - apiKey: "ollama", }); }); }); @@ -808,6 +837,12 @@ describe("completeCredentialSetup", () => { if (method === "GET" && path === `/api/tenants/${TENANT_ID}`) { return tenantResponse(); } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) @@ -1791,6 +1826,12 @@ describe("ensureSeeded (the slow half)", () => { let deploymentCreatePosts = 0; const api: ApiCall = async (method, path, body) => { + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return fixedOfferingsResponse(); + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/grants?`) diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index af1594cbc..b1feabe08 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -87,6 +87,41 @@ function collector() { return { lines, log: (line: string) => lines.push(line) }; } +// `ensureDeployment` resolves a real (non-noop-pinned) workflow's deploy +// source from the tenant's own catalog offerings (CL-7461); every test +// that seeds a real default workflow needs at least one listable, since +// none of these tests seed a catalog of its own. +function catalogOfferingsResponse( + method: string, + path: string, + tenantId: string, +): { status: number; data: unknown; cookies: string[] } | undefined { + if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + return undefined; + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; +} + function firstLoginSeedHub(args: { expectedParentId?: string }) { let principalsCalls = 0; const startedRuns: string[] = []; @@ -206,6 +241,33 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { ) { return { status: 200, data: [], cookies: [] }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if ( method === "GET" && path.startsWith(`/api/tenants/${TENANT_ID}/workflows/definitions`) @@ -638,6 +700,33 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + return { + status: 200, + data: { + data: [ + { + id: "off_1", + tenantId: TENANT_ID, + modelId: "mdl_1", + providerId: "mpr_1", + priority: 0, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + nextCursor: null, + }, + cookies: [], + }; + } if (method === "GET" && path === "/api/me/principals") { return membership(); } @@ -890,6 +979,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1019,6 +1110,8 @@ describe("provisionPersonalTenantIfNeeded", () => { SEEDED_MEMORY_TARBALL, ]); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1133,6 +1226,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1243,6 +1338,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1485,6 +1582,8 @@ describe("provisionPersonalTenantIfNeeded", () => { "missing", ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, @@ -1832,6 +1931,8 @@ describe("provisionPersonalTenantIfNeeded", () => { [], ); if (registry !== undefined) return registry; + const offerings = catalogOfferingsResponse(method, path, TENANT_ID); + if (offerings !== undefined) return offerings; if (method === "GET" && path === "/api/me/principals") { return { status: 200, diff --git a/packages/seeding/test/routine-launchability.test.ts b/packages/seeding/test/routine-launchability.test.ts index 427fa0b00..746e0e202 100644 --- a/packages/seeding/test/routine-launchability.test.ts +++ b/packages/seeding/test/routine-launchability.test.ts @@ -27,8 +27,6 @@ import { const FAKE_MODEL: ModelSource = { provider: "ollama", model: "qwen-test", - baseURL: "http://localhost:11434", - apiKey: "test-key", }; type SerializedStepDefinition = { diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index b6291fc94..720250165 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -38,10 +38,38 @@ installDisposableHubDataDir(); const MODEL = { provider: "anthropic", model: "claude-sonnet-4-5", - baseURL: "https://api.anthropic.com", - apiKey: "sk-test", }; +// The tenant's one real catalog offering — every `seedTenant` test that +// deploys a real (non-noop-pinned) workflow needs at least one of these +// listable, since `ensureDeployment` now resolves its deploy source from +// the tenant's catalog rather than a bare `ModelSource`. +const OFFERING_ID = "off_1"; + +function offeringsPage( + ids: readonly string[] = [OFFERING_ID], +): { status: number; data: unknown } { + return { + status: 200, + data: { + data: ids.map((id, index) => ({ + id, + tenantId: TENANT_ID, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + })), + nextCursor: null, + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -98,6 +126,40 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/workflows/deployments` ) return { status: 200, data: [] }; + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return offeringsPage(); + // The noop catalog chain `ensureNoopCatalogOffering` plants for a + // modelSource-pinned workflow (heartbeat) — every seedTenant test that + // deploys one needs this fresh chain to succeed, so it lives in the + // shared fallback rather than every such test repeating it. + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/models`) + return { status: 201, data: catalogModelRow("mdl_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_noop", "noop") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_noop", "prv_noop", "noop-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_noop", "noop", "cre_noop", "anthropic"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 201, + data: catalogOfferingRow("off_noop", "mdl_noop", "cpv_noop"), + }; const handshake = pristineScheduledDefinitionHandshake(method, path); if (handshake) return handshake; return undefined; @@ -860,11 +922,9 @@ describe("seedTenant", () => { expect(assistant?.displayName).toBe("Myra"); }); - test("NOOP_MODEL_SOURCE resolves to the hub's own noop-inference endpoint", () => { - const resolved = NOOP_MODEL_SOURCE("http://localhost:3000"); - expect(resolved.baseURL).toBe( - "http://localhost:3000/api/chat/noop-inference", - ); + test("NOOP_MODEL_SOURCE names the noop provider/model pair", () => { + const resolved = NOOP_MODEL_SOURCE(); + expect(resolved.provider).toBe("anthropic"); expect(resolved.model).toBe("noop"); }); @@ -934,8 +994,8 @@ describe("seedTenant", () => { (w) => w.assetName === "heartbeat", ); if (!heartbeat) throw new Error("expected the heartbeat workflow"); - const resolved = heartbeat.modelSource?.("http://localhost:3000"); - expect(resolved).toEqual(NOOP_MODEL_SOURCE("http://localhost:3000")); + const resolved = heartbeat.modelSource?.(); + expect(resolved).toEqual(NOOP_MODEL_SOURCE()); }); test("fresh run pushes, deploys, and confirms the heartbeat workflow against the noop source", async () => { @@ -1012,12 +1072,17 @@ describe("seedTenant", () => { expect(definition.triggers[0]?.to).toBe(`heartbeat@${TENANT_DOMAIN}`); expect(definition.stepOrder).toEqual(["heartbeat"]); - // The deploy's own source, not the tenant's real MODEL, is what - // proves the noop pin took effect: it must name the noop provider - // fixture, not the ordinary anthropic/claude-sonnet-4-5 model this - // test file's `args()` helper hands every other workflow. - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).toBe("noop"); + // The deploy's own source offering, not the tenant's real catalog + // offering, is what proves the noop pin took effect: it must name + // the dedicated noop offering `ensureNoopCatalogOffering` plants, + // never the tenant's own `OFFERING_ID` this test file's `baseRoutes` + // fixture hands every real workflow. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual(["off_noop"]); + expect(deployedBody.defaultSourceOfferingId).toBe("off_noop"); const output = lines.join("\n"); expect(output).toContain("deployed workflow heartbeat as dep_3"); @@ -1135,9 +1200,14 @@ describe("seedTenant", () => { ]); expect(definition.stepOrder).toEqual(["workbench-digest"]); - // Defaults deploy against the tenant's real model (not noop). - const deployedBody = deployedSources as { sources: { model: string }[] }; - expect(deployedBody.sources[0]?.model).not.toBe("noop"); + // Defaults deploy against the tenant's real catalog offering, never + // the noop one. + const deployedBody = deployedSources as { + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; + }; + expect(deployedBody.sourceOfferingIds).toEqual([OFFERING_ID]); + expect(deployedBody.defaultSourceOfferingId).toBe(OFFERING_ID); const output = lines.join("\n"); expect(output).toContain("deployed workflow workbench-digest as dep_4"); From 1ce35c4fa2202730eabe41578a95978cff05c8cf Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:00 -0700 Subject: [PATCH 05/20] Seeding: deploy default workflows by catalog offering ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureDeployment posted a bare {provider, baseURL, apiKey, model} source, but the hub's deploy route requires sourceOfferingIds and a defaultSourceOfferingId resolved from the tenant's catalog, so seeding a connect flow's "assistant" workflow 400'd. seedTenant now resolves the tenant's real catalog offerings (sorted by priority, lowest wins the default) for a normal workflow, and plants a dedicated noop catalog offering for a modelSource-pinned one (heartbeat), pointed at the hub's own noop-inference endpoint. ModelSource shrinks to a bare provider/model pair — the baseURL/apiKey it used to carry only mattered for the old deploy body, and callers that still need a key for seeding a catalog (system-seed's root tenant, config's env-key auto-plant) keep it on their own local type instead. --- apps/hub/src/config.ts | 8 +- apps/hub/src/system-seed.ts | 23 +- .../onboarding/src/complete-credential.ts | 51 +-- packages/seeding/src/seed.ts | 316 +++++++++++++++--- 4 files changed, 305 insertions(+), 93 deletions(-) diff --git a/apps/hub/src/config.ts b/apps/hub/src/config.ts index d9fdad334..1d3afcf79 100644 --- a/apps/hub/src/config.ts +++ b/apps/hub/src/config.ts @@ -272,7 +272,6 @@ function parsePositiveMsEnv( const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; // Matches boot-time seeding's own defaults (`system-seed.ts`) exactly, // so a zero-.env-edit local checkout that seeds its admin account @@ -282,10 +281,14 @@ const DEFAULT_PLANT_ADMIN_EMAIL = "alice@example.com"; const DEFAULT_PLANT_ADMIN_PASSWORD = "password123"; const DEFAULT_PLANT_ORG_SLUG = "workbench"; +/** The root tenant's boot-time seed model: a provider/model pair + * `seedTenant` names in every deployed definition, plus the real (or + * placeholder) key `seedCatalog` plants a launchable credential with. No + * `baseURL` — a workflow deploy resolves inference from the tenant's + * catalog offerings, never a bare source tuple (CL-7461). */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; readonly apiKey: string; }; @@ -624,7 +627,6 @@ function seedModelFrom(parsed: ParsedHubEnv): ModelSource | undefined { return { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey, }; } diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 7f9291fc5..52f0ff8e6 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -26,7 +26,6 @@ import { publishCorbitsToolsRegistry, DEFAULT_WORKFLOWS, PLACEHOLDER_CATALOG_API_KEY, - type ModelSource, } from "@corbits/seeding"; import { createHubAPI, signIn, type ApiCall } from "@corbits/hub-api-client"; import { findPersonalTenant } from "@workbench/onboarding"; @@ -43,14 +42,26 @@ const DEFAULT_POLL_INTERVAL_MS = 2_000; // and the catalog is still browsable, just not launchable. const SEED_MODEL_PROVIDER = "anthropic"; const SEED_MODEL = "claude-sonnet-5"; -const SEED_MODEL_BASE_URL = "https://api.anthropic.com"; -function resolvedModel(seedModel: ModelSource | undefined): ModelSource { +/** + * The root tenant's seed model: a provider/model pair for + * `seedTenant`'s deployed definitions, plus the real (or placeholder) + * API key `seedCatalog` needs to plant a launchable credential. Distinct + * from `@corbits/seeding`'s `ModelSource`, which carries no key — + * `seedTenant` never needs one; the deploy itself resolves inference + * from the tenant's catalog offerings (CL-7461). + */ +export type SeedModelConfig = { + readonly provider: string; + readonly model: string; + readonly apiKey: string; +}; + +function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, model: SEED_MODEL, - baseURL: SEED_MODEL_BASE_URL, apiKey: PLACEHOLDER_CATALOG_API_KEY, } ); @@ -60,7 +71,7 @@ export type SystemSeedDeps = { baseUrl: string; orgSlug: string; admin: { email: string; password: string }; - seedModel?: ModelSource; + seedModel?: SeedModelConfig; deadlineMs?: number; pollIntervalMs?: number; }; @@ -116,7 +127,7 @@ export async function runSystemSeed(deps: SystemSeedDeps): Promise { principalId: tenant.principalId, domain: tenant.tenantDomain, }, - model, + model: { provider: model.provider, model: model.model }, pushWorkflow, log: (line) => log.info`${line}`, workflows: DEFAULT_WORKFLOWS, diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index 5af80e3f7..f5f4a930a 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -60,10 +60,7 @@ import { TenantResponse, paginatedSchema, } from "@intx/types"; -import { - ollamaOpenAICompatBaseURL, - type SupportedCredentialProvider, -} from "@corbits/connections/credential-test"; +import { type SupportedCredentialProvider } from "@corbits/connections/credential-test"; import { preferCompletionCapable } from "@corbits/connections/model-capability"; import { CATALOG_SEEDS, @@ -280,13 +277,8 @@ async function resolveOllamaModelSource( api: ApiCall, cookies: string[], tenantId: string, - apiKey: string, - baseURLOverride: string | undefined, ): Promise { const catalogSeed = CATALOG_SEEDS.ollama; - const baseURL = ollamaOpenAICompatBaseURL( - baseURLOverride ?? catalogSeed.provider.baseURL, - ); const response = await api( "GET", @@ -368,39 +360,32 @@ async function resolveOllamaModelSource( return { provider: winner.plugin, model: winner.canonicalName, - baseURL, - apiKey, }; } -/** The `ModelSource` `ensureSeeded` deploys every default workflow - * against. Every provider but `ollama` has a fixed, always-available - * curated model list (`CATALOG_SEEDS`), so its curated default is a safe - * pin. `ollama` is the one provider whose actual model list is - * per-instance and can diverge from that curated name entirely - * (CL-6366) — its resolution defers to `resolveOllamaModelSource`, which - * reads back what `seedCatalog` actually found on the instance rather - * than repeating the static pin. `baseURLOverride` is the - * configurable-base-URL seam every provider but `ollama` ignores (a - * fixed origin); for `ollama` it is the root the person actually pointed - * their instance at, normalized to the OpenAI-compatible `/v1` form this - * deploys against. */ +/** The `ModelSource` `ensureSeeded` names in every deployed definition's + * inference preferences — never what the deploy itself resolves + * inference against (the tenant's catalog offerings, CL-7461). Every + * provider but `ollama` has a fixed, always-available curated model list + * (`CATALOG_SEEDS`), so its curated default is a safe pin. `ollama` is + * the one provider whose actual model list is per-instance and can + * diverge from that curated name entirely (CL-6366) — its resolution + * defers to `resolveOllamaModelSource`, which reads back what + * `seedCatalog` actually found on the instance rather than repeating the + * static pin. `apiKey`/`baseURLOverride` are accepted only because every + * caller already resolved them for `seedCatalog`'s own credential/catalog + * plant, which runs ahead of this in the same connect flow; this + * function itself needs neither. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - apiKey: string, - baseURLOverride?: string, + _apiKey: string, + _baseURLOverride?: string, ): Promise { if (provider === "ollama") { - return resolveOllamaModelSource( - api, - cookies, - tenantId, - apiKey, - baseURLOverride, - ); + return resolveOllamaModelSource(api, cookies, tenantId); } const catalogSeed = CATALOG_SEEDS[provider]; @@ -413,8 +398,6 @@ export async function modelSourceFor( return { provider: catalogSeed.provider.plugin, model: defaultModel.canonicalName, - baseURL: catalogSeed.provider.baseURL, - apiKey, }; } diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 2b34e98c3..20e350303 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -141,11 +141,6 @@ const DILIGENCE_BRIEF_TURN_TIMEOUT_MS = 2 * 60 * 1000; const RUN_START_TIMEOUT_MS = 30_000; const RUN_POLL_INTERVAL_MS = 1000; -// The deploy source the per-step agents launch against. The id is the -// routing key `defaultSource` must name; with exactly one source there -// is exactly one honest value for it. -const SEED_SOURCE_ID = "default"; - // The provider/model pair `noop-inference` (packages/chat/src/noop-inference.ts) // answers for any request, regardless of what is actually sent — the // route ignores its body and `x-api-key` entirely. Naming a distinct @@ -154,24 +149,31 @@ const SEED_SOURCE_ID = "default"; // UI and logs. const NOOP_PROVIDER = "anthropic"; const NOOP_MODEL = "noop"; +// The catalog provider/credential name a noop-pinned deployment's own +// offering is planted under — distinct from any real curated provider +// name (`CATALOG_SEEDS`'s keys), so `ensureNoopCatalogOffering` never +// collides with a tenant's real catalog and is trivially recognizable +// in the hub's own catalog UI. +const NOOP_CATALOG_PROVIDER_NAME = "noop"; +// Priority is meaningless for this offering — it is never included in +// a real workflow's `sourceOfferingIds` (`ensureNoopCatalogOffering`'s +// id is threaded through explicitly, never discovered by sorting), so +// any fixed value is honest here. +const NOOP_OFFERING_PRIORITY = 0; /** - * A `ModelSource` pointed at the hub's own `noop-inference` endpoint + * The provider/model pair naming the hub's own `noop-inference` endpoint * instead of a real provider — the same substitution * `packages/chat/src/platform-adapter.ts`'s `noopSourcesOverride` makes - * for workbench-host launches, reused here so a workflow deployed with - * this source resolves every turn instantly against a constant, - * locally served reply and never reaches a real model. `hubUrl` is the - * same base URL `seedTenant` already receives, so no new configuration - * is required to use it. + * for workbench-host launches, reused here so a workflow's rendered + * definition names this pair in its inference preferences. The actual + * deploy source is a catalog offering pinned at this same endpoint + * (`ensureNoopCatalogOffering`), not this pair directly — a workflow + * deployment resolves inference from catalog offering ids, never a bare + * provider/model/baseURL/apiKey tuple (CL-7461). */ -export function NOOP_MODEL_SOURCE(hubUrl: string): ModelSource { - return { - provider: NOOP_PROVIDER, - model: NOOP_MODEL, - baseURL: `${hubUrl}/api/chat/noop-inference`, - apiKey: "noop", - }; +export function NOOP_MODEL_SOURCE(): ModelSource { + return { provider: NOOP_PROVIDER, model: NOOP_MODEL }; } const GitTokenMintResponse = type({ id: "string", secret: "string" }); @@ -191,11 +193,18 @@ const WorkflowRunTriggerResponse = type({ messageId: "string", }); +/** + * A provider/model pair a deployed workflow's rendered definition names + * in its inference preferences (`DefaultWorkflow.buildJson`'s second + * argument). What a deployment actually resolves inference against is + * the tenant's catalog offerings, resolved separately by `ensureDeployment` + * (or, for a noop-pinned workflow, `ensureNoopCatalogOffering`) — this + * type carries no `baseURL`/`apiKey` because `seedTenant` never needs + * either (CL-7461). + */ export type ModelSource = { readonly provider: string; readonly model: string; - readonly baseURL: string; - readonly apiKey: string; }; /** @@ -240,9 +249,9 @@ export type DefaultWorkflow = { * Renders the definition's JSON given the tenant's mail domain and the * ordered provider/model preferences to deploy against. Takes the bare * preference list — never a full `ModelSource` — so this same - * function serves both `seedTenant`'s HTTP-deploy path (which also - * needs a `ModelSource`'s `baseURL`/`apiKey` for the deployment's - * `sources`, resolved separately) and a native in-process deploy path + * function serves both `seedTenant`'s HTTP-deploy path (whose actual + * deploy source is a set of catalog offering ids, resolved separately + * by `ensureDeployment`) and a native in-process deploy path * (`apps/hub/src/templates/block-workflows.ts`) that only ever has the * tenant's real, possibly multi-entry inference preferences on hand. */ @@ -251,15 +260,16 @@ export type DefaultWorkflow = { inferencePreferences: readonly InferencePreference[], ) => string; /** - * Overrides the deploy's inference source for this workflow only, - * given the hub's own base URL. Present on the catalog-test workflow - * `heartbeat`, which must stay free to run continuously: it names - * `NOOP_MODEL_SOURCE` instead of the tenant's real catalog model. - * Absent on every conversational workflow and on the seeded - * workbench-digest automation, which deploy against the tenant's real - * model. + * Overrides the deploy's inference source for this workflow only. + * Present on the catalog-test workflow `heartbeat`, which must stay + * free to run continuously: it names `NOOP_MODEL_SOURCE` instead of + * the tenant's real catalog model, and `seedTenant` deploys it against + * a dedicated noop catalog offering (`ensureNoopCatalogOffering`) + * rather than the tenant's own resolved offerings. Absent on every + * conversational workflow and on the seeded workbench-digest + * automation, which deploy against the tenant's real model. */ - modelSource?: (hubUrl: string) => ModelSource; + modelSource?: () => ModelSource; /** * When true, PUT the authored definition to `stopped` after deploy so * a native ScheduleTrigger does not fire every tenant at the next @@ -901,7 +911,8 @@ async function ensureDeployment( assetId: string; assetName: string; commitSha: string; - model: ModelSource; + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; }, log: (line: string) => void, ): Promise { @@ -950,16 +961,8 @@ async function ensureDeployment( package: { format: "source", commitSha: args.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SEED_SOURCE_ID, - provider: args.model.provider, - baseURL: args.model.baseURL, - apiKey: args.model.apiKey, - model: args.model.model, - }, - ], - defaultSource: SEED_SOURCE_ID, + sourceOfferingIds: args.sourceOfferingIds, + defaultSourceOfferingId: args.defaultSourceOfferingId, }, cookies, ); @@ -984,6 +987,170 @@ async function ensureDeployment( return deployment.id; } +/** + * Lists every catalog offering owned directly by a tenant, paging past + * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list + * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" + * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s + * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s + * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog + * directly on that same tenant, so the owned list already carries + * everything a plain `workbench seed` or the self-served connect flow + * ever seeded, with no HTTP surface exposing the resolved view to a + * caller outside the hub process. + */ +async function listTenantCatalogOfferings( + api: ApiCall, + cookies: string[], + tenantId: string, +): Promise<(typeof ModelOfferingResponse.infer)[]> { + const items: (typeof ModelOfferingResponse.infer)[] = []; + let cursor: string | undefined; + for (;;) { + const query = + cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings${query}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + items.push(...page.data); + if (page.nextCursor === null) return items; + if (items.length > 10_000) { + throw new HubApiError( + `catalog offerings list for tenant ${tenantId} did not terminate while paging`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + cursor = page.nextCursor; + } +} + +/** + * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow + * deploys against: every enabled offering the tenant owns, ordered by + * priority ascending, with the lowest-priority offering as the default — + * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and + * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply + * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the + * noop offering (`ensureNoopCatalogOffering`) from this list when one + * has already been planted on the same tenant this run — it must never + * win a real workflow's default just because it happens to sort first. + * + * Throws when nothing is left to deploy against: "seeded but not + * launchable" (`seedCatalog` with no credential) is a valid catalog + * state, but a workflow deploy that names zero sources is not a + * lesser success, it is the failure this function exists to catch + * before the hub's own `sourceOfferingIds` validation would. + */ +async function resolveRealSourceOfferingIds( + api: ApiCall, + cookies: string[], + tenantId: string, + excludeOfferingId: string | undefined, +): Promise<{ + sourceOfferingIds: readonly string[]; + defaultSourceOfferingId: string; +}> { + const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) + .filter( + (offering) => + !offering.disabled && offering.id !== excludeOfferingId, + ) + .sort((a, b) => a.priority - b.priority); + const defaultSourceOfferingId = offerings[0]?.id; + if (defaultSourceOfferingId === undefined) { + throw new HubApiError( + "this tenant has no catalog offerings to deploy against — nothing is launchable yet", + "seed the tenant's catalog with a real provider key (or a placeholder credential for a keyless dev/CI run), then re-run: workbench seed", + ); + } + return { + sourceOfferingIds: offerings.map((offering) => offering.id), + defaultSourceOfferingId, + }; +} + +/** + * Plants (or finds) the dedicated catalog offering a noop-pinned + * workflow (`DefaultWorkflow.modelSource`, e.g. `heartbeat`) deploys + * against: a catalog model named `NOOP_MODEL`, a provider pointed at the + * hub's own `noop-inference` endpoint, and a placeholder credential — + * the same `ensureCatalogModel` / `ensureProvider` / `ensureCredential` / + * `ensureCatalogProvider` / `ensureCatalogOffering` sequence + * `seedCatalog` runs for a real provider, run here for this one + * synthetic one. Idempotent, same as every other seed step: a re-run + * finds the existing rows by name and reuses them. + */ +async function ensureNoopCatalogOffering( + api: ApiCall, + cookies: string[], + tenantId: string, + hubUrl: string, + log: (line: string) => void, +): Promise { + const baseURL = `${hubUrl}/api/chat/noop-inference`; + const modelId = await ensureCatalogModel( + api, + cookies, + { tenantId, canonicalName: NOOP_MODEL }, + log, + ); + const providerId = await ensureProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + apiBaseUrl: baseURL, + }, + log, + ); + const credentialId = await ensureCredential( + api, + cookies, + { + tenantId, + providerId, + name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), + secret: PLACEHOLDER_CATALOG_API_KEY, + type: "api_key", + }, + log, + ); + const catalogProviderId = await ensureCatalogProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + baseURL, + credentialId, + }, + log, + ); + return ensureCatalogOffering( + api, + cookies, + { + tenantId, + modelId, + providerId: catalogProviderId, + priority: NOOP_OFFERING_PRIORITY, + capabilities: [], + }, + log, + ); +} + /** * After deploy, PUT a pristine authored definition to `stopped` so a native * ScheduleTrigger does not fire every tenant. A member-restored row @@ -1160,6 +1327,14 @@ export type SeedTenantArgs = { cookies: string[]; hubUrl: string; tenant: SeedTenant; + /** + * The provider/model pair a real (non-noop-pinned) workflow's rendered + * definition names in its inference preferences. The deploy itself + * resolves inference from the tenant's own catalog offerings + * (`resolveRealSourceOfferingIds`), not from this pair — a tenant with + * no catalog offerings still fails clearly at deploy time even if this + * pair happens to name a real model. + */ model: ModelSource; pushWorkflow: WorkflowPusher; log: (line: string) => void; @@ -1228,9 +1403,41 @@ export async function seedTenant(args: SeedTenantArgs): Promise { await plantDefaultSkills(api, cookies, tenant.tenantId, log); + // Resolved lazily and cached across the loop below: most seed runs + // deploy several workflows against the same tenant catalog state, so + // this fetches the tenant's real offerings (or plants the noop one) + // at most once each, however many workflows ask for them. + let noopOfferingId: string | undefined; + let realOfferings: + | { sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string } + | undefined; + let confirmed = 0; for (const workflow of workflows) { - const workflowModel = workflow.modelSource?.(hubUrl) ?? model; + const workflowModel = workflow.modelSource?.() ?? model; + + let sourceOfferingIds: readonly string[]; + let defaultSourceOfferingId: string; + if (workflow.modelSource !== undefined) { + noopOfferingId ??= await ensureNoopCatalogOffering( + api, + cookies, + tenant.tenantId, + hubUrl, + log, + ); + sourceOfferingIds = [noopOfferingId]; + defaultSourceOfferingId = noopOfferingId; + } else { + realOfferings ??= await resolveRealSourceOfferingIds( + api, + cookies, + tenant.tenantId, + noopOfferingId, + ); + ({ sourceOfferingIds, defaultSourceOfferingId } = realOfferings); + } + const assetId = await ensureWorkflowAsset( api, cookies, @@ -1265,7 +1472,8 @@ export async function seedTenant(args: SeedTenantArgs): Promise { assetId, assetName: workflow.assetName, commitSha: pushed.commitSha, - model: workflowModel, + sourceOfferingIds, + defaultSourceOfferingId, }, log, ); @@ -1666,7 +1874,7 @@ async function ensureCatalogOffering( quirks?: Record; }, log: (line: string) => void, -): Promise { +): Promise { const body: Record = { modelId: args.modelId, providerId: args.providerId, @@ -1681,9 +1889,13 @@ async function ensureCatalogOffering( cookies, ); if (created.status === 201) { - parseAs(ModelOfferingResponse, created.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + created.data, + "catalog offering response", + ); log("created catalog offering"); - return; + return offering.id; } if (created.status === 409) { let cursor: string | null = null; @@ -1715,7 +1927,7 @@ async function ensureCatalogOffering( } if (existing.priority === args.priority) { log("catalog offering already exists (skipped)"); - return; + return existing.id; } const updated = await api( @@ -1730,9 +1942,13 @@ async function ensureCatalogOffering( "check the hub logs for the underlying failure, then re-run: workbench seed", ); } - parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + const offering = parseAs( + ModelOfferingResponse, + updated.data, + "catalog offering response", + ); log("updated catalog offering priority"); - return; + return offering.id; } throw new HubApiError( `the hub rejected creation of the catalog offering with status ${created.status}: ${JSON.stringify(created.data)}`, From 2587cd2b5af3ecbb27bb13bcb652910bd10b29aa Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:46 -0700 Subject: [PATCH 06/20] longevity-sim: deploy workflows by catalog offering ids Another workflowDeployBody caller CI's typecheck job caught: this stack seeds one catalog offering per real inference target already, so track each one's id and pass it through to the deploy body instead of the raw provider/baseURL/apiKey shape. --- packages/longevity-sim/src/stack.ts | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/longevity-sim/src/stack.ts b/packages/longevity-sim/src/stack.ts index a7dc91776..59c4ff72a 100644 --- a/packages/longevity-sim/src/stack.ts +++ b/packages/longevity-sim/src/stack.ts @@ -437,7 +437,9 @@ export async function bootLongevityStack( // credential row still requires some string, so each target's own // `apiKey` (expected to be the same placeholder) is threaded through // unchanged rather than this package inventing its own convention. - async function seedCatalogChain(target: InferenceTarget): Promise { + async function seedCatalogChain( + target: InferenceTarget, + ): Promise<{ offeringId: string }> { const model = await api( hub.baseUrl, "POST", @@ -509,10 +511,15 @@ export async function bootLongevityStack( owner.cookies, ); expectStatus(`create catalog offering ${target.model}`, offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; } + const offeringIdByTargetLabel = new Map(); for (const target of options.realTargets) { - await seedCatalogChain(target); + const { offeringId } = await seedCatalogChain(target); + offeringIdByTargetLabel.set(target.label, offeringId); } for (const skill of options.skills ?? []) { @@ -668,6 +675,13 @@ export async function bootLongevityStack( workflowJson, }); + const offeringId = offeringIdByTargetLabel.get(target.label); + if (offeringId === undefined) { + throw new Error( + `agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`, + ); + } + const deployDeadline = Date.now() + 90_000; for (;;) { if (sidecar.exited()) { @@ -682,11 +696,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId, commitSha: pushed.commitSha, - sourceId: `src-agent-${spec.handle}`, - provider: target.provider, - baseURL: target.baseURL, - apiKey: target.apiKey, - model: target.model, + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }), owner.cookies, ); @@ -847,6 +858,12 @@ export async function bootLongevityStack( if (routineTarget === undefined) { throw new Error("unreachable: realTargets checked non-empty above"); } + const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label); + if (routineOfferingId === undefined) { + throw new Error( + `no catalog offering seeded for realTarget "${routineTarget.label}"`, + ); + } const assetName = "longevity-heartbeat"; const heartbeatAssetCreated = await api( hub.baseUrl, @@ -912,11 +929,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId: heartbeatAssetId, commitSha: heartbeatPushed.commitSha, - sourceId: "src-longevity-heartbeat", - provider: "openai-compatible", - baseURL: routineTarget.baseURL, - apiKey: routineTarget.apiKey, - model: routineTarget.model, + sourceOfferingIds: [routineOfferingId], + defaultSourceOfferingId: routineOfferingId, }), owner.cookies, ); From 2430449e41b11a77a87cc902e4e2f7c264736125 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:56:16 -0700 Subject: [PATCH 07/20] Fix formatting: prettier --- apps/hub/src/system-seed.ts | 4 +++- packages/seeding/src/seed.ts | 3 +-- packages/seeding/test/seed.test.ts | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/hub/src/system-seed.ts b/apps/hub/src/system-seed.ts index 52f0ff8e6..1637f2e89 100644 --- a/apps/hub/src/system-seed.ts +++ b/apps/hub/src/system-seed.ts @@ -57,7 +57,9 @@ export type SeedModelConfig = { readonly apiKey: string; }; -function resolvedModel(seedModel: SeedModelConfig | undefined): SeedModelConfig { +function resolvedModel( + seedModel: SeedModelConfig | undefined, +): SeedModelConfig { return ( seedModel ?? { provider: SEED_MODEL_PROVIDER, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 20e350303..24fc1ce6e 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -1060,8 +1060,7 @@ async function resolveRealSourceOfferingIds( }> { const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) .filter( - (offering) => - !offering.disabled && offering.id !== excludeOfferingId, + (offering) => !offering.disabled && offering.id !== excludeOfferingId, ) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 720250165..99f972804 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -46,9 +46,10 @@ const MODEL = { // the tenant's catalog rather than a bare `ModelSource`. const OFFERING_ID = "off_1"; -function offeringsPage( - ids: readonly string[] = [OFFERING_ID], -): { status: number; data: unknown } { +function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { return { status: 200, data: { From 602ad68f72898a1342ec88f5090eac80999493a4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:25 -0700 Subject: [PATCH 08/20] sandbox-sidecar: add tests for generation-0 ensure/destroy requests The allocation service's initial sidecar_allocation and workflow_probe rows start at generation 0 (@intx/db enforces generation >= 0, not > 0), so the first ensure() for a freshly allocated sidecar must be accepted at generation 0. These tests cover that acceptance, later generations, non-integer/negative rejection, and that stale-generation fencing still rejects an older generation once a newer one has been observed. --- .../sandbox-sidecar/src/provisioner.test.ts | 154 ++++++++++++++++++ .../sandbox-sidecar/src/state-store.test.ts | 10 ++ 2 files changed, 164 insertions(+) create mode 100644 packages/sandbox-sidecar/src/provisioner.test.ts diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts new file mode 100644 index 000000000..753478bf0 --- /dev/null +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { createSidecarProvisioner } from "./provisioner"; +import type { SidecarBackend } from "./backend"; + +const TOKEN = "s3cr3t-bootstrap-token"; + +let dataDir: string; +let store: AllocationStateStore; +let started: string[]; + +beforeEach(async () => { + dataDir = await mkdtemp(join(tmpdir(), "sandbox-sidecar-provisioner-")); + store = createAllocationStateStore(join(dataDir, "state.json")); + started = []; +}); + +afterEach(async () => { + await rm(dataDir, { recursive: true, force: true }); +}); + +function fakeBackend(): SidecarBackend { + let counter = 0; + return { + async startUnit() { + counter += 1; + const externalRef = `unit-${String(counter)}`; + started.push(externalRef); + return externalRef; + }, + async stopUnit() {}, + async findUnitsByAllocation() { + return []; + }, + }; +} + +function baseEnsureRequest(overrides: Partial> = {}) { + return { + allocationId: "alloc-1", + generation: 0, + tenantId: "tenant-1", + anchorRunId: "run-1", + sidecarId: "sidecar-1", + token: TOKEN, + hubWebSocketUrl: "wss://hub.example.com/ws", + ...overrides, + } as never; +} + +function makeProvisioner(backend: SidecarBackend = fakeBackend()) { + return createSidecarProvisioner({ + id: "test-provisioner", + apiVersion: 1, + bindingFingerprint: "test", + capabilities: [], + backend, + store, + }); +} + +describe("ensure request validation", () => { + test("accepts generation 0, the allocation service's initial generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); + }); + + test("accepts a later generation", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + + expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: -1 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("rejects a non-integer generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1.5 }), + ); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_ensure_request", + }); + expect(started).toHaveLength(0); + }); + + test("still rejects a stale generation once fencing has observed a later one", async () => { + const provisioner = makeProvisioner(); + + await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + expect(result).toMatchObject({ + kind: "rejected", + code: "stale_generation", + }); + expect(started).toHaveLength(1); + }); +}); + +describe("destroy request validation", () => { + test("accepts generation 0", async () => { + const provisioner = makeProvisioner(); + await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + + expect(result).toEqual({ kind: "destroyed" }); + }); + + test("rejects a negative generation", async () => { + const provisioner = makeProvisioner(); + + const result = await provisioner.destroy({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: -1, + }); + + expect(result).toMatchObject({ + kind: "rejected", + code: "invalid_destroy_request", + }); + }); +}); diff --git a/packages/sandbox-sidecar/src/state-store.test.ts b/packages/sandbox-sidecar/src/state-store.test.ts index d9bdadffb..abd723afd 100644 --- a/packages/sandbox-sidecar/src/state-store.test.ts +++ b/packages/sandbox-sidecar/src/state-store.test.ts @@ -30,6 +30,16 @@ describe("observeEnsure", () => { expect(result.kind).toBe("observed"); }); + test("accepts generation 0, the allocation service's initial generation", async () => { + const store = createAllocationStateStore(statePath()); + const result = await store.observeEnsure({ + allocationId: "alloc-1", + sidecarId: "sidecar-1", + generation: 0, + }); + expect(result.kind).toBe("observed"); + }); + test("is idempotent when observed again at the same generation", async () => { const store = createAllocationStateStore(statePath()); await store.observeEnsure({ From e01fedad96a1991c9b05eb612462fb18bd654db4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:35:30 -0700 Subject: [PATCH 09/20] sandbox-sidecar: accept the allocation service's initial generation @intx/db's sidecar_allocation and workflow_probe tables both check generation >= 0 and the allocation service records the first row for an allocation at generation 0, but validateEnsureRequest() and validateDestroyRequest() in packages/sandbox-sidecar rejected anything that wasn't strictly positive, and the on-disk fencing schema in state-store.ts required generation > 0. The first ensure() for any newly allocated sidecar was rejected with 'generation must be a positive integer', failing every deploy. Accept generation >= 0 in both validators and the state-store schema; stale-generation fencing in observeEnsure()/observeDestroy() already compares against the stored generation rather than assuming positivity, so it is unaffected. --- packages/sandbox-sidecar/src/provisioner.ts | 8 ++++---- packages/sandbox-sidecar/src/state-store.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.ts b/packages/sandbox-sidecar/src/provisioner.ts index 209d9e725..9771be319 100644 --- a/packages/sandbox-sidecar/src/provisioner.ts +++ b/packages/sandbox-sidecar/src/provisioner.ts @@ -218,8 +218,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { if (request.token === "") return "token must not be empty"; if (request.hubWebSocketUrl === "") return "hubWebSocketUrl must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } @@ -227,8 +227,8 @@ function validateEnsureRequest(request: EnsureSidecarRequest): string | null { function validateDestroyRequest(request: DestroySidecarRequest): string | null { if (request.allocationId === "") return "allocationId must not be empty"; if (request.sidecarId === "") return "sidecarId must not be empty"; - if (!Number.isInteger(request.generation) || request.generation <= 0) { - return "generation must be a positive integer"; + if (!Number.isInteger(request.generation) || request.generation < 0) { + return "generation must be a non-negative integer"; } return null; } diff --git a/packages/sandbox-sidecar/src/state-store.ts b/packages/sandbox-sidecar/src/state-store.ts index 12e5bed2a..9e600e228 100644 --- a/packages/sandbox-sidecar/src/state-store.ts +++ b/packages/sandbox-sidecar/src/state-store.ts @@ -12,7 +12,7 @@ import { type } from "arktype"; const AllocationRecord = type({ allocationId: "string > 0", sidecarId: "string > 0", - generation: "number.integer > 0", + generation: "number.integer >= 0", desiredState: "'ensured' | 'destroyed'", externalRef: "string | null", tokenHashSha256: "string | null", From c6da1fa7ba9e2607098f1e566456e8ebc8b84b9b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:37:36 -0700 Subject: [PATCH 10/20] sandbox-sidecar: fix formatting in provisioner test --- .../sandbox-sidecar/src/provisioner.test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/sandbox-sidecar/src/provisioner.test.ts b/packages/sandbox-sidecar/src/provisioner.test.ts index 753478bf0..b858c03cc 100644 --- a/packages/sandbox-sidecar/src/provisioner.test.ts +++ b/packages/sandbox-sidecar/src/provisioner.test.ts @@ -3,7 +3,10 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createAllocationStateStore, type AllocationStateStore } from "./state-store"; +import { + createAllocationStateStore, + type AllocationStateStore, +} from "./state-store"; import { createSidecarProvisioner } from "./provisioner"; import type { SidecarBackend } from "./backend"; @@ -67,7 +70,9 @@ describe("ensure request validation", () => { test("accepts generation 0, the allocation service's initial generation", async () => { const provisioner = makeProvisioner(); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-1" }); }); @@ -76,7 +81,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 0 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 1 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 1 }), + ); expect(result).toEqual({ kind: "accepted", externalRef: "unit-2" }); }); @@ -113,7 +120,9 @@ describe("ensure request validation", () => { const provisioner = makeProvisioner(); await provisioner.ensure(baseEnsureRequest({ generation: 1 })); - const result = await provisioner.ensure(baseEnsureRequest({ generation: 0 })); + const result = await provisioner.ensure( + baseEnsureRequest({ generation: 0 }), + ); expect(result).toMatchObject({ kind: "rejected", From 824c2b69f0c53998d64e5bebcb051c27b7d3ca4c Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:42 -0700 Subject: [PATCH 11/20] Add tests for resolved catalog offerings and modelSourceFor's trimmed signature Covers the new GET .../catalog/resolved-offerings route (denied grant, and both directly-owned and ancestor-inherited offerings sorted by priority) and updates seeding/onboarding test fixtures to the route's response shape ahead of the implementation change. --- .../test/resolved-offerings-routes.test.ts | 99 +++++++++++++++++++ .../test/complete-credential.test.ts | 94 ++++++------------ packages/onboarding/test/provision.test.ts | 48 +++------ packages/seeding/test/seed.test.ts | 23 +++++ 4 files changed, 166 insertions(+), 98 deletions(-) create mode 100644 packages/inference-catalog/test/resolved-offerings-routes.test.ts diff --git a/packages/inference-catalog/test/resolved-offerings-routes.test.ts b/packages/inference-catalog/test/resolved-offerings-routes.test.ts new file mode 100644 index 000000000..897e05313 --- /dev/null +++ b/packages/inference-catalog/test/resolved-offerings-routes.test.ts @@ -0,0 +1,99 @@ +// The resolved-offerings route exists so an out-of-process caller +// (`workbench seed`, CL-7461) can deploy against exactly what the hub's +// own workflow deployer would deploy against — including offerings a +// tenant only inherits from an ancestor, which the tenant-owned +// `GET .../catalog/offerings` route never lists. +import { Hono } from "hono"; +import { describe, expect, test } from "bun:test"; + +import type { TenantEnv } from "@intx/hub-api"; +import type { ResolvedOffering } from "@intx/db"; + +import { createResolvedOfferingsRoutes } from "../src/resolved-offerings-routes"; +import { offering } from "./fixtures"; + +const TENANT = { + id: "bench-1", + name: "Bench One", + slug: "bench-one", + domain: "bench-one.example", + parentId: null, + config: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-01-01T00:00:00.000Z"), +}; + +function app( + listOfferings: (tenantId: string) => Promise, + allow = true, +) { + const host = new Hono(); + host.use("*", async (c, next) => { + c.set("tenant", TENANT); + await next(); + }); + host.route( + "/", + createResolvedOfferingsRoutes({ + listOfferings, + requireGrant: () => async (c, next) => { + if (!allow) { + return c.json({ error: { code: "forbidden", message: "no" } }, 403); + } + return next(); + }, + }), + ); + return host; +} + +describe("GET resolved-offerings", () => { + test("a denied grant is rejected before the offerings are read", async () => { + const response = await app(async () => { + throw new Error("must not be called when the grant is denied"); + }, false).request("/"); + expect(response.status).toBe(403); + }); + + test("lists both directly-owned and ancestor-inherited offerings, sorted by priority", async () => { + const owned = offering({ + id: "owned", + canonicalName: "thrifty", + providerName: "globex", + capabilities: ["plain-text"], + priority: 2, + }); + const inherited = offering({ + id: "inherited", + canonicalName: "lavish", + providerName: "acme", + capabilities: ["plain-text"], + priority: 1, + inherited: true, + }); + + const response = await app(async (tenantId) => { + expect(tenantId).toBe(TENANT.id); + return [owned, inherited]; + }).request("/"); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + offerings: { + id: string; + priority: number; + modelId: string; + providerId: string; + origin: { tenantId: string; direct: boolean }; + }[]; + }; + expect(body.offerings.map((o) => o.id)).toEqual(["inherited", "owned"]); + const inheritedEntry = body.offerings.find((o) => o.id === "inherited"); + expect(inheritedEntry?.origin).toEqual({ + tenantId: "parent-bench", + direct: false, + }); + const ownedEntry = body.offerings.find((o) => o.id === "owned"); + expect(ownedEntry?.origin).toEqual({ tenantId: TENANT.id, direct: true }); + }); +}); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 8a4ae6a78..16e81d0df 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -72,22 +72,15 @@ function fixedOfferingsResponse() { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -179,19 +172,12 @@ function ownedCatalogModelsResponse(canonicalNames: string[]) { } describe("modelSourceFor", () => { - test("every other provider ignores a baseURLOverride and never calls the hub", async () => { + test("every other provider resolves to its curated default and never calls the hub", async () => { const api: ApiCall = (async () => { throw new Error("must not be called for a fixed curated provider"); }) as ApiCall; expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "anthropic", - "sk-ant", - "https://ignored", - ), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "anthropic"), ).toEqual({ provider: "anthropic", model: "claude-sonnet-5", @@ -216,7 +202,7 @@ describe("modelSourceFor", () => { }; expect( - await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama", "ollama"), + await modelSourceFor(api, ["session=abc"], TENANT_ID, "ollama"), ).toEqual({ provider: "openai-compatible", model: "llama3.2", @@ -250,7 +236,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -285,7 +270,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); @@ -323,7 +307,6 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("llama3.2"); }); @@ -360,44 +343,9 @@ describe("modelSourceFor", () => { ["session=abc"], TENANT_ID, "ollama", - "ollama", ); expect(result.model).toBe("gpt-oss:20b"); }); - - // `modelSourceFor` resolves a provider/model pair only — the deploy - // itself resolves inference from the tenant's own catalog offerings - // (CL-7461), so a `baseURLOverride` here is accepted (every caller - // already resolved one for `seedCatalog`'s earlier catalog plant) but - // never affects this resolution. - test("ollama resolves the same model whether or not a baseURLOverride is given", async () => { - const api: ApiCall = async (method, path) => { - if (method === "GET" && path === `/api/tenants/${TENANT_ID}/models`) { - return resolvedCatalogResponse([ - { - canonicalName: "qwen3.8:27b", - providerName: "ollama", - capabilities: ["plain-text"], - }, - ]); - } - throw new Error(`unexpected call: ${method} ${path}`); - }; - - expect( - await modelSourceFor( - api, - ["session=abc"], - TENANT_ID, - "ollama", - "ollama", - "https://home-mac.example.ts.net", - ), - ).toEqual({ - provider: "openai-compatible", - model: "qwen3.8:27b", - }); - }); }); describe("completeCredentialSetup", () => { @@ -780,12 +728,11 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, - data: { data: [], nextCursor: null }, + data: { offerings: [] }, cookies: [], }; } @@ -839,7 +786,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } @@ -1422,8 +1369,7 @@ describe("completeCredentialSetup", () => { } if ( method === "GET" && - (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || - path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { return { status: 200, @@ -1446,6 +1392,24 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) { + return { + status: 200, + data: { + offerings: catalogOfferings.map((o) => ({ + id: o.id, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -1828,7 +1792,7 @@ describe("ensureSeeded (the slow half)", () => { const api: ApiCall = async (method, path, body) => { if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return fixedOfferingsResponse(); } diff --git a/packages/onboarding/test/provision.test.ts b/packages/onboarding/test/provision.test.ts index b1feabe08..aff387d48 100644 --- a/packages/onboarding/test/provision.test.ts +++ b/packages/onboarding/test/provision.test.ts @@ -96,27 +96,23 @@ function catalogOfferingsResponse( path: string, tenantId: string, ): { status: number; data: unknown; cookies: string[] } | undefined { - if (method !== "GET" || path !== `/api/tenants/${tenantId}/catalog/offerings`) + if ( + method !== "GET" || + path !== `/api/tenants/${tenantId}/catalog/resolved-offerings` + ) return undefined; return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -243,27 +239,20 @@ function firstLoginSeedHub(args: { expectedParentId?: string }) { } if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; @@ -702,27 +691,20 @@ describe("provisionPersonalTenantIfNeeded", () => { if (registry !== undefined) return registry; if ( method === "GET" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` ) { return { status: 200, data: { - data: [ + offerings: [ { id: "off_1", - tenantId: TENANT_ID, + priority: 0, modelId: "mdl_1", providerId: "mpr_1", - priority: 0, - deploymentTags: [], - capabilities: [], - quirks: null, - disabled: false, - createdAt: "2026-01-01T00:00:00.000Z", - updatedAt: "2026-01-01T00:00:00.000Z", + origin: { tenantId: TENANT_ID, direct: true }, }, ], - nextCursor: null, }, cookies: [], }; diff --git a/packages/seeding/test/seed.test.ts b/packages/seeding/test/seed.test.ts index 99f972804..d1bea7bcd 100644 --- a/packages/seeding/test/seed.test.ts +++ b/packages/seeding/test/seed.test.ts @@ -71,6 +71,24 @@ function offeringsPage(ids: readonly string[] = [OFFERING_ID]): { }; } +function resolvedOfferingsPage(ids: readonly string[] = [OFFERING_ID]): { + status: number; + data: unknown; +} { + return { + status: 200, + data: { + offerings: ids.map((id, index) => ({ + id, + modelId: `mdl_${id}`, + providerId: `mpr_${id}`, + priority: index, + origin: { tenantId: TENANT_ID, direct: true }, + })), + }, + }; +} + const instantSleep = async (_ms: number) => {}; const recordingPusher = () => { @@ -132,6 +150,11 @@ function baseRoutes(method: string, path: string) { path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) return offeringsPage(); + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/resolved-offerings` + ) + return resolvedOfferingsPage(); // The noop catalog chain `ensureNoopCatalogOffering` plants for a // modelSource-pinned workflow (heartbeat) — every seedTenant test that // deploys one needs this fresh chain to succeed, so it lives in the From ec8755106e08316e1a4a12c57f8121849be6fcb6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 08:50:49 -0700 Subject: [PATCH 12/20] Seed deploys against a tenant's resolved catalog offerings, not just owned ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveRealSourceOfferingIds listed only offerings a tenant owns directly, missing anything inherited from an ancestor tenant — the hub's own workflow deployer already resolves that wider, ancestor-inheriting view via listVisibleOfferings. Adds GET /api/tenants/:tenantId/catalog/resolved-offerings (mounted from apps/hub, implemented in @corbits/inference-catalog so the deployer and this route share one source of truth) and points seed at it instead. Also drops the two long-dead unused parameters from onboarding's modelSourceFor, and exports ensureNoopCatalogOffering from @corbits/seeding for the e2e harness to reuse. --- apps/hub/src/index.ts | 16 ++++ packages/inference-catalog/src/index.ts | 4 + .../src/resolved-offerings-routes.ts | 44 ++++++++++ .../onboarding/src/complete-credential.ts | 9 +- packages/seeding/src/index.ts | 1 + packages/seeding/src/seed.ts | 84 +++++++++---------- 6 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 packages/inference-catalog/src/resolved-offerings-routes.ts diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 39d52b853..687bcde64 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -150,6 +150,7 @@ import { applyInferenceCatalogMigrations, createBenchModelPolicyRoutes, createPostgresBenchModelPolicyStore, + createResolvedOfferingsRoutes, createWorkflowCatalogRoutes, } from "@corbits/inference-catalog"; import { generateId } from "@intx/hub-common"; @@ -1877,6 +1878,21 @@ export async function createHub(config: HubConfig) { }), }), ); + // Resolved catalog offerings: the same ancestor-inheriting view + // `listVisibleOfferings` gives `workflowDeployer` above, exposed over + // HTTP so an out-of-process deployer (`workbench seed`) can deploy + // against exactly what the hub itself would deploy against, not just + // the offerings a tenant owns directly. + app.route( + `${TENANT_PREFIX}/catalog/resolved-offerings`, + createResolvedOfferingsRoutes({ + listOfferings: (tenantId) => listVisibleOfferings(db, tenantId), + requireGrant: createRequireGrant({ + grantStore: chatGrantStore, + conditionRegistry: chatConditionRegistry, + }), + }), + ); // Bench purpose/type: benches are Interchange tenants, so this is a // package-owned side-table keyed by tenant id, migrated at hub start // like insights and preferences. diff --git a/packages/inference-catalog/src/index.ts b/packages/inference-catalog/src/index.ts index b7e90138a..01af47a3e 100644 --- a/packages/inference-catalog/src/index.ts +++ b/packages/inference-catalog/src/index.ts @@ -63,6 +63,10 @@ export { createBenchModelPolicyRoutes, type CreateBenchModelPolicyRoutesDeps, } from "./routes"; +export { + createResolvedOfferingsRoutes, + type CreateResolvedOfferingsRoutesDeps, +} from "./resolved-offerings-routes"; export { createWorkflowCatalogRoutes, estimateUsd, diff --git a/packages/inference-catalog/src/resolved-offerings-routes.ts b/packages/inference-catalog/src/resolved-offerings-routes.ts new file mode 100644 index 000000000..ec1cfae2f --- /dev/null +++ b/packages/inference-catalog/src/resolved-offerings-routes.ts @@ -0,0 +1,44 @@ +// The tenant-owned `GET .../catalog/offerings` route only ever lists rows a +// tenant planted directly on itself; it has no way to show offerings a +// tenant inherits from an ancestor tenant. `listVisibleOfferings` +// (`@intx/db`) already computes that resolved, inheritance-aware view for +// the hub's own workflow deployer — this route exposes the same view over +// HTTP so an out-of-process caller (`workbench seed`, in particular) can +// deploy against exactly what the hub itself would deploy against. +// +// Takes a `listOfferings` port rather than a `db` handle, same as +// `workflow-catalog-routes.ts`, so the route is testable with literals. +import { Hono } from "hono"; +import type { ResolvedOffering } from "@intx/db"; +import type { RequireGrant, TenantEnv } from "@intx/hub-api"; + +export type CreateResolvedOfferingsRoutesDeps = { + readonly listOfferings: ( + tenantId: string, + ) => Promise; + readonly requireGrant: RequireGrant; +}; + +export function createResolvedOfferingsRoutes( + deps: CreateResolvedOfferingsRoutesDeps, +): Hono { + const app = new Hono(); + + app.get("/", deps.requireGrant("model-offering:*", "read"), async (c) => { + const tenant = c.get("tenant"); + const resolved = [...(await deps.listOfferings(tenant.id))].sort( + (a, b) => a.offering.priority - b.offering.priority, + ); + return c.json({ + offerings: resolved.map((entry) => ({ + id: entry.offering.id, + priority: entry.offering.priority, + modelId: entry.offering.modelId, + providerId: entry.offering.providerId, + origin: entry.origin, + })), + }); + }); + + return app; +} diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index f5f4a930a..c292bc897 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -372,17 +372,12 @@ async function resolveOllamaModelSource( * diverge from that curated name entirely (CL-6366) — its resolution * defers to `resolveOllamaModelSource`, which reads back what * `seedCatalog` actually found on the instance rather than repeating the - * static pin. `apiKey`/`baseURLOverride` are accepted only because every - * caller already resolved them for `seedCatalog`'s own credential/catalog - * plant, which runs ahead of this in the same connect flow; this - * function itself needs neither. */ + * static pin. */ export async function modelSourceFor( api: ApiCall, cookies: string[], tenantId: string, provider: SupportedCredentialProvider, - _apiKey: string, - _baseURLOverride?: string, ): Promise { if (provider === "ollama") { return resolveOllamaModelSource(api, cookies, tenantId); @@ -496,8 +491,6 @@ export async function ensureSeeded( args.cookies, args.tenant.tenantId, args.provider, - args.apiKey, - args.baseURLOverride, ), pushWorkflow: args.pushWorkflow, log: args.log, diff --git a/packages/seeding/src/index.ts b/packages/seeding/src/index.ts index 78506cd4f..e67e47a58 100644 --- a/packages/seeding/src/index.ts +++ b/packages/seeding/src/index.ts @@ -26,6 +26,7 @@ export { inferenceCredentialName, PLACEHOLDER_CATALOG_API_KEY, ensureCredential, + ensureNoopCatalogOffering, ensureProvider, reconcileSeedGrants, seedCatalog, diff --git a/packages/seeding/src/seed.ts b/packages/seeding/src/seed.ts index 24fc1ce6e..21ca42794 100644 --- a/packages/seeding/src/seed.ts +++ b/packages/seeding/src/seed.ts @@ -987,55 +987,51 @@ async function ensureDeployment( return deployment.id; } +const ResolvedOfferingResponse = type({ + id: "string", + priority: "number", + modelId: "string", + providerId: "string", + origin: { tenantId: "string", direct: "boolean" }, +}); + +const ResolvedOfferingsResponse = type({ + offerings: ResolvedOfferingResponse.array(), +}); + /** - * Lists every catalog offering owned directly by a tenant, paging past - * `paginatedSchema`'s cursor. This is deliberately the tenant-owned list - * (`GET .../catalog/offerings`, never the ancestor-inheriting "resolved" - * view `listVisibleOfferings` computes in-process, `apps/hub/src/index.ts`'s - * `workflowDeployer` and `packages/chat/src/platform-adapter.ts`'s - * `catalogOfferings`): `seedCatalog` always plants a tenant's catalog - * directly on that same tenant, so the owned list already carries - * everything a plain `workbench seed` or the self-served connect flow - * ever seeded, with no HTTP surface exposing the resolved view to a - * caller outside the hub process. + * Lists the offerings visible to a tenant, including any inherited from + * an ancestor tenant — the same resolved view `listVisibleOfferings` + * (`@intx/db`) gives the hub's own workflow deployer + * (`apps/hub/src/index.ts`'s `workflowDeployer`), exposed at + * `GET .../catalog/resolved-offerings`. Seeding a tenant whose catalog is + * inherited rather than directly owned (e.g. a sub-tenant under a + * parent that already carries a real provider key) needs this resolved + * list, not the tenant-owned-only `GET .../catalog/offerings`. */ -async function listTenantCatalogOfferings( +async function listResolvedCatalogOfferings( api: ApiCall, cookies: string[], tenantId: string, -): Promise<(typeof ModelOfferingResponse.infer)[]> { - const items: (typeof ModelOfferingResponse.infer)[] = []; - let cursor: string | undefined; - for (;;) { - const query = - cursor === undefined ? "" : `?cursor=${encodeURIComponent(cursor)}`; - const listed = await api( - "GET", - `/api/tenants/${tenantId}/catalog/offerings${query}`, - undefined, - cookies, - ); - const page = parseAs( - paginatedSchema(ModelOfferingResponse), - listed.data, - "catalog offerings response", - ); - items.push(...page.data); - if (page.nextCursor === null) return items; - if (items.length > 10_000) { - throw new HubApiError( - `catalog offerings list for tenant ${tenantId} did not terminate while paging`, - "check the hub logs for the underlying failure, then re-run: workbench seed", - ); - } - cursor = page.nextCursor; - } +): Promise<(typeof ResolvedOfferingResponse.infer)[]> { + const listed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/resolved-offerings`, + undefined, + cookies, + ); + return parseAs( + ResolvedOfferingsResponse, + listed.data, + "resolved catalog offerings response", + ).offerings; } /** * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow - * deploys against: every enabled offering the tenant owns, ordered by - * priority ascending, with the lowest-priority offering as the default — + * deploys against: every offering visible to the tenant (owned or + * inherited from an ancestor), ordered by priority ascending, with the + * lowest-priority offering as the default — * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the @@ -1058,10 +1054,8 @@ async function resolveRealSourceOfferingIds( sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string; }> { - const offerings = (await listTenantCatalogOfferings(api, cookies, tenantId)) - .filter( - (offering) => !offering.disabled && offering.id !== excludeOfferingId, - ) + const offerings = (await listResolvedCatalogOfferings(api, cookies, tenantId)) + .filter((offering) => offering.id !== excludeOfferingId) .sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; if (defaultSourceOfferingId === undefined) { @@ -1087,7 +1081,7 @@ async function resolveRealSourceOfferingIds( * synthetic one. Idempotent, same as every other seed step: a re-run * finds the existing rows by name and reuses them. */ -async function ensureNoopCatalogOffering( +export async function ensureNoopCatalogOffering( api: ApiCall, cookies: string[], tenantId: string, From bcb8703e02e1a669a5d16651045a604993aeaf2e Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:05 -0700 Subject: [PATCH 13/20] Add a regression test pinning the shared-sidecar auth gap (CL-7472) At the interchange pin this branch stacks on (692c3106), the hub authenticates a sidecar's register/reconnect frame only against a sidecar row that resolves to a live sidecar_allocation (or workflow_probe) row. Upstream dropped the older "shared" credential scope this repo's e2e harness used to provision a single, standalone sidecar identity for, so a provisionSidecar-only row now connects and is rejected with "invalid token" in a loop rather than ever dialing in. This test pins that negative so a future re-vendor that silently restores (or relies on) the old scope surfaces here first. --- scripts/e2e/harness.test.ts | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 37f4f93e4..1a4f3fce8 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -6,10 +6,14 @@ import { describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; +import path from "node:path"; +import { resetSchema, setupDatabase } from "../db-setup.ts"; import { createCleanupHarness, + e2eDatabaseUrl, parseEnvFileDatabaseUrl, + provisionSidecar, runCleanups, workflowDeployBody, type SpawnedApp, @@ -127,3 +131,71 @@ describe("parseEnvFileDatabaseUrl", () => { ).toBe("postgres://second/db"); }); }); + +// CL-7472 regression: at pin 692c3106, the hub authenticates a +// sidecar's register/reconnect frame only against a `sidecar` row that +// resolves to a live `sidecar_allocation` (or `workflow_probe`) row — +// upstream dropped the older "shared" credential scope this repo used +// to provision a single, standalone sidecar identity for (see +// `provisionSidecar`'s own doc comment). CI's e2e suites hit exactly +// this: a `provisionSidecar`-only row connects and is rejected with +// "invalid token" in a loop (never a working dial-in), which is why +// every suite now lets the hub's own process-provisioner spawn a +// dedicated, already-allocated sidecar per deployment instead (see +// `startHub`'s doc comment). This test pins the negative: an +// unallocated identity must keep failing to authenticate, so a future +// upstream re-vendor that silently restores (or a caller that +// silently starts relying on) the old "shared" scope surfaces here +// first rather than as a mystery disconnect loop in CI. +describe("provisionSidecar", () => { + const databaseUrl = e2eDatabaseUrl(); + const HUB_DIR = path.resolve(import.meta.dir, "..", "..", "apps", "hub"); + + test.skipIf(databaseUrl === undefined)( + "a provisioned identity with no allocation or probe never authenticates", + async () => { + const url = databaseUrl; + if (url === undefined) throw new Error("unreachable: suite is skipped"); + + await resetSchema(url); + await setupDatabase(url); + + const sidecarId = "harness-test-unallocated-sidecar"; + const token = crypto.randomUUID(); + await provisionSidecar(url, sidecarId, token); + + // Resolved through the hub's own dependency tree, exactly as + // `connectE2eDb` resolves `postgres` above — scripts/e2e is not + // itself a workspace member of these `@intx/*` packages. + const dbModule = (await import( + Bun.resolveSync("@intx/db", HUB_DIR) + )) as { createDB: (raw: unknown) => { db: unknown; close: () => Promise } }; + const hubSessionsModule = (await import( + Bun.resolveSync("@intx/hub-sessions", HUB_DIR) + )) as { + createSidecarCredentialResolver: (deps: { db: unknown }) => { + resolve: (token: string) => Promise; + }; + }; + + const parsed = new URL(url); + const db = dbModule.createDB({ + host: parsed.hostname, + port: parsed.port === "" ? 5432 : Number(parsed.port), + user: decodeURIComponent(parsed.username), + password: decodeURIComponent(parsed.password), + database: parsed.pathname.replace(/^\//, ""), + }); + try { + const resolver = hubSessionsModule.createSidecarCredentialResolver({ + db: db.db, + }); + const identity = await resolver.resolve(token); + expect(identity).toBeNull(); + } finally { + await db.close(); + } + }, + 60_000, + ); +}); From ccbe497e63bf84527e9ad01dcc840b7be2155e10 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:16 -0700 Subject: [PATCH 14/20] e2e: stop pre-provisioning a shared sidecar identity (CL-7472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every e2e suite dial-in relied on a single, harness-started sidecar process registered against a bare `provisionSidecar` identity row. At pin 692c3106, the hub only accepts a register/reconnect frame from a sidecar whose identity resolves to a live sidecar_allocation (or workflow_probe) row; a standalone identity with neither now fails the handshake and gets rejected with "invalid token" every ~3s, forever (chat mint 500s, deploys never dial in — CI run 34042907014). The hub's own process-provisioner already spawns a dedicated, correctly-allocated sidecar process per deployment on demand — the same path production already takes for a real workflow run or a minted agent chat. Every affected suite (walking skeleton, chat, heartbeat, workbench-digest, smoke-chat, smoke-webhook, greeting-delivery, channel-reconnect, local-rip, and the isolation suite) now lets that happen instead of dialing in its own sidecar: the existing 502/500 retry-until-ready loops already cover the wait, and diagnostics that used to read the one sidecar process's output now read the hub's — the process-provisioner backend spawns each sidecar with its stdio inherited from the hub, so the hub's own captured output already carries every spawned sidecar's log lines. `provisionSidecar`/`startSidecar` stay in the harness (documented as non-authenticating at this pin) for the handful of standalone proof scripts not yet migrated off a directly-controlled sidecar process. --- scripts/e2e/channel-reconnect.test.ts | 37 +++++----------- scripts/e2e/chat.test.ts | 61 ++++++++++----------------- scripts/e2e/greeting-delivery.test.ts | 36 ++++------------ scripts/e2e/harness.ts | 26 +++++++++--- scripts/e2e/heartbeat.test.ts | 29 ++----------- scripts/e2e/local-rip.test.ts | 24 ++--------- scripts/e2e/smoke-chat.test.ts | 29 ++----------- scripts/e2e/smoke-webhook.test.ts | 38 +++-------------- scripts/e2e/walking-skeleton.test.ts | 45 +++++--------------- scripts/e2e/workbench-digest.test.ts | 29 ++----------- test/isolation/setup.ts | 49 ++++++--------------- 11 files changed, 109 insertions(+), 294 deletions(-) diff --git a/scripts/e2e/channel-reconnect.test.ts b/scripts/e2e/channel-reconnect.test.ts index 59e254c19..3c3e8e637 100644 --- a/scripts/e2e/channel-reconnect.test.ts +++ b/scripts/e2e/channel-reconnect.test.ts @@ -29,12 +29,9 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -76,7 +73,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { let hubPort: number; let sessionSecret: string; let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let cookies: string[]; let tenantId: string; @@ -93,10 +89,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "workbench-reconnect-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hubPort = freePort(); sessionSecret = Buffer.from( crypto.getRandomValues(new Uint8Array(32)), @@ -110,14 +102,6 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-reconnect-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); const email = `workbench-reconnect-e2e-${crypto.randomUUID()}@example.invalid`; @@ -159,9 +143,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -174,7 +158,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -244,10 +228,10 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { // hub process on the same port against the same database, exactly // as a real restart looks from the sidecar's and the browser's // side: same DB rows, same session cookie, new process. The port - // must stay the same — the sidecar's own `HUB_WS_URL` is fixed at - // its boot and is never reconfigured here (only the hub restarts), - // so a different port would leave the sidecar dialing a hub that - // is no longer there. + // must stay the same — the process-provisioner-spawned sidecar's + // own `HUB_WS_URL` is fixed at its boot and is never reconfigured + // here (only the hub restarts), so a different port would leave the + // sidecar dialing a hub that is no longer there. await hub.stop(); const restarted = await startHub({ @@ -269,9 +253,9 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { const deadline = Date.now() + 180_000; let posted: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited after hub restart; output:\n${sidecar.output()}`, + `hub exited after hub restart; output:\n${hub.output()}`, ); } posted = await postMessage(after); @@ -279,8 +263,7 @@ describe.skipIf(databaseUrl === undefined)("workbench reconnect e2e", () => { if (Date.now() > deadline) { throw new Error( `message after hub restart never accepted: status=${posted.status} ` + - `${JSON.stringify(posted.data)}\nsidecar output:\n${sidecar.output()}` + - `\nhub output:\n${hub.output()}`, + `${JSON.stringify(posted.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index da9b255c8..c50da0894 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -46,14 +46,11 @@ import { e2eDatabaseUrl, expectStatus, freePort, - provisionSidecar, pushWorkflowSource, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const databaseUrl = e2eDatabaseUrl(); @@ -139,7 +136,6 @@ const textPart = (text: string): Part[] => [{ kind: "text", text }]; describe.skipIf(databaseUrl === undefined)("chat e2e", () => { let hub: HubHandle; - let sidecar: SpawnedApp; let api: ApiCall; let user1: SignedUpUser; let user2: SignedUpUser; @@ -159,10 +155,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { expect(report.action).toBe("migrated"); expect(report.migrations).toBeGreaterThan(0); - const sidecarId = "chat-e2e-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - hub = await startHub({ databaseUrl: url, port: freePort(), @@ -173,16 +165,6 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }); track(hub); - sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-chat-sidecar-data-"), - }); - track(sidecar); - api = createHubAPI(hub.baseUrl); // Two independent browser-shaped accounts, each carrying its own @@ -358,15 +340,15 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { ), }); - // Retries while the hub still answers 502 (the sidecar's dial-in - // may not have completed yet), matching `createWorkbench`'s own - // retry loop below. + // Retries while the hub still answers 502 (the process + // provisioner's spawned sidecar may not have dialed in yet), + // matching `createWorkbench`'s own retry loop below. const echoDeployDeadline = Date.now() + 60_000; let echoDeployed: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before echo deploy; output:\n${sidecar.output()}`, + `hub exited before echo deploy; output:\n${hub.output()}`, ); } echoDeployed = await api( @@ -384,7 +366,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > echoDeployDeadline) { throw new Error( `echo workflow never became deployable (hub kept answering 502): ` + - `${JSON.stringify(echoDeployed.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(echoDeployed.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -393,22 +375,22 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { }, 120_000); // Launching a workbench is the go/no-go signal for the whole suite: it - // launches the anchor instance in-process, which needs the - // sidecar's dial-in to have completed. `packages/chat/src/routes.ts` - // has no 502-style retry translation of its own (unlike the native - // workflow deploy route), so a launch attempted before the sidecar - // connects fails with an uncaught 500 — retried here directly until - // the sidecar is ready, exactly as the walking skeleton retries a - // 502 for the native deploy route. + // launches the anchor instance in-process, which needs its + // process-provisioner-spawned sidecar's dial-in to have completed. + // `packages/chat/src/routes.ts` has no 502-style retry translation of + // its own (unlike the native workflow deploy route), so a launch + // attempted before the sidecar connects fails with an uncaught 500 — + // retried here directly until the sidecar is ready, exactly as the + // walking skeleton retries a 502 for the native deploy route. async function createWorkbench( body: Record, ): Promise { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } res = await api( @@ -421,7 +403,7 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { if (Date.now() > deadline) { throw new Error( `workbench never became launchable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -1021,10 +1003,13 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { // temporarily reverting the self-anchor fix locally: the sidecar log // then carries ten `enqueueInbox failed, withholding ack` lines, one // per rejected pack, for this same suite; with the fix applied, zero. + // Every process-provisioner-spawned sidecar inherits the hub + // process's own stdio (`process-runner.ts`'s `stdout: "inherit"`), so + // `hub.output()` already carries every sidecar's log lines. test("no chat/folded run's workflow-run pack was ever permanently rejected", () => { - const sidecarOutput = sidecar.output(); - expect(sidecarOutput).not.toContain("enqueueInbox failed"); - expect(sidecarOutput).not.toContain("withholding ack"); - expect(sidecarOutput).not.toContain("rejecting inbound mail"); + const hubOutput = hub.output(); + expect(hubOutput).not.toContain("enqueueInbox failed"); + expect(hubOutput).not.toContain("withholding ack"); + expect(hubOutput).not.toContain("rejecting inbound mail"); }); }); diff --git a/scripts/e2e/greeting-delivery.test.ts b/scripts/e2e/greeting-delivery.test.ts index 0ac6e26bb..2521206f1 100644 --- a/scripts/e2e/greeting-delivery.test.ts +++ b/scripts/e2e/greeting-delivery.test.ts @@ -41,12 +41,9 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type ApiResult, type HubHandle, - type SpawnedApp, } from "./harness.ts"; const { tempDir, track } = createCleanupHarness(); @@ -135,10 +132,6 @@ describe.skipIf(databaseUrl === undefined)( expect(report.action).toBe("migrated"); }); - const sidecarId = "greeting-delivery-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -151,16 +144,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar: SpawnedApp = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-greeting-delivery-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -236,9 +219,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { @@ -269,9 +252,9 @@ describe.skipIf(databaseUrl === undefined)( async function deploySeededWorkflows(): Promise { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -356,9 +339,9 @@ describe.skipIf(databaseUrl === undefined)( const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before chat creation; output:\n${sidecar.output()}`, + `hub exited before chat creation; output:\n${hub.output()}`, ); } res = await api( @@ -372,7 +355,7 @@ describe.skipIf(databaseUrl === undefined)( if (Date.now() > deadline) { throw new Error( `chat never became mintable (hub kept answering 500): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -448,7 +431,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no agent-authored message landed in chat ${chatId} within ` + `60s of mint with zero user messages sent; messages seen: ` + - `${JSON.stringify(items)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(items)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -527,8 +510,7 @@ describe.skipIf(databaseUrl === undefined)( throw new Error( `no answer to turn ${turn} within 180s; agent messages: ` + `${JSON.stringify(agentTexts)}\nhub output (tail):\n` + - `${hub.output().slice(-60000)}\nsidecar output (tail):\n` + - `${sidecar.output().slice(-6000)}`, + `${hub.output().slice(-60000)}`, ); } await Bun.sleep(2000); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index cebc4e53d..1e884e0f9 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -221,6 +221,20 @@ export async function connectE2eDb(databaseUrl: string): Promise { * WebSocket dial-in against the token hash on the `sidecar` table, so * the row must exist before the sidecar process starts. Mirrors * Interchange's dev provisioning (id + sha256(token), placeholder url). + * + * At pin 692c3106, the hub only accepts a register/reconnect frame from + * a sidecar identity that resolves to a live `sidecar_allocation` (or + * `workflow_probe`) row — the standalone, unallocated ("shared" + * credential scope) sidecar this used to provision no longer + * authenticates at all (CL-7472), so a caller of this function gets a + * connect-and-immediately-reject loop, not a usable dial-in. Every + * suite this repo runs against CI now lets the hub's own + * process-provisioner spawn a dedicated sidecar per allocation instead + * (see `startHub`'s doc comment and `scripts/e2e/chat.test.ts`); this + * function is kept only for the handful of scripts not yet migrated off + * a directly-controlled sidecar process (e.g. ones that kill and + * restart a specific sidecar mid-scenario) and is not a working dial-in + * path until they are. */ export async function provisionSidecar( databaseUrl: string, @@ -400,11 +414,13 @@ export async function startHub(options: { } /** - * Boot the sidecar as a real process, pointed at the hub's WebSocket - * dial-in route with the provisioned identity. Connection readiness is - * observed by the deploy call itself (the hub answers 502 until a - * sidecar is connected), so this only guards against an immediate - * crash. + * Boot a standalone sidecar as a real process, pointed at the hub's + * WebSocket dial-in route with a `provisionSidecar`-issued identity. + * See `provisionSidecar`'s own doc comment: at pin 692c3106 this + * identity never authenticates (CL-7472), so the process this starts + * connects and is immediately rejected in a loop rather than dialing + * in. Kept only for the scripts not yet migrated to the hub's + * automatic, per-allocation process-provisioner spawn. */ export function startSidecar(options: { hubPort: number; diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 9aefb900f..8c924a6af 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -95,12 +93,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-heartbeat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -114,19 +106,6 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-heartbeat-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Heartbeat Tester", @@ -224,10 +203,8 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -240,7 +217,7 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/local-rip.test.ts b/scripts/e2e/local-rip.test.ts index 2b4f4ec9c..b1919de02 100644 --- a/scripts/e2e/local-rip.test.ts +++ b/scripts/e2e/local-rip.test.ts @@ -81,9 +81,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -187,10 +185,6 @@ describe.skipIf(databaseUrl === undefined)( } }); - const sidecarId = "local-rip-sidecar"; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(url, sidecarId, sidecarToken); - const hub: HubHandle = await hop("hub boot", async () => startHub({ databaseUrl: url, @@ -208,16 +202,6 @@ describe.skipIf(databaseUrl === undefined)( ); track(hub); - const sidecar = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-local-rip-sidecar-data-"), - }); - track(sidecar); - const hubApi: ApiCall = createHubAPI(hub.baseUrl); const user = await hop("sign-up", () => @@ -341,9 +325,9 @@ describe.skipIf(databaseUrl === undefined)( ): Promise>> { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before default workflows could deploy; output:\n${sidecar.output()}`, + `hub exited before default workflows could deploy; output:\n${hub.output()}`, ); } try { @@ -398,9 +382,9 @@ describe.skipIf(databaseUrl === undefined)( async () => { const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before ensureSeeded could run; output:\n${sidecar.output()}`, + `hub exited before ensureSeeded could run; output:\n${hub.output()}`, ); } try { diff --git a/scripts/e2e/smoke-chat.test.ts b/scripts/e2e/smoke-chat.test.ts index 98b598332..eaf0da399 100644 --- a/scripts/e2e/smoke-chat.test.ts +++ b/scripts/e2e/smoke-chat.test.ts @@ -20,9 +20,7 @@ import { expectStatus, freePort, hop, - provisionSidecar, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -58,13 +56,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { }); const hubDataDir = await tempDir("e2e-smoke-chat-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-chat-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-chat"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -78,18 +69,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -123,9 +102,9 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { // through the transient 500 the same way chat.test.ts does. const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { + if (hub.exited()) { throw new Error( - `sidecar exited before workbench creation; output:\n${sidecar.output()}`, + `hub exited before workbench creation; output:\n${hub.output()}`, ); } const res = await api( @@ -141,7 +120,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { } if (Date.now() > deadline) { throw new Error( - `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `workbench never became launchable (hub kept answering 500): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); @@ -171,7 +150,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: chat round-trip", () => { if (Date.now() > deadline) { throw new Error( `post message kept answering 500: ${JSON.stringify(res.data)}` + - `\nsidecar output:\n${sidecar.output()}`, + `\nhub output:\n${hub.output()}`, ); } await Bun.sleep(1000); diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 760aca6a3..d834aa623 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -12,9 +12,8 @@ // events by bare id (the native events route requires a deployment-anchor // run). This test instead reads the run row the ingress delivery created // straight out of the database — a harness-side fact with no route, -// exactly as `provisionSidecar` already does for the sidecar identity -// row — and confirms the trigger itself recorded the delivery -// (`lastFiredAt`). +// via `connectE2eDb` — and confirms the trigger itself recorded the +// delivery (`lastFiredAt`). import { signPayload, @@ -36,12 +35,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type HubHandle, } from "./harness.ts"; @@ -77,13 +74,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const hubDataDir = await tempDir("e2e-smoke-webhook-hub-data-"); - const sidecarDataDir = await tempDir("e2e-smoke-webhook-sidecar-data-"); - - const sidecarId = "sidecar-e2e-smoke-webhook"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); const hub: HubHandle = await hop("hub boot", () => startHub({ @@ -97,18 +87,6 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); track(hub); - const sidecar = await hop("sidecar boot", () => - Promise.resolve( - startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }), - ), - ); - track(sidecar); - { const cookies = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { @@ -204,10 +182,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); const deadline = Date.now() + 60_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } const res = await api( hub.baseUrl, @@ -219,7 +195,7 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { if (res.status === 502) { if (Date.now() > deadline) { throw new Error( - `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `sidecar never became deployable (hub kept answering 502): ${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); @@ -346,8 +322,8 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // No platform route reads a non-anchored run's row by bare id // (see the file header); this reads the fact the ingress // route's own launch is required to have produced, the same - // way `provisionSidecar` reaches the database directly for a - // fact no route exposes. + // way `connectE2eDb` reaches the database directly for a fact + // no route exposes. const sql = await connectE2eDb(url); try { const rows = await sql.unsafe( diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 8937f9b5e..e876f6bfa 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -33,12 +33,10 @@ import { expectStatus, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, type ApiResult, type HubHandle, } from "./harness.ts"; @@ -81,15 +79,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { expect(second.migrations).toBe(first.migrations); }); - // Hop: sidecar provisioning. The identity row the hub checks the - // sidecar's dial-in token against. - const sidecarId = "sidecar-e2e"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - - // Hop: hub boot. The composition root as a real process. + // Hop: hub boot. The composition root as a real process. The hub's + // own process provisioner spawns a dedicated sidecar for every + // allocation on demand (one `apps/sidecar` per deployment) — there + // is no shared, pre-provisioned sidecar identity to dial in here. const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -103,21 +96,6 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { return handle; }); - // Hop: sidecar boot. Dial-in readiness is observed at the deploy - // hop (the hub answers 502 until a sidecar is connected). - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-sidecar-data-"), - }); - track(app); - return app; - }); - // Hop: sign-up. A browser-shaped account creation through the // platform's better-auth surface; the session cookie carries the // rest of the skeleton. @@ -215,9 +193,10 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { ); // Hop: workflow deploy via the native deploy API. Retries while - // the hub still answers 502 (the sidecar's dial-in may not have - // completed yet); any other failure is final. The inference - // source is a placeholder — deployment does not call inference. + // the hub still answers 502 (the process provisioner's spawned + // sidecar may not have dialed in yet); any other failure is final. + // The inference source is a placeholder — deployment does not call + // inference. const deploymentId = await hop("workflow deploy", async () => { const body = workflowDeployBody({ assetId, @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 8917bf823..f3d41cd06 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -27,12 +27,10 @@ import { expectStepCompleted, freePort, hop, - provisionSidecar, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, startHub, - startSidecar, waitForRunCompletion, type ApiResult, type HubHandle, @@ -100,12 +98,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { await setupDatabase(url); }); - const sidecarId = "sidecar-e2e-workbench-digest"; - const sidecarToken = crypto.randomUUID(); - await hop("sidecar provisioning", () => - provisionSidecar(url, sidecarId, sidecarToken), - ); - const hub: HubHandle = await hop("hub boot", async () => { const handle = await startHub({ databaseUrl: url, @@ -119,19 +111,6 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { return handle; }); - const sidecar = await hop("sidecar boot", async () => { - const app = startSidecar({ - hubPort: new URL(hub.baseUrl).port - ? Number(new URL(hub.baseUrl).port) - : 80, - sidecarId, - token: sidecarToken, - dataDir: await tempDir("e2e-workbench-digest-sidecar-data-"), - }); - track(app); - return app; - }); - const user = await hop("sign-up", async () => { const res = await api(hub.baseUrl, "POST", "/api/auth/sign-up/email", { name: "Workbench Digest Tester", @@ -228,10 +207,8 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { const deadline = Date.now() + 60_000; let res: ApiResult; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before deploy; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before deploy; output:\n${hub.output()}`); } res = await api( hub.baseUrl, @@ -244,7 +221,7 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { if (Date.now() > deadline) { throw new Error( `sidecar never became deployable (hub kept answering 502): ` + - `${JSON.stringify(res.data)}\nsidecar output:\n${sidecar.output()}`, + `${JSON.stringify(res.data)}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(200); diff --git a/test/isolation/setup.ts b/test/isolation/setup.ts index 3644750fa..07271b0c1 100644 --- a/test/isolation/setup.ts +++ b/test/isolation/setup.ts @@ -10,11 +10,8 @@ import path from "node:path"; import { freePort, - provisionSidecar, startHub, - startSidecar, type HubHandle, - type SpawnedApp, } from "../../scripts/e2e/harness.ts"; import { assertDatabaseConfigured } from "../../scripts/e2e/db-gate.ts"; @@ -90,29 +87,21 @@ export async function prepareDatabase(databaseUrl: string): Promise { } /** - * Boots the real hub and a real sidecar as spawned processes, exactly - * as `scripts/e2e/harness.ts` does for the walking skeleton — a - * chat-workbench create launches a real agent instance over the - * sidecar's WebSocket dial-in, so an in-process hub with no sidecar - * attached cannot exercise the "chat workbench move" block below; it - * would fail every one of those cases with "no sidecar available" - * rather than proving isolation. `AppLike.request` wraps `fetch` - * against the hub's real port, matching how a client actually reaches - * it. + * Boots the real hub as a spawned process, exactly as + * `scripts/e2e/harness.ts` does for the walking skeleton — a + * chat-workbench create launches a real agent instance over its own + * process-provisioner-spawned sidecar's WebSocket dial-in, so an + * in-process hub with no sidecar plugin wired up cannot exercise the + * "chat workbench move" block below; it would fail every one of those + * cases with "no sidecar available" rather than proving isolation. + * `AppLike.request` wraps `fetch` against the hub's real port, matching + * how a client actually reaches it. */ export async function bootIsolationHub( databaseUrl: string, ): Promise { const root = mkdtempSync(path.join(tmpdir(), "isolation-suite-")); const hubDataDir = path.join(root, "hub-data"); - const sidecarDataDir = path.join(root, "sidecar-data"); - - // Unique per run: the `sidecar` table's id is a primary key, and a - // prior run's row is never cleaned up (the isolation suite runs - // against a shared database, never a scratch one it owns outright). - const sidecarId = `sidecar-isolation-suite-${crypto.randomUUID()}`; - const sidecarToken = crypto.randomUUID(); - await provisionSidecar(databaseUrl, sidecarId, sidecarToken); const hub: HubHandle = await startHub({ databaseUrl, @@ -133,23 +122,15 @@ export async function bootIsolationHub( }, }); - const sidecar: SpawnedApp = startSidecar({ - hubPort: Number(new URL(hub.baseUrl).port), - sidecarId, - token: sidecarToken, - dataDir: sidecarDataDir, - }); - const app: AppLike = { request: (requestPath, init) => fetch(`${hub.baseUrl}${requestPath}`, init), }; - await waitForSidecarDialIn(app, sidecar); + await waitForSidecarDialIn(app, hub); return { app, shutdown: async () => { - await sidecar.stop(); await hub.stop(); rmSync(root, { recursive: true, force: true }); }, @@ -169,7 +150,7 @@ export async function bootIsolationHub( */ async function waitForSidecarDialIn( app: AppLike, - sidecar: SpawnedApp, + hub: HubHandle, ): Promise { const cookie = await signUpUser( app, @@ -194,10 +175,8 @@ async function waitForSidecarDialIn( const deadline = Date.now() + 30_000; for (;;) { - if (sidecar.exited()) { - throw new Error( - `sidecar exited before dialing in; output:\n${sidecar.output()}`, - ); + if (hub.exited()) { + throw new Error(`hub exited before dialing in; output:\n${hub.output()}`); } const workbenchResponse = await app.request( `/api/tenants/${tenant.id}/chat/workbenches`, @@ -212,7 +191,7 @@ async function waitForSidecarDialIn( throw new Error( `sidecar never dialed in within 30s (workbench create kept ` + `failing with ${workbenchResponse.status}): ` + - `${await workbenchResponse.text()}\nsidecar output:\n${sidecar.output()}`, + `${await workbenchResponse.text()}\nhub output:\n${hub.output()}`, ); } await Bun.sleep(500); From 308ab2526a5018b2e65ae4bff4eb17805ecefdc7 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:19:21 -0700 Subject: [PATCH 15/20] ci: upload hub/sidecar e2e logs as a build artifact (CL-7472) The e2e job's hub/sidecar output was never captured, so the CI run that surfaced the sidecar disconnect loop (34042907014) had no sidecar-side log to read past the transcript. Point E2E_LOG_DIR at a runner-temp directory and upload it on every run, pass or fail. --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a35363400..4e67a69e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,15 @@ jobs: run: bun run test:e2e env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Upload hub/sidecar logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs + path: ${{ runner.temp }}/e2e-logs + if-no-files-found: ignore isolation: runs-on: ubuntu-latest From 72e5540351110a7250a5ce14af23eaeb04cf71cc Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:28:21 -0700 Subject: [PATCH 16/20] e2e: lint fixes --- scripts/e2e/harness.test.ts | 6 +++--- scripts/e2e/harness.ts | 1 - scripts/e2e/smoke-webhook.test.ts | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 1a4f3fce8..3503b8131 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -167,9 +167,9 @@ describe("provisionSidecar", () => { // Resolved through the hub's own dependency tree, exactly as // `connectE2eDb` resolves `postgres` above — scripts/e2e is not // itself a workspace member of these `@intx/*` packages. - const dbModule = (await import( - Bun.resolveSync("@intx/db", HUB_DIR) - )) as { createDB: (raw: unknown) => { db: unknown; close: () => Promise } }; + const dbModule = (await import(Bun.resolveSync("@intx/db", HUB_DIR))) as { + createDB: (raw: unknown) => { db: unknown; close: () => Promise }; + }; const hubSessionsModule = (await import( Bun.resolveSync("@intx/hub-sessions", HUB_DIR) )) as { diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 280fae27e..68c56ead1 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -693,4 +693,3 @@ export function workflowDeployBody(options: { defaultSourceOfferingId: options.defaultSourceOfferingId, }; } - diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 73fdf7410..28dd70d43 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -183,7 +183,9 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { const deadline = Date.now() + 60_000; for (;;) { if (hub.exited()) { - throw new Error(`hub exited before deploy; output:\n${hub.output()}`); + throw new Error( + `hub exited before deploy; output:\n${hub.output()}`, + ); } const res = await api( hub.baseUrl, From 9da40d7f8ef0c3f8988851bb8f7555d4a26b5a45 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:34:43 -0700 Subject: [PATCH 17/20] Add a test pinning the noop catalog offering's model name Exports NOOP_CATALOG_MODEL from the e2e harness and adds a unit test asserting seedNoopCatalogOffering plants a catalog model with that exact canonicalName, so a future edit can't silently drift the planted (provider, model) pair away from what a suite's workflow fixture declares as its inference preference. --- scripts/e2e/harness.test.ts | 53 +++++++++++++++++++++++++++++++++++++ scripts/e2e/harness.ts | 13 ++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 1a4f3fce8..9fdf40d73 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -12,10 +12,13 @@ import { resetSchema, setupDatabase } from "../db-setup.ts"; import { createCleanupHarness, e2eDatabaseUrl, + NOOP_CATALOG_MODEL, parseEnvFileDatabaseUrl, provisionSidecar, runCleanups, + seedNoopCatalogOffering, workflowDeployBody, + type ApiResult, type SpawnedApp, } from "./harness.ts"; @@ -95,6 +98,56 @@ describe("workflowDeployBody", () => { }); }); +describe("seedNoopCatalogOffering", () => { + // CL-7473: a suite's workflow definition must declare its inference + // preference as `("anthropic", NOOP_CATALOG_MODEL)` to match what this + // helper plants — the deploy-time capability walk only auto-approves the + // (provider, model) pair a step's own agent declares, so a preference + // naming any other model 409s "no approved inference source" at deploy + // even though the suite never calls real inference. This pins the + // catalog-model creation call's `canonicalName` to the exported constant + // so the two can never drift apart silently again. + test("plants a catalog model named NOOP_CATALOG_MODEL", async () => { + const calls: { path: string; body: unknown }[] = []; + const call = async ( + _method: string, + path: string, + body?: unknown, + ): Promise => { + calls.push({ path, body }); + if (path.endsWith("/catalog/models")) { + return { status: 201, data: { id: "model-1" }, cookies: [] }; + } + if (path.endsWith("/providers")) { + return { status: 201, data: { id: "provider-1" }, cookies: [] }; + } + if (path.endsWith("/credentials")) { + return { status: 201, data: { id: "credential-1" }, cookies: [] }; + } + if (path.endsWith("/catalog/providers")) { + return { + status: 201, + data: { id: "catalog-provider-1" }, + cookies: [], + }; + } + return { status: 201, data: { id: "offering-1" }, cookies: [] }; + }; + + await seedNoopCatalogOffering({ + call, + tenantId: "tenant-1", + cookies: [], + noopBaseUrl: "https://inference.invalid", + }); + + const modelCall = calls.find((entry) => + entry.path.endsWith("/catalog/models"), + ); + expect(modelCall?.body).toEqual({ canonicalName: NOOP_CATALOG_MODEL }); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 1e884e0f9..29ba10d47 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -700,7 +700,18 @@ export function workflowDeployBody(options: { * catalog offering joining them. Returns the offering id a deploy's * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + * + * The (provider, model) pair this plants is `("anthropic", + * NOOP_CATALOG_MODEL)`. A caller's workflow definition must declare that + * exact pair as its inference preference — the deploy-time capability walk + * only auto-approves the (provider, model) pairs a step's own agent + * declares (see `@intx/workflow-deploy`'s `pickStepInferenceSource`), so a + * preference naming any other model 409s "no approved inference source" + * even though a suite using this helper never calls real inference + * (CL-7473). */ +export const NOOP_CATALOG_MODEL = "noop"; + export async function seedNoopCatalogOffering(options: { call: ( method: string, @@ -718,7 +729,7 @@ export async function seedNoopCatalogOffering(options: { const model = await call( "POST", `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, + { canonicalName: NOOP_CATALOG_MODEL }, cookies, ); expectStatus("create catalog model", model, 201); From 1888f4221f1eb753b339b76afff8be639d9b2a52 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:34:49 -0700 Subject: [PATCH 18/20] walking-skeleton e2e: match the seeded catalog offering's model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite's echo workflow declared an inference preference of anthropic:claude-sonnet-5, while seedNoopCatalogOffering plants a catalog offering for anthropic:noop. At the re-vendored intx pin, the deploy-time capability walk only auto-approves the (provider, model) pair a step's own agent declares (@intx/workflow-deploy's pickStepInferenceSource), so the mismatch left neither the preferred source nor the deploy's defaultSource resolvable against an approved grant, and the deploy 409ed invalid_workflow before the run could ever start. Naming the same (provider, model) the seeded offering serves — like every other e2e suite already does — lets the deploy resolve a source it can actually approve. --- scripts/e2e/walking-skeleton.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index e876f6bfa..db3fd39ca 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -33,6 +33,7 @@ import { expectStatus, freePort, hop, + NOOP_CATALOG_MODEL, pushWorkflowSource, seedNoopCatalogOffering, workflowDeployBody, @@ -160,10 +161,18 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { ); expectStatus("mint git token", minted, 201); + // Must name the same (provider, model) `seedNoopCatalogOffering` + // below plants, like every other e2e suite's echo/agent fixture + // (see `heartbeat.test.ts`, `smoke-webhook.test.ts`, + // `workbench-digest.test.ts`) — the deploy-time capability walk + // only auto-approves the (provider, model) pairs a step's own + // agent declares (CL-7473), so a preference naming a model the + // seeded catalog never offers 409s with "no approved inference + // source" even though deployment never calls inference. const definition = buildEchoWorkflow({ triggerAddress: `echo@${slug}.localhost`, inferencePreferences: [ - { provider: "anthropic", model: "claude-sonnet-5" }, + { provider: "anthropic", model: NOOP_CATALOG_MODEL }, ], turnTimeoutMs: 60_000, }); From bb30c39e87ba7ca10e34d7c21df742a2792c2f14 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:43:10 -0700 Subject: [PATCH 19/20] walking-skeleton e2e: match the seeded catalog offering's model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite's echo workflow declared an inference preference of anthropic:claude-sonnet-5, while seedPlaceholderCatalogOffering plants a catalog offering for anthropic:noop. The deploy-time capability walk only auto-approves the (provider, model) pair a step's own agent declares (@intx/workflow-deploy's pickStepInferenceSource), so the mismatch left neither the preferred source nor the deploy's defaultSource resolvable against an approved grant, and the deploy 409ed invalid_workflow before the run could ever start. Naming PLACEHOLDER_MODEL — the same (provider, model) the seeded offering serves — lets the deploy resolve a source it can actually approve. --- scripts/e2e/walking-skeleton.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index ed13b71df..9cde2aa09 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -310,7 +310,7 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { const definition = buildEchoWorkflow({ triggerAddress: `echo@${slug}.localhost`, inferencePreferences: [ - { provider: "anthropic", model: "claude-sonnet-5" }, + { provider: "anthropic", model: PLACEHOLDER_MODEL }, ], turnTimeoutMs: 60_000, }); From 8a54953bfb529c4e9e279e7f546201d85796d62a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:48:22 -0700 Subject: [PATCH 20/20] walking-skeleton: drop the literal-asserting test Per the owner's 2026-09-06 ruling to test only non-obvious owned logic and never literals, remove the unit test asserting seedPlaceholderCatalogOffering names PLACEHOLDER_MODEL. --- scripts/e2e/walking-skeleton.test.ts | 49 ---------------------------- 1 file changed, 49 deletions(-) diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 9cde2aa09..3c78956da 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -158,55 +158,6 @@ async function seedPlaceholderCatalogOffering(options: { }; } -describe("seedPlaceholderCatalogOffering", () => { - // CL-7473: pins the helper's planted catalog model to PLACEHOLDER_MODEL - // — the same constant this suite's `buildEchoWorkflow` inference - // preference uses below. The two drifting apart is exactly what caused - // the deploy to 409 "no approved inference source": the capability walk - // only auto-approves the (provider, model) pair a step's own agent - // declares, so a preference naming a model the seeded catalog never - // offers can never resolve to an approved source. - test("plants a catalog model named PLACEHOLDER_MODEL", async () => { - const calls: { path: string; body: unknown }[] = []; - const call = async ( - _method: string, - path: string, - body?: unknown, - ): Promise => { - calls.push({ path, body }); - if (path.endsWith("/catalog/models")) { - return { status: 201, data: { id: "model-1" }, cookies: [] }; - } - if (path.endsWith("/providers")) { - return { status: 201, data: { id: "provider-1" }, cookies: [] }; - } - if (path.endsWith("/credentials")) { - return { status: 201, data: { id: "credential-1" }, cookies: [] }; - } - if (path.endsWith("/catalog/providers")) { - return { - status: 201, - data: { id: "catalog-provider-1" }, - cookies: [], - }; - } - return { status: 201, data: { id: "offering-1" }, cookies: [] }; - }; - - await seedPlaceholderCatalogOffering({ - call, - tenantId: "tenant-1", - cookies: [], - placeholderBaseUrl: "https://inference.invalid", - }); - - const modelCall = calls.find((entry) => - entry.path.endsWith("/catalog/models"), - ); - expect(modelCall?.body).toEqual({ canonicalName: PLACEHOLDER_MODEL }); - }); -}); - const { tempDir, track } = createCleanupHarness(); describe.skipIf(databaseUrl === undefined)("walking skeleton", () => {