From b1e6caa5d695b74c2023ec24af33b09afb344754 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:15 -0700 Subject: [PATCH 1/5] Add test for workflowDeployBody's catalog-offering shape --- scripts/e2e/harness.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/e2e/harness.test.ts b/scripts/e2e/harness.test.ts index 5b4d14800..37f4f93e4 100644 --- a/scripts/e2e/harness.test.ts +++ b/scripts/e2e/harness.test.ts @@ -11,6 +11,7 @@ import { createCleanupHarness, parseEnvFileDatabaseUrl, runCleanups, + workflowDeployBody, type SpawnedApp, } from "./harness.ts"; @@ -67,6 +68,29 @@ describe("createCleanupHarness", () => { }); }); +describe("workflowDeployBody", () => { + test("carries catalog offering ids, never a raw provider/apiKey source", () => { + const body = workflowDeployBody({ + assetId: "asset-1", + commitSha: "deadbeef", + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).toEqual({ + source: { + kind: "asset", + assetId: "asset-1", + package: { format: "source", commitSha: "deadbeef" }, + }, + entry: (body as { entry: string }).entry, + sourceOfferingIds: ["offering-1", "offering-2"], + defaultSourceOfferingId: "offering-1", + }); + expect(body).not.toHaveProperty("sources"); + expect(body).not.toHaveProperty("defaultSource"); + }); +}); + describe("parseEnvFileDatabaseUrl", () => { test("reads DATABASE_URL and ignores comments and blanks", () => { expect( From fae77c0682222b8149cb08b6b169e1caab79d33b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:41:23 -0700 Subject: [PATCH 2/5] e2e: deploy workflows by catalog offering ids The native workflow deploy route now requires sourceOfferingIds/ defaultSourceOfferingId resolved against the tenant's own catalog, rather than a raw provider/baseURL/apiKey source chain. Update workflowDeployBody accordingly, and extract the zero-cost noop catalog chain smoke-webhook.test.ts already built (an anthropic- plugin provider pointed at the hub's noop-inference endpoint) into a shared seedNoopCatalogOffering harness helper. Every deploying e2e suite now seeds or resolves a real catalog offering before deploying: walking-skeleton keeps a never-real placeholder host (assertNeverRealProvider still guards it), chat resolves the offering seedCatalog already planted rather than seeding a second chain, and heartbeat/workbench-digest/smoke-webhook seed the noop chain via the shared helper. cl-6324-launch-proof's section-mode deploy now reuses the offering id the proof already resolved when it narrowed the bench catalog to its pinned model. --- scripts/e2e/chat.test.ts | 48 +++++++++-- scripts/e2e/cl-6324-launch-proof.ts | 29 ++++--- scripts/e2e/harness.ts | 121 +++++++++++++++++++++++---- scripts/e2e/heartbeat.test.ts | 19 +++-- scripts/e2e/smoke-webhook.test.ts | 89 +++----------------- scripts/e2e/walking-skeleton.test.ts | 23 +++-- scripts/e2e/workbench-digest.test.ts | 19 +++-- 7 files changed, 219 insertions(+), 129 deletions(-) diff --git a/scripts/e2e/chat.test.ts b/scripts/e2e/chat.test.ts index deea7c826..da9b255c8 100644 --- a/scripts/e2e/chat.test.ts +++ b/scripts/e2e/chat.test.ts @@ -266,6 +266,47 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { log: () => undefined, }); + // The echo deploy resolves against the catalog offering `seedCatalog` + // just planted for the same provider/model this workflow declares, + // rather than seeding a second inference chain. + const echoModelsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/models`, + undefined, + user1.cookies, + ); + expectStatus("list catalog models", echoModelsListed, 200); + const echoModelRows = ( + echoModelsListed.data as { data: { id: string; canonicalName: string }[] } + ).data; + const echoModelId = echoModelRows.find( + (row) => row.canonicalName === "claude-sonnet-5", + )?.id; + if (echoModelId === undefined) { + throw new Error( + `no catalog model named "claude-sonnet-5": ${JSON.stringify(echoModelsListed.data)}`, + ); + } + + const echoOfferingsListed = await api( + "GET", + `/api/tenants/${tenantId}/catalog/offerings`, + undefined, + user1.cookies, + ); + expectStatus("list catalog offerings", echoOfferingsListed, 200); + const echoOfferingRows = ( + echoOfferingsListed.data as { data: { id: string; modelId: string }[] } + ).data; + const echoOfferingId = echoOfferingRows.find( + (row) => row.modelId === echoModelId, + )?.id; + if (echoOfferingId === undefined) { + throw new Error( + `no catalog offering for model "${echoModelId}": ${JSON.stringify(echoOfferingsListed.data)}`, + ); + } + // Seed the echo workflow as a deployed, invitable definition: the // same asset-publish → git-token → smart-HTTP push → native deploy // path `scripts/e2e/walking-skeleton.test.ts` proves end to end. @@ -334,11 +375,8 @@ describe.skipIf(databaseUrl === undefined)("chat e2e", () => { workflowDeployBody({ assetId: echoAssetId, commitSha: echoPushed.commitSha, - sourceId: "src-echo-e2e", - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [echoOfferingId], + defaultSourceOfferingId: echoOfferingId, }), user1.cookies, ); diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index 674315090..ef3be6297 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,7 +369,7 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - await hop("narrow the bench catalog to the pinned model", async () => { + const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { await plantGrant("model-offering:*", "read"); await plantGrant("model-offering:*", "manage"); await plantGrant("model:*", "read"); @@ -409,8 +409,13 @@ async function main(): Promise { "data", "model offerings", ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; for (const offering of offeringRows) { - if (offering.modelId === pinned.id || offering.disabled) continue; + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; const patched = await api( hub.baseUrl, "PATCH", @@ -420,10 +425,17 @@ async function main(): Promise { ); expectStatus(`disable offering ${offering.id}`, patched, 200); } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } console.log( ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); + return pinnedOffering.id; }); const assistantDefinitionId = await hop( @@ -694,7 +706,6 @@ async function main(): Promise { // own durable event log, which is the artefact the milestone's // `RunStarted` assertion was always about. const SECTION_ASSET_NAME = "cl6324-section"; - const SECTION_SOURCE_ID = "cl6324-section-source"; const SECTION_TURN_TIMEOUT_MS = 180_000; const sectionAddress = `${SECTION_ASSET_NAME}@${tenant.tenantDomain}`; @@ -780,16 +791,8 @@ async function main(): Promise { package: { format: "source", commitSha: pushed.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: SECTION_SOURCE_ID, - provider: model.provider, - baseURL: model.baseURL, - apiKey: model.apiKey, - model: model.model, - }, - ], - defaultSource: SECTION_SOURCE_ID, + sourceOfferingIds: [pinnedOfferingId], + defaultSourceOfferingId: pinnedOfferingId, }, user.cookies, ); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 099a7a210..27cd39204 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -653,16 +653,17 @@ export async function pushWorkflowSource(options: { /** * The deploy body a code-sourced asset deployment takes: the pushed * commit is the definition's pin, and the entry names the - * `interchange.workflow` module the sidecar evaluates. + * `interchange.workflow` module the sidecar evaluates. The native route + * resolves inference against the tenant's own catalog, so the caller + * supplies the ordered catalog offering ids to deploy against rather + * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` + * for the zero-cost way to obtain one. */ export function workflowDeployBody(options: { assetId: string; commitSha: string; - sourceId: string; - provider: string; - baseURL: string; - apiKey: string; - model: string; + sourceOfferingIds: string[]; + defaultSourceOfferingId: string; }): Record { return { source: { @@ -671,15 +672,103 @@ export function workflowDeployBody(options: { package: { format: "source", commitSha: options.commitSha }, }, entry: WORKFLOW_SOURCE_ENTRY, - sources: [ - { - id: options.sourceId, - provider: options.provider, - baseURL: options.baseURL, - apiKey: options.apiKey, - model: options.model, - }, - ], - defaultSource: options.sourceId, + sourceOfferingIds: options.sourceOfferingIds, + defaultSourceOfferingId: options.defaultSourceOfferingId, }; } + +/** + * Plants the zero-cost catalog chain every deploying e2e suite needs: a + * catalog model, an `anthropic`-plugin provider/credential pointed at + * `noopBaseUrl` (never a real provider host — enforced below), and the + * catalog offering joining them. Returns the offering id a deploy's + * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. + * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. + */ +export async function seedNoopCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + noopBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringFieldOf(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringFieldOf(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringFieldOf( + credential.data, + "id", + "create credential", + ); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.noopBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringFieldOf( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; +} + +function stringFieldOf(data: unknown, field: string, what: string): string { + if (typeof data === "object" && data !== null && field in data) { + const value = (data as Record)[field]; + if (typeof value === "string" && value !== "") return value; + } + throw new Error( + `${what}: missing string field "${field}": ${JSON.stringify(data)}`, + ); +} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index c19a6b6a2..9aefb900f 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -203,16 +204,22 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-heartbeat-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 1d248defa..760aca6a3 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -30,7 +30,6 @@ import { } from "../../workflows/heartbeat/src/index.ts"; import { api, - assertNeverRealProvider, connectE2eDb, createCleanupHarness, e2eDatabaseUrl, @@ -39,6 +38,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -138,77 +138,17 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real - // model. Mirrors `workbench-digest.test.ts`'s noop catalog seeding. + // model. const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - assertNeverRealProvider(noopBaseUrl, "noop catalog provider baseURL"); - await hop("noop catalog seeding", async () => { - const model = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringField(model.data, "id", "create catalog model"); - - const provider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringField(provider.data, "id", "create provider"); - - const credential = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringField( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: noopBaseUrl, - credentialId, - }, + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies2) => + api(hub.baseUrl, method, path, body, cookies2), + tenantId, cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringField( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await api( - hub.baseUrl, - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - }); + noopBaseUrl, + }), + ); const assetName = "heartbeat"; const { assetId, commitSha } = await hop( @@ -256,16 +196,11 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { ); const definitionId = await hop("workflow deploy", async () => { - const sourceId = "src-smoke-webhook-e2e"; - assertNeverRealProvider(noopBaseUrl, "workflow deploy source baseURL"); const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: noopBaseUrl, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; for (;;) { diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 833612dd6..8937f9b5e 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -35,6 +35,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -199,20 +200,30 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { }, ); + // Hop: noop catalog seeding. The native deploy resolves inference + // against a real catalog offering; this suite's chosen offering + // still names an unreachable placeholder host, since deployment + // never calls inference and full run-completion is not asserted. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: "https://inference.invalid", + }), + ); + // Hop: workflow deploy via the native deploy API. Retries while // the hub still answers 502 (the sidecar's dial-in may not have // completed yet); any other failure is final. The inference // source is a placeholder — deployment does not call inference. const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-echo-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: "https://inference.invalid", - apiKey: "e2e-placeholder", - model: "claude-sonnet-5", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index ecf2dc56e..8917bf823 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -29,6 +29,7 @@ import { hop, provisionSidecar, pushWorkflowSource, + seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -207,16 +208,22 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. + const { offeringId } = await hop("noop catalog seeding", () => + seedNoopCatalogOffering({ + call: (method, path, body, cookies) => + api(hub.baseUrl, method, path, body, cookies), + tenantId, + cookies: user.cookies, + noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, + }), + ); + const deploymentId = await hop("workflow deploy", async () => { - const sourceId = "src-workbench-digest-e2e"; const body = workflowDeployBody({ assetId, commitSha, - sourceId: sourceId, - provider: "anthropic", - baseURL: `${hub.baseUrl}/api/chat/noop-inference`, - apiKey: "noop", - model: "noop", + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }); const deadline = Date.now() + 60_000; let res: ApiResult; From b03ce16b54f9de7e1bb8f14a7efce6a50cd7f9ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:43:45 -0700 Subject: [PATCH 3/5] Fix formatting --- scripts/e2e/cl-6324-launch-proof.ts | 125 ++++++++++++++-------------- scripts/e2e/harness.ts | 4 +- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/scripts/e2e/cl-6324-launch-proof.ts b/scripts/e2e/cl-6324-launch-proof.ts index ef3be6297..80766c99d 100644 --- a/scripts/e2e/cl-6324-launch-proof.ts +++ b/scripts/e2e/cl-6324-launch-proof.ts @@ -369,74 +369,77 @@ async function main(): Promise { // on purpose, so it narrows the bench's own catalog to that model // through the catalog API rather than leaving the turn's model to a // coin flip the proof is not about. - const pinnedOfferingId = await hop("narrow the bench catalog to the pinned model", async () => { - await plantGrant("model-offering:*", "read"); - await plantGrant("model-offering:*", "manage"); - await plantGrant("model:*", "read"); + const pinnedOfferingId = await hop( + "narrow the bench catalog to the pinned model", + async () => { + await plantGrant("model-offering:*", "read"); + await plantGrant("model-offering:*", "manage"); + await plantGrant("model:*", "read"); - const models = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench catalog models", models, 200); - const modelRows = arrayField(models.data, "data", "catalog models") as { - id: string; - canonicalName: string; - }[]; - const pinned = modelRows.find( - (row) => row.canonicalName === proofModelSource.model, - ); - if (pinned === undefined) { - throw new Error( - `the bench catalog carries no model named ${proofModelSource.model}; ` + - `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + const models = await api( + hub.baseUrl, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/models?limit=200`, + undefined, + user.cookies, ); - } - - const offerings = await api( - hub.baseUrl, - "GET", - `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, - undefined, - user.cookies, - ); - expectStatus("list the bench model offerings", offerings, 200); - const offeringRows = arrayField( - offerings.data, - "data", - "model offerings", - ) as { id: string; modelId: string; disabled: boolean }[]; - let pinnedOffering: { id: string; modelId: string } | undefined; - for (const offering of offeringRows) { - if (offering.modelId === pinned.id) { - pinnedOffering = offering; - continue; + expectStatus("list the bench catalog models", models, 200); + const modelRows = arrayField(models.data, "data", "catalog models") as { + id: string; + canonicalName: string; + }[]; + const pinned = modelRows.find( + (row) => row.canonicalName === proofModelSource.model, + ); + if (pinned === undefined) { + throw new Error( + `the bench catalog carries no model named ${proofModelSource.model}; ` + + `it has ${JSON.stringify(modelRows.map((m) => m.canonicalName))}`, + ); } - if (offering.disabled) continue; - const patched = await api( + + const offerings = await api( hub.baseUrl, - "PATCH", - `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, - { disabled: true }, + "GET", + `/api/tenants/${tenant.tenantId}/catalog/offerings?limit=200`, + undefined, user.cookies, ); - expectStatus(`disable offering ${offering.id}`, patched, 200); - } - if (pinnedOffering === undefined) { - throw new Error( - `no catalog offering resolves the pinned model ${proofModelSource.model} ` + - `(id ${pinned.id})`, + expectStatus("list the bench model offerings", offerings, 200); + const offeringRows = arrayField( + offerings.data, + "data", + "model offerings", + ) as { id: string; modelId: string; disabled: boolean }[]; + let pinnedOffering: { id: string; modelId: string } | undefined; + for (const offering of offeringRows) { + if (offering.modelId === pinned.id) { + pinnedOffering = offering; + continue; + } + if (offering.disabled) continue; + const patched = await api( + hub.baseUrl, + "PATCH", + `/api/tenants/${tenant.tenantId}/catalog/offerings/${offering.id}`, + { disabled: true }, + user.cookies, + ); + expectStatus(`disable offering ${offering.id}`, patched, 200); + } + if (pinnedOffering === undefined) { + throw new Error( + `no catalog offering resolves the pinned model ${proofModelSource.model} ` + + `(id ${pinned.id})`, + ); + } + console.log( + ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + + `(${String(offeringRows.length - 1)} other offerings disabled)`, ); - } - console.log( - ` TRANSCRIPT — bench catalog narrowed to ${proofModelSource.model} ` + - `(${String(offeringRows.length - 1)} other offerings disabled)`, - ); - return pinnedOffering.id; - }); + return pinnedOffering.id; + }, + ); const assistantDefinitionId = await hop( "PROOF 1 — 'assistant' is invitable tenant-wide", diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 27cd39204..cebc4e53d 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -760,7 +760,9 @@ export async function seedNoopCatalogOffering(options: { cookies, ); expectStatus("create catalog offering", offering, 201); - return { offeringId: stringFieldOf(offering.data, "id", "create catalog offering") }; + return { + offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), + }; } function stringFieldOf(data: unknown, field: string, what: string): string { From 2587cd2b5af3ecbb27bb13bcb652910bd10b29aa Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sat, 5 Sep 2026 23:54:46 -0700 Subject: [PATCH 4/5] longevity-sim: deploy workflows by catalog offering ids Another workflowDeployBody caller CI's typecheck job caught: this stack seeds one catalog offering per real inference target already, so track each one's id and pass it through to the deploy body instead of the raw provider/baseURL/apiKey shape. --- packages/longevity-sim/src/stack.ts | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/longevity-sim/src/stack.ts b/packages/longevity-sim/src/stack.ts index a7dc91776..59c4ff72a 100644 --- a/packages/longevity-sim/src/stack.ts +++ b/packages/longevity-sim/src/stack.ts @@ -437,7 +437,9 @@ export async function bootLongevityStack( // credential row still requires some string, so each target's own // `apiKey` (expected to be the same placeholder) is threaded through // unchanged rather than this package inventing its own convention. - async function seedCatalogChain(target: InferenceTarget): Promise { + async function seedCatalogChain( + target: InferenceTarget, + ): Promise<{ offeringId: string }> { const model = await api( hub.baseUrl, "POST", @@ -509,10 +511,15 @@ export async function bootLongevityStack( owner.cookies, ); expectStatus(`create catalog offering ${target.model}`, offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; } + const offeringIdByTargetLabel = new Map(); for (const target of options.realTargets) { - await seedCatalogChain(target); + const { offeringId } = await seedCatalogChain(target); + offeringIdByTargetLabel.set(target.label, offeringId); } for (const skill of options.skills ?? []) { @@ -668,6 +675,13 @@ export async function bootLongevityStack( workflowJson, }); + const offeringId = offeringIdByTargetLabel.get(target.label); + if (offeringId === undefined) { + throw new Error( + `agent ${spec.key}: no catalog offering seeded for realTarget "${target.label}"`, + ); + } + const deployDeadline = Date.now() + 90_000; for (;;) { if (sidecar.exited()) { @@ -682,11 +696,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId, commitSha: pushed.commitSha, - sourceId: `src-agent-${spec.handle}`, - provider: target.provider, - baseURL: target.baseURL, - apiKey: target.apiKey, - model: target.model, + sourceOfferingIds: [offeringId], + defaultSourceOfferingId: offeringId, }), owner.cookies, ); @@ -847,6 +858,12 @@ export async function bootLongevityStack( if (routineTarget === undefined) { throw new Error("unreachable: realTargets checked non-empty above"); } + const routineOfferingId = offeringIdByTargetLabel.get(routineTarget.label); + if (routineOfferingId === undefined) { + throw new Error( + `no catalog offering seeded for realTarget "${routineTarget.label}"`, + ); + } const assetName = "longevity-heartbeat"; const heartbeatAssetCreated = await api( hub.baseUrl, @@ -912,11 +929,8 @@ export async function bootLongevityStack( workflowDeployBody({ assetId: heartbeatAssetId, commitSha: heartbeatPushed.commitSha, - sourceId: "src-longevity-heartbeat", - provider: "openai-compatible", - baseURL: routineTarget.baseURL, - apiKey: routineTarget.apiKey, - model: routineTarget.model, + sourceOfferingIds: [routineOfferingId], + defaultSourceOfferingId: routineOfferingId, }), owner.cookies, ); From 97480d0b63dd1729159311d7e70a42bd26fb7811 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 09:02:16 -0700 Subject: [PATCH 5/5] e2e harness: reuse @corbits/seeding's noop catalog offering chain harness.ts's seedNoopCatalogOffering duplicated the exact chain @corbits/seeding's ensureNoopCatalogOffering already plants (CL-7461). Every deploying suite that pointed the offering at the hub's own real noop-inference endpoint (heartbeat, smoke-webhook, workbench-digest) now calls the shared helper instead; walking-skeleton keeps its own local copy, since it deliberately points the offering at an unreachable placeholder host rather than the hub's noop-inference endpoint, which the shared helper does not support. --- scripts/e2e/harness.ts | 103 +-------------------------- scripts/e2e/heartbeat.test.ts | 15 ++-- scripts/e2e/smoke-webhook.test.ts | 16 ++--- scripts/e2e/walking-skeleton.test.ts | 93 +++++++++++++++++++++++- scripts/e2e/workbench-digest.test.ts | 15 ++-- 5 files changed, 117 insertions(+), 125 deletions(-) diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index cebc4e53d..39aa8393c 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -656,8 +656,9 @@ export async function pushWorkflowSource(options: { * `interchange.workflow` module the sidecar evaluates. The native route * resolves inference against the tenant's own catalog, so the caller * supplies the ordered catalog offering ids to deploy against rather - * than a raw provider/baseURL/apiKey triple — see `seedNoopCatalogOffering` - * for the zero-cost way to obtain one. + * than a raw provider/baseURL/apiKey triple — see + * `@corbits/seeding`'s `ensureNoopCatalogOffering` for the zero-cost way + * to obtain one. */ export function workflowDeployBody(options: { assetId: string; @@ -676,101 +677,3 @@ export function workflowDeployBody(options: { defaultSourceOfferingId: options.defaultSourceOfferingId, }; } - -/** - * Plants the zero-cost catalog chain every deploying e2e suite needs: a - * catalog model, an `anthropic`-plugin provider/credential pointed at - * `noopBaseUrl` (never a real provider host — enforced below), and the - * catalog offering joining them. Returns the offering id a deploy's - * `sourceOfferingIds`/`defaultSourceOfferingId` resolves against. - * Mirrors the chain `smoke-webhook.test.ts` proved out by hand. - */ -export async function seedNoopCatalogOffering(options: { - call: ( - method: string, - path: string, - body?: unknown, - cookies?: string[], - ) => Promise; - tenantId: string; - cookies: string[]; - noopBaseUrl: string; -}): Promise<{ offeringId: string }> { - assertNeverRealProvider(options.noopBaseUrl, "noop catalog provider baseURL"); - const { call, tenantId, cookies } = options; - - const model = await call( - "POST", - `/api/tenants/${tenantId}/catalog/models`, - { canonicalName: "noop" }, - cookies, - ); - expectStatus("create catalog model", model, 201); - const modelId = stringFieldOf(model.data, "id", "create catalog model"); - - const provider = await call( - "POST", - `/api/tenants/${tenantId}/providers`, - { name: "anthropic", plugin: "anthropic" }, - cookies, - ); - expectStatus("create provider", provider, 201); - const providerId = stringFieldOf(provider.data, "id", "create provider"); - - const credential = await call( - "POST", - `/api/tenants/${tenantId}/credentials`, - { - providerId, - name: "anthropic-default", - type: "api_key", - secret: "noop", - }, - cookies, - ); - expectStatus("create credential", credential, 201); - const credentialId = stringFieldOf( - credential.data, - "id", - "create credential", - ); - - const catalogProvider = await call( - "POST", - `/api/tenants/${tenantId}/catalog/providers`, - { - name: "anthropic", - plugin: "anthropic", - baseURL: options.noopBaseUrl, - credentialId, - }, - cookies, - ); - expectStatus("create catalog provider", catalogProvider, 201); - const catalogProviderId = stringFieldOf( - catalogProvider.data, - "id", - "create catalog provider", - ); - - const offering = await call( - "POST", - `/api/tenants/${tenantId}/catalog/offerings`, - { modelId, providerId: catalogProviderId }, - cookies, - ); - expectStatus("create catalog offering", offering, 201); - return { - offeringId: stringFieldOf(offering.data, "id", "create catalog offering"), - }; -} - -function stringFieldOf(data: unknown, field: string, what: string): string { - if (typeof data === "object" && data !== null && field in data) { - const value = (data as Record)[field]; - if (typeof value === "string" && value !== "") return value; - } - throw new Error( - `${what}: missing string field "${field}": ${JSON.stringify(data)}`, - ); -} diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 9aefb900f..668ed3b02 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -19,6 +19,7 @@ import { buildHeartbeatWorkflow, serializeHeartbeatWorkflow, } from "../../workflows/heartbeat/src/index.ts"; +import { ensureNoopCatalogOffering } from "../../packages/seeding/src/index.ts"; import { api, createCleanupHarness, @@ -29,7 +30,6 @@ import { hop, provisionSidecar, pushWorkflowSource, - seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -204,14 +204,15 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. - const { offeringId } = await hop("noop catalog seeding", () => - seedNoopCatalogOffering({ - call: (method, path, body, cookies) => + const offeringId = await hop("noop catalog seeding", () => + ensureNoopCatalogOffering( + (method, path, body, cookies) => api(hub.baseUrl, method, path, body, cookies), + user.cookies, tenantId, - cookies: user.cookies, - noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, - }), + hub.baseUrl, + () => {}, + ), ); const deploymentId = await hop("workflow deploy", async () => { diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 760aca6a3..66a258c60 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -28,6 +28,7 @@ import { buildHeartbeatWorkflow, serializeHeartbeatWorkflow, } from "../../workflows/heartbeat/src/index.ts"; +import { ensureNoopCatalogOffering } from "../../packages/seeding/src/index.ts"; import { api, connectE2eDb, @@ -38,7 +39,6 @@ import { hop, provisionSidecar, pushWorkflowSource, - seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -139,15 +139,15 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { // The zero-cost catalog chain: an anthropic-plugin provider whose // base URL is the hub's own noop-inference endpoint, never a real // model. - const noopBaseUrl = `${hub.baseUrl}/api/chat/noop-inference`; - const { offeringId } = await hop("noop catalog seeding", () => - seedNoopCatalogOffering({ - call: (method, path, body, cookies2) => + const offeringId = await hop("noop catalog seeding", () => + ensureNoopCatalogOffering( + (method, path, body, cookies2) => api(hub.baseUrl, method, path, body, cookies2), - tenantId, cookies, - noopBaseUrl, - }), + tenantId, + hub.baseUrl, + () => {}, + ), ); const assetName = "heartbeat"; diff --git a/scripts/e2e/walking-skeleton.test.ts b/scripts/e2e/walking-skeleton.test.ts index 8937f9b5e..4713eba04 100644 --- a/scripts/e2e/walking-skeleton.test.ts +++ b/scripts/e2e/walking-skeleton.test.ts @@ -28,6 +28,7 @@ import { } from "../../workflows/echo/src/index.ts"; import { api, + assertNeverRealProvider, createCleanupHarness, e2eDatabaseUrl, expectStatus, @@ -35,7 +36,6 @@ import { hop, provisionSidecar, pushWorkflowSource, - seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -62,6 +62,93 @@ function stringField(data: unknown, field: string, what: string): string { ); } +/** + * Plants the same catalog-model/provider/credential/offering chain as + * `@corbits/seeding`'s `ensureNoopCatalogOffering`, but pointed at an + * unreachable placeholder host rather than the hub's own + * `noop-inference` endpoint — deliberately, since this suite never + * calls inference and asserts only that the deploy is + * address-reachable (see the module comment above). + */ +async function seedPlaceholderCatalogOffering(options: { + call: ( + method: string, + path: string, + body?: unknown, + cookies?: string[], + ) => Promise; + tenantId: string; + cookies: string[]; + placeholderBaseUrl: string; +}): Promise<{ offeringId: string }> { + assertNeverRealProvider( + options.placeholderBaseUrl, + "placeholder catalog provider baseURL", + ); + const { call, tenantId, cookies } = options; + + const model = await call( + "POST", + `/api/tenants/${tenantId}/catalog/models`, + { canonicalName: "noop" }, + cookies, + ); + expectStatus("create catalog model", model, 201); + const modelId = stringField(model.data, "id", "create catalog model"); + + const provider = await call( + "POST", + `/api/tenants/${tenantId}/providers`, + { name: "anthropic", plugin: "anthropic" }, + cookies, + ); + expectStatus("create provider", provider, 201); + const providerId = stringField(provider.data, "id", "create provider"); + + const credential = await call( + "POST", + `/api/tenants/${tenantId}/credentials`, + { + providerId, + name: "anthropic-default", + type: "api_key", + secret: "noop", + }, + cookies, + ); + expectStatus("create credential", credential, 201); + const credentialId = stringField(credential.data, "id", "create credential"); + + const catalogProvider = await call( + "POST", + `/api/tenants/${tenantId}/catalog/providers`, + { + name: "anthropic", + plugin: "anthropic", + baseURL: options.placeholderBaseUrl, + credentialId, + }, + cookies, + ); + expectStatus("create catalog provider", catalogProvider, 201); + const catalogProviderId = stringField( + catalogProvider.data, + "id", + "create catalog provider", + ); + + const offering = await call( + "POST", + `/api/tenants/${tenantId}/catalog/offerings`, + { modelId, providerId: catalogProviderId }, + cookies, + ); + expectStatus("create catalog offering", offering, 201); + return { + offeringId: stringField(offering.data, "id", "create catalog offering"), + }; +} + const { tempDir, track } = createCleanupHarness(); describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { @@ -205,12 +292,12 @@ describe.skipIf(databaseUrl === undefined)("walking skeleton", () => { // still names an unreachable placeholder host, since deployment // never calls inference and full run-completion is not asserted. const { offeringId } = await hop("noop catalog seeding", () => - seedNoopCatalogOffering({ + seedPlaceholderCatalogOffering({ call: (method, path, body, cookies) => api(hub.baseUrl, method, path, body, cookies), tenantId, cookies: user.cookies, - noopBaseUrl: "https://inference.invalid", + placeholderBaseUrl: "https://inference.invalid", }), ); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index 8917bf823..b560c98a2 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -19,6 +19,7 @@ import { buildWorkbenchDigestWorkflow, serializeWorkbenchDigestWorkflow, } from "../../workflows/workbench-digest/src/index.ts"; +import { ensureNoopCatalogOffering } from "../../packages/seeding/src/index.ts"; import { api, createCleanupHarness, @@ -29,7 +30,6 @@ import { hop, provisionSidecar, pushWorkflowSource, - seedNoopCatalogOffering, workflowDeployBody, startHub, startSidecar, @@ -208,14 +208,15 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { // whole point of this suite: a run started against this source // actually completes an inference call, at zero cost, because // noop-inference answers it locally without reaching a real model. - const { offeringId } = await hop("noop catalog seeding", () => - seedNoopCatalogOffering({ - call: (method, path, body, cookies) => + const offeringId = await hop("noop catalog seeding", () => + ensureNoopCatalogOffering( + (method, path, body, cookies) => api(hub.baseUrl, method, path, body, cookies), + user.cookies, tenantId, - cookies: user.cookies, - noopBaseUrl: `${hub.baseUrl}/api/chat/noop-inference`, - }), + hub.baseUrl, + () => {}, + ), ); const deploymentId = await hop("workflow deploy", async () => {