From eac034c62d4e2cbebaef3e8b4ce1963cdd930c83 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 17 Sep 2026 12:09:41 -0700 Subject: [PATCH] feat(hub)!: drop the setup-status and noop-inference mounts (CL-8160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hub no longer mounts /api/setup (setup-status.ts) or /api/chat/noop-inference (e2e/noop-inference.ts): a scheduled step that needs no model must not call one, and setup status is now derived client-side from the converge state (findOwnedTenants over createFetchStockHub) instead of a hub-served route. - apps/web/src/onboarding.ts's triggerFirstLoginProvisioning now reads ownership of tenants directly over stock routes instead of GET /api/setup/status. - packages/connections/src/seed-catalog.ts drops ensureNoopCatalogOffering and its NOOP_* constants; its underlying ensureCatalogModel/Provider/Offering primitives are now exported for reuse. packages/onboarding/src/tenant-seed.ts drops the dead noop-pinned-workflow branch (modelSource) it fed — no seeded workflow set that field. - workflows/heartbeat and workflows/workbench-digest stop documenting a deployer-side noop ModelSource pin: with no non-agent step primitive wired on the deployed host (@intx/workflow's `action` primitive has no invokeAction callback anywhere), these now deploy against the tenant's own real, resolved catalog offering like every other workflow, per owner decision. - E2E suites that exercised the noop path (scripts/e2e/heartbeat.test.ts, workbench-digest.test.ts, smoke-webhook.test.ts) now start their own local noop inference server (scripts/e2e/noop-inference-server.ts, moved handler) and plant a catalog offering against it directly, never against the hub. Deleted tests (hard cutover, encoded the removed hub routes): - apps/hub/src/setup-status.test.ts - apps/hub/src/e2e/noop-inference.test.ts - apps/web/test/onboarding.test.tsx - the "ensureNoopCatalogOffering" describe block in packages/connections/src/seed-catalog.test.ts --- apps/hub/src/e2e/noop-inference.test.ts | 141 ------ apps/hub/src/e2e/noop-inference.ts | 83 ---- apps/hub/src/index.ts | 26 +- apps/hub/src/setup-status.test.ts | 74 --- apps/hub/src/setup-status.ts | 53 --- apps/web/src/onboarding.ts | 46 +- apps/web/test/onboarding.test.tsx | 448 ------------------ packages/connections/src/seed-catalog.test.ts | 37 +- packages/connections/src/seed-catalog.ts | 113 +---- .../credential-wiring-e2e.drizzle.test.ts | 2 +- packages/onboarding/src/tenant-seed.ts | 84 +--- scripts/checks/report-error-baseline.txt | 1 - scripts/e2e/harness.ts | 33 +- scripts/e2e/heartbeat.test.ts | 27 +- scripts/e2e/noop-inference-server.ts | 201 ++++++++ scripts/e2e/smoke-webhook.test.ts | 23 +- scripts/e2e/workbench-digest.test.ts | 38 +- workflows/heartbeat/src/index.ts | 24 +- workflows/workbench-digest/src/index.ts | 20 +- 19 files changed, 361 insertions(+), 1113 deletions(-) delete mode 100644 apps/hub/src/e2e/noop-inference.test.ts delete mode 100644 apps/hub/src/e2e/noop-inference.ts delete mode 100644 apps/hub/src/setup-status.test.ts delete mode 100644 apps/hub/src/setup-status.ts delete mode 100644 apps/web/test/onboarding.test.tsx create mode 100644 scripts/e2e/noop-inference-server.ts diff --git a/apps/hub/src/e2e/noop-inference.test.ts b/apps/hub/src/e2e/noop-inference.test.ts deleted file mode 100644 index 8b89fee9c3..0000000000 --- a/apps/hub/src/e2e/noop-inference.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -// Proves `createNoopInferenceRoutes()` serves a wire-valid Anthropic -// SSE stream: `POST /v1/messages` (whatever body, whatever — or no — -// `x-api-key`) returns a `message_start`/`content_block_start`/ -// `content_block_delta`/`content_block_stop`/`message_delta`/ -// `message_stop` sequence, parsed here with the exact `parseSSE` byte -// framing and the exact per-event arktype schemas -// `@intx/inference/src/providers/anthropic.ts` uses to accept -// real Anthropic traffic — so a drift in this endpoint's shape shows -// up here, not only when a real workbench host tries to use it. -import { describe, expect, test } from "bun:test"; -import { type } from "arktype"; -import { createNoopInferenceRoutes } from "./noop-inference"; - -const ContentBlockDelta = type({ - type: "'content_block_delta'", - index: "number", - delta: { type: "string", "text?": "string" }, -}); -const ContentBlockStart = type({ - type: "'content_block_start'", - index: "number", - "content_block?": { type: "string" }, -}); -const ContentBlockStop = type({ - type: "'content_block_stop'", - index: "number", -}); -const MessageDelta = type({ - type: "'message_delta'", - "usage?": { "output_tokens?": "number" }, -}); -const MessageStart = type({ - type: "'message_start'", - "message?": { - "usage?": { - "input_tokens?": "number", - "output_tokens?": "number", - "cache_read_input_tokens?": "number", - "cache_creation_input_tokens?": "number", - }, - }, -}); -const MessageStop = type({ type: "'message_stop'" }); - -const AnthropicSSEEvent = ContentBlockDelta.or(ContentBlockStart) - .or(ContentBlockStop) - .or(MessageDelta) - .or(MessageStart) - .or(MessageStop); - -async function readSSEEvents(response: Response): Promise { - const text = await response.text(); - const payloads = text - .split("\n\n") - .map((chunk) => chunk.trim()) - .filter((chunk) => chunk.length > 0) - .map((chunk) => chunk.replace(/^data:\s?/, "")); - return payloads.map((payload) => JSON.parse(payload)); -} - -describe("createNoopInferenceRoutes", () => { - test("serves a parseable Anthropic-shaped SSE stream for any body and no x-api-key", async () => { - const app = createNoopInferenceRoutes(); - - const response = await app.request("/v1/messages", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "claude-sonnet-5", messages: [] }), - }); - - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("text/event-stream"); - - const events = await readSSEEvents(response); - expect(events).toHaveLength(6); - - const parsed = events.map((event) => { - const result = AnthropicSSEEvent(event); - if (result instanceof type.errors) { - throw new Error( - `event failed the Anthropic SSE schema: ${result.summary}`, - ); - } - return result; - }); - - expect(parsed.map((event) => event.type)).toEqual([ - "message_start", - "content_block_start", - "content_block_delta", - "content_block_stop", - "message_delta", - "message_stop", - ]); - - const messageStart = parsed[0] as { - type: "message_start"; - message?: { usage?: { input_tokens?: number; output_tokens?: number } }; - }; - expect(messageStart.message?.usage?.input_tokens).toBe(0); - expect(messageStart.message?.usage?.output_tokens).toBe(0); - - const delta = parsed[2] as { - type: "content_block_delta"; - index: number; - delta: { type: string; text?: string }; - }; - expect(delta.index).toBe(0); - expect(delta.delta.type).toBe("text_delta"); - // Deliberately empty — see `noop-inference.ts`'s own doc: a - // non-empty delta would make `connector.reply` treat this as a - // real reply and post it into the workbench's mailbox. - expect(delta.delta.text).toBe(""); - - const usageDelta = parsed[4] as { - type: "message_delta"; - usage?: { output_tokens?: number }; - }; - expect(usageDelta.usage?.output_tokens).toBeGreaterThan(0); - }); - - test("accepts a request carrying an arbitrary x-api-key exactly the same as one carrying none", async () => { - const app = createNoopInferenceRoutes(); - - const withKey = await app.request("/v1/messages", { - method: "POST", - headers: { "x-api-key": "not-checked-at-all" }, - body: "{}", - }); - const withoutKey = await app.request("/v1/messages", { - method: "POST", - body: "{}", - }); - - expect(withKey.status).toBe(200); - expect(withoutKey.status).toBe(200); - expect(await readSSEEvents(withKey)).toEqual( - await readSSEEvents(withoutKey), - ); - }); -}); diff --git a/apps/hub/src/e2e/noop-inference.ts b/apps/hub/src/e2e/noop-inference.ts deleted file mode 100644 index eabb658ebf..0000000000 --- a/apps/hub/src/e2e/noop-inference.ts +++ /dev/null @@ -1,83 +0,0 @@ -// A hub-served stand-in for a real Anthropic endpoint, spoken by -// agents that must never produce text: `workflows/heartbeat` and -// `workflows/workbench-digest` pin their `ModelSource` here (see -// `@corbits/onboarding`'s `tenant-seed`) so their scheduled turns complete -// instantly against a constant empty reply instead of burning a live -// model call whose output would be discarded. -// -// The wire shape matches exactly what `@intx/inference/src/providers/anthropic.ts`'s -// `parseResponse` accepts (see that file's `AnthropicSSEEvent` union, -// ~line 490-560): `message_start` (with zeroed usage), one -// `content_block_start` at index 0, one `content_block_delta` carrying -// a `text_delta`, `content_block_stop`, `message_delta` (with a -// nonzero `output_tokens` so downstream usage accounting sees a real -// number), then `message_stop`. `@intx/inference`'s `parseSSE` (see -// that file) needs no `[DONE]` sentinel — the stream simply ends. -// -// The delta's text is deliberately empty, not merely short: an empty -// reply is exactly the posture a real model obeying "never reply" -// would earn by producing no text, so nothing downstream mistakes the -// constant for a real answer. -import { Hono } from "hono"; - -const NOOP_REPLY_TEXT = ""; - -function sseLine(event: Record): string { - return `data: ${JSON.stringify(event)}\n\n`; -} - -/** - * `POST /v1/messages` — the same path `buildRequest` posts to - * (`{baseURL}/v1/messages`) — accepting any body and any `x-api-key` - * (including none) and always returning the same constant SSE stream. - * No auth check is safe here specifically because the handler takes no - * action, reads no state, and returns a constant: there is nothing an - * attacker could use this endpoint to do, see, or change regardless of - * what credential (if any) they present. - */ -export function createNoopInferenceRoutes(): Hono { - const app = new Hono(); - - app.post("/v1/messages", (_c) => { - const body = new ReadableStream({ - start(controller) { - const encoder = new TextEncoder(); - const events: Record[] = [ - { - type: "message_start", - message: { - usage: { - input_tokens: 0, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "text" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: NOOP_REPLY_TEXT }, - }, - { type: "content_block_stop", index: 0 }, - { type: "message_delta", usage: { output_tokens: 1 } }, - { type: "message_stop" }, - ]; - for (const event of events) { - controller.enqueue(encoder.encode(sseLine(event))); - } - controller.close(); - }, - }); - return new Response(body, { - headers: { "content-type": "text/event-stream" }, - }); - }); - - return app; -} diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index f5aac11c1d..aec7b4ab3d 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -22,7 +22,7 @@ import { user as userTable, workflowDefinition, } from "@intx/db/schema"; -import { and, count, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { createEnvKeyCredentialCipher, createNoopCredentialCipher, @@ -142,7 +142,6 @@ import { import { createLaunchCaches } from "./launch-caches"; import { hubErrorHandler } from "./hub-error-handler"; import { grantConditionRegistry } from "./grant-conditions"; -import { createNoopInferenceRoutes } from "./e2e/noop-inference"; import { wireMailRedelivery } from "./mail-redelivery"; import { getLogger, setup } from "@intx/log"; import { hexEncode } from "@intx/types"; @@ -204,7 +203,6 @@ import { import { type } from "arktype"; import { betterAuth } from "better-auth"; import { createSignInAttemptLimiter } from "./sign-in-rate-limit"; -import { createSetupStatusRoutes } from "./setup-status"; import { drizzleAdapter } from "better-auth/adapters/drizzle"; import { type Context, Hono, type Next } from "hono"; @@ -1111,28 +1109,6 @@ export async function createHub(config: HubConfig) { // record the reply path reads back to thread an agent's answer under // the message that woke its turn. Same `db` handle, same reasoning. const turnMailCorrelation = createDrizzleTurnMailCorrelationStore(db); - // Mounted outside the tenant prefix — the sidecar reaches it as a - // plain inference endpoint, never through tenant-scoped auth, the - // same way it reaches a real provider's API. Pinned by the heartbeat - // and workbench-digest workflow seeds (from the deleted seeding - // package), whose - // agents never produce text. `config.baseUrl` (not `localhost`) is - // what makes the URL usable from a sidecar on another machine. - app.route("/api/chat/noop-inference", createNoopInferenceRoutes()); - // Native cold-boot setup status (CL-8112). Mounted outside the tenant - // prefix like the noop-inference route: an empty hub has no tenant to - // scope to, and the first-login hook must read it before any bench - // exists. Counts come straight off the native user/tenant tables — no - // workbench package involved. - app.route( - "/api/setup", - createSetupStatusRoutes({ - countUsers: async () => - (await db.select({ n: count() }).from(userTable))[0]?.n ?? 0, - countTenants: async () => - (await db.select({ n: count() }).from(tenantTable))[0]?.n ?? 0, - }), - ); // The chat platform's invite-launch fallback: a definition with no // model requirements of its own resolves the tenant-catalog default. const chatHostInferencePreferencesResolver = diff --git a/apps/hub/src/setup-status.test.ts b/apps/hub/src/setup-status.test.ts deleted file mode 100644 index db01b83878..0000000000 --- a/apps/hub/src/setup-status.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -// The native cold-boot status read (CL-8112): empty hub demands setup, -// any tenant means setup is behind us, and a count failure answers the -// hub's error envelope (never a leak, never a bare 500). -import { describe, expect, test } from "bun:test"; - -import { - createSetupStatusRoutes, - type SetupStatusCounts, -} from "./setup-status"; - -function counts(counters: SetupStatusCounts) { - return createSetupStatusRoutes(counters); -} - -describe("createSetupStatusRoutes", () => { - test("an empty hub reports setupRequired with zero counts", async () => { - const res = await counts({ - countUsers: async () => 0, - countTenants: async () => 0, - }).request("/status"); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - setupRequired: true, - userCount: 0, - tenantCount: 0, - }); - }); - - test("a hub with tenants reports setupRequired false with real counts", async () => { - const res = await counts({ - countUsers: async () => 3, - countTenants: async () => 2, - }).request("/status"); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - setupRequired: false, - userCount: 3, - tenantCount: 2, - }); - }); - - test("users with zero tenants still require setup — no bench exists", async () => { - const res = await counts({ - countUsers: async () => 1, - countTenants: async () => 0, - }).request("/status"); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - setupRequired: true, - userCount: 1, - tenantCount: 0, - }); - }); - - test("a count failure answers the error envelope with a refId", async () => { - const res = await counts({ - countUsers: async () => 0, - countTenants: async () => { - throw new Error("connection refused"); - }, - }).request("/status"); - - expect(res.status).toBe(500); - const body = (await res.json()) as { - error: { code: string; userMessage: string; refId: string }; - }; - expect(body.error.code).toBe("setup_status_failed"); - expect(body.error.userMessage).not.toContain("connection refused"); - expect(body.error.refId.length).toBeGreaterThan(0); - }); -}); diff --git a/apps/hub/src/setup-status.ts b/apps/hub/src/setup-status.ts deleted file mode 100644 index f661b62bda..0000000000 --- a/apps/hub/src/setup-status.ts +++ /dev/null @@ -1,53 +0,0 @@ -// Native cold-boot setup status (CL-8112). A read-only, unauthenticated -// `GET /status` reporting whether this hub has ever been set up, derived -// from native user/tenant counts — the same genesis rule the seed path -// uses: zero tenants means no bench has ever been minted, so the first -// login must route into setup. Replaces the web first-login hook's old -// provision-POST read, which belonged to the unmounted onboarding router. No policy, no auth logic (those are -// T6/T7) — just counts. - -import { Hono } from "hono"; -import { makeErrorEnvelope, reportError } from "@corbits/error-sink"; - -export type SetupStatusCounts = { - /** Native user-row count. */ - countUsers(): Promise; - /** Native tenant-row count. */ - countTenants(): Promise; -}; - -export type SetupStatus = { - readonly setupRequired: boolean; - readonly userCount: number; - readonly tenantCount: number; -}; - -export function createSetupStatusRoutes(deps: SetupStatusCounts): Hono { - const app = new Hono(); - app.get("/status", async (c) => { - try { - const [userCount, tenantCount] = await Promise.all([ - deps.countUsers(), - deps.countTenants(), - ]); - const body: SetupStatus = { - setupRequired: tenantCount === 0, - userCount, - tenantCount, - }; - return c.json(body); - } catch (error) { - const refId = reportError(error, { operation: "hub.setup_status" }); - return c.json( - makeErrorEnvelope({ - code: "setup_status_failed", - userMessage: - "Checking your workbench hit a snag. Try again in a moment.", - refId, - }), - 500, - ); - } - }); - return app; -} diff --git a/apps/web/src/onboarding.ts b/apps/web/src/onboarding.ts index 90687ac37e..46895c6981 100644 --- a/apps/web/src/onboarding.ts +++ b/apps/web/src/onboarding.ts @@ -1,19 +1,15 @@ -// The browser side of the first-login hook: one GET against the hub's -// native setup-status route, made once per session. A hub with zero -// tenants reports setup-required so the UI routes into the setup wizard; -// any tenant means a bench exists and the shell loads normally. Read-only -// on purpose (CL-8112): provisioning itself back onto the hub is T6/T7's -// surface, so this never mints anything — a broken status read blocks the -// shell loudly rather than leaving the user silently benchless. +// The browser side of the first-login hook: reads the client converge +// state (CL-8160) instead of a hub-served setup-status route — a hub with +// zero owned tenants means setup-required so the UI routes into the setup +// wizard; any owned tenant means a bench exists and the shell loads +// normally. Read-only on purpose (CL-8112): provisioning itself back onto +// the hub is T6/T7's surface, so this never mints anything — a failed +// read blocks the shell loudly rather than leaving the user silently +// benchless. import { type } from "arktype"; import { reportError } from "@corbits/error-sink"; - -const SetupStatus = type({ - setupRequired: "boolean", - "userCount?": "number", - "tenantCount?": "number", -}); +import { createFetchStockHub, findOwnedTenants } from "./needs-converge"; /** Any credential row this bench actually has stored — the cheap * pre-skip read the home page's first-workbench flow uses to tell "no @@ -84,27 +80,13 @@ export type ProvisionOutcome = export async function triggerFirstLoginProvisioning(): Promise { try { - const response = await fetch("/api/setup/status"); - const body: unknown = await response.json().catch(() => null); - if (!response.ok) { - const envelope = ErrorEnvelope(body); - return envelope instanceof type.errors - ? { kind: "error", message: FALLBACK_ERROR_MESSAGE } - : { - kind: "error", - message: envelope.error.userMessage, - refId: envelope.error.refId, - }; - } - const parsed = SetupStatus(body); - if (parsed instanceof type.errors) { - return { kind: "error", message: FALLBACK_ERROR_MESSAGE }; - } - return parsed.setupRequired + const owned = await findOwnedTenants(createFetchStockHub()); + return owned.length === 0 ? { kind: "needs-onboarding" } : { kind: "existing-member" }; - } catch { - return { kind: "error", message: FALLBACK_ERROR_MESSAGE }; + } catch (cause) { + const refId = reportError(cause, { operation: "first_login_provisioning" }); + return { kind: "error", message: FALLBACK_ERROR_MESSAGE, refId }; } } diff --git a/apps/web/test/onboarding.test.tsx b/apps/web/test/onboarding.test.tsx deleted file mode 100644 index 104bc10e2a..0000000000 --- a/apps/web/test/onboarding.test.tsx +++ /dev/null @@ -1,448 +0,0 @@ -// The setup gate (CL-8112): the first-login hook is a read-only status -// read now — `triggerFirstLoginProvisioning` GETs the hub's native -// `/api/setup/status` and reports only a routing verdict -// (`existing-member` / `needs-onboarding` / `error`), never minting -// anything. The screen at the onboarding path is a thin gate over that -// verdict: an empty hub renders a static pending panel, a set-up hub -// bounces into the shell, and a broken status read blocks with retry. -// Nothing here may touch `/api/onboarding/*` — those routes went with -// the hub mount, and the setup flow itself (T6/T7) has yet to be built. - -import { ThemeProvider } from "@corbits/react-ui"; -import { afterEach, describe, expect, test } from "bun:test"; -import { act } from "react"; -import type { ReactElement } from "react"; -import { createRoot } from "react-dom/client"; -import type { Root } from "react-dom/client"; -import { renderToStaticMarkup } from "react-dom/server"; - -import { App } from "../src/app"; -import { - fetchAgentReadiness, - hasActiveCredential, - triggerFirstLoginProvisioning, -} from "../src/onboarding"; -import { ONBOARDING_PATH } from "../src/routes"; -import type { SessionState } from "../src/session"; - -const realFetch = globalThis.fetch; - -afterEach(() => { - globalThis.fetch = realFetch; -}); - -const json = (body: unknown, status = 200) => - new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json" }, - }); - -const noop = () => undefined; - -/** A `navigate` that records every call — the gate hands off to `/` - * with a navigate call, so proving the hand-off means proving the - * call, not scraping ending copy. */ -function trackedNavigate() { - const calls: string[] = []; - return { navigate: (to: string) => calls.push(to), calls }; -} - -const signedIn: SessionState = { - kind: "signed-in", - user: { id: "user_1", name: "Ada", email: "ada@example.com" }, -}; - -describe("triggerFirstLoginProvisioning", () => { - test("a structured error envelope becomes an error outcome, not null", async () => { - globalThis.fetch = (async () => - json( - { - error: { - code: "setup_status_failed", - userMessage: - "Checking your workbench hit a snag. Try again in a moment.", - refId: "abc123", - }, - }, - 500, - )) as unknown as typeof fetch; - - const result = await triggerFirstLoginProvisioning(); - expect(result).toEqual({ - kind: "error", - message: "Checking your workbench hit a snag. Try again in a moment.", - refId: "abc123", - }); - }); - - // CL-6360: a raw network failure (or any body that doesn't match the - // hub's error envelope) must never surface its own text to the user — - // only the fixed consumer sentence. - test("a network failure becomes a consumer-language error outcome, never the raw cause", async () => { - globalThis.fetch = (async () => { - throw new Error("connection refused"); - }) as unknown as typeof fetch; - - const result = await triggerFirstLoginProvisioning(); - expect(result.kind).toBe("error"); - if (result.kind !== "error") throw new Error("unreachable"); - expect(result.message).not.toContain("connection refused"); - expect(result.message).toBe( - "Setting up your workbench hit a snag — we're on it. Try again in a moment.", - ); - }); - - test("an empty hub reports needs-onboarding, never a silent pass-through", async () => { - globalThis.fetch = (async () => - json({ - setupRequired: true, - userCount: 1, - tenantCount: 0, - })) as unknown as typeof fetch; - - const result = await triggerFirstLoginProvisioning(); - expect(result).toEqual({ kind: "needs-onboarding" }); - }); - - test("a hub with tenants reports existing-member", async () => { - globalThis.fetch = (async () => - json({ - setupRequired: false, - userCount: 1, - tenantCount: 1, - })) as unknown as typeof fetch; - - const result = await triggerFirstLoginProvisioning(); - expect(result).toEqual({ kind: "existing-member" }); - }); - - test("an unparseable status body is an error, never a fabricated verdict", async () => { - globalThis.fetch = (async () => - json({ not: "a status" })) as unknown as typeof fetch; - - const result = await triggerFirstLoginProvisioning(); - expect(result.kind).toBe("error"); - }); - - test("reads the native status route with a bare GET — no body, no minting", async () => { - const seen: { url: string; init: RequestInit | undefined }[] = []; - globalThis.fetch = (async (url: string, init: RequestInit) => { - seen.push({ url, init }); - return json({ setupRequired: false, userCount: 1, tenantCount: 1 }); - }) as unknown as typeof fetch; - - await triggerFirstLoginProvisioning(); - expect(seen.map((s) => s.url)).toEqual(["/api/setup/status"]); - expect(seen[0]?.init?.method ?? "GET").toBe("GET"); - expect(seen[0]?.init?.body).toBeUndefined(); - }); -}); - -describe("fetchAgentReadiness", () => { - // CL-8112 T1: the legacy `/api/onboarding/provisioning-status` route is - // gone with the hub mount — a 404/410 must answer `route-gone`, never - // the generic agent error, so no caller can read a deleted route as - // "your agent is broken". - test("a 404 is route-gone, not a generic agent error", async () => { - globalThis.fetch = (async () => - new Response("not found", { status: 404 })) as unknown as typeof fetch; - - expect(await fetchAgentReadiness("tnt_1")).toEqual({ - kind: "route-gone", - }); - }); - - test("a 410 is route-gone, not a generic agent error", async () => { - globalThis.fetch = (async () => - new Response("gone", { status: 410 })) as unknown as typeof fetch; - - expect(await fetchAgentReadiness("tnt_1")).toEqual({ - kind: "route-gone", - }); - }); -}); - -describe("hasActiveCredential", () => { - // CL-6868: a transient credentials read must never coerce to "no key" — - // that path opens paste-a-key as if none exists when one may already be - // connected. Probe failures are a distinct outcome from a confirmed miss. - test("an active credential is reported as active", async () => { - globalThis.fetch = (async () => - json({ - data: [{ id: "cred_1", status: "active" }], - nextCursor: null, - })) as unknown as typeof fetch; - - expect(await hasActiveCredential("ten_1")).toEqual({ kind: "active" }); - }); - - test("an empty credentials page is a confirmed none, not a probe failure", async () => { - globalThis.fetch = (async () => - json({ data: [], nextCursor: null })) as unknown as typeof fetch; - - expect(await hasActiveCredential("ten_1")).toEqual({ kind: "none" }); - }); - - test("a network failure is a probe error, never coerced to none", async () => { - globalThis.fetch = (async () => { - throw new Error("connection refused"); - }) as unknown as typeof fetch; - - expect(await hasActiveCredential("ten_1")).toEqual({ kind: "error" }); - }); - - test("a non-OK credentials response is a probe error, never coerced to none", async () => { - globalThis.fetch = (async () => - json({ error: "boom" }, 500)) as unknown as typeof fetch; - - expect(await hasActiveCredential("ten_1")).toEqual({ kind: "error" }); - }); - - test("an unparseable credentials body is a probe error, never coerced to none", async () => { - globalThis.fetch = (async () => - json({ not: "credentials" })) as unknown as typeof fetch; - - expect(await hasActiveCredential("ten_1")).toEqual({ kind: "error" }); - }); -}); - -describe("App with a provisioning error", () => { - test("blocks the shell with a retry action instead of rendering it", () => { - const markup = renderToStaticMarkup( - , - ); - - expect(markup).toContain("set up your workbench"); - expect(markup).toContain( - "Could not provision a workbench for this account.", - ); - expect(markup).not.toContain("shell-frame"); - }); -}); - -describe("App at the onboarding path", () => { - const renderOnboarding = () => - renderToStaticMarkup( - , - ); - - test("never renders the shell — no rail, no bench dock, nothing", () => { - const markup = renderOnboarding(); - expect(markup).not.toContain("shell-frame"); - expect(markup).not.toContain("shell-bench-dock"); - }); - - test("lands on the checking phase first — the verdict arrives after the status read", () => { - const markup = renderOnboarding(); - expect(markup).toContain("Checking your workbench"); - }); -}); - -describe("App once onboarding is behind you", () => { - test("mounts the shell for an ordinary route", () => { - globalThis.fetch = (async () => - json({ data: [] })) as unknown as typeof fetch; - - const markup = renderToStaticMarkup( - - - , - ); - - expect(markup).toContain("shell-frame"); - }); -}); - -describe("the setup gate", () => { - function renderGateAtOnboarding(statusBody: unknown, status = 200) { - const seen: string[] = []; - globalThis.fetch = (async (url: string) => { - seen.push(url); - // Anything outside the native status read is a failure: the gate - // must never touch the deleted `/api/onboarding/*` routes. - if (url !== "/api/setup/status") { - throw new Error(`unexpected fetch: ${url}`); - } - return json(statusBody, status); - }) as unknown as typeof fetch; - - const container = document.createElement("div"); - document.body.appendChild(container); - const root: Root = createRoot(container); - const { navigate, calls } = trackedNavigate(); - return { container, root, navigate, calls, seen }; - } - - async function settle(root: Root, element: ReactElement) { - act(() => { - root.render(element); - }); - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 10)); - }); - } - - function gateElement(navigate: (to: string) => void) { - return ( - - - - ); - } - - test("an empty hub drives the installer, which mints the primary tenant then asks the operator to connect a provider (CL-8154)", async () => { - // The installer's own flow (CL-8131, extended by CL-8154): the first - // pass finds no owned primary tenant, mints one over stock - // `POST /api/tenants`, then checks whether it already resolves a - // catalog offering. A fresh tenant has none, so the gate renders the - // credential-connect step rather than the old "no myraDeploy - // configured" gap — the whole point of CL-8154 is that this page - // supplies `myraDeploy` itself instead of stopping there. - let ownsPrimary = false; - const container = document.createElement("div"); - document.body.appendChild(container); - const root: Root = createRoot(container); - const { navigate, calls } = trackedNavigate(); - const seen: string[] = []; - globalThis.fetch = (async (url: string, init?: RequestInit) => { - seen.push(url); - const path = url.split("?")[0] ?? url; - if (path === "/api/setup/status") { - return json({ setupRequired: true, userCount: 1, tenantCount: 0 }); - } - if (path === "/api/me/principals") { - return json({ - data: ownsPrimary - ? [ - { - principalId: "prn_user", - tenantId: "tnt_primary", - tenantName: "Ada's Workbench", - tenantSlug: "ada", - kind: "user", - status: "active", - roles: [{ id: "role_owner", name: "owner" }], - }, - ] - : [], - nextCursor: null, - }); - } - if (path === "/api/tenants" && init?.method === "POST") { - ownsPrimary = true; - return json( - { - id: "tnt_primary", - name: "Ada's Workbench", - slug: "ada", - domain: "ada.workbench.localhost", - parentId: null, - }, - 201, - ); - } - if (path === "/api/tenants/tnt_primary") { - return json({ - id: "tnt_primary", - name: "Ada's Workbench", - slug: "ada", - domain: "ada.workbench.localhost", - parentId: null, - }); - } - if (path === "/api/tenants/tnt_primary/principals") { - return json({ data: [], nextCursor: null }); - } - if (path === "/api/tenants/tnt_primary/models") { - return json([]); - } - // Anything outside the mocked stock routes is a failure: the gate - // must never touch the deleted `/api/onboarding/*` routes. - throw new Error(`unexpected fetch: ${url}`); - }) as unknown as typeof fetch; - - try { - await settle(root, gateElement(navigate)); - - expect(container.textContent).toContain("Connect a model provider"); - expect(calls).toEqual([]); - expect(seen).toContain("/api/setup/status"); - expect(seen).toContain("/api/tenants/tnt_primary/models"); - } finally { - act(() => root.unmount()); - container.remove(); - } - }); - - test("a hub with tenants bounces straight into the shell", async () => { - const { container, root, navigate, calls, seen } = renderGateAtOnboarding({ - setupRequired: false, - userCount: 1, - tenantCount: 1, - }); - try { - await settle(root, gateElement(navigate)); - - expect(calls).toEqual(["/"]); - expect(seen).toEqual(["/api/setup/status"]); - } finally { - act(() => root.unmount()); - container.remove(); - } - }); - - test("a broken status read blocks with retry instead of the shell", async () => { - const { container, root, navigate, calls } = renderGateAtOnboarding( - { - error: { - code: "setup_status_failed", - userMessage: - "Checking your workbench hit a snag. Try again in a moment.", - refId: "r1", - }, - }, - 500, - ); - try { - await settle(root, gateElement(navigate)); - - expect(container.textContent).toContain("Couldn't check your workbench"); - expect(container.textContent).toContain("Try again"); - expect(container.textContent).toContain("r1"); - expect(calls).toEqual([]); - } finally { - act(() => root.unmount()); - container.remove(); - } - }); -}); diff --git a/packages/connections/src/seed-catalog.test.ts b/packages/connections/src/seed-catalog.test.ts index 3b9adb9fe5..342b739843 100644 --- a/packages/connections/src/seed-catalog.test.ts +++ b/packages/connections/src/seed-catalog.test.ts @@ -1,15 +1,13 @@ // Tests for the credential + catalog planting now owned by this package -// (CL-7585): `seedCatalog` plants -// one provider's curated catalog idempotently, refuses a placeholder -// credential for OAuth-only providers, and `ensureNoopCatalogOffering` -// plants the synthetic noop-inference offering. The hub HTTP API is an -// in-memory stub below — no network, no database. +// (CL-7585): `seedCatalog` plants one provider's curated catalog +// idempotently and refuses a placeholder credential for OAuth-only +// providers. The hub HTTP API is an in-memory stub below — no network, +// no database. import { describe, expect, test } from "bun:test"; import type { ApiCall } from "@corbits/hub-api-client"; import { CATALOG_SEEDS } from "./catalog-seed-data"; import { - ensureNoopCatalogOffering, inferenceCredentialName, PLACEHOLDER_CATALOG_API_KEY, PlaceholderCredentialError, @@ -233,33 +231,6 @@ describe("seedCatalog", () => { }); }); -describe("ensureNoopCatalogOffering", () => { - test("plants the synthetic noop offering under the placeholder secret", async () => { - const { api, providers, credentials } = stubHub(); - const lines: string[] = []; - const offeringId = await ensureNoopCatalogOffering( - api, - [], - "tnt_1", - "https://hub.invalid", - (line) => lines.push(line), - ); - - expect(offeringId).toStartWith("off_mdl_noop_cprv_noop"); - expect(providers.map((p) => p.name)).toEqual(["noop"]); - expect(credentials.map((c) => c.name)).toEqual(["noop-default"]); - - const second = await ensureNoopCatalogOffering( - api, - [], - "tnt_1", - "https://hub.invalid", - () => undefined, - ); - expect(second).toBe(offeringId); - }); -}); - describe("catalog seed data", () => { test("every curated provider names at least one model", () => { for (const [provider, seed] of Object.entries(CATALOG_SEEDS)) { diff --git a/packages/connections/src/seed-catalog.ts b/packages/connections/src/seed-catalog.ts index 574a717597..81947a2ab8 100644 --- a/packages/connections/src/seed-catalog.ts +++ b/packages/connections/src/seed-catalog.ts @@ -1,9 +1,7 @@ // Credential + catalog planting over the hub HTTP API — idempotent // ensure-by-name helpers (`ensureProvider`, `ensureCredential`, -// `ensureCatalogModel/Provider/Offering`), the curated-catalog planter -// (`seedCatalog`, driven by `./catalog-seed-data.ts`), and the -// noop-inference offering planter tests pin catalog-less workflows -// against (`ensureNoopCatalogOffering`). +// `ensureCatalogModel/Provider/Offering`) and the curated-catalog planter +// (`seedCatalog`, driven by `./catalog-seed-data.ts`). // // Relocated here by CL-7585: this is the credential domain's own connect-time planting, // not bench seeding, so it lives with the connect flows that call it @@ -281,7 +279,7 @@ export async function ensureCredential( return existing.id; } -async function ensureCatalogModel( +export async function ensureCatalogModel( api: ApiCall, cookies: string[], args: { tenantId: string; canonicalName: string }, @@ -331,7 +329,7 @@ async function ensureCatalogModel( return existing.id; } -async function ensureCatalogProvider( +export async function ensureCatalogProvider( api: ApiCall, cookies: string[], args: { @@ -392,7 +390,7 @@ async function ensureCatalogProvider( return existing.id; } -async function ensureCatalogOffering( +export async function ensureCatalogOffering( api: ApiCall, cookies: string[], args: { @@ -823,96 +821,11 @@ export async function seedCatalog( }; } -// The provider/model pair `noop-inference` (packages/chat/src/noop-inference.ts) -// answers for any request, regardless of what is actually sent — the -// route ignores its body and `x-api-key` entirely. Naming a distinct -// pair here (rather than reusing the tenant's real model id) keeps a -// noop-pinned deployment visually distinct from a real one in the hub's -// UI and logs. -const NOOP_PROVIDER = "anthropic"; -const NOOP_MODEL = "noop"; -// The catalog provider/credential name a noop-pinned deployment's own -// offering is planted under — distinct from any real curated provider -// name (`CATALOG_SEEDS`'s keys), so `ensureNoopCatalogOffering` never -// collides with a tenant's real catalog and is trivially recognizable -// in the hub's own catalog UI. -const NOOP_CATALOG_PROVIDER_NAME = "noop"; -// Priority is meaningless for this offering — it is never included in -// a real workflow's `sourceOfferingIds` (`ensureNoopCatalogOffering`'s -// id is threaded through explicitly, never discovered by sorting), so -// any fixed value is honest here. -const NOOP_OFFERING_PRIORITY = 0; - -/** - * Plants (or finds) the dedicated catalog offering a noop-pinned - * workflow deploys against: a catalog model named `NOOP_MODEL`, a - * provider pointed at the hub's own `noop-inference` endpoint, and a - * placeholder credential — the same `ensureCatalogModel` / - * `ensureProvider` / `ensureCredential` / `ensureCatalogProvider` / - * `ensureCatalogOffering` sequence `seedCatalog` runs for a real - * provider, run here for this one synthetic one. Idempotent, same as - * every other seed step: a re-run finds the existing rows by name and - * reuses them. - */ -export async function ensureNoopCatalogOffering( - api: ApiCall, - cookies: string[], - tenantId: string, - hubUrl: string, - log: (line: string) => void, -): Promise { - const baseURL = `${hubUrl}/api/chat/noop-inference`; - const modelId = await ensureCatalogModel( - api, - cookies, - { tenantId, canonicalName: NOOP_MODEL }, - log, - ); - const providerId = await ensureProvider( - api, - cookies, - { - tenantId, - name: NOOP_CATALOG_PROVIDER_NAME, - plugin: NOOP_PROVIDER, - apiBaseUrl: baseURL, - }, - log, - ); - const credentialId = await ensureCredential( - api, - cookies, - { - tenantId, - providerId, - name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), - secret: PLACEHOLDER_CATALOG_API_KEY, - type: "api_key", - }, - log, - ); - const catalogProviderId = await ensureCatalogProvider( - api, - cookies, - { - tenantId, - name: NOOP_CATALOG_PROVIDER_NAME, - plugin: NOOP_PROVIDER, - baseURL, - credentialId, - }, - log, - ); - return ensureCatalogOffering( - api, - cookies, - { - tenantId, - modelId, - providerId: catalogProviderId, - priority: NOOP_OFFERING_PRIORITY, - capabilities: [], - }, - log, - ); -} +// The noop-inference offering planter (`ensureNoopCatalogOffering`) that +// used to live here moved to `scripts/e2e/noop-inference-server.ts` +// (CL-8160): the hub no longer mounts a noop-inference route at all, so +// planting a catalog offering against one is exclusively an e2e-suite +// concern now, not something this connect-time planting module should +// know about. That file reuses `ensureCatalogModel`, `ensureProvider`, +// `ensureCredential`, `ensureCatalogProvider`, and `ensureCatalogOffering` +// exported from here against its own tiny local noop server. diff --git a/packages/granola-tools/test/credential-wiring-e2e.drizzle.test.ts b/packages/granola-tools/test/credential-wiring-e2e.drizzle.test.ts index 814ab3482c..6f6c905b51 100644 --- a/packages/granola-tools/test/credential-wiring-e2e.drizzle.test.ts +++ b/packages/granola-tools/test/credential-wiring-e2e.drizzle.test.ts @@ -27,7 +27,7 @@ // LLM inference cycle (`agent.send()` deciding to call the tool). No // scripted/deterministic tool-call inference adapter exists anywhere in // this repo today (`@intx/agent` ships no test fixtures for one, -// and the hub's `noop-inference` route emits an empty delta that never +// and the e2e suites' local noop inference server (`scripts/e2e/noop-inference-server.ts`) emits an empty delta that never // triggers a tool call) -- a genuine testing-infrastructure gap, not // something this test works around. This test instead drives the tool // bundle directly with a `ToolCall`, exactly as `../src/tool.test.ts` diff --git a/packages/onboarding/src/tenant-seed.ts b/packages/onboarding/src/tenant-seed.ts index ecf825399b..d1b97296c6 100644 --- a/packages/onboarding/src/tenant-seed.ts +++ b/packages/onboarding/src/tenant-seed.ts @@ -3,10 +3,10 @@ // the seed grants, skill planting, and // `seedTenant`, the grants-plus-workflows installer the first-login // provisioning hook drives. The catalog half (`seedCatalog`, -// `ensureCredential`, `ensureProvider`, `ensureNoopCatalogOffering`, -// `CATALOG_SEEDS`) lives at `@corbits/connections/seed-catalog`, and -// the git-push transport (`WorkflowPusher`, `createGitWorkflowPusher`) -// at `@corbits/connections/workflow-push`. +// `ensureCredential`, `ensureProvider`, `CATALOG_SEEDS`) lives at +// `@corbits/connections/seed-catalog`, and the git-push transport +// (`WorkflowPusher`, `createGitWorkflowPusher`) at +// `@corbits/connections/workflow-push`. // // Dropped in the move, on purpose: `CATALOG_TEST_WORKFLOWS` (the // heartbeat platform-exercise entry) and `EXCLUDED_WORKFLOW_SOURCES` @@ -104,7 +104,6 @@ import { type ApiCall, } from "@corbits/hub-api-client"; import { DEFAULT_SKILLS } from "@corbits/connections/default-skills"; -import { ensureNoopCatalogOffering } from "@corbits/connections/seed-catalog"; import type { WorkflowPusher } from "@corbits/connections/workflow-push"; const GIT_TOKEN_TTL_MS = 10 * 60 * 1000; @@ -158,10 +157,9 @@ const WorkflowRunTriggerResponse = type({ * A provider/model pair a deployed workflow's rendered definition names * in its inference preferences (`DefaultWorkflow.buildJson`'s second * argument). What a deployment actually resolves inference against is - * the tenant's catalog offerings, resolved separately by `ensureDeployment` - * (or, for a noop-pinned workflow, `ensureNoopCatalogOffering`) — this - * type carries no `baseURL`/`apiKey` because `seedTenant` never needs - * either (CL-7461). + * the tenant's catalog offerings, resolved separately by + * `resolveRealSourceOfferingIds` — this type carries no `baseURL`/`apiKey` + * because `seedTenant` never needs either (CL-7461). */ export type ModelSource = { readonly provider: string; @@ -202,17 +200,6 @@ export type DefaultWorkflow = { tenantDomain: string, inferencePreferences: readonly InferencePreference[], ) => string; - /** - * Overrides the deploy's inference source for this workflow only. - * Present on a workflow that must stay free to run continuously: it - * names a synthetic model source instead of the tenant's real catalog - * model, and `seedTenant` deploys it against a dedicated noop catalog - * offering (`ensureNoopCatalogOffering`) rather than the tenant's own - * resolved offerings. Absent on every conversational workflow and on - * the seeded workbench-digest automation, which deploy against the - * tenant's real model. - */ - modelSource?: () => ModelSource; /** * When true, PUT the authored definition to `stopped` after deploy so * a native ScheduleTrigger does not fire every tenant at the next @@ -1021,16 +1008,12 @@ async function listDiscoveredModelOfferings( } /** - * Resolves the catalog offering ids a REAL (non-noop-pinned) workflow - * deploys against: every offering visible to the tenant (owned or - * inherited from an ancestor), ordered by priority ascending, with the - * lowest-priority offering as the default — - * the SAME rule `apps/hub/src/index.ts`'s `workflowDeployer.deploy` and - * `packages/chat/src/platform-adapter.ts`'s `catalogOfferings` apply - * in-process via `listVisibleOfferings`. `excludeOfferingId` drops the - * noop offering (`ensureNoopCatalogOffering`) from this list when one - * has already been planted on the same tenant this run — it must never - * win a real workflow's default just because it happens to sort first. + * Resolves the catalog offering ids a workflow deploys against: every + * offering visible to the tenant (owned or inherited from an ancestor), + * ordered by priority ascending, with the lowest-priority offering as + * the default — the SAME rule `apps/hub/src/index.ts`'s + * `workflowDeployer.deploy` and `packages/chat/src/platform-adapter.ts`'s + * `catalogOfferings` apply in-process via `listVisibleOfferings`. * * Throws when nothing is left to deploy against: "seeded but not * launchable" (`seedCatalog` with no credential) is a valid catalog @@ -1042,14 +1025,13 @@ async function resolveRealSourceOfferingIds( api: ApiCall, cookies: string[], tenantId: string, - excludeOfferingId: string | undefined, ): Promise<{ sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string; }> { - const offerings = (await listDiscoveredModelOfferings(api, cookies, tenantId)) - .filter((offering) => offering.id !== excludeOfferingId) - .sort((a, b) => a.priority - b.priority); + const offerings = ( + await listDiscoveredModelOfferings(api, cookies, tenantId) + ).sort((a, b) => a.priority - b.priority); const defaultSourceOfferingId = offerings[0]?.id; if (defaultSourceOfferingId === undefined) { throw new HubApiError( @@ -1215,38 +1197,22 @@ export async function seedTenant(args: SeedTenantArgs): Promise { // Resolved lazily and cached across the loop below: most seed runs // deploy several workflows against the same tenant catalog state, so - // this fetches the tenant's real offerings (or plants the noop one) - // at most once each, however many workflows ask for them. - let noopOfferingId: string | undefined; + // this fetches the tenant's real offerings at most once, however many + // workflows ask for them. let realOfferings: | { sourceOfferingIds: readonly string[]; defaultSourceOfferingId: string } | undefined; let confirmed = 0; for (const workflow of workflows) { - const workflowModel = workflow.modelSource?.() ?? model; + const workflowModel = model; - let sourceOfferingIds: readonly string[]; - let defaultSourceOfferingId: string; - if (workflow.modelSource !== undefined) { - noopOfferingId ??= await ensureNoopCatalogOffering( - api, - cookies, - tenant.tenantId, - hubUrl, - log, - ); - sourceOfferingIds = [noopOfferingId]; - defaultSourceOfferingId = noopOfferingId; - } else { - realOfferings ??= await resolveRealSourceOfferingIds( - api, - cookies, - tenant.tenantId, - noopOfferingId, - ); - ({ sourceOfferingIds, defaultSourceOfferingId } = realOfferings); - } + realOfferings ??= await resolveRealSourceOfferingIds( + api, + cookies, + tenant.tenantId, + ); + const { sourceOfferingIds, defaultSourceOfferingId } = realOfferings; const assetId = await ensureWorkflowAsset( api, diff --git a/scripts/checks/report-error-baseline.txt b/scripts/checks/report-error-baseline.txt index 03fa8b17c9..7d9ec09ff1 100644 --- a/scripts/checks/report-error-baseline.txt +++ b/scripts/checks/report-error-baseline.txt @@ -65,7 +65,6 @@ apps/web/src/connect-service-actions.ts 2 const message = apps/web/src/last-workbench.ts 1 // A private-browsing tab with storage disabled loses this signal — apps/web/src/last-workbench.ts 1 return null; apps/web/src/onboarding.ts 1 return { kind: "error" }; -apps/web/src/onboarding.ts 1 return { kind: "error", message: FALLBACK_ERROR_MESSAGE }; apps/web/src/pages/agent-detail-page.tsx 1 // A handle collision is the one failure retrying can never clear, so apps/web/src/pages/agent-detail-page.tsx 1 setLifecycle({ apps/web/src/pages/agent-detail-page.tsx 1 setSave({ kind: "idle" }); diff --git a/scripts/e2e/harness.ts b/scripts/e2e/harness.ts index 08fd5026cb..e43eb16c0f 100644 --- a/scripts/e2e/harness.ts +++ b/scripts/e2e/harness.ts @@ -428,19 +428,20 @@ export function expectStatus( /** * Real inference provider hosts. This is not a repo-wide invariant — * each e2e suite chooses per file whether it stays zero-network: a - * suite that pins every inference source at the hub's own - * noop-inference endpoint or an unreachable placeholder host imports - * this guard and calls it at every baseURL/apiKey it constructs, so an - * accidental live-provider reference fails immediately instead of - * silently attempting a real call. A suite that deliberately does dial - * a real provider host (`chat.test.ts`'s echo-invite test, and - * `local-rip.test.ts`'s task leg — see each file's own header comment - * for why) skips this guard on purpose and says so inline, rather than - * importing it and then routing around it. `startHub` only forwards an - * explicit env allowlist (see `osEnv`), so a real ANTHROPIC_API_KEY - * sitting in a developer's shell never reaches the spawned hub either - * way; this guard is the second, explicit line of defense for the - * suites that opt into it. + * suite that pins every inference source at its own local noop + * inference server (`./noop-inference-server.ts`; the hub itself mounts + * no such route as of CL-8160) or an unreachable placeholder host + * imports this guard and calls it at every baseURL/apiKey it + * constructs, so an accidental live-provider reference fails + * immediately instead of silently attempting a real call. A suite that + * deliberately does dial a real provider host (`chat.test.ts`'s + * echo-invite test, and `local-rip.test.ts`'s task leg — see each + * file's own header comment for why) skips this guard on purpose and + * says so inline, rather than importing it and then routing around it. + * `startHub` only forwards an explicit env allowlist (see `osEnv`), so + * a real ANTHROPIC_API_KEY sitting in a developer's shell never reaches + * the spawned hub either way; this guard is the second, explicit line + * of defense for the suites that opt into it. */ const REAL_PROVIDER_HOSTS = [ "api.anthropic.com", @@ -464,9 +465,9 @@ export function assertNeverRealProvider(value: string, what: string): void { if (hit !== undefined) { throw new Error( `${what} references a real inference provider host ("${hit}"); ` + - "the e2e suite must never reach a live provider — use the hub's " + - "own noop-inference endpoint or an unreachable placeholder host " + - "instead.", + "the e2e suite must never reach a live provider — use a local " + + "noop-inference-server.ts server or an unreachable placeholder " + + "host instead.", ); } } diff --git a/scripts/e2e/heartbeat.test.ts b/scripts/e2e/heartbeat.test.ts index 2f3ac524d2..3bcc30b824 100644 --- a/scripts/e2e/heartbeat.test.ts +++ b/scripts/e2e/heartbeat.test.ts @@ -19,7 +19,10 @@ import { buildHeartbeatWorkflow, serializeHeartbeatWorkflow, } from "../../workflows/heartbeat/src/index.ts"; -import { ensureNoopCatalogOffering } from "../../packages/connections/src/seed-catalog.ts"; +import { + ensureNoopCatalogOffering, + startNoopInferenceServer, +} from "./noop-inference-server.ts"; import { api, createCleanupHarness, @@ -176,19 +179,27 @@ describe.skipIf(databaseUrl === undefined)("heartbeat workflow", () => { }, ); - // The deploy's source is the hub's own, really-reachable - // noop-inference endpoint — not a placeholder like the walking - // skeleton's `https://inference.invalid`. That distinction is the - // whole point of this suite: a run started against this source - // actually completes an inference call, at zero cost, because - // noop-inference answers it locally without reaching a real model. + // The deploy's source is this suite's own, really-reachable noop + // inference server — not a placeholder like the walking skeleton's + // `https://inference.invalid`, and never the hub (CL-8160 dropped + // its noop-inference mount). That distinction is the whole point of + // this suite: a run started against this source actually completes + // an inference call, at zero cost, because the noop server answers + // it locally without reaching a real model. + const noopServer = startNoopInferenceServer(); + track({ + label: "noop-inference-server", + output: () => "", + exited: () => false, + stop: async () => noopServer.stop(), + }); const offeringId = await hop("noop catalog seeding", () => ensureNoopCatalogOffering( (method, path, body, cookies) => api(hub.baseUrl, method, path, body, cookies), user.cookies, tenantId, - hub.baseUrl, + noopServer.baseUrl, () => {}, ), ); diff --git a/scripts/e2e/noop-inference-server.ts b/scripts/e2e/noop-inference-server.ts new file mode 100644 index 0000000000..84574a57e2 --- /dev/null +++ b/scripts/e2e/noop-inference-server.ts @@ -0,0 +1,201 @@ +// A standalone stand-in for a real Anthropic endpoint, moved here from +// the hub (`apps/hub/src/e2e/noop-inference.ts`, CL-8160): a scheduled +// step that needs no model must not call one, so the hub no longer +// mounts this route at all. An e2e suite that wants a deployment costing +// zero real inference now starts this tiny server itself +// (`startNoopInferenceServer`) and plants a catalog offering pointed at +// its own `baseUrl` (`ensureNoopCatalogOffering`), never at the hub. +// +// The wire shape matches exactly what +// `@intx/inference/src/providers/anthropic.ts`'s `parseResponse` accepts +// (see that file's `AnthropicSSEEvent` union, ~line 490-560): +// `message_start` (with zeroed usage), one `content_block_start` at +// index 0, one `content_block_delta` carrying a `text_delta`, +// `content_block_stop`, `message_delta` (with a nonzero `output_tokens` +// so downstream usage accounting sees a real number), then +// `message_stop`. `@intx/inference`'s `parseSSE` needs no `[DONE]` +// sentinel — the stream simply ends. +// +// The delta's text is deliberately empty, not merely short: an empty +// reply is exactly the posture a real model obeying "never reply" would +// earn by producing no text, so nothing downstream mistakes the +// constant for a real answer. + +import type { ApiCall } from "../../packages/hub-api-client/src/index.ts"; +import { + ensureCatalogModel, + ensureCatalogOffering, + ensureCatalogProvider, + ensureCredential, + ensureProvider, + inferenceCredentialName, + PLACEHOLDER_CATALOG_API_KEY, +} from "../../packages/connections/src/seed-catalog.ts"; + +const NOOP_REPLY_TEXT = ""; + +function sseLine(event: Record): string { + return `data: ${JSON.stringify(event)}\n\n`; +} + +function noopInferenceResponse(): Response { + const body = new ReadableStream({ + start(controller) { + const encoder = new TextEncoder(); + const events: Record[] = [ + { + type: "message_start", + message: { + usage: { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { type: "text" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: NOOP_REPLY_TEXT }, + }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", usage: { output_tokens: 1 } }, + { type: "message_stop" }, + ]; + for (const event of events) { + controller.enqueue(encoder.encode(sseLine(event))); + } + controller.close(); + }, + }); + return new Response(body, { + headers: { "content-type": "text/event-stream" }, + }); +} + +export type NoopInferenceServer = { + readonly baseUrl: string; + stop(): void; +}; + +/** + * Starts a standalone HTTP server answering `POST /v1/messages` with the + * constant noop SSE stream above, on an OS-assigned free port. No auth + * check is needed here specifically because the handler takes no + * action, reads no state, and returns a constant: there is nothing an + * attacker could use this endpoint to do, see, or change regardless of + * what credential (if any) they present. Callers must `stop()` it in + * their own test cleanup — nothing here tracks it for them. + */ +export function startNoopInferenceServer(): NoopInferenceServer { + const server = Bun.serve({ + port: 0, + fetch(request) { + const url = new URL(request.url); + if (request.method === "POST" && url.pathname === "/v1/messages") { + return noopInferenceResponse(); + } + return new Response("not found", { status: 404 }); + }, + }); + return { + baseUrl: `http://127.0.0.1:${server.port}`, + stop: () => server.stop(true), + }; +} + +const NOOP_PROVIDER = "anthropic"; +const NOOP_MODEL = "noop"; +// The catalog provider/credential name a noop-pinned deployment's own +// offering is planted under — distinct from any real curated provider +// name (`CATALOG_SEEDS`'s keys), so this never collides with a tenant's +// real catalog and is trivially recognizable in the hub's own catalog UI. +const NOOP_CATALOG_PROVIDER_NAME = "noop"; +// Priority is meaningless for this offering — it is never included in a +// real workflow's `sourceOfferingIds` (this offering's id is threaded +// through explicitly, never discovered by sorting), so any fixed value +// is honest here. +const NOOP_OFFERING_PRIORITY = 0; + +/** + * Plants (or finds) the dedicated catalog offering an e2e suite deploys + * a zero-cost workflow against: a catalog model named `NOOP_MODEL`, a + * provider pointed at this file's own `startNoopInferenceServer` + * `baseUrl` (never the hub — CL-8160 dropped the hub's own mount), and a + * placeholder credential — the same `ensureCatalogModel` / `ensureProvider` + * / `ensureCredential` / `ensureCatalogProvider` / `ensureCatalogOffering` + * sequence `seedCatalog` runs for a real provider, run here for this one + * synthetic one. Idempotent, same as every other seed step: a re-run + * finds the existing rows by name and reuses them. + */ +export async function ensureNoopCatalogOffering( + api: ApiCall, + cookies: string[], + tenantId: string, + noopServerBaseUrl: string, + log: (line: string) => void, +): Promise { + // Matches the hub-mounted route's old convention: `baseURL` is the + // endpoint's base, and the inference client itself appends + // `/v1/messages` (`buildRequest` posts to `{baseURL}/v1/messages`). + const baseURL = noopServerBaseUrl; + const modelId = await ensureCatalogModel( + api, + cookies, + { tenantId, canonicalName: NOOP_MODEL }, + log, + ); + const providerId = await ensureProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + apiBaseUrl: baseURL, + }, + log, + ); + const credentialId = await ensureCredential( + api, + cookies, + { + tenantId, + providerId, + name: inferenceCredentialName(NOOP_CATALOG_PROVIDER_NAME), + secret: PLACEHOLDER_CATALOG_API_KEY, + type: "api_key", + }, + log, + ); + const catalogProviderId = await ensureCatalogProvider( + api, + cookies, + { + tenantId, + name: NOOP_CATALOG_PROVIDER_NAME, + plugin: NOOP_PROVIDER, + baseURL, + credentialId, + }, + log, + ); + return ensureCatalogOffering( + api, + cookies, + { + tenantId, + modelId, + providerId: catalogProviderId, + priority: NOOP_OFFERING_PRIORITY, + capabilities: [], + }, + log, + ); +} diff --git a/scripts/e2e/smoke-webhook.test.ts b/scripts/e2e/smoke-webhook.test.ts index 4f8dc8c840..4048ba6e4b 100644 --- a/scripts/e2e/smoke-webhook.test.ts +++ b/scripts/e2e/smoke-webhook.test.ts @@ -2,9 +2,9 @@ // a `@corbits/webhook-triggers` row instead of a cadence; a correctly // HMAC-signed payload delivered to the public ingress route launches a // run. Zero-cost like `scripts/e2e/workbench-digest.test.ts`: the -// deployed workflow's only inference source is the hub's own -// noop-inference endpoint, so nothing here ever calls a real model -// provider. +// deployed workflow's only inference source is this suite's own local +// noop inference server (`./noop-inference-server.ts`), so nothing here +// ever calls a real model provider. // // A webhook-fired run launches a standalone folded run via // `@corbits/folded-runs` (see `packages/webhook-triggers/src/launch.ts`). @@ -27,7 +27,10 @@ import { buildHeartbeatWorkflow, serializeHeartbeatWorkflow, } from "../../workflows/heartbeat/src/index.ts"; -import { ensureNoopCatalogOffering } from "../../packages/connections/src/seed-catalog.ts"; +import { + ensureNoopCatalogOffering, + startNoopInferenceServer, +} from "./noop-inference-server.ts"; import { publishCorbitsToolsRegistry } from "../../packages/tool-registry-publish/src/publish.ts"; import { createHubAPI } from "../../packages/hub-api-client/src/index.ts"; import { @@ -116,15 +119,23 @@ describe.skipIf(databaseUrl === undefined)("smoke: webhook trigger", () => { }); // The zero-cost catalog chain: an anthropic-plugin provider whose - // base URL is the hub's own noop-inference endpoint, never a real + // base URL is this suite's own noop inference server, never the + // hub (CL-8160 dropped its noop-inference mount) and never a real // model. + const noopServer = startNoopInferenceServer(); + track({ + label: "noop-inference-server", + output: () => "", + exited: () => false, + stop: async () => noopServer.stop(), + }); const offeringId = await hop("noop catalog seeding", () => ensureNoopCatalogOffering( (method, path, body, cookies2) => api(hub.baseUrl, method, path, body, cookies2), cookies, tenantId, - hub.baseUrl, + noopServer.baseUrl, () => {}, ), ); diff --git a/scripts/e2e/workbench-digest.test.ts b/scripts/e2e/workbench-digest.test.ts index efc3aa2e78..8f069d93ad 100644 --- a/scripts/e2e/workbench-digest.test.ts +++ b/scripts/e2e/workbench-digest.test.ts @@ -1,15 +1,16 @@ // A light end-to-end smoke test for the workbench-digest workflow: the // real hub and sidecar as spawned processes against a real Postgres, a -// workbench-digest deployment whose inference source is the hub's own -// `noop-inference` endpoint (not a placeholder, not a real provider), -// and a trigger that runs the step to completion. +// workbench-digest deployment whose inference source is this suite's +// own local noop inference server (not a placeholder, not a real +// provider, and never the hub — CL-8160 dropped its noop-inference +// mount), and a trigger that runs the step to completion. // // This is the proof-by-construction that workbench-digest costs nothing // to run frequently: the deploy's source is a real, reachable endpoint // (unlike the walking skeleton's `https://inference.invalid` // placeholder), so a run started against it actually resolves its -// inference call — against `noop-inference`'s constant, locally -// served reply, never a real model. +// inference call — against the noop server's constant, locally served +// reply, never a real model. import { describe, test } from "bun:test"; @@ -19,7 +20,10 @@ import { buildWorkbenchDigestWorkflow, serializeWorkbenchDigestWorkflow, } from "../../workflows/workbench-digest/src/index.ts"; -import { ensureNoopCatalogOffering } from "../../packages/connections/src/seed-catalog.ts"; +import { + ensureNoopCatalogOffering, + startNoopInferenceServer, +} from "./noop-inference-server.ts"; import { api, createCleanupHarness, @@ -180,19 +184,27 @@ describe.skipIf(databaseUrl === undefined)("workbench-digest workflow", () => { }, ); - // The deploy's source is the hub's own, really-reachable - // noop-inference endpoint — not a placeholder like the walking - // skeleton's `https://inference.invalid`. That distinction is the - // whole point of this suite: a run started against this source - // actually completes an inference call, at zero cost, because - // noop-inference answers it locally without reaching a real model. + // The deploy's source is this suite's own, really-reachable noop + // inference server — not a placeholder like the walking skeleton's + // `https://inference.invalid`, and never the hub (CL-8160 dropped + // its noop-inference mount). That distinction is the whole point of + // this suite: a run started against this source actually completes + // an inference call, at zero cost, because the noop server answers + // it locally without reaching a real model. + const noopServer = startNoopInferenceServer(); + track({ + label: "noop-inference-server", + output: () => "", + exited: () => false, + stop: async () => noopServer.stop(), + }); const offeringId = await hop("noop catalog seeding", () => ensureNoopCatalogOffering( (method, path, body, cookies) => api(hub.baseUrl, method, path, body, cookies), user.cookies, tenantId, - hub.baseUrl, + noopServer.baseUrl, () => {}, ), ); diff --git a/workflows/heartbeat/src/index.ts b/workflows/heartbeat/src/index.ts index b946b47a3c..836bcfe036 100644 --- a/workflows/heartbeat/src/index.ts +++ b/workflows/heartbeat/src/index.ts @@ -8,15 +8,21 @@ // completion) is the "timestamp result" this workflow exists to // produce, not anything the agent says. // -// Zero inference cost: the DSL has no agent-free step primitive that -// runs on the deployed host today (the `action` primitive exists in -// `@intx/workflow`, but no shipped host wires an `invokeAction` -// callback to resolve it — see VENDORED.md-adjacent research notes in -// this package's README). A deployer therefore pins this definition's -// `inferencePreferences` to the hub's `noop-inference` endpoint (see -// `packages/chat/src/noop-inference.ts`): the turn -// completes instantly against a constant, never reaching a real model -// provider, so running this workflow every few seconds costs nothing. +// No agent-free step primitive runs on the deployed host today (the +// `action` primitive exists in `@intx/workflow`, but no shipped host +// wires an `invokeAction` callback to resolve it — see +// VENDORED.md-adjacent research notes in this package's README), so +// this step is unavoidably an agent step. CL-8160 stopped pinning a +// deployer-supplied synthetic noop `ModelSource` here: a scheduled step +// that needs no model must not call one, but with no non-agent step +// shape available the least-bad option is to deploy against the +// tenant's own real, resolved catalog offering like every other +// workflow (`resolveRealSourceOfferingIds` in +// `packages/onboarding/src/tenant-seed.ts`) rather than inventing a +// fake provider to route around a step that will still make a real +// call. This definition still accepts `inferencePreferences` as plain +// deploy-time data; it is the deployer's choice, never this +// definition's, what those preferences resolve against. // // This package is installable data. It imports only published platform // packages, and nothing imports it statically: a host publishes the diff --git a/workflows/workbench-digest/src/index.ts b/workflows/workbench-digest/src/index.ts index 4c447345ff..a734423b54 100644 --- a/workflows/workbench-digest/src/index.ts +++ b/workflows/workbench-digest/src/index.ts @@ -9,17 +9,15 @@ // relay an already-deterministic digest line back out verbatim, so // nothing about the reply's content is left to the model. // -// Zero inference cost: like `@corbits/heartbeat-workflow`, this -// definition is deployed with its `inferencePreferences` pinned to the -// hub's `noop-inference` endpoint (see -// `packages/chat/src/noop-inference.ts`). Under that -// pin the turn completes instantly against a constant, empty reply — -// by design, `noop-inference` never produces real text (see that -// file's header comment) — so this deployment proves the scheduling -// and workbench-mail-posting paths stay alive at zero cost, without -// posting visible digest text. Pin `inferencePreferences` at a real -// catalog model instead to get an actual, human-visible digest line -// posted on every trigger, at that model's ordinary per-turn cost. +// No agent-free step primitive runs on the deployed host today (see +// `@corbits/heartbeat-workflow`'s own header comment for why), so this +// step is unavoidably an agent step. CL-8160 stopped pinning a +// deployer-supplied synthetic noop `ModelSource` here: `seedTenant` +// (`packages/onboarding/src/tenant-seed.ts`) now deploys this workflow +// against the tenant's own real, resolved catalog offering like every +// other workflow (`resolveRealSourceOfferingIds`), so a trigger posts +// an actual, human-visible digest line at that model's ordinary +// per-turn cost rather than a constant empty reply. import { defineAgent } from "@intx/agent"; import type { InferencePreference } from "@intx/agent";