From ac70dd84d6378824f15cc4da8729016dfeea35c4 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 15 Sep 2026 14:04:27 -0700 Subject: [PATCH 1/2] feat(onboarding)!: delete pending-seed drain and bench provisioner (CL-7586) Provisioning is the explicit setup-CLI/installer flow plus the desired-state reconcile only: remove onboarding.pending_seed (schema, store, migrations live on as 0001 + 0002_drop), the bench-provisioning background drain, and every caller. Connect and status routes now kick the reconcile fire-and-forget and answer from hub reads. Retire docs/bench-provisioning.md and shrink the report-error baseline to match. --- apps/hub/src/bench-session.ts | 38 +- apps/hub/src/index.ts | 83 ++- apps/hub/src/slack-channel-mint-session.ts | 2 +- apps/hub/src/slack-tag-mount.ts | 2 +- apps/hub/src/tenant-create-onboard.ts | 32 +- apps/web/test/onboarding.test.tsx | 7 +- docs/bench-provisioning.md | 99 ---- docs/local-dev.md | 6 +- docs/model-seeding.md | 5 +- docs/onboarding-huggingface-connect.md | 15 +- docs/onboarding-openrouter-connect.md | 47 +- docs/seed-reconciliation.md | 10 +- .../access-policy/test/store.drizzle.test.ts | 2 +- packages/connections/src/connected-hook.ts | 11 +- packages/connections/src/oauth-routes.ts | 12 +- packages/connections/src/routes.ts | 10 +- packages/onboarding/README.md | 25 +- packages/onboarding/src/bench-provisioning.ts | 389 -------------- .../onboarding/src/complete-credential.ts | 24 +- packages/onboarding/src/desired-state.ts | 4 +- packages/onboarding/src/index.ts | 23 - packages/onboarding/src/migrations.ts | 10 + packages/onboarding/src/pending-seed.test.ts | 311 ------------ packages/onboarding/src/pending-seed.ts | 474 ------------------ packages/onboarding/src/routes.ts | 242 +++++---- packages/onboarding/src/schema.ts | 26 - .../test/bench-provisioning.test.ts | 404 --------------- .../test/complete-credential.test.ts | 15 +- .../test/complete-setup-routes.test.ts | 383 ++++---------- .../test/connect-deploys-nothing.test.ts | 202 ++++---- .../test/desired-state-reconcile.test.ts | 4 +- .../test/huggingface-connect-routes.test.ts | 46 +- packages/onboarding/test/migrations.test.ts | 21 +- .../test/openrouter-connect-routes.test.ts | 42 +- .../test/pending-seed-store.drizzle.test.ts | 275 ---------- .../test/report-error-routing.test.ts | 52 +- packages/onboarding/test/routes.test.ts | 49 +- scripts/checks/no-product-tenancy.ts | 5 - scripts/checks/report-error-baseline.txt | 2 - 39 files changed, 582 insertions(+), 2827 deletions(-) delete mode 100644 docs/bench-provisioning.md delete mode 100644 packages/onboarding/src/bench-provisioning.ts delete mode 100644 packages/onboarding/src/pending-seed.test.ts delete mode 100644 packages/onboarding/src/pending-seed.ts delete mode 100644 packages/onboarding/src/schema.ts delete mode 100644 packages/onboarding/test/bench-provisioning.test.ts delete mode 100644 packages/onboarding/test/pending-seed-store.drizzle.test.ts diff --git a/apps/hub/src/bench-session.ts b/apps/hub/src/bench-session.ts index ec824c43c8..426a0163a9 100644 --- a/apps/hub/src/bench-session.ts +++ b/apps/hub/src/bench-session.ts @@ -1,7 +1,7 @@ // The one thing a background loop cannot inherit: a session. The bench -// provisioner (`@workbench/onboarding`'s `createBenchProvisioner`) runs -// with no request to borrow cookies from, yet everything it drives — -// `seedTenant`'s asset creates, deployments, grants, and the git push +// provisioned by an explicit flow (a desired-state revisit kick, the +// setup CLI) runs with no request to borrow cookies from, yet everything +// it drives — asset creates, deployments, grants, and the git push // underneath them — speaks the hub's own HTTP API as the bench's owner. // This mints that session in-process. // @@ -11,17 +11,27 @@ // parent-org admin does not inherit rights over a child bench — RBAC // resolves grants within a single tenant — so "just use the operator // account" would 403 on every call. The owner's own session is the only -// identity that can provision their bench, which is also the honest +// identity that can act for their bench, which is also the honest // one: the work is theirs, done on their behalf, and it shows up in the // session table as such (tagged by user agent, so these are greppable // and never mistaken for a human sign-in). // // Sessions are cached per user and re-minted well before expiry, so a -// drain tick every few seconds does not write a session row every few +// kick every few seconds does not write a session row every few // seconds. import { makeSignature } from "better-auth/crypto"; -import type { SessionForUser } from "@workbench/onboarding"; + +/** + * Mints the hub session an explicit provisioning flow acts under for + * one user's own bench, or `undefined` when no session can be minted + * (an account since deleted, an auth backend briefly unavailable). + * Returning `undefined` skips that run — it never discards anything. + */ +export type SessionForUser = (args: { + userId: string; + tenantId: string; +}) => Promise; /** Re-mint this far ahead of a cached session's own expiry, so a long * provisioning pass can never have its session expire mid-flight. */ @@ -53,16 +63,16 @@ export type BenchSessionAuth = { }; /** - * Builds the `sessionFor` seam the bench provisioner takes. Returns the - * bare `name=value` cookie pairs `ApiCall` sends, signed exactly the way - * better-auth's own cookie writer signs them — the same HMAC helper the - * library uses, rather than a hand-rolled copy that could drift from the - * verifier. + * Builds the `sessionFor` seam the explicit provisioning flows take. + * Returns the bare `name=value` cookie pairs `ApiCall` sends, signed + * exactly the way better-auth's own cookie writer signs them — the same + * HMAC helper the library uses, rather than a hand-rolled copy that + * could drift from the verifier. * * `undefined` means "no session could be minted right now" (an account - * since deleted, an auth backend briefly unavailable). The provisioner - * treats that as a reason to hold the bench for a later pass, never as a - * reason to discard its pending work. + * since deleted, an auth backend briefly unavailable). The caller + * treats that as a reason to skip the run, never as a reason to + * discard anything. */ export function createBenchSessionMinter(args: { auth: BenchSessionAuth; diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index a47115e48f..1fa74ce8e5 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -281,11 +281,7 @@ import { import { supportedCredentialProviders } from "@corbits/connections/credential-test"; import { CATALOG_WORKFLOWS, createGitWorkflowPusher } from "@corbits/seeding"; import { createHubAPI } from "@corbits/hub-api-client"; -import { - createDrizzlePendingSeedStore, - createBenchProvisioner, - createOnboardingRoutes, -} from "@workbench/onboarding"; +import { createOnboardingRoutes } from "@workbench/onboarding"; import { createConnectionRoutes, isInferenceProvider, @@ -474,13 +470,9 @@ export function createStaticHandler(staticDir: string) { /** * The `CredentialCipher` (see `@intx/types`) every secret-at-rest seam * in this composition root shares — `webhookTriggerStore`'s signing - * secrets, `@workbench/onboarding`'s in-flight OAuth connect state + * secrets and `@workbench/onboarding`'s in-flight OAuth connect state * (the PKCE verifier parked between `/start` and `/callback`, sealed - * into the state itself so it survives a restart between the two), and - * (since CL-6031) the same package's `pending_seed` table — a - * just-connected credential's plaintext key, parked server-side - * between the OAuth callback and the onboarding page's own - * `/complete-setup` follow-up (see `packages/onboarding/src/pending-seed.ts`). + * into the state itself so it survives a restart between the two). * A real key (`CREDENTIAL_ENCRYPTION_KEY`) builds an AES-256-GCM * cipher. An unset key hard-fails boot — a self-hosting operator who * forgets this variable must not silently end up storing those secrets @@ -496,9 +488,9 @@ export function credentialCipherFrom( throw new Error( [ "CREDENTIAL_ENCRYPTION_KEY is not set.", - "It encrypts secrets at rest — webhook-trigger signing secrets,", - "onboarding's OAuth PKCE connect state, and its pending-seed", - "table — so the hub refuses to boot without it. Generate one and", + "It encrypts secrets at rest — webhook-trigger signing secrets", + "and onboarding's OAuth PKCE connect state — so the hub refuses", + "to boot without it. Generate one and", "add it to .env:", "", " openssl rand -hex 32", @@ -509,7 +501,7 @@ export function credentialCipherFrom( ].join("\n"), ); } - log.warn`No CREDENTIAL_ENCRYPTION_KEY configured; secrets (e.g. webhook-trigger signing secrets, onboarding OAuth connect state, onboarding's pending-seed table) will NOT be encrypted at rest. ALLOW_PLAINTEXT_SECRETS is set — expected in dev/test only, never for a real deployment.`; + log.warn`No CREDENTIAL_ENCRYPTION_KEY configured; secrets (e.g. webhook-trigger signing secrets, onboarding OAuth connect state) will NOT be encrypted at rest. ALLOW_PLAINTEXT_SECRETS is set — expected in dev/test only, never for a real deployment.`; return createNoopCredentialCipher(); } return createEnvKeyCredentialCipher( @@ -2621,31 +2613,33 @@ export async function createHub(config: HubConfig) { // CL-6568's other half: a tenant whose only provider is one it // connected itself through Settings — never an operator-configured // hub key — must converge on Myra and the default workflow set the - // same way an onboarding-connected one does. `pendingSeedStore` and - // `benchProvisioner` are declared further down this function, but + // same way an onboarding-connected one does. The drain is gone + // (CL-7586), so this kicks the same desired-state reconcile the + // tenant-create observer and the onboarding routes kick, under the + // connecting user's own minted session — `sessionFor` is declared + // further up this function, and `observerRef` further down, but // this closure only runs on a future request, well after both are - // constructed below — the same forward-reference this file already - // relies on for `onboardingDeps`. + // constructed below. Best-effort like every other kick: absent or + // unmintable means convergence waits for the revisit probe. onInferenceCredentialUsable: async (info) => { const provider = supportedCredentialProviders().find( (candidate) => candidate.id === info.provider, )?.id; if (provider === undefined) { - log.error`onInferenceCredentialUsable fired for an unsupported provider ${info.provider} on tenant ${info.tenantId}; skipping the pending-seed row`; + log.error`onInferenceCredentialUsable fired for an unsupported provider ${info.provider} on tenant ${info.tenantId}; skipping the desired-state kick`; return; } - await pendingSeedStore.put({ + const cookies = await sessionFor({ userId: info.userId, tenantId: info.tenantId, - principalId: info.principalId, - tenantDomain: info.tenantDomain, - provider, - apiKey: info.apiKey, - ...(info.baseURLOverride !== undefined - ? { baseURLOverride: info.baseURLOverride } - : {}), }); - benchProvisioner.wake(); + if (cookies === undefined) { + log.error`onInferenceCredentialUsable could not mint a session for user ${info.userId} on tenant ${info.tenantId}; skipping the desired-state kick`; + return; + } + void observerRef.current + ?.kick({ tenantId: info.tenantId, cookies }) + .catch(() => undefined); }, }), ); @@ -3378,29 +3372,17 @@ export async function createHub(config: HubConfig) { // session it serves belongs to no tenant yet. The route is // `@workbench/onboarding`'s; what it decides is documented in that // package's provision.ts. - // Connecting a provider deploys nothing (CL-6457): the onboarding - // routes persist the credential and hand the workflow deploys to this - // drain, which converges every bench with a pending row — including - // one a previous process died halfway through, since the row itself is - // the durable work item. - const pendingSeedStore = createDrizzlePendingSeedStore(db, credentialCipher); - const benchProvisioner = createBenchProvisioner({ - api: selfApi, - hubUrl: config.baseUrl, - store: pendingSeedStore, - pushWorkflow: createGitWorkflowPusher(), - sessionFor, - log: (line) => log.info`${line}`, - logError: (line) => log.error`${line}`, - }); - benchProvisioner.start(); + // Connecting a provider deploys nothing by itself (CL-6457): the + // onboarding routes persist the credential and kick the desired-state + // reconcile below, which converges every bench — including one a + // previous process died halfway through, since the reconcile re-reads + // the pins on every kick rather than consuming a durable work item. + // (CL-7586 removed the pending-seed drain that used to do this.) // CL-7584: the tenant-create trigger. A 201 from the native // `POST /api/tenants` route kicks a fire-and-forget desired-state - // reconcile for the new tenant under the creator's minted session — - // the revisit kick below and the drain above share this one - // reconciler. No durable row: the pending_seed row stays the only - // durable work item, and a kick lost to a restart is re-covered by + // reconcile for the new tenant under the creator's minted session. + // No durable row anywhere: a kick lost to a restart is re-covered by // the revisit kick on the tenant's next visit. The observer itself is // composed just before the guard wrap, after every route mount: Hono // copies routes at `.route()` time, so wrapping earlier would strand @@ -3417,8 +3399,6 @@ export async function createHub(config: HubConfig) { log: (line) => log.info`${line}`, logError: (line) => log.error`${line}`, credentialCipher, - pendingSeedStore, - benchProvisioner, desiredStateKick: (args) => { // Fire-and-forget; the route already decided pins are pending. void observerRef.current @@ -3687,7 +3667,6 @@ export async function createHub(config: HubConfig) { workflowScheduler.stop(); credentialExpirySweep.stop(); inboxUnsnoozeSweep.stop(); - benchProvisioner.stop(); await insightsUsage.close(); await insightsLatency.close(); await preferences.close(); diff --git a/apps/hub/src/slack-channel-mint-session.ts b/apps/hub/src/slack-channel-mint-session.ts index 7b4df3f01d..f607266868 100644 --- a/apps/hub/src/slack-channel-mint-session.ts +++ b/apps/hub/src/slack-channel-mint-session.ts @@ -1,7 +1,7 @@ import { generateId } from "@intx/hub-common"; import { reportError } from "@corbits/error-sink"; import type { WorkbenchTenancyStore } from "@corbits/chat"; -import type { SessionForUser } from "@workbench/onboarding"; +import type { SessionForUser } from "./bench-session"; /** * Slack auto-provisioned identities are members. POST /api/tenants is diff --git a/apps/hub/src/slack-tag-mount.ts b/apps/hub/src/slack-tag-mount.ts index 87a81804a3..d5eb4e21d0 100644 --- a/apps/hub/src/slack-tag-mount.ts +++ b/apps/hub/src/slack-tag-mount.ts @@ -32,7 +32,7 @@ import { type ChatStore, type RoomMessageStore, } from "@corbits/chat"; -import type { SessionForUser } from "@workbench/onboarding"; +import type { SessionForUser } from "./bench-session"; import { createAutoProvisionPrincipalResolver, createDrizzleSlackChannelBindingStore, diff --git a/apps/hub/src/tenant-create-onboard.ts b/apps/hub/src/tenant-create-onboard.ts index bbd2655761..0f9b5f7d27 100644 --- a/apps/hub/src/tenant-create-onboard.ts +++ b/apps/hub/src/tenant-create-onboard.ts @@ -7,12 +7,11 @@ // the kick never touches the DB directly — so a fresh tenant gets Myra // and the core pins without hub boot seeding anything. // -// There is no durable work item here on purpose: the pending_seed row -// stays the only durable queue (a connect's credential), and a -// tenant-create kick that dies with the process is re-covered by the +// There is no durable work item here on purpose: nothing is queued, and +// a tenant-create kick that dies with the process is re-covered by the // revisit kick (`POST /api/onboarding/provision`) on the next visit. -// In-process dedupe by tenantId is the same class of optimization the -// provisioner's in-flight map is: never a fact the system needs correct. +// In-process dedupe by tenantId is purely an optimization against double +// kicks, never a fact the system needs correct. import { Hono } from "hono"; import type { AppEnv } from "@intx/hub-api"; import { cookiesFromHeader, type ApiCall } from "@corbits/hub-api-client"; @@ -35,10 +34,10 @@ export type TenantCreateOnboardDeps = { * The convergence step. Production resolves the tenant's deploy model * from its resolved catalog plus the creator's deploy identity (owner * principal + domain, via `resolveTenantDeployer` — a fresh native - * tenant has no pending-seed row to read those from) and delegates to - * `reconcileTenantDesiredState`; with no offerings it reports the - * workflow pins blocked (logged, never thrown). Tests replace the - * whole thing. + * tenant carries no credential material to read those from) and + * delegates to `reconcileTenantDesiredState`; with no offerings it + * reports the workflow pins blocked (logged, never thrown). Tests + * replace the whole thing. */ reconcileFn?: (args: { tenantId: string; @@ -77,8 +76,8 @@ export function createTenantCreateObserver( wrapped: Hono, ): TenantCreateObserver { const logError = deps.logError ?? deps.log; - // In-process tenantId dedupe, same pattern as the provisioner's - // in-flight map: an optimization against double kicks, never a fact. + // In-process tenantId dedupe against double kicks: an optimization, + // never a fact the system needs correct. const kicked = new Set(); const inFlight = new Set>(); let stopped = false; @@ -99,9 +98,8 @@ export function createTenantCreateObserver( ); if (model === undefined) { // No catalog offerings yet — nothing is launchable. Report the - // workflow pins blocked; the next trigger (a connect's drain - // pass, a revisit probe) sees the pins still pending and - // re-kicks. + // workflow pins blocked; the next trigger (a revisit probe) + // sees the pins still pending and re-kicks. deps.log( `tenant-create onboarding for ${reconcileArgs.tenantId} is blocked: no catalog offerings to deploy against yet`, ); @@ -115,8 +113,8 @@ export function createTenantCreateObserver( })), } satisfies ReconcileReport; } - // A fresh native tenant has no pending-seed row to read deploy - // identity from — resolve the creator's owner principal (minted + // A fresh native tenant carries no credential material to read + // deploy identity from — resolve the creator's owner principal // in the create transaction) plus the tenant domain before // reconciling, so the deploy lands under a real identity. const deployer = await resolveTenantDeployer( @@ -157,7 +155,7 @@ export function createTenantCreateObserver( const operation = runReconcile(args) .catch((cause: unknown) => { logError( - `tenant-create onboarding for ${args.tenantId} failed (the revisit kick or drain will cover it): ${cause instanceof Error ? cause.message : String(cause)}`, + `tenant-create onboarding for ${args.tenantId} failed (the revisit kick will cover it): ${cause instanceof Error ? cause.message : String(cause)}`, ); }) .finally(() => { diff --git a/apps/web/test/onboarding.test.tsx b/apps/web/test/onboarding.test.tsx index ed040429ee..2377a33aa4 100644 --- a/apps/web/test/onboarding.test.tsx +++ b/apps/web/test/onboarding.test.tsx @@ -1042,9 +1042,10 @@ describe("the OpenRouter connect card", () => { }); test("a connected return falls back to the credential phase, not a dead end, when completeSetup reports unseeded", async () => { - // The degraded-but-correct path: the pending-seed cookie can be - // missing (a loser duplicate-callback response the browser never - // applied, a cookie that already expired) even though the + // The degraded-but-correct path: nothing is parked server-side for + // the follow-up (CL-7586 deleted the pending-seed drain — a loser + // duplicate-callback response the browser never applied, or an + // already-expired cookie, leaves no row behind) even though the // credential itself connected fine. `completeSetup` answers // `unseeded` — not an error — and the wizard must land somewhere a // person can actually finish from, never stuck on the spinner or a diff --git a/docs/bench-provisioning.md b/docs/bench-provisioning.md deleted file mode 100644 index 390bfd17d2..0000000000 --- a/docs/bench-provisioning.md +++ /dev/null @@ -1,99 +0,0 @@ -# Bench provisioning - -Connecting a provider deploys nothing. - -That is the whole rule, and it exists because the alternative was -measured: pasting an Anthropic key on onboarding sat on a static -"Connecting…" for over two minutes, because the connect request -deployed five default workflows — around twenty seconds each — before it -answered. Long enough that the owner concluded the app had frozen. - -## The split - -Connecting a credential has a fast half and a slow half, and only the -fast half is allowed on a request. - -- **Fast half** (`testAndPersistCredential`) — persist the credential, - seed its model catalog. Durable, and measured in seconds. This is all - `POST /api/onboarding/complete` does before it answers. -- **Slow half** (`ensureSeeded` → `seedTenant`) — deploy the bench's - default workflows. This runs in the background, in - `createBenchProvisioner` (`packages/onboarding/src/bench-provisioning.ts`). - -No HTTP route runs the slow half. `POST /complete` and `POST -/complete-setup` both answer with where the bench's agents stand and -hand the deploying to the drain; `GET /api/onboarding/provisioning-status` -reports the same thing for anything that needs to poll. - -`completeCredentialSetup` still composes both halves back to back, but -only for harnesses that need a fully deployed bench before a scenario -can begin (`@workbench/evals`' real target). A route that calls it -re-creates the freeze. - -## Why the drain converges - -The drain is not a job queue. It is a pass over -`onboarding.pending_seed`, where the row _is_ the work item and means -"this bench has a credential and does not yet have its agents". Three -properties follow from that framing rather than having to be built: - -- **Idempotent** — every pass re-reads the bench's real asset and - deployment state before acting, and `seedTenant` underneath is - ensure-then-create at every step. A pass over a finished bench deploys - nothing and clears the row. -- **Convergent** — a pass that gets partway leaves the row in place, so - the next pass picks up exactly the workflows still missing. -- **Restart-safe** — nothing about outstanding work lives in the - process. A hub that dies mid-deploy leaves the row; the next boot's - first tick finishes it. In-memory state is only ever a dedupe guard or - a retry backoff, never a fact the system needs to be correct. - -A failing bench is held off with exponential backoff (15s, doubling to a -10-minute ceiling) so a sidecar outage is not hammered, and a bench is -never dropped for failing — the row stays until it converges or its TTL -(24 hours) expires. That backoff bookkeeping is reclaimed once the row -itself is gone — TTL-expiry or otherwise — not only when the bench -converges, so a permanently-failing bench never leaves a stale backoff -behind for a later, unrelated connect to the same user/tenant to -inherit (CL-7233). - -## Sessions - -A background loop has no request to borrow cookies from, and everything -`seedTenant` drives speaks the hub's own HTTP API. The provisioner takes -`sessionFor` as a seam; the hub fills it (`apps/hub/src/bench-session.ts`) -by minting the bench owner's own session in process. - -It has to be the owner's session. The hub resolves a tenant by looking -up a principal for that user, and rights do not flow from a parent org -down to a child bench — an administrator acting on someone else's -personal bench is refused outright. These sessions are tagged with a -`workbench-bench-provisioner` user agent so they are never mistaken for -a human sign-in, and are cached per user rather than minted per tick. - -## What someone sees - -Connecting hands the person forward immediately, and the wait on the -other side is as short as it can honestly be: the setup agent (Myra) -leads `DEFAULT_WORKFLOWS`, so she is deployed before anything else, and -the land hop drops the person into her room the moment she can answer. -The other catalog workflows (digest, recurring-task, research) are not -seeded at all — they are deployable through the catalog instantiate route -(CL-7073) once someone picks one, not on a critical path nobody is waiting -on. - -While she is still coming up, the shared warm loader is the whole -screen: one honest headline and a rotating tip, never an internal count. -How many workflows a bench seeds is an implementation detail, and "0 of -5 ready" told a waiting person nothing they could act on. A wait that -runs past a sensible interval says so plainly and offers another go, -rather than showing an unchanging number. - -Once the person has landed, anything still deploying is at most a single -dismissible line — never a screen that blocks them. - -Whether a bench can be started is asked of the bench, never read off an -error message: `GET /api/onboarding/provisioning-status` answers with -`setupAgentReady`, which is the only field a waiting surface should -branch on. `deployed`/`pending` stay in the response for operators and -logs. diff --git a/docs/local-dev.md b/docs/local-dev.md index 62a3065feb..5c0b43143c 100644 --- a/docs/local-dev.md +++ b/docs/local-dev.md @@ -75,9 +75,9 @@ Convergence has three triggers, all driving the same reconciler: 1. A tenant-create observation: a 201 from `POST /api/tenants` fires a fire-and-forget reconcile under the creator's minted session (`apps/hub/src/tenant-create-onboard.ts`). -2. The pending-seed drain: `runOnce` delegates to the same reconcile; - the `pending_seed` row stays the only durable work item (ready - clears it, blocked keeps it, failed keeps it and backs off). +2. The credential-connect kick: persisting a credential (onboarding's + `/complete`, or a provider connected through Settings) fires a + fire-and-forget reconcile for that tenant. 3. The revisit kick: `POST /api/onboarding/provision` fires a kick when the caller's tenant still has pending pins — how a joined member's bench converges. diff --git a/docs/model-seeding.md b/docs/model-seeding.md index 9a0579932a..22b2267985 100644 --- a/docs/model-seeding.md +++ b/docs/model-seeding.md @@ -45,8 +45,9 @@ API — never discovered at runtime: that same entry's first model (`CATALOG_SEEDS[provider].models[0]`) — the one place a provider's default model is named, so the deploy target and the catalog it is chosen from can never drift apart — but that - deploy happens in the background, never in the connect request itself - (CL-6457; see [bench-provisioning.md](bench-provisioning.md)). This is also how a + deploy happens through the desired-state reconcile, never in the + connect request itself (CL-6457; see + [seed-reconciliation.md](seed-reconciliation.md)). This is also how a bench the hub's own sign-in hook could only mark `bench_unseeded` (no hub-owned `ANTHROPIC_API_KEY` configured) finishes seeding: the first working credential a user connects, through whichever onboarding path diff --git a/docs/onboarding-huggingface-connect.md b/docs/onboarding-huggingface-connect.md index 60375cd77f..f629cad588 100644 --- a/docs/onboarding-huggingface-connect.md +++ b/docs/onboarding-huggingface-connect.md @@ -83,13 +83,14 @@ expiring token. routines against that token never happens here** — see `docs/onboarding-openrouter-connect.md`'s step 4 for why (the same duplicate-callback defect and fast/slow split apply here unchanged). -5. **The plaintext token rides forward server-side**, the same - `@workbench/onboarding` pending-seed store OpenRouter's flow uses - (step 5 in `docs/onboarding-openrouter-connect.md`) — a row keyed by - `(userId, tenantId)` carrying `provider: "huggingface"` and the - minted access token, never the expiry metadata (that's already - stored on the credential row by this point) and never anything the - browser sees. +5. **Nothing rides forward — the credential row is the handoff**, the + same kick OpenRouter's flow fires (step 5 in + `docs/onboarding-openrouter-connect.md`): the token is stored typed + `oauth_token`, the desired-state reconcile is kicked fire-and-forget + for the bench, and the callback 302s immediately. No row keyed by + `(userId, tenantId)`, no expiry metadata parked anywhere (that's + already on the credential row by this point), nothing the browser + sees. 6. **Back to the wizard.** Every ending 302s to `/onboarding?connect=huggingface&...`, parsed by `readHuggingFaceConnectReturn` exactly as `readOpenRouterConnectReturn` diff --git a/docs/onboarding-openrouter-connect.md b/docs/onboarding-openrouter-connect.md index 830d2dae1f..5e4d9f425a 100644 --- a/docs/onboarding-openrouter-connect.md +++ b/docs/onboarding-openrouter-connect.md @@ -62,23 +62,19 @@ at `/api/onboarding`); the wizard only navigates. now returns as soon as the key is proven and stored — see `complete-credential.ts`'s module comment for the fast/slow split, and `ensureSeeded` for the deploy step's new home. -5. **The plaintext key rides forward server-side, never to the - browser.** Credential secrets are write-only through the hub's own - API, so nothing can re-fetch the key once this request ends. It is - instead sealed (AEAD, the same `CredentialCipher` the state above - uses) into a row in `@workbench/onboarding`'s own `onboarding` - Postgres schema (`pending-seed.ts`, `createDrizzlePendingSeedStore`) - keyed by `(userId, tenantId)`, with a ten-minute TTL — long enough - for the wizard's own follow-up call. The browser gets nothing from - this step: no cookie, no ciphertext, not even a token — its ordinary - session cookie is what scopes `/complete-setup`'s read to exactly - this row. (Before CL-6031 this rode forward as an HttpOnly - `workbench_pending_seed` cookie; moved server-side because the - browser had no business custodying a sealed copy of the key even - though it could never read it. See `pending-seed.ts`'s module - comment for the full rationale, including the future - `InferenceSource` credential-by-reference primitive that would - remove this store entirely.) +5. **Nothing rides forward — the credential row is the handoff.** + Credential secrets are write-only through the hub's own API, so + nothing needs to re-fetch the key once this request ends. The + callback persists the credential (`testAndPersistCredential`), fires + the desired-state reconcile as a fire-and-forget kick for the bench, + and 302s immediately — no sealed row, no parked state, nothing the + browser carries. (Before CL-7586 the plaintext key rode forward + server-side in `@workbench/onboarding`'s own `onboarding` Postgres + schema, keyed by `(userId, tenantId)` with a ten-minute TTL, for the + wizard's own follow-up call to deploy with — and before CL-6031 it + rode as an HttpOnly `workbench_pending_seed` cookie. The reconcile + reads the bench's real state instead, so the key never needs to be + parked anywhere.) 6. **Back to the wizard.** Every ending 302s to `/onboarding?connect=openrouter&...`: `outcome=connected` with the bench slug (no routine list yet — that comes from step 7), or @@ -95,15 +91,14 @@ at `/api/onboarding`); the wizard only navigates. redirect, or a log line. 7. **The wizard finishes the job.** Landing on `outcome=connected`, the wizard shows a brief "setting up your workbench" state and calls - `POST /api/onboarding/complete-setup`. That route reads the pending - row for the caller's own `(userId, tenantId)`, runs `ensureSeeded` - (`seedTenant` with - `confirmDeployments: false`, same as before), and answers `seeded` - with the deployed routine names once done. It answers `unseeded` - (never an error) if there is nothing to seed with yet, and two - overlapping calls never double-deploy — every step `ensureSeeded` - drives is itself ensure-then-create (a 409 falls back to a list), - the same tolerance `seedTenant` has always had. + `POST /api/onboarding/complete-setup`. That route answers from hub + reads — `ready` once every default workflow is live, `provisioning` + while the kicked reconcile still converges (it re-kicks + fire-and-forget on each poll), `unseeded` (never an error) if there + is nothing to converge with yet — and two overlapping calls never + double-deploy, because every step the reconcile drives is itself + ensure-then-create (a 409 falls back to a list), the same tolerance + `seedTenant` has always had. ## CSRF model diff --git a/docs/seed-reconciliation.md b/docs/seed-reconciliation.md index 752244ec3b..a1db5dfb34 100644 --- a/docs/seed-reconciliation.md +++ b/docs/seed-reconciliation.md @@ -272,11 +272,11 @@ Sidecar-unavailable failures report `blocked` (the same class `failed` and are safe to re-run. Three triggers drive it, one reconciler: a tenant-create observation -(`POST /api/tenants` 201), the pending-seed drain over a connected -credential, and the revisit kick from `POST -/api/onboarding/provision`. The `pending_seed` row remains the only -durable work item; the tenant-create kick and the revisit kick are -deliberately in-memory — convergence, not delivery. +(`POST /api/tenants` 201), the credential-connect kick over a freshly +persisted credential, and the revisit kick from `POST +/api/onboarding/provision`. Every kick is deliberately in-memory — there +is no parked row and no durable work item anywhere; convergence comes +from re-reading the bench's real state on each pass, not delivery. Member-edit posture: a reconcile only ever installs what the document names. A member (or anyone) deleting or editing a seeded workflow or diff --git a/packages/access-policy/test/store.drizzle.test.ts b/packages/access-policy/test/store.drizzle.test.ts index ff1a5cdfb4..95ebdde545 100644 --- a/packages/access-policy/test/store.drizzle.test.ts +++ b/packages/access-policy/test/store.drizzle.test.ts @@ -1,6 +1,6 @@ // DB-gated: skipped when no DATABASE_URL is reachable (a fresh checkout // still runs the unit gates), mirroring `@workbench/onboarding`'s own -// `pending-seed-store.drizzle.test.ts`. Runs against its own scratch +// `migrations.test.ts`. Runs against its own scratch // database, never the developer's or the walking-skeleton suite's. // // Proves what the in-memory fake cannot: that diff --git a/packages/connections/src/connected-hook.ts b/packages/connections/src/connected-hook.ts index 3b3372892e..c9834e11ce 100644 --- a/packages/connections/src/connected-hook.ts +++ b/packages/connections/src/connected-hook.ts @@ -43,10 +43,9 @@ export async function fireConnectedHook( // `/complete` just left the tenant with `hasUsableModel` true — a model // with a resolvable offering, whether or not this tenant's bench has // ever deployed its default workflows. A composition wires this to the -// same durable pending-seed drain the onboarding credential step already -// feeds (`@workbench/onboarding`'s `pendingSeedStore` + `benchProvisioner`), -// so a tenant that connects its own provider through Settings converges -// on Myra and the default workflow set exactly like one that connects +// same desired-state kick the onboarding credential step fires, so a +// tenant that connects its own provider through Settings converges on +// Myra and the default workflow set exactly like one that connects // through onboarding — never stuck waiting on an operator-configured // hub key. `apiKey` here is the same secret `/complete` just stored // (the URL placeholder for a `credentialInputKind: "url"` connector like @@ -61,8 +60,8 @@ export type InferenceCredentialSeedableInfo = { readonly apiKey: string; /** The real instance origin a `credentialInputKind: "url"` connector * (Ollama) was just pointed at — absent for every other provider. - * Carried through to the drain so its deploy targets this tenant's - * actual endpoint rather than a curated default. */ + * Carried through to the kick so the reconcile's deploy targets this + * tenant's actual endpoint rather than a curated default. */ readonly baseURLOverride?: string; }; diff --git a/packages/connections/src/oauth-routes.ts b/packages/connections/src/oauth-routes.ts index 8668518ee3..a12d479d7f 100644 --- a/packages/connections/src/oauth-routes.ts +++ b/packages/connections/src/oauth-routes.ts @@ -16,13 +16,13 @@ // exchanged material and persisting it as a credential // (`deps.connectCredential`), recovering a duplicate callback // (`deps.recentlyConnected`), and anything that runs after a credential -// is durably stored — like onboarding's pending-seed / deploy-default- -// workflows step (`deps.afterConnected`) — are all injected. This keeps +// is durably stored — like onboarding's persist-then-kick step +// (`deps.afterConnected`) — are all injected. This keeps // `packages/connections` ignorant of tenant provisioning and workflow // deployment; `packages/onboarding` supplies its own -// `testAndPersistCredential`/`sealPendingSeed` unchanged as those deps -// when it mounts this factory, exactly preserving the sealed-state and -// pending-seed machinery the survey found already hardened. +// `testAndPersistCredential` and desired-state kick as those deps when +// it mounts this factory, and this package itself never deploys +// anything. // // `returnPath` (so both `/onboarding` and `/settings/connections` can // use the same routes): read from `?return=` on `/start`, carried across @@ -199,7 +199,7 @@ export type CreateOAuthConnectRoutesDeps = { }) => Promise<{ tenantSlug: string } | undefined>; /** Runs once a credential is durably stored, only for a connector * whose `oauth.deploysDefaultWorkflows` is true — onboarding's - * pending-seed sealing lives here, entirely outside this package. + * desired-state kick lives here, entirely outside this package. * Given the Hono `Context` directly so it can set its own cookie. */ readonly afterConnected?: (args: { c: Context; diff --git a/packages/connections/src/routes.ts b/packages/connections/src/routes.ts index 8fa49114b4..9549662030 100644 --- a/packages/connections/src/routes.ts +++ b/packages/connections/src/routes.ts @@ -290,11 +290,11 @@ export type CreateConnectionRoutesDeps = { * non-inference connector, and never merely because a credential row * exists (seeding plants that row regardless of whether it actually * resolves an offering). The composition wires this to the same - * durable pending-seed drain onboarding's own credential step feeds, - * so a provider connected through Settings deploys the tenant's - * default workflows exactly like one connected through onboarding — - * see `./connected-hook.ts`. Absent means this hub build never - * re-seeds off a Settings connect (every existing test double). */ + * desired-state kick onboarding's own credential step fires, so a + * provider connected through Settings converges the tenant's default + * workflows exactly like one connected through onboarding — see + * `./connected-hook.ts`. Absent means this hub build never re-kicks + * off a Settings connect (every existing test double). */ onInferenceCredentialUsable?: InferenceCredentialSeedableHook; }; diff --git a/packages/onboarding/README.md b/packages/onboarding/README.md index eb3efde72a..972d336b1a 100644 --- a/packages/onboarding/README.md +++ b/packages/onboarding/README.md @@ -21,10 +21,11 @@ workflow deploys. `@corbits/connections` (CL-6028 generalized both flows into that package's OAuth route factory) and are kept here only so existing imports don't break. -- `@intx/crypto`'s `CredentialCipher` seals the plaintext key carried from - an OAuth callback to onboarding's own follow-up request (see - `pending-seed.ts`); `@workbench/access-policy` and `@corbits/hub-api-client` - supply the grant and tenant primitives provisioning reuses. +- `@intx/crypto`'s `CredentialCipher` seals the OAuth connect state + (PKCE verifier included) parked between `/start` and `/callback` + (see `@corbits/connections`' `pkce.ts`); `@workbench/access-policy` + and `@corbits/hub-api-client` supply the grant and tenant primitives + provisioning reuses. ## Key modules @@ -36,10 +37,14 @@ workflow deploys. testAndPersistCredential` (fast, safe for an OAuth callback to await) and `ensureSeeded` (slow, the workflow-deploy half, run separately so a browser is never left waiting mid-redirect). -- `pending-seed.ts` — server-side custody of a just-connected credential's - plaintext key between the OAuth callback and the follow-up seed request. -- `schema.ts` / `migrations.ts` — the package's own `onboarding.pending_seed` - table, in its own Postgres schema, with its own migration ledger. +- `desired-state.ts` — the desired-state reconcile and its status reader: + connecting or polling a bench kicks the reconcile fire-and-forget and + answers from hub reads (`ready` / `provisioning` / `unseeded`). There + is no parked row and no deferred deploy step anywhere. +- `migrations.ts` — the package's migration ledger. `0001_pending_seed` + stays so replays from an old base still run; `0002_drop_pending_seed` + (CL-7586) drops the retired `onboarding.pending_seed` table, so a + migrated database holds no pending-seed table anywhere. ## Running tests @@ -47,5 +52,5 @@ testAndPersistCredential` (fast, safe for an OAuth callback to await) cd packages/onboarding && bun test ``` -`test/migrations.test.ts` and `test/pending-seed-store.drizzle.test.ts` -need a live Postgres: `DATABASE_URL=postgres://localhost:5432/workbench_e2e`. +`test/migrations.test.ts` needs a live Postgres: +`DATABASE_URL=postgres://localhost:5432/workbench_e2e`. diff --git a/packages/onboarding/src/bench-provisioning.ts b/packages/onboarding/src/bench-provisioning.ts deleted file mode 100644 index 27507518b6..0000000000 --- a/packages/onboarding/src/bench-provisioning.ts +++ /dev/null @@ -1,389 +0,0 @@ -// The one place a bench's default workflows are ever deployed -// (CL-6457). Connecting a provider used to do it inline: a pasted key -// sat on "Connecting…" for minutes while five workflows deployed at -// ~20s each, and the owner reasonably concluded the app had frozen. -// Connect now does the durable, fast half only — persist the credential, -// seed its catalog — and this converges the bench afterwards, off the -// request path entirely. -// -// It is a drain over `onboarding.pending_seed`, not a queue of jobs: the -// row IS the work item, and what it means is "this bench has a -// credential and does not yet have its agents". That framing is what -// makes the three properties fall out rather than have to be engineered: -// -// - Idempotent. Every pass reconciles the bench against the tenant -// desired-state document (`reconcileTenantDesiredState`) — ensure- -// then-create at every step — so a pass over a converged bench -// deploys nothing and simply clears the row. -// - Convergent. A pass that gets partway — pins reported `blocked` -// (sidecar unavailable) or `failed` — leaves the row in place, so -// the next pass picks up exactly the pins that are still missing. -// - Restart-safe. Nothing about a bench's outstanding work lives in -// this process. A hub that dies mid-deploy leaves the row behind, -// and the next boot's first tick finishes it. In-memory state here -// is only ever an optimization (a dedupe guard, a retry backoff), -// never a fact the system needs to be correct. -// -// Sessions are the one thing a background loop cannot inherit: there is -// no request to borrow cookies from. `sessionFor` is that seam — the -// composition root decides how a session is minted for a user, and this -// module stays out of the auth mechanism entirely. - -import { reportError } from "@corbits/error-sink"; -import type { ModelSource, WorkflowPusher } from "@corbits/seeding"; -import { type ApiCall } from "@corbits/hub-api-client"; -import { - reconcileTenantDesiredState, - resolveTenantModelSource, - type ReconcileReport, -} from "./desired-state"; -import { - PENDING_SEED_SCAN_LIMIT, - type PendingSeed, - type PendingSeedListCursor, - type PendingSeedStore, -} from "./pending-seed"; - -export const PROVISIONING_POLL_INTERVAL_MS = 15_000; -const RETRY_BACKOFF_BASE_MS = 15_000; -const RETRY_BACKOFF_CEILING_MS = 10 * 60 * 1000; - -/** - * Mints the hub session the drain acts under for one user's own bench, - * or `undefined` when no session can be minted (an account since - * deleted, an auth backend briefly unavailable). Returning `undefined` - * holds the bench for a later pass — it never discards the row. - */ -export type SessionForUser = (args: { - userId: string; - tenantId: string; -}) => Promise; - -export type BenchProvisionerDeps = { - api: ApiCall; - hubUrl: string; - store: PendingSeedStore; - pushWorkflow: WorkflowPusher; - sessionFor: SessionForUser; - log: (line: string) => void; - logError?: (line: string) => void; - /** - * The convergence step, doc-driven (CL-7584): reads the tenant's real - * state against `TENANT_DESIRED_STATE` and installs only absent pins. - * Production resolves the deploy model from the tenant's catalog and - * delegates to `reconcileTenantDesiredState`; tests replace the whole - * thing. A `blocked` report keeps the row; `failed` keeps it too and - * counts as a failure for backoff. - */ - reconcileFn?: (args: { - api: ApiCall; - cookies: string[]; - hubUrl: string; - tenant: { tenantId: string; principalId: string; domain: string }; - pushWorkflow: WorkflowPusher; - log: (line: string) => void; - }) => Promise; - now?: () => number; -}; - -/** What one pass over one bench concluded. `converged` and `pending` are - * both healthy — the difference is only whether there is more to do. */ -export type BenchProvisionOutcome = "converged" | "pending" | "failed"; - -export type DrainReport = { - readonly converged: number; - readonly pending: number; - readonly failed: number; - /** Benches skipped this tick because a previous failure's backoff has - * not elapsed, or because a pass over them is still running. */ - readonly deferred: number; - /** True when this tick stopped because its scan page filled the limit - * with more rows still in the table. The next tick continues after this - * page's cursor rather than restarting at the oldest row. False when - * the scan reached the end of the table. */ - readonly truncated: boolean; -}; - -export type BenchProvisioner = { - /** One bench, start to finish. Exposed so a connect can kick its own - * bench immediately instead of waiting out a poll interval. */ - provisionBench( - seed: PendingSeed, - ): Promise; - drainOnce(args?: { ignoreBackoff?: boolean }): Promise; - /** Fire-and-forget drain for callers that must not wait on it — a - * route that just wrote a pending row, or hub boot. Never rejects. */ - wake(): void; - start(args?: { intervalMs?: number }): void; - stop(): void; -}; - -function benchKey(seed: { userId: string; tenantId: string }): string { - return `${seed.userId}:${seed.tenantId}`; -} - -export function createBenchProvisioner( - deps: BenchProvisionerDeps, -): BenchProvisioner { - const now = deps.now ?? Date.now; - const logError = deps.logError ?? deps.log; - - /** - * The production reconcile: resolve the tenant's deploy model from its - * resolved catalog, then install only the desired-state pins that are - * absent. `undefined` model means no launchable offering — reconcile - * reports the workflow pins blocked rather than throwing. - */ - const runReconcile: NonNullable = - deps.reconcileFn ?? - (async (args) => { - const model: ModelSource | undefined = await resolveTenantModelSource( - args.api, - args.cookies, - args.tenant.tenantId, - ); - return reconcileTenantDesiredState({ ...args, model }); - }); - - const inFlight = new Map>(); - // Backoff bookkeeping for a failing bench, keyed the same way - // `inFlight` is. This is process-local, in-memory state — never a - // fact the system needs to be correct, only a hammering guard — but - // with no eviction it survives its own bench forever: a bench that - // fails permanently accumulates here even after its `pending_seed` - // row TTL-expires out from under it (CL-7233). `userId`/`tenantId` - // are carried alongside the counters (not re-derived from the map - // key) so `pruneOrphanedHolds` can ask the store directly whether a - // hold's row still exists, rather than assuming a compound string key - // splits back apart cleanly. - const holds = new Map< - string, - { - retryAfter: number; - failureCount: number; - userId: string; - tenantId: string; - } - >(); - let timer: ReturnType | undefined; - let scanAfter: PendingSeedListCursor | undefined; - - function holdOff(seed: { userId: string; tenantId: string }): void { - const key = benchKey(seed); - const failures = (holds.get(key)?.failureCount ?? 0) + 1; - const backoff = Math.min( - RETRY_BACKOFF_BASE_MS * 2 ** (failures - 1), - RETRY_BACKOFF_CEILING_MS, - ); - holds.set(key, { - retryAfter: now() + backoff, - failureCount: failures, - userId: seed.userId, - tenantId: seed.tenantId, - }); - } - - function clearHold(key: string): void { - holds.delete(key); - } - - /** - * Reclaims a hold whose bench is permanently gone rather than merely - * quiet this tick (CL-7233). A hold already appearing in `due` is - * left alone unconditionally — its row still exists, backoff or not. - * For every other hold, `store.read` is the authoritative check - * (unlike `due`, which is `listDue`'s capped-and-unordered page and - * can omit a row that still exists): a row is deleted the moment it - * is read past its TTL, so `undefined` here means the row is truly - * gone, not just off this tick's page. - */ - async function pruneOrphanedHolds( - due: readonly PendingSeed[], - ): Promise { - const dueKeys = new Set(due.map(benchKey)); - for (const [key, hold] of holds) { - if (dueKeys.has(key)) continue; - const row = await deps.store.read({ - userId: hold.userId, - tenantId: hold.tenantId, - }); - if (row === undefined) holds.delete(key); - } - } - - async function runOnce(seed: PendingSeed): Promise { - const cookies = await deps.sessionFor({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - if (cookies === undefined) { - logError( - `bench provisioning for tenant ${seed.tenantId} has no session to act under; holding for a later pass`, - ); - return "failed"; - } - - // Doc-driven convergence (CL-7584): reconcile installs only the - // desired-state pins this bench is still missing. Row semantics are - // unchanged — ready clears the row, blocked keeps it as pending, - // failed keeps it and backs off. - let report: ReconcileReport; - try { - report = await runReconcile({ - api: deps.api, - cookies, - hubUrl: deps.hubUrl, - tenant: { - tenantId: seed.tenantId, - principalId: seed.principalId, - domain: seed.tenantDomain, - }, - pushWorkflow: deps.pushWorkflow, - log: deps.log, - }); - } catch (cause) { - reportError(cause, { - operation: "pending_seed_reconcile", - tenantId: seed.tenantId, - }); - logError( - `bench provisioning for tenant ${seed.tenantId} failed; its pending row stays for a retry: ${cause instanceof Error ? cause.message : String(cause)}`, - ); - return "failed"; - } - - if (report.ready) { - await deps.store.clear({ userId: seed.userId, tenantId: seed.tenantId }); - deps.log( - `bench ${seed.tenantId} finished provisioning (${report.pins.length} pins present)`, - ); - return "converged"; - } - - if (report.pins.some((pin) => pin.status === "failed")) { - deps.log( - `bench provisioning for tenant ${seed.tenantId} failed; its pending row stays for a retry`, - ); - return "failed"; - } - - deps.log( - `bench ${seed.tenantId} is partly provisioned; its pending row stays for the next pass`, - ); - return "pending"; - } - - async function provisionBench( - seed: PendingSeed, - ): Promise { - const key = benchKey(seed); - const running = inFlight.get(key); - if (running !== undefined) return "deferred"; - - const operation = (async (): Promise => { - try { - const outcome = await runOnce(seed); - if (outcome === "converged") clearHold(key); - else holdOff(seed); - return outcome; - } catch (cause) { - // report-error-ignore: CL-7234 routes this drain catch through - // reportError the way provision.ts already does - const message = cause instanceof Error ? cause.message : String(cause); - logError( - `bench provisioning for tenant ${seed.tenantId} failed; its pending row stays for a retry: ${message}`, - ); - holdOff(seed); - return "failed"; - } - })(); - - inFlight.set(key, operation); - try { - return await operation; - } finally { - if (inFlight.get(key) === operation) inFlight.delete(key); - } - } - - async function drainOnce( - args: { ignoreBackoff?: boolean } = {}, - ): Promise { - const page = await deps.store.listDue({ - limit: PENDING_SEED_SCAN_LIMIT, - ...(scanAfter !== undefined ? { after: scanAfter } : {}), - }); - let converged = 0; - let pending = 0; - let failed = 0; - let deferred = 0; - - for (const seed of page.seeds) { - const key = benchKey(seed); - const heldUntil = holds.get(key)?.retryAfter; - if ( - args.ignoreBackoff !== true && - heldUntil !== undefined && - heldUntil > now() - ) { - deferred += 1; - continue; - } - if (args.ignoreBackoff === true) clearHold(key); - - const outcome = await provisionBench(seed); - if (outcome === "converged") converged += 1; - else if (outcome === "pending") pending += 1; - else if (outcome === "failed") failed += 1; - else deferred += 1; - } - - if (page.truncated && page.next !== undefined) { - scanAfter = page.next; - } else { - scanAfter = undefined; - } - - await pruneOrphanedHolds(page.seeds); - - return { - converged, - pending, - failed, - deferred, - truncated: page.truncated, - }; - } - - function wake(): void { - void drainOnce().catch((cause: unknown) => { - const message = cause instanceof Error ? cause.message : String(cause); - logError(`bench provisioning drain failed: ${message}`); - // Not scoped to any one bench — a whole-tick failure (listDue - // itself throwing, say) rather than one seed's own provisioning - // failure, which already carries tenant context via holdOff - // (CL-7234). - reportError(cause, { operation: "bench_provisioning_drain" }); - }); - } - - return { - provisionBench, - drainOnce, - wake, - start(args = {}) { - if (timer !== undefined) return; - timer = setInterval( - wake, - args.intervalMs ?? PROVISIONING_POLL_INTERVAL_MS, - ); - timer.unref?.(); - wake(); - }, - stop() { - if (timer === undefined) return; - clearInterval(timer); - timer = undefined; - }, - }; -} diff --git a/packages/onboarding/src/complete-credential.ts b/packages/onboarding/src/complete-credential.ts index 52b7eaf6b0..d5398b6094 100644 --- a/packages/onboarding/src/complete-credential.ts +++ b/packages/onboarding/src/complete-credential.ts @@ -23,14 +23,14 @@ // // CL-6457 finished that split: NO HTTP route runs the slow half any // more. Connecting a provider persists the credential, seeds its -// catalog, and returns in seconds; the deploys belong to the background -// drain in `./bench-provisioning.ts`, which is also what makes them -// survive a hub restart. `completeCredentialSetup` at the bottom of this -// file still composes both halves back-to-back, but it is an -// eval-harness convenience — a bench that must be fully deployed before -// a scenario runs — and never the connect path. A route that calls it -// re-creates the 2+ minute "Connecting…" freeze this split exists to -// prevent. +// catalog, and returns in seconds; the deploys converge afterwards +// through the desired-state revisit kicks (`desiredStateKick`, fired +// from the connect routes) and the tenant-create observer — never on a +// request path. `completeCredentialSetup` at the bottom of this file +// still composes both halves back-to-back, but it is an eval-harness +// convenience — a bench that must be fully deployed before a scenario +// runs — and never the connect path. A route that calls it re-creates +// the 2+ minute "Connecting…" freeze this split exists to prevent. // // `ensureSeeded` runs with `confirmDeployments: false`: there is nothing // to confirm by triggering a real, billed inference call against an @@ -141,12 +141,8 @@ export type CompleteCredentialResult = readonly kind: "seeded-pending-agents"; readonly tenantId: string; readonly tenantSlug: string; - /** Threaded through to `routes.ts`'s `/complete` handler, which - * writes them into the same `pendingSeedStore` row an OAuth - * connect writes (see `./pending-seed.ts`) so `POST - * /complete-setup`'s existing retry path — no new queue — finishes - * the deferred workflows on this account's next onboarding-page - * visit. */ + /** Carried through to `routes.ts`'s `/complete` handler for the + * desired-state revisit kick and status answer. */ readonly principalId: string; readonly tenantDomain: string; readonly deployed: string[]; diff --git a/packages/onboarding/src/desired-state.ts b/packages/onboarding/src/desired-state.ts index 254f0902c6..cbd6c56844 100644 --- a/packages/onboarding/src/desired-state.ts +++ b/packages/onboarding/src/desired-state.ts @@ -373,8 +373,8 @@ export type TenantDeployer = { * The deploy identity for a tenant nobody has connected a credential * to yet. The tenant-create kick (CL-7584) reconciles exactly such a * tenant — a native `POST /api/tenants` that mints the creator's owner - * principal in the same transaction — so there is no pending-seed row - * to read `principalId`/`tenantDomain` from. This resolves both from + * principal in the same transaction — so there is no stored row to + * read `principalId`/`tenantDomain` from. This resolves both from * the API instead: the session's own active user principal on the * tenant, plus the tenant row's domain. `undefined` when the session * holds no deployable principal there (suspended, invited, agent, or diff --git a/packages/onboarding/src/index.ts b/packages/onboarding/src/index.ts index 8db4158df1..ae7e057f13 100644 --- a/packages/onboarding/src/index.ts +++ b/packages/onboarding/src/index.ts @@ -54,29 +54,6 @@ export type { ToolPackagePin, WorkflowPin, } from "./desired-state"; -export { - createBenchProvisioner, - PROVISIONING_POLL_INTERVAL_MS, -} from "./bench-provisioning"; -export type { - BenchProvisioner, - BenchProvisionerDeps, - BenchProvisionOutcome, - DrainReport, - SessionForUser, -} from "./bench-provisioning"; -export { - createDrizzlePendingSeedStore, - createInMemoryPendingSeedStore, - PENDING_SEED_TTL_MS, -} from "./pending-seed"; -export type { - PendingSeed, - PendingSeedDb, - PendingSeedDuePage, - PendingSeedListCursor, - PendingSeedStore, -} from "./pending-seed"; export { envProviderBaseUrlsFrom, envProviderKeysFrom, diff --git a/packages/onboarding/src/migrations.ts b/packages/onboarding/src/migrations.ts index 88a445e918..4d498ed0e9 100644 --- a/packages/onboarding/src/migrations.ts +++ b/packages/onboarding/src/migrations.ts @@ -27,6 +27,16 @@ export const onboardingMigrations: readonly OnboardingMigration[] = [ ); `, }, + // CL-7586: the pending-seed drain is gone — provisioning is the + // explicit setup-CLI/installer flow now — so the table it converged + // goes with it. 0001 stays: replays from an empty database still need + // the full history. + { + name: "0002_drop_pending_seed", + sql: ` + DROP TABLE IF EXISTS "onboarding"."pending_seed"; + `, + }, ]; const LEDGER_TABLE = "onboarding_migrations"; diff --git a/packages/onboarding/src/pending-seed.test.ts b/packages/onboarding/src/pending-seed.test.ts deleted file mode 100644 index 6d64f0d9b9..0000000000 --- a/packages/onboarding/src/pending-seed.test.ts +++ /dev/null @@ -1,311 +0,0 @@ -// The in-memory store exercises the same encrypt/validate/TTL logic -// `createDrizzlePendingSeedStore` runs against Postgres (see -// `createPendingSeedStore` in `./pending-seed.ts`) — round-trip, -// per-provider AAD, wrong-session, expiry, and cleared-on-success all -// have to hold regardless of which `RowAccess` backs the store. -// `test/pending-seed-store.test.ts` DB-gates the same scenarios against -// the real Postgres-backed store. -import { describe, expect, test } from "bun:test"; -import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import type { CredentialCipher } from "@intx/types"; -import { - createInMemoryPendingSeedStore, - PENDING_SEED_TTL_MS, - type PendingSeed, -} from "./pending-seed"; - -const TEST_KEY = Buffer.alloc(32, 11); -function testCipher(): CredentialCipher { - return createEnvKeyCredentialCipher(TEST_KEY); -} - -const SEED: PendingSeed = { - userId: "user_1", - tenantId: "ten_1", - principalId: "prn_1", - tenantDomain: "alice-user1.bench.local", - provider: "openrouter", - apiKey: "sk-or-v1-minted", -}; - -describe("createInMemoryPendingSeedStore", () => { - test("round-trips for the exact user and tenant it was written for", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - - expect(read).toEqual(SEED); - }); - - test("stays readable more than once inside its TTL — unlike the PKCE state, this is not single-use", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const first = await store.read({ userId: "user_1", tenantId: "ten_1" }); - const second = await store.read({ userId: "user_1", tenantId: "ten_1" }); - - expect(first).toEqual(SEED); - expect(second).toEqual(SEED); - }); - - test("a row written for one user is invisible to another", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const read = await store.read({ userId: "user_2", tenantId: "ten_1" }); - - expect(read).toBeUndefined(); - }); - - test("a row written for one tenant is invisible against another", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const read = await store.read({ - userId: "user_1", - tenantId: "ten_other", - }); - - expect(read).toBeUndefined(); - }); - - test("an expired row reads as absent and is deleted, not merely ignored", async () => { - let clock = 0; - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED, { now: () => clock }); - - clock = PENDING_SEED_TTL_MS; - const read = await store.read({ - userId: "user_1", - tenantId: "ten_1", - now: () => clock, - }); - expect(read).toBeUndefined(); - - // Deleted, not just expired-and-skipped: a fresh read (even with - // the clock rewound) finds nothing, proving the row is gone. - const rereadEarlier = await store.read({ - userId: "user_1", - tenantId: "ten_1", - now: () => 0, - }); - expect(rereadEarlier).toBeUndefined(); - }); - - test("a ciphertext sealed under one key is worthless after a key rotation", async () => { - const cipher = testCipher(); - const aad = JSON.stringify(["onboarding-pending-seed", "openrouter"]); - const payload = await cipher.encrypt( - JSON.stringify({ - principalId: SEED.principalId, - tenantDomain: SEED.tenantDomain, - apiKey: SEED.apiKey, - }), - aad, - ); - - const rotated = createEnvKeyCredentialCipher(Buffer.alloc(32, 12)); - await expect(rotated.decrypt(payload, aad)).rejects.toThrow(); - }); - - test("a fresh connect upserts over whatever pending seed came before it — one active row per (userId, tenantId)", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const replacement: PendingSeed = { - ...SEED, - provider: "huggingface", - apiKey: "hf_replaced", - }; - await store.put(replacement); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - expect(read).toEqual(replacement); - }); - - test("listDue answers every unexpired row, whoever owns it — the drain has no request to scope it", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - await store.put({ - ...SEED, - userId: "user_2", - tenantId: "ten_2", - apiKey: "sk-or-v1-second", - }); - - const due = await store.listDue({}); - - expect(due.truncated).toBe(false); - expect(due).not.toHaveProperty("next"); - expect(due.seeds).toHaveLength(2); - expect(due.seeds).toEqual( - expect.arrayContaining([ - SEED, - { - ...SEED, - userId: "user_2", - tenantId: "ten_2", - apiKey: "sk-or-v1-second", - }, - ]), - ); - }); - - test("listDue drops an expired row instead of handing the drain a dead key", async () => { - let clock = 0; - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED, { now: () => clock }); - await store.put( - { ...SEED, userId: "user_2", tenantId: "ten_2" }, - { now: () => clock, ttlMs: PENDING_SEED_TTL_MS * 2 }, - ); - - clock = PENDING_SEED_TTL_MS + 1; - const due = await store.listDue({ now: () => clock }); - - expect(due.truncated).toBe(false); - expect(due.seeds).toEqual([ - { ...SEED, userId: "user_2", tenantId: "ten_2" }, - ]); - // Swept, not merely skipped — the expired row is gone for good. - expect( - await store.read({ userId: "user_1", tenantId: "ten_1", now: () => 0 }), - ).toBeUndefined(); - }); - - test("listDue honors its limit so one drain tick can never scan the whole table", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - for (let index = 0; index < 5; index += 1) { - await store.put({ - ...SEED, - userId: `user_${index}`, - tenantId: `ten_${index}`, - }); - } - - const due = await store.listDue({ limit: 2 }); - - expect(due.seeds).toHaveLength(2); - expect(due.truncated).toBe(true); - expect(due.next).toEqual({ - expiresAt: expect.any(Date), - userId: expect.any(String), - tenantId: expect.any(String), - }); - }); - - test("listDue returns oldest-due first and continues from an (expiresAt, userId, tenantId) cursor", async () => { - const clock = 0; - const store = createInMemoryPendingSeedStore(testCipher()); - for (let index = 0; index < 5; index += 1) { - await store.put( - { - ...SEED, - userId: `user_${index}`, - tenantId: `ten_${index}`, - }, - { now: () => clock, ttlMs: 5_000 - index }, - ); - } - - const first = await store.listDue({ limit: 2, now: () => clock }); - expect(first.seeds.map((seed) => seed.userId)).toEqual([ - "user_4", - "user_3", - ]); - expect(first.truncated).toBe(true); - - const second = await store.listDue({ - limit: 2, - now: () => clock, - ...(first.next !== undefined ? { after: first.next } : {}), - }); - expect(second.seeds.map((seed) => seed.userId)).toEqual([ - "user_2", - "user_1", - ]); - expect(second.truncated).toBe(true); - - const third = await store.listDue({ - limit: 2, - now: () => clock, - ...(second.next !== undefined ? { after: second.next } : {}), - }); - expect(third.seeds.map((seed) => seed.userId)).toEqual(["user_0"]); - expect(third.truncated).toBe(false); - expect(third).not.toHaveProperty("next"); - }); - - test("cleared on successful seed — the row is gone after clear", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - await store.clear({ userId: "user_1", tenantId: "ten_1" }); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - expect(read).toBeUndefined(); - }); - - test("clearing a row that was never written is a harmless no-op", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await expect( - store.clear({ userId: "user_1", tenantId: "ten_1" }), - ).resolves.toBeUndefined(); - }); - - test("round-trips baseURLOverride for an ollama-shaped seed — the drain needs the real instance URL, not a curated default", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - const ollamaSeed: PendingSeed = { - ...SEED, - provider: "ollama", - baseURLOverride: "https://home-mac-studio.tail87f5aa.ts.net", - }; - await store.put(ollamaSeed); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - - expect(read).toEqual(ollamaSeed); - }); - - test("omits baseURLOverride when the seed carries none, rather than round-tripping it as undefined", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - await store.put(SEED); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - - expect(read).not.toHaveProperty("baseURLOverride"); - }); - - test("round-trips for a provider other than the first — every supported provider seals and opens correctly", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - const hfSeed: PendingSeed = { ...SEED, provider: "huggingface" }; - await store.put(hfSeed); - - const read = await store.read({ userId: "user_1", tenantId: "ten_1" }); - - expect(read).toEqual(hfSeed); - }); - - test("domain separation: a ciphertext sealed for one provider's AAD cannot decrypt under another's", async () => { - const cipher = testCipher(); - const openrouterAad = JSON.stringify([ - "onboarding-pending-seed", - "openrouter", - ]); - const huggingfaceAad = JSON.stringify([ - "onboarding-pending-seed", - "huggingface", - ]); - const payload = await cipher.encrypt( - JSON.stringify({ - principalId: SEED.principalId, - tenantDomain: SEED.tenantDomain, - apiKey: SEED.apiKey, - }), - openrouterAad, - ); - - await expect(cipher.decrypt(payload, huggingfaceAad)).rejects.toThrow(); - }); -}); diff --git a/packages/onboarding/src/pending-seed.ts b/packages/onboarding/src/pending-seed.ts deleted file mode 100644 index 5ab12155dc..0000000000 --- a/packages/onboarding/src/pending-seed.ts +++ /dev/null @@ -1,474 +0,0 @@ -// Server-side custody for a just-connected credential's plaintext key, -// carried from the OAuth callback (which must never wait on a workflow -// deploy — see `complete-credential.ts`) to the onboarding page's own -// follow-up request, the one that actually runs `ensureSeeded`. The -// hub's credential rows are write-only by design (a secret is never -// handed back over `GET /api/tenants/:id/credentials`), so once the -// callback's own request ends, the plaintext key is gone from memory -// unless something carries it forward — this is that something. -// -// CL-6031 moved this off the browser entirely. It used to be an -// HttpOnly cookie sealed the same way the PKCE connect state is -// (`pkce.ts`); a cookie meant the browser briefly custodied a sealed -// copy of the plaintext key even though it could never read it. Now -// it is one row per (userId, tenantId) in this package's own -// `onboarding.pending_seed` table (see `./schema.ts`), and the browser -// carries nothing but its ordinary session cookie — which already -// scopes every read to the exact user and tenant the row was written -// for, the same guarantee the cookie's own userId/tenantId check used -// to provide by hand. -// -// Same AEAD discipline the cookie used: sealed via the shared -// `CredentialCipher`, AAD `["onboarding-pending-seed", provider]`, so a -// value minted moments before a hub restart survives it, and a row -// sealed for one provider cannot decrypt under another's. Unlike the -// PKCE connect state, this is not single-use — the workflow-deploy step -// it feeds (`seedTenant`/`ensureSeeded`) is itself idempotent -// (409-then-list on every create), so two overlapping "finish setup" -// calls both reading the same still-valid row is exactly as safe as -// one, and a legit retry after a transient failure can reuse it within -// the same short window instead of being stranded with no key to retry -// with. A single active row per (userId, tenantId) — writing a new one -// replaces whatever was there (upsert) — mirrors the cookie's -// single-active-token behavior. -// -// CL-6457 turned this row into something more than a handoff: it is now -// the durable work item for background bench provisioning (see -// `./bench-provisioning.ts`). Connecting a provider deploys nothing, so -// the row is what says "this bench has a credential and does not yet -// have its agents" — and because it lives in Postgres rather than in -// the process that wrote it, a hub that dies mid-deploy resumes the -// bench on its next boot instead of stranding it half-provisioned. That -// is also why the TTL is a day rather than ten minutes: it now has to -// outlive provisioning, not just a redirect. -// -// TTL is enforced at read time: a row past its `expiresAt` is deleted -// and treated as though it were never there, rather than by a periodic -// sweep job — `listDue` applies exactly the same rule as `read` on its -// way past, so the drain is never handed a dead key. A row that is never -// read again is dead weight, not a correctness or security problem — the -// ciphertext is worthless without the cipher key, and a forgotten row -// does not accumulate: the next connect upserts over it. -// -// Future work: once Interchange ships an `InferenceSource` -// credential-by-reference primitive (the [Intx ask] tracked on this -// ticket), the callback can hand `ensureSeeded` a reference to the -// credential it already just persisted instead of smuggling the -// plaintext key forward at all — this entire store, table, and AEAD -// dance goes away. - -import { and, asc, eq, gt, or } from "drizzle-orm"; -import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; -import { type } from "arktype"; -import type { CredentialCipher } from "@intx/types"; -import { - supportedCredentialProviders, - type SupportedCredentialProvider, -} from "@corbits/connections/credential-test"; -import { pendingSeed } from "./schema"; - -/** - * How long a row stays usable. This is a provisioning queue's durability - * window, not a handoff token's lifetime (CL-6457): the row is what lets - * a background drain finish — or, after a hub restart, resume — the - * workflow deploys that connect deliberately no longer waits on. It is - * deleted the moment provisioning converges, so this bound only governs - * a bench that never converged at all: long enough that an overnight - * sidecar outage still resolves itself, short enough that an abandoned - * one does not keep a duplicate of the key indefinitely. The key itself - * already lives on, encrypted, as the tenant's own credential row — this - * copy is a duplicate of material the hub holds either way, never a new - * class of secret. - */ -export const PENDING_SEED_TTL_MS = 24 * 60 * 60 * 1000; - -const PROVIDER_IDS = supportedCredentialProviders().map((p) => p.id) as [ - SupportedCredentialProvider, - ...SupportedCredentialProvider[], -]; - -const PendingSeedSecret = type({ - principalId: "string > 0", - tenantDomain: "string > 0", - apiKey: "string > 0", - "baseURLOverride?": "string > 0", -}); - -export type PendingSeed = { - readonly userId: string; - readonly tenantId: string; - readonly principalId: string; - readonly tenantDomain: string; - readonly provider: SupportedCredentialProvider; - readonly apiKey: string; - /** The instance origin a `credentialInputKind: "url"` connector - * (Ollama) was actually pointed at, connect time — never a curated - * default. Absent for every other provider. Threaded through so the - * drain's `ensureSeeded` call resolves `modelSourceFor` against the - * real instance rather than falling back to `CATALOG_SEEDS.ollama`'s - * fixed origin (CL-6366's same failure mode, one hop later). */ - readonly baseURLOverride?: string; -}; - -export type PendingSeedDb< - TSchema extends Record = Record, -> = PostgresJsDatabase; - -/** Domain separation on top of the AEAD tag, mirroring `pkce.ts`'s - * `connectStateAad`: a row sealed for one provider's connect flow - * cannot decrypt under another's. */ -function pendingSeedAad(provider: string): string { - return JSON.stringify(["onboarding-pending-seed", provider]); -} - -export interface PendingSeedStore { - /** Upserts the single active row for (userId, tenantId) — a fresh - * connect always replaces whatever pending seed came before it. */ - put( - seed: PendingSeed, - args?: { ttlMs?: number; now?: () => number }, - ): Promise; - /** - * Reads and validates the row for exactly the signed-in user and - * tenant this request already resolved — a row belonging to a - * different user or tenant is simply a different key and is never - * found. A row that fails to validate (expired, corrupt, sealed - * under a since-rotated key) is deleted as dead weight before this - * returns `undefined`, rather than left to linger out its TTL — - * this is the read-time TTL cleanup described in this module's - * header comment. A row that validates is left in place; the caller - * decides when to `clear` it (on successful seed). - */ - read(args: { - userId: string; - tenantId: string; - now?: () => number; - }): Promise; - /** - * One ordered page of unexpired rows, for the background drain that has - * no request — and therefore no (userId, tenantId) — to scope itself by. - * This is what makes provisioning survive a hub restart: the rows a - * crashed process was mid-way through are still here, and the next - * boot's first tick picks them straight back up. Expired rows are - * deleted on the way past, the same read-time sweep `read` performs, so - * a dead key is never handed to a drain. - * - * Rows are ordered oldest-due first (`expiresAt`, then `userId`, - * `tenantId`). `limit` bounds one page so a tick cannot scan the whole - * table; pass `after` the previous page's `next` cursor to continue. - * `truncated` is true when more rows remain behind this page. - */ - listDue(args: { - now?: () => number; - limit?: number; - after?: PendingSeedListCursor; - }): Promise; - /** Deletes the row for (userId, tenantId), if any. Called once the - * pending seed has done its job (seeded successfully) or once the - * bench already reads as fully seeded some other way. */ - clear(args: { userId: string; tenantId: string }): Promise; -} - -export const PENDING_SEED_SCAN_LIMIT = 50; - -/** Keyset cursor for `listDue` — the last row of a truncated page. */ -export type PendingSeedListCursor = { - readonly expiresAt: Date; - readonly userId: string; - readonly tenantId: string; -}; - -export type PendingSeedDuePage = { - readonly seeds: PendingSeed[]; - readonly truncated: boolean; - readonly next?: PendingSeedListCursor; -}; - -interface StoredRow { - provider: string; - payload: string; - expiresAt: Date; -} - -interface IdentifiedRow extends StoredRow { - userId: string; - tenantId: string; -} - -interface RowAccess { - get(userId: string, tenantId: string): Promise; - put(userId: string, tenantId: string, row: StoredRow): Promise; - delete(userId: string, tenantId: string): Promise; - list(args: { - limit: number; - after?: PendingSeedListCursor; - }): Promise; -} - -function comparePendingSeedCursor( - a: PendingSeedListCursor, - b: PendingSeedListCursor, -): number { - const byExpires = a.expiresAt.getTime() - b.expiresAt.getTime(); - if (byExpires !== 0) return byExpires; - if (a.userId < b.userId) return -1; - if (a.userId > b.userId) return 1; - if (a.tenantId < b.tenantId) return -1; - if (a.tenantId > b.tenantId) return 1; - return 0; -} - -function createPendingSeedStore( - access: RowAccess, - cipher: CredentialCipher, -): PendingSeedStore { - /** - * The single validity rule both readers apply: a row that is expired, - * sealed for an unsupported provider, or undecryptable under the - * current key is dead weight — deleted here and reported as absent, - * rather than left to linger or handed onward as a usable seed. - */ - async function decodeRow( - row: IdentifiedRow, - nowMs: number, - ): Promise { - const drop = async (): Promise => { - await access.delete(row.userId, row.tenantId); - return undefined; - }; - - if (row.expiresAt.getTime() <= nowMs) return drop(); - if (!PROVIDER_IDS.includes(row.provider as SupportedCredentialProvider)) { - return drop(); - } - const provider = row.provider as SupportedCredentialProvider; - - try { - const plaintext = await cipher.decrypt( - row.payload, - pendingSeedAad(provider), - ); - const parsed = PendingSeedSecret(JSON.parse(plaintext)); - if (parsed instanceof type.errors) return drop(); - return { - userId: row.userId, - tenantId: row.tenantId, - provider, - principalId: parsed.principalId, - tenantDomain: parsed.tenantDomain, - apiKey: parsed.apiKey, - ...(parsed.baseURLOverride !== undefined - ? { baseURLOverride: parsed.baseURLOverride } - : {}), - }; - } catch { - return drop(); - } - } - - return { - async put(seed, args = {}) { - const now = args.now ?? Date.now; - const ttlMs = args.ttlMs ?? PENDING_SEED_TTL_MS; - const payload = await cipher.encrypt( - JSON.stringify({ - principalId: seed.principalId, - tenantDomain: seed.tenantDomain, - apiKey: seed.apiKey, - ...(seed.baseURLOverride !== undefined - ? { baseURLOverride: seed.baseURLOverride } - : {}), - }), - pendingSeedAad(seed.provider), - ); - await access.put(seed.userId, seed.tenantId, { - provider: seed.provider, - payload, - expiresAt: new Date(now() + ttlMs), - }); - }, - - async read(args) { - const now = args.now ?? Date.now; - const row = await access.get(args.userId, args.tenantId); - if (row === undefined) return undefined; - return decodeRow( - { ...row, userId: args.userId, tenantId: args.tenantId }, - now(), - ); - }, - - async listDue(args) { - const now = args.now ?? Date.now; - const nowMs = now(); - const limit = args.limit ?? PENDING_SEED_SCAN_LIMIT; - const fetched = await access.list({ - limit: limit + 1, - ...(args.after !== undefined ? { after: args.after } : {}), - }); - const truncated = fetched.length > limit; - const rows = truncated ? fetched.slice(0, limit) : fetched; - const due: PendingSeed[] = []; - for (const row of rows) { - const seed = await decodeRow(row, nowMs); - if (seed !== undefined) due.push(seed); - } - const last = rows[rows.length - 1]; - return { - seeds: due, - truncated, - ...(truncated && last !== undefined - ? { - next: { - expiresAt: last.expiresAt, - userId: last.userId, - tenantId: last.tenantId, - }, - } - : {}), - }; - }, - - async clear(args) { - await access.delete(args.userId, args.tenantId); - }, - }; -} - -export function createDrizzlePendingSeedStore< - TSchema extends Record, ->(db: PendingSeedDb, cipher: CredentialCipher): PendingSeedStore { - return createPendingSeedStore( - { - async get(userId, tenantId) { - const [row] = await db - .select() - .from(pendingSeed) - .where( - and( - eq(pendingSeed.userId, userId), - eq(pendingSeed.tenantId, tenantId), - ), - ); - return row === undefined - ? undefined - : { - provider: row.provider, - payload: row.payload, - expiresAt: row.expiresAt, - }; - }, - async put(userId, tenantId, row) { - await db - .insert(pendingSeed) - .values({ - userId, - tenantId, - provider: row.provider, - payload: row.payload, - expiresAt: row.expiresAt, - }) - .onConflictDoUpdate({ - target: [pendingSeed.userId, pendingSeed.tenantId], - set: { - provider: row.provider, - payload: row.payload, - expiresAt: row.expiresAt, - }, - }); - }, - async delete(userId, tenantId) { - await db - .delete(pendingSeed) - .where( - and( - eq(pendingSeed.userId, userId), - eq(pendingSeed.tenantId, tenantId), - ), - ); - }, - async list({ limit, after }) { - const rows = - after === undefined - ? await db - .select() - .from(pendingSeed) - .orderBy( - asc(pendingSeed.expiresAt), - asc(pendingSeed.userId), - asc(pendingSeed.tenantId), - ) - .limit(limit) - : await db - .select() - .from(pendingSeed) - .where( - or( - gt(pendingSeed.expiresAt, after.expiresAt), - and( - eq(pendingSeed.expiresAt, after.expiresAt), - gt(pendingSeed.userId, after.userId), - ), - and( - eq(pendingSeed.expiresAt, after.expiresAt), - eq(pendingSeed.userId, after.userId), - gt(pendingSeed.tenantId, after.tenantId), - ), - ), - ) - .orderBy( - asc(pendingSeed.expiresAt), - asc(pendingSeed.userId), - asc(pendingSeed.tenantId), - ) - .limit(limit); - return rows.map((row) => ({ - userId: row.userId, - tenantId: row.tenantId, - provider: row.provider, - payload: row.payload, - expiresAt: row.expiresAt, - })); - }, - }, - cipher, - ); -} - -/** The fake this package's own tests drive routes through — no - * Postgres required, same encrypt/validate/TTL semantics as the real - * store. */ -export function createInMemoryPendingSeedStore( - cipher: CredentialCipher, -): PendingSeedStore { - const rows = new Map(); - const keyOf = (userId: string, tenantId: string) => `${userId}:${tenantId}`; - return createPendingSeedStore( - { - async get(userId, tenantId) { - return rows.get(keyOf(userId, tenantId)); - }, - async put(userId, tenantId, row) { - rows.set(keyOf(userId, tenantId), row); - }, - async delete(userId, tenantId) { - rows.delete(keyOf(userId, tenantId)); - }, - async list({ limit, after }) { - const identified = [...rows.entries()].map(([key, row]) => { - const [userId = "", tenantId = ""] = key.split(":"); - return { ...row, userId, tenantId }; - }); - identified.sort(comparePendingSeedCursor); - const filtered = - after === undefined - ? identified - : identified.filter( - (row) => comparePendingSeedCursor(row, after) > 0, - ); - return filtered.slice(0, limit); - }, - }, - cipher, - ); -} diff --git a/packages/onboarding/src/routes.ts b/packages/onboarding/src/routes.ts index 46bfb2b312..bcdea835a2 100644 --- a/packages/onboarding/src/routes.ts +++ b/packages/onboarding/src/routes.ts @@ -51,13 +51,11 @@ import { import type { HubSignupTenancy } from "./genesis"; import { - ensureSeeded, findPersonalTenant, testAndPersistCredential, type PersonalTenant, type TestAndPersistCredentialResult, } from "./complete-credential"; -import type { BenchProvisioner } from "./bench-provisioning"; import { createOAuthConnectRoutes, DEFAULT_RETURN_PATH_ALLOWLIST, @@ -65,7 +63,6 @@ import { type OAuthExchangeResult, } from "@corbits/connections"; import { CONNECTOR_REGISTRY } from "@workbench/templates/connectors"; -import type { PendingSeedStore } from "./pending-seed"; import { exchangeCodeForKey } from "./openrouter-connect"; import { exchangeCodeForToken as exchangeHuggingFaceCodeForToken } from "./huggingface-connect"; import type { ProviderHealthStore } from "@corbits/connections/provider-health"; @@ -173,28 +170,14 @@ export type CreateOnboardingRoutesDeps = { exchange?: typeof exchangeHuggingFaceCodeForToken; connectCredential?: typeof testAndPersistCredential; }; - /** - * The background drain these routes hand provisioning work to - * (CL-6457). No route here ever deploys a workflow itself; the most a - * route does is write the pending-seed row and nudge this. Absent - * only means no nudge — the drain's own poll still picks the row up on - * its next tick, which is why this is a latency optimization rather - * than a correctness dependency. - */ - benchProvisioner?: Pick; /** * CL-7584 revisit kick: fire-and-forget desired-state reconcile for the * caller's tenant, fired unconditionally — never gated on a pending * check (the reconciler itself is reads-only when converged). The hub - * wires this to the same reconciler the tenant-create observer and the - * drain use; the route never awaits it. Absent means no kick — - * convergence then relies on the drain's own poll, which is why this is - * a latency optimization rather than a correctness dependency. + * wires this to the same reconciler the tenant-create observer uses; + * the route never awaits it. Absent means no kick. */ desiredStateKick?: (args: { tenantId: string; cookies: string[] }) => void; - /** Test seam standing in for the deploy step, so a route test can - * prove the response never waits on one. */ - ensureSeededFn?: typeof ensureSeeded; /** Test seam for `POST /complete`'s fast half — credential persist and * catalog seed, the only work that call still does inline. */ testAndPersistCredentialFn?: typeof testAndPersistCredential; @@ -208,13 +191,6 @@ export type CreateOnboardingRoutesDeps = { * optional dep here. */ providerHealth?: ProviderHealthStore; - /** Server-side custody for a just-connected credential's plaintext - * key between the OAuth callback and this package's own - * `/complete-setup` follow-up — see `./pending-seed.ts`'s module - * comment for why this replaced an HttpOnly cookie (CL-6031). Built - * from `createDrizzlePendingSeedStore(db, credentialCipher)` in - * production; tests inject `createInMemoryPendingSeedStore`. */ - pendingSeedStore: PendingSeedStore; /** The closed-by-default access-policy gate threaded straight into * `provisionPersonalTenantIfNeeded` — see that function's own * `accessPolicy` doc comment. Absent means no access-policy package @@ -239,8 +215,8 @@ export type CreateOnboardingRoutesDeps = { * What every provisioning-aware route answers with (CL-6457): where this * bench's agents actually are, read from the bench's own asset and * deployment state rather than from anything a caller remembers. `ready` - * means every default workflow is live; `provisioning` means the drain - * still has work to do. + * means every default workflow is live; `provisioning` means pins are + * still missing. * * `setupAgentReady` is the only field a waiting surface should ever * branch on (CL-6462): it says whether the one agent a person talks to @@ -340,6 +316,55 @@ async function recentlyConnectedCredential( } } +/** + * Whether the tenant holds any active inference credential — what + * `/complete-setup` answers `unseeded` from now that no pending-seed row + * records "a credential was just connected" (CL-7586). Best-effort like + * the duplicate-callback recovery above: a hub read failure answers + * "none", and the caller falls back to the ordinary credential step + * rather than surfacing a second failure mode. + */ +async function hasActiveInferenceCredential( + api: ApiCall, + cookies: string[], + tenantId: string, + log: (line: string) => void, +): Promise { + try { + const listed = await api( + "GET", + `/api/tenants/${tenantId}/credentials`, + undefined, + cookies, + ); + const credentials = parseAs( + paginatedSchema(CredentialResponse), + listed.data, + "credentials response", + ).data; + const names = new Set( + supportedCredentialProviders().map((provider) => + inferenceCredentialName(provider.id), + ), + ); + return credentials.some( + (credential) => + credential.status === "active" && names.has(credential.name), + ); + } catch (cause) { + // Never the raw cause detail — this path lists credentials after a + // connect and can see secret-shaped hub/parse errors (CL-7255). + const refId = reportError(cause, { + operation: "onboarding_complete_setup_credential_check", + extra: { tenantId }, + }); + log( + `complete-setup credential check failed for tenant ${tenantId} [${refId}]`, + ); + return false; + } +} + export function createOnboardingRoutes( deps: CreateOnboardingRoutesDeps, ): Hono { @@ -502,7 +527,7 @@ export function createOnboardingRoutes( // every pin present), so a pending-check here would only ever delay // this response; the reconcile itself re-reads the pins behind the // fire-and-forget boundary. Repeat probes re-kick safely - // (idempotent) and `pending_seed` stays the only durable row. + // (idempotent). if ( (result.kind === "provisioned" || result.kind === "existing-member") && deps.desiredStateKick !== undefined @@ -512,11 +537,10 @@ export function createOnboardingRoutes( try { kick({ tenantId, cookies }); } catch (cause) { - // report-error-ignore: the kick is best-effort — convergence - // falls back to the pending_seed drain, so a failed kick only - // ever costs one delayed pass. + // report-error-ignore: the kick is best-effort — a failed + // kick only ever costs one missed reconcile. deps.log( - `desired-state kick for user ${user.id} failed (convergence falls back to the drain): ${cause instanceof Error ? cause.message : String(cause)}`, + `desired-state kick for user ${user.id} failed: ${cause instanceof Error ? cause.message : String(cause)}`, ); } }; @@ -540,11 +564,10 @@ export function createOnboardingRoutes( ); if (found?.tenantId !== undefined) fire(found.tenantId); } catch (cause) { - // report-error-ignore: the kick lookup is best-effort — - // convergence falls back to the pending_seed drain, so a - // failed lookup only ever costs one delayed pass. + // report-error-ignore: the kick lookup is best-effort — a + // failed lookup only ever costs one missed reconcile. deps.log( - `desired-state kick lookup for user ${user.id} failed (convergence falls back to the drain): ${cause instanceof Error ? cause.message : String(cause)}`, + `desired-state kick lookup for user ${user.id} failed: ${cause instanceof Error ? cause.message : String(cause)}`, ); } })(); @@ -608,9 +631,9 @@ export function createOnboardingRoutes( // entries. What stays here, unchanged: persisting the exchanged // material (`testAndPersistCredential`, the fast half — no probe, no // workflow deploy), the duplicate-callback recovery lookup - // (`recentlyConnectedCredential`, below), and writing the pending-seed - // row the deferred `/complete-setup` deploy step reads (see - // `./pending-seed.ts`). Every test seam this package's deps already + // (`recentlyConnectedCredential`, below), and firing the desired-state + // kick `/complete-setup` and the revisit probe build on (see + // `afterConnected`, below). Every test seam this package's deps already // exposed (`openrouterConnect`/`huggingfaceConnect` overrides) still // works — they're threaded into the registry entries' `oauth.exchange` // below. @@ -782,11 +805,11 @@ export function createOnboardingRoutes( } /** Runs only for a connector whose `oauth.deploysDefaultWorkflows` is - * true (both OpenRouter and Hugging Face) — writes the plaintext - * material into the pending-seed store `/complete-setup` reads, so - * the deferred workflow deploy never blocks this redirect. The - * browser gets nothing from this call: no cookie, no ciphertext, only - * the ordinary redirect — see `./pending-seed.ts`'s module comment. */ + * true (both OpenRouter and Hugging Face) — fires the desired-state + * revisit kick for the just-connected bench, so its default workflows + * converge without the redirect waiting on a deploy. Best-effort: + * the onboarding page's own `/complete-setup` call kicks again, and + * the reconciler itself is reads-only when converged. */ async function afterConnected(args: { c: import("hono").Context; connectorId: string; @@ -800,17 +823,23 @@ export function createOnboardingRoutes( const provider = onboardingOAuthProvider(args.connectorId); if (provider === undefined) { throw new Error( - `onboarding's pending-seed store only holds its own providers, not ${args.connectorId}`, + `onboarding only converges its own providers, not ${args.connectorId}`, + ); + } + if (deps.desiredStateKick === undefined) return; + try { + deps.desiredStateKick({ + tenantId: args.tenantId, + cookies: cookiesFromHeader(args.c.req.header("cookie")), + }); + } catch (cause) { + // report-error-ignore: the kick is best-effort — `/complete-setup` + // kicks again when the onboarding page lands, so a failed kick + // here only ever costs one missed reconcile. + deps.log( + `desired-state kick after OAuth connect for user ${args.userId} failed: ${cause instanceof Error ? cause.message : String(cause)}`, ); } - await deps.pendingSeedStore.put({ - userId: args.userId, - tenantId: args.tenantId, - principalId: args.principalId, - tenantDomain: args.tenantDomain, - provider, - apiKey: args.apiKey, - }); } app.route( @@ -880,9 +909,10 @@ export function createOnboardingRoutes( try { // The fast half, and only the fast half (CL-6457): persist the // credential, seed its catalog, answer. Deploying this bench's - // default workflows is the background drain's job — a request that + // default workflows never blocks this response — a request that // waits on it is the 2+ minute "Connecting…" this route exists to - // never reproduce. + // never reproduce. Convergence is kicked below; the response only + // ever reports status. const result = await runTestAndPersistCredential( parsed.baseURL !== undefined ? { ...baseCompleteCredentialArgs, baseURLOverride: parsed.baseURL } @@ -920,29 +950,25 @@ export function createOnboardingRoutes( tenantId = result.tenantId; const status = await provisioningStatus(cookies, result); - if (status.kind === "ready") { - await deps.pendingSeedStore.clear({ - userId: user.id, - tenantId: result.tenantId, - }); - return c.json(status, 200); - } + if (status.kind === "ready") return c.json(status, 200); - // The row is the drain's work item — durable, so a hub that dies - // mid-deploy resumes this bench on its next boot rather than - // stranding it half-provisioned. - await deps.pendingSeedStore.put({ - userId: user.id, - tenantId: result.tenantId, - principalId: result.principalId, - tenantDomain: result.tenantDomain, - provider: parsed.provider, - apiKey: parsed.apiKey, - ...(parsed.baseURL !== undefined - ? { baseURLOverride: parsed.baseURL } - : {}), - }); - deps.benchProvisioner?.wake(); + // Pins still missing: kick the desired-state reconcile for this + // bench (fire-and-forget — the response answers from the status + // read above, never waits on the deploy) and answer the same + // status, so the waiting surface polls `/provisioning-status` for + // live progress. + if (deps.desiredStateKick !== undefined) { + try { + deps.desiredStateKick({ tenantId: result.tenantId, cookies }); + } catch (cause) { + // report-error-ignore: the kick is best-effort — the revisit + // kick on the tenant's next visit covers it, so a failed kick + // here only ever costs one missed reconcile. + deps.log( + `desired-state kick after credential connect for user ${user.id} failed: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } return c.json(status, 200); } catch (cause) { // Neither `ProvisionError` nor `CliError` messages are safe to show @@ -965,16 +991,16 @@ export function createOnboardingRoutes( }); // Runs after the onboarding page lands — from a fresh connect - // (`outcome=connected`) or a plain reload — and drives the slow half - // the OAuth callback never runs: deploying the default workflows - // against whichever credential is already on the caller's own - // personal bench. Already-seeded is answered from a single read, no - // pending token required, so a returning fully-set-up account (or a - // second overlapping call once the first finishes) gets the same - // `seeded` answer without redoing any work. `kind: "unseeded"` (200, - // not an error) means there is nothing this call can do yet — no - // pending credential to seed with — and the caller should fall back - // to the ordinary credential step rather than treat it as a failure. + // (`outcome=connected`) or a plain reload — and answers where the + // caller's own personal bench stands. Already-seeded is answered from + // reads alone, so a returning fully-set-up account gets the same + // status without redoing any work. `kind: "unseeded"` (200, not an + // error) means the bench holds no active inference credential yet — + // nothing this call can converge — and the caller should fall back to + // the ordinary credential step rather than treat it as a failure. + // Otherwise the bench has a credential and missing pins, so this call + // kicks the desired-state reconcile (fire-and-forget) and answers the + // status for the waiting surface to poll on. app.post("/complete-setup", async (c) => { const user = c.get("user"); if (!user) { @@ -1009,24 +1035,32 @@ export function createOnboardingRoutes( tenantId = tenant.tenantId; const status = await provisioningStatus(cookies, tenant); - if (status.kind === "ready") { - await deps.pendingSeedStore.clear({ - userId: user.id, - tenantId: tenant.tenantId, - }); - return c.json(status, 200); - } + if (status.kind === "ready") return c.json(status, 200); - // Not yet provisioned, and no credential parked to provision with - // — nothing this call can do, and not a failure: the caller falls - // back to the ordinary credential step. - const pending = await deps.pendingSeedStore.read({ - userId: user.id, - tenantId: tenant.tenantId, - }); - if (pending === undefined) return c.json({ kind: "unseeded" }, 200); + // Not yet provisioned — whether there is a credential to converge + // with is read from the bench itself. None: nothing this call can + // do, and not a failure — the caller falls back to the ordinary + // credential step. + const seeded = await hasActiveInferenceCredential( + api, + cookies, + tenant.tenantId, + deps.log, + ); + if (!seeded) return c.json({ kind: "unseeded" }, 200); - deps.benchProvisioner?.wake(); + if (deps.desiredStateKick !== undefined) { + try { + deps.desiredStateKick({ tenantId: tenant.tenantId, cookies }); + } catch (cause) { + // report-error-ignore: the kick is best-effort — the revisit + // kick on the tenant's next visit covers it, so a failed kick + // here only ever costs one missed reconcile. + deps.log( + `desired-state kick from complete-setup for user ${user.id} failed: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } return c.json(status, 200); } catch (cause) { // report-error-ignore: CL-7234 — reportOnboardingError itself needs diff --git a/packages/onboarding/src/schema.ts b/packages/onboarding/src/schema.ts deleted file mode 100644 index 46b9c823ab..0000000000 --- a/packages/onboarding/src/schema.ts +++ /dev/null @@ -1,26 +0,0 @@ -// The one product table `@corbits/onboarding` owns: a single pending -// deferred-seed row per (user, tenant), keyed for upsert. Lives in its -// own `onboarding` Postgres schema, never `public` — see -// docs/package-migrations.md. See `./pending-seed.ts` for what the row -// holds and why. -import { pgSchema, primaryKey, text, timestamp } from "drizzle-orm/pg-core"; - -export const onboardingSchema = pgSchema("onboarding"); - -export const pendingSeed = onboardingSchema.table( - "pending_seed", - { - userId: text("user_id").notNull(), - tenantId: text("tenant_id").notNull(), - provider: text("provider").notNull(), - /** - * `CredentialCipher`-encrypted JSON (`principalId`, `tenantDomain`, - * `apiKey`) — see `./pending-seed.ts` for the AAD discipline. - */ - payload: text("payload").notNull(), - expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), - }, - (table) => [primaryKey({ columns: [table.userId, table.tenantId] })], -); - -export type PendingSeedRow = typeof pendingSeed.$inferSelect; diff --git a/packages/onboarding/test/bench-provisioning.test.ts b/packages/onboarding/test/bench-provisioning.test.ts deleted file mode 100644 index 4a3c7f2eca..0000000000 --- a/packages/onboarding/test/bench-provisioning.test.ts +++ /dev/null @@ -1,404 +0,0 @@ -// The background provisioner: the only thing in the system that -// converges a connected bench (CL-6457, doc-driven since CL-7584). -// Connect persists a credential and returns; this reconciles the bench -// against the tenant desired-state document afterwards, and has to hold -// three properties no HTTP request can hold for it — it is idempotent (a -// second pass over an already-seeded bench installs nothing), -// convergent (a half-provisioned bench finishes on a later pass), and -// restart-safe (a fresh process with nothing in memory picks up -// whatever the crashed one left in the pending-seed table). -import { describe, expect, test } from "bun:test"; -import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import type { CredentialCipher } from "@intx/types"; -import { DEFAULT_WORKFLOWS } from "@corbits/seeding"; -import { - createBenchProvisioner, - type BenchProvisionerDeps, -} from "../src/bench-provisioning"; -import { - createInMemoryPendingSeedStore, - PENDING_SEED_SCAN_LIMIT, - type PendingSeed, - type PendingSeedStore, -} from "../src/pending-seed"; - -type ReconcileArgsLike = Parameters< - NonNullable ->[0]; - -const TEST_KEY = Buffer.alloc(32, 33); -function testCipher(): CredentialCipher { - return createEnvKeyCredentialCipher(TEST_KEY); -} - -const SEED: PendingSeed = { - userId: "user_1", - tenantId: "ten_1", - principalId: "prn_1", - tenantDomain: "user-1.bench.local", - provider: "anthropic", - apiKey: "sk-ant-connected", -}; - -const ALL_WORKFLOWS = DEFAULT_WORKFLOWS.map((workflow) => workflow.assetName); - -function readyReport(tenantId: string) { - return { - tenantId, - ready: true as const, - pins: ALL_WORKFLOWS.map((name) => ({ - name, - kind: "workflow" as const, - status: "installed" as const, - })), - }; -} - -function blockedReport(tenantId: string) { - return { - tenantId, - ready: false as const, - pins: [ - { - name: "assistant", - kind: "workflow" as const, - status: "blocked" as const, - }, - ], - }; -} - -function failedReport(tenantId: string) { - return { - tenantId, - ready: false as const, - pins: [ - { - name: "assistant", - kind: "workflow" as const, - status: "failed" as const, - }, - ], - }; -} - -/** A provisioner wired entirely to fakes: no hub, no sidecar, no git - * push. `deployedByTenant` is the fake bench state the reconcile seam - * reads and writes, so idempotence and convergence are observable as - * call counts rather than asserted by inspection. */ -function harness( - overrides: Partial & { store?: PendingSeedStore } = {}, -) { - const store = overrides.store ?? createInMemoryPendingSeedStore(testCipher()); - const deployedByTenant = new Map(); - const calls = { - reconcile: 0, - sessionFor: 0, - }; - const logged: string[] = []; - - const deps: BenchProvisionerDeps = { - api: (async () => { - throw new Error("the fakes below stand in for every hub call"); - }) as unknown as BenchProvisionerDeps["api"], - hubUrl: "https://bench.example.com", - store, - pushWorkflow: async () => ({ - outcome: "pushed" as const, - commitSha: "a".repeat(40), - }), - sessionFor: async () => { - calls.sessionFor += 1; - return ["better-auth.session_token=minted"]; - }, - log: (line) => logged.push(line), - reconcileFn: async (args) => { - calls.reconcile += 1; - deployedByTenant.set(args.tenant.tenantId, [...ALL_WORKFLOWS]); - return readyReport(args.tenant.tenantId); - }, - ...overrides, - }; - - return { - provisioner: createBenchProvisioner(deps), - store, - calls, - logged, - deployedByTenant, - }; -} - -describe("createBenchProvisioner", () => { - test("converges a freshly connected bench and clears its pending row", async () => { - const { provisioner, store, calls, deployedByTenant } = harness(); - await store.put(SEED); - - const report = await provisioner.drainOnce(); - - expect(calls.reconcile).toBe(1); - expect(deployedByTenant.get("ten_1")).toEqual(ALL_WORKFLOWS); - expect(report).toMatchObject({ converged: 1, truncated: false }); - expect( - await store.read({ userId: "user_1", tenantId: "ten_1" }), - ).toBeUndefined(); - }); - - test("is idempotent: a second drain over an already-seeded bench installs nothing", async () => { - const { provisioner, store, calls } = harness(); - await store.put(SEED); - - await provisioner.drainOnce(); - // The row is gone after the first pass, so re-arm it the way a - // duplicate connect would and prove the reconcile short-circuits. - await store.put(SEED); - await provisioner.drainOnce(); - - expect(calls.reconcile).toBe(2); - }); - - test("reconciling a bench someone else already converged reports ready and still clears the row", async () => { - const { provisioner, store } = harness(); - await store.put(SEED); - - const report = await provisioner.drainOnce(); - - expect(report).toMatchObject({ converged: 1, truncated: false }); - expect( - await store.read({ userId: "user_1", tenantId: "ten_1" }), - ).toBeUndefined(); - }); - - test("a blocked bench keeps its row and converges on a later pass", async () => { - let attempt = 0; - const { provisioner, store } = harness({ - reconcileFn: async (args) => { - attempt += 1; - if (attempt > 1) { - return readyReport(args.tenant.tenantId); - } - return blockedReport(args.tenant.tenantId); - }, - }); - await store.put(SEED); - - const first = await provisioner.drainOnce(); - expect(first).toMatchObject({ pending: 1 }); - // The row survives precisely so the next pass can finish the job. - expect(await store.read({ userId: "user_1", tenantId: "ten_1" })).toEqual( - SEED, - ); - - const second = await provisioner.drainOnce({ ignoreBackoff: true }); - expect(second).toMatchObject({ converged: 1 }); - expect( - await store.read({ userId: "user_1", tenantId: "ten_1" }), - ).toBeUndefined(); - }); - - test("a reconcile failure leaves the row for the next pass rather than losing the bench", async () => { - let attempt = 0; - const { provisioner, store, logged } = harness({ - reconcileFn: async (args) => { - attempt += 1; - if (attempt === 1) throw new Error("sidecar exploded"); - return readyReport(args.tenant.tenantId); - }, - }); - await store.put(SEED); - - const first = await provisioner.drainOnce(); - expect(first).toMatchObject({ failed: 1 }); - expect(await store.read({ userId: "user_1", tenantId: "ten_1" })).toEqual( - SEED, - ); - expect(logged.some((line) => line.includes("sidecar exploded"))).toBe(true); - - const second = await provisioner.drainOnce({ ignoreBackoff: true }); - expect(second).toMatchObject({ converged: 1 }); - }); - - test("a failed-pin report counts as failed for backoff purposes", async () => { - const { provisioner, store } = harness({ - reconcileFn: async (args) => failedReport(args.tenant.tenantId), - }); - await store.put(SEED); - - await provisioner.drainOnce(); - const held = await provisioner.drainOnce(); - - expect(held).toMatchObject({ deferred: 1 }); - expect(await store.read({ userId: "user_1", tenantId: "ten_1" })).toEqual( - SEED, - ); - }); - - test("restart-resume: a fresh provisioner with empty memory finishes what a crashed one left behind", async () => { - const store = createInMemoryPendingSeedStore(testCipher()); - // The "crashed" process: it wrote the row, then died before its - // deploy ever ran. - await store.put(SEED); - - // A brand-new provisioner — no in-flight map, no backoff state, no - // knowledge of the connect that wrote the row — boots and drains. - const { provisioner, calls, deployedByTenant } = harness({ store }); - const report = await provisioner.drainOnce(); - - expect(calls.reconcile).toBe(1); - expect(deployedByTenant.get("ten_1")).toEqual(ALL_WORKFLOWS); - expect(report).toMatchObject({ converged: 1, truncated: false }); - }); - - test("overlapping drains never double-provision the same bench", async () => { - let inFlight = 0; - let maxConcurrent = 0; - let reconciles = 0; - const { provisioner, store } = harness({ - reconcileFn: async (args) => { - reconciles += 1; - inFlight += 1; - maxConcurrent = Math.max(maxConcurrent, inFlight); - await new Promise((resolve) => setTimeout(resolve, 20)); - inFlight -= 1; - return readyReport(args.tenant.tenantId); - }, - }); - await store.put(SEED); - - await Promise.all([provisioner.drainOnce(), provisioner.drainOnce()]); - - expect(maxConcurrent).toBe(1); - expect(reconciles).toBe(1); - }); - - test("a blocked report holds the pending row as pending, not failed", async () => { - const { provisioner, store } = harness({ - reconcileFn: async (args) => blockedReport(args.tenant.tenantId), - }); - await store.put(SEED); - - const report = await provisioner.drainOnce(); - - expect(report).toMatchObject({ pending: 1, failed: 0 }); - expect(await store.read({ userId: "user_1", tenantId: "ten_1" })).toEqual( - SEED, - ); - }); - - test("a bench whose user has no mintable session is left alone, not dropped", async () => { - const { provisioner, store, calls } = harness({ - sessionFor: async () => undefined, - }); - await store.put(SEED); - - const report = await provisioner.drainOnce(); - - expect(calls.reconcile).toBe(0); - expect(report).toMatchObject({ failed: 1 }); - expect(await store.read({ userId: "user_1", tenantId: "ten_1" })).toEqual( - SEED, - ); - }); - - test("a permanently-failing bench's backoff is reclaimed once its row is gone, not only on success", async () => { - // Simulates the CL-7233 orphan case: the pending_seed row disappears - // (TTL-expiry or otherwise) while the bench is still backed off from - // repeated failures — the retry-hold bookkeeping must not survive - // the row that justified it. - const { provisioner, store } = harness({ - reconcileFn: async () => { - throw new Error("sidecar still down"); - }, - }); - await store.put(SEED); - - const first = await provisioner.drainOnce(); - expect(first).toMatchObject({ failed: 1 }); - // Confirm the hold is actually in effect before the row disappears — - // otherwise this test would pass for the wrong reason. - const stillBackedOff = await provisioner.drainOnce(); - expect(stillBackedOff).toMatchObject({ deferred: 1 }); - - // The row is gone by some path other than this provisioner's own - // convergence (an admin action, or read-time TTL expiry elsewhere). - await store.clear({ userId: "user_1", tenantId: "ten_1" }); - const afterRowGone = await provisioner.drainOnce(); - expect(afterRowGone).toMatchObject({ - converged: 0, - pending: 0, - failed: 0, - deferred: 0, - }); - - // A brand-new connect for the same user/tenant must not inherit the - // dead bench's backoff — without eviction, this would come back - // deferred instead of attempted. - await store.put(SEED); - const freshAttempt = await provisioner.drainOnce(); - expect(freshAttempt).toMatchObject({ failed: 1 }); - }); - - test("drains every waiting bench in one tick, not just the first", async () => { - const { provisioner, store, calls } = harness(); - await store.put(SEED); - await store.put({ ...SEED, userId: "user_2", tenantId: "ten_2" }); - - const report = await provisioner.drainOnce(); - - expect(calls.reconcile).toBe(2); - expect(report).toMatchObject({ converged: 2, truncated: false }); - }); - - test("DrainReport.truncated is true when more due rows remain behind this tick's page", async () => { - const seen = new Set(); - const { provisioner, store } = harness({ - reconcileFn: async (args: ReconcileArgsLike) => { - seen.add(args.tenant.tenantId); - return blockedReport(args.tenant.tenantId); - }, - }); - for (let index = 0; index < PENDING_SEED_SCAN_LIMIT + 3; index += 1) { - await store.put({ - ...SEED, - userId: `user_${index}`, - tenantId: `ten_${index}`, - }); - } - - const first = await provisioner.drainOnce(); - expect(first.truncated).toBe(true); - expect(first.pending).toBe(PENDING_SEED_SCAN_LIMIT); - expect(seen.size).toBe(PENDING_SEED_SCAN_LIMIT); - - const second = await provisioner.drainOnce(); - expect(second.truncated).toBe(false); - expect(second.pending).toBe(3); - expect(seen.size).toBe(PENDING_SEED_SCAN_LIMIT + 3); - }); - - test("rows past the scan limit still get a drain pass across ticks, even when the first page never converges", async () => { - const seen = new Set(); - const { provisioner, store } = harness({ - reconcileFn: async (args: ReconcileArgsLike) => { - seen.add(args.tenant.tenantId); - return blockedReport(args.tenant.tenantId); - }, - }); - const total = PENDING_SEED_SCAN_LIMIT + 3; - for (let index = 0; index < total; index += 1) { - await store.put({ - ...SEED, - userId: `user_${index}`, - tenantId: `ten_${index}`, - }); - } - - await provisioner.drainOnce(); - await provisioner.drainOnce(); - - expect(seen.size).toBe(total); - for (let index = 0; index < total; index += 1) { - expect(seen.has(`ten_${index}`)).toBe(true); - } - }); -}); diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index 64eec9f9d1..fd712672fa 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -1588,9 +1588,10 @@ describe("completeCredentialSetup", () => { // CL-6264: the credential-persist half already succeeded (the tenant, // principal, and credential are real and durable) by the time the - // deploy step hits a sidecar-unavailable failure, so `/complete`'s own - // route can write a pending-seed row for the retry path — it needs - // `principalId`/`tenantDomain` threaded all the way out here to do it. + // deploy step hits a sidecar-unavailable failure, so the retry path — + // the desired-state reconcile's redeploy — still has a durable tenant + // identity to deploy under: it needs `principalId`/`tenantDomain` + // threaded all the way out here to do it. test("a sidecar-unavailable deploy still reports the durable tenant identity, not just deployed/pending", async () => { const api: ApiCall = async (method, path) => { if (method === "GET" && path === "/api/me/principals") { @@ -1848,10 +1849,10 @@ describe("ensureSeeded (the slow half)", () => { }); test("deploys the setup agent before anything else, so a waiting person can start as soon as she is live", async () => { - // CL-6462. This is the deploy path the background drain runs - // (bench-provisioning's `runOnce` → `ensureSeeded` → `seedTenant`), - // and `seedTenant` works through the list in order at ~20s each — so - // the order handed in here is the order a fresh signup experiences. + // CL-6462. This is the deploy path the explicit setup flow runs + // (`ensureSeeded` → `seedTenant`), and `seedTenant` works through + // the list in order at ~20s each — so the order handed in here is + // the order a fresh signup experiences. // CL-7074 narrowed DEFAULT_WORKFLOWS to just the setup agent, so // "before anything else" is now trivially satisfied by being the // only entry — this still pins that fact rather than assuming it. diff --git a/packages/onboarding/test/complete-setup-routes.test.ts b/packages/onboarding/test/complete-setup-routes.test.ts index c22918c893..c848d61359 100644 --- a/packages/onboarding/test/complete-setup-routes.test.ts +++ b/packages/onboarding/test/complete-setup-routes.test.ts @@ -1,44 +1,24 @@ // `POST /complete-setup` reports where a bench stands; it never -// deploys. CL-6457 moved every workflow deploy onto the background -// drain in `../src/bench-provisioning.ts`, because a request that -// waits on deploys is the two-minute "Connecting…" onboarding was -// stuck behind. What is left here is a status reporter over two cheap -// hub reads, and it has to answer three cases correctly: every default -// workflow live (`ready` — clear the pending row, the drain has -// nothing left to do), still deploying with a pending row parked -// (`provisioning` — nudge the drain and leave the row alone, it is the -// drain's durable work item), and nothing live with no row to work -// from (`unseeded`, a 200 and not an error, telling the caller to fall -// back to the ordinary credential step). +// deploys. CL-6457 moved every workflow deploy off the request path, +// and CL-7586 deleted the pending-seed drain entirely: there is no +// parked row, no deferred deploy step, no nudge target. What is left +// here is a status reporter over hub reads, and it has to answer three +// cases correctly: every default workflow live (`ready` — answered +// from the read alone), a credential with pins still missing +// (`provisioning` — kick the desired-state reconcile fire-and-forget +// and answer the status for the waiting surface to poll on), and +// nothing to converge with (`unseeded`, a 200 and not an error, +// telling the caller to fall back to the ordinary credential step). // -// The deploying itself — its idempotency, its retries, its -// half-provisioned recovery — is covered where it now lives, in -// `./bench-provisioning.test.ts`. -// -// CL-6031 moved the pending credential off the browser: what used to -// be a sealed HttpOnly cookie is now a row in -// `createInMemoryPendingSeedStore` (the same store shape -// `createDrizzlePendingSeedStore` gives Postgres — see -// `../src/pending-seed.test.ts` for that logic's own direct coverage), -// written by calling `store.put(...)` the way the OAuth callback would, -// instead of round-tripping a cookie header. +// The converging itself — the desired-state reconcile's idempotency, +// its retries, its half-provisioned recovery — is covered where it +// now lives, in `./desired-state.test.ts`. import { describe, expect, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import type { CredentialCipher } from "@intx/types"; -import { DEFAULT_WORKFLOWS } from "@corbits/seeding"; +import { DEFAULT_WORKFLOWS, inferenceCredentialName } from "@corbits/seeding"; import { createOnboardingRoutes } from "../src/routes"; -import { - createInMemoryPendingSeedStore, - type PendingSeedStore, -} from "../src/pending-seed"; - -const TEST_KEY = Buffer.alloc(32, 21); -function testCipher(): CredentialCipher { - return createEnvKeyCredentialCipher(TEST_KEY); -} // These tests never exercise the genesis-or-join join path — the stub // satisfies the required tenancy wiring without standing up a DB. @@ -101,20 +81,32 @@ function principalsRoute(hub: Hono) { ); } -async function withPendingSeed( - store: PendingSeedStore, - args: { userId?: string; ttlMs?: number } = {}, -): Promise { - await store.put( - { - userId: args.userId ?? "user_1", - tenantId: TENANT_ID, - principalId: PRINCIPAL_ID, - tenantDomain: TENANT_DOMAIN, - provider: "openrouter", - apiKey: "sk-or-v1-minted", - }, - args.ttlMs !== undefined ? { ttlMs: args.ttlMs } : {}, +/** The one row `/complete-setup`'s credential check looks for: an + * active credential under the name the setup catalog persists + * inference keys as. */ +function activeCredentialRow() { + return { + id: "cre_1", + tenantId: TENANT_ID, + providerId: "prv_1", + name: inferenceCredentialName("anthropic"), + type: "api_key", + status: "active", + metadata: null, + createdAt: TIMESTAMP, + updatedAt: TIMESTAMP, + }; +} + +function credentialsRoute( + hub: Hono, + args: { inferenceCredential?: boolean } = {}, +) { + hub.get(`/api/tenants/${TENANT_ID}/credentials`, (c) => + c.json({ + data: args.inferenceCredential === true ? [activeCredentialRow()] : [], + nextCursor: null, + }), ); } @@ -132,7 +124,6 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: createInMemoryPendingSeedStore(testCipher()), }), ); @@ -160,7 +151,6 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: createInMemoryPendingSeedStore(testCipher()), }), ); @@ -223,7 +213,6 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: createInMemoryPendingSeedStore(testCipher()), }), ); @@ -265,7 +254,7 @@ describe("POST /complete-setup", () => { })), ), ); - let ensureSeededCalls = 0; + const kicks: string[] = []; const server = Bun.serve({ port: 0, fetch: hub.fetch }); try { const app = mountAuthenticated( @@ -278,16 +267,14 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: createInMemoryPendingSeedStore(testCipher()), - ensureSeededFn: async () => { - ensureSeededCalls += 1; - return { kind: "seeded", workflows: [] }; + desiredStateKick: ({ tenantId }) => { + kicks.push(tenantId); }, }), ); - // No pending-seed row at all — an already-seeded bench must - // answer from the read alone, no pending row required. + // An already-seeded bench answers from the read alone — and a + // `ready` read kicks nothing, there is nothing to converge. const response = await app.request("/api/onboarding/complete-setup", { method: "POST", }); @@ -305,13 +292,13 @@ describe("POST /complete-setup", () => { DEFAULT_WORKFLOWS.map((w) => w.assetName).sort(), ); expect(body.pending).toEqual([]); - expect(ensureSeededCalls).toBe(0); + expect(kicks).toEqual([]); } finally { server.stop(true); } }); - test("unseeded with no pending row reports unseeded, not an error", async () => { + test("nothing live and no credential reports unseeded, not an error", async () => { const hub = new Hono(); principalsRoute(hub); hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json([])); @@ -330,7 +317,6 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: createInMemoryPendingSeedStore(testCipher()), }), ); @@ -346,17 +332,18 @@ describe("POST /complete-setup", () => { } }); - test("a bench still deploying reports provisioning and never deploys inline", async () => { + test("a credential with nothing live reports provisioning and kicks without waiting", async () => { const hub = new Hono(); principalsRoute(hub); + credentialsRoute(hub, { inferenceCredential: true }); hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json([])); hub.get(`/api/tenants/${TENANT_ID}/workflows/deployments`, (c) => c.json([]), ); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); + let kickStarted = false; + let kickFinished = false; try { - let ensureSeededCalls = 0; const app = mountAuthenticated( createOnboardingRoutes({ tenancy: emptyHubTenancy, @@ -367,18 +354,22 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, - ensureSeededFn: async () => { - ensureSeededCalls += 1; - return { kind: "seeded", workflows: [] }; + desiredStateKick: () => { + kickStarted = true; + void new Promise((resolve) => setTimeout(resolve, 5_000)).then( + () => { + kickFinished = true; + }, + ); }, }), ); - await withPendingSeed(pendingSeedStore); + const startedAt = Date.now(); const response = await app.request("/api/onboarding/complete-setup", { method: "POST", }); + const elapsedMs = Date.now() - startedAt; expect(response.status).toBe(200); const body = (await response.json()) as { @@ -399,33 +390,29 @@ describe("POST /complete-setup", () => { pending: DEFAULT_WORKFLOWS.map((w) => w.assetName), steps: expect.any(Array), }); - // The point of CL-6457: a pending row in front of it is not a - // licence to deploy on the request path. The seam still exists, - // it just belongs to the drain now. - expect(ensureSeededCalls).toBe(0); - // And the row stays exactly where it is — it is the drain's - // durable work item, not this request's scratch state. - const stillThere = await pendingSeedStore.read({ - userId: "user_1", - tenantId: TENANT_ID, - }); - expect(stillThere).toBeDefined(); + // The point of CL-6457, kept by CL-7586: a credential waiting on + // pins is not a licence to do the converge on the request path. + // The kick started, but the route never waited on its five + // seconds — it answered from the status read alone. + expect(elapsedMs).toBeLessThan(1_000); + expect(kickStarted).toBe(true); + expect(kickFinished).toBe(false); } finally { server.stop(true); } }); - test("a still-provisioning bench nudges the drain instead of waiting on it", async () => { + test("a still-provisioning bench kicks the reconcile instead of waiting on it", async () => { const hub = new Hono(); principalsRoute(hub); + credentialsRoute(hub, { inferenceCredential: true }); hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json([])); hub.get(`/api/tenants/${TENANT_ID}/workflows/deployments`, (c) => c.json([]), ); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); try { - let wakes = 0; + const kicks: string[] = []; const app = mountAuthenticated( createOnboardingRoutes({ tenancy: emptyHubTenancy, @@ -436,15 +423,11 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, - benchProvisioner: { - wake: () => { - wakes += 1; - }, + desiredStateKick: ({ tenantId }) => { + kicks.push(tenantId); }, }), ); - await withPendingSeed(pendingSeedStore); const response = await app.request("/api/onboarding/complete-setup", { method: "POST", @@ -453,178 +436,27 @@ describe("POST /complete-setup", () => { expect(response.status).toBe(200); const body = (await response.json()) as { kind: string }; expect(body.kind).toBe("provisioning"); - // Without the nudge a freshly parked row waits out the drain's - // whole tick interval before anything happens — the reason a - // waiting onboarding page calls this route at all. - expect(wakes).toBe(1); - } finally { - server.stop(true); - } - }); - - test("an already fully seeded bench also clears a stray pending row, not just the never-written case", async () => { - const hub = new Hono(); - principalsRoute(hub); - hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => - c.json( - DEFAULT_WORKFLOWS.map((workflow, index) => ({ - id: `ast_${index}`, - tenantId: TENANT_ID, - kind: "workflow", - name: workflow.assetName, - displayName: workflow.displayName, - creatorPrincipalId: PRINCIPAL_ID, - createdAt: TIMESTAMP, - updatedAt: TIMESTAMP, - origin: { tenantId: TENANT_ID, direct: true }, - })), - ), - ); - hub.get(`/api/tenants/${TENANT_ID}/workflows/deployments`, (c) => - c.json( - DEFAULT_WORKFLOWS.map((_workflow, index) => ({ - definitionAssetId: `ast_${index}`, - status: "deployed", - })), - ), - ); - const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); - try { - const app = mountAuthenticated( - createOnboardingRoutes({ - tenancy: emptyHubTenancy, - defaultTenantSlug: "workbench", - hubUrl: `http://localhost:${server.port}`, - pushWorkflow: async () => ({ - outcome: "pushed" as const, - commitSha: "a".repeat(40), - }), - log: () => undefined, - pendingSeedStore, - }), - ); - - // The drain finished this bench a moment ago but its row is - // still sitting there — a crash between the last deploy and the - // clear, or a connect that parked a fresh row over an already - // complete bench. A `ready` read is the authority: the row has no - // work left in it and must not be left for the drain to pick up - // again. - await withPendingSeed(pendingSeedStore); - - const response = await app.request("/api/onboarding/complete-setup", { - method: "POST", - }); - - expect(response.status).toBe(200); - const body = (await response.json()) as { kind: string }; - expect(body.kind).toBe("ready"); - const stillThere = await pendingSeedStore.read({ - userId: "user_1", - tenantId: TENANT_ID, - }); - expect(stillThere).toBeUndefined(); + // Without the kick a credential with missing pins waits out the + // reconcile's whole tick interval before anything happens — the + // reason a waiting onboarding page calls this route at all. + expect(kicks).toEqual([TENANT_ID]); } finally { server.stop(true); } }); - test("an expired pending row is cleared rather than left to linger unused", async () => { - const hub = new Hono(); - principalsRoute(hub); - hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json([])); - hub.get(`/api/tenants/${TENANT_ID}/workflows/deployments`, (c) => - c.json([]), - ); - const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); - try { - await withPendingSeed(pendingSeedStore, { ttlMs: -1 }); - const app = mountAuthenticated( - createOnboardingRoutes({ - tenancy: emptyHubTenancy, - defaultTenantSlug: "workbench", - hubUrl: `http://localhost:${server.port}`, - pushWorkflow: async () => ({ - outcome: "pushed" as const, - commitSha: "a".repeat(40), - }), - log: () => undefined, - pendingSeedStore, - }), - ); - - const response = await app.request("/api/onboarding/complete-setup", { - method: "POST", - }); - - expect(response.status).toBe(200); - const body = (await response.json()) as { kind: string }; - expect(body.kind).toBe("unseeded"); - const stillThere = await pendingSeedStore.read({ - userId: "user_1", - tenantId: TENANT_ID, - }); - expect(stillThere).toBeUndefined(); - } finally { - server.stop(true); - } - }); - - test("a pending row written for a different user is invisible to this session", async () => { - const hub = new Hono(); - principalsRoute(hub); - hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json([])); - hub.get(`/api/tenants/${TENANT_ID}/workflows/deployments`, (c) => - c.json([]), - ); - const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); - try { - await withPendingSeed(pendingSeedStore, { userId: "someone_else" }); - let ensureSeededCalls = 0; - const app = mountAuthenticated( - createOnboardingRoutes({ - tenancy: emptyHubTenancy, - defaultTenantSlug: "workbench", - hubUrl: `http://localhost:${server.port}`, - pushWorkflow: async () => ({ - outcome: "pushed" as const, - commitSha: "a".repeat(40), - }), - log: () => undefined, - pendingSeedStore, - ensureSeededFn: async () => { - ensureSeededCalls += 1; - return { kind: "seeded", workflows: [] }; - }, - }), - ); - - const response = await app.request("/api/onboarding/complete-setup", { - method: "POST", - }); - - expect(response.status).toBe(200); - const body = (await response.json()) as { kind: string }; - expect(body.kind).toBe("unseeded"); - expect(ensureSeededCalls).toBe(0); - } finally { - server.stop(true); - } - }); - - test("two overlapping calls both report provisioning and neither deploys", async () => { + test("two overlapping calls both report provisioning and each kicks", async () => { // Two "finish setup" requests racing (a double effect fire, a // retried fetch) used to be this route's sharpest edge, because // both would deploy. Post-CL-6457 the route deploys nothing at all, // so the only thing left to hold is that overlapping callers get // the same honest status and still start no work of their own. The - // dedupe that matters now lives one layer down and is covered - // there: "overlapping drains never double-deploy the same bench" in - // ./bench-provisioning.test.ts. + // route deliberately does not dedupe the kicks either — one + // fire-and-forget nudge per caller, and the reconcile itself + // converges (covered where it lives, in + // `./desired-state-reconcile.test.ts`). const hub = new Hono(); + credentialsRoute(hub, { inferenceCredential: true }); // Deterministic overlap, not a race against real wall-clock // scheduling: `findPersonalTenant` is the first hub call each // `/complete-setup` request makes, so gating it on "both requests @@ -671,9 +503,8 @@ describe("POST /complete-setup", () => { c.json([]), ); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); try { - let ensureSeededCalls = 0; + const kicks: string[] = []; const app = mountAuthenticated( createOnboardingRoutes({ tenancy: emptyHubTenancy, @@ -684,14 +515,11 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, - ensureSeededFn: async () => { - ensureSeededCalls += 1; - return { kind: "seeded", workflows: [] }; + desiredStateKick: ({ tenantId }) => { + kicks.push(tenantId); }, }), ); - await withPendingSeed(pendingSeedStore); const [first, second] = await Promise.all([ app.request("/api/onboarding/complete-setup", { @@ -708,38 +536,35 @@ describe("POST /complete-setup", () => { const secondBody = (await second.json()) as { kind: string }; expect(firstBody.kind).toBe("provisioning"); expect(secondBody.kind).toBe("provisioning"); - expect(ensureSeededCalls).toBe(0); - // One work item, however many callers ask about it. - const stillThere = await pendingSeedStore.read({ - userId: "user_1", - tenantId: TENANT_ID, - }); - expect(stillThere).toBeDefined(); + // One kick per caller — the route never dedupes, the reconcile + // converges on its own. + expect(kicks).toEqual([TENANT_ID, TENANT_ID]); } finally { server.stop(true); } }); - // CL-6264, re-homed by CL-6457: a bench that got partway through its - // workflows must read as still provisioning, and must keep its - // pending row so the drain can finish the rest. The convergence - // itself — deploying only what is missing on a later pass — is - // covered by "a half-provisioned bench keeps its row and converges on - // a later pass" in ./bench-provisioning.test.ts. + // CL-6264, re-homed by CL-6457 and kept by CL-7586: a bench that got + // partway through its workflows must read as still provisioning, and + // must kick the reconcile so the rest actually goes live. The + // convergence itself — deploying only what is missing on a later + // pass — is covered by "a non-sidecar failure reports failed, and a + // re-run can converge" in ./desired-state-reconcile.test.ts. // // CL-7074 narrowed DEFAULT_WORKFLOWS to just the setup agent, so // "partway through" no longer means "one of several live, the rest // pending" — it means the one default workflow's asset exists but // has not gone live yet (the sidecar push landed, the deploy // confirmation has not). - test("a bench whose agent is not yet live keeps its pending row for the drain", async () => { + test("a bench whose agent is not yet live reports provisioning and kicks", async () => { const liveWorkflow = DEFAULT_WORKFLOWS[0]; if (liveWorkflow === undefined) { throw new Error("DEFAULT_WORKFLOWS is empty"); } const hub = new Hono(); principalsRoute(hub); + credentialsRoute(hub, { inferenceCredential: true }); hub.get(`/api/tenants/${TENANT_ID}/assets`, (c) => c.json( DEFAULT_WORKFLOWS.map((workflow, index) => ({ @@ -760,8 +585,8 @@ describe("POST /complete-setup", () => { c.json([]), ); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const pendingSeedStore = createInMemoryPendingSeedStore(testCipher()); try { + const kicks: string[] = []; const app = mountAuthenticated( createOnboardingRoutes({ tenancy: emptyHubTenancy, @@ -772,10 +597,11 @@ describe("POST /complete-setup", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, + desiredStateKick: ({ tenantId }) => { + kicks.push(tenantId); + }, }), ); - await withPendingSeed(pendingSeedStore); const response = await app.request("/api/onboarding/complete-setup", { method: "POST", @@ -801,13 +627,8 @@ describe("POST /complete-setup", () => { steps: expect.any(Array), }); - // Not finished yet — clearing the row here would strand the - // remaining agents with nothing left to deploy them. - const stillThere = await pendingSeedStore.read({ - userId: "user_1", - tenantId: TENANT_ID, - }); - expect(stillThere).toBeDefined(); + // Not finished yet — and the kick is what finishes it. + expect(kicks).toEqual([TENANT_ID]); } finally { server.stop(true); } diff --git a/packages/onboarding/test/connect-deploys-nothing.test.ts b/packages/onboarding/test/connect-deploys-nothing.test.ts index 6ae3f8f36c..c9d507a3f8 100644 --- a/packages/onboarding/test/connect-deploys-nothing.test.ts +++ b/packages/onboarding/test/connect-deploys-nothing.test.ts @@ -2,27 +2,23 @@ // layer. The live repro: pasting a key sat on "Connecting…" for 2+ // minutes because `POST /complete` deployed five default workflows — // ~20s each — before it answered. The fix is structural, so the test is -// too: the deploy step is handed in as a seam that takes five seconds -// and records when it ran, and the route has to answer long before it -// could possibly have waited on that. A route that ever awaits a deploy -// again fails here on the clock, not on a mock's call count alone. +// too: the follow-up work is handed in as a kick seam that takes five +// seconds and records when it ran, and the route has to answer long +// before it could possibly have waited on that. A route that ever awaits +// a deploy again fails here on the clock, not on a mock's call count +// alone. CL-7586 deleted the pending-seed drain the kick used to wake: +// the kick now names the tenant for the desired-state reconcile, and no +// row is parked anywhere. import { describe, expect, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { createEnvKeyCredentialCipher } from "@intx/crypto"; -import type { CredentialCipher } from "@intx/types"; -import { DEFAULT_WORKFLOWS, SETUP_AGENT_ASSET_NAME } from "@corbits/seeding"; -import { createOnboardingRoutes } from "../src/routes"; import { - createInMemoryPendingSeedStore, - type PendingSeedStore, -} from "../src/pending-seed"; - -const TEST_KEY = Buffer.alloc(32, 44); -function testCipher(): CredentialCipher { - return createEnvKeyCredentialCipher(TEST_KEY); -} + DEFAULT_WORKFLOWS, + SETUP_AGENT_ASSET_NAME, + inferenceCredentialName, +} from "@corbits/seeding"; +import { createOnboardingRoutes } from "../src/routes"; const TENANT_ID = "ten_1"; const PRINCIPAL_ID = "prn_1"; @@ -41,7 +37,11 @@ function asUser(): MiddlewareHandler { * traffic would have to go somewhere else entirely — and the deploy seam * below proves it never even starts. */ function fakeHub( - args: { seededWorkflows?: string[]; tenantSlug?: string } = {}, + args: { + seededWorkflows?: string[]; + tenantSlug?: string; + inferenceCredential?: boolean; + } = {}, ) { const seeded = args.seededWorkflows ?? []; const hub = new Hono(); @@ -118,6 +118,12 @@ function fakeHub( }, ]), ); + hub.get("/api/tenants/:id/credentials", (c) => + c.json({ + data: args.inferenceCredential === true ? [activeCredentialRow()] : [], + nextCursor: null, + }), + ); hub.get("/api/tenants/:id/skills/:name", (c) => c.json({ name: c.req.param("name") }), ); @@ -132,11 +138,26 @@ function fakeHub( return { hub, requests }; } +/** The one row `/complete-setup`'s credential check looks for: an + * active credential under the name `seedCatalog` persists inference + * keys as. */ +function activeCredentialRow() { + return { + id: "cre_1", + tenantId: TENANT_ID, + providerId: "prv_1", + name: inferenceCredentialName("anthropic"), + type: "api_key", + status: "active", + metadata: null, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }; +} + function routeDeps(args: { hubUrl: string; - store: PendingSeedStore; - onDeploy?: () => void; - deployDelayMs?: number; + onKick?: (tenantId: string) => void; }) { return { hubUrl: args.hubUrl, @@ -156,7 +177,6 @@ function routeDeps(args: { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore: args.store, testAndPersistCredentialFn: async () => ({ kind: "connected" as const, tenantId: TENANT_ID, @@ -164,13 +184,17 @@ function routeDeps(args: { principalId: PRINCIPAL_ID, tenantDomain: TENANT_DOMAIN, }), - ensureSeededFn: async () => { - args.onDeploy?.(); - await new Promise((resolve) => - setTimeout(resolve, args.deployDelayMs ?? 0), - ); - return { kind: "seeded" as const, workflows: ALL_WORKFLOWS }; - }, + // CL-7586 deleted the pending-seed drain: the route hands follow-up + // work to the desired-state reconcile as a fire-and-forget kick + // naming the tenant. Tests hand in slow kicks to prove the route + // answers without waiting on them. + ...(args.onKick === undefined + ? {} + : { + desiredStateKick: ({ tenantId }: { tenantId: string }) => { + args.onKick?.(tenantId); + }, + }), }; } @@ -182,20 +206,23 @@ function mountAuthenticated(routes: Hono): Hono { } describe("POST /complete — connecting deploys nothing", () => { - test("answers in a moment even when a deploy would take five seconds", async () => { + test("answers in a moment even when the kicked reconcile would take five seconds", async () => { const { hub, requests } = fakeHub(); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); - let deployStarted = false; + let kickStarted = false; + let kickFinished = false; try { const app = mountAuthenticated( createOnboardingRoutes( routeDeps({ hubUrl: `http://localhost:${server.port}`, - store, - deployDelayMs: 5_000, - onDeploy: () => { - deployStarted = true; + onKick: () => { + kickStarted = true; + void new Promise((resolve) => setTimeout(resolve, 5_000)).then( + () => { + kickFinished = true; + }, + ); }, }), ), @@ -211,8 +238,10 @@ describe("POST /complete — connecting deploys nothing", () => { expect(response.status).toBe(200); expect(elapsedMs).toBeLessThan(1_000); - // The response never waited on a deploy — the whole point. - expect(deployStarted).toBe(false); + // The response never waited on the kicked reconcile — the whole + // point: the kick started, but its five seconds are still running. + expect(kickStarted).toBe(true); + expect(kickFinished).toBe(false); // And nothing deploy-shaped was even attempted against the hub. expect( requests.filter( @@ -233,11 +262,10 @@ describe("POST /complete — connecting deploys nothing", () => { test("reports the bench as provisioning, naming the agents still to come", async () => { const { hub } = fakeHub(); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); @@ -262,33 +290,32 @@ describe("POST /complete — connecting deploys nothing", () => { } }); - test("hands the drain a pending row carrying the key it will deploy against", async () => { + test("kicks the desired-state reconcile for the bench instead of parking a row", async () => { const { hub } = fakeHub(); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); + const kickedTenants: string[] = []; try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ + hubUrl: `http://localhost:${server.port}`, + onKick: (tenantId) => { + kickedTenants.push(tenantId); + }, + }), ), ); - await app.request("/api/onboarding/complete", { + const response = await app.request("/api/onboarding/complete", { method: "POST", body: JSON.stringify({ provider: "anthropic", apiKey: "sk-ant-x" }), headers: { "content-type": "application/json" }, }); - expect( - await store.read({ userId: "user_1", tenantId: TENANT_ID }), - ).toEqual({ - userId: "user_1", - tenantId: TENANT_ID, - principalId: PRINCIPAL_ID, - tenantDomain: TENANT_DOMAIN, - provider: "anthropic", - apiKey: "sk-ant-x", - }); + expect(response.status).toBe(200); + // No row is parked anywhere: the reconcile learns the tenant from + // the kick's arguments, not from a drain table. + expect(kickedTenants).toEqual([TENANT_ID]); } finally { server.stop(true); } @@ -297,11 +324,16 @@ describe("POST /complete — connecting deploys nothing", () => { test("an already-provisioned bench reconnecting reports ready, with nothing left pending", async () => { const { hub } = fakeHub({ seededWorkflows: ALL_WORKFLOWS }); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); + const kickedTenants: string[] = []; try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ + hubUrl: `http://localhost:${server.port}`, + onKick: (tenantId) => { + kickedTenants.push(tenantId); + }, + }), ), ); @@ -317,6 +349,8 @@ describe("POST /complete — connecting deploys nothing", () => { expect(body.kind).toBe("ready"); expect(body.pending).toEqual([]); + // Nothing left to reconcile, so no kick either. + expect(kickedTenants).toEqual([]); } finally { server.stop(true); } @@ -333,11 +367,10 @@ describe("GET /provisioning-status", () => { async (query, expectedStatus) => { const { hub, requests } = fakeHub(); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); const response = await app.request( @@ -365,11 +398,10 @@ describe("GET /provisioning-status", () => { test("reports provisioning, with the setup agent not yet ready, before she deploys", async () => { const { hub } = fakeHub({ seededWorkflows: [] }); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); @@ -399,11 +431,10 @@ describe("GET /provisioning-status", () => { ); const { hub } = fakeHub({ seededWorkflows: others }); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); @@ -424,11 +455,10 @@ describe("GET /provisioning-status", () => { tenantSlug: "workbench", }); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); @@ -444,29 +474,27 @@ describe("GET /provisioning-status", () => { }); }); -describe("POST /complete-setup — no longer deploys inline either", () => { - test("returns provisioning status without waiting on a deploy", async () => { - const { hub } = fakeHub(); +describe("POST /complete-setup — kicks instead of deploying", () => { + test("kicks the reconcile and answers provisioning without waiting on it", async () => { + // The key is already persisted under the setup catalog's name, so + // the route finds something to reconcile toward; the workflows are + // not deployed yet, so the answer is provisioning either way. + const { hub } = fakeHub({ inferenceCredential: true }); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); - let deployStarted = false; + let kickStarted = false; + let kickFinished = false; try { - await store.put({ - userId: "user_1", - tenantId: TENANT_ID, - principalId: PRINCIPAL_ID, - tenantDomain: TENANT_DOMAIN, - provider: "openrouter", - apiKey: "sk-or-v1-minted", - }); const app = mountAuthenticated( createOnboardingRoutes( routeDeps({ hubUrl: `http://localhost:${server.port}`, - store, - deployDelayMs: 5_000, - onDeploy: () => { - deployStarted = true; + onKick: () => { + kickStarted = true; + void new Promise((resolve) => setTimeout(resolve, 5_000)).then( + () => { + kickFinished = true; + }, + ); }, }), ), @@ -481,25 +509,23 @@ describe("POST /complete-setup — no longer deploys inline either", () => { expect(response.status).toBe(200); expect(elapsedMs).toBeLessThan(1_000); - expect(deployStarted).toBe(false); expect(body.kind).toBe("provisioning"); - // The row stays: it is the drain's work item, not a spent token. - expect( - await store.read({ userId: "user_1", tenantId: TENANT_ID }), - ).toBeDefined(); + // Same structural guarantee as /complete: the kicked reconcile + // started, but the route never waited on its five seconds. + expect(kickStarted).toBe(true); + expect(kickFinished).toBe(false); } finally { server.stop(true); } }); - test("still reports unseeded when there is nothing to provision with yet", async () => { + test("still reports unseeded when there is no key to reconcile toward yet", async () => { const { hub } = fakeHub(); const server = Bun.serve({ port: 0, fetch: hub.fetch }); - const store = createInMemoryPendingSeedStore(testCipher()); try { const app = mountAuthenticated( createOnboardingRoutes( - routeDeps({ hubUrl: `http://localhost:${server.port}`, store }), + routeDeps({ hubUrl: `http://localhost:${server.port}` }), ), ); diff --git a/packages/onboarding/test/desired-state-reconcile.test.ts b/packages/onboarding/test/desired-state-reconcile.test.ts index 9c75851d70..b15ef72d64 100644 --- a/packages/onboarding/test/desired-state-reconcile.test.ts +++ b/packages/onboarding/test/desired-state-reconcile.test.ts @@ -385,8 +385,8 @@ describe("resolveTenantModelSource", () => { describe("resolveTenantDeployer", () => { // The tenant-create kick (CL-7584) reconciles a tenant nobody has - // connected a credential to yet, so there is no pending-seed row to - // read deploy identity from — it resolves the creator's owner + // connected a credential to yet, so there is no connected identity to + // deploy under — it resolves the creator's owner // principal plus the tenant domain straight from the API instead of // deploying under empty-string ids. function deployerPrincipalsPage( diff --git a/packages/onboarding/test/huggingface-connect-routes.test.ts b/packages/onboarding/test/huggingface-connect-routes.test.ts index 62ad959212..4fb8f53942 100644 --- a/packages/onboarding/test/huggingface-connect-routes.test.ts +++ b/packages/onboarding/test/huggingface-connect-routes.test.ts @@ -12,18 +12,11 @@ import { describe, expect, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { - createEnvKeyCredentialCipher, - createNoopCredentialCipher, -} from "@intx/crypto"; +import { createEnvKeyCredentialCipher } from "@intx/crypto"; import { createOnboardingRoutes } from "../src/routes"; import type { CreateOnboardingRoutesDeps } from "../src/routes"; import { testAndPersistCredential } from "../src/complete-credential"; import { s256Challenge } from "@corbits/connections"; -import { - createInMemoryPendingSeedStore, - type PendingSeedStore, -} from "../src/pending-seed"; // Stands in for a stable `CREDENTIAL_ENCRYPTION_KEY`: a fresh cipher // built from these same bytes is indistinguishable, to the state store, @@ -214,10 +207,9 @@ function connectRoutes( overrides.pushWorkflow ?? (async () => ({ outcome: "pushed" as const, commitSha: "a".repeat(40) })), log: overrides.log ?? (() => undefined), - pendingSeedStore: - overrides.pendingSeedStore ?? - createInMemoryPendingSeedStore(createNoopCredentialCipher()), }; + if (overrides.desiredStateKick !== undefined) + deps.desiredStateKick = overrides.desiredStateKick; if (overrides.omitClientId !== true) { deps.huggingfaceClientId = overrides.huggingfaceClientId ?? "hf_client_1"; } @@ -323,11 +315,11 @@ describe("GET /oauth/huggingface/callback", () => { expiresAt?: string; credentialMetadata?: Record; }[] = []; - const pendingSeedStore: PendingSeedStore = createInMemoryPendingSeedStore( - createNoopCredentialCipher(), - ); + const kickedTenants: string[] = []; const app = connectRoutes({ - pendingSeedStore, + desiredStateKick: ({ tenantId }) => { + kickedTenants.push(tenantId); + }, huggingfaceConnect: { exchange: async ({ code, codeVerifier, redirectUri, clientId }) => { exchanges.push({ code, codeVerifier, redirectUri, clientId }); @@ -406,25 +398,15 @@ describe("GET /oauth/huggingface/callback", () => { }, ]); - // The plaintext token is carried forward for the deferred deploy - // step server-side, in the pending-seed store — never as a cookie - // (this response still clears the connect-state cookies, but never - // sets the pre-CL-6031 `workbench_pending_seed` one) or a redirect - // query parameter. + // The callback kicks the desired-state reconcile for the + // just-connected bench and carries the minted token nowhere: no + // pending-seed row (CL-7586 deleted the drain), no cookie (this + // response still clears the connect-state cookies, but never sets + // the pre-CL-6031 `workbench_pending_seed` one), no redirect query + // parameter. const setCookie = response.headers.get("set-cookie") ?? ""; expect(setCookie).not.toContain("workbench_pending_seed="); - const pending = await pendingSeedStore.read({ - userId: "user_1", - tenantId: "ten_1", - }); - expect(pending).toEqual({ - userId: "user_1", - tenantId: "ten_1", - principalId: "prn_1", - tenantDomain: "alice-user1.bench.local", - provider: "huggingface", - apiKey: "hf_oauth_minted", - }); + expect(kickedTenants).toEqual(["ten_1"]); }); test("a callback whose query state disagrees with the cookie never exchanges", async () => { diff --git a/packages/onboarding/test/migrations.test.ts b/packages/onboarding/test/migrations.test.ts index 8514f268e2..dd55152934 100644 --- a/packages/onboarding/test/migrations.test.ts +++ b/packages/onboarding/test/migrations.test.ts @@ -19,7 +19,7 @@ function scratchUrlFor(e2eUrl: string): string { const databaseUrl = e2eDatabaseUrl(); const describeIfDb = dbGate(databaseUrl, import.meta.path); -const migrationNames = ["0001_pending_seed"]; +const migrationNames = ["0001_pending_seed", "0002_drop_pending_seed"]; describeIfDb("applyOnboardingMigrations", () => { const scratchUrl = scratchUrlFor( @@ -56,7 +56,11 @@ describeIfDb("applyOnboardingMigrations", () => { } }, 20000); - test("applies the pending_seed table into its own schema and is idempotent on a second run", async () => { + // CL-7586 retired the pending-seed drain: 0001 stays so replays from + // an empty database still see the full history, and 0002 drops the + // table the drain converged. A fresh bench ends with the ledger full + // and no pending_seed table anywhere. + test("applies the full history into its own schema, drops the retired pending_seed table, and is idempotent on a second run", async () => { const first = await applyOnboardingMigrations(scratchUrl); expect(first.applied).toEqual(migrationNames); @@ -70,24 +74,13 @@ describeIfDb("applyOnboardingMigrations", () => { `SELECT table_name FROM information_schema.tables ` + `WHERE table_schema = 'onboarding' AND table_name = 'pending_seed'`, ); - expect(tables.map((row) => String(row["table_name"]))).toEqual([ - "pending_seed", - ]); + expect(tables).toHaveLength(0); const inPublic = await sql.unsafe( `SELECT table_name FROM information_schema.tables ` + `WHERE table_schema = 'public' AND table_name = 'pending_seed'`, ); expect(inPublic).toHaveLength(0); - - const primaryKeyColumns = await sql.unsafe( - `SELECT a.attname FROM pg_index i ` + - `JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) ` + - `WHERE i.indrelid = 'onboarding.pending_seed'::regclass AND i.indisprimary`, - ); - expect( - primaryKeyColumns.map((row) => String(row["attname"])).sort(), - ).toEqual(["tenant_id", "user_id"]); } finally { await sql.end(); } diff --git a/packages/onboarding/test/openrouter-connect-routes.test.ts b/packages/onboarding/test/openrouter-connect-routes.test.ts index 33ed954c45..244cea6e5f 100644 --- a/packages/onboarding/test/openrouter-connect-routes.test.ts +++ b/packages/onboarding/test/openrouter-connect-routes.test.ts @@ -11,18 +11,11 @@ import { describe, expect, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { - createEnvKeyCredentialCipher, - createNoopCredentialCipher, -} from "@intx/crypto"; +import { createEnvKeyCredentialCipher } from "@intx/crypto"; import { createOnboardingRoutes } from "../src/routes"; import type { CreateOnboardingRoutesDeps } from "../src/routes"; import { testAndPersistCredential } from "../src/complete-credential"; import { s256Challenge } from "../src/openrouter-connect"; -import { - createInMemoryPendingSeedStore, - type PendingSeedStore, -} from "../src/pending-seed"; const MOCK_TIMESTAMP = "2026-01-01T00:00:00.000Z"; @@ -224,10 +217,9 @@ function connectRoutes( overrides.pushWorkflow ?? (async () => ({ outcome: "pushed" as const, commitSha: "a".repeat(40) })), log: overrides.log ?? (() => undefined), - pendingSeedStore: - overrides.pendingSeedStore ?? - createInMemoryPendingSeedStore(createNoopCredentialCipher()), }; + if (overrides.desiredStateKick !== undefined) + deps.desiredStateKick = overrides.desiredStateKick; if (overrides.openrouterConnect !== undefined) deps.openrouterConnect = overrides.openrouterConnect; if (overrides.credentialCipher !== undefined) @@ -323,11 +315,11 @@ describe("GET /oauth/openrouter/callback", () => { apiKey: string; userId: string; }[] = []; - const pendingSeedStore: PendingSeedStore = createInMemoryPendingSeedStore( - createNoopCredentialCipher(), - ); + const kickedTenants: string[] = []; const app = connectRoutes({ - pendingSeedStore, + desiredStateKick: ({ tenantId }) => { + kickedTenants.push(tenantId); + }, openrouterConnect: { exchange: async ({ code, codeVerifier }) => { exchanges.push({ code, codeVerifier }); @@ -384,25 +376,15 @@ describe("GET /oauth/openrouter/callback", () => { { provider: "openrouter", apiKey: "sk-or-v1-minted", userId: "user_1" }, ]); - // The plaintext key is carried forward for the deferred deploy step - // server-side, in the pending-seed store — never as a cookie (this + // The callback kicks the desired-state reconcile for the + // just-connected bench and carries the minted key nowhere: no + // pending-seed row (CL-7586 deleted the drain), no cookie (this // response still clears the connect-state cookies, but never sets - // the pre-CL-6031 `workbench_pending_seed` one) or a redirect query + // the pre-CL-6031 `workbench_pending_seed` one), no redirect query // parameter. const setCookie = response.headers.get("set-cookie") ?? ""; expect(setCookie).not.toContain("workbench_pending_seed="); - const pending = await pendingSeedStore.read({ - userId: "user_1", - tenantId: "ten_1", - }); - expect(pending).toEqual({ - userId: "user_1", - tenantId: "ten_1", - principalId: "prn_1", - tenantDomain: "alice-user1.bench.local", - provider: "openrouter", - apiKey: "sk-or-v1-minted", - }); + expect(kickedTenants).toEqual(["ten_1"]); }); test("a callback without the state cookie never exchanges", async () => { diff --git a/packages/onboarding/test/pending-seed-store.drizzle.test.ts b/packages/onboarding/test/pending-seed-store.drizzle.test.ts deleted file mode 100644 index e6a33cd0d8..0000000000 --- a/packages/onboarding/test/pending-seed-store.drizzle.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -// DB-gated: skipped when no DATABASE_URL is reachable (a fresh checkout -// still runs the unit gates), mirroring `@corbits/webhook-triggers`' -// own `store.drizzle.test.ts` and this package's `migrations.test.ts`. -// Runs against its own scratch database, never the developer's or the -// walking-skeleton suite's. -// -// Proves what an in-memory `PendingSeedStore` fake cannot: that -// `createDrizzlePendingSeedStore` actually persists across separate -// connections (surviving a restart by construction — it's a real -// table, not process memory), that a real cipher's ciphertext on disk -// is not the plaintext key, that a fresh connect upserts the single -// active row per (userId, tenantId) rather than accumulating rows, and -// that a row whose stored `provider` column disagrees with the AAD its -// payload was actually sealed under (simulated here via a raw SQL -// UPDATE — the kind of tamper the application layer never produces on -// its own) fails to decrypt and is swept away rather than silently -// misattributed. -import { afterAll, beforeAll, expect, test } from "bun:test"; -import { drizzle } from "drizzle-orm/postgres-js"; -import postgres from "postgres"; -import { - createEnvKeyCredentialCipher, - createNoopCredentialCipher, -} from "@intx/crypto"; - -import { e2eDatabaseUrl } from "../../../scripts/e2e/database-url"; -import { applyOnboardingMigrations } from "../src/migrations"; -import { dbGate } from "../../../scripts/e2e/db-gate"; -import { - createDrizzlePendingSeedStore, - type PendingSeed, -} from "../src/pending-seed"; - -function scratchUrlFor(e2eUrl: string): string { - const url = new URL(e2eUrl); - const database = url.pathname.replace(/^\//, ""); - url.pathname = `/${database}_onboarding_pending_seed_store_test`; - return url.toString(); -} - -const databaseUrl = e2eDatabaseUrl(); -const describeIfDb = dbGate(databaseUrl, import.meta.path); - -const KEY = new Uint8Array(32).fill(9); - -const SEED: PendingSeed = { - userId: "user_1", - tenantId: "ten_1", - principalId: "prn_1", - tenantDomain: "alice-user1.bench.local", - provider: "openrouter", - apiKey: "sk-or-v1-minted", -}; - -describeIfDb("createDrizzlePendingSeedStore", () => { - const scratchUrl = scratchUrlFor( - databaseUrl ?? "postgres://localhost:5432/unused", - ); - const scratchDatabase = new URL(scratchUrl).pathname.replace(/^\//, ""); - - beforeAll(async () => { - const maintenanceUrl = new URL(scratchUrl); - maintenanceUrl.pathname = "/postgres"; - const maintenance = postgres(maintenanceUrl.toString(), { - max: 1, - onnotice: () => undefined, - }); - try { - await maintenance.unsafe(`DROP DATABASE IF EXISTS "${scratchDatabase}"`); - await maintenance.unsafe(`CREATE DATABASE "${scratchDatabase}"`); - } finally { - await maintenance.end(); - } - await applyOnboardingMigrations(scratchUrl); - }, 20000); - - afterAll(async () => { - const maintenanceUrl = new URL(scratchUrl); - maintenanceUrl.pathname = "/postgres"; - const maintenance = postgres(maintenanceUrl.toString(), { - max: 1, - onnotice: () => undefined, - }); - try { - await maintenance.unsafe(`DROP DATABASE IF EXISTS "${scratchDatabase}"`); - } finally { - await maintenance.end(); - } - }, 20000); - - test("a real cipher's stored payload is not the plaintext, and round-trips correctly", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const cipher = createEnvKeyCredentialCipher(KEY); - const store = createDrizzlePendingSeedStore(db, cipher); - - await store.put(SEED); - - const [rawRow] = - await sql`select payload from onboarding.pending_seed where user_id = ${SEED.userId} and tenant_id = ${SEED.tenantId}`; - expect(String(rawRow?.["payload"])).not.toContain(SEED.apiKey); - expect(String(rawRow?.["payload"])).toContain("enc:"); - - const read = await store.read({ - userId: SEED.userId, - tenantId: SEED.tenantId, - }); - expect(read).toEqual(SEED); - } finally { - await sql.end(); - } - }); - - test("a noop cipher (dev/test default) round-trips as plaintext identity", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const store = createDrizzlePendingSeedStore( - db, - createNoopCredentialCipher(), - ); - const seed: PendingSeed = { ...SEED, userId: "user_noop" }; - - await store.put(seed); - - const [rawRow] = - await sql`select payload from onboarding.pending_seed where user_id = ${seed.userId} and tenant_id = ${seed.tenantId}`; - expect(String(rawRow?.["payload"])).toContain(seed.apiKey); - - const read = await store.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toEqual(seed); - } finally { - await sql.end(); - } - }); - - test("a fresh connect upserts the single active row per (userId, tenantId) — never a second row", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const cipher = createEnvKeyCredentialCipher(KEY); - const store = createDrizzlePendingSeedStore(db, cipher); - const seed: PendingSeed = { ...SEED, userId: "user_upsert" }; - - await store.put(seed); - const replacement: PendingSeed = { - ...seed, - provider: "huggingface", - apiKey: "hf_replaced", - }; - await store.put(replacement); - - const rows = - await sql`select provider from onboarding.pending_seed where user_id = ${seed.userId} and tenant_id = ${seed.tenantId}`; - expect(rows).toHaveLength(1); - expect(rows[0]?.["provider"]).toBe("huggingface"); - - const read = await store.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toEqual(replacement); - } finally { - await sql.end(); - } - }); - - test("survives across separate connections — DB-backed, not process memory", async () => { - const cipher = createEnvKeyCredentialCipher(KEY); - const seed: PendingSeed = { ...SEED, userId: "user_restart" }; - - const writeSql = postgres(scratchUrl, { max: 1 }); - try { - const writeStore = createDrizzlePendingSeedStore( - drizzle(writeSql), - cipher, - ); - await writeStore.put(seed); - } finally { - await writeSql.end(); - } - - // A brand-new connection and store instance — nothing shared with - // the one above except the database and the cipher key, exactly - // what survives a real hub restart. - const readSql = postgres(scratchUrl, { max: 1 }); - try { - const readStore = createDrizzlePendingSeedStore(drizzle(readSql), cipher); - const read = await readStore.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toEqual(seed); - } finally { - await readSql.end(); - } - }); - - test("a row whose provider column disagrees with the AAD its payload was sealed under fails closed and is swept away", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const cipher = createEnvKeyCredentialCipher(KEY); - const store = createDrizzlePendingSeedStore(db, cipher); - const seed: PendingSeed = { ...SEED, userId: "user_tamper" }; - - await store.put(seed); - // Simulate a tamper the application layer never produces: - // relabel the row's provider column without re-sealing the - // payload under the new AAD. - await sql`update onboarding.pending_seed set provider = 'huggingface' where user_id = ${seed.userId} and tenant_id = ${seed.tenantId}`; - - const read = await store.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toBeUndefined(); - - const rows = - await sql`select 1 from onboarding.pending_seed where user_id = ${seed.userId} and tenant_id = ${seed.tenantId}`; - expect(rows).toHaveLength(0); - } finally { - await sql.end(); - } - }); - - test("an expired row is deleted at read time, not merely skipped", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const cipher = createEnvKeyCredentialCipher(KEY); - const store = createDrizzlePendingSeedStore(db, cipher); - const seed: PendingSeed = { ...SEED, userId: "user_expired" }; - - await store.put(seed, { ttlMs: -1 }); - - const read = await store.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toBeUndefined(); - - const rows = - await sql`select 1 from onboarding.pending_seed where user_id = ${seed.userId} and tenant_id = ${seed.tenantId}`; - expect(rows).toHaveLength(0); - } finally { - await sql.end(); - } - }); - - test("clear deletes the row", async () => { - const sql = postgres(scratchUrl, { max: 1 }); - try { - const db = drizzle(sql); - const cipher = createEnvKeyCredentialCipher(KEY); - const store = createDrizzlePendingSeedStore(db, cipher); - const seed: PendingSeed = { ...SEED, userId: "user_clear" }; - - await store.put(seed); - await store.clear({ userId: seed.userId, tenantId: seed.tenantId }); - - const read = await store.read({ - userId: seed.userId, - tenantId: seed.tenantId, - }); - expect(read).toBeUndefined(); - } finally { - await sql.end(); - } - }); -}); diff --git a/packages/onboarding/test/report-error-routing.test.ts b/packages/onboarding/test/report-error-routing.test.ts index 66b42461a7..fb51a0e5e0 100644 --- a/packages/onboarding/test/report-error-routing.test.ts +++ b/packages/onboarding/test/report-error-routing.test.ts @@ -1,5 +1,5 @@ -// CL-7234: every caught failure in this package's routes, background -// drain, and env-credential-plant paths must reach @corbits/error-sink's +// CL-7234: every caught failure in this package's routes and +// env-credential-plant paths must reach @corbits/error-sink's // reportError with the operation/tenant context it expects — the same // precedent ../src/provision.ts already sets. This mocks // @corbits/error-sink and dynamically imports each module under test @@ -11,7 +11,6 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { createNoopCredentialCipher } from "@intx/crypto"; let reportErrorCalls: [unknown, Record][] = []; beforeEach(async () => { @@ -28,15 +27,9 @@ afterEach(() => { }); const { createOnboardingRoutes } = await import("../src/routes"); -const { createInMemoryPendingSeedStore } = await import("../src/pending-seed"); -const { createBenchProvisioner } = await import("../src/bench-provisioning"); const { plantEnvProviderCredentials } = await import("../src/plant-env-credentials"); -const pendingSeedStore = createInMemoryPendingSeedStore( - createNoopCredentialCipher(), -); - // These tests never exercise the genesis-or-join join path — the stub // satisfies the required tenancy wiring without standing up a DB. const emptyHubTenancy = { @@ -71,7 +64,6 @@ describe("routes.ts routes caught errors through reportError", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -137,7 +129,6 @@ describe("routes.ts routes caught errors through reportError", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -160,44 +151,6 @@ describe("routes.ts routes caught errors through reportError", () => { }); }); -describe("bench-provisioning.ts's whole-drain failure reports through reportError", () => { - test("a listDue failure (not scoped to any one bench) reports operation only", async () => { - const provisioner = createBenchProvisioner({ - api: (async () => { - throw new Error("unused"); - }) as unknown as Parameters[0]["api"], - hubUrl: "https://bench.example.com", - store: { - listDue: async () => { - throw new Error("db unreachable"); - }, - read: async () => undefined, - put: async () => undefined, - clear: async () => undefined, - }, - pushWorkflow: async () => ({ - outcome: "pushed" as const, - commitSha: "a".repeat(40), - }), - sessionFor: async () => ["better-auth.session_token=minted"], - log: () => undefined, - }); - - provisioner.wake(); - // wake() is fire-and-forget; give its internal drainOnce().catch a - // turn to run before asserting. - await new Promise((resolve) => setTimeout(resolve, 10)); - - expect(reportErrorCalls).toHaveLength(1); - const [, context] = reportErrorCalls[0] as [ - unknown, - Record, - ]; - expect(context.operation).toBe("bench_provisioning_drain"); - expect(context.tenantId).toBeUndefined(); - }); -}); - describe("plant-env-credentials.ts catches report through reportError and never leak the raw cause", () => { // No existing provider row for any credential this test plants — the // real network calls `findProviderId`/`findActiveCredential` make @@ -287,7 +240,6 @@ describe("recentlyConnectedCredential reports through reportError and still find log: (line) => { logs.push(line); }, - pendingSeedStore, }); const app = mountAuthenticated(routes); diff --git a/packages/onboarding/test/routes.test.ts b/packages/onboarding/test/routes.test.ts index 6634b831bd..f16ae1f9fb 100644 --- a/packages/onboarding/test/routes.test.ts +++ b/packages/onboarding/test/routes.test.ts @@ -8,21 +8,11 @@ import { describe, expect, test } from "bun:test"; import type { AppEnv } from "@intx/hub-api"; import type { MiddlewareHandler } from "hono"; import { Hono } from "hono"; -import { createNoopCredentialCipher } from "@intx/crypto"; import { createOnboardingRoutes } from "../src/routes"; import { testAndPersistCredential } from "../src/complete-credential"; -import { createInMemoryPendingSeedStore } from "../src/pending-seed"; import { createProviderHealthStore } from "@corbits/connections/provider-health"; import { HubApiError } from "@corbits/hub-api-client"; -// These tests never exercise the pending-seed store — it is required -// wiring for `createOnboardingRoutes`, and its dedicated coverage lives -// in `./complete-setup-routes.test.ts`, `./connect-deploys-nothing.test.ts` -// and `../src/pending-seed.test.ts`. -const pendingSeedStore = createInMemoryPendingSeedStore( - createNoopCredentialCipher(), -); - // These tests never exercise the genesis-or-join join path — the stub // satisfies the required tenancy wiring without standing up a DB. const emptyHubTenancy = { @@ -60,7 +50,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: (line) => lines.push(line), - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -106,7 +95,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -148,7 +136,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -176,7 +163,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); const named = { @@ -217,7 +203,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -262,7 +247,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -299,7 +283,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -329,7 +312,6 @@ describe("POST /provision", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const response = await routes.request("/provision", { method: "POST" }); @@ -353,7 +335,6 @@ describe("POST /complete", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const response = await routes.request("/complete", { @@ -382,7 +363,6 @@ describe("POST /complete", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, }); const app = mountAuthenticated(routes); @@ -425,7 +405,6 @@ describe("POST /complete", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, providerHealth, testAndPersistCredentialFn: async () => ({ kind: "connected", @@ -462,7 +441,6 @@ describe("POST /complete", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, providerHealth, testAndPersistCredentialFn: async () => ({ kind: "invalid-credential", @@ -483,14 +461,17 @@ describe("POST /complete", () => { ); }); - // CL-6457 moved partial-deploy convergence off this route entirely: - // `/complete` no longer deploys anything, so there is no half-finished - // deploy for it to report. What the route still owes the drain — a - // durable pending row carrying the key the deploy runs against — is - // covered by `./connect-deploys-nothing.test.ts` ("hands the drain a - // pending row carrying the key it will deploy against"), and the - // convergence that row buys is covered by `./bench-provisioning.test.ts` - // ("a half-provisioned bench keeps its row and converges on a later pass"). + // CL-6457 moved partial-deploy convergence off this route entirely, + // and CL-7586 deleted the pending-seed drain: `/complete` no longer + // deploys anything and parks no row, so there is no half-finished + // deploy for it to report. What the route still owes the waiting + // surface — a fire-and-forget kick naming the tenant for the + // desired-state reconcile — is covered by + // `./connect-deploys-nothing.test.ts` ("hands follow-up work to the + // desired-state reconcile as a fire-and-forget kick"), and the + // convergence that kick buys is covered by + // `./desired-state-reconcile.test.ts` ("a non-sidecar failure reports + // failed, and a re-run can converge"). test("a non-sidecar failure during setup still fails loudly with the existing 500 envelope", async () => { const routes = createOnboardingRoutes({ @@ -502,7 +483,6 @@ describe("POST /complete", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, testAndPersistCredentialFn: async () => { throw new Error("the hub rejected deployment with status 500"); }, @@ -539,7 +519,6 @@ describe("POST /complete", () => { }), log: () => undefined, logError: (line) => lines.push(line), - pendingSeedStore, testAndPersistCredentialFn: async () => { throw new HubApiError( "publishing the corbits-tools package-registry asset failed: " + @@ -627,7 +606,6 @@ describe("POST /complete — seeded-admin fallback", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, testAndPersistCredentialFn: (args) => testAndPersistCredential({ ...args, @@ -773,7 +751,6 @@ describe("CL-7584 desired-state kicks and steps", () => { commitSha: "a".repeat(40), }), log: () => undefined, - pendingSeedStore, desiredStateKick: (args) => kicks.push(args.tenantId), }); return { server, app: mountAuthenticated(routes) }; @@ -853,8 +830,8 @@ describe("CL-7584 desired-state kicks and steps", () => { expect(kicks).toEqual(["ten_root"]); // The retry is the same idempotent probe, not a new queue table or // endpoint: the revisit fires again unconditionally, and the - // reconcile itself re-reads the pins. The `pending_seed` row covers - // the credential half; this re-kick covers the desired-state half. + // reconcile itself re-reads the pins. There is no durable queue + // left — this re-kick is the whole recovery story. const revisit = await app.request("/provision", { method: "POST" }); expect(revisit.status).toBe(200); await awaitKicks(kicks, 2); diff --git a/scripts/checks/no-product-tenancy.ts b/scripts/checks/no-product-tenancy.ts index 0722010273..6876d6b8c1 100644 --- a/scripts/checks/no-product-tenancy.ts +++ b/scripts/checks/no-product-tenancy.ts @@ -132,11 +132,6 @@ const ALLOWLIST: readonly { maxOccurrences: 1, tables: ["policy"], }, - { - relPath: "packages/onboarding/src/schema.ts", - maxOccurrences: 1, - tables: ["pending_seed"], - }, { // `@corbits/mailbox`'s enrichment carries only priority, // classification and status — there is no column for a snooze's diff --git a/scripts/checks/report-error-baseline.txt b/scripts/checks/report-error-baseline.txt index 03115ba460..277150cd70 100644 --- a/scripts/checks/report-error-baseline.txt +++ b/scripts/checks/report-error-baseline.txt @@ -191,8 +191,6 @@ packages/memory-tools/src/tool.ts 3 if (err instanceof MemoryUnavailableError) { packages/mocks/src/ollama/capture.ts 1 return raw; packages/ollama-adapter/src/inline-tool-json.ts 1 return false; packages/ollama-adapter/src/inline-tool-json.ts 1 return { kind: "incomplete" }; -packages/onboarding/src/pending-seed.ts 1 return drop(); -packages/onboarding/src/routes.ts 1 // Neither `ProvisionError` nor `CliError` messages are safe to show packages/onboarding/src/routes.ts 1 return c.json( packages/presence/src/artifact-persistence.ts 1 deps.onSnapshotError?.(key, error); packages/presence/src/client.ts 1 // A malformed update is dropped rather than crashing the client; From 61bcd8d170a9c30fc86d0ee8a0a6c02adf1b0eb9 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 15 Sep 2026 14:12:06 -0700 Subject: [PATCH 2/2] docs(onboarding): correct reconcile scope after drain deletion (CL-7586) --- apps/hub/src/index.ts | 5 +++-- docs/onboarding-openrouter-connect.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 1fa74ce8e5..32f0e6bec7 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -3374,8 +3374,9 @@ export async function createHub(config: HubConfig) { // package's provision.ts. // Connecting a provider deploys nothing by itself (CL-6457): the // onboarding routes persist the credential and kick the desired-state - // reconcile below, which converges every bench — including one a - // previous process died halfway through, since the reconcile re-reads + // reconcile below, which converges the bench on the next kick (tenant + // create, connect, revisit probe) — including one a previous process + // died halfway through, since the reconcile re-reads // the pins on every kick rather than consuming a durable work item. // (CL-7586 removed the pending-seed drain that used to do this.) diff --git a/docs/onboarding-openrouter-connect.md b/docs/onboarding-openrouter-connect.md index 5e4d9f425a..0e66443941 100644 --- a/docs/onboarding-openrouter-connect.md +++ b/docs/onboarding-openrouter-connect.md @@ -61,7 +61,7 @@ at `/api/onboarding`); the wizard only navigates. second, for a connection that had actually succeeded. The callback now returns as soon as the key is proven and stored — see `complete-credential.ts`'s module comment for the fast/slow split, - and `ensureSeeded` for the deploy step's new home. + and the desired-state reconcile for the deploy step. 5. **Nothing rides forward — the credential row is the handoff.** Credential secrets are write-only through the hub's own API, so nothing needs to re-fetch the key once this request ends. The