diff --git a/apps/hub/src/config.ts b/apps/hub/src/config.ts index 03f55c1981..aaac6a14e7 100644 --- a/apps/hub/src/config.ts +++ b/apps/hub/src/config.ts @@ -216,6 +216,14 @@ const DEFAULT_TENANT_SLUG = "workbench"; // package, add a member here with its settings, add its id to // `SIDECAR_PROVISIONER_IDS`, and register it in // `sidecarProvisionerFrom`'s per-id parsing below. +// +// CL-7324: this is the CLOSED dispatchable-backend set — dispatchable +// work may only ever land on one of these hub-provisioned backends, and +// only ever tenant-bound (the anchor run's tenant; see +// `./dispatch-tenant-guard.ts`). No schema change carries that rule: it is +// enforced at the dispatch seams, not declared in config, so a new backend +// id added here is provisionable but never implicitly dispatchable +// cross-tenant. export type DockerSidecarProvisionerConfig = { readonly id: "docker"; readonly image: string; diff --git a/apps/hub/src/dispatch-tenant-guard.test.ts b/apps/hub/src/dispatch-tenant-guard.test.ts new file mode 100644 index 0000000000..333258248f --- /dev/null +++ b/apps/hub/src/dispatch-tenant-guard.test.ts @@ -0,0 +1,220 @@ +// CL-7324: hermetic tests for the Workbench-side dispatch containment +// (`dispatch-tenant-guard.ts`). No database: the service wrapper gets a +// stub anchor-tenant resolver and a stub delegate, and the route guard +// gets a stub tenancy loader in front of a counting inner app — proving +// the decision logic itself, not Postgres. +import { expect, test } from "bun:test"; +import { Hono } from "hono"; +import type { AppEnv } from "@intx/hub-api"; +import type { SidecarAllocation } from "@intx/db"; +import type { WorkflowAllocationService } from "@intx/hub-sessions"; + +import { + ALLOCATION_TENANT_MISMATCH_CODE, + AllocationTenantMismatchError, + TENANT_MISMATCH_FAILURE_CODE, + withDispatchTenantGuard, + withTenantBoundAllocationService, +} from "./dispatch-tenant-guard"; + +function allocationFor(args: { + id?: string; + anchorRunId?: string; + tenantId?: string; +}): SidecarAllocation { + return { + id: args.id ?? "sal_test", + anchorRunId: args.anchorRunId ?? "wfr_anchor", + tenantId: args.tenantId ?? "ten_a", + provisionerId: "process", + provisionerApiVersion: 1, + provisionerBindingFingerprint: "fp", + status: "allocated", + generation: 1, + ensureAttempts: 0, + destroyAttempts: 0, + createdAt: new Date(0), + updatedAt: new Date(0), + }; +} + +function stubAllocationService(): WorkflowAllocationService & { + deployCalls: SidecarAllocation[]; +} { + const deployCalls: SidecarAllocation[] = []; + const service = { + deployCalls, + async prepareProvisionedDeployment() { + throw new Error("not implemented in stub"); + }, + async deployReadyAllocation(allocation: SidecarAllocation) { + deployCalls.push(allocation); + return null; + }, + } as unknown as WorkflowAllocationService & { + deployCalls: SidecarAllocation[]; + }; + return service; +} + +test("deployReadyAllocation delegates when allocation and anchor tenants match", async () => { + const inner = stubAllocationService(); + const guarded = withTenantBoundAllocationService(inner, { + resolveAnchorTenantId: async () => "ten_a", + }); + const allocation = allocationFor({ tenantId: "ten_a" }); + const result = await guarded.deployReadyAllocation(allocation); + expect(result).toBeNull(); + expect(inner.deployCalls).toEqual([allocation]); +}); + +test("deployReadyAllocation throws tenant_mismatch without delegating on a foreign allocation", async () => { + const inner = stubAllocationService(); + const guarded = withTenantBoundAllocationService(inner, { + resolveAnchorTenantId: async () => "ten_a", + }); + const thrown = await guarded + .deployReadyAllocation(allocationFor({ tenantId: "ten_b" })) + .then( + () => null, + (error: unknown) => error, + ); + expect(thrown).toBeInstanceOf(AllocationTenantMismatchError); + expect((thrown as AllocationTenantMismatchError).code).toBe( + TENANT_MISMATCH_FAILURE_CODE, + ); + // Never falls through to another tenant: the delegate never ran, so the + // allocation can never deploy — the reconciler retries instead. + expect(inner.deployCalls).toEqual([]); +}); + +test("deployReadyAllocation fails closed when the anchor row is gone", async () => { + const inner = stubAllocationService(); + const guarded = withTenantBoundAllocationService(inner, { + resolveAnchorTenantId: async () => undefined, + }); + const thrown = await guarded + .deployReadyAllocation(allocationFor({ tenantId: "ten_a" })) + .then( + () => null, + (error: unknown) => error, + ); + expect(thrown).toBeInstanceOf(AllocationTenantMismatchError); + expect(inner.deployCalls).toEqual([]); +}); + +function guardedTestApp( + loadAnchorDispatch: ( + anchorRunId: string, + ) => Promise< + | { anchorTenantId: string; allocationTenantIds: readonly string[] } + | undefined + >, +): { app: Hono; state: { innerHits: number } } { + const state = { innerHits: 0 }; + const inner = new Hono(); + inner.all("*", (c) => { + state.innerHits += 1; + return c.json({ reached: true }, 200); + }); + const app = withDispatchTenantGuard(inner, { loadAnchorDispatch }); + return { app, state }; +} + +async function postApp( + app: Hono, + path: string, +): Promise<{ status: number; body: unknown; app: Hono }> { + const res = await app.request(path, { method: "POST" }); + return { status: res.status, body: await res.json(), app }; +} + +test("mail trigger with a foreign allocation answers 403 allocation_tenant_mismatch without reaching the inner app", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_a", + allocationTenantIds: ["ten_b"], + })); + const { status, body } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_anchor/mail", + ); + expect(status).toBe(403); + expect(body).toMatchObject({ + error: expect.objectContaining({ code: ALLOCATION_TENANT_MISMATCH_CODE }), + }); + expect(state.innerHits).toBe(0); +}); + +test("signals route with a foreign allocation answers 403 without reaching the inner app", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_a", + allocationTenantIds: ["ten_a", "ten_b"], + })); + const { status } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_anchor/signals", + ); + expect(status).toBe(403); + expect(state.innerHits).toBe(0); +}); + +test("matching anchor and allocation tenants fall through to the inner app", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_a", + allocationTenantIds: ["ten_a"], + })); + const { status } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_anchor/mail", + ); + expect(status).toBe(200); + expect(state.innerHits).toBe(1); +}); + +test("a pre-provisioning anchor with no allocation rows falls through", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_a", + allocationTenantIds: [], + })); + const { status } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_anchor/signals", + ); + expect(status).toBe(200); + expect(state.innerHits).toBe(1); +}); + +test("an absent anchor falls through so the native lookup owns the 404", async () => { + const { app, state } = guardedTestApp(async () => undefined); + const { status } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_missing/mail", + ); + expect(status).toBe(200); + expect(state.innerHits).toBe(1); +}); + +test("an anchor owned by another tenant falls through so the native lookup owns the 404", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_other", + allocationTenantIds: ["ten_other"], + })); + const { status } = await postApp( + app, + "/api/tenants/ten_a/workflows/wfr_anchor/mail", + ); + expect(status).toBe(200); + expect(state.innerHits).toBe(1); +}); + +test("non-dispatch paths are never intercepted", async () => { + const { app, state } = guardedTestApp(async () => ({ + anchorTenantId: "ten_a", + allocationTenantIds: ["ten_b"], + })); + const res = await app.request("/api/tenants/ten_a/workflows", { + method: "GET", + }); + expect(res.status).toBe(200); + expect(state.innerHits).toBe(1); +}); diff --git a/apps/hub/src/dispatch-tenant-guard.ts b/apps/hub/src/dispatch-tenant-guard.ts new file mode 100644 index 0000000000..6ab5690a95 --- /dev/null +++ b/apps/hub/src/dispatch-tenant-guard.ts @@ -0,0 +1,247 @@ +// [Intx gap] CL-7324: the native dispatch path joins anchor + allocation +// on `anchorRunId` alone — neither +// `vendor/intx/hub-api/src/run-grant-materialization.ts`'s +// `lockDispatchableAllocation` nor the signal route's deployment query +// (`vendor/intx/hub-api/src/routes/workflows.ts`) filters the allocation +// leg by tenant. A sidecar allocation row bound to an anchor but carrying +// another tenant's id would therefore dispatch — and materialize grants — +// cross-tenant. Filed upstream; vendor is read-only, so this hub contains +// at its own seams instead of patching the join: +// +// 1. `withTenantBoundAllocationService` wraps the allocation service this +// composition root hands to `createApp` (and every Workbench-owned call +// site): `deployReadyAllocation` throws `AllocationTenantMismatchError` +// before delegating whenever the allocation's tenant is not the anchor +// run's tenant. The reconciler's `onReady` catch-all turns any throw +// into `scheduleRetry` (`sidecar_initialization_failed`), so a mismatch +// is retryable, never terminal, and never falls through to another +// tenant — the allocation simply never deploys. +// 2. `withDispatchTenantGuard` outer-wraps the fully-built app (the same +// composition as `tenant-create-guard.ts`'s `guardedHubApp`) and denies +// the two dispatch routes — `POST +// /api/tenants/:tenantId/workflows/:runId/mail` and `POST +// /api/tenants/:tenantId/workflows/:runId/signals` — with 403 +// `allocation_tenant_mismatch` when an allocation bound to the anchor +// carries a foreign tenant, before the native handler can enqueue or +// materialize grants. An absent anchor (or an anchor owned by another +// tenant) falls through: the native tenant-scoped lookup owns that 404. +// +// Bench-owner, hub-provisioned, tenant-bound dispatch only (operator +// ruling 2026-09-15, Option A). No enrollment or allowlist surfaces live +// here — those were deferred (B/C), so there is nothing to consult: any +// cross-tenant join is denied, not admitted. +import { eq } from "drizzle-orm"; +import { Hono } from "hono"; +import type { DB } from "@intx/db"; +import { sidecarAllocation, workflowRun } from "@intx/db/schema"; +import type { AppEnv } from "@intx/hub-api"; +import type { SidecarAllocation } from "@intx/db"; +import type { WorkflowAllocationService } from "@intx/hub-sessions"; +import { makeErrorEnvelope, reportError } from "@corbits/error-sink"; + +/** Error-code surfaced at the route seam on a foreign-tenant allocation. */ +export const ALLOCATION_TENANT_MISMATCH_CODE = "allocation_tenant_mismatch"; +/** Reconciler failure code a mismatch surfaces as: scheduled for retry, + * never terminal. */ +export const TENANT_MISMATCH_FAILURE_CODE = "tenant_mismatch"; + +/** + * Thrown (never returned) when an allocation's tenant is not its anchor + * run's tenant. Retryable by construction: nothing downstream treats this + * as terminal, and the throw precedes any delegate call, so a mismatched + * allocation can never deploy or dispatch into another tenant. + */ +export class AllocationTenantMismatchError extends Error { + readonly code = TENANT_MISMATCH_FAILURE_CODE; + readonly anchorRunId: string; + readonly anchorTenantId: string | undefined; + readonly allocationId: string | undefined; + readonly allocationTenantId: string; + + constructor(args: { + anchorRunId: string; + anchorTenantId: string | undefined; + allocationId: string | undefined; + allocationTenantId: string; + }) { + super( + `allocation ${args.allocationId ?? ""} carries tenant ` + + `${args.allocationTenantId} but anchors run ${args.anchorRunId} of ` + + `tenant ${args.anchorTenantId ?? ""}`, + ); + this.name = "AllocationTenantMismatchError"; + this.anchorRunId = args.anchorRunId; + this.anchorTenantId = args.anchorTenantId; + this.allocationId = args.allocationId; + this.allocationTenantId = args.allocationTenantId; + } +} + +function mismatchContext(error: AllocationTenantMismatchError): { + operation: string; + tenantId: string; + extra: Record; +} { + const extra: Record = { + anchorRunId: error.anchorRunId, + allocationTenantId: error.allocationTenantId, + }; + if (error.allocationId !== undefined) { + extra["allocationId"] = error.allocationId; + } + if (error.anchorTenantId !== undefined) { + extra["anchorTenantId"] = error.anchorTenantId; + } + return { + operation: "workflow-dispatch-tenant-guard", + tenantId: error.allocationTenantId, + extra, + }; +} + +/** Anchor run's owning tenant — `undefined` when the anchor row is gone. */ +export async function resolveAnchorTenantId( + db: DB["db"], + anchorRunId: string, +): Promise { + const [row] = await db + .select({ tenantId: workflowRun.tenantId }) + .from(workflowRun) + .where(eq(workflowRun.id, anchorRunId)) + .limit(1); + return row?.tenantId; +} + +export type TenantBoundAllocationServiceDeps = { + /** Anchor run's owning tenant (`undefined` when the row is gone), kept + * injectable so the wrapper's decision is testable without a database. */ + resolveAnchorTenantId: (anchorRunId: string) => Promise; +}; + +/** + * Wraps the allocation service so `deployReadyAllocation` enforces the + * anchor/allocation tenant join: on a mismatch (or a missing anchor row) + * it reports through the error sink and throws + * `AllocationTenantMismatchError` WITHOUT delegating — the reconciler + * retries, the allocation never deploys, and dispatch never crosses + * tenants. Every other method passes straight through. + */ +export function withTenantBoundAllocationService( + service: WorkflowAllocationService, + deps: TenantBoundAllocationServiceDeps, +): WorkflowAllocationService { + return { + ...service, + async deployReadyAllocation(allocation: SidecarAllocation) { + const anchorTenantId = await deps.resolveAnchorTenantId( + allocation.anchorRunId, + ); + if ( + anchorTenantId === undefined || + anchorTenantId !== allocation.tenantId + ) { + const mismatch = new AllocationTenantMismatchError({ + anchorRunId: allocation.anchorRunId, + anchorTenantId, + allocationId: allocation.id, + allocationTenantId: allocation.tenantId, + }); + reportError(mismatch, mismatchContext(mismatch)); + throw mismatch; + } + return service.deployReadyAllocation(allocation); + }, + }; +} + +export type AnchorDispatchTenancy = { + /** Owning tenant of the anchor run named in the URL. */ + readonly anchorTenantId: string; + /** Tenant of every allocation row bound to that anchor (empty when the + * deployment has no allocation yet — pre-provisioning deploys). */ + readonly allocationTenantIds: readonly string[]; +}; + +export type DispatchTenantGuardDeps = { + /** Anchor + bound-allocation tenancy (`undefined` when the anchor row is + * gone), injected so the guard's decision stays DB-free in tests. */ + loadAnchorDispatch: ( + anchorRunId: string, + ) => Promise; +}; + +/** Production read: anchor row plus every allocation row bound to it. */ +export async function loadAnchorDispatch( + db: DB["db"], + anchorRunId: string, +): Promise { + const [anchor] = await db + .select({ tenantId: workflowRun.tenantId }) + .from(workflowRun) + .where(eq(workflowRun.id, anchorRunId)) + .limit(1); + if (anchor === undefined) return undefined; + const rows = await db + .select({ tenantId: sidecarAllocation.tenantId }) + .from(sidecarAllocation) + .where(eq(sidecarAllocation.anchorRunId, anchorRunId)); + return { + anchorTenantId: anchor.tenantId, + allocationTenantIds: rows.map((row) => row.tenantId), + }; +} + +const DISPATCH_GUARD_PATHS = [ + "/api/tenants/:tenantId/workflows/:runId/mail", + "/api/tenants/:tenantId/workflows/:runId/signals", +] as const; + +/** + * Wraps the fully-built hub app in an outer app that denies the two + * dispatch routes with 403 `allocation_tenant_mismatch` when any + * allocation bound to the anchor carries a tenant other than the URL + * tenant — before the native handler joins anchor+allocation and + * materializes grants or enqueues. Every other path, an anchor with no + * foreign allocation, and an absent/foreign anchor (the native + * tenant-scoped lookup owns that 404) fall straight through. + */ +export function withDispatchTenantGuard( + nativeApp: Hono, + deps: DispatchTenantGuardDeps, +): Hono { + const guarded = new Hono(); + + for (const path of DISPATCH_GUARD_PATHS) { + guarded.use(path, async (c, next) => { + if (c.req.method !== "POST") return next(); + const tenantId = c.req.param("tenantId"); + const anchorRunId = c.req.param("runId"); + const anchor = await deps.loadAnchorDispatch(anchorRunId); + if (anchor === undefined || anchor.anchorTenantId !== tenantId) { + return next(); + } + const foreign = anchor.allocationTenantIds.find( + (allocationTenantId) => allocationTenantId !== tenantId, + ); + if (foreign === undefined) return next(); + const mismatch = new AllocationTenantMismatchError({ + anchorRunId, + anchorTenantId: anchor.anchorTenantId, + allocationId: undefined, + allocationTenantId: foreign, + }); + reportError(mismatch, mismatchContext(mismatch)); + return c.json( + makeErrorEnvelope({ + code: ALLOCATION_TENANT_MISMATCH_CODE, + userMessage: + "This deployment's allocation belongs to another workbench and cannot be dispatched here.", + }), + 403, + ); + }); + } + + guarded.route("/", nativeApp); + return guarded; +} diff --git a/apps/hub/src/index.ts b/apps/hub/src/index.ts index 53f75a1bbe..a47115e48f 100644 --- a/apps/hub/src/index.ts +++ b/apps/hub/src/index.ts @@ -313,6 +313,12 @@ import { createDrizzleAccessPolicyStore, } from "@workbench/access-policy"; import { guardedHubApp, resolveCallerRoleNames } from "./tenant-create-guard"; +import { + loadAnchorDispatch, + resolveAnchorTenantId, + withDispatchTenantGuard, + withTenantBoundAllocationService, +} from "./dispatch-tenant-guard"; import { createTenantCreateObserver } from "./tenant-create-onboard"; import { createInMemoryNotifyDispatchStore, @@ -1129,7 +1135,7 @@ export async function createHub(config: HubConfig) { : {}), }); const sidecarPlugins = buildSidecarPlugins("deployment"); - const workflowAllocationService = createWorkflowAllocationService({ + const nativeWorkflowAllocationService = createWorkflowAllocationService({ db, deploymentPlugins: sidecarPlugins, probePlugins: buildSidecarPlugins("probe"), @@ -1138,6 +1144,19 @@ export async function createHub(config: HubConfig) { allocationRouter: sidecarRouter, hubWebSocketUrl, }); + // [Intx gap] CL-7324: the native dispatch path joins anchor + allocation + // on `anchorRunId` alone, so a foreign-tenant allocation bound to an + // anchor would dispatch cross-tenant. Vendor is read-only — wrap the + // service here, once, so `createApp`, the reconciler `onReady`, and + // every Workbench-owned call site below all share the tenant-bound + // instance (see ./dispatch-tenant-guard.ts). + const workflowAllocationService = withTenantBoundAllocationService( + nativeWorkflowAllocationService, + { + resolveAnchorTenantId: (anchorRunId) => + resolveAnchorTenantId(db, anchorRunId), + }, + ); const sidecarAllocationStore = createSidecarAllocationStore(db); const workflowDispatchService = createWorkflowDispatchService({ dispatchStore: createWorkflowRunDispatchStore(db), @@ -3619,8 +3638,15 @@ export async function createHub(config: HubConfig) { ); observerRef.current = observer; const guardedApp = guardedHubApp(observer.app, guardDeps); + // [Intx gap] CL-7324: deny foreign-tenant allocations at the two dispatch + // routes (403 `allocation_tenant_mismatch`) before the native handler can + // enqueue or materialize grants — same outer-wrap composition as the + // tenant-create guard above. See ./dispatch-tenant-guard.ts. + const dispatchGuardedApp = withDispatchTenantGuard(guardedApp, { + loadAnchorDispatch: (anchorRunId) => loadAnchorDispatch(db, anchorRunId), + }); const inFlight = createInFlightRequestTracker(); - const servingApp = withInFlightRequestTracking(guardedApp, inFlight); + const servingApp = withInFlightRequestTracking(dispatchGuardedApp, inFlight); return { app: servingApp,