From b5fae832bbf918a311a8b254d668b6e90a303724 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 08:08:41 -0700 Subject: [PATCH 1/4] test(e2e): three benches from /new stay isolated child tenants (CL-9507) --- e2e/browser/new-bench.test.ts | 133 ++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 e2e/browser/new-bench.test.ts diff --git a/e2e/browser/new-bench.test.ts b/e2e/browser/new-bench.test.ts new file mode 100644 index 000000000..66f5e1b84 --- /dev/null +++ b/e2e/browser/new-bench.test.ts @@ -0,0 +1,133 @@ +// Three benches created from /new in one session: each is its own child +// tenant with exactly one agent, the sidebar lists them by short title, and +// no bench's thread shows another's opening message. Inference is not +// scripted in this harness, so the agent's self-naming reply is not covered. +import { expect, test } from "bun:test"; +import type { Page } from "puppeteer-core"; +import { bootBrowserApp, browserGate } from "../lib/browser"; + +const describeBrowser = browserGate(import.meta.path); +const STEP_TIMEOUT = 60_000; +const MESSAGES = [ + "Summarize what shipped this week across every repository we track, grouped by team", + "Draft a launch checklist for the onboarding redesign including owners and due dates", + "Research competing pricing pages and list the three patterns we should borrow first", +] as const; + +// Page scripts are strings: this tsconfig has no DOM lib. +async function waitForText(page: Page, label: string): Promise { + await page.waitForFunction(`document.body.innerText.includes(${JSON.stringify(label)})`, { + timeout: STEP_TIMEOUT, + }); +} + +async function clickText(page: Page, selector: string, label: string): Promise { + const script = `Array.from(document.querySelectorAll(${JSON.stringify(selector)})).find((e) => e.textContent.includes(${JSON.stringify(label)}))`; + await page.waitForFunction(script, { timeout: STEP_TIMEOUT }); + await page.evaluate(`(${script}).click()`); +} + +const VIEWPORT = { width: 1400, height: 900 }; + +type Principal = { tenantId: string; kind: string; status: string }; +type Tenant = { id: string; parentId: string | null }; + +async function hubJson(page: Page, url: string): Promise { + return (await page.evaluate(`fetch(${JSON.stringify(url)}).then((r) => r.json())`)) as T; +} + +async function signUp(page: Page, origin: string): Promise { + await page.goto(origin, { waitUntil: "networkidle0" }); + await clickText(page, "button", "Create an account"); + await waitForText(page, "Create your account"); + await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); + await page.type("input[type=password]", "correct-horse-battery-staple"); + await page.click("button[type=submit]"); + await waitForText(page, "Connect a brain"); + await clickText(page, "label", "Anthropic"); + await page.waitForSelector("input[type=password]"); + await page.type("input[type=password]", "sk-ant-placeholder"); + await clickText(page, "button", "Connect"); + await clickText(page, "button", "Start your first workbench"); +} + +async function createBench(page: Page, message: string): Promise { + await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); + await page.type("textarea", message); + await page.click("button[aria-label='Start this workbench']"); + await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); + await waitForText(page, message); + return new URL(page.url()).pathname.slice("/w/".length); +} + +async function runFlow(page: Page, origin: string): Promise { + await page.setViewport(VIEWPORT); + await signUp(page, origin); + const ids: string[] = []; + for (const [i, message] of MESSAGES.entries()) { + if (i > 0) await page.goto(`${origin}/new`, { waitUntil: "networkidle0" }); + ids.push(await createBench(page, message)); + } + expect(new Set(ids).size).toBe(3); + + // (a) three child tenants under the one workspace, via stock routes. + const principals = await hubJson<{ data: Principal[] }>(page, "/api/me/principals"); + const tenantIds = principals.data + .filter((p) => p.kind === "user" && p.status === "active") + .map((p) => p.tenantId); + const tenants = await Promise.all( + tenantIds.map((id) => hubJson(page, `/api/tenants/${id}`)), + ); + const workspace = tenants.find((t) => t.parentId === null || t.parentId === undefined); + expect(workspace).toBeDefined(); + const children = tenants.filter((t) => t.parentId === workspace?.id); + expect(children.map((t) => t.id).sort()).toEqual([...ids].sort()); + + // (b) each bench deploys exactly one agent (one definition asset). + for (const id of ids) { + const deployments = await hubJson<{ definitionAssetId: string }[]>( + page, + `/api/tenants/${id}/workflows/deployments`, + ); + expect(new Set(deployments.map((d) => d.definitionAssetId)).size).toBe(1); + } + + // (c) sidebar: three bench rows with short titles, plus the worker group. + await page.goto(`${origin}/w/${ids[0]}`, { waitUntil: "networkidle0" }); + await waitForText(page, "Workers"); + const rows = (await page.evaluate( + `Array.from(document.querySelectorAll(".panel-stack-group")).map((g) => ({ label: g.querySelector(".shell-panel-section-label")?.textContent, names: Array.from(g.querySelectorAll(".shell-ch-name")).map((n) => n.textContent) }))`, + )) as { label: string; names: string[] }[]; + const benches = rows.find((g) => g.label === "Workbenches")?.names ?? []; + expect(benches).toHaveLength(3); + for (const name of benches) expect(name.length).toBeLessThanOrEqual(40); + expect(rows.find((g) => g.label === "Workers")?.names.length).toBeGreaterThanOrEqual(1); + + // (d) no bench shows another's opening message. + for (const [i, id] of ids.entries()) { + await page.goto(`${origin}/w/${id}`, { waitUntil: "networkidle0" }); + await waitForText(page, MESSAGES[i] as string); + const text = (await page.evaluate("document.body.innerText")) as string; + for (const [j, other] of MESSAGES.entries()) { + if (j !== i) expect(text.includes(other)).toBe(false); + } + } +} + +describeBrowser("new bench", () => { + const app = bootBrowserApp(); + + test("three benches from /new are isolated child tenants with one agent each", async () => { + const { page, errors } = await app().newPage(); + try { + await runFlow(page, app().origin); + } catch (cause) { + const body = await page.evaluate("document.body.innerText"); + process.stderr.write( + `new-bench failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\n`, + ); + throw cause; + } + await page.close(); + }, 420_000); +}); From c06b387d0befc6396b856b0ea2627055079e3fba Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 08:16:10 -0700 Subject: [PATCH 2/4] test(e2e): widen the viewport only after onboarding (CL-9507) --- e2e/browser/new-bench.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/browser/new-bench.test.ts b/e2e/browser/new-bench.test.ts index 66f5e1b84..40d5b1780 100644 --- a/e2e/browser/new-bench.test.ts +++ b/e2e/browser/new-bench.test.ts @@ -61,8 +61,8 @@ async function createBench(page: Page, message: string): Promise { } async function runFlow(page: Page, origin: string): Promise { - await page.setViewport(VIEWPORT); await signUp(page, origin); + await page.setViewport(VIEWPORT); const ids: string[] = []; for (const [i, message] of MESSAGES.entries()) { if (i > 0) await page.goto(`${origin}/new`, { waitUntil: "networkidle0" }); From af61af54641cf32dae16e858177dd35572797ef6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 08:23:08 -0700 Subject: [PATCH 3/4] test(e2e): fold the three-bench flow into the first-run test (CL-9507) --- e2e/browser/first-run.test.ts | 113 ++++++++++++++++++++++++++--- e2e/browser/new-bench.test.ts | 133 ---------------------------------- 2 files changed, 104 insertions(+), 142 deletions(-) delete mode 100644 e2e/browser/new-bench.test.ts diff --git a/e2e/browser/first-run.test.ts b/e2e/browser/first-run.test.ts index ed4347573..c2ec564a5 100644 --- a/e2e/browser/first-run.test.ts +++ b/e2e/browser/first-run.test.ts @@ -1,20 +1,115 @@ -// A fresh user signs up in a real browser, the client sets up its workspace -// over stock routes, connects a placeholder provider, and creates a first -// workbench that shows up in the sidebar. Asserts on visible UI only. -import { test } from "bun:test"; +// A fresh user signs up in a real browser, connects a placeholder provider, +// and creates three benches from /new: each is its own child tenant with +// exactly one agent, the sidebar lists them by short title, and no bench's +// thread shows another's opening message. One test, because a third hub boot +// in the same `bun test` process hangs its deploys. Inference is not +// scripted in this harness, so the agent's self-naming reply is not covered. +import { expect, test } from "bun:test"; +import type { Page } from "puppeteer-core"; import { bootBrowserApp, browserGate } from "../lib/browser"; -import { runFirstRunFlow } from "../lib/first-run"; +import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run"; const describeBrowser = browserGate(import.meta.path); +const MESSAGES = [ + "Summarize what shipped this week across every repository we track, grouped by team", + "Draft a launch checklist for the onboarding redesign including owners and due dates", + "Research competing pricing pages and list the three patterns we should borrow first", +] as const; + +const VIEWPORT = { width: 1400, height: 900 }; + +type Principal = { tenantId: string; kind: string; status: string }; +type Tenant = { id: string; parentId: string | null }; + +async function hubJson(page: Page, url: string): Promise { + return (await page.evaluate(`fetch(${JSON.stringify(url)}).then((r) => r.json())`)) as T; +} + +async function signUp(page: Page, origin: string): Promise { + await page.goto(origin, { waitUntil: "networkidle0" }); + await clickText(page, "button", "Create an account"); + await waitForText(page, "Create your account"); + await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); + await page.type("input[type=password]", "correct-horse-battery-staple"); + await page.click("button[type=submit]"); + await waitForText(page, "Connect a brain"); + await clickText(page, "label", "Anthropic"); + await page.waitForSelector("input[type=password]"); + await page.type("input[type=password]", "sk-ant-placeholder"); + await clickText(page, "button", "Connect"); + await clickText(page, "button", "Start your first workbench"); +} + +async function createBench(page: Page, message: string): Promise { + await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); + await page.type("textarea", message); + await page.click("button[aria-label='Start this workbench']"); + await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); + await waitForText(page, message); + return new URL(page.url()).pathname.slice("/w/".length); +} + +async function runFlow(page: Page, origin: string): Promise { + await signUp(page, origin); + await page.setViewport(VIEWPORT); + const ids: string[] = []; + for (const [i, message] of MESSAGES.entries()) { + if (i > 0) await page.goto(`${origin}/new`, { waitUntil: "networkidle0" }); + ids.push(await createBench(page, message)); + } + expect(new Set(ids).size).toBe(3); + + // (a) three child tenants under the one workspace, via stock routes. + const principals = await hubJson<{ data: Principal[] }>(page, "/api/me/principals"); + const tenantIds = principals.data + .filter((p) => p.kind === "user" && p.status === "active") + .map((p) => p.tenantId); + const tenants = await Promise.all( + tenantIds.map((id) => hubJson(page, `/api/tenants/${id}`)), + ); + const workspace = tenants.find((t) => t.parentId === null || t.parentId === undefined); + expect(workspace).toBeDefined(); + const children = tenants.filter((t) => t.parentId === workspace?.id); + expect(children.map((t) => t.id).sort()).toEqual([...ids].sort()); + + // (b) each bench deploys exactly one agent (one definition asset). + for (const id of ids) { + const deployments = await hubJson<{ definitionAssetId: string }[]>( + page, + `/api/tenants/${id}/workflows/deployments`, + ); + expect(new Set(deployments.map((d) => d.definitionAssetId)).size).toBe(1); + } + + // (c) sidebar: three bench rows with short titles, plus the worker group. + await page.goto(`${origin}/w/${ids[0]}`, { waitUntil: "networkidle0" }); + await waitForText(page, "Workers"); + const rows = (await page.evaluate( + `Array.from(document.querySelectorAll(".panel-stack-group")).map((g) => ({ label: g.querySelector(".shell-panel-section-label")?.textContent, names: Array.from(g.querySelectorAll(".shell-ch-name")).map((n) => n.textContent) }))`, + )) as { label: string; names: string[] }[]; + const benches = rows.find((g) => g.label === "Workbenches")?.names ?? []; + expect(benches).toHaveLength(3); + for (const name of benches) expect(name.length).toBeLessThanOrEqual(40); + expect(rows.find((g) => g.label === "Workers")?.names.length).toBeGreaterThanOrEqual(1); + + // (d) no bench shows another's opening message. + for (const [i, id] of ids.entries()) { + await page.goto(`${origin}/w/${id}`, { waitUntil: "networkidle0" }); + await waitForText(page, MESSAGES[i] as string); + const text = (await page.evaluate("document.body.innerText")) as string; + for (const [j, other] of MESSAGES.entries()) { + if (j !== i) expect(text.includes(other)).toBe(false); + } + } +} -// The test database outlives a run, so each run signs up a distinct alice. describeBrowser("first run", () => { const app = bootBrowserApp(); - test("sign up, connect a provider, create a first workbench", async () => { + test("sign up, connect a provider, create three isolated benches", async () => { const { page, errors } = await app().newPage(); try { - await runFirstRunFlow(page, app().origin); + await runFlow(page, app().origin); } catch (cause) { const body = await page.evaluate("document.body.innerText"); process.stderr.write( @@ -23,5 +118,5 @@ describeBrowser("first run", () => { throw cause; } await page.close(); - }, 240_000); + }, 420_000); }); diff --git a/e2e/browser/new-bench.test.ts b/e2e/browser/new-bench.test.ts deleted file mode 100644 index 40d5b1780..000000000 --- a/e2e/browser/new-bench.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -// Three benches created from /new in one session: each is its own child -// tenant with exactly one agent, the sidebar lists them by short title, and -// no bench's thread shows another's opening message. Inference is not -// scripted in this harness, so the agent's self-naming reply is not covered. -import { expect, test } from "bun:test"; -import type { Page } from "puppeteer-core"; -import { bootBrowserApp, browserGate } from "../lib/browser"; - -const describeBrowser = browserGate(import.meta.path); -const STEP_TIMEOUT = 60_000; -const MESSAGES = [ - "Summarize what shipped this week across every repository we track, grouped by team", - "Draft a launch checklist for the onboarding redesign including owners and due dates", - "Research competing pricing pages and list the three patterns we should borrow first", -] as const; - -// Page scripts are strings: this tsconfig has no DOM lib. -async function waitForText(page: Page, label: string): Promise { - await page.waitForFunction(`document.body.innerText.includes(${JSON.stringify(label)})`, { - timeout: STEP_TIMEOUT, - }); -} - -async function clickText(page: Page, selector: string, label: string): Promise { - const script = `Array.from(document.querySelectorAll(${JSON.stringify(selector)})).find((e) => e.textContent.includes(${JSON.stringify(label)}))`; - await page.waitForFunction(script, { timeout: STEP_TIMEOUT }); - await page.evaluate(`(${script}).click()`); -} - -const VIEWPORT = { width: 1400, height: 900 }; - -type Principal = { tenantId: string; kind: string; status: string }; -type Tenant = { id: string; parentId: string | null }; - -async function hubJson(page: Page, url: string): Promise { - return (await page.evaluate(`fetch(${JSON.stringify(url)}).then((r) => r.json())`)) as T; -} - -async function signUp(page: Page, origin: string): Promise { - await page.goto(origin, { waitUntil: "networkidle0" }); - await clickText(page, "button", "Create an account"); - await waitForText(page, "Create your account"); - await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); - await page.type("input[type=password]", "correct-horse-battery-staple"); - await page.click("button[type=submit]"); - await waitForText(page, "Connect a brain"); - await clickText(page, "label", "Anthropic"); - await page.waitForSelector("input[type=password]"); - await page.type("input[type=password]", "sk-ant-placeholder"); - await clickText(page, "button", "Connect"); - await clickText(page, "button", "Start your first workbench"); -} - -async function createBench(page: Page, message: string): Promise { - await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); - await page.type("textarea", message); - await page.click("button[aria-label='Start this workbench']"); - await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); - await waitForText(page, message); - return new URL(page.url()).pathname.slice("/w/".length); -} - -async function runFlow(page: Page, origin: string): Promise { - await signUp(page, origin); - await page.setViewport(VIEWPORT); - const ids: string[] = []; - for (const [i, message] of MESSAGES.entries()) { - if (i > 0) await page.goto(`${origin}/new`, { waitUntil: "networkidle0" }); - ids.push(await createBench(page, message)); - } - expect(new Set(ids).size).toBe(3); - - // (a) three child tenants under the one workspace, via stock routes. - const principals = await hubJson<{ data: Principal[] }>(page, "/api/me/principals"); - const tenantIds = principals.data - .filter((p) => p.kind === "user" && p.status === "active") - .map((p) => p.tenantId); - const tenants = await Promise.all( - tenantIds.map((id) => hubJson(page, `/api/tenants/${id}`)), - ); - const workspace = tenants.find((t) => t.parentId === null || t.parentId === undefined); - expect(workspace).toBeDefined(); - const children = tenants.filter((t) => t.parentId === workspace?.id); - expect(children.map((t) => t.id).sort()).toEqual([...ids].sort()); - - // (b) each bench deploys exactly one agent (one definition asset). - for (const id of ids) { - const deployments = await hubJson<{ definitionAssetId: string }[]>( - page, - `/api/tenants/${id}/workflows/deployments`, - ); - expect(new Set(deployments.map((d) => d.definitionAssetId)).size).toBe(1); - } - - // (c) sidebar: three bench rows with short titles, plus the worker group. - await page.goto(`${origin}/w/${ids[0]}`, { waitUntil: "networkidle0" }); - await waitForText(page, "Workers"); - const rows = (await page.evaluate( - `Array.from(document.querySelectorAll(".panel-stack-group")).map((g) => ({ label: g.querySelector(".shell-panel-section-label")?.textContent, names: Array.from(g.querySelectorAll(".shell-ch-name")).map((n) => n.textContent) }))`, - )) as { label: string; names: string[] }[]; - const benches = rows.find((g) => g.label === "Workbenches")?.names ?? []; - expect(benches).toHaveLength(3); - for (const name of benches) expect(name.length).toBeLessThanOrEqual(40); - expect(rows.find((g) => g.label === "Workers")?.names.length).toBeGreaterThanOrEqual(1); - - // (d) no bench shows another's opening message. - for (const [i, id] of ids.entries()) { - await page.goto(`${origin}/w/${id}`, { waitUntil: "networkidle0" }); - await waitForText(page, MESSAGES[i] as string); - const text = (await page.evaluate("document.body.innerText")) as string; - for (const [j, other] of MESSAGES.entries()) { - if (j !== i) expect(text.includes(other)).toBe(false); - } - } -} - -describeBrowser("new bench", () => { - const app = bootBrowserApp(); - - test("three benches from /new are isolated child tenants with one agent each", async () => { - const { page, errors } = await app().newPage(); - try { - await runFlow(page, app().origin); - } catch (cause) { - const body = await page.evaluate("document.body.innerText"); - process.stderr.write( - `new-bench failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\n`, - ); - throw cause; - } - await page.close(); - }, 420_000); -}); From 7ab7025c13813dd4f8d88fed260edbe67fd8bdc1 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 30 Sep 2026 08:58:03 -0700 Subject: [PATCH 4/4] fix(web): e2e suites shut down the hubs they boot (CL-9620) A hub left running keeps reconciling the shared database, and fails the deployment allocations of any hub booted after it in the same process with provisioner_unavailable. That ended the first run's Myra before the flow returned to /new, which sent the page back to /onboarding. --- e2e/grants/grants-not-walkable.test.ts | 10 +++++++++- e2e/hub/composition.test.ts | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/e2e/grants/grants-not-walkable.test.ts b/e2e/grants/grants-not-walkable.test.ts index 8415cfeb8..1fe9d7c52 100644 --- a/e2e/grants/grants-not-walkable.test.ts +++ b/e2e/grants/grants-not-walkable.test.ts @@ -2,7 +2,7 @@ // from an ancestor tenant, grants never do. Booting the hub runs package // migrations, so the suite skips without a reachable DATABASE_URL. -import { expect, test } from "bun:test"; +import { afterAll, expect, test } from "bun:test"; import { type } from "arktype"; import { dbGate } from "../lib/db-gate"; import { installDisposableHubDataDir } from "../lib/disposable-hub-data-dir"; @@ -23,6 +23,13 @@ if (databaseUrl !== "") { process.env["DB_NAME"] = url.pathname.replace(/^\//, ""); } +// A hub left running keeps reconciling the shared database and fails the +// allocations of every hub booted after it in the same process. +const closers: (() => Promise)[] = []; +afterAll(async () => { + for (const close of closers) await close(); +}); + const Id = type({ id: "string" }); const GrantPage = type({ data: type({ id: "string" }).array() }); const Evaluation = type({ effect: "string", matchingGrants: type({ id: "string" }).array() }); @@ -31,6 +38,7 @@ describeIfDb("grants are explicit per tenant", () => { test("a credential resolves from a child tenant; a grant does not", async () => { const { createHubServer } = await import("../../apps/hub/src/server"); const hub = await createHubServer(); + closers.push(() => hub.shutdown()); const origin = "http://localhost"; const suffix = crypto.randomUUID().slice(0, 8); diff --git a/e2e/hub/composition.test.ts b/e2e/hub/composition.test.ts index a4dc7b2db..deb4ebd8c 100644 --- a/e2e/hub/composition.test.ts +++ b/e2e/hub/composition.test.ts @@ -43,6 +43,7 @@ describeIfDb("boot", () => { test("serves stock platform routes", async () => { const { createHubServer } = await import("../../apps/hub/src/server"); const opts = await createHubServer(); + closers.push(() => opts.shutdown()); const status = await opts.fetch(new Request("http://localhost/status")); expect(status.status).toBe(200); @@ -54,6 +55,7 @@ describeIfDb("boot", () => { test("a Corbits mount (mailbox) sits inside the native tenant middleware", async () => { const { createHubServer } = await import("../../apps/hub/src/server"); const opts = await createHubServer(); + closers.push(() => opts.shutdown()); // Anonymous request to a Corbits-mounted route: the platform's tenant // middleware answers 401 before the mailbox library's own handler runs.