diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93be0be42..afe4561b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,18 @@ jobs: - run: bun run fmt:check # Integration and browser suites under e2e/. CI=true turns DB- and - # Chrome-gated skips into hard failures. - e2e: + # Chrome-gated skips into hard failures. Browser suites shard by + # round-robin over the sorted file list, so a new test file joins a shard + # automatically — no per-file list to keep in sync. Fewer files per shard + # also means fewer hub boots per `bun test` process, which is the + # documented flake boundary (see e2e/browser/first-run.test.ts). + e2e-browser: runs-on: ubuntu-latest timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2] services: postgres: image: pgvector/pgvector:pg16 @@ -60,4 +68,55 @@ jobs: - run: | mkdir -p "$RUNNER_TEMP/hub-data" echo "HUB_DATA_DIR=$RUNNER_TEMP/hub-data" >> "$GITHUB_ENV" - - run: bun run test:e2e + - run: | + LC_ALL=C + i=0 + files="" + for f in e2e/browser/*.test.ts e2e/mcp/*.test.ts; do + if [ "$((i % 3))" = "${{ matrix.shard }}" ]; then files="$files $f"; fi + i=$((i + 1)) + done + echo "shard ${{ matrix.shard }} files:$files" + # shellcheck disable=SC2086 + bun test $files + + # Hub-level suites: no Chrome, no browser boots. + e2e-hub: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: workbench + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + CI: "true" + DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + # Throwaway keys for the ephemeral CI hub; same placeholders as .env.example. + CREDENTIAL_ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" + PRINCIPAL_KEY_ENCRYPTION_KEY: "1111111111111111111111111111111111111111111111111111111111111111" + SIDECAR_CREDENTIAL_ENCRYPTION_KEY: "2222222222222222222222222222222222222222222222222222222222222222" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-workbench + - run: | + mkdir -p "$RUNNER_TEMP/hub-data" + echo "HUB_DATA_DIR=$RUNNER_TEMP/hub-data" >> "$GITHUB_ENV" + # The worker bundle is gitignored build output (see packages/worker): + # browser shards get it as a side effect of the web build, so the hub + # job builds it directly. The web-build freshness unit suite lives in + # e2e/lib (no Chrome, no DB), which no shard glob and no package test + # root would otherwise run. + - run: bun run build + working-directory: packages/worker + - run: bun test e2e/lib/web-build.test.ts + - run: bun test e2e/grants e2e/hub e2e/install e2e/workflows diff --git a/e2e/browser/composer-layout.test.ts b/e2e/browser/composer-layout.test.ts deleted file mode 100644 index 7d64f07f3..000000000 --- a/e2e/browser/composer-layout.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -// CL-9785: the composer never scrolls horizontally (long unbroken tokens -// wrap) and sidebar rows go full-width with their radius intact. -import { expect, test } from "bun:test"; -import { bootBrowserApp, browserGate } from "../lib/browser"; -import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run"; - -const describeBrowser = browserGate(import.meta.path); - -const VIEWPORT = { width: 1400, height: 900 }; -// One unbroken token wider than any composer: without wrapping it would force -// a horizontal scrollbar. -const LONG_TOKEN = "a".repeat(400); - -type RowBox = { - active: string | null; - offsetWidth: number; - containerWidth: number; - radius: string; -}; - -const ROW_BOX_SCRIPT = `(() => { - const out = []; - for (const row of document.querySelectorAll(".shell-ch-row")) { - const parent = row.parentElement; - out.push({ - active: row.getAttribute("aria-current"), - offsetWidth: row.offsetWidth, - containerWidth: parent ? parent.clientWidth : -1, - radius: getComputedStyle(row).borderTopLeftRadius, - }); - } - return out; -})()`; - -describeBrowser("composer layout", () => { - const app = bootBrowserApp(); - - test("long tokens wrap and sidebar rows go full-width", async () => { - const { page, errors } = await app().newPage(); - try { - await page.setViewport(VIEWPORT); - await page.goto(app().origin, { waitUntil: "networkidle0" }); - await clickText(page, "button", "Create an account"); - await waitForText(page, "Create your account"); - await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); - await page.type("input[type=password]", "correct-horse-battery-staple"); - await page.click("button[type=submit]"); - - // A placeholder key is enough: connecting stores a credential without - // calling the provider, and creating a bench needs no inference. - await waitForText(page, "Choose your AI model"); - await clickText(page, "label", "Anthropic"); - await page.waitForSelector("input[type=password]"); - await page.type("input[type=password]", "sk-ant-placeholder"); - await clickText(page, "button", "Connect"); - - // The new-workbench prompt is a bare composer: a long unbroken token - // must wrap (no horizontal overflow) instead of scrolling. - await page.waitForSelector(".chat-composer-input", { timeout: STEP_TIMEOUT }); - await page.type(".chat-composer-input", LONG_TOKEN); - await page.waitForFunction( - `(() => { const el = document.querySelector(".chat-composer-input"); return el !== null && el.scrollWidth <= el.clientWidth + 1; })()`, - { timeout: STEP_TIMEOUT }, - ); - const overflowX = (await page.evaluate( - `getComputedStyle(document.querySelector(".chat-composer-input")).overflowX`, - )) as string; - expect(overflowX).toBe("hidden"); - - // Create the bench; its sidebar row is populated and active. - await page.goto(`${app().origin}/new`, { waitUntil: "networkidle0" }); - await page.waitForSelector(".chat-composer-input", { timeout: STEP_TIMEOUT }); - await page.type(".chat-composer-input", "Summarize what shipped this week"); - await page.click("button[aria-label='Start this workbench']"); - await page.waitForFunction(`location.pathname.startsWith("/w/")`, { - timeout: STEP_TIMEOUT, - }); - await page.waitForSelector(".shell-ch-row", { timeout: STEP_TIMEOUT }); - - const rows = (await page.evaluate(ROW_BOX_SCRIPT)) as RowBox[]; - expect(rows.length).toBeGreaterThan(0); - for (const row of rows) { - expect(Math.abs(row.offsetWidth - row.containerWidth)).toBeLessThanOrEqual(1); - expect(row.radius).not.toBe("0px"); - } - expect(rows.some((row) => row.active === "true")).toBe(true); - - // Full-width did not smush the radius: it is stable under hover. - const hovered = (await page.evaluate( - `(() => { const row = document.querySelector(".shell-ch-row"); return row === null ? null : getComputedStyle(row).borderTopLeftRadius; })()`, - )) as string | null; - await page.hover(".shell-ch-row"); - const hoverRadius = (await page.evaluate( - `(() => { const row = document.querySelector(".shell-ch-row:hover"); return row === null ? null : getComputedStyle(row).borderTopLeftRadius; })()`, - )) as string | null; - expect(hovered).not.toBeNull(); - expect(hoverRadius).toBe(hovered); - } catch (cause) { - const body = await page.evaluate("document.body.innerText"); - process.stderr.write( - `composer-layout failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\n`, - ); - throw cause; - } - await page.close(); - }, 240_000); -}); diff --git a/e2e/browser/first-run.test.ts b/e2e/browser/first-run.test.ts index d65e68d1a..71aef9857 100644 --- a/e2e/browser/first-run.test.ts +++ b/e2e/browser/first-run.test.ts @@ -1,30 +1,85 @@ -// A fresh user signs up in a real browser, connects a placeholder provider, -// and creates three benches from /new: each is its own child tenant with -// exactly one agent, the sidebar lists them by short title, and no bench's -// thread shows another's opening message. One test, because a third hub boot -// in the same `bun test` process hangs its deploys. Inference is not -// scripted in this harness, so the agent's self-naming reply is not covered. +// First-run journeys sharing one hub boot, in declaration order: the hub +// seeds nothing, signup connects a placeholder provider, three benches +// isolate by tenant, and the composer lays out. One bootBrowserApp for all +// three tests — an extra hub boot in the same `bun test` process risks +// hanging deploys, and every boot multiplies CI minutes. The seed-nothing +// test stays first: it probes the hub before any signup on this boot exists. +// Each test signs up its own user, so tenants never leak across tests. import { expect, test } from "bun:test"; import type { Page } from "puppeteer-core"; import { bootBrowserApp, browserGate } from "../lib/browser"; import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run"; const describeBrowser = browserGate(import.meta.path); + +const VIEWPORT = { width: 1400, height: 900 }; +// One unbroken token wider than any composer: without wrapping it would force +// a horizontal scrollbar. +const LONG_TOKEN = "a".repeat(400); const MESSAGES = [ "Summarize what shipped this week across every repository we track, grouped by team", "Draft a launch checklist for the onboarding redesign including owners and due dates", "Research competing pricing pages and list the three patterns we should borrow first", ] as const; -const VIEWPORT = { width: 1400, height: 900 }; - type Principal = { tenantId: string; kind: string; status: string }; type Tenant = { id: string; parentId: string | null }; +type Probe = { status: number; body: unknown }; +type Snapshot = { tenants: Record[]; deployments: Record[] }; +type RowBox = { + active: string | null; + offsetWidth: number; + containerWidth: number; + radius: string; +}; async function hubJson(page: Page, url: string): Promise { return (await page.evaluate(`fetch(${JSON.stringify(url)}).then((r) => r.json())`)) as T; } +// Same-origin fetch from the page, so the session cookie rides along. +async function probe(page: Page, path: string): Promise { + return (await page.evaluate( + `fetch(${JSON.stringify(path)}).then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }))`, + )) as Probe; +} + +function items(body: unknown): Record[] { + const list = Array.isArray(body) ? body : ((body as { data?: unknown } | null)?.data ?? []); + return list as Record[]; +} + +async function snapshot(page: Page): Promise { + const principals = await probe(page, "/api/me/principals"); + expect(principals.status).toBe(200); + const tenantIds = [...new Set(items(principals.body).map((p) => String(p["tenantId"])))]; + const tenants: Record[] = []; + const deployments: Record[] = []; + for (const id of tenantIds) { + const tenant = await probe(page, `/api/tenants/${id}`); + expect(tenant.status).toBe(200); + tenants.push(tenant.body as Record); + const deps = await probe(page, `/api/tenants/${id}/workflows/deployments`); + expect(deps.status).toBe(200); + deployments.push(...items(deps.body)); + } + return { tenants, deployments }; +} + +const ROW_BOX_SCRIPT = `(() => { + const out = []; + for (const row of document.querySelectorAll(".shell-ch-row")) { + const parent = row.parentElement; + out.push({ + active: row.getAttribute("aria-current"), + offsetWidth: row.offsetWidth, + containerWidth: parent ? parent.clientWidth : -1, + radius: getComputedStyle(row).borderTopLeftRadius, + }); + } + return out; +})()`; + async function signUp(page: Page, origin: string): Promise { await page.goto(origin, { waitUntil: "networkidle0" }); await clickText(page, "button", "Create an account"); @@ -35,7 +90,7 @@ async function signUp(page: Page, origin: string): Promise { await waitForText(page, "Choose your AI model"); await clickText(page, "label", "Anthropic"); await page.waitForSelector("input[type=password]"); - await page.type("input[type=password]", "sk-ant-placeholder"); + await page.type("input[type=password]", "[REDACTED]"); await clickText(page, "button", "Connect"); } @@ -124,6 +179,73 @@ async function runFlow(page: Page, origin: string): Promise { describeBrowser("first run", () => { const app = bootBrowserApp(); + test("the hub seeds nothing and reloads never duplicate setup", async () => { + const { page } = await app().newPage(); + await page.goto(app().origin, { waitUntil: "networkidle0" }); + + // Before signup the hub has nothing for this visitor. + expect((await probe(page, "/api/me/principals")).status).toBe(401); + expect((await probe(page, "/api/tenants")).status).toBeOneOf([401, 403, 404]); + + await clickText(page, "button", "Create an account"); + await waitForText(page, "Create your account"); + await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); + await page.type("input[type=password]", "correct-horse-battery-staple"); + await page.click("button[type=submit]"); + + await waitForText(page, "Choose your AI model"); + await clickText(page, "label", "Anthropic"); + await page.waitForSelector("input[type=password]"); + await page.type("input[type=password]", "[REDACTED]"); + await clickText(page, "button", "Connect"); + await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); + + // Setup leaves one workspace tenant and no deployment anywhere. + const setup = await snapshot(page); + expect(setup.tenants).toHaveLength(1); + expect(setup.tenants[0]?.["parentId"] ?? null).toBeNull(); + expect(setup.deployments).toHaveLength(0); + // The stock tenant carries no creator field; the signed-in user's own + // principal holding owner on it is the client-created proof. + const mine = items((await probe(page, "/api/me/principals")).body); + expect(mine).toHaveLength(1); + expect(mine[0]?.["tenantId"]).toBe(setup.tenants[0]?.["id"]); + expect(mine[0]?.["kind"]).toBe("user"); + expect(JSON.stringify(mine[0]?.["roles"])).toContain("owner"); + + // The first bench carries the first worker. + await page.type("textarea", "Summarize what shipped this week"); + await page.click("button[aria-label='Start this workbench']"); + await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); + const benchId = new URL(page.url()).pathname.slice("/w/".length); + + const first = await snapshot(page); + expect(first.tenants).toHaveLength(2); + expect(first.tenants.find((t) => t["id"] === benchId)?.["parentId"]).toBe( + setup.tenants[0]?.["id"], + ); + expect(first.deployments).toHaveLength(1); + expect(first.deployments[0]?.["tenantId"]).toBe(benchId); + + for (let i = 0; i < 2; i++) { + await page.reload({ waitUntil: "networkidle0" }); + // Give the client's setup time to (wrongly) run again. + await new Promise((resolve) => setTimeout(resolve, 4_000)); + const again = await snapshot(page); + expect(again.tenants.map((t) => t["id"])).toEqual(first.tenants.map((t) => t["id"])); + // Status moves (pending -> deployed) and a deployment the hub itself + // failed is retried by design, so assert no second live deployment and + // that the first one is never replaced out from under us. + const live = again.deployments.filter( + (d) => !["failed", "released", "destroy_failed"].includes(String(d["status"])), + ); + expect(live.length).toBeLessThanOrEqual(1); + const ids = again.deployments.map((d) => d["id"]); + expect(ids).toContain(first.deployments[0]?.["id"]); + } + await page.close(); + }, 240_000); + test("sign up, connect a provider, create three isolated benches", async () => { const { page, errors } = await app().newPage(); try { @@ -137,4 +259,74 @@ describeBrowser("first run", () => { } await page.close(); }, 420_000); + + test("long tokens wrap and sidebar rows go full-width", async () => { + const { page, errors } = await app().newPage(); + try { + await page.setViewport(VIEWPORT); + await page.goto(app().origin, { waitUntil: "networkidle0" }); + await clickText(page, "button", "Create an account"); + await waitForText(page, "Create your account"); + await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); + await page.type("input[type=password]", "correct-horse-battery-staple"); + await page.click("button[type=submit]"); + + // A placeholder key is enough: connecting stores a credential without + // calling the provider, and creating a bench needs no inference. + await waitForText(page, "Choose your AI model"); + await clickText(page, "label", "Anthropic"); + await page.waitForSelector("input[type=password]"); + await page.type("input[type=password]", "[REDACTED]"); + await clickText(page, "button", "Connect"); + + // The new-workbench prompt is a bare composer: a long unbroken token + // must wrap (no horizontal overflow) instead of scrolling. + await page.waitForSelector(".chat-composer-input", { timeout: STEP_TIMEOUT }); + await page.type(".chat-composer-input", LONG_TOKEN); + await page.waitForFunction( + `(() => { const el = document.querySelector(".chat-composer-input"); return el !== null && el.scrollWidth <= el.clientWidth + 1; })()`, + { timeout: STEP_TIMEOUT }, + ); + const overflowX = (await page.evaluate( + `getComputedStyle(document.querySelector(".chat-composer-input")).overflowX`, + )) as string; + expect(overflowX).toBe("hidden"); + + // Create the bench; its sidebar row is populated and active. + await page.goto(`${app().origin}/new`, { waitUntil: "networkidle0" }); + await page.waitForSelector(".chat-composer-input", { timeout: STEP_TIMEOUT }); + await page.type(".chat-composer-input", "Summarize what shipped this week"); + await page.click("button[aria-label='Start this workbench']"); + await page.waitForFunction(`location.pathname.startsWith("/w/")`, { + timeout: STEP_TIMEOUT, + }); + await page.waitForSelector(".shell-ch-row", { timeout: STEP_TIMEOUT }); + + const rows = (await page.evaluate(ROW_BOX_SCRIPT)) as RowBox[]; + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(Math.abs(row.offsetWidth - row.containerWidth)).toBeLessThanOrEqual(1); + expect(row.radius).not.toBe("0px"); + } + expect(rows.some((row) => row.active === "true")).toBe(true); + + // Full-width did not smush the radius: it is stable under hover. + const hovered = (await page.evaluate( + `(() => { const row = document.querySelector(".shell-ch-row"); return row === null ? null : getComputedStyle(row).borderTopLeftRadius; })()`, + )) as string | null; + await page.hover(".shell-ch-row"); + const hoverRadius = (await page.evaluate( + `(() => { const row = document.querySelector(".shell-ch-row:hover"); return row === null ? null : getComputedStyle(row).borderTopLeftRadius; })()`, + )) as string | null; + expect(hovered).not.toBeNull(); + expect(hoverRadius).toBe(hovered); + } catch (cause) { + const body = await page.evaluate("document.body.innerText"); + process.stderr.write( + `composer-layout failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\n`, + ); + throw cause; + } + await page.close(); + }, 240_000); }); diff --git a/e2e/browser/reply.test.ts b/e2e/browser/reply.test.ts deleted file mode 100644 index 3842f65be..000000000 --- a/e2e/browser/reply.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -// A fresh user connects an inference provider served by aimock, creates a -// bench from /new, and the worker's reply lands in the thread while the -// bench stops showing "working". The reply names the worker, which then shows -// everywhere; a rename in the bench drawer survives a reload. -import { expect, test } from "bun:test"; -import { bootAimock, MOCK_REPLY } from "../lib/aimock"; -import { bootBrowserApp, browserGate } from "../lib/browser"; -import { clickText, STEP_TIMEOUT, waitForText } from "../lib/first-run"; - -const describeBrowser = browserGate(import.meta.path); - -describeBrowser("worker reply", () => { - const aimock = bootAimock([ - { - match: { predicate: () => true }, - response: { content: `${MOCK_REPLY}\n\nName: Ada` }, - }, - ]); - const app = bootBrowserApp(); - - test("the worker's reply lands in the thread", async () => { - const { page, errors } = await app().newPage(); - try { - await page.goto(app().origin, { waitUntil: "networkidle0" }); - await clickText(page, "button", "Create an account"); - await waitForText(page, "Create your account"); - await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); - await page.type("input[type=password]", "correct-horse-battery-staple"); - await page.click("button[type=submit]"); - - // The Anthropic option pins api.anthropic.com; only Custom takes a base URL. - await waitForText(page, "Choose your AI model"); - await clickText(page, "label", "Custom"); - await page.waitForSelector("input[type=password]"); - const inputs = await page.$$("form input:not([type=radio])"); - expect(inputs.length).toBeGreaterThanOrEqual(3); - await page.type("form input:not([type=radio]):not([type=password])", `${aimock().url}/v1`); - await page.type("input[type=password]", "mock"); - const model = await page.$$("form input:not([type=radio]):not([type=password])"); - await model[1]?.type("mock-model"); - await clickText(page, "button", "Connect"); - - await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); - await page.type("textarea", "Say hello"); - await page.click("button[aria-label='Start this workbench']"); - await page.waitForFunction(`location.pathname.startsWith("/w/")`, { - timeout: STEP_TIMEOUT, - }); - - await waitForText(page, MOCK_REPLY); - await page.waitForFunction(`!document.body.innerText.toLowerCase().includes("working")`, { - timeout: STEP_TIMEOUT, - }); - expect(aimock().journal().length).toBeGreaterThan(0); - - // The worker named itself: pill, sidebar and message header show it, - // and the marker line is never rendered. - const benchUrl = page.url(); - await page.waitForFunction( - `document.querySelector(".bench-pill")?.innerText.includes("Ada")`, - { - timeout: STEP_TIMEOUT, - }, - ); - await page.waitForFunction( - `document.querySelector("[aria-label='Workbenches and workers']")?.textContent.includes("Ada")`, - { timeout: 20_000 }, - ); - expect(await page.evaluate("document.body.innerText.includes('Name: Ada')")).toBe(false); - - await page.goto(`${app().origin}/workers`, { waitUntil: "networkidle0" }); - await waitForText(page, "Ada"); - - // Rename from the drawer; the new name persists across a reload. - await page.goto(benchUrl, { waitUntil: "networkidle0" }); - await page.click(".bench-pill"); - await page.waitForSelector("input[aria-label='Worker name']"); - await page.evaluate(`document.querySelector("input[aria-label='Worker name']").select()`); - await page.type("input[aria-label='Worker name']", "Bea"); - await clickText(page, "button", "Rename"); - await page.waitForFunction( - `document.querySelector(".bench-pill")?.innerText.includes("Bea")`, - { - timeout: STEP_TIMEOUT, - }, - ); - await page.reload({ waitUntil: "networkidle0" }); - await page.waitForFunction( - `document.querySelector(".bench-pill")?.innerText.includes("Bea")`, - { - timeout: STEP_TIMEOUT, - }, - ); - } catch (cause) { - const body = await page.evaluate("document.body.innerText"); - process.stderr.write( - `reply failed at ${page.url()}:\n${String(body)}\nconsole: ${errors.join(" | ")}\njournal: ${JSON.stringify(aimock().journal()).slice(0, 4000)}\n`, - ); - throw cause; - } - await page.close(); - }, 240_000); -}); diff --git a/e2e/browser/setup.test.ts b/e2e/browser/setup.test.ts deleted file mode 100644 index cf9edb970..000000000 --- a/e2e/browser/setup.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -// The hub seeds nothing: before signup it has no tenants for the visitor, -// and after the client's setup the workspace exists with no deployment, -// created by the signed-in principal; the first bench brings the first -// worker. Reloading the app never duplicates either. -import { expect, test } from "bun:test"; -import type { Page } from "puppeteer-core"; -import { bootBrowserApp, browserGate } from "../lib/browser"; - -const describeBrowser = browserGate(import.meta.path); -const STEP_TIMEOUT = 60_000; - -async function waitForText(page: Page, label: string): Promise { - await page.waitForFunction(`document.body.innerText.includes(${JSON.stringify(label)})`, { - timeout: STEP_TIMEOUT, - }); -} - -async function clickText(page: Page, selector: string, label: string): Promise { - const script = `Array.from(document.querySelectorAll(${JSON.stringify(selector)})).find((e) => e.textContent.includes(${JSON.stringify(label)}))`; - await page.waitForFunction(script, { timeout: STEP_TIMEOUT }); - await page.evaluate(`(${script}).click()`); -} - -type Probe = { status: number; body: unknown }; - -// Same-origin fetch from the page, so the session cookie rides along. -async function probe(page: Page, path: string): Promise { - return (await page.evaluate( - `fetch(${JSON.stringify(path)}).then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }))`, - )) as Probe; -} - -function items(body: unknown): Record[] { - const list = Array.isArray(body) ? body : ((body as { data?: unknown } | null)?.data ?? []); - return list as Record[]; -} - -type Snapshot = { tenants: Record[]; deployments: Record[] }; - -async function snapshot(page: Page): Promise { - const principals = await probe(page, "/api/me/principals"); - expect(principals.status).toBe(200); - const tenantIds = [...new Set(items(principals.body).map((p) => String(p["tenantId"])))]; - const tenants: Record[] = []; - const deployments: Record[] = []; - for (const id of tenantIds) { - const tenant = await probe(page, `/api/tenants/${id}`); - expect(tenant.status).toBe(200); - tenants.push(tenant.body as Record); - const deps = await probe(page, `/api/tenants/${id}/workflows/deployments`); - expect(deps.status).toBe(200); - deployments.push(...items(deps.body)); - } - return { tenants, deployments }; -} - -describeBrowser("client setup", () => { - const app = bootBrowserApp(); - - test("the hub seeds nothing and reloads never duplicate setup", async () => { - const { page } = await app().newPage(); - await page.goto(app().origin, { waitUntil: "networkidle0" }); - - // Before signup the hub has nothing for this visitor. - expect((await probe(page, "/api/me/principals")).status).toBe(401); - expect((await probe(page, "/api/tenants")).status).toBeOneOf([401, 403, 404]); - - await clickText(page, "button", "Create an account"); - await waitForText(page, "Create your account"); - await page.type("input[type=email]", `alice+${String(Date.now())}@example.com`); - await page.type("input[type=password]", "correct-horse-battery-staple"); - await page.click("button[type=submit]"); - - await waitForText(page, "Choose your AI model"); - await clickText(page, "label", "Anthropic"); - await page.waitForSelector("input[type=password]"); - await page.type("input[type=password]", "sk-ant-placeholder"); - await clickText(page, "button", "Connect"); - await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); - - // Setup leaves one workspace tenant and no deployment anywhere. - const setup = await snapshot(page); - expect(setup.tenants).toHaveLength(1); - expect(setup.tenants[0]?.["parentId"] ?? null).toBeNull(); - expect(setup.deployments).toHaveLength(0); - // The stock tenant carries no creator field; the signed-in user's own - // principal holding owner on it is the client-created proof. - const mine = items((await probe(page, "/api/me/principals")).body); - expect(mine).toHaveLength(1); - expect(mine[0]?.["tenantId"]).toBe(setup.tenants[0]?.["id"]); - expect(mine[0]?.["kind"]).toBe("user"); - expect(JSON.stringify(mine[0]?.["roles"])).toContain("owner"); - - // The first bench carries the first worker. - await page.type("textarea", "Summarize what shipped this week"); - await page.click("button[aria-label='Start this workbench']"); - await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); - const benchId = new URL(page.url()).pathname.slice("/w/".length); - - const first = await snapshot(page); - expect(first.tenants).toHaveLength(2); - expect(first.tenants.find((t) => t["id"] === benchId)?.["parentId"]).toBe( - setup.tenants[0]?.["id"], - ); - expect(first.deployments).toHaveLength(1); - expect(first.deployments[0]?.["tenantId"]).toBe(benchId); - - for (let i = 0; i < 2; i++) { - await page.reload({ waitUntil: "networkidle0" }); - // Give the client's setup time to (wrongly) run again. - await new Promise((resolve) => setTimeout(resolve, 4_000)); - const again = await snapshot(page); - expect(again.tenants.map((t) => t["id"])).toEqual(first.tenants.map((t) => t["id"])); - // Status moves (pending -> deployed) and a deployment the hub itself - // failed is retried by design, so assert no second live deployment and - // that the first one is never replaced out from under us. - const live = again.deployments.filter( - (d) => !["failed", "released", "destroy_failed"].includes(String(d["status"])), - ); - expect(live.length).toBeLessThanOrEqual(1); - const ids = again.deployments.map((d) => d["id"]); - expect(ids).toContain(first.deployments[0]?.["id"]); - } - await page.close(); - }, 240_000); -}); diff --git a/e2e/browser/zero-to-one.test.ts b/e2e/browser/zero-to-one.test.ts index ed1d5dea5..e45ed1850 100644 --- a/e2e/browser/zero-to-one.test.ts +++ b/e2e/browser/zero-to-one.test.ts @@ -1,6 +1,7 @@ -// The 0-to-1 gate against scripted inference: signup, provider, a bench from -// /new, a worker reply, a follow-up reply, an artifact the worker saves, and a -// routine "Run now" brief. The Linear-connect step is not here (CL-9513). +// The 0-to-1 gate against scripted inference, one flow: signup, provider, a +// bench from /new, a worker reply that names the worker, a follow-up reply, +// an artifact the worker saves, a routine "Run now" brief, and a worker +// rename that survives reload. The Linear-connect step is not here (CL-9513). import { expect, test } from "bun:test"; import { bootAimock, MOCK_REPLY, type Fixture } from "../lib/aimock"; import { bootBrowserApp, browserGate } from "../lib/browser"; @@ -15,6 +16,14 @@ const ARTIFACT_TITLE = "Gate artifact"; const ARTIFACT_DONE = "Saved the gate artifact."; const BRIEF = "Brief: nothing shipped this week."; +// The worker-reply test's worker names itself. Registered after the scripted +// turns so exact matches win, ahead of bootAimock's default MOCK_REPLY +// fallback — which the gate's own first reply still falls through to. +const ADA_REPLY: Fixture = { + match: { predicate: () => true }, + response: { content: `${MOCK_REPLY}\n\nName: Ada` }, +}; + const FIXTURES: Fixture[] = [ { match: { userMessage: FOLLOW_UP }, response: { content: FOLLOW_UP_REPLY } }, { @@ -75,7 +84,7 @@ async function send(page: import("puppeteer-core").Page, text: string): Promise< } describeBrowser("0-to-1 gate", () => { - const aimock = bootAimock(FIXTURES); + const aimock = bootAimock([...FIXTURES, ADA_REPLY]); const app = bootBrowserApp(); test("signup to reply, follow-up, artifact, and routine brief", async () => { @@ -96,13 +105,48 @@ describeBrowser("0-to-1 gate", () => { await page.type("input[type=password]", "mock"); const fields = await page.$$("form input:not([type=radio]):not([type=password])"); await fields[1]?.type("mock-model"); + // Custom takes base URL, key, and model: three non-radio inputs. + const inputs = await page.$$("form input:not([type=radio])"); + expect(inputs.length).toBeGreaterThanOrEqual(3); await clickText(page, "button", "Connect"); await page.waitForSelector("textarea", { timeout: STEP_TIMEOUT }); await page.type("textarea", "Say hello"); + // Wait for the composer's enabled state before clicking: the send + // button enables after the workspace roles load (CL-9780), and a + // click while the first bench is still bootstrapping races the + // navigate-to-/w/ assertion below (a known non-deterministic flake). + await page.waitForFunction( + `(() => { + const send = document.querySelector("button[aria-label='Start this workbench']"); + return send instanceof HTMLButtonElement && !send.disabled; + })()`, + { timeout: STEP_TIMEOUT }, + ); await page.click("button[aria-label='Start this workbench']"); await page.waitForFunction(`location.pathname.startsWith("/w/")`, { timeout: STEP_TIMEOUT }); + const benchUrl = page.url(); await waitForText(page, MOCK_REPLY); + // The worker reply proves the scripted inference provider was dialed, + // not just the UI shell loading. + expect(aimock().journal().length).toBeGreaterThan(0); + + // The worker named itself: pill, sidebar and message header show it, + // and the marker line is never rendered. + await page.waitForFunction( + `document.querySelector(".bench-pill")?.innerText.includes("Ada")`, + { + timeout: STEP_TIMEOUT, + }, + ); + await page.waitForFunction( + `document.querySelector("[aria-label='Workbenches and workers']")?.textContent.includes("Ada")`, + { timeout: 20_000 }, + ); + expect(await page.evaluate("document.body.innerText.includes('Name: Ada')")).toBe(false); + await page.goto(`${app().origin}/workers`, { waitUntil: "networkidle0" }); + await waitForText(page, "Ada"); + await page.goto(benchUrl, { waitUntil: "networkidle0" }); await send(page, FOLLOW_UP); await waitForText(page, FOLLOW_UP_REPLY); @@ -124,6 +168,27 @@ describeBrowser("0-to-1 gate", () => { }; expect(run.trigger?.status).toBe(202); await waitForText(page, BRIEF); + + // Rename from the drawer; the new name persists across a reload. + await page.goto(benchUrl, { waitUntil: "networkidle0" }); + await page.click(".bench-pill"); + await page.waitForSelector("input[aria-label='Worker name']"); + await page.evaluate(`document.querySelector("input[aria-label='Worker name']").select()`); + await page.type("input[aria-label='Worker name']", "Bea"); + await clickText(page, "button", "Rename"); + await page.waitForFunction( + `document.querySelector(".bench-pill")?.innerText.includes("Bea")`, + { + timeout: STEP_TIMEOUT, + }, + ); + await page.reload({ waitUntil: "networkidle0" }); + await page.waitForFunction( + `document.querySelector(".bench-pill")?.innerText.includes("Bea")`, + { + timeout: STEP_TIMEOUT, + }, + ); } catch (cause) { const body = await page.evaluate("document.body.innerText"); process.stderr.write( diff --git a/e2e/lib/browser.ts b/e2e/lib/browser.ts index 40a45fc9a..62194e763 100644 --- a/e2e/lib/browser.ts +++ b/e2e/lib/browser.ts @@ -10,6 +10,7 @@ import puppeteer, { type Browser, type Page } from "puppeteer-core"; import { dbGate } from "./db-gate"; import { bootHub, type BootedHub } from "./hub"; import { REPO_ROOT } from "./database-url"; +import { isWebBuildFresh, writeWebBuildMarker } from "./web-build"; const DEFAULT_CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; const WEB_DIR = path.join(REPO_ROOT, "apps", "web"); @@ -44,14 +45,27 @@ export type BrowserApp = { newPage: () => Promise<{ page: Page; errors: string[] }>; }; -// Always rebuild: a leftover dist from an older checkout serves stale UI. +// The web build also emits the worker bundle (see apps/web/vite.config.ts), +// which non-browser suites consume without ever building the web bundle. +const REQUIRED_ARTIFACTS = [ + "packages/worker/bundle/workflow-bundle.js", + "packages/worker/bundle/directors-bundle.js", + "packages/worker/bundle/tool-names.json", +]; + +// One build per process: files in one `bun test e2e` run serially, so the +// first browser suite builds and the rest reuse its dist while the tree is +// unchanged. A stale dist still rebuilds — the marker only matches the tree +// that produced it. async function ensureWebBuild(): Promise { + if (isWebBuildFresh(WEB_DIR, REPO_ROOT, REQUIRED_ARTIFACTS)) return; const proc = Bun.spawn(["bun", "run", "build"], { cwd: WEB_DIR, stdout: "inherit", stderr: "inherit", }); if ((await proc.exited) !== 0) throw new Error("apps/web build failed"); + writeWebBuildMarker(WEB_DIR, REPO_ROOT); } /** Registers beforeAll/afterAll that boot and tear down hub + web + Chrome. */ @@ -100,7 +114,12 @@ export function bootBrowserApp(): () => BrowserApp { app = { origin, newPage: async () => { - const page = await launched.newPage(); + // A fresh incognito context per page: tests sharing one boot must + // not share sessions — a second signup would land on an already + // signed-in shell instead of the signed-out screen. Contexts die + // with the browser in afterAll. + const context = await launched.createBrowserContext(); + const page = await context.newPage(); const errors: string[] = []; page.on("console", (msg) => { if (msg.type() === "error") errors.push(msg.text()); diff --git a/e2e/lib/web-build.test.ts b/e2e/lib/web-build.test.ts new file mode 100644 index 000000000..fcfb91ebb --- /dev/null +++ b/e2e/lib/web-build.test.ts @@ -0,0 +1,112 @@ +// Freshness gate for the apps/web production bundle the browser suites +// serve: a dist marker matching the working-tree fingerprint means the +// current tree already built it, so bootBrowserApp can skip its rebuild. +import { expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { isWebBuildFresh, webBuildFingerprint, writeWebBuildMarker } from "./web-build"; + +function git(dir: string, args: readonly string[]): void { + const proc = Bun.spawnSync(["git", ...args], { + cwd: dir, + stdout: "pipe", + stderr: "pipe", + }); + if (proc.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed in ${dir}`); +} + +function initRepo(): { repo: string; web: string } { + const repo = mkdtempSync(path.join(tmpdir(), "web-build-")); + git(repo, ["init"]); + git(repo, ["config", "user.email", "test@example.com"]); + git(repo, ["config", "user.name", "test"]); + // The developer's global git config may sign commits or install hooks; + // neither exists usefully inside a throwaway repo. + const hooks = path.join(repo, "no-hooks"); + mkdirSync(hooks, { recursive: true }); + git(repo, ["config", "commit.gpgsign", "false"]); + git(repo, ["config", "core.hooksPath", hooks]); + writeFileSync(path.join(repo, "src.txt"), "v1"); + git(repo, ["add", "."]); + git(repo, ["commit", "-m", "init"]); + const web = path.join(repo, "web"); + mkdirSync(path.join(web, "dist"), { recursive: true }); + return { repo, web }; +} + +test("a matching marker means fresh; any tree change means rebuild", () => { + const { repo, web } = initRepo(); + expect(webBuildFingerprint(repo)).not.toBeNull(); + // No marker yet: build. + expect(isWebBuildFresh(web, repo)).toBe(false); + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo)).toBe(true); + // A tracked modification invalidates the marker. + writeFileSync(path.join(repo, "src.txt"), "v2"); + expect(isWebBuildFresh(web, repo)).toBe(false); + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo)).toBe(true); + // So does an untracked file: it may be a new build input. + writeFileSync(path.join(repo, "new-input.txt"), "untracked"); + expect(isWebBuildFresh(web, repo)).toBe(false); +}); + +test("a new commit invalidates the marker", () => { + const { repo, web } = initRepo(); + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo)).toBe(true); + writeFileSync(path.join(repo, "src.txt"), "v2"); + git(repo, ["add", "."]); + git(repo, ["commit", "-m", "change"]); + expect(isWebBuildFresh(web, repo)).toBe(false); +}); + +test("the fingerprint does not depend on BASE_URL", () => { + // The production dist does not bake any env: apps/web/vite.config.ts reads + // BASE_URL only for the dev-server proxy, and apps/web/src has no + // process.env / import.meta.env references. But e2e/lib/hub.ts sets + // process.env.BASE_URL to each hub's port at boot and never restores it; + // with every browser file in one `bun test` process, a leaked BASE_URL + // must not flip the fingerprint or the shared-build freshness gate would + // rebuild apps/web per suite. Guard exactly that. + const { repo, web } = initRepo(); + const previous = process.env["BASE_URL"]; + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo)).toBe(true); + // Capture the fingerprint after the marker exists so both sides of the + // comparison see the same (marker-present) tree. + const before = webBuildFingerprint(repo); + try { + process.env["BASE_URL"] = "http://localhost:4000"; + expect(webBuildFingerprint(repo)).toBe(before); + expect(isWebBuildFresh(web, repo)).toBe(true); + } finally { + if (previous === undefined) delete process.env["BASE_URL"]; + else process.env["BASE_URL"] = previous; + } +}); + +test("a missing build artifact means rebuild even with a matching marker", () => { + const { repo, web } = initRepo(); + mkdirSync(path.join(repo, "pkg"), { recursive: true }); + writeFileSync(path.join(repo, "pkg", "out.js"), "built"); + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo, ["pkg/out.js"])).toBe(true); + // Deleting the artifact (and re-marking, so status matches again) must + // still read stale: only the existence check can catch this. + rmSync(path.join(repo, "pkg", "out.js")); + writeWebBuildMarker(web, repo); + expect(isWebBuildFresh(web, repo, ["pkg/out.js"])).toBe(false); + expect(isWebBuildFresh(web, repo)).toBe(true); +}); + +test("without git the build is never fresh", () => { + const dir = mkdtempSync(path.join(tmpdir(), "web-build-nogit-")); + const web = path.join(dir, "web"); + mkdirSync(path.join(web, "dist"), { recursive: true }); + expect(webBuildFingerprint(dir)).toBeNull(); + expect(isWebBuildFresh(web, dir)).toBe(false); + writeWebBuildMarker(web, dir); + expect(isWebBuildFresh(web, dir)).toBe(false); +}); diff --git a/e2e/lib/web-build.ts b/e2e/lib/web-build.ts new file mode 100644 index 000000000..d947df168 --- /dev/null +++ b/e2e/lib/web-build.ts @@ -0,0 +1,78 @@ +// Freshness gate for the apps/web production bundle the browser suites +// serve: files in one `bun test e2e` process run serially, so the first +// browser suite builds and the rest reuse its dist when the tree hasn't +// changed since. The fingerprint is HEAD plus the full worktree status, so +// any tracked change, new file, or new commit rebuilds; ignored paths +// (node_modules, .worktrees) never participate. Without git there is no +// fingerprint, and the caller always rebuilds — the old behavior. +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +export const WEB_BUILD_MARKER = ".e2e-build-fingerprint"; + +function gitOutput(repoRoot: string, args: readonly string[]): string | null { + const proc = Bun.spawnSync(["git", ...args], { + cwd: repoRoot, + stdout: "pipe", + stderr: "pipe", + }); + if (proc.exitCode !== 0) return null; + return proc.stdout.toString(); +} + +export function webBuildFingerprint( + repoRoot: string, + excludeStatusPaths: readonly string[] = [], +): string | null { + const head = gitOutput(repoRoot, ["rev-parse", "HEAD"]); + const status = gitOutput(repoRoot, ["status", "--porcelain=v1", "--untracked-files=all"]); + if (head === null || status === null) return null; + const excluded = new Set(excludeStatusPaths); + const lines = status.split("\n").filter((line) => { + if (line === "") return false; + // Porcelain v1: `XY PATH`; quote-wrapped when the path needs it. + const quoted = line.slice(3); + const unquoted = + quoted.length >= 2 && quoted.startsWith('"') && quoted.endsWith('"') + ? quoted.slice(1, -1) + : quoted; + return !excluded.has(unquoted); + }); + return `${head.trim()}\n${lines.join("\n")}${lines.length > 0 ? "\n" : ""}`; +} + +function markerPath(webDir: string): string { + return path.join(webDir, "dist", WEB_BUILD_MARKER); +} + +// The marker is itself an untracked file, so it must not participate in the +// fingerprint it records — otherwise writing it would invalidate it. +function markerStatusPath(webDir: string, repoRoot: string): string { + return path.relative(repoRoot, markerPath(webDir)); +} + +export function isWebBuildFresh( + webDir: string, + repoRoot: string, + // Repo-root-relative build artifacts the web build produces alongside + // dist (today the worker bundle): the marker matches the tree, but a + // deleted artifact still means rebuild. + requiredArtifacts: readonly string[] = [], +): boolean { + const fingerprint = webBuildFingerprint(repoRoot, [markerStatusPath(webDir, repoRoot)]); + if (fingerprint === null) return false; + let marker: string; + try { + marker = readFileSync(markerPath(webDir), "utf8"); + } catch { + return false; + } + if (marker !== fingerprint) return false; + return requiredArtifacts.every((artifact) => existsSync(path.join(repoRoot, artifact))); +} + +export function writeWebBuildMarker(webDir: string, repoRoot: string): void { + const fingerprint = webBuildFingerprint(repoRoot, [markerStatusPath(webDir, repoRoot)]); + if (fingerprint === null) return; + writeFileSync(markerPath(webDir), fingerprint); +}