From 06e9aeb56ec2f6dfa95cf341b432eaf7561f0b99 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:47:36 -0700 Subject: [PATCH 1/3] Add tests for the e2e suite matrix job split Pins the expected shape of an e2e-plan/e2e-suite/e2e job trio and a list-suites helper that folds the non-hub-booting suites (harness, db-gate, db-setup, list-suites) into one "unit" entry. --- scripts/ci-jobs.test.ts | 29 ++++++++++++++++++++++++++++- scripts/e2e/list-suites.test.ts | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 scripts/e2e/list-suites.test.ts diff --git a/scripts/ci-jobs.test.ts b/scripts/ci-jobs.test.ts index f9b9fab069..75b53aa57c 100644 --- a/scripts/ci-jobs.test.ts +++ b/scripts/ci-jobs.test.ts @@ -26,7 +26,7 @@ function jobBodies(yaml: string): Map { const SETUP = "./.github/actions/setup-workbench"; const POSTGRES_IMAGE = "pgvector/pgvector:pg17"; -const DB_JOBS = ["e2e", "isolation", "db-suites"] as const; +const DB_JOBS = ["e2e-suite", "isolation", "db-suites"] as const; const MERGE_BASE_JOBS = ["typecheck", "build-test", "structural"] as const; test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", async () => { @@ -69,6 +69,33 @@ test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", asy expect(e2eSetupIndex).toBeLessThan(hubSuiteIndex); }); +test('e2e runs as a plan/matrix/summary trio so branch protection\'s "e2e" check still exists', async () => { + const yaml = await readFile(join(ROOT, ".github/workflows/ci.yml"), "utf8"); + const jobs = jobBodies(yaml); + + const plan = jobs.get("e2e-plan"); + expect(plan).toBeDefined(); + expect(plan).toContain("scripts/e2e/list-suites.ts"); + expect(plan).toContain("outputs:"); + + const suite = jobs.get("e2e-suite"); + expect(suite).toBeDefined(); + expect(suite).toContain("needs: e2e-plan"); + expect(suite).toContain("fail-fast: false"); + expect(suite).toContain("matrix:"); + expect(suite).toContain("fromJson(needs.e2e-plan.outputs.suites)"); + expect(suite).toContain("postgres:"); + expect(suite).toContain(`image: ${POSTGRES_IMAGE}`); + expect(suite).toContain("E2E_LOG_DIR"); + expect(suite).toContain("actions/upload-artifact@v4"); + expect(suite).toContain("if: always()"); + + const summary = jobs.get("e2e"); + expect(summary).toBeDefined(); + expect(summary).toContain("needs: e2e-suite"); + expect(summary).toContain("if: always()"); +}); + test("the structural job runs the same list as local check:structural", async () => { const yaml = await readFile(join(ROOT, ".github/workflows/ci.yml"), "utf8"); const structural = jobBodies(yaml).get("structural") ?? ""; diff --git a/scripts/e2e/list-suites.test.ts b/scripts/e2e/list-suites.test.ts new file mode 100644 index 0000000000..8a932908ca --- /dev/null +++ b/scripts/e2e/list-suites.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { listSuites } from "./list-suites.ts"; + +test("listSuites includes a fixture suite and folds the unit files together", async () => { + const dir = await mkdtemp(join(tmpdir(), "e2e-list-suites-")); + try { + for (const name of [ + "chat.test.ts", + "harness.test.ts", + "db-gate.test.ts", + "db-setup.test.ts", + "not-a-suite.ts", + ]) { + await writeFile(join(dir, name), ""); + } + + const suites = await listSuites(dir); + + expect(suites).toContain("chat"); + expect(suites).toContain("unit"); + expect(suites).not.toContain("harness"); + expect(suites).not.toContain("db-gate"); + expect(suites).not.toContain("db-setup"); + expect(suites).not.toContain("list-suites"); + expect(suites).not.toContain("not-a-suite"); + expect(suites.filter((suite) => suite === "unit")).toHaveLength(1); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); From 8c4f13124a300a0517cdd3354cdd01c2c5e562b6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:47:41 -0700 Subject: [PATCH 2/3] CI: run each e2e suite as its own job The e2e job ran every scripts/e2e/*.test.ts suite serially in one job, each booting hub and sidecar against Postgres, so a PR's e2e wall time scaled with the sum of every suite (4 to 12 minutes) rather than the slowest one. An e2e-plan job now lists the suites and feeds them into an e2e-suite matrix job, one job per suite, fail-fast off, each uploading its E2E_LOG_DIR as an artifact. A summary e2e job keeps that name for branch protection while the matrix does the work. --- .github/workflows/ci.yml | 56 +++++++++++++++++++++++++++++++++++--- scripts/e2e/list-suites.ts | 32 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 scripts/e2e/list-suites.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a353634006..02b2bbac26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,9 +87,29 @@ jobs: - uses: ./.github/actions/setup-workbench - run: bun run check:structural - e2e: + e2e-plan: runs-on: ubuntu-latest - timeout-minutes: 20 + outputs: + suites: ${{ steps.list-suites.outputs.suites }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-workbench + - name: List e2e suites + id: list-suites + run: echo "suites=$(bun run scripts/e2e/list-suites.ts)" >> "$GITHUB_OUTPUT" + + # One job per e2e suite so a PR's e2e wall-clock is the slowest suite + # rather than the sum of all of them, and a failing suite names itself + # instead of hiding behind "e2e failed". fail-fast is off so one flaky + # suite doesn't cancel every other suite's run. + e2e-suite: + needs: e2e-plan + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + suite: ${{ fromJson(needs.e2e-plan.outputs.suites) }} services: postgres: @@ -135,10 +155,38 @@ jobs: pg_isready -h localhost -p 5432 git --version - - name: Run the walking skeleton - run: bun run test:e2e + - name: Run the suite + run: | + if [ "${{ matrix.suite }}" = "unit" ]; then + bun test scripts/e2e/harness.test.ts scripts/e2e/db-gate.test.ts scripts/e2e/db-setup.test.ts scripts/e2e/list-suites.test.ts --max-concurrency=1 + else + bun test "scripts/e2e/${{ matrix.suite }}.test.ts" --max-concurrency=1 + fi env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench + E2E_LOG_DIR: e2e-logs + + - name: Upload e2e logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs-${{ matrix.suite }} + path: e2e-logs + if-no-files-found: ignore + + # Branch protection's required check is named "e2e"; this summary job + # keeps that name while the actual work runs as per-suite e2e-suite jobs. + e2e: + needs: e2e-suite + if: always() + runs-on: ubuntu-latest + steps: + - name: Check e2e-suite matrix result + run: | + if [ "${{ needs.e2e-suite.result }}" != "success" ]; then + echo "one or more e2e suites failed" + exit 1 + fi isolation: runs-on: ubuntu-latest diff --git a/scripts/e2e/list-suites.ts b/scripts/e2e/list-suites.ts new file mode 100644 index 0000000000..1e4f4af39c --- /dev/null +++ b/scripts/e2e/list-suites.ts @@ -0,0 +1,32 @@ +// Prints a JSON array of e2e suite ids for the CI matrix: one id per +// scripts/e2e/*.test.ts file, except harness.test.ts, db-gate.test.ts, and +// db-setup.test.ts, which fold into a single "unit" entry (they cover pure +// helpers rather than a full hub+sidecar boot, so a single fast job covers +// all three instead of paying three boot costs). +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; + +export const UNIT_SUITE_FILES = [ + "harness", + "db-gate", + "db-setup", + "list-suites", +] as const; + +export async function listSuites(e2eDir: string): Promise { + const entries = await readdir(e2eDir, { withFileTypes: true }); + const suiteNames = entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".test.ts")) + .map((entry) => entry.name.slice(0, -".test.ts".length)); + + const unitFiles = new Set(UNIT_SUITE_FILES); + const suites = suiteNames.filter((name) => !unitFiles.has(name)); + if (suiteNames.some((name) => unitFiles.has(name))) suites.push("unit"); + + return suites.sort(); +} + +if (import.meta.main) { + const suites = await listSuites(join(import.meta.dir)); + console.log(JSON.stringify(suites)); +} From 98fd69a485642e8a62e4850b22257c8b2366a643 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Sun, 6 Sep 2026 13:47:45 -0700 Subject: [PATCH 3/3] Update docs: e2e job split Renames the e2e Postgres job reference to e2e-suite and notes that the e2e-suite matrix job now sets E2E_LOG_DIR and uploads it per suite. --- AGENTS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e14d3474df..32a93215dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,7 +70,7 @@ check:structural `. - Remaining test/CI flags (what breaks if unset): - `DATABASE_URL` — DB-gated suites skip locally with a banner naming `docker compose -f docker-compose.test.yml up -d`. On CI jobs that - provision Postgres (`e2e`, `isolation`, `db-suites`), `CI=true` + provision Postgres (`e2e-suite`, `isolation`, `db-suites`), `CI=true` turns a missing URL into a hard failure via `dbGate`. The unit jobs (`build-test`, `structural`, `lint`, `typecheck`) never provision Postgres and still skip. @@ -88,8 +88,9 @@ check:structural `. - `WORKBENCH_CHECK_CONCURRENCY` — override `scripts/run-all.ts` parallelism. Unset uses a core-count heuristic (all cores on GitHub Actions, cores-2 locally). - - `E2E_LOG_DIR` — optional directory for hub/sidecar e2e logs. - Unset does not capture them. + - `E2E_LOG_DIR` — optional directory for hub/sidecar e2e logs. Unset + does not capture them; CI's `e2e-suite` matrix job sets it and + uploads the directory as a per-suite artifact. - `CHROME_PATH` — browser walkthrough looks for Chrome here first. Unset falls through to platform defaults and fails if none exist. - Tests are meaningful red/green tests only — no coverage theater. Unit