CI: run each e2e suite as its own job (CL-7485)

.github/workflows/ci.yml

@@ -87,9 +87,29 @@ jobs:
       - uses: ./.github/actions/setup-workbench
       - run: bun run check:structural
 
-  e2e:
+  e2e-plan:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    outputs:
+      suites: ${{ steps.list-suites.outputs.suites }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-workbench
+      - name: List e2e suites
+        id: list-suites
+        run: echo "suites=$(bun run scripts/e2e/list-suites.ts)" >> "$GITHUB_OUTPUT"
+
+  # One job per e2e suite so a PR's e2e wall-clock is the slowest suite
+  # rather than the sum of all of them, and a failing suite names itself
+  # instead of hiding behind "e2e failed". fail-fast is off so one flaky
+  # suite doesn't cancel every other suite's run.
+  e2e-suite:
+    needs: e2e-plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: ${{ fromJson(needs.e2e-plan.outputs.suites) }}
 
     services:
       postgres:
@@ -135,10 +155,38 @@ jobs:
           pg_isready -h localhost -p 5432
           git --version
 
-      - name: Run the walking skeleton
-        run: bun run test:e2e
+      - name: Run the suite
+        run: |
+          if [ "${{ matrix.suite }}" = "unit" ]; then
+            bun test scripts/e2e/harness.test.ts scripts/e2e/db-gate.test.ts scripts/e2e/db-setup.test.ts scripts/e2e/list-suites.test.ts --max-concurrency=1
+          else
+            bun test "scripts/e2e/${{ matrix.suite }}.test.ts" --max-concurrency=1
+          fi
         env:
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/workbench
+          E2E_LOG_DIR: e2e-logs
+
+      - name: Upload e2e logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-logs-${{ matrix.suite }}
+          path: e2e-logs
+          if-no-files-found: ignore
+
+  # Branch protection's required check is named "e2e"; this summary job
+  # keeps that name while the actual work runs as per-suite e2e-suite jobs.
+  e2e:
+    needs: e2e-suite
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check e2e-suite matrix result
+        run: |
+          if [ "${{ needs.e2e-suite.result }}" != "success" ]; then
+            echo "one or more e2e suites failed"
+            exit 1
+          fi
 
   isolation:
     runs-on: ubuntu-latest

AGENTS.md

@@ -70,7 +70,7 @@ check:structural <name>`.
 - Remaining test/CI flags (what breaks if unset):
   - `DATABASE_URL` — DB-gated suites skip locally with a banner naming
     `docker compose -f docker-compose.test.yml up -d`. On CI jobs that
-    provision Postgres (`e2e`, `isolation`, `db-suites`), `CI=true`
+    provision Postgres (`e2e-suite`, `isolation`, `db-suites`), `CI=true`
     turns a missing URL into a hard failure via `dbGate`. The unit
     jobs (`build-test`, `structural`, `lint`, `typecheck`) never
     provision Postgres and still skip.
@@ -88,8 +88,9 @@ check:structural <name>`.
   - `WORKBENCH_CHECK_CONCURRENCY` — override `scripts/run-all.ts`
     parallelism. Unset uses a core-count heuristic (all cores on
     GitHub Actions, cores-2 locally).
-  - `E2E_LOG_DIR` — optional directory for hub/sidecar e2e logs.
-    Unset does not capture them.
+  - `E2E_LOG_DIR` — optional directory for hub/sidecar e2e logs. Unset
+    does not capture them; CI's `e2e-suite` matrix job sets it and
+    uploads the directory as a per-suite artifact.
   - `CHROME_PATH` — browser walkthrough looks for Chrome here first.
     Unset falls through to platform defaults and fails if none exist.
 - Tests are meaningful red/green tests only — no coverage theater. Unit

scripts/ci-jobs.test.ts

@@ -26,7 +26,7 @@ function jobBodies(yaml: string): Map<string, string> {
 
 const SETUP = "./.github/actions/setup-workbench";
 const POSTGRES_IMAGE = "pgvector/pgvector:pg17";
-const DB_JOBS = ["e2e", "isolation", "db-suites"] as const;
+const DB_JOBS = ["e2e-suite", "isolation", "db-suites"] as const;
 const MERGE_BASE_JOBS = ["typecheck", "build-test", "structural"] as const;
 
 test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", async () => {
@@ -69,6 +69,33 @@ test("CI splits e2e, isolation, and db-suites onto their own Postgres jobs", asy
   expect(e2eSetupIndex).toBeLessThan(hubSuiteIndex);
 });
 
+test('e2e runs as a plan/matrix/summary trio so branch protection\'s "e2e" check still exists', async () => {
+  const yaml = await readFile(join(ROOT, ".github/workflows/ci.yml"), "utf8");
+  const jobs = jobBodies(yaml);
+
+  const plan = jobs.get("e2e-plan");
+  expect(plan).toBeDefined();
+  expect(plan).toContain("scripts/e2e/list-suites.ts");
+  expect(plan).toContain("outputs:");
+
+  const suite = jobs.get("e2e-suite");
+  expect(suite).toBeDefined();
+  expect(suite).toContain("needs: e2e-plan");
+  expect(suite).toContain("fail-fast: false");
+  expect(suite).toContain("matrix:");
+  expect(suite).toContain("fromJson(needs.e2e-plan.outputs.suites)");
+  expect(suite).toContain("postgres:");
+  expect(suite).toContain(`image: ${POSTGRES_IMAGE}`);
+  expect(suite).toContain("E2E_LOG_DIR");
+  expect(suite).toContain("actions/upload-artifact@v4");
+  expect(suite).toContain("if: always()");
+
+  const summary = jobs.get("e2e");
+  expect(summary).toBeDefined();
+  expect(summary).toContain("needs: e2e-suite");
+  expect(summary).toContain("if: always()");
+});
+
 test("the structural job runs the same list as local check:structural", async () => {
   const yaml = await readFile(join(ROOT, ".github/workflows/ci.yml"), "utf8");
   const structural = jobBodies(yaml).get("structural") ?? "";

scripts/e2e/list-suites.test.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { listSuites } from "./list-suites.ts";
+
+test("listSuites includes a fixture suite and folds the unit files together", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "e2e-list-suites-"));
+  try {
+    for (const name of [
+      "chat.test.ts",
+      "harness.test.ts",
+      "db-gate.test.ts",
+      "db-setup.test.ts",
+      "not-a-suite.ts",
+    ]) {
+      await writeFile(join(dir, name), "");
+    }
+
+    const suites = await listSuites(dir);
+
+    expect(suites).toContain("chat");
+    expect(suites).toContain("unit");
+    expect(suites).not.toContain("harness");
+    expect(suites).not.toContain("db-gate");
+    expect(suites).not.toContain("db-setup");
+    expect(suites).not.toContain("list-suites");
+    expect(suites).not.toContain("not-a-suite");
+    expect(suites.filter((suite) => suite === "unit")).toHaveLength(1);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});

scripts/e2e/list-suites.ts

@@ -0,0 +1,32 @@
+// Prints a JSON array of e2e suite ids for the CI matrix: one id per
+// scripts/e2e/*.test.ts file, except harness.test.ts, db-gate.test.ts, and
+// db-setup.test.ts, which fold into a single "unit" entry (they cover pure
+// helpers rather than a full hub+sidecar boot, so a single fast job covers
+// all three instead of paying three boot costs).
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+
+export const UNIT_SUITE_FILES = [
+  "harness",
+  "db-gate",
+  "db-setup",
+  "list-suites",
+] as const;
+
+export async function listSuites(e2eDir: string): Promise<string[]> {
+  const entries = await readdir(e2eDir, { withFileTypes: true });
+  const suiteNames = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".test.ts"))
+    .map((entry) => entry.name.slice(0, -".test.ts".length));
+
+  const unitFiles = new Set<string>(UNIT_SUITE_FILES);
+  const suites = suiteNames.filter((name) => !unitFiles.has(name));
+  if (suiteNames.some((name) => unitFiles.has(name))) suites.push("unit");
+
+  return suites.sort();
+}
+
+if (import.meta.main) {
+  const suites = await listSuites(join(import.meta.dir));
+  console.log(JSON.stringify(suites));
+}