diff --git a/.github/workflows/nightly-deep.yml b/.github/workflows/nightly-deep.yml index acb8d064..0095d537 100644 --- a/.github/workflows/nightly-deep.yml +++ b/.github/workflows/nightly-deep.yml @@ -123,7 +123,7 @@ jobs: - run: pnpm verify browser: - name: Browser journeys (${{ matrix.browser }}) + name: Browser journeys (${{ matrix.browser }} / ${{ matrix.project }} / ${{ matrix.shard }} of 2) needs: changes if: needs.changes.outputs.run_deep == 'true' runs-on: ubuntu-latest @@ -132,10 +132,17 @@ jobs: fail-fast: false matrix: browser: [chromium, firefox, webkit] + # Each runner owns one database and one Next dev server for about 75 tests. + # The prior job sent all 302 tests and six browser workers through one server. + project: [mobile, desktop] + shard: [1, 2] env: CI: true DATABASE_URL: postgresql://peanut:peanut@localhost:5432/peanut_split_dev E2E_BROWSER: ${{ matrix.browser }} + # A single browser shares the runner with Next and Postgres. Hosted evidence + # showed three workers still starved long journeys; one was stable locally. + E2E_WORKERS: 1 services: postgres: image: postgres:16-alpine @@ -157,14 +164,16 @@ jobs: node-version: 22 - run: corepack enable - run: pnpm bootstrap + - name: Validate the ${{ matrix.project }} shard ${{ matrix.shard }} of 2 plan + run: pnpm --dir apps/web e2e:plan ${{ matrix.project }} ${{ matrix.shard }}/2 - run: pnpm --dir apps/web exec prisma migrate deploy - run: pnpm --dir apps/web exec playwright install --with-deps ${{ matrix.browser }} - - run: pnpm --dir apps/web e2e + - run: pnpm --dir apps/web exec playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/2 - name: Keep the report and traces if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: browser-${{ matrix.browser }}-${{ github.sha }} + name: browser-${{ matrix.browser }}-${{ matrix.project }}-${{ matrix.shard }}of2-${{ github.sha }} path: | apps/web/playwright-report apps/web/test-results diff --git a/apps/web/package.json b/apps/web/package.json index 344e6796..c04ce2c0 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -19,6 +19,7 @@ "classes:audit": "node scripts/tailwind-class-audit.mjs", "test:watch": "vitest", "e2e": "playwright test", + "e2e:plan": "node scripts/validate-e2e-project-plan.mjs", "e2e:v2": "playwright test --config playwright.v2.config.ts", "icons": "node scripts/generate-icons.mjs" }, diff --git a/apps/web/scripts/validate-e2e-project-plan.mjs b/apps/web/scripts/validate-e2e-project-plan.mjs new file mode 100644 index 00000000..75e93ee4 --- /dev/null +++ b/apps/web/scripts/validate-e2e-project-plan.mjs @@ -0,0 +1,69 @@ +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' + +const project = process.argv[2] +const shard = process.argv[3] +const knownProjects = ['mobile', 'desktop'] +const knownShards = ['1/2', '2/2'] + +if (!knownProjects.includes(project)) { + console.error(`Expected one browser shard project (${knownProjects.join(' or ')}), got ${project ?? 'nothing'}`) + process.exit(1) +} +if (!knownShards.includes(shard)) { + console.error(`Expected one browser shard (${knownShards.join(' or ')}), got ${shard ?? 'nothing'}`) + process.exit(1) +} + +const root = resolve(import.meta.dirname, '..') +const fail = (message, result) => { + process.stdout.write(result?.stdout ?? '') + process.stderr.write(result?.stderr ?? '') + console.error(message) + process.exit(1) +} + +const list = (selectedShard) => { + const args = ['exec', 'playwright', 'test', '--list', `--project=${project}`] + if (selectedShard) args.push(`--shard=${selectedShard}`) + const result = spawnSync('pnpm', args, { + cwd: root, + encoding: 'utf8', + env: { ...process.env, CI: 'true' }, + }) + + if (result.error) fail(`Could not list ${project} ${selectedShard ?? 'unsharded'}: ${result.error.message}`, result) + if (result.status !== 0) fail(`Playwright could not list ${project} ${selectedShard ?? 'unsharded'}`, result) + + const tests = [...result.stdout.matchAll(/^\s+\[([^\]]+)] › (.+)$/gm)].map((match) => ({ + project: match[1], + identity: match[2], + })) + const summary = result.stdout.match(/^Total: (\d+) tests in (\d+) files$/m) + const foreignProjects = [...new Set(tests.filter((test) => test.project !== project).map((test) => test.project))] + + if (!summary || tests.length === 0) fail(`${project} ${selectedShard ?? 'unsharded'} selected no tests`, result) + if (foreignProjects.length > 0) + fail(`${project} ${selectedShard ?? 'unsharded'} also selected: ${foreignProjects.join(', ')}`, result) + if (Number(summary[1]) !== tests.length) + fail(`Playwright reported ${summary[1]} tests but listed ${tests.length}`, result) + + return { tests: new Set(tests.map((test) => test.identity)), count: tests.length, files: Number(summary[2]) } +} + +const full = list() +const halves = knownShards.map(list) +const overlap = [...halves[0].tests].filter((test) => halves[1].tests.has(test)) +const combined = new Set(halves.flatMap((half) => [...half.tests])) +const missing = [...full.tests].filter((test) => !combined.has(test)) +const extra = [...combined].filter((test) => !full.tests.has(test)) + +if (overlap.length > 0) fail(`${project} shards overlap on ${overlap.length} tests`) +if (missing.length > 0) fail(`${project} shards omit ${missing.length} tests`) +if (extra.length > 0) fail(`${project} shards add ${extra.length} tests`) + +const selected = halves[knownShards.indexOf(shard)] +console.log( + `Validated ${project} ${shard}: ${selected.count} tests in ${selected.files} files; ` + + `both shards partition all ${full.count} tests` +)