From f8ca350ab82c31d0272c1b4621d5883e96c32459 Mon Sep 17 00:00:00 2001 From: 0xkkonrad Date: Tue, 11 Aug 2026 11:51:38 +0000 Subject: [PATCH 1/2] ci: shard nightly browser projects --- .github/workflows/nightly-deep.yml | 14 +++++-- apps/web/package.json | 1 + .../web/scripts/validate-e2e-project-plan.mjs | 40 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 apps/web/scripts/validate-e2e-project-plan.mjs diff --git a/.github/workflows/nightly-deep.yml b/.github/workflows/nightly-deep.yml index acb8d064..e925383c 100644 --- a/.github/workflows/nightly-deep.yml +++ b/.github/workflows/nightly-deep.yml @@ -123,7 +123,7 @@ jobs: - run: pnpm verify browser: - name: Browser journeys (${{ matrix.browser }}) + name: Browser journeys (${{ matrix.browser }} / ${{ matrix.project }}) needs: changes if: needs.changes.outputs.run_deep == 'true' runs-on: ubuntu-latest @@ -132,10 +132,16 @@ jobs: fail-fast: false matrix: browser: [chromium, firefox, webkit] + # One project per runner means one database and one Next dev server per + # 151-test shard. The prior job sent all 302 tests through one server. + project: [mobile, desktop] env: CI: true DATABASE_URL: postgresql://peanut:peanut@localhost:5432/peanut_split_dev E2E_BROWSER: ${{ matrix.browser }} + # Preserve six workers of aggregate throughput across the two project + # shards without asking either single Next dev server to serve all six. + E2E_WORKERS: 3 services: postgres: image: postgres:16-alpine @@ -157,14 +163,16 @@ jobs: node-version: 22 - run: corepack enable - run: pnpm bootstrap + - name: Validate the ${{ matrix.project }} shard plan + run: pnpm --dir apps/web e2e:plan ${{ matrix.project }} - run: pnpm --dir apps/web exec prisma migrate deploy - run: pnpm --dir apps/web exec playwright install --with-deps ${{ matrix.browser }} - - run: pnpm --dir apps/web e2e + - run: pnpm --dir apps/web exec playwright test --project=${{ matrix.project }} - name: Keep the report and traces if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: browser-${{ matrix.browser }}-${{ github.sha }} + name: browser-${{ matrix.browser }}-${{ matrix.project }}-${{ github.sha }} path: | apps/web/playwright-report apps/web/test-results diff --git a/apps/web/package.json b/apps/web/package.json index 344e6796..c04ce2c0 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -19,6 +19,7 @@ "classes:audit": "node scripts/tailwind-class-audit.mjs", "test:watch": "vitest", "e2e": "playwright test", + "e2e:plan": "node scripts/validate-e2e-project-plan.mjs", "e2e:v2": "playwright test --config playwright.v2.config.ts", "icons": "node scripts/generate-icons.mjs" }, diff --git a/apps/web/scripts/validate-e2e-project-plan.mjs b/apps/web/scripts/validate-e2e-project-plan.mjs new file mode 100644 index 00000000..b2cbf398 --- /dev/null +++ b/apps/web/scripts/validate-e2e-project-plan.mjs @@ -0,0 +1,40 @@ +import { spawnSync } from 'node:child_process' +import { resolve } from 'node:path' + +const project = process.argv[2] +const knownProjects = ['mobile', 'desktop'] + +if (!knownProjects.includes(project)) { + console.error(`Expected one browser shard project (${knownProjects.join(' or ')}), got ${project ?? 'nothing'}`) + process.exit(1) +} + +const root = resolve(import.meta.dirname, '..') +const result = spawnSync('pnpm', ['exec', 'playwright', 'test', '--list', `--project=${project}`], { + cwd: root, + encoding: 'utf8', + env: { ...process.env, CI: 'true' }, +}) + +const output = result.stdout ?? '' +const diagnostics = result.stderr ?? '' +const fail = (message) => { + process.stdout.write(output) + process.stderr.write(diagnostics) + console.error(message) + process.exit(1) +} + +if (result.error) fail(`Could not list the ${project} browser shard: ${result.error.message}`) +if (result.status !== 0) fail(`Playwright could not list the ${project} browser shard`) + +const listedProjects = [...output.matchAll(/^\s+\[([^\]]+)] › /gm)].map((match) => match[1]) +const summary = output.match(/^Total: (\d+) tests in (\d+) files$/m) +const foreignProjects = [...new Set(listedProjects.filter((listed) => listed !== project))] + +if (!summary || listedProjects.length === 0) fail(`The ${project} browser shard selected no tests`) +if (foreignProjects.length > 0) fail(`The ${project} browser shard also selected: ${foreignProjects.join(', ')}`) +if (Number(summary[1]) !== listedProjects.length) + fail(`Playwright reported ${summary[1]} tests but listed ${listedProjects.length}`) + +console.log(`Validated ${project} browser shard: ${summary[1]} tests in ${summary[2]} files`) From 9b1ad14a751968f3f995b44c086a061a296cba4b Mon Sep 17 00:00:00 2001 From: 0xkkonrad Date: Tue, 11 Aug 2026 12:31:19 +0000 Subject: [PATCH 2/2] ci: isolate nightly browser shard workers --- .github/workflows/nightly-deep.yml | 21 +++--- .../web/scripts/validate-e2e-project-plan.mjs | 71 +++++++++++++------ 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nightly-deep.yml b/.github/workflows/nightly-deep.yml index e925383c..0095d537 100644 --- a/.github/workflows/nightly-deep.yml +++ b/.github/workflows/nightly-deep.yml @@ -123,7 +123,7 @@ jobs: - run: pnpm verify browser: - name: Browser journeys (${{ matrix.browser }} / ${{ matrix.project }}) + name: Browser journeys (${{ matrix.browser }} / ${{ matrix.project }} / ${{ matrix.shard }} of 2) needs: changes if: needs.changes.outputs.run_deep == 'true' runs-on: ubuntu-latest @@ -132,16 +132,17 @@ jobs: fail-fast: false matrix: browser: [chromium, firefox, webkit] - # One project per runner means one database and one Next dev server per - # 151-test shard. The prior job sent all 302 tests through one server. + # Each runner owns one database and one Next dev server for about 75 tests. + # The prior job sent all 302 tests and six browser workers through one server. project: [mobile, desktop] + shard: [1, 2] env: CI: true DATABASE_URL: postgresql://peanut:peanut@localhost:5432/peanut_split_dev E2E_BROWSER: ${{ matrix.browser }} - # Preserve six workers of aggregate throughput across the two project - # shards without asking either single Next dev server to serve all six. - E2E_WORKERS: 3 + # A single browser shares the runner with Next and Postgres. Hosted evidence + # showed three workers still starved long journeys; one was stable locally. + E2E_WORKERS: 1 services: postgres: image: postgres:16-alpine @@ -163,16 +164,16 @@ jobs: node-version: 22 - run: corepack enable - run: pnpm bootstrap - - name: Validate the ${{ matrix.project }} shard plan - run: pnpm --dir apps/web e2e:plan ${{ matrix.project }} + - name: Validate the ${{ matrix.project }} shard ${{ matrix.shard }} of 2 plan + run: pnpm --dir apps/web e2e:plan ${{ matrix.project }} ${{ matrix.shard }}/2 - run: pnpm --dir apps/web exec prisma migrate deploy - run: pnpm --dir apps/web exec playwright install --with-deps ${{ matrix.browser }} - - run: pnpm --dir apps/web exec playwright test --project=${{ matrix.project }} + - run: pnpm --dir apps/web exec playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/2 - name: Keep the report and traces if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: browser-${{ matrix.browser }}-${{ matrix.project }}-${{ github.sha }} + name: browser-${{ matrix.browser }}-${{ matrix.project }}-${{ matrix.shard }}of2-${{ github.sha }} path: | apps/web/playwright-report apps/web/test-results diff --git a/apps/web/scripts/validate-e2e-project-plan.mjs b/apps/web/scripts/validate-e2e-project-plan.mjs index b2cbf398..75e93ee4 100644 --- a/apps/web/scripts/validate-e2e-project-plan.mjs +++ b/apps/web/scripts/validate-e2e-project-plan.mjs @@ -2,39 +2,68 @@ import { spawnSync } from 'node:child_process' import { resolve } from 'node:path' const project = process.argv[2] +const shard = process.argv[3] const knownProjects = ['mobile', 'desktop'] +const knownShards = ['1/2', '2/2'] if (!knownProjects.includes(project)) { console.error(`Expected one browser shard project (${knownProjects.join(' or ')}), got ${project ?? 'nothing'}`) process.exit(1) } +if (!knownShards.includes(shard)) { + console.error(`Expected one browser shard (${knownShards.join(' or ')}), got ${shard ?? 'nothing'}`) + process.exit(1) +} const root = resolve(import.meta.dirname, '..') -const result = spawnSync('pnpm', ['exec', 'playwright', 'test', '--list', `--project=${project}`], { - cwd: root, - encoding: 'utf8', - env: { ...process.env, CI: 'true' }, -}) - -const output = result.stdout ?? '' -const diagnostics = result.stderr ?? '' -const fail = (message) => { - process.stdout.write(output) - process.stderr.write(diagnostics) +const fail = (message, result) => { + process.stdout.write(result?.stdout ?? '') + process.stderr.write(result?.stderr ?? '') console.error(message) process.exit(1) } -if (result.error) fail(`Could not list the ${project} browser shard: ${result.error.message}`) -if (result.status !== 0) fail(`Playwright could not list the ${project} browser shard`) +const list = (selectedShard) => { + const args = ['exec', 'playwright', 'test', '--list', `--project=${project}`] + if (selectedShard) args.push(`--shard=${selectedShard}`) + const result = spawnSync('pnpm', args, { + cwd: root, + encoding: 'utf8', + env: { ...process.env, CI: 'true' }, + }) + + if (result.error) fail(`Could not list ${project} ${selectedShard ?? 'unsharded'}: ${result.error.message}`, result) + if (result.status !== 0) fail(`Playwright could not list ${project} ${selectedShard ?? 'unsharded'}`, result) + + const tests = [...result.stdout.matchAll(/^\s+\[([^\]]+)] › (.+)$/gm)].map((match) => ({ + project: match[1], + identity: match[2], + })) + const summary = result.stdout.match(/^Total: (\d+) tests in (\d+) files$/m) + const foreignProjects = [...new Set(tests.filter((test) => test.project !== project).map((test) => test.project))] + + if (!summary || tests.length === 0) fail(`${project} ${selectedShard ?? 'unsharded'} selected no tests`, result) + if (foreignProjects.length > 0) + fail(`${project} ${selectedShard ?? 'unsharded'} also selected: ${foreignProjects.join(', ')}`, result) + if (Number(summary[1]) !== tests.length) + fail(`Playwright reported ${summary[1]} tests but listed ${tests.length}`, result) + + return { tests: new Set(tests.map((test) => test.identity)), count: tests.length, files: Number(summary[2]) } +} -const listedProjects = [...output.matchAll(/^\s+\[([^\]]+)] › /gm)].map((match) => match[1]) -const summary = output.match(/^Total: (\d+) tests in (\d+) files$/m) -const foreignProjects = [...new Set(listedProjects.filter((listed) => listed !== project))] +const full = list() +const halves = knownShards.map(list) +const overlap = [...halves[0].tests].filter((test) => halves[1].tests.has(test)) +const combined = new Set(halves.flatMap((half) => [...half.tests])) +const missing = [...full.tests].filter((test) => !combined.has(test)) +const extra = [...combined].filter((test) => !full.tests.has(test)) -if (!summary || listedProjects.length === 0) fail(`The ${project} browser shard selected no tests`) -if (foreignProjects.length > 0) fail(`The ${project} browser shard also selected: ${foreignProjects.join(', ')}`) -if (Number(summary[1]) !== listedProjects.length) - fail(`Playwright reported ${summary[1]} tests but listed ${listedProjects.length}`) +if (overlap.length > 0) fail(`${project} shards overlap on ${overlap.length} tests`) +if (missing.length > 0) fail(`${project} shards omit ${missing.length} tests`) +if (extra.length > 0) fail(`${project} shards add ${extra.length} tests`) -console.log(`Validated ${project} browser shard: ${summary[1]} tests in ${summary[2]} files`) +const selected = halves[knownShards.indexOf(shard)] +console.log( + `Validated ${project} ${shard}: ${selected.count} tests in ${selected.files} files; ` + + `both shards partition all ${full.count} tests` +)