diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a3bfd1..00879b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,35 @@ jobs: - run: npm ci - run: npm test + # End-to-end smoke tests against a production build. Not yet a required + # check — continue-on-error keeps failures visible without blocking merges + # while the suite is still being built out. + e2e: + name: e2e (non-blocking) + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm run test:e2e + - name: Upload playwright report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: playwright-report + path: playwright-report/ + retention-days: 14 + # public/llms.txt, llms-full.txt, and AGENTS.md are generated from the route # tree and committed. They go stale whenever a route is added, renamed, or # removed, so this fails if they differ from a fresh generation (it also diff --git a/.gitignore b/.gitignore index fd6624f..fa6719c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,12 @@ yarn-error.log* # typescript *.tsbuildinfo next-env.d.ts + +# playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ .claude/worktrees/ .agents/skills/ .claude/skills/ diff --git a/AGENTS.md b/AGENTS.md index b801449..3b90fc3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,31 @@ of a single commit, use `SKIP_DOCS_HOOK=1` or put `[skip-docs]` in the message. ## Testing -- Run the llms-kit suite: `node --test tests/` +Two separate suites, for two separate purposes — don't mix their files. + +- **Unit/component/contract tests — Vitest.** `*.test.ts(x)`, colocated next to + the code they cover (e.g. `app/snapshots/networks.contract.test.ts`, + `deploy.config.test.mjs`). Run with `npm test`. Fast, no browser, no build — + this is where logic, data transforms, and contract/matrix assertions live. + Blocking CI job (`test`). +- **End-to-end journeys — Playwright.** `e2e/*.spec.ts`. Run with + `npm run test:e2e`, which builds the app and drives it in a real browser + against that production build. Non-blocking CI job (`e2e (non-blocking)`, + `continue-on-error: true`) — failures are visible but don't block merges + while this suite is still young. `vitest.config.mts` excludes `e2e/**` from + Vitest's default test glob so the two runners never pick up each other's + files. + + **When to add an e2e test**: only for surfaces that are stable — established + pages/journeys not expected to change shape week to week (e.g. the home + page, core nav, Snapshots, Upgrades). Do not add e2e coverage for a new demo + or anything still being iterated on (e.g. work under `app/vibenet/demos/`) + — the UI will keep shifting under the test faster than the test catches + real regressions, making it pure maintenance overhead. Add a smoke test + once a demo's surface has settled, not before. + +- Run the llms-kit suite (generation scripts under `scripts/`, unrelated to + the app tests above): `node --test tests/` - Include liveness checks against the deployed site: `LLMS_LIVE=1 node --test tests/` ## Analytics diff --git a/README.md b/README.md index cae6783..d0aefdb 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ to run the **internal** build locally with those sections visible. See - `npm run lint` — eslint (next/core-web-vitals) - `npm run typecheck` — `tsc --noEmit` - `npm test` — vitest +- `npm run test:e2e` — playwright end-to-end tests (builds and serves the app first) - `npm run llms` / `npm run agents` — regenerate the agent index files - `npm run docs:check` — verify the agent index is current @@ -49,6 +50,9 @@ to run the **internal** build locally with those sections visible. See - **public build excludes internal-only surfaces** — builds the default (external) target and asserts that internal-only routes 404 and never appear in the nav or sitemap, so the deployment matrix can't silently regress +- **e2e (non-blocking)** — playwright smoke tests against a production build. + `continue-on-error`, so failures show up without blocking merges while the + suite is still being built out. CodeQL, StepSecurity, Heimdall, and the Vercel preview build are configured outside this repo at the org/platform level. diff --git a/e2e/home.spec.ts b/e2e/home.spec.ts new file mode 100644 index 0000000..3f7ba93 --- /dev/null +++ b/e2e/home.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test'; + +test('home page loads and shows the landing copy', async ({ page }) => { + await page.goto('/'); + + await expect(page.getByText('Monitor and test Base, all in one place.')).toBeVisible(); + await expect(page.getByRole('main').getByRole('link', { name: /vibenet/i })).toBeVisible(); +}); diff --git a/package-lock.json b/package-lock.json index 75e37ab..16b1c5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "viem": "^2.55.10" }, "devDependencies": { + "@playwright/test": "^1.62.1", "@types/d3": "^7.4.3", "@types/node": "^24", "@types/react": "19.2.18", @@ -1082,9 +1083,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1101,9 +1099,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1120,9 +1115,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1139,9 +1131,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1158,9 +1147,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1177,9 +1163,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1196,9 +1179,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1215,9 +1195,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1234,9 +1211,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1259,9 +1233,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1284,9 +1255,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1309,9 +1277,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1334,9 +1299,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1359,9 +1321,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1384,9 +1343,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1409,9 +1365,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1644,9 +1597,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1663,9 +1613,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1682,9 +1629,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1701,9 +1645,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1842,6 +1783,22 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@radix-ui/number": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.2.tgz", @@ -2488,9 +2445,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2508,9 +2462,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2528,9 +2479,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2548,9 +2496,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2568,9 +2513,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2588,9 +2530,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -7391,9 +7330,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7415,9 +7351,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7439,9 +7372,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7463,9 +7393,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8246,6 +8173,53 @@ "node": ">= 6" } }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", diff --git a/package.json b/package.json index 5fa19a9..b119694 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "lint": "eslint .", "typecheck": "tsc --noEmit", "test": "vitest run", + "test:e2e": "playwright test", "llms": "node scripts/llms.mjs", "agents": "node scripts/agents.mjs", "docs:check": "node scripts/llms.mjs --check && node scripts/agents.mjs --check" @@ -38,6 +39,7 @@ "viem": "^2.55.10" }, "devDependencies": { + "@playwright/test": "^1.62.1", "@types/d3": "^7.4.3", "@types/node": "^24", "@types/react": "19.2.18", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..097a899 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,29 @@ +import { defineConfig, devices } from '@playwright/test'; + +const PORT = 3100; +const baseURL = `http://localhost:${PORT}`; + +export default defineConfig({ + testDir: './e2e', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + reporter: process.env.CI ? [['github'], ['html', { open: 'never' }]] : 'html', + use: { + baseURL, + trace: 'on-first-retry', + screenshot: 'only-on-failure', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: { + command: `npm run build && npm run start -- -p ${PORT}`, + url: baseURL, + reuseExistingServer: !process.env.CI, + timeout: 180_000, + }, +}); diff --git a/vitest.config.mts b/vitest.config.mts index 7b5f19c..0f768ce 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,6 +1,6 @@ import path from 'node:path'; -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; export default defineConfig({ resolve: { @@ -10,5 +10,8 @@ export default defineConfig({ test: { globals: true, environment: 'node', + // Playwright owns e2e/*.spec.ts; vitest's default glob would otherwise + // pick them up and fail on the Playwright-only `test()` fixture API. + exclude: [...configDefaults.exclude, 'e2e/**'], }, });